arch/s390/kvm/interrupt.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * handling kvm guest interrupts
   4  *
   5  * Copyright IBM Corp. 2008, 2020
   6  *
   7  *    Author(s): Carsten Otte <cotte@de.ibm.com>
   8  */
   9
  10 #define KMSG_COMPONENT "kvm-s390"
  11 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  12
  13 #include <linux/interrupt.h>
  14 #include <linux/kvm_host.h>
  15 #include <linux/hrtimer.h>
  16 #include <linux/mmu_context.h>
  17 #include <linux/nospec.h>
  18 #include <linux/signal.h>
  19 #include <linux/slab.h>
  20 #include <linux/bitmap.h>
  21 #include <linux/vmalloc.h>
  22 #include <asm/asm-offsets.h>
  23 #include <asm/dis.h>
  24 #include <linux/uaccess.h>
  25 #include <asm/sclp.h>
  26 #include <asm/isc.h>
  27 #include <asm/gmap.h>
  28 #include <asm/switch_to.h>
  29 #include <asm/nmi.h>
  30 #include <asm/airq.h>
  31 #include <asm/tpi.h>
  32 #include "kvm-s390.h"
  33 #include "gaccess.h"
  34 #include "trace-s390.h"
  35 #include "pci.h"
  36
  37 #define PFAULT_INIT 0x0600
  38 #define PFAULT_DONE 0x0680
  39 #define VIRTIO_PARAM 0x0d00
  40
  41 static struct kvm_s390_gib *gib;
  42
  43 /* handle external calls via sigp interpretation facility */
  44 static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id)
  45 {
  46         int c, scn;
  47
  48         if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND))
  49                 return 0;
  50
  51         BUG_ON(!kvm_s390_use_sca_entries());
  52         read_lock(&vcpu->kvm->arch.sca_lock);
  53         if (vcpu->kvm->arch.use_esca) {
  54                 struct esca_block *sca = vcpu->kvm->arch.sca;
  55                 union esca_sigp_ctrl sigp_ctrl =
  56                         sca->cpu[vcpu->vcpu_id].sigp_ctrl;
  57
  58                 c = sigp_ctrl.c;
  59                 scn = sigp_ctrl.scn;
  60         } else {
  61                 struct bsca_block *sca = vcpu->kvm->arch.sca;
  62                 union bsca_sigp_ctrl sigp_ctrl =
  63                         sca->cpu[vcpu->vcpu_id].sigp_ctrl;
  64
  65                 c = sigp_ctrl.c;
  66                 scn = sigp_ctrl.scn;
  67         }
  68         read_unlock(&vcpu->kvm->arch.sca_lock);
  69
  70         if (src_id)
  71                 *src_id = scn;
  72
  73         return c;
  74 }
  75
  76 static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
  77 {
  78         int expect, rc;
  79
  80         BUG_ON(!kvm_s390_use_sca_entries());
  81         read_lock(&vcpu->kvm->arch.sca_lock);
  82         if (vcpu->kvm->arch.use_esca) {
  83                 struct esca_block *sca = vcpu->kvm->arch.sca;
  84                 union esca_sigp_ctrl *sigp_ctrl =
  85                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
  86                 union esca_sigp_ctrl new_val = {0}, old_val;
  87
  88                 old_val = READ_ONCE(*sigp_ctrl);
  89                 new_val.scn = src_id;
  90                 new_val.c = 1;
  91                 old_val.c = 0;
  92
  93                 expect = old_val.value;
  94                 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
  95         } else {
  96                 struct bsca_block *sca = vcpu->kvm->arch.sca;
  97                 union bsca_sigp_ctrl *sigp_ctrl =
  98                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
  99                 union bsca_sigp_ctrl new_val = {0}, old_val;
 100
 101                 old_val = READ_ONCE(*sigp_ctrl);
 102                 new_val.scn = src_id;
 103                 new_val.c = 1;
 104                 old_val.c = 0;
 105
 106                 expect = old_val.value;
 107                 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
 108         }
 109         read_unlock(&vcpu->kvm->arch.sca_lock);
 110
 111         if (rc != expect) {
 112                 /* another external call is pending */
 113                 return -EBUSY;
 114         }
 115         kvm_s390_set_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
 116         return 0;
 117 }
 118
 119 static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
 120 {
 121         int rc, expect;
 122
 123         if (!kvm_s390_use_sca_entries())
 124                 return;
 125         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
 126         read_lock(&vcpu->kvm->arch.sca_lock);
 127         if (vcpu->kvm->arch.use_esca) {
 128                 struct esca_block *sca = vcpu->kvm->arch.sca;
 129                 union esca_sigp_ctrl *sigp_ctrl =
 130                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
 131                 union esca_sigp_ctrl old;
 132
 133                 old = READ_ONCE(*sigp_ctrl);
 134                 expect = old.value;
 135                 rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
 136         } else {
 137                 struct bsca_block *sca = vcpu->kvm->arch.sca;
 138                 union bsca_sigp_ctrl *sigp_ctrl =
 139                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
 140                 union bsca_sigp_ctrl old;
 141
 142                 old = READ_ONCE(*sigp_ctrl);
 143                 expect = old.value;
 144                 rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
 145         }
 146         read_unlock(&vcpu->kvm->arch.sca_lock);
 147         WARN_ON(rc != expect); /* cannot clear? */
 148 }
 149
 150 int psw_extint_disabled(struct kvm_vcpu *vcpu)
 151 {
 152         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
 153 }
 154
 155 static int psw_ioint_disabled(struct kvm_vcpu *vcpu)
 156 {
 157         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO);
 158 }
 159
 160 static int psw_mchk_disabled(struct kvm_vcpu *vcpu)
 161 {
 162         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK);
 163 }
 164
 165 static int psw_interrupts_disabled(struct kvm_vcpu *vcpu)
 166 {
 167         return psw_extint_disabled(vcpu) &&
 168                psw_ioint_disabled(vcpu) &&
 169                psw_mchk_disabled(vcpu);
 170 }
 171
 172 static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
 173 {
 174         if (psw_extint_disabled(vcpu) ||
 175             !(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
 176                 return 0;
 177         if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu))
 178                 /* No timer interrupts when single stepping */
 179                 return 0;
 180         return 1;
 181 }
 182
 183 static int ckc_irq_pending(struct kvm_vcpu *vcpu)
 184 {
 185         const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
 186         const u64 ckc = vcpu->arch.sie_block->ckc;
 187
 188         if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
 189                 if ((s64)ckc >= (s64)now)
 190                         return 0;
 191         } else if (ckc >= now) {
 192                 return 0;
 193         }
 194         return ckc_interrupts_enabled(vcpu);
 195 }
 196
 197 static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu)
 198 {
 199         return !psw_extint_disabled(vcpu) &&
 200                (vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK);
 201 }
 202
 203 static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
 204 {
 205         if (!cpu_timer_interrupts_enabled(vcpu))
 206                 return 0;
 207         return kvm_s390_get_cpu_timer(vcpu) >> 63;
 208 }
 209
 210 static uint64_t isc_to_isc_bits(int isc)
 211 {
 212         return (0x80 >> isc) << 24;
 213 }
 214
 215 static inline u32 isc_to_int_word(u8 isc)
 216 {
 217         return ((u32)isc << 27) | 0x80000000;
 218 }
 219
 220 static inline u8 int_word_to_isc(u32 int_word)
 221 {
 222         return (int_word & 0x38000000) >> 27;
 223 }
 224
 225 /*
 226  * To use atomic bitmap functions, we have to provide a bitmap address
 227  * that is u64 aligned. However, the ipm might be u32 aligned.
 228  * Therefore, we logically start the bitmap at the very beginning of the
 229  * struct and fixup the bit number.
 230  */
 231 #define IPM_BIT_OFFSET (offsetof(struct kvm_s390_gisa, ipm) * BITS_PER_BYTE)
 232
 233 /**
 234  * gisa_set_iam - change the GISA interruption alert mask
 235  *
 236  * @gisa: gisa to operate on
 237  * @iam: new IAM value to use
 238  *
 239  * Change the IAM atomically with the next alert address and the IPM
 240  * of the GISA if the GISA is not part of the GIB alert list. All three
 241  * fields are located in the first long word of the GISA.
 242  *
 243  * Returns: 0 on success
 244  *          -EBUSY in case the gisa is part of the alert list
 245  */
 246 static inline int gisa_set_iam(struct kvm_s390_gisa *gisa, u8 iam)
 247 {
 248         u64 word, _word;
 249
 250         do {
 251                 word = READ_ONCE(gisa->u64.word[0]);
 252                 if ((u64)gisa != word >> 32)
 253                         return -EBUSY;
 254                 _word = (word & ~0xffUL) | iam;
 255         } while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
 256
 257         return 0;
 258 }
 259
 260 /**
 261  * gisa_clear_ipm - clear the GISA interruption pending mask
 262  *
 263  * @gisa: gisa to operate on
 264  *
 265  * Clear the IPM atomically with the next alert address and the IAM
 266  * of the GISA unconditionally. All three fields are located in the
 267  * first long word of the GISA.
 268  */
 269 static inline void gisa_clear_ipm(struct kvm_s390_gisa *gisa)
 270 {
 271         u64 word, _word;
 272
 273         do {
 274                 word = READ_ONCE(gisa->u64.word[0]);
 275                 _word = word & ~(0xffUL << 24);
 276         } while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
 277 }
 278
 279 /**
 280  * gisa_get_ipm_or_restore_iam - return IPM or restore GISA IAM
 281  *
 282  * @gi: gisa interrupt struct to work on
 283  *
 284  * Atomically restores the interruption alert mask if none of the
 285  * relevant ISCs are pending and return the IPM.
 286  *
 287  * Returns: the relevant pending ISCs
 288  */
 289 static inline u8 gisa_get_ipm_or_restore_iam(struct kvm_s390_gisa_interrupt *gi)
 290 {
 291         u8 pending_mask, alert_mask;
 292         u64 word, _word;
 293
 294         do {
 295                 word = READ_ONCE(gi->origin->u64.word[0]);
 296                 alert_mask = READ_ONCE(gi->alert.mask);
 297                 pending_mask = (u8)(word >> 24) & alert_mask;
 298                 if (pending_mask)
 299                         return pending_mask;
 300                 _word = (word & ~0xffUL) | alert_mask;
 301         } while (cmpxchg(&gi->origin->u64.word[0], word, _word) != word);
 302
 303         return 0;
 304 }
 305
 306 static inline int gisa_in_alert_list(struct kvm_s390_gisa *gisa)
 307 {
 308         return READ_ONCE(gisa->next_alert) != (u32)virt_to_phys(gisa);
 309 }
 310
 311 static inline void gisa_set_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
 312 {
 313         set_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
 314 }
 315
 316 static inline u8 gisa_get_ipm(struct kvm_s390_gisa *gisa)
 317 {
 318         return READ_ONCE(gisa->ipm);
 319 }
 320
 321 static inline int gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
 322 {
 323         return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
 324 }
 325
 326 static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu)
 327 {
 328         unsigned long pending = vcpu->kvm->arch.float_int.pending_irqs |
 329                                 vcpu->arch.local_int.pending_irqs;
 330
 331         pending &= ~vcpu->kvm->arch.float_int.masked_irqs;
 332         return pending;
 333 }
 334
 335 static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu)
 336 {
 337         struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
 338         unsigned long pending_mask;
 339
 340         pending_mask = pending_irqs_no_gisa(vcpu);
 341         if (gi->origin)
 342                 pending_mask |= gisa_get_ipm(gi->origin) << IRQ_PEND_IO_ISC_7;
 343         return pending_mask;
 344 }
 345
 346 static inline int isc_to_irq_type(unsigned long isc)
 347 {
 348         return IRQ_PEND_IO_ISC_0 - isc;
 349 }
 350
 351 static inline int irq_type_to_isc(unsigned long irq_type)
 352 {
 353         return IRQ_PEND_IO_ISC_0 - irq_type;
 354 }
 355
 356 static unsigned long disable_iscs(struct kvm_vcpu *vcpu,
 357                                    unsigned long active_mask)
 358 {
 359         int i;
 360
 361         for (i = 0; i <= MAX_ISC; i++)
 362                 if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i)))
 363                         active_mask &= ~(1UL << (isc_to_irq_type(i)));
 364
 365         return active_mask;
 366 }
 367
 368 static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
 369 {
 370         unsigned long active_mask;
 371
 372         active_mask = pending_irqs(vcpu);
 373         if (!active_mask)
 374                 return 0;
 375
 376         if (psw_extint_disabled(vcpu))
 377                 active_mask &= ~IRQ_PEND_EXT_MASK;
 378         if (psw_ioint_disabled(vcpu))
 379                 active_mask &= ~IRQ_PEND_IO_MASK;
 380         else
 381                 active_mask = disable_iscs(vcpu, active_mask);
 382         if (!(vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
 383                 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask);
 384         if (!(vcpu->arch.sie_block->gcr[0] & CR0_EMERGENCY_SIGNAL_SUBMASK))
 385                 __clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask);
 386         if (!(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
 387                 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask);
 388         if (!(vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK))
 389                 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask);
 390         if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK)) {
 391                 __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask);
 392                 __clear_bit(IRQ_PEND_EXT_SERVICE_EV, &active_mask);
 393         }
 394         if (psw_mchk_disabled(vcpu))
 395                 active_mask &= ~IRQ_PEND_MCHK_MASK;
 396         /* PV guest cpus can have a single interruption injected at a time. */
 397         if (kvm_s390_pv_cpu_get_handle(vcpu) &&
 398             vcpu->arch.sie_block->iictl != IICTL_CODE_NONE)
 399                 active_mask &= ~(IRQ_PEND_EXT_II_MASK |
 400                                  IRQ_PEND_IO_MASK |
 401                                  IRQ_PEND_MCHK_MASK);
 402         /*
 403          * Check both floating and local interrupt's cr14 because
 404          * bit IRQ_PEND_MCHK_REP could be set in both cases.
 405          */
 406         if (!(vcpu->arch.sie_block->gcr[14] &
 407            (vcpu->kvm->arch.float_int.mchk.cr14 |
 408            vcpu->arch.local_int.irq.mchk.cr14)))
 409                 __clear_bit(IRQ_PEND_MCHK_REP, &active_mask);
 410
 411         /*
 412          * STOP irqs will never be actively delivered. They are triggered via
 413          * intercept requests and cleared when the stop intercept is performed.
 414          */
 415         __clear_bit(IRQ_PEND_SIGP_STOP, &active_mask);
 416
 417         return active_mask;
 418 }
 419
 420 static void __set_cpu_idle(struct kvm_vcpu *vcpu)
 421 {
 422         kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT);
 423         set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
 424 }
 425
 426 static void __unset_cpu_idle(struct kvm_vcpu *vcpu)
 427 {
 428         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT);
 429         clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
 430 }
 431
 432 static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
 433 {
 434         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IO_INT | CPUSTAT_EXT_INT |
 435                                       CPUSTAT_STOP_INT);
 436         vcpu->arch.sie_block->lctl = 0x0000;
 437         vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT);
 438
 439         if (guestdbg_enabled(vcpu)) {
 440                 vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 |
 441                                                LCTL_CR10 | LCTL_CR11);
 442                 vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT);
 443         }
 444 }
 445
 446 static void set_intercept_indicators_io(struct kvm_vcpu *vcpu)
 447 {
 448         if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_IO_MASK))
 449                 return;
 450         if (psw_ioint_disabled(vcpu))
 451                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT);
 452         else
 453                 vcpu->arch.sie_block->lctl |= LCTL_CR6;
 454 }
 455
 456 static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu)
 457 {
 458         if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_EXT_MASK))
 459                 return;
 460         if (psw_extint_disabled(vcpu))
 461                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
 462         else
 463                 vcpu->arch.sie_block->lctl |= LCTL_CR0;
 464 }
 465
 466 static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu)
 467 {
 468         if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_MCHK_MASK))
 469                 return;
 470         if (psw_mchk_disabled(vcpu))
 471                 vcpu->arch.sie_block->ictl |= ICTL_LPSW;
 472         else
 473                 vcpu->arch.sie_block->lctl |= LCTL_CR14;
 474 }
 475
 476 static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu)
 477 {
 478         if (kvm_s390_is_stop_irq_pending(vcpu))
 479                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
 480 }
 481
 482 /* Set interception request for non-deliverable interrupts */
 483 static void set_intercept_indicators(struct kvm_vcpu *vcpu)
 484 {
 485         set_intercept_indicators_io(vcpu);
 486         set_intercept_indicators_ext(vcpu);
 487         set_intercept_indicators_mchk(vcpu);
 488         set_intercept_indicators_stop(vcpu);
 489 }
 490
 491 static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu)
 492 {
 493         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 494         int rc = 0;
 495
 496         vcpu->stat.deliver_cputm++;
 497         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
 498                                          0, 0);
 499         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 500                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 501                 vcpu->arch.sie_block->eic = EXT_IRQ_CPU_TIMER;
 502         } else {
 503                 rc  = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER,
 504                                    (u16 *)__LC_EXT_INT_CODE);
 505                 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 506                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 507                                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 508                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 509                                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 510         }
 511         clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
 512         return rc ? -EFAULT : 0;
 513 }
 514
 515 static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu)
 516 {
 517         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 518         int rc = 0;
 519
 520         vcpu->stat.deliver_ckc++;
 521         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
 522                                          0, 0);
 523         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 524                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 525                 vcpu->arch.sie_block->eic = EXT_IRQ_CLK_COMP;
 526         } else {
 527                 rc  = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP,
 528                                    (u16 __user *)__LC_EXT_INT_CODE);
 529                 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 530                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 531                                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 532                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 533                                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 534         }
 535         clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
 536         return rc ? -EFAULT : 0;
 537 }
 538
 539 static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu)
 540 {
 541         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 542         struct kvm_s390_ext_info ext;
 543         int rc;
 544
 545         spin_lock(&li->lock);
 546         ext = li->irq.ext;
 547         clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
 548         li->irq.ext.ext_params2 = 0;
 549         spin_unlock(&li->lock);
 550
 551         VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx",
 552                    ext.ext_params2);
 553         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 554                                          KVM_S390_INT_PFAULT_INIT,
 555                                          0, ext.ext_params2);
 556
 557         rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE);
 558         rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR);
 559         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 560                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 561         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 562                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 563         rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2);
 564         return rc ? -EFAULT : 0;
 565 }
 566
 567 static int __write_machine_check(struct kvm_vcpu *vcpu,
 568                                  struct kvm_s390_mchk_info *mchk)
 569 {
 570         unsigned long ext_sa_addr;
 571         unsigned long lc;
 572         freg_t fprs[NUM_FPRS];
 573         union mci mci;
 574         int rc;
 575
 576         /*
 577          * All other possible payload for a machine check (e.g. the register
 578          * contents in the save area) will be handled by the ultravisor, as
 579          * the hypervisor does not not have the needed information for
 580          * protected guests.
 581          */
 582         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 583                 vcpu->arch.sie_block->iictl = IICTL_CODE_MCHK;
 584                 vcpu->arch.sie_block->mcic = mchk->mcic;
 585                 vcpu->arch.sie_block->faddr = mchk->failing_storage_address;
 586                 vcpu->arch.sie_block->edc = mchk->ext_damage_code;
 587                 return 0;
 588         }
 589
 590         mci.val = mchk->mcic;
 591         /* take care of lazy register loading */
 592         save_fpu_regs();
 593         save_access_regs(vcpu->run->s.regs.acrs);
 594         if (MACHINE_HAS_GS && vcpu->arch.gs_enabled)
 595                 save_gs_cb(current->thread.gs_cb);
 596
 597         /* Extended save area */
 598         rc = read_guest_lc(vcpu, __LC_MCESAD, &ext_sa_addr,
 599                            sizeof(unsigned long));
 600         /* Only bits 0 through 63-LC are used for address formation */
 601         lc = ext_sa_addr & MCESA_LC_MASK;
 602         if (test_kvm_facility(vcpu->kvm, 133)) {
 603                 switch (lc) {
 604                 case 0:
 605                 case 10:
 606                         ext_sa_addr &= ~0x3ffUL;
 607                         break;
 608                 case 11:
 609                         ext_sa_addr &= ~0x7ffUL;
 610                         break;
 611                 case 12:
 612                         ext_sa_addr &= ~0xfffUL;
 613                         break;
 614                 default:
 615                         ext_sa_addr = 0;
 616                         break;
 617                 }
 618         } else {
 619                 ext_sa_addr &= ~0x3ffUL;
 620         }
 621
 622         if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(vcpu->kvm, 129)) {
 623                 if (write_guest_abs(vcpu, ext_sa_addr, vcpu->run->s.regs.vrs,
 624                                     512))
 625                         mci.vr = 0;
 626         } else {
 627                 mci.vr = 0;
 628         }
 629         if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(vcpu->kvm, 133)
 630             && (lc == 11 || lc == 12)) {
 631                 if (write_guest_abs(vcpu, ext_sa_addr + 1024,
 632                                     &vcpu->run->s.regs.gscb, 32))
 633                         mci.gs = 0;
 634         } else {
 635                 mci.gs = 0;
 636         }
 637
 638         /* General interruption information */
 639         rc |= put_guest_lc(vcpu, 1, (u8 __user *) __LC_AR_MODE_ID);
 640         rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
 641                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 642         rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
 643                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 644         rc |= put_guest_lc(vcpu, mci.val, (u64 __user *) __LC_MCCK_CODE);
 645
 646         /* Register-save areas */
 647         if (MACHINE_HAS_VX) {
 648                 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
 649                 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, fprs, 128);
 650         } else {
 651                 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA,
 652                                      vcpu->run->s.regs.fprs, 128);
 653         }
 654         rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA,
 655                              vcpu->run->s.regs.gprs, 128);
 656         rc |= put_guest_lc(vcpu, current->thread.fpu.fpc,
 657                            (u32 __user *) __LC_FP_CREG_SAVE_AREA);
 658         rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr,
 659                            (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA);
 660         rc |= put_guest_lc(vcpu, kvm_s390_get_cpu_timer(vcpu),
 661                            (u64 __user *) __LC_CPU_TIMER_SAVE_AREA);
 662         rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->ckc >> 8,
 663                            (u64 __user *) __LC_CLOCK_COMP_SAVE_AREA);
 664         rc |= write_guest_lc(vcpu, __LC_AREGS_SAVE_AREA,
 665                              &vcpu->run->s.regs.acrs, 64);
 666         rc |= write_guest_lc(vcpu, __LC_CREGS_SAVE_AREA,
 667                              &vcpu->arch.sie_block->gcr, 128);
 668
 669         /* Extended interruption information */
 670         rc |= put_guest_lc(vcpu, mchk->ext_damage_code,
 671                            (u32 __user *) __LC_EXT_DAMAGE_CODE);
 672         rc |= put_guest_lc(vcpu, mchk->failing_storage_address,
 673                            (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
 674         rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, &mchk->fixed_logout,
 675                              sizeof(mchk->fixed_logout));
 676         return rc ? -EFAULT : 0;
 677 }
 678
 679 static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu)
 680 {
 681         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
 682         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 683         struct kvm_s390_mchk_info mchk = {};
 684         int deliver = 0;
 685         int rc = 0;
 686
 687         spin_lock(&fi->lock);
 688         spin_lock(&li->lock);
 689         if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) ||
 690             test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) {
 691                 /*
 692                  * If there was an exigent machine check pending, then any
 693                  * repressible machine checks that might have been pending
 694                  * are indicated along with it, so always clear bits for
 695                  * repressible and exigent interrupts
 696                  */
 697                 mchk = li->irq.mchk;
 698                 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
 699                 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
 700                 memset(&li->irq.mchk, 0, sizeof(mchk));
 701                 deliver = 1;
 702         }
 703         /*
 704          * We indicate floating repressible conditions along with
 705          * other pending conditions. Channel Report Pending and Channel
 706          * Subsystem damage are the only two and are indicated by
 707          * bits in mcic and masked in cr14.
 708          */
 709         if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
 710                 mchk.mcic |= fi->mchk.mcic;
 711                 mchk.cr14 |= fi->mchk.cr14;
 712                 memset(&fi->mchk, 0, sizeof(mchk));
 713                 deliver = 1;
 714         }
 715         spin_unlock(&li->lock);
 716         spin_unlock(&fi->lock);
 717
 718         if (deliver) {
 719                 VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx",
 720                            mchk.mcic);
 721                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 722                                                  KVM_S390_MCHK,
 723                                                  mchk.cr14, mchk.mcic);
 724                 vcpu->stat.deliver_machine_check++;
 725                 rc = __write_machine_check(vcpu, &mchk);
 726         }
 727         return rc;
 728 }
 729
 730 static int __must_check __deliver_restart(struct kvm_vcpu *vcpu)
 731 {
 732         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 733         int rc = 0;
 734
 735         VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart");
 736         vcpu->stat.deliver_restart_signal++;
 737         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
 738
 739         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 740                 vcpu->arch.sie_block->iictl = IICTL_CODE_RESTART;
 741         } else {
 742                 rc  = write_guest_lc(vcpu,
 743                                      offsetof(struct lowcore, restart_old_psw),
 744                                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 745                 rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw),
 746                                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 747         }
 748         clear_bit(IRQ_PEND_RESTART, &li->pending_irqs);
 749         return rc ? -EFAULT : 0;
 750 }
 751
 752 static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu)
 753 {
 754         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 755         struct kvm_s390_prefix_info prefix;
 756
 757         spin_lock(&li->lock);
 758         prefix = li->irq.prefix;
 759         li->irq.prefix.address = 0;
 760         clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
 761         spin_unlock(&li->lock);
 762
 763         vcpu->stat.deliver_prefix_signal++;
 764         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 765                                          KVM_S390_SIGP_SET_PREFIX,
 766                                          prefix.address, 0);
 767
 768         kvm_s390_set_prefix(vcpu, prefix.address);
 769         return 0;
 770 }
 771
 772 static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu)
 773 {
 774         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 775         int rc;
 776         int cpu_addr;
 777
 778         spin_lock(&li->lock);
 779         cpu_addr = find_first_bit(li->sigp_emerg_pending, KVM_MAX_VCPUS);
 780         clear_bit(cpu_addr, li->sigp_emerg_pending);
 781         if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS))
 782                 clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
 783         spin_unlock(&li->lock);
 784
 785         VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg");
 786         vcpu->stat.deliver_emergency_signal++;
 787         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
 788                                          cpu_addr, 0);
 789         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 790                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 791                 vcpu->arch.sie_block->eic = EXT_IRQ_EMERGENCY_SIG;
 792                 vcpu->arch.sie_block->extcpuaddr = cpu_addr;
 793                 return 0;
 794         }
 795
 796         rc  = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG,
 797                            (u16 *)__LC_EXT_INT_CODE);
 798         rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR);
 799         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 800                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 801         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 802                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 803         return rc ? -EFAULT : 0;
 804 }
 805
 806 static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu)
 807 {
 808         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 809         struct kvm_s390_extcall_info extcall;
 810         int rc;
 811
 812         spin_lock(&li->lock);
 813         extcall = li->irq.extcall;
 814         li->irq.extcall.code = 0;
 815         clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
 816         spin_unlock(&li->lock);
 817
 818         VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call");
 819         vcpu->stat.deliver_external_call++;
 820         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 821                                          KVM_S390_INT_EXTERNAL_CALL,
 822                                          extcall.code, 0);
 823         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 824                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 825                 vcpu->arch.sie_block->eic = EXT_IRQ_EXTERNAL_CALL;
 826                 vcpu->arch.sie_block->extcpuaddr = extcall.code;
 827                 return 0;
 828         }
 829
 830         rc  = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL,
 831                            (u16 *)__LC_EXT_INT_CODE);
 832         rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR);
 833         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 834                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 835         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw,
 836                             sizeof(psw_t));
 837         return rc ? -EFAULT : 0;
 838 }
 839
 840 static int __deliver_prog_pv(struct kvm_vcpu *vcpu, u16 code)
 841 {
 842         switch (code) {
 843         case PGM_SPECIFICATION:
 844                 vcpu->arch.sie_block->iictl = IICTL_CODE_SPECIFICATION;
 845                 break;
 846         case PGM_OPERAND:
 847                 vcpu->arch.sie_block->iictl = IICTL_CODE_OPERAND;
 848                 break;
 849         default:
 850                 return -EINVAL;
 851         }
 852         return 0;
 853 }
 854
 855 static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
 856 {
 857         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 858         struct kvm_s390_pgm_info pgm_info;
 859         int rc = 0, nullifying = false;
 860         u16 ilen;
 861
 862         spin_lock(&li->lock);
 863         pgm_info = li->irq.pgm;
 864         clear_bit(IRQ_PEND_PROG, &li->pending_irqs);
 865         memset(&li->irq.pgm, 0, sizeof(pgm_info));
 866         spin_unlock(&li->lock);
 867
 868         ilen = pgm_info.flags & KVM_S390_PGM_FLAGS_ILC_MASK;
 869         VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilen:%d",
 870                    pgm_info.code, ilen);
 871         vcpu->stat.deliver_program++;
 872         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
 873                                          pgm_info.code, 0);
 874
 875         /* PER is handled by the ultravisor */
 876         if (kvm_s390_pv_cpu_is_protected(vcpu))
 877                 return __deliver_prog_pv(vcpu, pgm_info.code & ~PGM_PER);
 878
 879         switch (pgm_info.code & ~PGM_PER) {
 880         case PGM_AFX_TRANSLATION:
 881         case PGM_ASX_TRANSLATION:
 882         case PGM_EX_TRANSLATION:
 883         case PGM_LFX_TRANSLATION:
 884         case PGM_LSTE_SEQUENCE:
 885         case PGM_LSX_TRANSLATION:
 886         case PGM_LX_TRANSLATION:
 887         case PGM_PRIMARY_AUTHORITY:
 888         case PGM_SECONDARY_AUTHORITY:
 889                 nullifying = true;
 890                 fallthrough;
 891         case PGM_SPACE_SWITCH:
 892                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 893                                   (u64 *)__LC_TRANS_EXC_CODE);
 894                 break;
 895         case PGM_ALEN_TRANSLATION:
 896         case PGM_ALE_SEQUENCE:
 897         case PGM_ASTE_INSTANCE:
 898         case PGM_ASTE_SEQUENCE:
 899         case PGM_ASTE_VALIDITY:
 900         case PGM_EXTENDED_AUTHORITY:
 901                 rc = put_guest_lc(vcpu, pgm_info.exc_access_id,
 902                                   (u8 *)__LC_EXC_ACCESS_ID);
 903                 nullifying = true;
 904                 break;
 905         case PGM_ASCE_TYPE:
 906         case PGM_PAGE_TRANSLATION:
 907         case PGM_REGION_FIRST_TRANS:
 908         case PGM_REGION_SECOND_TRANS:
 909         case PGM_REGION_THIRD_TRANS:
 910         case PGM_SEGMENT_TRANSLATION:
 911                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 912                                   (u64 *)__LC_TRANS_EXC_CODE);
 913                 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
 914                                    (u8 *)__LC_EXC_ACCESS_ID);
 915                 rc |= put_guest_lc(vcpu, pgm_info.op_access_id,
 916                                    (u8 *)__LC_OP_ACCESS_ID);
 917                 nullifying = true;
 918                 break;
 919         case PGM_MONITOR:
 920                 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr,
 921                                   (u16 *)__LC_MON_CLASS_NR);
 922                 rc |= put_guest_lc(vcpu, pgm_info.mon_code,
 923                                    (u64 *)__LC_MON_CODE);
 924                 break;
 925         case PGM_VECTOR_PROCESSING:
 926         case PGM_DATA:
 927                 rc = put_guest_lc(vcpu, pgm_info.data_exc_code,
 928                                   (u32 *)__LC_DATA_EXC_CODE);
 929                 break;
 930         case PGM_PROTECTION:
 931                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 932                                   (u64 *)__LC_TRANS_EXC_CODE);
 933                 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
 934                                    (u8 *)__LC_EXC_ACCESS_ID);
 935                 break;
 936         case PGM_STACK_FULL:
 937         case PGM_STACK_EMPTY:
 938         case PGM_STACK_SPECIFICATION:
 939         case PGM_STACK_TYPE:
 940         case PGM_STACK_OPERATION:
 941         case PGM_TRACE_TABEL:
 942         case PGM_CRYPTO_OPERATION:
 943                 nullifying = true;
 944                 break;
 945         }
 946
 947         if (pgm_info.code & PGM_PER) {
 948                 rc |= put_guest_lc(vcpu, pgm_info.per_code,
 949                                    (u8 *) __LC_PER_CODE);
 950                 rc |= put_guest_lc(vcpu, pgm_info.per_atmid,
 951                                    (u8 *)__LC_PER_ATMID);
 952                 rc |= put_guest_lc(vcpu, pgm_info.per_address,
 953                                    (u64 *) __LC_PER_ADDRESS);
 954                 rc |= put_guest_lc(vcpu, pgm_info.per_access_id,
 955                                    (u8 *) __LC_PER_ACCESS_ID);
 956         }
 957
 958         if (nullifying && !(pgm_info.flags & KVM_S390_PGM_FLAGS_NO_REWIND))
 959                 kvm_s390_rewind_psw(vcpu, ilen);
 960
 961         /* bit 1+2 of the target are the ilc, so we can directly use ilen */
 962         rc |= put_guest_lc(vcpu, ilen, (u16 *) __LC_PGM_ILC);
 963         rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea,
 964                                  (u64 *) __LC_PGM_LAST_BREAK);
 965         rc |= put_guest_lc(vcpu, pgm_info.code,
 966                            (u16 *)__LC_PGM_INT_CODE);
 967         rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW,
 968                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 969         rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW,
 970                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 971         return rc ? -EFAULT : 0;
 972 }
 973
 974 #define SCCB_MASK 0xFFFFFFF8
 975 #define SCCB_EVENT_PENDING 0x3
 976
 977 static int write_sclp(struct kvm_vcpu *vcpu, u32 parm)
 978 {
 979         int rc;
 980
 981         if (kvm_s390_pv_cpu_get_handle(vcpu)) {
 982                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 983                 vcpu->arch.sie_block->eic = EXT_IRQ_SERVICE_SIG;
 984                 vcpu->arch.sie_block->eiparams = parm;
 985                 return 0;
 986         }
 987
 988         rc  = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE);
 989         rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 990         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 991                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 992         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 993                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 994         rc |= put_guest_lc(vcpu, parm,
 995                            (u32 *)__LC_EXT_PARAMS);
 996
 997         return rc ? -EFAULT : 0;
 998 }
 999
1000 static int __must_check __deliver_service(struct kvm_vcpu *vcpu)
1001 {
1002         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1003         struct kvm_s390_ext_info ext;
1004
1005         spin_lock(&fi->lock);
1006         if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs) ||
1007             !(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) {
1008                 spin_unlock(&fi->lock);
1009                 return 0;
1010         }
1011         ext = fi->srv_signal;
1012         memset(&fi->srv_signal, 0, sizeof(ext));
1013         clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1014         clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1015         if (kvm_s390_pv_cpu_is_protected(vcpu))
1016                 set_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs);
1017         spin_unlock(&fi->lock);
1018
1019         VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x",
1020                    ext.ext_params);
1021         vcpu->stat.deliver_service_signal++;
1022         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
1023                                          ext.ext_params, 0);
1024
1025         return write_sclp(vcpu, ext.ext_params);
1026 }
1027
1028 static int __must_check __deliver_service_ev(struct kvm_vcpu *vcpu)
1029 {
1030         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1031         struct kvm_s390_ext_info ext;
1032
1033         spin_lock(&fi->lock);
1034         if (!(test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs))) {
1035                 spin_unlock(&fi->lock);
1036                 return 0;
1037         }
1038         ext = fi->srv_signal;
1039         /* only clear the event bit */
1040         fi->srv_signal.ext_params &= ~SCCB_EVENT_PENDING;
1041         clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1042         spin_unlock(&fi->lock);
1043
1044         VCPU_EVENT(vcpu, 4, "%s", "deliver: sclp parameter event");
1045         vcpu->stat.deliver_service_signal++;
1046         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
1047                                          ext.ext_params, 0);
1048
1049         return write_sclp(vcpu, SCCB_EVENT_PENDING);
1050 }
1051
1052 static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu)
1053 {
1054         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1055         struct kvm_s390_interrupt_info *inti;
1056         int rc = 0;
1057
1058         spin_lock(&fi->lock);
1059         inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT],
1060                                         struct kvm_s390_interrupt_info,
1061                                         list);
1062         if (inti) {
1063                 list_del(&inti->list);
1064                 fi->counters[FIRQ_CNTR_PFAULT] -= 1;
1065         }
1066         if (list_empty(&fi->lists[FIRQ_LIST_PFAULT]))
1067                 clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1068         spin_unlock(&fi->lock);
1069
1070         if (inti) {
1071                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1072                                                  KVM_S390_INT_PFAULT_DONE, 0,
1073                                                  inti->ext.ext_params2);
1074                 VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx",
1075                            inti->ext.ext_params2);
1076
1077                 rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
1078                                 (u16 *)__LC_EXT_INT_CODE);
1079                 rc |= put_guest_lc(vcpu, PFAULT_DONE,
1080                                 (u16 *)__LC_EXT_CPU_ADDR);
1081                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
1082                                 &vcpu->arch.sie_block->gpsw,
1083                                 sizeof(psw_t));
1084                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
1085                                 &vcpu->arch.sie_block->gpsw,
1086                                 sizeof(psw_t));
1087                 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
1088                                 (u64 *)__LC_EXT_PARAMS2);
1089                 kfree(inti);
1090         }
1091         return rc ? -EFAULT : 0;
1092 }
1093
1094 static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu)
1095 {
1096         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1097         struct kvm_s390_interrupt_info *inti;
1098         int rc = 0;
1099
1100         spin_lock(&fi->lock);
1101         inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO],
1102                                         struct kvm_s390_interrupt_info,
1103                                         list);
1104         if (inti) {
1105                 VCPU_EVENT(vcpu, 4,
1106                            "deliver: virtio parm: 0x%x,parm64: 0x%llx",
1107                            inti->ext.ext_params, inti->ext.ext_params2);
1108                 vcpu->stat.deliver_virtio++;
1109                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1110                                 inti->type,
1111                                 inti->ext.ext_params,
1112                                 inti->ext.ext_params2);
1113                 list_del(&inti->list);
1114                 fi->counters[FIRQ_CNTR_VIRTIO] -= 1;
1115         }
1116         if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO]))
1117                 clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1118         spin_unlock(&fi->lock);
1119
1120         if (inti) {
1121                 rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
1122                                 (u16 *)__LC_EXT_INT_CODE);
1123                 rc |= put_guest_lc(vcpu, VIRTIO_PARAM,
1124                                 (u16 *)__LC_EXT_CPU_ADDR);
1125                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
1126                                 &vcpu->arch.sie_block->gpsw,
1127                                 sizeof(psw_t));
1128                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
1129                                 &vcpu->arch.sie_block->gpsw,
1130                                 sizeof(psw_t));
1131                 rc |= put_guest_lc(vcpu, inti->ext.ext_params,
1132                                 (u32 *)__LC_EXT_PARAMS);
1133                 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
1134                                 (u64 *)__LC_EXT_PARAMS2);
1135                 kfree(inti);
1136         }
1137         return rc ? -EFAULT : 0;
1138 }
1139
1140 static int __do_deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info *io)
1141 {
1142         int rc;
1143
1144         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
1145                 vcpu->arch.sie_block->iictl = IICTL_CODE_IO;
1146                 vcpu->arch.sie_block->subchannel_id = io->subchannel_id;
1147                 vcpu->arch.sie_block->subchannel_nr = io->subchannel_nr;
1148                 vcpu->arch.sie_block->io_int_parm = io->io_int_parm;
1149                 vcpu->arch.sie_block->io_int_word = io->io_int_word;
1150                 return 0;
1151         }
1152
1153         rc  = put_guest_lc(vcpu, io->subchannel_id, (u16 *)__LC_SUBCHANNEL_ID);
1154         rc |= put_guest_lc(vcpu, io->subchannel_nr, (u16 *)__LC_SUBCHANNEL_NR);
1155         rc |= put_guest_lc(vcpu, io->io_int_parm, (u32 *)__LC_IO_INT_PARM);
1156         rc |= put_guest_lc(vcpu, io->io_int_word, (u32 *)__LC_IO_INT_WORD);
1157         rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
1158                              &vcpu->arch.sie_block->gpsw,
1159                              sizeof(psw_t));
1160         rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
1161                             &vcpu->arch.sie_block->gpsw,
1162                             sizeof(psw_t));
1163         return rc ? -EFAULT : 0;
1164 }
1165
1166 static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
1167                                      unsigned long irq_type)
1168 {
1169         struct list_head *isc_list;
1170         struct kvm_s390_float_interrupt *fi;
1171         struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
1172         struct kvm_s390_interrupt_info *inti = NULL;
1173         struct kvm_s390_io_info io;
1174         u32 isc;
1175         int rc = 0;
1176
1177         fi = &vcpu->kvm->arch.float_int;
1178
1179         spin_lock(&fi->lock);
1180         isc = irq_type_to_isc(irq_type);
1181         isc_list = &fi->lists[isc];
1182         inti = list_first_entry_or_null(isc_list,
1183                                         struct kvm_s390_interrupt_info,
1184                                         list);
1185         if (inti) {
1186                 if (inti->type & KVM_S390_INT_IO_AI_MASK)
1187                         VCPU_EVENT(vcpu, 4, "%s", "deliver: I/O (AI)");
1188                 else
1189                         VCPU_EVENT(vcpu, 4, "deliver: I/O %x ss %x schid %04x",
1190                         inti->io.subchannel_id >> 8,
1191                         inti->io.subchannel_id >> 1 & 0x3,
1192                         inti->io.subchannel_nr);
1193
1194                 vcpu->stat.deliver_io++;
1195                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1196                                 inti->type,
1197                                 ((__u32)inti->io.subchannel_id << 16) |
1198                                 inti->io.subchannel_nr,
1199                                 ((__u64)inti->io.io_int_parm << 32) |
1200                                 inti->io.io_int_word);
1201                 list_del(&inti->list);
1202                 fi->counters[FIRQ_CNTR_IO] -= 1;
1203         }
1204         if (list_empty(isc_list))
1205                 clear_bit(irq_type, &fi->pending_irqs);
1206         spin_unlock(&fi->lock);
1207
1208         if (inti) {
1209                 rc = __do_deliver_io(vcpu, &(inti->io));
1210                 kfree(inti);
1211                 goto out;
1212         }
1213
1214         if (gi->origin && gisa_tac_ipm_gisc(gi->origin, isc)) {
1215                 /*
1216                  * in case an adapter interrupt was not delivered
1217                  * in SIE context KVM will handle the delivery
1218                  */
1219                 VCPU_EVENT(vcpu, 4, "%s isc %u", "deliver: I/O (AI/gisa)", isc);
1220                 memset(&io, 0, sizeof(io));
1221                 io.io_int_word = isc_to_int_word(isc);
1222                 vcpu->stat.deliver_io++;
1223                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1224                         KVM_S390_INT_IO(1, 0, 0, 0),
1225                         ((__u32)io.subchannel_id << 16) |
1226                         io.subchannel_nr,
1227                         ((__u64)io.io_int_parm << 32) |
1228                         io.io_int_word);
1229                 rc = __do_deliver_io(vcpu, &io);
1230         }
1231 out:
1232         return rc;
1233 }
1234
1235 /* Check whether an external call is pending (deliverable or not) */
1236 int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
1237 {
1238         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1239
1240         if (!sclp.has_sigpif)
1241                 return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
1242
1243         return sca_ext_call_pending(vcpu, NULL);
1244 }
1245
1246 int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
1247 {
1248         if (deliverable_irqs(vcpu))
1249                 return 1;
1250
1251         if (kvm_cpu_has_pending_timer(vcpu))
1252                 return 1;
1253
1254         /* external call pending and deliverable */
1255         if (kvm_s390_ext_call_pending(vcpu) &&
1256             !psw_extint_disabled(vcpu) &&
1257             (vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
1258                 return 1;
1259
1260         if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu))
1261                 return 1;
1262         return 0;
1263 }
1264
1265 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
1266 {
1267         return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu);
1268 }
1269
1270 static u64 __calculate_sltime(struct kvm_vcpu *vcpu)
1271 {
1272         const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
1273         const u64 ckc = vcpu->arch.sie_block->ckc;
1274         u64 cputm, sltime = 0;
1275
1276         if (ckc_interrupts_enabled(vcpu)) {
1277                 if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
1278                         if ((s64)now < (s64)ckc)
1279                                 sltime = tod_to_ns((s64)ckc - (s64)now);
1280                 } else if (now < ckc) {
1281                         sltime = tod_to_ns(ckc - now);
1282                 }
1283                 /* already expired */
1284                 if (!sltime)
1285                         return 0;
1286                 if (cpu_timer_interrupts_enabled(vcpu)) {
1287                         cputm = kvm_s390_get_cpu_timer(vcpu);
1288                         /* already expired? */
1289                         if (cputm >> 63)
1290                                 return 0;
1291                         return min_t(u64, sltime, tod_to_ns(cputm));
1292                 }
1293         } else if (cpu_timer_interrupts_enabled(vcpu)) {
1294                 sltime = kvm_s390_get_cpu_timer(vcpu);
1295                 /* already expired? */
1296                 if (sltime >> 63)
1297                         return 0;
1298         }
1299         return sltime;
1300 }
1301
1302 int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
1303 {
1304         struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
1305         u64 sltime;
1306
1307         vcpu->stat.exit_wait_state++;
1308
1309         /* fast path */
1310         if (kvm_arch_vcpu_runnable(vcpu))
1311                 return 0;
1312
1313         if (psw_interrupts_disabled(vcpu)) {
1314                 VCPU_EVENT(vcpu, 3, "%s", "disabled wait");
1315                 return -EOPNOTSUPP; /* disabled wait */
1316         }
1317
1318         if (gi->origin &&
1319             (gisa_get_ipm_or_restore_iam(gi) &
1320              vcpu->arch.sie_block->gcr[6] >> 24))
1321                 return 0;
1322
1323         if (!ckc_interrupts_enabled(vcpu) &&
1324             !cpu_timer_interrupts_enabled(vcpu)) {
1325                 VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer");
1326                 __set_cpu_idle(vcpu);
1327                 goto no_timer;
1328         }
1329
1330         sltime = __calculate_sltime(vcpu);
1331         if (!sltime)
1332                 return 0;
1333
1334         __set_cpu_idle(vcpu);
1335         hrtimer_start(&vcpu->arch.ckc_timer, sltime, HRTIMER_MODE_REL);
1336         VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime);
1337 no_timer:
1338         kvm_vcpu_srcu_read_unlock(vcpu);
1339         kvm_vcpu_halt(vcpu);
1340         vcpu->valid_wakeup = false;
1341         __unset_cpu_idle(vcpu);
1342         kvm_vcpu_srcu_read_lock(vcpu);
1343
1344         hrtimer_cancel(&vcpu->arch.ckc_timer);
1345         return 0;
1346 }
1347
1348 void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu)
1349 {
1350         vcpu->valid_wakeup = true;
1351         kvm_vcpu_wake_up(vcpu);
1352
1353         /*
1354          * The VCPU might not be sleeping but rather executing VSIE. Let's
1355          * kick it, so it leaves the SIE to process the request.
1356          */
1357         kvm_s390_vsie_kick(vcpu);
1358 }
1359
1360 enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer)
1361 {
1362         struct kvm_vcpu *vcpu;
1363         u64 sltime;
1364
1365         vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer);
1366         sltime = __calculate_sltime(vcpu);
1367
1368         /*
1369          * If the monotonic clock runs faster than the tod clock we might be
1370          * woken up too early and have to go back to sleep to avoid deadlocks.
1371          */
1372         if (sltime && hrtimer_forward_now(timer, ns_to_ktime(sltime)))
1373                 return HRTIMER_RESTART;
1374         kvm_s390_vcpu_wakeup(vcpu);
1375         return HRTIMER_NORESTART;
1376 }
1377
1378 void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu)
1379 {
1380         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1381
1382         spin_lock(&li->lock);
1383         li->pending_irqs = 0;
1384         bitmap_zero(li->sigp_emerg_pending, KVM_MAX_VCPUS);
1385         memset(&li->irq, 0, sizeof(li->irq));
1386         spin_unlock(&li->lock);
1387
1388         sca_clear_ext_call(vcpu);
1389 }
1390
1391 int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
1392 {
1393         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1394         int rc = 0;
1395         unsigned long irq_type;
1396         unsigned long irqs;
1397
1398         __reset_intercept_indicators(vcpu);
1399
1400         /* pending ckc conditions might have been invalidated */
1401         clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1402         if (ckc_irq_pending(vcpu))
1403                 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1404
1405         /* pending cpu timer conditions might have been invalidated */
1406         clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1407         if (cpu_timer_irq_pending(vcpu))
1408                 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1409
1410         while ((irqs = deliverable_irqs(vcpu)) && !rc) {
1411                 /* bits are in the reverse order of interrupt priority */
1412                 irq_type = find_last_bit(&irqs, IRQ_PEND_COUNT);
1413                 switch (irq_type) {
1414                 case IRQ_PEND_IO_ISC_0:
1415                 case IRQ_PEND_IO_ISC_1:
1416                 case IRQ_PEND_IO_ISC_2:
1417                 case IRQ_PEND_IO_ISC_3:
1418                 case IRQ_PEND_IO_ISC_4:
1419                 case IRQ_PEND_IO_ISC_5:
1420                 case IRQ_PEND_IO_ISC_6:
1421                 case IRQ_PEND_IO_ISC_7:
1422                         rc = __deliver_io(vcpu, irq_type);
1423                         break;
1424                 case IRQ_PEND_MCHK_EX:
1425                 case IRQ_PEND_MCHK_REP:
1426                         rc = __deliver_machine_check(vcpu);
1427                         break;
1428                 case IRQ_PEND_PROG:
1429                         rc = __deliver_prog(vcpu);
1430                         break;
1431                 case IRQ_PEND_EXT_EMERGENCY:
1432                         rc = __deliver_emergency_signal(vcpu);
1433                         break;
1434                 case IRQ_PEND_EXT_EXTERNAL:
1435                         rc = __deliver_external_call(vcpu);
1436                         break;
1437                 case IRQ_PEND_EXT_CLOCK_COMP:
1438                         rc = __deliver_ckc(vcpu);
1439                         break;
1440                 case IRQ_PEND_EXT_CPU_TIMER:
1441                         rc = __deliver_cpu_timer(vcpu);
1442                         break;
1443                 case IRQ_PEND_RESTART:
1444                         rc = __deliver_restart(vcpu);
1445                         break;
1446                 case IRQ_PEND_SET_PREFIX:
1447                         rc = __deliver_set_prefix(vcpu);
1448                         break;
1449                 case IRQ_PEND_PFAULT_INIT:
1450                         rc = __deliver_pfault_init(vcpu);
1451                         break;
1452                 case IRQ_PEND_EXT_SERVICE:
1453                         rc = __deliver_service(vcpu);
1454                         break;
1455                 case IRQ_PEND_EXT_SERVICE_EV:
1456                         rc = __deliver_service_ev(vcpu);
1457                         break;
1458                 case IRQ_PEND_PFAULT_DONE:
1459                         rc = __deliver_pfault_done(vcpu);
1460                         break;
1461                 case IRQ_PEND_VIRTIO:
1462                         rc = __deliver_virtio(vcpu);
1463                         break;
1464                 default:
1465                         WARN_ONCE(1, "Unknown pending irq type %ld", irq_type);
1466                         clear_bit(irq_type, &li->pending_irqs);
1467                 }
1468         }
1469
1470         set_intercept_indicators(vcpu);
1471
1472         return rc;
1473 }
1474
1475 static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1476 {
1477         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1478
1479         vcpu->stat.inject_program++;
1480         VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code);
1481         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
1482                                    irq->u.pgm.code, 0);
1483
1484         if (!(irq->u.pgm.flags & KVM_S390_PGM_FLAGS_ILC_VALID)) {
1485                 /* auto detection if no valid ILC was given */
1486                 irq->u.pgm.flags &= ~KVM_S390_PGM_FLAGS_ILC_MASK;
1487                 irq->u.pgm.flags |= kvm_s390_get_ilen(vcpu);
1488                 irq->u.pgm.flags |= KVM_S390_PGM_FLAGS_ILC_VALID;
1489         }
1490
1491         if (irq->u.pgm.code == PGM_PER) {
1492                 li->irq.pgm.code |= PGM_PER;
1493                 li->irq.pgm.flags = irq->u.pgm.flags;
1494                 /* only modify PER related information */
1495                 li->irq.pgm.per_address = irq->u.pgm.per_address;
1496                 li->irq.pgm.per_code = irq->u.pgm.per_code;
1497                 li->irq.pgm.per_atmid = irq->u.pgm.per_atmid;
1498                 li->irq.pgm.per_access_id = irq->u.pgm.per_access_id;
1499         } else if (!(irq->u.pgm.code & PGM_PER)) {
1500                 li->irq.pgm.code = (li->irq.pgm.code & PGM_PER) |
1501                                    irq->u.pgm.code;
1502                 li->irq.pgm.flags = irq->u.pgm.flags;
1503                 /* only modify non-PER information */
1504                 li->irq.pgm.trans_exc_code = irq->u.pgm.trans_exc_code;
1505                 li->irq.pgm.mon_code = irq->u.pgm.mon_code;
1506                 li->irq.pgm.data_exc_code = irq->u.pgm.data_exc_code;
1507                 li->irq.pgm.mon_class_nr = irq->u.pgm.mon_class_nr;
1508                 li->irq.pgm.exc_access_id = irq->u.pgm.exc_access_id;
1509                 li->irq.pgm.op_access_id = irq->u.pgm.op_access_id;
1510         } else {
1511                 li->irq.pgm = irq->u.pgm;
1512         }
1513         set_bit(IRQ_PEND_PROG, &li->pending_irqs);
1514         return 0;
1515 }
1516
1517 static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1518 {
1519         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1520
1521         vcpu->stat.inject_pfault_init++;
1522         VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx",
1523                    irq->u.ext.ext_params2);
1524         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT,
1525                                    irq->u.ext.ext_params,
1526                                    irq->u.ext.ext_params2);
1527
1528         li->irq.ext = irq->u.ext;
1529         set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
1530         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1531         return 0;
1532 }
1533
1534 static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1535 {
1536         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1537         struct kvm_s390_extcall_info *extcall = &li->irq.extcall;
1538         uint16_t src_id = irq->u.extcall.code;
1539
1540         vcpu->stat.inject_external_call++;
1541         VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u",
1542                    src_id);
1543         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL,
1544                                    src_id, 0);
1545
1546         /* sending vcpu invalid */
1547         if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL)
1548                 return -EINVAL;
1549
1550         if (sclp.has_sigpif && !kvm_s390_pv_cpu_get_handle(vcpu))
1551                 return sca_inject_ext_call(vcpu, src_id);
1552
1553         if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs))
1554                 return -EBUSY;
1555         *extcall = irq->u.extcall;
1556         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1557         return 0;
1558 }
1559
1560 static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1561 {
1562         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1563         struct kvm_s390_prefix_info *prefix = &li->irq.prefix;
1564
1565         vcpu->stat.inject_set_prefix++;
1566         VCPU_EVENT(vcpu, 3, "inject: set prefix to %x",
1567                    irq->u.prefix.address);
1568         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX,
1569                                    irq->u.prefix.address, 0);
1570
1571         if (!is_vcpu_stopped(vcpu))
1572                 return -EBUSY;
1573
1574         *prefix = irq->u.prefix;
1575         set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
1576         return 0;
1577 }
1578
1579 #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS)
1580 static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1581 {
1582         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1583         struct kvm_s390_stop_info *stop = &li->irq.stop;
1584         int rc = 0;
1585
1586         vcpu->stat.inject_stop_signal++;
1587         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0);
1588
1589         if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS)
1590                 return -EINVAL;
1591
1592         if (is_vcpu_stopped(vcpu)) {
1593                 if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS)
1594                         rc = kvm_s390_store_status_unloaded(vcpu,
1595                                                 KVM_S390_STORE_STATUS_NOADDR);
1596                 return rc;
1597         }
1598
1599         if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs))
1600                 return -EBUSY;
1601         stop->flags = irq->u.stop.flags;
1602         kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
1603         return 0;
1604 }
1605
1606 static int __inject_sigp_restart(struct kvm_vcpu *vcpu)
1607 {
1608         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1609
1610         vcpu->stat.inject_restart++;
1611         VCPU_EVENT(vcpu, 3, "%s", "inject: restart int");
1612         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
1613
1614         set_bit(IRQ_PEND_RESTART, &li->pending_irqs);
1615         return 0;
1616 }
1617
1618 static int __inject_sigp_emergency(struct kvm_vcpu *vcpu,
1619                                    struct kvm_s390_irq *irq)
1620 {
1621         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1622
1623         vcpu->stat.inject_emergency_signal++;
1624         VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u",
1625                    irq->u.emerg.code);
1626         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
1627                                    irq->u.emerg.code, 0);
1628
1629         /* sending vcpu invalid */
1630         if (kvm_get_vcpu_by_id(vcpu->kvm, irq->u.emerg.code) == NULL)
1631                 return -EINVAL;
1632
1633         set_bit(irq->u.emerg.code, li->sigp_emerg_pending);
1634         set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
1635         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1636         return 0;
1637 }
1638
1639 static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1640 {
1641         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1642         struct kvm_s390_mchk_info *mchk = &li->irq.mchk;
1643
1644         vcpu->stat.inject_mchk++;
1645         VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx",
1646                    irq->u.mchk.mcic);
1647         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0,
1648                                    irq->u.mchk.mcic);
1649
1650         /*
1651          * Because repressible machine checks can be indicated along with
1652          * exigent machine checks (PoP, Chapter 11, Interruption action)
1653          * we need to combine cr14, mcic and external damage code.
1654          * Failing storage address and the logout area should not be or'ed
1655          * together, we just indicate the last occurrence of the corresponding
1656          * machine check
1657          */
1658         mchk->cr14 |= irq->u.mchk.cr14;
1659         mchk->mcic |= irq->u.mchk.mcic;
1660         mchk->ext_damage_code |= irq->u.mchk.ext_damage_code;
1661         mchk->failing_storage_address = irq->u.mchk.failing_storage_address;
1662         memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout,
1663                sizeof(mchk->fixed_logout));
1664         if (mchk->mcic & MCHK_EX_MASK)
1665                 set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
1666         else if (mchk->mcic & MCHK_REP_MASK)
1667                 set_bit(IRQ_PEND_MCHK_REP,  &li->pending_irqs);
1668         return 0;
1669 }
1670
1671 static int __inject_ckc(struct kvm_vcpu *vcpu)
1672 {
1673         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1674
1675         vcpu->stat.inject_ckc++;
1676         VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external");
1677         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
1678                                    0, 0);
1679
1680         set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1681         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1682         return 0;
1683 }
1684
1685 static int __inject_cpu_timer(struct kvm_vcpu *vcpu)
1686 {
1687         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1688
1689         vcpu->stat.inject_cputm++;
1690         VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external");
1691         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
1692                                    0, 0);
1693
1694         set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1695         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1696         return 0;
1697 }
1698
1699 static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
1700                                                   int isc, u32 schid)
1701 {
1702         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1703         struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1704         struct kvm_s390_interrupt_info *iter;
1705         u16 id = (schid & 0xffff0000U) >> 16;
1706         u16 nr = schid & 0x0000ffffU;
1707
1708         spin_lock(&fi->lock);
1709         list_for_each_entry(iter, isc_list, list) {
1710                 if (schid && (id != iter->io.subchannel_id ||
1711                               nr != iter->io.subchannel_nr))
1712                         continue;
1713                 /* found an appropriate entry */
1714                 list_del_init(&iter->list);
1715                 fi->counters[FIRQ_CNTR_IO] -= 1;
1716                 if (list_empty(isc_list))
1717                         clear_bit(isc_to_irq_type(isc), &fi->pending_irqs);
1718                 spin_unlock(&fi->lock);
1719                 return iter;
1720         }
1721         spin_unlock(&fi->lock);
1722         return NULL;
1723 }
1724
1725 static struct kvm_s390_interrupt_info *get_top_io_int(struct kvm *kvm,
1726                                                       u64 isc_mask, u32 schid)
1727 {
1728         struct kvm_s390_interrupt_info *inti = NULL;
1729         int isc;
1730
1731         for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
1732                 if (isc_mask & isc_to_isc_bits(isc))
1733                         inti = get_io_int(kvm, isc, schid);
1734         }
1735         return inti;
1736 }
1737
1738 static int get_top_gisa_isc(struct kvm *kvm, u64 isc_mask, u32 schid)
1739 {
1740         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1741         unsigned long active_mask;
1742         int isc;
1743
1744         if (schid)
1745                 goto out;
1746         if (!gi->origin)
1747                 goto out;
1748
1749         active_mask = (isc_mask & gisa_get_ipm(gi->origin) << 24) << 32;
1750         while (active_mask) {
1751                 isc = __fls(active_mask) ^ (BITS_PER_LONG - 1);
1752                 if (gisa_tac_ipm_gisc(gi->origin, isc))
1753                         return isc;
1754                 clear_bit_inv(isc, &active_mask);
1755         }
1756 out:
1757         return -EINVAL;
1758 }
1759
1760 /*
1761  * Dequeue and return an I/O interrupt matching any of the interruption
1762  * subclasses as designated by the isc mask in cr6 and the schid (if != 0).
1763  * Take into account the interrupts pending in the interrupt list and in GISA.
1764  *
1765  * Note that for a guest that does not enable I/O interrupts
1766  * but relies on TPI, a flood of classic interrupts may starve
1767  * out adapter interrupts on the same isc. Linux does not do
1768  * that, and it is possible to work around the issue by configuring
1769  * different iscs for classic and adapter interrupts in the guest,
1770  * but we may want to revisit this in the future.
1771  */
1772 struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
1773                                                     u64 isc_mask, u32 schid)
1774 {
1775         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1776         struct kvm_s390_interrupt_info *inti, *tmp_inti;
1777         int isc;
1778
1779         inti = get_top_io_int(kvm, isc_mask, schid);
1780
1781         isc = get_top_gisa_isc(kvm, isc_mask, schid);
1782         if (isc < 0)
1783                 /* no AI in GISA */
1784                 goto out;
1785
1786         if (!inti)
1787                 /* AI in GISA but no classical IO int */
1788                 goto gisa_out;
1789
1790         /* both types of interrupts present */
1791         if (int_word_to_isc(inti->io.io_int_word) <= isc) {
1792                 /* classical IO int with higher priority */
1793                 gisa_set_ipm_gisc(gi->origin, isc);
1794                 goto out;
1795         }
1796 gisa_out:
1797         tmp_inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
1798         if (tmp_inti) {
1799                 tmp_inti->type = KVM_S390_INT_IO(1, 0, 0, 0);
1800                 tmp_inti->io.io_int_word = isc_to_int_word(isc);
1801                 if (inti)
1802                         kvm_s390_reinject_io_int(kvm, inti);
1803                 inti = tmp_inti;
1804         } else
1805                 gisa_set_ipm_gisc(gi->origin, isc);
1806 out:
1807         return inti;
1808 }
1809
1810 static int __inject_service(struct kvm *kvm,
1811                              struct kvm_s390_interrupt_info *inti)
1812 {
1813         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1814
1815         kvm->stat.inject_service_signal++;
1816         spin_lock(&fi->lock);
1817         fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING;
1818
1819         /* We always allow events, track them separately from the sccb ints */
1820         if (fi->srv_signal.ext_params & SCCB_EVENT_PENDING)
1821                 set_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1822
1823         /*
1824          * Early versions of the QEMU s390 bios will inject several
1825          * service interrupts after another without handling a
1826          * condition code indicating busy.
1827          * We will silently ignore those superfluous sccb values.
1828          * A future version of QEMU will take care of serialization
1829          * of servc requests
1830          */
1831         if (fi->srv_signal.ext_params & SCCB_MASK)
1832                 goto out;
1833         fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK;
1834         set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1835 out:
1836         spin_unlock(&fi->lock);
1837         kfree(inti);
1838         return 0;
1839 }
1840
1841 static int __inject_virtio(struct kvm *kvm,
1842                             struct kvm_s390_interrupt_info *inti)
1843 {
1844         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1845
1846         kvm->stat.inject_virtio++;
1847         spin_lock(&fi->lock);
1848         if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) {
1849                 spin_unlock(&fi->lock);
1850                 return -EBUSY;
1851         }
1852         fi->counters[FIRQ_CNTR_VIRTIO] += 1;
1853         list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]);
1854         set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1855         spin_unlock(&fi->lock);
1856         return 0;
1857 }
1858
1859 static int __inject_pfault_done(struct kvm *kvm,
1860                                  struct kvm_s390_interrupt_info *inti)
1861 {
1862         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1863
1864         kvm->stat.inject_pfault_done++;
1865         spin_lock(&fi->lock);
1866         if (fi->counters[FIRQ_CNTR_PFAULT] >=
1867                 (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) {
1868                 spin_unlock(&fi->lock);
1869                 return -EBUSY;
1870         }
1871         fi->counters[FIRQ_CNTR_PFAULT] += 1;
1872         list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]);
1873         set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1874         spin_unlock(&fi->lock);
1875         return 0;
1876 }
1877
1878 #define CR_PENDING_SUBCLASS 28
1879 static int __inject_float_mchk(struct kvm *kvm,
1880                                 struct kvm_s390_interrupt_info *inti)
1881 {
1882         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1883
1884         kvm->stat.inject_float_mchk++;
1885         spin_lock(&fi->lock);
1886         fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS);
1887         fi->mchk.mcic |= inti->mchk.mcic;
1888         set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs);
1889         spin_unlock(&fi->lock);
1890         kfree(inti);
1891         return 0;
1892 }
1893
1894 static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1895 {
1896         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1897         struct kvm_s390_float_interrupt *fi;
1898         struct list_head *list;
1899         int isc;
1900
1901         kvm->stat.inject_io++;
1902         isc = int_word_to_isc(inti->io.io_int_word);
1903
1904         /*
1905          * We do not use the lock checking variant as this is just a
1906          * performance optimization and we do not hold the lock here.
1907          * This is ok as the code will pick interrupts from both "lists"
1908          * for delivery.
1909          */
1910         if (gi->origin && inti->type & KVM_S390_INT_IO_AI_MASK) {
1911                 VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc);
1912                 gisa_set_ipm_gisc(gi->origin, isc);
1913                 kfree(inti);
1914                 return 0;
1915         }
1916
1917         fi = &kvm->arch.float_int;
1918         spin_lock(&fi->lock);
1919         if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) {
1920                 spin_unlock(&fi->lock);
1921                 return -EBUSY;
1922         }
1923         fi->counters[FIRQ_CNTR_IO] += 1;
1924
1925         if (inti->type & KVM_S390_INT_IO_AI_MASK)
1926                 VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)");
1927         else
1928                 VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x",
1929                         inti->io.subchannel_id >> 8,
1930                         inti->io.subchannel_id >> 1 & 0x3,
1931                         inti->io.subchannel_nr);
1932         list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1933         list_add_tail(&inti->list, list);
1934         set_bit(isc_to_irq_type(isc), &fi->pending_irqs);
1935         spin_unlock(&fi->lock);
1936         return 0;
1937 }
1938
1939 /*
1940  * Find a destination VCPU for a floating irq and kick it.
1941  */
1942 static void __floating_irq_kick(struct kvm *kvm, u64 type)
1943 {
1944         struct kvm_vcpu *dst_vcpu;
1945         int sigcpu, online_vcpus, nr_tries = 0;
1946
1947         online_vcpus = atomic_read(&kvm->online_vcpus);
1948         if (!online_vcpus)
1949                 return;
1950
1951         /* find idle VCPUs first, then round robin */
1952         sigcpu = find_first_bit(kvm->arch.idle_mask, online_vcpus);
1953         if (sigcpu == online_vcpus) {
1954                 do {
1955                         sigcpu = kvm->arch.float_int.next_rr_cpu++;
1956                         kvm->arch.float_int.next_rr_cpu %= online_vcpus;
1957                         /* avoid endless loops if all vcpus are stopped */
1958                         if (nr_tries++ >= online_vcpus)
1959                                 return;
1960                 } while (is_vcpu_stopped(kvm_get_vcpu(kvm, sigcpu)));
1961         }
1962         dst_vcpu = kvm_get_vcpu(kvm, sigcpu);
1963
1964         /* make the VCPU drop out of the SIE, or wake it up if sleeping */
1965         switch (type) {
1966         case KVM_S390_MCHK:
1967                 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT);
1968                 break;
1969         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1970                 if (!(type & KVM_S390_INT_IO_AI_MASK &&
1971                       kvm->arch.gisa_int.origin) ||
1972                       kvm_s390_pv_cpu_get_handle(dst_vcpu))
1973                         kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT);
1974                 break;
1975         default:
1976                 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT);
1977                 break;
1978         }
1979         kvm_s390_vcpu_wakeup(dst_vcpu);
1980 }
1981
1982 static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1983 {
1984         u64 type = READ_ONCE(inti->type);
1985         int rc;
1986
1987         switch (type) {
1988         case KVM_S390_MCHK:
1989                 rc = __inject_float_mchk(kvm, inti);
1990                 break;
1991         case KVM_S390_INT_VIRTIO:
1992                 rc = __inject_virtio(kvm, inti);
1993                 break;
1994         case KVM_S390_INT_SERVICE:
1995                 rc = __inject_service(kvm, inti);
1996                 break;
1997         case KVM_S390_INT_PFAULT_DONE:
1998                 rc = __inject_pfault_done(kvm, inti);
1999                 break;
2000         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2001                 rc = __inject_io(kvm, inti);
2002                 break;
2003         default:
2004                 rc = -EINVAL;
2005         }
2006         if (rc)
2007                 return rc;
2008
2009         __floating_irq_kick(kvm, type);
2010         return 0;
2011 }
2012
2013 int kvm_s390_inject_vm(struct kvm *kvm,
2014                        struct kvm_s390_interrupt *s390int)
2015 {
2016         struct kvm_s390_interrupt_info *inti;
2017         int rc;
2018
2019         inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
2020         if (!inti)
2021                 return -ENOMEM;
2022
2023         inti->type = s390int->type;
2024         switch (inti->type) {
2025         case KVM_S390_INT_VIRTIO:
2026                 VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx",
2027                          s390int->parm, s390int->parm64);
2028                 inti->ext.ext_params = s390int->parm;
2029                 inti->ext.ext_params2 = s390int->parm64;
2030                 break;
2031         case KVM_S390_INT_SERVICE:
2032                 VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm);
2033                 inti->ext.ext_params = s390int->parm;
2034                 break;
2035         case KVM_S390_INT_PFAULT_DONE:
2036                 inti->ext.ext_params2 = s390int->parm64;
2037                 break;
2038         case KVM_S390_MCHK:
2039                 VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx",
2040                          s390int->parm64);
2041                 inti->mchk.cr14 = s390int->parm; /* upper bits are not used */
2042                 inti->mchk.mcic = s390int->parm64;
2043                 break;
2044         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2045                 inti->io.subchannel_id = s390int->parm >> 16;
2046                 inti->io.subchannel_nr = s390int->parm & 0x0000ffffu;
2047                 inti->io.io_int_parm = s390int->parm64 >> 32;
2048                 inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull;
2049                 break;
2050         default:
2051                 kfree(inti);
2052                 return -EINVAL;
2053         }
2054         trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64,
2055                                  2);
2056
2057         rc = __inject_vm(kvm, inti);
2058         if (rc)
2059                 kfree(inti);
2060         return rc;
2061 }
2062
2063 int kvm_s390_reinject_io_int(struct kvm *kvm,
2064                               struct kvm_s390_interrupt_info *inti)
2065 {
2066         return __inject_vm(kvm, inti);
2067 }
2068
2069 int s390int_to_s390irq(struct kvm_s390_interrupt *s390int,
2070                        struct kvm_s390_irq *irq)
2071 {
2072         irq->type = s390int->type;
2073         switch (irq->type) {
2074         case KVM_S390_PROGRAM_INT:
2075                 if (s390int->parm & 0xffff0000)
2076                         return -EINVAL;
2077                 irq->u.pgm.code = s390int->parm;
2078                 break;
2079         case KVM_S390_SIGP_SET_PREFIX:
2080                 irq->u.prefix.address = s390int->parm;
2081                 break;
2082         case KVM_S390_SIGP_STOP:
2083                 irq->u.stop.flags = s390int->parm;
2084                 break;
2085         case KVM_S390_INT_EXTERNAL_CALL:
2086                 if (s390int->parm & 0xffff0000)
2087                         return -EINVAL;
2088                 irq->u.extcall.code = s390int->parm;
2089                 break;
2090         case KVM_S390_INT_EMERGENCY:
2091                 if (s390int->parm & 0xffff0000)
2092                         return -EINVAL;
2093                 irq->u.emerg.code = s390int->parm;
2094                 break;
2095         case KVM_S390_MCHK:
2096                 irq->u.mchk.mcic = s390int->parm64;
2097                 break;
2098         case KVM_S390_INT_PFAULT_INIT:
2099                 irq->u.ext.ext_params = s390int->parm;
2100                 irq->u.ext.ext_params2 = s390int->parm64;
2101                 break;
2102         case KVM_S390_RESTART:
2103         case KVM_S390_INT_CLOCK_COMP:
2104         case KVM_S390_INT_CPU_TIMER:
2105                 break;
2106         default:
2107                 return -EINVAL;
2108         }
2109         return 0;
2110 }
2111
2112 int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu)
2113 {
2114         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2115
2116         return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
2117 }
2118
2119 int kvm_s390_is_restart_irq_pending(struct kvm_vcpu *vcpu)
2120 {
2121         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2122
2123         return test_bit(IRQ_PEND_RESTART, &li->pending_irqs);
2124 }
2125
2126 void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu)
2127 {
2128         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2129
2130         spin_lock(&li->lock);
2131         li->irq.stop.flags = 0;
2132         clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
2133         spin_unlock(&li->lock);
2134 }
2135
2136 static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
2137 {
2138         int rc;
2139
2140         switch (irq->type) {
2141         case KVM_S390_PROGRAM_INT:
2142                 rc = __inject_prog(vcpu, irq);
2143                 break;
2144         case KVM_S390_SIGP_SET_PREFIX:
2145                 rc = __inject_set_prefix(vcpu, irq);
2146                 break;
2147         case KVM_S390_SIGP_STOP:
2148                 rc = __inject_sigp_stop(vcpu, irq);
2149                 break;
2150         case KVM_S390_RESTART:
2151                 rc = __inject_sigp_restart(vcpu);
2152                 break;
2153         case KVM_S390_INT_CLOCK_COMP:
2154                 rc = __inject_ckc(vcpu);
2155                 break;
2156         case KVM_S390_INT_CPU_TIMER:
2157                 rc = __inject_cpu_timer(vcpu);
2158                 break;
2159         case KVM_S390_INT_EXTERNAL_CALL:
2160                 rc = __inject_extcall(vcpu, irq);
2161                 break;
2162         case KVM_S390_INT_EMERGENCY:
2163                 rc = __inject_sigp_emergency(vcpu, irq);
2164                 break;
2165         case KVM_S390_MCHK:
2166                 rc = __inject_mchk(vcpu, irq);
2167                 break;
2168         case KVM_S390_INT_PFAULT_INIT:
2169                 rc = __inject_pfault_init(vcpu, irq);
2170                 break;
2171         case KVM_S390_INT_VIRTIO:
2172         case KVM_S390_INT_SERVICE:
2173         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2174         default:
2175                 rc = -EINVAL;
2176         }
2177
2178         return rc;
2179 }
2180
2181 int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
2182 {
2183         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2184         int rc;
2185
2186         spin_lock(&li->lock);
2187         rc = do_inject_vcpu(vcpu, irq);
2188         spin_unlock(&li->lock);
2189         if (!rc)
2190                 kvm_s390_vcpu_wakeup(vcpu);
2191         return rc;
2192 }
2193
2194 static inline void clear_irq_list(struct list_head *_list)
2195 {
2196         struct kvm_s390_interrupt_info *inti, *n;
2197
2198         list_for_each_entry_safe(inti, n, _list, list) {
2199                 list_del(&inti->list);
2200                 kfree(inti);
2201         }
2202 }
2203
2204 static void inti_to_irq(struct kvm_s390_interrupt_info *inti,
2205                        struct kvm_s390_irq *irq)
2206 {
2207         irq->type = inti->type;
2208         switch (inti->type) {
2209         case KVM_S390_INT_PFAULT_INIT:
2210         case KVM_S390_INT_PFAULT_DONE:
2211         case KVM_S390_INT_VIRTIO:
2212                 irq->u.ext = inti->ext;
2213                 break;
2214         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2215                 irq->u.io = inti->io;
2216                 break;
2217         }
2218 }
2219
2220 void kvm_s390_clear_float_irqs(struct kvm *kvm)
2221 {
2222         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2223         int i;
2224
2225         mutex_lock(&kvm->lock);
2226         if (!kvm_s390_pv_is_protected(kvm))
2227                 fi->masked_irqs = 0;
2228         mutex_unlock(&kvm->lock);
2229         spin_lock(&fi->lock);
2230         fi->pending_irqs = 0;
2231         memset(&fi->srv_signal, 0, sizeof(fi->srv_signal));
2232         memset(&fi->mchk, 0, sizeof(fi->mchk));
2233         for (i = 0; i < FIRQ_LIST_COUNT; i++)
2234                 clear_irq_list(&fi->lists[i]);
2235         for (i = 0; i < FIRQ_MAX_COUNT; i++)
2236                 fi->counters[i] = 0;
2237         spin_unlock(&fi->lock);
2238         kvm_s390_gisa_clear(kvm);
2239 };
2240
2241 static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
2242 {
2243         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
2244         struct kvm_s390_interrupt_info *inti;
2245         struct kvm_s390_float_interrupt *fi;
2246         struct kvm_s390_irq *buf;
2247         struct kvm_s390_irq *irq;
2248         int max_irqs;
2249         int ret = 0;
2250         int n = 0;
2251         int i;
2252
2253         if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0)
2254                 return -EINVAL;
2255
2256         /*
2257          * We are already using -ENOMEM to signal
2258          * userspace it may retry with a bigger buffer,
2259          * so we need to use something else for this case
2260          */
2261         buf = vzalloc(len);
2262         if (!buf)
2263                 return -ENOBUFS;
2264
2265         max_irqs = len / sizeof(struct kvm_s390_irq);
2266
2267         if (gi->origin && gisa_get_ipm(gi->origin)) {
2268                 for (i = 0; i <= MAX_ISC; i++) {
2269                         if (n == max_irqs) {
2270                                 /* signal userspace to try again */
2271                                 ret = -ENOMEM;
2272                                 goto out_nolock;
2273                         }
2274                         if (gisa_tac_ipm_gisc(gi->origin, i)) {
2275                                 irq = (struct kvm_s390_irq *) &buf[n];
2276                                 irq->type = KVM_S390_INT_IO(1, 0, 0, 0);
2277                                 irq->u.io.io_int_word = isc_to_int_word(i);
2278                                 n++;
2279                         }
2280                 }
2281         }
2282         fi = &kvm->arch.float_int;
2283         spin_lock(&fi->lock);
2284         for (i = 0; i < FIRQ_LIST_COUNT; i++) {
2285                 list_for_each_entry(inti, &fi->lists[i], list) {
2286                         if (n == max_irqs) {
2287                                 /* signal userspace to try again */
2288                                 ret = -ENOMEM;
2289                                 goto out;
2290                         }
2291                         inti_to_irq(inti, &buf[n]);
2292                         n++;
2293                 }
2294         }
2295         if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs) ||
2296             test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs)) {
2297                 if (n == max_irqs) {
2298                         /* signal userspace to try again */
2299                         ret = -ENOMEM;
2300                         goto out;
2301                 }
2302                 irq = (struct kvm_s390_irq *) &buf[n];
2303                 irq->type = KVM_S390_INT_SERVICE;
2304                 irq->u.ext = fi->srv_signal;
2305                 n++;
2306         }
2307         if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
2308                 if (n == max_irqs) {
2309                                 /* signal userspace to try again */
2310                                 ret = -ENOMEM;
2311                                 goto out;
2312                 }
2313                 irq = (struct kvm_s390_irq *) &buf[n];
2314                 irq->type = KVM_S390_MCHK;
2315                 irq->u.mchk = fi->mchk;
2316                 n++;
2317 }
2318
2319 out:
2320         spin_unlock(&fi->lock);
2321 out_nolock:
2322         if (!ret && n > 0) {
2323                 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n))
2324                         ret = -EFAULT;
2325         }
2326         vfree(buf);
2327
2328         return ret < 0 ? ret : n;
2329 }
2330
2331 static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr)
2332 {
2333         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2334         struct kvm_s390_ais_all ais;
2335
2336         if (attr->attr < sizeof(ais))
2337                 return -EINVAL;
2338
2339         if (!test_kvm_facility(kvm, 72))
2340                 return -EOPNOTSUPP;
2341
2342         mutex_lock(&fi->ais_lock);
2343         ais.simm = fi->simm;
2344         ais.nimm = fi->nimm;
2345         mutex_unlock(&fi->ais_lock);
2346
2347         if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais)))
2348                 return -EFAULT;
2349
2350         return 0;
2351 }
2352
2353 static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2354 {
2355         int r;
2356
2357         switch (attr->group) {
2358         case KVM_DEV_FLIC_GET_ALL_IRQS:
2359                 r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr,
2360                                           attr->attr);
2361                 break;
2362         case KVM_DEV_FLIC_AISM_ALL:
2363                 r = flic_ais_mode_get_all(dev->kvm, attr);
2364                 break;
2365         default:
2366                 r = -EINVAL;
2367         }
2368
2369         return r;
2370 }
2371
2372 static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti,
2373                                      u64 addr)
2374 {
2375         struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr;
2376         void *target = NULL;
2377         void __user *source;
2378         u64 size;
2379
2380         if (get_user(inti->type, (u64 __user *)addr))
2381                 return -EFAULT;
2382
2383         switch (inti->type) {
2384         case KVM_S390_INT_PFAULT_INIT:
2385         case KVM_S390_INT_PFAULT_DONE:
2386         case KVM_S390_INT_VIRTIO:
2387         case KVM_S390_INT_SERVICE:
2388                 target = (void *) &inti->ext;
2389                 source = &uptr->u.ext;
2390                 size = sizeof(inti->ext);
2391                 break;
2392         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2393                 target = (void *) &inti->io;
2394                 source = &uptr->u.io;
2395                 size = sizeof(inti->io);
2396                 break;
2397         case KVM_S390_MCHK:
2398                 target = (void *) &inti->mchk;
2399                 source = &uptr->u.mchk;
2400                 size = sizeof(inti->mchk);
2401                 break;
2402         default:
2403                 return -EINVAL;
2404         }
2405
2406         if (copy_from_user(target, source, size))
2407                 return -EFAULT;
2408
2409         return 0;
2410 }
2411
2412 static int enqueue_floating_irq(struct kvm_device *dev,
2413                                 struct kvm_device_attr *attr)
2414 {
2415         struct kvm_s390_interrupt_info *inti = NULL;
2416         int r = 0;
2417         int len = attr->attr;
2418
2419         if (len % sizeof(struct kvm_s390_irq) != 0)
2420                 return -EINVAL;
2421         else if (len > KVM_S390_FLIC_MAX_BUFFER)
2422                 return -EINVAL;
2423
2424         while (len >= sizeof(struct kvm_s390_irq)) {
2425                 inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
2426                 if (!inti)
2427                         return -ENOMEM;
2428
2429                 r = copy_irq_from_user(inti, attr->addr);
2430                 if (r) {
2431                         kfree(inti);
2432                         return r;
2433                 }
2434                 r = __inject_vm(dev->kvm, inti);
2435                 if (r) {
2436                         kfree(inti);
2437                         return r;
2438                 }
2439                 len -= sizeof(struct kvm_s390_irq);
2440                 attr->addr += sizeof(struct kvm_s390_irq);
2441         }
2442
2443         return r;
2444 }
2445
2446 static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id)
2447 {
2448         if (id >= MAX_S390_IO_ADAPTERS)
2449                 return NULL;
2450         id = array_index_nospec(id, MAX_S390_IO_ADAPTERS);
2451         return kvm->arch.adapters[id];
2452 }
2453
2454 static int register_io_adapter(struct kvm_device *dev,
2455                                struct kvm_device_attr *attr)
2456 {
2457         struct s390_io_adapter *adapter;
2458         struct kvm_s390_io_adapter adapter_info;
2459
2460         if (copy_from_user(&adapter_info,
2461                            (void __user *)attr->addr, sizeof(adapter_info)))
2462                 return -EFAULT;
2463
2464         if (adapter_info.id >= MAX_S390_IO_ADAPTERS)
2465                 return -EINVAL;
2466
2467         adapter_info.id = array_index_nospec(adapter_info.id,
2468                                              MAX_S390_IO_ADAPTERS);
2469
2470         if (dev->kvm->arch.adapters[adapter_info.id] != NULL)
2471                 return -EINVAL;
2472
2473         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL_ACCOUNT);
2474         if (!adapter)
2475                 return -ENOMEM;
2476
2477         adapter->id = adapter_info.id;
2478         adapter->isc = adapter_info.isc;
2479         adapter->maskable = adapter_info.maskable;
2480         adapter->masked = false;
2481         adapter->swap = adapter_info.swap;
2482         adapter->suppressible = (adapter_info.flags) &
2483                                 KVM_S390_ADAPTER_SUPPRESSIBLE;
2484         dev->kvm->arch.adapters[adapter->id] = adapter;
2485
2486         return 0;
2487 }
2488
2489 int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked)
2490 {
2491         int ret;
2492         struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2493
2494         if (!adapter || !adapter->maskable)
2495                 return -EINVAL;
2496         ret = adapter->masked;
2497         adapter->masked = masked;
2498         return ret;
2499 }
2500
2501 void kvm_s390_destroy_adapters(struct kvm *kvm)
2502 {
2503         int i;
2504
2505         for (i = 0; i < MAX_S390_IO_ADAPTERS; i++)
2506                 kfree(kvm->arch.adapters[i]);
2507 }
2508
2509 static int modify_io_adapter(struct kvm_device *dev,
2510                              struct kvm_device_attr *attr)
2511 {
2512         struct kvm_s390_io_adapter_req req;
2513         struct s390_io_adapter *adapter;
2514         int ret;
2515
2516         if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
2517                 return -EFAULT;
2518
2519         adapter = get_io_adapter(dev->kvm, req.id);
2520         if (!adapter)
2521                 return -EINVAL;
2522         switch (req.type) {
2523         case KVM_S390_IO_ADAPTER_MASK:
2524                 ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask);
2525                 if (ret > 0)
2526                         ret = 0;
2527                 break;
2528         /*
2529          * The following operations are no longer needed and therefore no-ops.
2530          * The gpa to hva translation is done when an IRQ route is set up. The
2531          * set_irq code uses get_user_pages_remote() to do the actual write.
2532          */
2533         case KVM_S390_IO_ADAPTER_MAP:
2534         case KVM_S390_IO_ADAPTER_UNMAP:
2535                 ret = 0;
2536                 break;
2537         default:
2538                 ret = -EINVAL;
2539         }
2540
2541         return ret;
2542 }
2543
2544 static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr)
2545
2546 {
2547         const u64 isc_mask = 0xffUL << 24; /* all iscs set */
2548         u32 schid;
2549
2550         if (attr->flags)
2551                 return -EINVAL;
2552         if (attr->attr != sizeof(schid))
2553                 return -EINVAL;
2554         if (copy_from_user(&schid, (void __user *) attr->addr, sizeof(schid)))
2555                 return -EFAULT;
2556         if (!schid)
2557                 return -EINVAL;
2558         kfree(kvm_s390_get_io_int(kvm, isc_mask, schid));
2559         /*
2560          * If userspace is conforming to the architecture, we can have at most
2561          * one pending I/O interrupt per subchannel, so this is effectively a
2562          * clear all.
2563          */
2564         return 0;
2565 }
2566
2567 static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr)
2568 {
2569         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2570         struct kvm_s390_ais_req req;
2571         int ret = 0;
2572
2573         if (!test_kvm_facility(kvm, 72))
2574                 return -EOPNOTSUPP;
2575
2576         if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
2577                 return -EFAULT;
2578
2579         if (req.isc > MAX_ISC)
2580                 return -EINVAL;
2581
2582         trace_kvm_s390_modify_ais_mode(req.isc,
2583                                        (fi->simm & AIS_MODE_MASK(req.isc)) ?
2584                                        (fi->nimm & AIS_MODE_MASK(req.isc)) ?
2585                                        2 : KVM_S390_AIS_MODE_SINGLE :
2586                                        KVM_S390_AIS_MODE_ALL, req.mode);
2587
2588         mutex_lock(&fi->ais_lock);
2589         switch (req.mode) {
2590         case KVM_S390_AIS_MODE_ALL:
2591                 fi->simm &= ~AIS_MODE_MASK(req.isc);
2592                 fi->nimm &= ~AIS_MODE_MASK(req.isc);
2593                 break;
2594         case KVM_S390_AIS_MODE_SINGLE:
2595                 fi->simm |= AIS_MODE_MASK(req.isc);
2596                 fi->nimm &= ~AIS_MODE_MASK(req.isc);
2597                 break;
2598         default:
2599                 ret = -EINVAL;
2600         }
2601         mutex_unlock(&fi->ais_lock);
2602
2603         return ret;
2604 }
2605
2606 static int kvm_s390_inject_airq(struct kvm *kvm,
2607                                 struct s390_io_adapter *adapter)
2608 {
2609         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2610         struct kvm_s390_interrupt s390int = {
2611                 .type = KVM_S390_INT_IO(1, 0, 0, 0),
2612                 .parm = 0,
2613                 .parm64 = isc_to_int_word(adapter->isc),
2614         };
2615         int ret = 0;
2616
2617         if (!test_kvm_facility(kvm, 72) || !adapter->suppressible)
2618                 return kvm_s390_inject_vm(kvm, &s390int);
2619
2620         mutex_lock(&fi->ais_lock);
2621         if (fi->nimm & AIS_MODE_MASK(adapter->isc)) {
2622                 trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc);
2623                 goto out;
2624         }
2625
2626         ret = kvm_s390_inject_vm(kvm, &s390int);
2627         if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) {
2628                 fi->nimm |= AIS_MODE_MASK(adapter->isc);
2629                 trace_kvm_s390_modify_ais_mode(adapter->isc,
2630                                                KVM_S390_AIS_MODE_SINGLE, 2);
2631         }
2632 out:
2633         mutex_unlock(&fi->ais_lock);
2634         return ret;
2635 }
2636
2637 static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr)
2638 {
2639         unsigned int id = attr->attr;
2640         struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2641
2642         if (!adapter)
2643                 return -EINVAL;
2644
2645         return kvm_s390_inject_airq(kvm, adapter);
2646 }
2647
2648 static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr)
2649 {
2650         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2651         struct kvm_s390_ais_all ais;
2652
2653         if (!test_kvm_facility(kvm, 72))
2654                 return -EOPNOTSUPP;
2655
2656         if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais)))
2657                 return -EFAULT;
2658
2659         mutex_lock(&fi->ais_lock);
2660         fi->simm = ais.simm;
2661         fi->nimm = ais.nimm;
2662         mutex_unlock(&fi->ais_lock);
2663
2664         return 0;
2665 }
2666
2667 static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2668 {
2669         int r = 0;
2670         unsigned long i;
2671         struct kvm_vcpu *vcpu;
2672
2673         switch (attr->group) {
2674         case KVM_DEV_FLIC_ENQUEUE:
2675                 r = enqueue_floating_irq(dev, attr);
2676                 break;
2677         case KVM_DEV_FLIC_CLEAR_IRQS:
2678                 kvm_s390_clear_float_irqs(dev->kvm);
2679                 break;
2680         case KVM_DEV_FLIC_APF_ENABLE:
2681                 dev->kvm->arch.gmap->pfault_enabled = 1;
2682                 break;
2683         case KVM_DEV_FLIC_APF_DISABLE_WAIT:
2684                 dev->kvm->arch.gmap->pfault_enabled = 0;
2685                 /*
2686                  * Make sure no async faults are in transition when
2687                  * clearing the queues. So we don't need to worry
2688                  * about late coming workers.
2689                  */
2690                 synchronize_srcu(&dev->kvm->srcu);
2691                 kvm_for_each_vcpu(i, vcpu, dev->kvm)
2692                         kvm_clear_async_pf_completion_queue(vcpu);
2693                 break;
2694         case KVM_DEV_FLIC_ADAPTER_REGISTER:
2695                 r = register_io_adapter(dev, attr);
2696                 break;
2697         case KVM_DEV_FLIC_ADAPTER_MODIFY:
2698                 r = modify_io_adapter(dev, attr);
2699                 break;
2700         case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2701                 r = clear_io_irq(dev->kvm, attr);
2702                 break;
2703         case KVM_DEV_FLIC_AISM:
2704                 r = modify_ais_mode(dev->kvm, attr);
2705                 break;
2706         case KVM_DEV_FLIC_AIRQ_INJECT:
2707                 r = flic_inject_airq(dev->kvm, attr);
2708                 break;
2709         case KVM_DEV_FLIC_AISM_ALL:
2710                 r = flic_ais_mode_set_all(dev->kvm, attr);
2711                 break;
2712         default:
2713                 r = -EINVAL;
2714         }
2715
2716         return r;
2717 }
2718
2719 static int flic_has_attr(struct kvm_device *dev,
2720                              struct kvm_device_attr *attr)
2721 {
2722         switch (attr->group) {
2723         case KVM_DEV_FLIC_GET_ALL_IRQS:
2724         case KVM_DEV_FLIC_ENQUEUE:
2725         case KVM_DEV_FLIC_CLEAR_IRQS:
2726         case KVM_DEV_FLIC_APF_ENABLE:
2727         case KVM_DEV_FLIC_APF_DISABLE_WAIT:
2728         case KVM_DEV_FLIC_ADAPTER_REGISTER:
2729         case KVM_DEV_FLIC_ADAPTER_MODIFY:
2730         case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2731         case KVM_DEV_FLIC_AISM:
2732         case KVM_DEV_FLIC_AIRQ_INJECT:
2733         case KVM_DEV_FLIC_AISM_ALL:
2734                 return 0;
2735         }
2736         return -ENXIO;
2737 }
2738
2739 static int flic_create(struct kvm_device *dev, u32 type)
2740 {
2741         if (!dev)
2742                 return -EINVAL;
2743         if (dev->kvm->arch.flic)
2744                 return -EINVAL;
2745         dev->kvm->arch.flic = dev;
2746         return 0;
2747 }
2748
2749 static void flic_destroy(struct kvm_device *dev)
2750 {
2751         dev->kvm->arch.flic = NULL;
2752         kfree(dev);
2753 }
2754
2755 /* s390 floating irq controller (flic) */
2756 struct kvm_device_ops kvm_flic_ops = {
2757         .name = "kvm-flic",
2758         .get_attr = flic_get_attr,
2759         .set_attr = flic_set_attr,
2760         .has_attr = flic_has_attr,
2761         .create = flic_create,
2762         .destroy = flic_destroy,
2763 };
2764
2765 static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap)
2766 {
2767         unsigned long bit;
2768
2769         bit = bit_nr + (addr % PAGE_SIZE) * 8;
2770
2771         return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit;
2772 }
2773
2774 static struct page *get_map_page(struct kvm *kvm, u64 uaddr)
2775 {
2776         struct page *page = NULL;
2777
2778         mmap_read_lock(kvm->mm);
2779         get_user_pages_remote(kvm->mm, uaddr, 1, FOLL_WRITE,
2780                               &page, NULL);
2781         mmap_read_unlock(kvm->mm);
2782         return page;
2783 }
2784
2785 static int adapter_indicators_set(struct kvm *kvm,
2786                                   struct s390_io_adapter *adapter,
2787                                   struct kvm_s390_adapter_int *adapter_int)
2788 {
2789         unsigned long bit;
2790         int summary_set, idx;
2791         struct page *ind_page, *summary_page;
2792         void *map;
2793
2794         ind_page = get_map_page(kvm, adapter_int->ind_addr);
2795         if (!ind_page)
2796                 return -1;
2797         summary_page = get_map_page(kvm, adapter_int->summary_addr);
2798         if (!summary_page) {
2799                 put_page(ind_page);
2800                 return -1;
2801         }
2802
2803         idx = srcu_read_lock(&kvm->srcu);
2804         map = page_address(ind_page);
2805         bit = get_ind_bit(adapter_int->ind_addr,
2806                           adapter_int->ind_offset, adapter->swap);
2807         set_bit(bit, map);
2808         mark_page_dirty(kvm, adapter_int->ind_addr >> PAGE_SHIFT);
2809         set_page_dirty_lock(ind_page);
2810         map = page_address(summary_page);
2811         bit = get_ind_bit(adapter_int->summary_addr,
2812                           adapter_int->summary_offset, adapter->swap);
2813         summary_set = test_and_set_bit(bit, map);
2814         mark_page_dirty(kvm, adapter_int->summary_addr >> PAGE_SHIFT);
2815         set_page_dirty_lock(summary_page);
2816         srcu_read_unlock(&kvm->srcu, idx);
2817
2818         put_page(ind_page);
2819         put_page(summary_page);
2820         return summary_set ? 0 : 1;
2821 }
2822
2823 /*
2824  * < 0 - not injected due to error
2825  * = 0 - coalesced, summary indicator already active
2826  * > 0 - injected interrupt
2827  */
2828 static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e,
2829                            struct kvm *kvm, int irq_source_id, int level,
2830                            bool line_status)
2831 {
2832         int ret;
2833         struct s390_io_adapter *adapter;
2834
2835         /* We're only interested in the 0->1 transition. */
2836         if (!level)
2837                 return 0;
2838         adapter = get_io_adapter(kvm, e->adapter.adapter_id);
2839         if (!adapter)
2840                 return -1;
2841         ret = adapter_indicators_set(kvm, adapter, &e->adapter);
2842         if ((ret > 0) && !adapter->masked) {
2843                 ret = kvm_s390_inject_airq(kvm, adapter);
2844                 if (ret == 0)
2845                         ret = 1;
2846         }
2847         return ret;
2848 }
2849
2850 /*
2851  * Inject the machine check to the guest.
2852  */
2853 void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu,
2854                                      struct mcck_volatile_info *mcck_info)
2855 {
2856         struct kvm_s390_interrupt_info inti;
2857         struct kvm_s390_irq irq;
2858         struct kvm_s390_mchk_info *mchk;
2859         union mci mci;
2860         __u64 cr14 = 0;         /* upper bits are not used */
2861         int rc;
2862
2863         mci.val = mcck_info->mcic;
2864         if (mci.sr)
2865                 cr14 |= CR14_RECOVERY_SUBMASK;
2866         if (mci.dg)
2867                 cr14 |= CR14_DEGRADATION_SUBMASK;
2868         if (mci.w)
2869                 cr14 |= CR14_WARNING_SUBMASK;
2870
2871         mchk = mci.ck ? &inti.mchk : &irq.u.mchk;
2872         mchk->cr14 = cr14;
2873         mchk->mcic = mcck_info->mcic;
2874         mchk->ext_damage_code = mcck_info->ext_damage_code;
2875         mchk->failing_storage_address = mcck_info->failing_storage_address;
2876         if (mci.ck) {
2877                 /* Inject the floating machine check */
2878                 inti.type = KVM_S390_MCHK;
2879                 rc = __inject_vm(vcpu->kvm, &inti);
2880         } else {
2881                 /* Inject the machine check to specified vcpu */
2882                 irq.type = KVM_S390_MCHK;
2883                 rc = kvm_s390_inject_vcpu(vcpu, &irq);
2884         }
2885         WARN_ON_ONCE(rc);
2886 }
2887
2888 int kvm_set_routing_entry(struct kvm *kvm,
2889                           struct kvm_kernel_irq_routing_entry *e,
2890                           const struct kvm_irq_routing_entry *ue)
2891 {
2892         u64 uaddr;
2893
2894         switch (ue->type) {
2895         /* we store the userspace addresses instead of the guest addresses */
2896         case KVM_IRQ_ROUTING_S390_ADAPTER:
2897                 e->set = set_adapter_int;
2898                 uaddr =  gmap_translate(kvm->arch.gmap, ue->u.adapter.summary_addr);
2899                 if (uaddr == -EFAULT)
2900                         return -EFAULT;
2901                 e->adapter.summary_addr = uaddr;
2902                 uaddr =  gmap_translate(kvm->arch.gmap, ue->u.adapter.ind_addr);
2903                 if (uaddr == -EFAULT)
2904                         return -EFAULT;
2905                 e->adapter.ind_addr = uaddr;
2906                 e->adapter.summary_offset = ue->u.adapter.summary_offset;
2907                 e->adapter.ind_offset = ue->u.adapter.ind_offset;
2908                 e->adapter.adapter_id = ue->u.adapter.adapter_id;
2909                 return 0;
2910         default:
2911                 return -EINVAL;
2912         }
2913 }
2914
2915 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
2916                 int irq_source_id, int level, bool line_status)
2917 {
2918         return -EINVAL;
2919 }
2920
2921 int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len)
2922 {
2923         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2924         struct kvm_s390_irq *buf;
2925         int r = 0;
2926         int n;
2927
2928         buf = vmalloc(len);
2929         if (!buf)
2930                 return -ENOMEM;
2931
2932         if (copy_from_user((void *) buf, irqstate, len)) {
2933                 r = -EFAULT;
2934                 goto out_free;
2935         }
2936
2937         /*
2938          * Don't allow setting the interrupt state
2939          * when there are already interrupts pending
2940          */
2941         spin_lock(&li->lock);
2942         if (li->pending_irqs) {
2943                 r = -EBUSY;
2944                 goto out_unlock;
2945         }
2946
2947         for (n = 0; n < len / sizeof(*buf); n++) {
2948                 r = do_inject_vcpu(vcpu, &buf[n]);
2949                 if (r)
2950                         break;
2951         }
2952
2953 out_unlock:
2954         spin_unlock(&li->lock);
2955 out_free:
2956         vfree(buf);
2957
2958         return r;
2959 }
2960
2961 static void store_local_irq(struct kvm_s390_local_interrupt *li,
2962                             struct kvm_s390_irq *irq,
2963                             unsigned long irq_type)
2964 {
2965         switch (irq_type) {
2966         case IRQ_PEND_MCHK_EX:
2967         case IRQ_PEND_MCHK_REP:
2968                 irq->type = KVM_S390_MCHK;
2969                 irq->u.mchk = li->irq.mchk;
2970                 break;
2971         case IRQ_PEND_PROG:
2972                 irq->type = KVM_S390_PROGRAM_INT;
2973                 irq->u.pgm = li->irq.pgm;
2974                 break;
2975         case IRQ_PEND_PFAULT_INIT:
2976                 irq->type = KVM_S390_INT_PFAULT_INIT;
2977                 irq->u.ext = li->irq.ext;
2978                 break;
2979         case IRQ_PEND_EXT_EXTERNAL:
2980                 irq->type = KVM_S390_INT_EXTERNAL_CALL;
2981                 irq->u.extcall = li->irq.extcall;
2982                 break;
2983         case IRQ_PEND_EXT_CLOCK_COMP:
2984                 irq->type = KVM_S390_INT_CLOCK_COMP;
2985                 break;
2986         case IRQ_PEND_EXT_CPU_TIMER:
2987                 irq->type = KVM_S390_INT_CPU_TIMER;
2988                 break;
2989         case IRQ_PEND_SIGP_STOP:
2990                 irq->type = KVM_S390_SIGP_STOP;
2991                 irq->u.stop = li->irq.stop;
2992                 break;
2993         case IRQ_PEND_RESTART:
2994                 irq->type = KVM_S390_RESTART;
2995                 break;
2996         case IRQ_PEND_SET_PREFIX:
2997                 irq->type = KVM_S390_SIGP_SET_PREFIX;
2998                 irq->u.prefix = li->irq.prefix;
2999                 break;
3000         }
3001 }
3002
3003 int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len)
3004 {
3005         int scn;
3006         DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS);
3007         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
3008         unsigned long pending_irqs;
3009         struct kvm_s390_irq irq;
3010         unsigned long irq_type;
3011         int cpuaddr;
3012         int n = 0;
3013
3014         spin_lock(&li->lock);
3015         pending_irqs = li->pending_irqs;
3016         memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending,
3017                sizeof(sigp_emerg_pending));
3018         spin_unlock(&li->lock);
3019
3020         for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) {
3021                 memset(&irq, 0, sizeof(irq));
3022                 if (irq_type == IRQ_PEND_EXT_EMERGENCY)
3023                         continue;
3024                 if (n + sizeof(irq) > len)
3025                         return -ENOBUFS;
3026                 store_local_irq(&vcpu->arch.local_int, &irq, irq_type);
3027                 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3028                         return -EFAULT;
3029                 n += sizeof(irq);
3030         }
3031
3032         if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) {
3033                 for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) {
3034                         memset(&irq, 0, sizeof(irq));
3035                         if (n + sizeof(irq) > len)
3036                                 return -ENOBUFS;
3037                         irq.type = KVM_S390_INT_EMERGENCY;
3038                         irq.u.emerg.code = cpuaddr;
3039                         if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3040                                 return -EFAULT;
3041                         n += sizeof(irq);
3042                 }
3043         }
3044
3045         if (sca_ext_call_pending(vcpu, &scn)) {
3046                 if (n + sizeof(irq) > len)
3047                         return -ENOBUFS;
3048                 memset(&irq, 0, sizeof(irq));
3049                 irq.type = KVM_S390_INT_EXTERNAL_CALL;
3050                 irq.u.extcall.code = scn;
3051                 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3052                         return -EFAULT;
3053                 n += sizeof(irq);
3054         }
3055
3056         return n;
3057 }
3058
3059 static void __airqs_kick_single_vcpu(struct kvm *kvm, u8 deliverable_mask)
3060 {
3061         int vcpu_idx, online_vcpus = atomic_read(&kvm->online_vcpus);
3062         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3063         struct kvm_vcpu *vcpu;
3064         u8 vcpu_isc_mask;
3065
3066         for_each_set_bit(vcpu_idx, kvm->arch.idle_mask, online_vcpus) {
3067                 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
3068                 if (psw_ioint_disabled(vcpu))
3069                         continue;
3070                 vcpu_isc_mask = (u8)(vcpu->arch.sie_block->gcr[6] >> 24);
3071                 if (deliverable_mask & vcpu_isc_mask) {
3072                         /* lately kicked but not yet running */
3073                         if (test_and_set_bit(vcpu_idx, gi->kicked_mask))
3074                                 return;
3075                         kvm_s390_vcpu_wakeup(vcpu);
3076                         return;
3077                 }
3078         }
3079 }
3080
3081 static enum hrtimer_restart gisa_vcpu_kicker(struct hrtimer *timer)
3082 {
3083         struct kvm_s390_gisa_interrupt *gi =
3084                 container_of(timer, struct kvm_s390_gisa_interrupt, timer);
3085         struct kvm *kvm =
3086                 container_of(gi->origin, struct sie_page2, gisa)->kvm;
3087         u8 pending_mask;
3088
3089         pending_mask = gisa_get_ipm_or_restore_iam(gi);
3090         if (pending_mask) {
3091                 __airqs_kick_single_vcpu(kvm, pending_mask);
3092                 hrtimer_forward_now(timer, ns_to_ktime(gi->expires));
3093                 return HRTIMER_RESTART;
3094         }
3095
3096         return HRTIMER_NORESTART;
3097 }
3098
3099 #define NULL_GISA_ADDR 0x00000000UL
3100 #define NONE_GISA_ADDR 0x00000001UL
3101 #define GISA_ADDR_MASK 0xfffff000UL
3102
3103 static void process_gib_alert_list(void)
3104 {
3105         struct kvm_s390_gisa_interrupt *gi;
3106         u32 final, gisa_phys, origin = 0UL;
3107         struct kvm_s390_gisa *gisa;
3108         struct kvm *kvm;
3109
3110         do {
3111                 /*
3112                  * If the NONE_GISA_ADDR is still stored in the alert list
3113                  * origin, we will leave the outer loop. No further GISA has
3114                  * been added to the alert list by millicode while processing
3115                  * the current alert list.
3116                  */
3117                 final = (origin & NONE_GISA_ADDR);
3118                 /*
3119                  * Cut off the alert list and store the NONE_GISA_ADDR in the
3120                  * alert list origin to avoid further GAL interruptions.
3121                  * A new alert list can be build up by millicode in parallel
3122                  * for guests not in the yet cut-off alert list. When in the
3123                  * final loop, store the NULL_GISA_ADDR instead. This will re-
3124                  * enable GAL interruptions on the host again.
3125                  */
3126                 origin = xchg(&gib->alert_list_origin,
3127                               (!final) ? NONE_GISA_ADDR : NULL_GISA_ADDR);
3128                 /*
3129                  * Loop through the just cut-off alert list and start the
3130                  * gisa timers to kick idle vcpus to consume the pending
3131                  * interruptions asap.
3132                  */
3133                 while (origin & GISA_ADDR_MASK) {
3134                         gisa_phys = origin;
3135                         gisa = phys_to_virt(gisa_phys);
3136                         origin = gisa->next_alert;
3137                         gisa->next_alert = gisa_phys;
3138                         kvm = container_of(gisa, struct sie_page2, gisa)->kvm;
3139                         gi = &kvm->arch.gisa_int;
3140                         if (hrtimer_active(&gi->timer))
3141                                 hrtimer_cancel(&gi->timer);
3142                         hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL);
3143                 }
3144         } while (!final);
3145
3146 }
3147
3148 void kvm_s390_gisa_clear(struct kvm *kvm)
3149 {
3150         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3151
3152         if (!gi->origin)
3153                 return;
3154         gisa_clear_ipm(gi->origin);
3155         VM_EVENT(kvm, 3, "gisa 0x%pK cleared", gi->origin);
3156 }
3157
3158 void kvm_s390_gisa_init(struct kvm *kvm)
3159 {
3160         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3161
3162         if (!css_general_characteristics.aiv)
3163                 return;
3164         gi->origin = &kvm->arch.sie_page2->gisa;
3165         gi->alert.mask = 0;
3166         spin_lock_init(&gi->alert.ref_lock);
3167         gi->expires = 50 * 1000; /* 50 usec */
3168         hrtimer_init(&gi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3169         gi->timer.function = gisa_vcpu_kicker;
3170         memset(gi->origin, 0, sizeof(struct kvm_s390_gisa));
3171         gi->origin->next_alert = (u32)virt_to_phys(gi->origin);
3172         VM_EVENT(kvm, 3, "gisa 0x%pK initialized", gi->origin);
3173 }
3174
3175 void kvm_s390_gisa_enable(struct kvm *kvm)
3176 {
3177         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3178         struct kvm_vcpu *vcpu;
3179         unsigned long i;
3180         u32 gisa_desc;
3181
3182         if (gi->origin)
3183                 return;
3184         kvm_s390_gisa_init(kvm);
3185         gisa_desc = kvm_s390_get_gisa_desc(kvm);
3186         if (!gisa_desc)
3187                 return;
3188         kvm_for_each_vcpu(i, vcpu, kvm) {
3189                 mutex_lock(&vcpu->mutex);
3190                 vcpu->arch.sie_block->gd = gisa_desc;
3191                 vcpu->arch.sie_block->eca |= ECA_AIV;
3192                 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3193                            vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3194                 mutex_unlock(&vcpu->mutex);
3195         }
3196 }
3197
3198 void kvm_s390_gisa_destroy(struct kvm *kvm)
3199 {
3200         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3201         struct kvm_s390_gisa *gisa = gi->origin;
3202
3203         if (!gi->origin)
3204                 return;
3205         if (gi->alert.mask)
3206                 KVM_EVENT(3, "vm 0x%pK has unexpected iam 0x%02x",
3207                           kvm, gi->alert.mask);
3208         while (gisa_in_alert_list(gi->origin))
3209                 cpu_relax();
3210         hrtimer_cancel(&gi->timer);
3211         gi->origin = NULL;
3212         VM_EVENT(kvm, 3, "gisa 0x%pK destroyed", gisa);
3213 }
3214
3215 void kvm_s390_gisa_disable(struct kvm *kvm)
3216 {
3217         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3218         struct kvm_vcpu *vcpu;
3219         unsigned long i;
3220
3221         if (!gi->origin)
3222                 return;
3223         kvm_for_each_vcpu(i, vcpu, kvm) {
3224                 mutex_lock(&vcpu->mutex);
3225                 vcpu->arch.sie_block->eca &= ~ECA_AIV;
3226                 vcpu->arch.sie_block->gd = 0U;
3227                 mutex_unlock(&vcpu->mutex);
3228                 VCPU_EVENT(vcpu, 3, "AIV disabled for cpu %03u", vcpu->vcpu_id);
3229         }
3230         kvm_s390_gisa_destroy(kvm);
3231 }
3232
3233 /**
3234  * kvm_s390_gisc_register - register a guest ISC
3235  *
3236  * @kvm:  the kernel vm to work with
3237  * @gisc: the guest interruption sub class to register
3238  *
3239  * The function extends the vm specific alert mask to use.
3240  * The effective IAM mask in the GISA is updated as well
3241  * in case the GISA is not part of the GIB alert list.
3242  * It will be updated latest when the IAM gets restored
3243  * by gisa_get_ipm_or_restore_iam().
3244  *
3245  * Returns: the nonspecific ISC (NISC) the gib alert mechanism
3246  *          has registered with the channel subsystem.
3247  *          -ENODEV in case the vm uses no GISA
3248  *          -ERANGE in case the guest ISC is invalid
3249  */
3250 int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc)
3251 {
3252         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3253
3254         if (!gi->origin)
3255                 return -ENODEV;
3256         if (gisc > MAX_ISC)
3257                 return -ERANGE;
3258
3259         spin_lock(&gi->alert.ref_lock);
3260         gi->alert.ref_count[gisc]++;
3261         if (gi->alert.ref_count[gisc] == 1) {
3262                 gi->alert.mask |= 0x80 >> gisc;
3263                 gisa_set_iam(gi->origin, gi->alert.mask);
3264         }
3265         spin_unlock(&gi->alert.ref_lock);
3266
3267         return gib->nisc;
3268 }
3269 EXPORT_SYMBOL_GPL(kvm_s390_gisc_register);
3270
3271 /**
3272  * kvm_s390_gisc_unregister - unregister a guest ISC
3273  *
3274  * @kvm:  the kernel vm to work with
3275  * @gisc: the guest interruption sub class to register
3276  *
3277  * The function reduces the vm specific alert mask to use.
3278  * The effective IAM mask in the GISA is updated as well
3279  * in case the GISA is not part of the GIB alert list.
3280  * It will be updated latest when the IAM gets restored
3281  * by gisa_get_ipm_or_restore_iam().
3282  *
3283  * Returns: the nonspecific ISC (NISC) the gib alert mechanism
3284  *          has registered with the channel subsystem.
3285  *          -ENODEV in case the vm uses no GISA
3286  *          -ERANGE in case the guest ISC is invalid
3287  *          -EINVAL in case the guest ISC is not registered
3288  */
3289 int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc)
3290 {
3291         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3292         int rc = 0;
3293
3294         if (!gi->origin)
3295                 return -ENODEV;
3296         if (gisc > MAX_ISC)
3297                 return -ERANGE;
3298
3299         spin_lock(&gi->alert.ref_lock);
3300         if (gi->alert.ref_count[gisc] == 0) {
3301                 rc = -EINVAL;
3302                 goto out;
3303         }
3304         gi->alert.ref_count[gisc]--;
3305         if (gi->alert.ref_count[gisc] == 0) {
3306                 gi->alert.mask &= ~(0x80 >> gisc);
3307                 gisa_set_iam(gi->origin, gi->alert.mask);
3308         }
3309 out:
3310         spin_unlock(&gi->alert.ref_lock);
3311
3312         return rc;
3313 }
3314 EXPORT_SYMBOL_GPL(kvm_s390_gisc_unregister);
3315
3316 static void aen_host_forward(unsigned long si)
3317 {
3318         struct kvm_s390_gisa_interrupt *gi;
3319         struct zpci_gaite *gaite;
3320         struct kvm *kvm;
3321
3322         gaite = (struct zpci_gaite *)aift->gait +
3323                 (si * sizeof(struct zpci_gaite));
3324         if (gaite->count == 0)
3325                 return;
3326         if (gaite->aisb != 0)
3327                 set_bit_inv(gaite->aisbo, phys_to_virt(gaite->aisb));
3328
3329         kvm = kvm_s390_pci_si_to_kvm(aift, si);
3330         if (!kvm)
3331                 return;
3332         gi = &kvm->arch.gisa_int;
3333
3334         if (!(gi->origin->g1.simm & AIS_MODE_MASK(gaite->gisc)) ||
3335             !(gi->origin->g1.nimm & AIS_MODE_MASK(gaite->gisc))) {
3336                 gisa_set_ipm_gisc(gi->origin, gaite->gisc);
3337                 if (hrtimer_active(&gi->timer))
3338                         hrtimer_cancel(&gi->timer);
3339                 hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL);
3340                 kvm->stat.aen_forward++;
3341         }
3342 }
3343
3344 static void aen_process_gait(u8 isc)
3345 {
3346         bool found = false, first = true;
3347         union zpci_sic_iib iib = {{0}};
3348         unsigned long si, flags;
3349
3350         spin_lock_irqsave(&aift->gait_lock, flags);
3351
3352         if (!aift->gait) {
3353                 spin_unlock_irqrestore(&aift->gait_lock, flags);
3354                 return;
3355         }
3356
3357         for (si = 0;;) {
3358                 /* Scan adapter summary indicator bit vector */
3359                 si = airq_iv_scan(aift->sbv, si, airq_iv_end(aift->sbv));
3360                 if (si == -1UL) {
3361                         if (first || found) {
3362                                 /* Re-enable interrupts. */
3363                                 zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, isc,
3364                                                   &iib);
3365                                 first = found = false;
3366                         } else {
3367                                 /* Interrupts on and all bits processed */
3368                                 break;
3369                         }
3370                         found = false;
3371                         si = 0;
3372                         /* Scan again after re-enabling interrupts */
3373                         continue;
3374                 }
3375                 found = true;
3376                 aen_host_forward(si);
3377         }
3378
3379         spin_unlock_irqrestore(&aift->gait_lock, flags);
3380 }
3381
3382 static void gib_alert_irq_handler(struct airq_struct *airq,
3383                                   struct tpi_info *tpi_info)
3384 {
3385         struct tpi_adapter_info *info = (struct tpi_adapter_info *)tpi_info;
3386
3387         inc_irq_stat(IRQIO_GAL);
3388
3389         if ((info->forward || info->error) &&
3390             IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
3391                 aen_process_gait(info->isc);
3392                 if (info->aism != 0)
3393                         process_gib_alert_list();
3394         } else {
3395                 process_gib_alert_list();
3396         }
3397 }
3398
3399 static struct airq_struct gib_alert_irq = {
3400         .handler = gib_alert_irq_handler,
3401         .lsi_ptr = &gib_alert_irq.lsi_mask,
3402 };
3403
3404 void kvm_s390_gib_destroy(void)
3405 {
3406         if (!gib)
3407                 return;
3408         if (kvm_s390_pci_interp_allowed() && aift) {
3409                 mutex_lock(&aift->aift_lock);
3410                 kvm_s390_pci_aen_exit();
3411                 mutex_unlock(&aift->aift_lock);
3412         }
3413         chsc_sgib(0);
3414         unregister_adapter_interrupt(&gib_alert_irq);
3415         free_page((unsigned long)gib);
3416         gib = NULL;
3417 }
3418
3419 int __init kvm_s390_gib_init(u8 nisc)
3420 {
3421         u32 gib_origin;
3422         int rc = 0;
3423
3424         if (!css_general_characteristics.aiv) {
3425                 KVM_EVENT(3, "%s", "gib not initialized, no AIV facility");
3426                 goto out;
3427         }
3428
3429         gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
3430         if (!gib) {
3431                 rc = -ENOMEM;
3432                 goto out;
3433         }
3434
3435         gib_alert_irq.isc = nisc;
3436         if (register_adapter_interrupt(&gib_alert_irq)) {
3437                 pr_err("Registering the GIB alert interruption handler failed\n");
3438                 rc = -EIO;
3439                 goto out_free_gib;
3440         }
3441
3442         gib->nisc = nisc;
3443         gib_origin = virt_to_phys(gib);
3444         if (chsc_sgib(gib_origin)) {
3445                 pr_err("Associating the GIB with the AIV facility failed\n");
3446                 free_page((unsigned long)gib);
3447                 gib = NULL;
3448                 rc = -EIO;
3449                 goto out_unreg_gal;
3450         }
3451
3452         if (kvm_s390_pci_interp_allowed()) {
3453                 if (kvm_s390_pci_aen_init(nisc)) {
3454                         pr_err("Initializing AEN for PCI failed\n");
3455                         rc = -EIO;
3456                         goto out_unreg_gal;
3457                 }
3458         }
3459
3460         KVM_EVENT(3, "gib 0x%pK (nisc=%d) initialized", gib, gib->nisc);
3461         goto out;
3462
3463 out_unreg_gal:
3464         unregister_adapter_interrupt(&gib_alert_irq);
3465 out_free_gib:
3466         free_page((unsigned long)gib);
3467         gib = NULL;
3468 out:
3469         return rc;
3470 }