arch/s390/kvm/kvm-s390.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * hosting IBM Z kernel virtual machines (s390x)
   4  *
   5  * Copyright IBM Corp. 2008, 2017
   6  *
   7  *    Author(s): Carsten Otte <cotte@de.ibm.com>
   8  *               Christian Borntraeger <borntraeger@de.ibm.com>
   9  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  10  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  11  *               Jason J. Herne <jjherne@us.ibm.com>
  12  */
  13
  14 #include <linux/compiler.h>
  15 #include <linux/err.h>
  16 #include <linux/fs.h>
  17 #include <linux/hrtimer.h>
  18 #include <linux/init.h>
  19 #include <linux/kvm.h>
  20 #include <linux/kvm_host.h>
  21 #include <linux/mman.h>
  22 #include <linux/module.h>
  23 #include <linux/moduleparam.h>
  24 #include <linux/random.h>
  25 #include <linux/slab.h>
  26 #include <linux/timer.h>
  27 #include <linux/vmalloc.h>
  28 #include <linux/bitmap.h>
  29 #include <linux/sched/signal.h>
  30 #include <linux/string.h>
  31
  32 #include <asm/asm-offsets.h>
  33 #include <asm/lowcore.h>
  34 #include <asm/stp.h>
  35 #include <asm/pgtable.h>
  36 #include <asm/gmap.h>
  37 #include <asm/nmi.h>
  38 #include <asm/switch_to.h>
  39 #include <asm/isc.h>
  40 #include <asm/sclp.h>
  41 #include <asm/cpacf.h>
  42 #include <asm/timex.h>
  43 #include "kvm-s390.h"
  44 #include "gaccess.h"
  45
  46 #define KMSG_COMPONENT "kvm-s390"
  47 #undef pr_fmt
  48 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  49
  50 #define CREATE_TRACE_POINTS
  51 #include "trace.h"
  52 #include "trace-s390.h"
  53
  54 #define MEM_OP_MAX_SIZE 65536   /* Maximum transfer size for KVM_S390_MEM_OP */
  55 #define LOCAL_IRQS 32
  56 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
  57                            (KVM_MAX_VCPUS + LOCAL_IRQS))
  58
  59 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  60
  61 struct kvm_stats_debugfs_item debugfs_entries[] = {
  62         { "userspace_handled", VCPU_STAT(exit_userspace) },
  63         { "exit_null", VCPU_STAT(exit_null) },
  64         { "exit_validity", VCPU_STAT(exit_validity) },
  65         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  66         { "exit_external_request", VCPU_STAT(exit_external_request) },
  67         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  68         { "exit_instruction", VCPU_STAT(exit_instruction) },
  69         { "exit_pei", VCPU_STAT(exit_pei) },
  70         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  71         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  72         { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
  73         { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
  74         { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
  75         { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
  76         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  77         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  78         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  79         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
  80         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
  81         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  82         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  83         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  84         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
  85         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  86         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  87         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  88         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
  89         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  90         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
  91         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
  92         { "instruction_spx", VCPU_STAT(instruction_spx) },
  93         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
  94         { "instruction_stap", VCPU_STAT(instruction_stap) },
  95         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
  96         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
  97         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
  98         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
  99         { "instruction_essa", VCPU_STAT(instruction_essa) },
 100         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
 101         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
 102         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
 103         { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
 104         { "instruction_sie", VCPU_STAT(instruction_sie) },
 105         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
 106         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
 107         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
 108         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
 109         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
 110         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
 111         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
 112         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
 113         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
 114         { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
 115         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
 116         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
 117         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
 118         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
 119         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
 120         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
 121         { "diagnose_10", VCPU_STAT(diagnose_10) },
 122         { "diagnose_44", VCPU_STAT(diagnose_44) },
 123         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
 124         { "diagnose_258", VCPU_STAT(diagnose_258) },
 125         { "diagnose_308", VCPU_STAT(diagnose_308) },
 126         { "diagnose_500", VCPU_STAT(diagnose_500) },
 127         { NULL }
 128 };
 129
 130 struct kvm_s390_tod_clock_ext {
 131         __u8 epoch_idx;
 132         __u64 tod;
 133         __u8 reserved[7];
 134 } __packed;
 135
 136 /* allow nested virtualization in KVM (if enabled by user space) */
 137 static int nested;
 138 module_param(nested, int, S_IRUGO);
 139 MODULE_PARM_DESC(nested, "Nested virtualization support");
 140
 141 /* upper facilities limit for kvm */
 142 unsigned long kvm_s390_fac_list_mask[16] = { FACILITIES_KVM };
 143
 144 unsigned long kvm_s390_fac_list_mask_size(void)
 145 {
 146         BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
 147         return ARRAY_SIZE(kvm_s390_fac_list_mask);
 148 }
 149
 150 /* available cpu features supported by kvm */
 151 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
 152 /* available subfunctions indicated via query / "test bit" */
 153 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
 154
 155 static struct gmap_notifier gmap_notifier;
 156 static struct gmap_notifier vsie_gmap_notifier;
 157 debug_info_t *kvm_s390_dbf;
 158
 159 /* Section: not file related */
 160 int kvm_arch_hardware_enable(void)
 161 {
 162         /* every s390 is virtualization enabled ;-) */
 163         return 0;
 164 }
 165
 166 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
 167                               unsigned long end);
 168
 169 /*
 170  * This callback is executed during stop_machine(). All CPUs are therefore
 171  * temporarily stopped. In order not to change guest behavior, we have to
 172  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
 173  * so a CPU won't be stopped while calculating with the epoch.
 174  */
 175 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
 176                           void *v)
 177 {
 178         struct kvm *kvm;
 179         struct kvm_vcpu *vcpu;
 180         int i;
 181         unsigned long long *delta = v;
 182
 183         list_for_each_entry(kvm, &vm_list, vm_list) {
 184                 kvm->arch.epoch -= *delta;
 185                 kvm_for_each_vcpu(i, vcpu, kvm) {
 186                         vcpu->arch.sie_block->epoch -= *delta;
 187                         if (vcpu->arch.cputm_enabled)
 188                                 vcpu->arch.cputm_start += *delta;
 189                         if (vcpu->arch.vsie_block)
 190                                 vcpu->arch.vsie_block->epoch -= *delta;
 191                 }
 192         }
 193         return NOTIFY_OK;
 194 }
 195
 196 static struct notifier_block kvm_clock_notifier = {
 197         .notifier_call = kvm_clock_sync,
 198 };
 199
 200 int kvm_arch_hardware_setup(void)
 201 {
 202         gmap_notifier.notifier_call = kvm_gmap_notifier;
 203         gmap_register_pte_notifier(&gmap_notifier);
 204         vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
 205         gmap_register_pte_notifier(&vsie_gmap_notifier);
 206         atomic_notifier_chain_register(&s390_epoch_delta_notifier,
 207                                        &kvm_clock_notifier);
 208         return 0;
 209 }
 210
 211 void kvm_arch_hardware_unsetup(void)
 212 {
 213         gmap_unregister_pte_notifier(&gmap_notifier);
 214         gmap_unregister_pte_notifier(&vsie_gmap_notifier);
 215         atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
 216                                          &kvm_clock_notifier);
 217 }
 218
 219 static void allow_cpu_feat(unsigned long nr)
 220 {
 221         set_bit_inv(nr, kvm_s390_available_cpu_feat);
 222 }
 223
 224 static inline int plo_test_bit(unsigned char nr)
 225 {
 226         register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
 227         int cc;
 228
 229         asm volatile(
 230                 /* Parameter registers are ignored for "test bit" */
 231                 "       plo     0,0,0,0(0)\n"
 232                 "       ipm     %0\n"
 233                 "       srl     %0,28\n"
 234                 : "=d" (cc)
 235                 : "d" (r0)
 236                 : "cc");
 237         return cc == 0;
 238 }
 239
 240 static void kvm_s390_cpu_feat_init(void)
 241 {
 242         int i;
 243
 244         for (i = 0; i < 256; ++i) {
 245                 if (plo_test_bit(i))
 246                         kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
 247         }
 248
 249         if (test_facility(28)) /* TOD-clock steering */
 250                 ptff(kvm_s390_available_subfunc.ptff,
 251                      sizeof(kvm_s390_available_subfunc.ptff),
 252                      PTFF_QAF);
 253
 254         if (test_facility(17)) { /* MSA */
 255                 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
 256                               kvm_s390_available_subfunc.kmac);
 257                 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
 258                               kvm_s390_available_subfunc.kmc);
 259                 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
 260                               kvm_s390_available_subfunc.km);
 261                 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
 262                               kvm_s390_available_subfunc.kimd);
 263                 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
 264                               kvm_s390_available_subfunc.klmd);
 265         }
 266         if (test_facility(76)) /* MSA3 */
 267                 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
 268                               kvm_s390_available_subfunc.pckmo);
 269         if (test_facility(77)) { /* MSA4 */
 270                 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
 271                               kvm_s390_available_subfunc.kmctr);
 272                 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
 273                               kvm_s390_available_subfunc.kmf);
 274                 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
 275                               kvm_s390_available_subfunc.kmo);
 276                 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
 277                               kvm_s390_available_subfunc.pcc);
 278         }
 279         if (test_facility(57)) /* MSA5 */
 280                 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
 281                               kvm_s390_available_subfunc.ppno);
 282
 283         if (test_facility(146)) /* MSA8 */
 284                 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
 285                               kvm_s390_available_subfunc.kma);
 286
 287         if (MACHINE_HAS_ESOP)
 288                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
 289         /*
 290          * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
 291          * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
 292          */
 293         if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
 294             !test_facility(3) || !nested)
 295                 return;
 296         allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
 297         if (sclp.has_64bscao)
 298                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
 299         if (sclp.has_siif)
 300                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
 301         if (sclp.has_gpere)
 302                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
 303         if (sclp.has_gsls)
 304                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
 305         if (sclp.has_ib)
 306                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
 307         if (sclp.has_cei)
 308                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
 309         if (sclp.has_ibs)
 310                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
 311         if (sclp.has_kss)
 312                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
 313         /*
 314          * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
 315          * all skey handling functions read/set the skey from the PGSTE
 316          * instead of the real storage key.
 317          *
 318          * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
 319          * pages being detected as preserved although they are resident.
 320          *
 321          * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
 322          * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
 323          *
 324          * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
 325          * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
 326          * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
 327          *
 328          * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
 329          * cannot easily shadow the SCA because of the ipte lock.
 330          */
 331 }
 332
 333 int kvm_arch_init(void *opaque)
 334 {
 335         kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
 336         if (!kvm_s390_dbf)
 337                 return -ENOMEM;
 338
 339         if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
 340                 debug_unregister(kvm_s390_dbf);
 341                 return -ENOMEM;
 342         }
 343
 344         kvm_s390_cpu_feat_init();
 345
 346         /* Register floating interrupt controller interface. */
 347         return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 348 }
 349
 350 void kvm_arch_exit(void)
 351 {
 352         debug_unregister(kvm_s390_dbf);
 353 }
 354
 355 /* Section: device related */
 356 long kvm_arch_dev_ioctl(struct file *filp,
 357                         unsigned int ioctl, unsigned long arg)
 358 {
 359         if (ioctl == KVM_S390_ENABLE_SIE)
 360                 return s390_enable_sie();
 361         return -EINVAL;
 362 }
 363
 364 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 365 {
 366         int r;
 367
 368         switch (ext) {
 369         case KVM_CAP_S390_PSW:
 370         case KVM_CAP_S390_GMAP:
 371         case KVM_CAP_SYNC_MMU:
 372 #ifdef CONFIG_KVM_S390_UCONTROL
 373         case KVM_CAP_S390_UCONTROL:
 374 #endif
 375         case KVM_CAP_ASYNC_PF:
 376         case KVM_CAP_SYNC_REGS:
 377         case KVM_CAP_ONE_REG:
 378         case KVM_CAP_ENABLE_CAP:
 379         case KVM_CAP_S390_CSS_SUPPORT:
 380         case KVM_CAP_IOEVENTFD:
 381         case KVM_CAP_DEVICE_CTRL:
 382         case KVM_CAP_ENABLE_CAP_VM:
 383         case KVM_CAP_S390_IRQCHIP:
 384         case KVM_CAP_VM_ATTRIBUTES:
 385         case KVM_CAP_MP_STATE:
 386         case KVM_CAP_IMMEDIATE_EXIT:
 387         case KVM_CAP_S390_INJECT_IRQ:
 388         case KVM_CAP_S390_USER_SIGP:
 389         case KVM_CAP_S390_USER_STSI:
 390         case KVM_CAP_S390_SKEYS:
 391         case KVM_CAP_S390_IRQ_STATE:
 392         case KVM_CAP_S390_USER_INSTR0:
 393         case KVM_CAP_S390_CMMA_MIGRATION:
 394         case KVM_CAP_S390_AIS:
 395         case KVM_CAP_S390_AIS_MIGRATION:
 396                 r = 1;
 397                 break;
 398         case KVM_CAP_S390_MEM_OP:
 399                 r = MEM_OP_MAX_SIZE;
 400                 break;
 401         case KVM_CAP_NR_VCPUS:
 402         case KVM_CAP_MAX_VCPUS:
 403                 r = KVM_S390_BSCA_CPU_SLOTS;
 404                 if (!kvm_s390_use_sca_entries())
 405                         r = KVM_MAX_VCPUS;
 406                 else if (sclp.has_esca && sclp.has_64bscao)
 407                         r = KVM_S390_ESCA_CPU_SLOTS;
 408                 break;
 409         case KVM_CAP_NR_MEMSLOTS:
 410                 r = KVM_USER_MEM_SLOTS;
 411                 break;
 412         case KVM_CAP_S390_COW:
 413                 r = MACHINE_HAS_ESOP;
 414                 break;
 415         case KVM_CAP_S390_VECTOR_REGISTERS:
 416                 r = MACHINE_HAS_VX;
 417                 break;
 418         case KVM_CAP_S390_RI:
 419                 r = test_facility(64);
 420                 break;
 421         case KVM_CAP_S390_GS:
 422                 r = test_facility(133);
 423                 break;
 424         default:
 425                 r = 0;
 426         }
 427         return r;
 428 }
 429
 430 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
 431                                         struct kvm_memory_slot *memslot)
 432 {
 433         gfn_t cur_gfn, last_gfn;
 434         unsigned long address;
 435         struct gmap *gmap = kvm->arch.gmap;
 436
 437         /* Loop over all guest pages */
 438         last_gfn = memslot->base_gfn + memslot->npages;
 439         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
 440                 address = gfn_to_hva_memslot(memslot, cur_gfn);
 441
 442                 if (test_and_clear_guest_dirty(gmap->mm, address))
 443                         mark_page_dirty(kvm, cur_gfn);
 444                 if (fatal_signal_pending(current))
 445                         return;
 446                 cond_resched();
 447         }
 448 }
 449
 450 /* Section: vm related */
 451 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
 452
 453 /*
 454  * Get (and clear) the dirty memory log for a memory slot.
 455  */
 456 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 457                                struct kvm_dirty_log *log)
 458 {
 459         int r;
 460         unsigned long n;
 461         struct kvm_memslots *slots;
 462         struct kvm_memory_slot *memslot;
 463         int is_dirty = 0;
 464
 465         if (kvm_is_ucontrol(kvm))
 466                 return -EINVAL;
 467
 468         mutex_lock(&kvm->slots_lock);
 469
 470         r = -EINVAL;
 471         if (log->slot >= KVM_USER_MEM_SLOTS)
 472                 goto out;
 473
 474         slots = kvm_memslots(kvm);
 475         memslot = id_to_memslot(slots, log->slot);
 476         r = -ENOENT;
 477         if (!memslot->dirty_bitmap)
 478                 goto out;
 479
 480         kvm_s390_sync_dirty_log(kvm, memslot);
 481         r = kvm_get_dirty_log(kvm, log, &is_dirty);
 482         if (r)
 483                 goto out;
 484
 485         /* Clear the dirty log */
 486         if (is_dirty) {
 487                 n = kvm_dirty_bitmap_bytes(memslot);
 488                 memset(memslot->dirty_bitmap, 0, n);
 489         }
 490         r = 0;
 491 out:
 492         mutex_unlock(&kvm->slots_lock);
 493         return r;
 494 }
 495
 496 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
 497 {
 498         unsigned int i;
 499         struct kvm_vcpu *vcpu;
 500
 501         kvm_for_each_vcpu(i, vcpu, kvm) {
 502                 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
 503         }
 504 }
 505
 506 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 507 {
 508         int r;
 509
 510         if (cap->flags)
 511                 return -EINVAL;
 512
 513         switch (cap->cap) {
 514         case KVM_CAP_S390_IRQCHIP:
 515                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
 516                 kvm->arch.use_irqchip = 1;
 517                 r = 0;
 518                 break;
 519         case KVM_CAP_S390_USER_SIGP:
 520                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
 521                 kvm->arch.user_sigp = 1;
 522                 r = 0;
 523                 break;
 524         case KVM_CAP_S390_VECTOR_REGISTERS:
 525                 mutex_lock(&kvm->lock);
 526                 if (kvm->created_vcpus) {
 527                         r = -EBUSY;
 528                 } else if (MACHINE_HAS_VX) {
 529                         set_kvm_facility(kvm->arch.model.fac_mask, 129);
 530                         set_kvm_facility(kvm->arch.model.fac_list, 129);
 531                         if (test_facility(134)) {
 532                                 set_kvm_facility(kvm->arch.model.fac_mask, 134);
 533                                 set_kvm_facility(kvm->arch.model.fac_list, 134);
 534                         }
 535                         if (test_facility(135)) {
 536                                 set_kvm_facility(kvm->arch.model.fac_mask, 135);
 537                                 set_kvm_facility(kvm->arch.model.fac_list, 135);
 538                         }
 539                         r = 0;
 540                 } else
 541                         r = -EINVAL;
 542                 mutex_unlock(&kvm->lock);
 543                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
 544                          r ? "(not available)" : "(success)");
 545                 break;
 546         case KVM_CAP_S390_RI:
 547                 r = -EINVAL;
 548                 mutex_lock(&kvm->lock);
 549                 if (kvm->created_vcpus) {
 550                         r = -EBUSY;
 551                 } else if (test_facility(64)) {
 552                         set_kvm_facility(kvm->arch.model.fac_mask, 64);
 553                         set_kvm_facility(kvm->arch.model.fac_list, 64);
 554                         r = 0;
 555                 }
 556                 mutex_unlock(&kvm->lock);
 557                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
 558                          r ? "(not available)" : "(success)");
 559                 break;
 560         case KVM_CAP_S390_AIS:
 561                 mutex_lock(&kvm->lock);
 562                 if (kvm->created_vcpus) {
 563                         r = -EBUSY;
 564                 } else {
 565                         set_kvm_facility(kvm->arch.model.fac_mask, 72);
 566                         set_kvm_facility(kvm->arch.model.fac_list, 72);
 567                         r = 0;
 568                 }
 569                 mutex_unlock(&kvm->lock);
 570                 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
 571                          r ? "(not available)" : "(success)");
 572                 break;
 573         case KVM_CAP_S390_GS:
 574                 r = -EINVAL;
 575                 mutex_lock(&kvm->lock);
 576                 if (atomic_read(&kvm->online_vcpus)) {
 577                         r = -EBUSY;
 578                 } else if (test_facility(133)) {
 579                         set_kvm_facility(kvm->arch.model.fac_mask, 133);
 580                         set_kvm_facility(kvm->arch.model.fac_list, 133);
 581                         r = 0;
 582                 }
 583                 mutex_unlock(&kvm->lock);
 584                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
 585                          r ? "(not available)" : "(success)");
 586                 break;
 587         case KVM_CAP_S390_USER_STSI:
 588                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
 589                 kvm->arch.user_stsi = 1;
 590                 r = 0;
 591                 break;
 592         case KVM_CAP_S390_USER_INSTR0:
 593                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
 594                 kvm->arch.user_instr0 = 1;
 595                 icpt_operexc_on_all_vcpus(kvm);
 596                 r = 0;
 597                 break;
 598         default:
 599                 r = -EINVAL;
 600                 break;
 601         }
 602         return r;
 603 }
 604
 605 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 606 {
 607         int ret;
 608
 609         switch (attr->attr) {
 610         case KVM_S390_VM_MEM_LIMIT_SIZE:
 611                 ret = 0;
 612                 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
 613                          kvm->arch.mem_limit);
 614                 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
 615                         ret = -EFAULT;
 616                 break;
 617         default:
 618                 ret = -ENXIO;
 619                 break;
 620         }
 621         return ret;
 622 }
 623
 624 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 625 {
 626         int ret;
 627         unsigned int idx;
 628         switch (attr->attr) {
 629         case KVM_S390_VM_MEM_ENABLE_CMMA:
 630                 ret = -ENXIO;
 631                 if (!sclp.has_cmma)
 632                         break;
 633
 634                 ret = -EBUSY;
 635                 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
 636                 mutex_lock(&kvm->lock);
 637                 if (!kvm->created_vcpus) {
 638                         kvm->arch.use_cmma = 1;
 639                         ret = 0;
 640                 }
 641                 mutex_unlock(&kvm->lock);
 642                 break;
 643         case KVM_S390_VM_MEM_CLR_CMMA:
 644                 ret = -ENXIO;
 645                 if (!sclp.has_cmma)
 646                         break;
 647                 ret = -EINVAL;
 648                 if (!kvm->arch.use_cmma)
 649                         break;
 650
 651                 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
 652                 mutex_lock(&kvm->lock);
 653                 idx = srcu_read_lock(&kvm->srcu);
 654                 s390_reset_cmma(kvm->arch.gmap->mm);
 655                 srcu_read_unlock(&kvm->srcu, idx);
 656                 mutex_unlock(&kvm->lock);
 657                 ret = 0;
 658                 break;
 659         case KVM_S390_VM_MEM_LIMIT_SIZE: {
 660                 unsigned long new_limit;
 661
 662                 if (kvm_is_ucontrol(kvm))
 663                         return -EINVAL;
 664
 665                 if (get_user(new_limit, (u64 __user *)attr->addr))
 666                         return -EFAULT;
 667
 668                 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
 669                     new_limit > kvm->arch.mem_limit)
 670                         return -E2BIG;
 671
 672                 if (!new_limit)
 673                         return -EINVAL;
 674
 675                 /* gmap_create takes last usable address */
 676                 if (new_limit != KVM_S390_NO_MEM_LIMIT)
 677                         new_limit -= 1;
 678
 679                 ret = -EBUSY;
 680                 mutex_lock(&kvm->lock);
 681                 if (!kvm->created_vcpus) {
 682                         /* gmap_create will round the limit up */
 683                         struct gmap *new = gmap_create(current->mm, new_limit);
 684
 685                         if (!new) {
 686                                 ret = -ENOMEM;
 687                         } else {
 688                                 gmap_remove(kvm->arch.gmap);
 689                                 new->private = kvm;
 690                                 kvm->arch.gmap = new;
 691                                 ret = 0;
 692                         }
 693                 }
 694                 mutex_unlock(&kvm->lock);
 695                 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
 696                 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
 697                          (void *) kvm->arch.gmap->asce);
 698                 break;
 699         }
 700         default:
 701                 ret = -ENXIO;
 702                 break;
 703         }
 704         return ret;
 705 }
 706
 707 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
 708
 709 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
 710 {
 711         struct kvm_vcpu *vcpu;
 712         int i;
 713
 714         if (!test_kvm_facility(kvm, 76))
 715                 return -EINVAL;
 716
 717         mutex_lock(&kvm->lock);
 718         switch (attr->attr) {
 719         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 720                 get_random_bytes(
 721                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
 722                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 723                 kvm->arch.crypto.aes_kw = 1;
 724                 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
 725                 break;
 726         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 727                 get_random_bytes(
 728                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
 729                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 730                 kvm->arch.crypto.dea_kw = 1;
 731                 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
 732                 break;
 733         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 734                 kvm->arch.crypto.aes_kw = 0;
 735                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
 736                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 737                 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
 738                 break;
 739         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 740                 kvm->arch.crypto.dea_kw = 0;
 741                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
 742                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 743                 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
 744                 break;
 745         default:
 746                 mutex_unlock(&kvm->lock);
 747                 return -ENXIO;
 748         }
 749
 750         kvm_for_each_vcpu(i, vcpu, kvm) {
 751                 kvm_s390_vcpu_crypto_setup(vcpu);
 752                 exit_sie(vcpu);
 753         }
 754         mutex_unlock(&kvm->lock);
 755         return 0;
 756 }
 757
 758 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
 759 {
 760         int cx;
 761         struct kvm_vcpu *vcpu;
 762
 763         kvm_for_each_vcpu(cx, vcpu, kvm)
 764                 kvm_s390_sync_request(req, vcpu);
 765 }
 766
 767 /*
 768  * Must be called with kvm->srcu held to avoid races on memslots, and with
 769  * kvm->lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
 770  */
 771 static int kvm_s390_vm_start_migration(struct kvm *kvm)
 772 {
 773         struct kvm_s390_migration_state *mgs;
 774         struct kvm_memory_slot *ms;
 775         /* should be the only one */
 776         struct kvm_memslots *slots;
 777         unsigned long ram_pages;
 778         int slotnr;
 779
 780         /* migration mode already enabled */
 781         if (kvm->arch.migration_state)
 782                 return 0;
 783
 784         slots = kvm_memslots(kvm);
 785         if (!slots || !slots->used_slots)
 786                 return -EINVAL;
 787
 788         mgs = kzalloc(sizeof(*mgs), GFP_KERNEL);
 789         if (!mgs)
 790                 return -ENOMEM;
 791         kvm->arch.migration_state = mgs;
 792
 793         if (kvm->arch.use_cmma) {
 794                 /*
 795                  * Get the first slot. They are reverse sorted by base_gfn, so
 796                  * the first slot is also the one at the end of the address
 797                  * space. We have verified above that at least one slot is
 798                  * present.
 799                  */
 800                 ms = slots->memslots;
 801                 /* round up so we only use full longs */
 802                 ram_pages = roundup(ms->base_gfn + ms->npages, BITS_PER_LONG);
 803                 /* allocate enough bytes to store all the bits */
 804                 mgs->pgste_bitmap = vmalloc(ram_pages / 8);
 805                 if (!mgs->pgste_bitmap) {
 806                         kfree(mgs);
 807                         kvm->arch.migration_state = NULL;
 808                         return -ENOMEM;
 809                 }
 810
 811                 mgs->bitmap_size = ram_pages;
 812                 atomic64_set(&mgs->dirty_pages, ram_pages);
 813                 /* mark all the pages in active slots as dirty */
 814                 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
 815                         ms = slots->memslots + slotnr;
 816                         bitmap_set(mgs->pgste_bitmap, ms->base_gfn, ms->npages);
 817                 }
 818
 819                 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
 820         }
 821         return 0;
 822 }
 823
 824 /*
 825  * Must be called with kvm->lock to avoid races with ourselves and
 826  * kvm_s390_vm_start_migration.
 827  */
 828 static int kvm_s390_vm_stop_migration(struct kvm *kvm)
 829 {
 830         struct kvm_s390_migration_state *mgs;
 831
 832         /* migration mode already disabled */
 833         if (!kvm->arch.migration_state)
 834                 return 0;
 835         mgs = kvm->arch.migration_state;
 836         kvm->arch.migration_state = NULL;
 837
 838         if (kvm->arch.use_cmma) {
 839                 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
 840                 vfree(mgs->pgste_bitmap);
 841         }
 842         kfree(mgs);
 843         return 0;
 844 }
 845
 846 static int kvm_s390_vm_set_migration(struct kvm *kvm,
 847                                      struct kvm_device_attr *attr)
 848 {
 849         int idx, res = -ENXIO;
 850
 851         mutex_lock(&kvm->lock);
 852         switch (attr->attr) {
 853         case KVM_S390_VM_MIGRATION_START:
 854                 idx = srcu_read_lock(&kvm->srcu);
 855                 res = kvm_s390_vm_start_migration(kvm);
 856                 srcu_read_unlock(&kvm->srcu, idx);
 857                 break;
 858         case KVM_S390_VM_MIGRATION_STOP:
 859                 res = kvm_s390_vm_stop_migration(kvm);
 860                 break;
 861         default:
 862                 break;
 863         }
 864         mutex_unlock(&kvm->lock);
 865
 866         return res;
 867 }
 868
 869 static int kvm_s390_vm_get_migration(struct kvm *kvm,
 870                                      struct kvm_device_attr *attr)
 871 {
 872         u64 mig = (kvm->arch.migration_state != NULL);
 873
 874         if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
 875                 return -ENXIO;
 876
 877         if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
 878                 return -EFAULT;
 879         return 0;
 880 }
 881
 882 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
 883 {
 884         struct kvm_s390_vm_tod_clock gtod;
 885
 886         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
 887                 return -EFAULT;
 888
 889         if (test_kvm_facility(kvm, 139))
 890                 kvm_s390_set_tod_clock_ext(kvm, &gtod);
 891         else if (gtod.epoch_idx == 0)
 892                 kvm_s390_set_tod_clock(kvm, gtod.tod);
 893         else
 894                 return -EINVAL;
 895
 896         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
 897                 gtod.epoch_idx, gtod.tod);
 898
 899         return 0;
 900 }
 901
 902 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 903 {
 904         u8 gtod_high;
 905
 906         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
 907                                            sizeof(gtod_high)))
 908                 return -EFAULT;
 909
 910         if (gtod_high != 0)
 911                 return -EINVAL;
 912         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
 913
 914         return 0;
 915 }
 916
 917 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 918 {
 919         u64 gtod;
 920
 921         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
 922                 return -EFAULT;
 923
 924         kvm_s390_set_tod_clock(kvm, gtod);
 925         VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
 926         return 0;
 927 }
 928
 929 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 930 {
 931         int ret;
 932
 933         if (attr->flags)
 934                 return -EINVAL;
 935
 936         switch (attr->attr) {
 937         case KVM_S390_VM_TOD_EXT:
 938                 ret = kvm_s390_set_tod_ext(kvm, attr);
 939                 break;
 940         case KVM_S390_VM_TOD_HIGH:
 941                 ret = kvm_s390_set_tod_high(kvm, attr);
 942                 break;
 943         case KVM_S390_VM_TOD_LOW:
 944                 ret = kvm_s390_set_tod_low(kvm, attr);
 945                 break;
 946         default:
 947                 ret = -ENXIO;
 948                 break;
 949         }
 950         return ret;
 951 }
 952
 953 static void kvm_s390_get_tod_clock_ext(struct kvm *kvm,
 954                                         struct kvm_s390_vm_tod_clock *gtod)
 955 {
 956         struct kvm_s390_tod_clock_ext htod;
 957
 958         preempt_disable();
 959
 960         get_tod_clock_ext((char *)&htod);
 961
 962         gtod->tod = htod.tod + kvm->arch.epoch;
 963         gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
 964
 965         if (gtod->tod < htod.tod)
 966                 gtod->epoch_idx += 1;
 967
 968         preempt_enable();
 969 }
 970
 971 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
 972 {
 973         struct kvm_s390_vm_tod_clock gtod;
 974
 975         memset(&gtod, 0, sizeof(gtod));
 976
 977         if (test_kvm_facility(kvm, 139))
 978                 kvm_s390_get_tod_clock_ext(kvm, &gtod);
 979         else
 980                 gtod.tod = kvm_s390_get_tod_clock_fast(kvm);
 981
 982         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
 983                 return -EFAULT;
 984
 985         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
 986                 gtod.epoch_idx, gtod.tod);
 987         return 0;
 988 }
 989
 990 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 991 {
 992         u8 gtod_high = 0;
 993
 994         if (copy_to_user((void __user *)attr->addr, &gtod_high,
 995                                          sizeof(gtod_high)))
 996                 return -EFAULT;
 997         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
 998
 999         return 0;
1000 }
1001
1002 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1003 {
1004         u64 gtod;
1005
1006         gtod = kvm_s390_get_tod_clock_fast(kvm);
1007         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1008                 return -EFAULT;
1009         VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1010
1011         return 0;
1012 }
1013
1014 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1015 {
1016         int ret;
1017
1018         if (attr->flags)
1019                 return -EINVAL;
1020
1021         switch (attr->attr) {
1022         case KVM_S390_VM_TOD_EXT:
1023                 ret = kvm_s390_get_tod_ext(kvm, attr);
1024                 break;
1025         case KVM_S390_VM_TOD_HIGH:
1026                 ret = kvm_s390_get_tod_high(kvm, attr);
1027                 break;
1028         case KVM_S390_VM_TOD_LOW:
1029                 ret = kvm_s390_get_tod_low(kvm, attr);
1030                 break;
1031         default:
1032                 ret = -ENXIO;
1033                 break;
1034         }
1035         return ret;
1036 }
1037
1038 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1039 {
1040         struct kvm_s390_vm_cpu_processor *proc;
1041         u16 lowest_ibc, unblocked_ibc;
1042         int ret = 0;
1043
1044         mutex_lock(&kvm->lock);
1045         if (kvm->created_vcpus) {
1046                 ret = -EBUSY;
1047                 goto out;
1048         }
1049         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1050         if (!proc) {
1051                 ret = -ENOMEM;
1052                 goto out;
1053         }
1054         if (!copy_from_user(proc, (void __user *)attr->addr,
1055                             sizeof(*proc))) {
1056                 kvm->arch.model.cpuid = proc->cpuid;
1057                 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1058                 unblocked_ibc = sclp.ibc & 0xfff;
1059                 if (lowest_ibc && proc->ibc) {
1060                         if (proc->ibc > unblocked_ibc)
1061                                 kvm->arch.model.ibc = unblocked_ibc;
1062                         else if (proc->ibc < lowest_ibc)
1063                                 kvm->arch.model.ibc = lowest_ibc;
1064                         else
1065                                 kvm->arch.model.ibc = proc->ibc;
1066                 }
1067                 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1068                        S390_ARCH_FAC_LIST_SIZE_BYTE);
1069                 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1070                          kvm->arch.model.ibc,
1071                          kvm->arch.model.cpuid);
1072                 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1073                          kvm->arch.model.fac_list[0],
1074                          kvm->arch.model.fac_list[1],
1075                          kvm->arch.model.fac_list[2]);
1076         } else
1077                 ret = -EFAULT;
1078         kfree(proc);
1079 out:
1080         mutex_unlock(&kvm->lock);
1081         return ret;
1082 }
1083
1084 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1085                                        struct kvm_device_attr *attr)
1086 {
1087         struct kvm_s390_vm_cpu_feat data;
1088         int ret = -EBUSY;
1089
1090         if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1091                 return -EFAULT;
1092         if (!bitmap_subset((unsigned long *) data.feat,
1093                            kvm_s390_available_cpu_feat,
1094                            KVM_S390_VM_CPU_FEAT_NR_BITS))
1095                 return -EINVAL;
1096
1097         mutex_lock(&kvm->lock);
1098         if (!atomic_read(&kvm->online_vcpus)) {
1099                 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1100                             KVM_S390_VM_CPU_FEAT_NR_BITS);
1101                 ret = 0;
1102         }
1103         mutex_unlock(&kvm->lock);
1104         return ret;
1105 }
1106
1107 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1108                                           struct kvm_device_attr *attr)
1109 {
1110         /*
1111          * Once supported by kernel + hw, we have to store the subfunctions
1112          * in kvm->arch and remember that user space configured them.
1113          */
1114         return -ENXIO;
1115 }
1116
1117 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1118 {
1119         int ret = -ENXIO;
1120
1121         switch (attr->attr) {
1122         case KVM_S390_VM_CPU_PROCESSOR:
1123                 ret = kvm_s390_set_processor(kvm, attr);
1124                 break;
1125         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1126                 ret = kvm_s390_set_processor_feat(kvm, attr);
1127                 break;
1128         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1129                 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1130                 break;
1131         }
1132         return ret;
1133 }
1134
1135 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1136 {
1137         struct kvm_s390_vm_cpu_processor *proc;
1138         int ret = 0;
1139
1140         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1141         if (!proc) {
1142                 ret = -ENOMEM;
1143                 goto out;
1144         }
1145         proc->cpuid = kvm->arch.model.cpuid;
1146         proc->ibc = kvm->arch.model.ibc;
1147         memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1148                S390_ARCH_FAC_LIST_SIZE_BYTE);
1149         VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1150                  kvm->arch.model.ibc,
1151                  kvm->arch.model.cpuid);
1152         VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1153                  kvm->arch.model.fac_list[0],
1154                  kvm->arch.model.fac_list[1],
1155                  kvm->arch.model.fac_list[2]);
1156         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1157                 ret = -EFAULT;
1158         kfree(proc);
1159 out:
1160         return ret;
1161 }
1162
1163 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1164 {
1165         struct kvm_s390_vm_cpu_machine *mach;
1166         int ret = 0;
1167
1168         mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1169         if (!mach) {
1170                 ret = -ENOMEM;
1171                 goto out;
1172         }
1173         get_cpu_id((struct cpuid *) &mach->cpuid);
1174         mach->ibc = sclp.ibc;
1175         memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1176                S390_ARCH_FAC_LIST_SIZE_BYTE);
1177         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
1178                sizeof(S390_lowcore.stfle_fac_list));
1179         VM_EVENT(kvm, 3, "GET: host ibc:  0x%4.4x, host cpuid:  0x%16.16llx",
1180                  kvm->arch.model.ibc,
1181                  kvm->arch.model.cpuid);
1182         VM_EVENT(kvm, 3, "GET: host facmask:  0x%16.16llx.%16.16llx.%16.16llx",
1183                  mach->fac_mask[0],
1184                  mach->fac_mask[1],
1185                  mach->fac_mask[2]);
1186         VM_EVENT(kvm, 3, "GET: host faclist:  0x%16.16llx.%16.16llx.%16.16llx",
1187                  mach->fac_list[0],
1188                  mach->fac_list[1],
1189                  mach->fac_list[2]);
1190         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1191                 ret = -EFAULT;
1192         kfree(mach);
1193 out:
1194         return ret;
1195 }
1196
1197 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1198                                        struct kvm_device_attr *attr)
1199 {
1200         struct kvm_s390_vm_cpu_feat data;
1201
1202         bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1203                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1204         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1205                 return -EFAULT;
1206         return 0;
1207 }
1208
1209 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1210                                      struct kvm_device_attr *attr)
1211 {
1212         struct kvm_s390_vm_cpu_feat data;
1213
1214         bitmap_copy((unsigned long *) data.feat,
1215                     kvm_s390_available_cpu_feat,
1216                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1217         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1218                 return -EFAULT;
1219         return 0;
1220 }
1221
1222 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1223                                           struct kvm_device_attr *attr)
1224 {
1225         /*
1226          * Once we can actually configure subfunctions (kernel + hw support),
1227          * we have to check if they were already set by user space, if so copy
1228          * them from kvm->arch.
1229          */
1230         return -ENXIO;
1231 }
1232
1233 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1234                                         struct kvm_device_attr *attr)
1235 {
1236         if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1237             sizeof(struct kvm_s390_vm_cpu_subfunc)))
1238                 return -EFAULT;
1239         return 0;
1240 }
1241 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1242 {
1243         int ret = -ENXIO;
1244
1245         switch (attr->attr) {
1246         case KVM_S390_VM_CPU_PROCESSOR:
1247                 ret = kvm_s390_get_processor(kvm, attr);
1248                 break;
1249         case KVM_S390_VM_CPU_MACHINE:
1250                 ret = kvm_s390_get_machine(kvm, attr);
1251                 break;
1252         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1253                 ret = kvm_s390_get_processor_feat(kvm, attr);
1254                 break;
1255         case KVM_S390_VM_CPU_MACHINE_FEAT:
1256                 ret = kvm_s390_get_machine_feat(kvm, attr);
1257                 break;
1258         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1259                 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1260                 break;
1261         case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1262                 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1263                 break;
1264         }
1265         return ret;
1266 }
1267
1268 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1269 {
1270         int ret;
1271
1272         switch (attr->group) {
1273         case KVM_S390_VM_MEM_CTRL:
1274                 ret = kvm_s390_set_mem_control(kvm, attr);
1275                 break;
1276         case KVM_S390_VM_TOD:
1277                 ret = kvm_s390_set_tod(kvm, attr);
1278                 break;
1279         case KVM_S390_VM_CPU_MODEL:
1280                 ret = kvm_s390_set_cpu_model(kvm, attr);
1281                 break;
1282         case KVM_S390_VM_CRYPTO:
1283                 ret = kvm_s390_vm_set_crypto(kvm, attr);
1284                 break;
1285         case KVM_S390_VM_MIGRATION:
1286                 ret = kvm_s390_vm_set_migration(kvm, attr);
1287                 break;
1288         default:
1289                 ret = -ENXIO;
1290                 break;
1291         }
1292
1293         return ret;
1294 }
1295
1296 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1297 {
1298         int ret;
1299
1300         switch (attr->group) {
1301         case KVM_S390_VM_MEM_CTRL:
1302                 ret = kvm_s390_get_mem_control(kvm, attr);
1303                 break;
1304         case KVM_S390_VM_TOD:
1305                 ret = kvm_s390_get_tod(kvm, attr);
1306                 break;
1307         case KVM_S390_VM_CPU_MODEL:
1308                 ret = kvm_s390_get_cpu_model(kvm, attr);
1309                 break;
1310         case KVM_S390_VM_MIGRATION:
1311                 ret = kvm_s390_vm_get_migration(kvm, attr);
1312                 break;
1313         default:
1314                 ret = -ENXIO;
1315                 break;
1316         }
1317
1318         return ret;
1319 }
1320
1321 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1322 {
1323         int ret;
1324
1325         switch (attr->group) {
1326         case KVM_S390_VM_MEM_CTRL:
1327                 switch (attr->attr) {
1328                 case KVM_S390_VM_MEM_ENABLE_CMMA:
1329                 case KVM_S390_VM_MEM_CLR_CMMA:
1330                         ret = sclp.has_cmma ? 0 : -ENXIO;
1331                         break;
1332                 case KVM_S390_VM_MEM_LIMIT_SIZE:
1333                         ret = 0;
1334                         break;
1335                 default:
1336                         ret = -ENXIO;
1337                         break;
1338                 }
1339                 break;
1340         case KVM_S390_VM_TOD:
1341                 switch (attr->attr) {
1342                 case KVM_S390_VM_TOD_LOW:
1343                 case KVM_S390_VM_TOD_HIGH:
1344                         ret = 0;
1345                         break;
1346                 default:
1347                         ret = -ENXIO;
1348                         break;
1349                 }
1350                 break;
1351         case KVM_S390_VM_CPU_MODEL:
1352                 switch (attr->attr) {
1353                 case KVM_S390_VM_CPU_PROCESSOR:
1354                 case KVM_S390_VM_CPU_MACHINE:
1355                 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1356                 case KVM_S390_VM_CPU_MACHINE_FEAT:
1357                 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1358                         ret = 0;
1359                         break;
1360                 /* configuring subfunctions is not supported yet */
1361                 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1362                 default:
1363                         ret = -ENXIO;
1364                         break;
1365                 }
1366                 break;
1367         case KVM_S390_VM_CRYPTO:
1368                 switch (attr->attr) {
1369                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1370                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1371                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1372                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1373                         ret = 0;
1374                         break;
1375                 default:
1376                         ret = -ENXIO;
1377                         break;
1378                 }
1379                 break;
1380         case KVM_S390_VM_MIGRATION:
1381                 ret = 0;
1382                 break;
1383         default:
1384                 ret = -ENXIO;
1385                 break;
1386         }
1387
1388         return ret;
1389 }
1390
1391 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1392 {
1393         uint8_t *keys;
1394         uint64_t hva;
1395         int srcu_idx, i, r = 0;
1396
1397         if (args->flags != 0)
1398                 return -EINVAL;
1399
1400         /* Is this guest using storage keys? */
1401         if (!mm_use_skey(current->mm))
1402                 return KVM_S390_GET_SKEYS_NONE;
1403
1404         /* Enforce sane limit on memory allocation */
1405         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1406                 return -EINVAL;
1407
1408         keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1409         if (!keys)
1410                 return -ENOMEM;
1411
1412         down_read(&current->mm->mmap_sem);
1413         srcu_idx = srcu_read_lock(&kvm->srcu);
1414         for (i = 0; i < args->count; i++) {
1415                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1416                 if (kvm_is_error_hva(hva)) {
1417                         r = -EFAULT;
1418                         break;
1419                 }
1420
1421                 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1422                 if (r)
1423                         break;
1424         }
1425         srcu_read_unlock(&kvm->srcu, srcu_idx);
1426         up_read(&current->mm->mmap_sem);
1427
1428         if (!r) {
1429                 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1430                                  sizeof(uint8_t) * args->count);
1431                 if (r)
1432                         r = -EFAULT;
1433         }
1434
1435         kvfree(keys);
1436         return r;
1437 }
1438
1439 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1440 {
1441         uint8_t *keys;
1442         uint64_t hva;
1443         int srcu_idx, i, r = 0;
1444
1445         if (args->flags != 0)
1446                 return -EINVAL;
1447
1448         /* Enforce sane limit on memory allocation */
1449         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1450                 return -EINVAL;
1451
1452         keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1453         if (!keys)
1454                 return -ENOMEM;
1455
1456         r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1457                            sizeof(uint8_t) * args->count);
1458         if (r) {
1459                 r = -EFAULT;
1460                 goto out;
1461         }
1462
1463         /* Enable storage key handling for the guest */
1464         r = s390_enable_skey();
1465         if (r)
1466                 goto out;
1467
1468         down_read(&current->mm->mmap_sem);
1469         srcu_idx = srcu_read_lock(&kvm->srcu);
1470         for (i = 0; i < args->count; i++) {
1471                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1472                 if (kvm_is_error_hva(hva)) {
1473                         r = -EFAULT;
1474                         break;
1475                 }
1476
1477                 /* Lowest order bit is reserved */
1478                 if (keys[i] & 0x01) {
1479                         r = -EINVAL;
1480                         break;
1481                 }
1482
1483                 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1484                 if (r)
1485                         break;
1486         }
1487         srcu_read_unlock(&kvm->srcu, srcu_idx);
1488         up_read(&current->mm->mmap_sem);
1489 out:
1490         kvfree(keys);
1491         return r;
1492 }
1493
1494 /*
1495  * Base address and length must be sent at the start of each block, therefore
1496  * it's cheaper to send some clean data, as long as it's less than the size of
1497  * two longs.
1498  */
1499 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1500 /* for consistency */
1501 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1502
1503 /*
1504  * This function searches for the next page with dirty CMMA attributes, and
1505  * saves the attributes in the buffer up to either the end of the buffer or
1506  * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
1507  * no trailing clean bytes are saved.
1508  * In case no dirty bits were found, or if CMMA was not enabled or used, the
1509  * output buffer will indicate 0 as length.
1510  */
1511 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
1512                                   struct kvm_s390_cmma_log *args)
1513 {
1514         struct kvm_s390_migration_state *s = kvm->arch.migration_state;
1515         unsigned long bufsize, hva, pgstev, i, next, cur;
1516         int srcu_idx, peek, r = 0, rr;
1517         u8 *res;
1518
1519         cur = args->start_gfn;
1520         i = next = pgstev = 0;
1521
1522         if (unlikely(!kvm->arch.use_cmma))
1523                 return -ENXIO;
1524         /* Invalid/unsupported flags were specified */
1525         if (args->flags & ~KVM_S390_CMMA_PEEK)
1526                 return -EINVAL;
1527         /* Migration mode query, and we are not doing a migration */
1528         peek = !!(args->flags & KVM_S390_CMMA_PEEK);
1529         if (!peek && !s)
1530                 return -EINVAL;
1531         /* CMMA is disabled or was not used, or the buffer has length zero */
1532         bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
1533         if (!bufsize || !kvm->mm->context.use_cmma) {
1534                 memset(args, 0, sizeof(*args));
1535                 return 0;
1536         }
1537
1538         if (!peek) {
1539                 /* We are not peeking, and there are no dirty pages */
1540                 if (!atomic64_read(&s->dirty_pages)) {
1541                         memset(args, 0, sizeof(*args));
1542                         return 0;
1543                 }
1544                 cur = find_next_bit(s->pgste_bitmap, s->bitmap_size,
1545                                     args->start_gfn);
1546                 if (cur >= s->bitmap_size)      /* nothing found, loop back */
1547                         cur = find_next_bit(s->pgste_bitmap, s->bitmap_size, 0);
1548                 if (cur >= s->bitmap_size) {    /* again! (very unlikely) */
1549                         memset(args, 0, sizeof(*args));
1550                         return 0;
1551                 }
1552                 next = find_next_bit(s->pgste_bitmap, s->bitmap_size, cur + 1);
1553         }
1554
1555         res = vmalloc(bufsize);
1556         if (!res)
1557                 return -ENOMEM;
1558
1559         args->start_gfn = cur;
1560
1561         down_read(&kvm->mm->mmap_sem);
1562         srcu_idx = srcu_read_lock(&kvm->srcu);
1563         while (i < bufsize) {
1564                 hva = gfn_to_hva(kvm, cur);
1565                 if (kvm_is_error_hva(hva)) {
1566                         r = -EFAULT;
1567                         break;
1568                 }
1569                 /* decrement only if we actually flipped the bit to 0 */
1570                 if (!peek && test_and_clear_bit(cur, s->pgste_bitmap))
1571                         atomic64_dec(&s->dirty_pages);
1572                 r = get_pgste(kvm->mm, hva, &pgstev);
1573                 if (r < 0)
1574                         pgstev = 0;
1575                 /* save the value */
1576                 res[i++] = (pgstev >> 24) & 0x43;
1577                 /*
1578                  * if the next bit is too far away, stop.
1579                  * if we reached the previous "next", find the next one
1580                  */
1581                 if (!peek) {
1582                         if (next > cur + KVM_S390_MAX_BIT_DISTANCE)
1583                                 break;
1584                         if (cur == next)
1585                                 next = find_next_bit(s->pgste_bitmap,
1586                                                      s->bitmap_size, cur + 1);
1587                 /* reached the end of the bitmap or of the buffer, stop */
1588                         if ((next >= s->bitmap_size) ||
1589                             (next >= args->start_gfn + bufsize))
1590                                 break;
1591                 }
1592                 cur++;
1593         }
1594         srcu_read_unlock(&kvm->srcu, srcu_idx);
1595         up_read(&kvm->mm->mmap_sem);
1596         args->count = i;
1597         args->remaining = s ? atomic64_read(&s->dirty_pages) : 0;
1598
1599         rr = copy_to_user((void __user *)args->values, res, args->count);
1600         if (rr)
1601                 r = -EFAULT;
1602
1603         vfree(res);
1604         return r;
1605 }
1606
1607 /*
1608  * This function sets the CMMA attributes for the given pages. If the input
1609  * buffer has zero length, no action is taken, otherwise the attributes are
1610  * set and the mm->context.use_cmma flag is set.
1611  */
1612 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
1613                                   const struct kvm_s390_cmma_log *args)
1614 {
1615         unsigned long hva, mask, pgstev, i;
1616         uint8_t *bits;
1617         int srcu_idx, r = 0;
1618
1619         mask = args->mask;
1620
1621         if (!kvm->arch.use_cmma)
1622                 return -ENXIO;
1623         /* invalid/unsupported flags */
1624         if (args->flags != 0)
1625                 return -EINVAL;
1626         /* Enforce sane limit on memory allocation */
1627         if (args->count > KVM_S390_CMMA_SIZE_MAX)
1628                 return -EINVAL;
1629         /* Nothing to do */
1630         if (args->count == 0)
1631                 return 0;
1632
1633         bits = vmalloc(sizeof(*bits) * args->count);
1634         if (!bits)
1635                 return -ENOMEM;
1636
1637         r = copy_from_user(bits, (void __user *)args->values, args->count);
1638         if (r) {
1639                 r = -EFAULT;
1640                 goto out;
1641         }
1642
1643         down_read(&kvm->mm->mmap_sem);
1644         srcu_idx = srcu_read_lock(&kvm->srcu);
1645         for (i = 0; i < args->count; i++) {
1646                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1647                 if (kvm_is_error_hva(hva)) {
1648                         r = -EFAULT;
1649                         break;
1650                 }
1651
1652                 pgstev = bits[i];
1653                 pgstev = pgstev << 24;
1654                 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
1655                 set_pgste_bits(kvm->mm, hva, mask, pgstev);
1656         }
1657         srcu_read_unlock(&kvm->srcu, srcu_idx);
1658         up_read(&kvm->mm->mmap_sem);
1659
1660         if (!kvm->mm->context.use_cmma) {
1661                 down_write(&kvm->mm->mmap_sem);
1662                 kvm->mm->context.use_cmma = 1;
1663                 up_write(&kvm->mm->mmap_sem);
1664         }
1665 out:
1666         vfree(bits);
1667         return r;
1668 }
1669
1670 long kvm_arch_vm_ioctl(struct file *filp,
1671                        unsigned int ioctl, unsigned long arg)
1672 {
1673         struct kvm *kvm = filp->private_data;
1674         void __user *argp = (void __user *)arg;
1675         struct kvm_device_attr attr;
1676         int r;
1677
1678         switch (ioctl) {
1679         case KVM_S390_INTERRUPT: {
1680                 struct kvm_s390_interrupt s390int;
1681
1682                 r = -EFAULT;
1683                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1684                         break;
1685                 r = kvm_s390_inject_vm(kvm, &s390int);
1686                 break;
1687         }
1688         case KVM_ENABLE_CAP: {
1689                 struct kvm_enable_cap cap;
1690                 r = -EFAULT;
1691                 if (copy_from_user(&cap, argp, sizeof(cap)))
1692                         break;
1693                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1694                 break;
1695         }
1696         case KVM_CREATE_IRQCHIP: {
1697                 struct kvm_irq_routing_entry routing;
1698
1699                 r = -EINVAL;
1700                 if (kvm->arch.use_irqchip) {
1701                         /* Set up dummy routing. */
1702                         memset(&routing, 0, sizeof(routing));
1703                         r = kvm_set_irq_routing(kvm, &routing, 0, 0);
1704                 }
1705                 break;
1706         }
1707         case KVM_SET_DEVICE_ATTR: {
1708                 r = -EFAULT;
1709                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1710                         break;
1711                 r = kvm_s390_vm_set_attr(kvm, &attr);
1712                 break;
1713         }
1714         case KVM_GET_DEVICE_ATTR: {
1715                 r = -EFAULT;
1716                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1717                         break;
1718                 r = kvm_s390_vm_get_attr(kvm, &attr);
1719                 break;
1720         }
1721         case KVM_HAS_DEVICE_ATTR: {
1722                 r = -EFAULT;
1723                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1724                         break;
1725                 r = kvm_s390_vm_has_attr(kvm, &attr);
1726                 break;
1727         }
1728         case KVM_S390_GET_SKEYS: {
1729                 struct kvm_s390_skeys args;
1730
1731                 r = -EFAULT;
1732                 if (copy_from_user(&args, argp,
1733                                    sizeof(struct kvm_s390_skeys)))
1734                         break;
1735                 r = kvm_s390_get_skeys(kvm, &args);
1736                 break;
1737         }
1738         case KVM_S390_SET_SKEYS: {
1739                 struct kvm_s390_skeys args;
1740
1741                 r = -EFAULT;
1742                 if (copy_from_user(&args, argp,
1743                                    sizeof(struct kvm_s390_skeys)))
1744                         break;
1745                 r = kvm_s390_set_skeys(kvm, &args);
1746                 break;
1747         }
1748         case KVM_S390_GET_CMMA_BITS: {
1749                 struct kvm_s390_cmma_log args;
1750
1751                 r = -EFAULT;
1752                 if (copy_from_user(&args, argp, sizeof(args)))
1753                         break;
1754                 r = kvm_s390_get_cmma_bits(kvm, &args);
1755                 if (!r) {
1756                         r = copy_to_user(argp, &args, sizeof(args));
1757                         if (r)
1758                                 r = -EFAULT;
1759                 }
1760                 break;
1761         }
1762         case KVM_S390_SET_CMMA_BITS: {
1763                 struct kvm_s390_cmma_log args;
1764
1765                 r = -EFAULT;
1766                 if (copy_from_user(&args, argp, sizeof(args)))
1767                         break;
1768                 r = kvm_s390_set_cmma_bits(kvm, &args);
1769                 break;
1770         }
1771         default:
1772                 r = -ENOTTY;
1773         }
1774
1775         return r;
1776 }
1777
1778 static int kvm_s390_query_ap_config(u8 *config)
1779 {
1780         u32 fcn_code = 0x04000000UL;
1781         u32 cc = 0;
1782
1783         memset(config, 0, 128);
1784         asm volatile(
1785                 "lgr 0,%1\n"
1786                 "lgr 2,%2\n"
1787                 ".long 0xb2af0000\n"            /* PQAP(QCI) */
1788                 "0: ipm %0\n"
1789                 "srl %0,28\n"
1790                 "1:\n"
1791                 EX_TABLE(0b, 1b)
1792                 : "+r" (cc)
1793                 : "r" (fcn_code), "r" (config)
1794                 : "cc", "0", "2", "memory"
1795         );
1796
1797         return cc;
1798 }
1799
1800 static int kvm_s390_apxa_installed(void)
1801 {
1802         u8 config[128];
1803         int cc;
1804
1805         if (test_facility(12)) {
1806                 cc = kvm_s390_query_ap_config(config);
1807
1808                 if (cc)
1809                         pr_err("PQAP(QCI) failed with cc=%d", cc);
1810                 else
1811                         return config[0] & 0x40;
1812         }
1813
1814         return 0;
1815 }
1816
1817 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1818 {
1819         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1820
1821         if (kvm_s390_apxa_installed())
1822                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1823         else
1824                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1825 }
1826
1827 static u64 kvm_s390_get_initial_cpuid(void)
1828 {
1829         struct cpuid cpuid;
1830
1831         get_cpu_id(&cpuid);
1832         cpuid.version = 0xff;
1833         return *((u64 *) &cpuid);
1834 }
1835
1836 static void kvm_s390_crypto_init(struct kvm *kvm)
1837 {
1838         if (!test_kvm_facility(kvm, 76))
1839                 return;
1840
1841         kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
1842         kvm_s390_set_crycb_format(kvm);
1843
1844         /* Enable AES/DEA protected key functions by default */
1845         kvm->arch.crypto.aes_kw = 1;
1846         kvm->arch.crypto.dea_kw = 1;
1847         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1848                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1849         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1850                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1851 }
1852
1853 static void sca_dispose(struct kvm *kvm)
1854 {
1855         if (kvm->arch.use_esca)
1856                 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1857         else
1858                 free_page((unsigned long)(kvm->arch.sca));
1859         kvm->arch.sca = NULL;
1860 }
1861
1862 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1863 {
1864         gfp_t alloc_flags = GFP_KERNEL;
1865         int i, rc;
1866         char debug_name[16];
1867         static unsigned long sca_offset;
1868
1869         rc = -EINVAL;
1870 #ifdef CONFIG_KVM_S390_UCONTROL
1871         if (type & ~KVM_VM_S390_UCONTROL)
1872                 goto out_err;
1873         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1874                 goto out_err;
1875 #else
1876         if (type)
1877                 goto out_err;
1878 #endif
1879
1880         rc = s390_enable_sie();
1881         if (rc)
1882                 goto out_err;
1883
1884         rc = -ENOMEM;
1885
1886         kvm->arch.use_esca = 0; /* start with basic SCA */
1887         if (!sclp.has_64bscao)
1888                 alloc_flags |= GFP_DMA;
1889         rwlock_init(&kvm->arch.sca_lock);
1890         kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
1891         if (!kvm->arch.sca)
1892                 goto out_err;
1893         spin_lock(&kvm_lock);
1894         sca_offset += 16;
1895         if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1896                 sca_offset = 0;
1897         kvm->arch.sca = (struct bsca_block *)
1898                         ((char *) kvm->arch.sca + sca_offset);
1899         spin_unlock(&kvm_lock);
1900
1901         sprintf(debug_name, "kvm-%u", current->pid);
1902
1903         kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1904         if (!kvm->arch.dbf)
1905                 goto out_err;
1906
1907         kvm->arch.sie_page2 =
1908              (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1909         if (!kvm->arch.sie_page2)
1910                 goto out_err;
1911
1912         /* Populate the facility mask initially. */
1913         memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
1914                sizeof(S390_lowcore.stfle_fac_list));
1915         for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1916                 if (i < kvm_s390_fac_list_mask_size())
1917                         kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
1918                 else
1919                         kvm->arch.model.fac_mask[i] = 0UL;
1920         }
1921
1922         /* Populate the facility list initially. */
1923         kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1924         memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
1925                S390_ARCH_FAC_LIST_SIZE_BYTE);
1926
1927         /* we are always in czam mode - even on pre z14 machines */
1928         set_kvm_facility(kvm->arch.model.fac_mask, 138);
1929         set_kvm_facility(kvm->arch.model.fac_list, 138);
1930         /* we emulate STHYI in kvm */
1931         set_kvm_facility(kvm->arch.model.fac_mask, 74);
1932         set_kvm_facility(kvm->arch.model.fac_list, 74);
1933         if (MACHINE_HAS_TLB_GUEST) {
1934                 set_kvm_facility(kvm->arch.model.fac_mask, 147);
1935                 set_kvm_facility(kvm->arch.model.fac_list, 147);
1936         }
1937
1938         kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
1939         kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1940
1941         kvm_s390_crypto_init(kvm);
1942
1943         mutex_init(&kvm->arch.float_int.ais_lock);
1944         kvm->arch.float_int.simm = 0;
1945         kvm->arch.float_int.nimm = 0;
1946         spin_lock_init(&kvm->arch.float_int.lock);
1947         for (i = 0; i < FIRQ_LIST_COUNT; i++)
1948                 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1949         init_waitqueue_head(&kvm->arch.ipte_wq);
1950         mutex_init(&kvm->arch.ipte_mutex);
1951
1952         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1953         VM_EVENT(kvm, 3, "vm created with type %lu", type);
1954
1955         if (type & KVM_VM_S390_UCONTROL) {
1956                 kvm->arch.gmap = NULL;
1957                 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
1958         } else {
1959                 if (sclp.hamax == U64_MAX)
1960                         kvm->arch.mem_limit = TASK_SIZE_MAX;
1961                 else
1962                         kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
1963                                                     sclp.hamax + 1);
1964                 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
1965                 if (!kvm->arch.gmap)
1966                         goto out_err;
1967                 kvm->arch.gmap->private = kvm;
1968                 kvm->arch.gmap->pfault_enabled = 0;
1969         }
1970
1971         kvm->arch.css_support = 0;
1972         kvm->arch.use_irqchip = 0;
1973         kvm->arch.epoch = 0;
1974
1975         spin_lock_init(&kvm->arch.start_stop_lock);
1976         kvm_s390_vsie_init(kvm);
1977         KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
1978
1979         return 0;
1980 out_err:
1981         free_page((unsigned long)kvm->arch.sie_page2);
1982         debug_unregister(kvm->arch.dbf);
1983         sca_dispose(kvm);
1984         KVM_EVENT(3, "creation of vm failed: %d", rc);
1985         return rc;
1986 }
1987
1988 bool kvm_arch_has_vcpu_debugfs(void)
1989 {
1990         return false;
1991 }
1992
1993 int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
1994 {
1995         return 0;
1996 }
1997
1998 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1999 {
2000         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
2001         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
2002         kvm_s390_clear_local_irqs(vcpu);
2003         kvm_clear_async_pf_completion_queue(vcpu);
2004         if (!kvm_is_ucontrol(vcpu->kvm))
2005                 sca_del_vcpu(vcpu);
2006
2007         if (kvm_is_ucontrol(vcpu->kvm))
2008                 gmap_remove(vcpu->arch.gmap);
2009
2010         if (vcpu->kvm->arch.use_cmma)
2011                 kvm_s390_vcpu_unsetup_cmma(vcpu);
2012         free_page((unsigned long)(vcpu->arch.sie_block));
2013
2014         kvm_vcpu_uninit(vcpu);
2015         kmem_cache_free(kvm_vcpu_cache, vcpu);
2016 }
2017
2018 static void kvm_free_vcpus(struct kvm *kvm)
2019 {
2020         unsigned int i;
2021         struct kvm_vcpu *vcpu;
2022
2023         kvm_for_each_vcpu(i, vcpu, kvm)
2024                 kvm_arch_vcpu_destroy(vcpu);
2025
2026         mutex_lock(&kvm->lock);
2027         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2028                 kvm->vcpus[i] = NULL;
2029
2030         atomic_set(&kvm->online_vcpus, 0);
2031         mutex_unlock(&kvm->lock);
2032 }
2033
2034 void kvm_arch_destroy_vm(struct kvm *kvm)
2035 {
2036         kvm_free_vcpus(kvm);
2037         sca_dispose(kvm);
2038         debug_unregister(kvm->arch.dbf);
2039         free_page((unsigned long)kvm->arch.sie_page2);
2040         if (!kvm_is_ucontrol(kvm))
2041                 gmap_remove(kvm->arch.gmap);
2042         kvm_s390_destroy_adapters(kvm);
2043         kvm_s390_clear_float_irqs(kvm);
2044         kvm_s390_vsie_destroy(kvm);
2045         if (kvm->arch.migration_state) {
2046                 vfree(kvm->arch.migration_state->pgste_bitmap);
2047                 kfree(kvm->arch.migration_state);
2048         }
2049         KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
2050 }
2051
2052 /* Section: vcpu related */
2053 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2054 {
2055         vcpu->arch.gmap = gmap_create(current->mm, -1UL);
2056         if (!vcpu->arch.gmap)
2057                 return -ENOMEM;
2058         vcpu->arch.gmap->private = vcpu->kvm;
2059
2060         return 0;
2061 }
2062
2063 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2064 {
2065         if (!kvm_s390_use_sca_entries())
2066                 return;
2067         read_lock(&vcpu->kvm->arch.sca_lock);
2068         if (vcpu->kvm->arch.use_esca) {
2069                 struct esca_block *sca = vcpu->kvm->arch.sca;
2070
2071                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2072                 sca->cpu[vcpu->vcpu_id].sda = 0;
2073         } else {
2074                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2075
2076                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2077                 sca->cpu[vcpu->vcpu_id].sda = 0;
2078         }
2079         read_unlock(&vcpu->kvm->arch.sca_lock);
2080 }
2081
2082 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
2083 {
2084         if (!kvm_s390_use_sca_entries()) {
2085                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2086
2087                 /* we still need the basic sca for the ipte control */
2088                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2089                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2090         }
2091         read_lock(&vcpu->kvm->arch.sca_lock);
2092         if (vcpu->kvm->arch.use_esca) {
2093                 struct esca_block *sca = vcpu->kvm->arch.sca;
2094
2095                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2096                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2097                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
2098                 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2099                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2100         } else {
2101                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2102
2103                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2104                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2105                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2106                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2107         }
2108         read_unlock(&vcpu->kvm->arch.sca_lock);
2109 }
2110
2111 /* Basic SCA to Extended SCA data copy routines */
2112 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2113 {
2114         d->sda = s->sda;
2115         d->sigp_ctrl.c = s->sigp_ctrl.c;
2116         d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2117 }
2118
2119 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2120 {
2121         int i;
2122
2123         d->ipte_control = s->ipte_control;
2124         d->mcn[0] = s->mcn;
2125         for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2126                 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2127 }
2128
2129 static int sca_switch_to_extended(struct kvm *kvm)
2130 {
2131         struct bsca_block *old_sca = kvm->arch.sca;
2132         struct esca_block *new_sca;
2133         struct kvm_vcpu *vcpu;
2134         unsigned int vcpu_idx;
2135         u32 scaol, scaoh;
2136
2137         new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2138         if (!new_sca)
2139                 return -ENOMEM;
2140
2141         scaoh = (u32)((u64)(new_sca) >> 32);
2142         scaol = (u32)(u64)(new_sca) & ~0x3fU;
2143
2144         kvm_s390_vcpu_block_all(kvm);
2145         write_lock(&kvm->arch.sca_lock);
2146
2147         sca_copy_b_to_e(new_sca, old_sca);
2148
2149         kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2150                 vcpu->arch.sie_block->scaoh = scaoh;
2151                 vcpu->arch.sie_block->scaol = scaol;
2152                 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2153         }
2154         kvm->arch.sca = new_sca;
2155         kvm->arch.use_esca = 1;
2156
2157         write_unlock(&kvm->arch.sca_lock);
2158         kvm_s390_vcpu_unblock_all(kvm);
2159
2160         free_page((unsigned long)old_sca);
2161
2162         VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2163                  old_sca, kvm->arch.sca);
2164         return 0;
2165 }
2166
2167 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2168 {
2169         int rc;
2170
2171         if (!kvm_s390_use_sca_entries()) {
2172                 if (id < KVM_MAX_VCPUS)
2173                         return true;
2174                 return false;
2175         }
2176         if (id < KVM_S390_BSCA_CPU_SLOTS)
2177                 return true;
2178         if (!sclp.has_esca || !sclp.has_64bscao)
2179                 return false;
2180
2181         mutex_lock(&kvm->lock);
2182         rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2183         mutex_unlock(&kvm->lock);
2184
2185         return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
2186 }
2187
2188 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
2189 {
2190         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2191         kvm_clear_async_pf_completion_queue(vcpu);
2192         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
2193                                     KVM_SYNC_GPRS |
2194                                     KVM_SYNC_ACRS |
2195                                     KVM_SYNC_CRS |
2196                                     KVM_SYNC_ARCH0 |
2197                                     KVM_SYNC_PFAULT;
2198         kvm_s390_set_prefix(vcpu, 0);
2199         if (test_kvm_facility(vcpu->kvm, 64))
2200                 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
2201         if (test_kvm_facility(vcpu->kvm, 133))
2202                 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
2203         /* fprs can be synchronized via vrs, even if the guest has no vx. With
2204          * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
2205          */
2206         if (MACHINE_HAS_VX)
2207                 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
2208         else
2209                 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
2210
2211         if (kvm_is_ucontrol(vcpu->kvm))
2212                 return __kvm_ucontrol_vcpu_init(vcpu);
2213
2214         return 0;
2215 }
2216
2217 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2218 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2219 {
2220         WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
2221         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2222         vcpu->arch.cputm_start = get_tod_clock_fast();
2223         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2224 }
2225
2226 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2227 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2228 {
2229         WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
2230         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2231         vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2232         vcpu->arch.cputm_start = 0;
2233         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2234 }
2235
2236 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2237 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2238 {
2239         WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2240         vcpu->arch.cputm_enabled = true;
2241         __start_cpu_timer_accounting(vcpu);
2242 }
2243
2244 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2245 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2246 {
2247         WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2248         __stop_cpu_timer_accounting(vcpu);
2249         vcpu->arch.cputm_enabled = false;
2250 }
2251
2252 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2253 {
2254         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2255         __enable_cpu_timer_accounting(vcpu);
2256         preempt_enable();
2257 }
2258
2259 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2260 {
2261         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2262         __disable_cpu_timer_accounting(vcpu);
2263         preempt_enable();
2264 }
2265
2266 /* set the cpu timer - may only be called from the VCPU thread itself */
2267 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
2268 {
2269         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2270         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2271         if (vcpu->arch.cputm_enabled)
2272                 vcpu->arch.cputm_start = get_tod_clock_fast();
2273         vcpu->arch.sie_block->cputm = cputm;
2274         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2275         preempt_enable();
2276 }
2277
2278 /* update and get the cpu timer - can also be called from other VCPU threads */
2279 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
2280 {
2281         unsigned int seq;
2282         __u64 value;
2283
2284         if (unlikely(!vcpu->arch.cputm_enabled))
2285                 return vcpu->arch.sie_block->cputm;
2286
2287         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2288         do {
2289                 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
2290                 /*
2291                  * If the writer would ever execute a read in the critical
2292                  * section, e.g. in irq context, we have a deadlock.
2293                  */
2294                 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
2295                 value = vcpu->arch.sie_block->cputm;
2296                 /* if cputm_start is 0, accounting is being started/stopped */
2297                 if (likely(vcpu->arch.cputm_start))
2298                         value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2299         } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
2300         preempt_enable();
2301         return value;
2302 }
2303
2304 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2305 {
2306
2307         gmap_enable(vcpu->arch.enabled_gmap);
2308         atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
2309         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2310                 __start_cpu_timer_accounting(vcpu);
2311         vcpu->cpu = cpu;
2312 }
2313
2314 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2315 {
2316         vcpu->cpu = -1;
2317         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2318                 __stop_cpu_timer_accounting(vcpu);
2319         atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
2320         vcpu->arch.enabled_gmap = gmap_get_enabled();
2321         gmap_disable(vcpu->arch.enabled_gmap);
2322
2323 }
2324
2325 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
2326 {
2327         /* this equals initial cpu reset in pop, but we don't switch to ESA */
2328         vcpu->arch.sie_block->gpsw.mask = 0UL;
2329         vcpu->arch.sie_block->gpsw.addr = 0UL;
2330         kvm_s390_set_prefix(vcpu, 0);
2331         kvm_s390_set_cpu_timer(vcpu, 0);
2332         vcpu->arch.sie_block->ckc       = 0UL;
2333         vcpu->arch.sie_block->todpr     = 0;
2334         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
2335         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
2336         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
2337         /* make sure the new fpc will be lazily loaded */
2338         save_fpu_regs();
2339         current->thread.fpu.fpc = 0;
2340         vcpu->arch.sie_block->gbea = 1;
2341         vcpu->arch.sie_block->pp = 0;
2342         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2343         kvm_clear_async_pf_completion_queue(vcpu);
2344         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
2345                 kvm_s390_vcpu_stop(vcpu);
2346         kvm_s390_clear_local_irqs(vcpu);
2347 }
2348
2349 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
2350 {
2351         mutex_lock(&vcpu->kvm->lock);
2352         preempt_disable();
2353         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
2354         preempt_enable();
2355         mutex_unlock(&vcpu->kvm->lock);
2356         if (!kvm_is_ucontrol(vcpu->kvm)) {
2357                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
2358                 sca_add_vcpu(vcpu);
2359         }
2360         if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
2361                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2362         /* make vcpu_load load the right gmap on the first trigger */
2363         vcpu->arch.enabled_gmap = vcpu->arch.gmap;
2364 }
2365
2366 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
2367 {
2368         if (!test_kvm_facility(vcpu->kvm, 76))
2369                 return;
2370
2371         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
2372
2373         if (vcpu->kvm->arch.crypto.aes_kw)
2374                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
2375         if (vcpu->kvm->arch.crypto.dea_kw)
2376                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
2377
2378         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
2379 }
2380
2381 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
2382 {
2383         free_page(vcpu->arch.sie_block->cbrlo);
2384         vcpu->arch.sie_block->cbrlo = 0;
2385 }
2386
2387 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
2388 {
2389         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
2390         if (!vcpu->arch.sie_block->cbrlo)
2391                 return -ENOMEM;
2392
2393         vcpu->arch.sie_block->ecb2 &= ~ECB2_PFMFI;
2394         return 0;
2395 }
2396
2397 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
2398 {
2399         struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
2400
2401         vcpu->arch.sie_block->ibc = model->ibc;
2402         if (test_kvm_facility(vcpu->kvm, 7))
2403                 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
2404 }
2405
2406 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
2407 {
2408         int rc = 0;
2409
2410         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
2411                                                     CPUSTAT_SM |
2412                                                     CPUSTAT_STOPPED);
2413
2414         if (test_kvm_facility(vcpu->kvm, 78))
2415                 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
2416         else if (test_kvm_facility(vcpu->kvm, 8))
2417                 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
2418
2419         kvm_s390_vcpu_setup_model(vcpu);
2420
2421         /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2422         if (MACHINE_HAS_ESOP)
2423                 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
2424         if (test_kvm_facility(vcpu->kvm, 9))
2425                 vcpu->arch.sie_block->ecb |= ECB_SRSI;
2426         if (test_kvm_facility(vcpu->kvm, 73))
2427                 vcpu->arch.sie_block->ecb |= ECB_TE;
2428
2429         if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
2430                 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
2431         if (test_kvm_facility(vcpu->kvm, 130))
2432                 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
2433         vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
2434         if (sclp.has_cei)
2435                 vcpu->arch.sie_block->eca |= ECA_CEI;
2436         if (sclp.has_ib)
2437                 vcpu->arch.sie_block->eca |= ECA_IB;
2438         if (sclp.has_siif)
2439                 vcpu->arch.sie_block->eca |= ECA_SII;
2440         if (sclp.has_sigpif)
2441                 vcpu->arch.sie_block->eca |= ECA_SIGPI;
2442         if (test_kvm_facility(vcpu->kvm, 129)) {
2443                 vcpu->arch.sie_block->eca |= ECA_VX;
2444                 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
2445         }
2446         if (test_kvm_facility(vcpu->kvm, 139))
2447                 vcpu->arch.sie_block->ecd |= ECD_MEF;
2448
2449         vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2450                                         | SDNXC;
2451         vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
2452
2453         if (sclp.has_kss)
2454                 atomic_or(CPUSTAT_KSS, &vcpu->arch.sie_block->cpuflags);
2455         else
2456                 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
2457
2458         if (vcpu->kvm->arch.use_cmma) {
2459                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
2460                 if (rc)
2461                         return rc;
2462         }
2463         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2464         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
2465
2466         kvm_s390_vcpu_crypto_setup(vcpu);
2467
2468         return rc;
2469 }
2470
2471 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
2472                                       unsigned int id)
2473 {
2474         struct kvm_vcpu *vcpu;
2475         struct sie_page *sie_page;
2476         int rc = -EINVAL;
2477
2478         if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
2479                 goto out;
2480
2481         rc = -ENOMEM;
2482
2483         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
2484         if (!vcpu)
2485                 goto out;
2486
2487         BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
2488         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2489         if (!sie_page)
2490                 goto out_free_cpu;
2491
2492         vcpu->arch.sie_block = &sie_page->sie_block;
2493         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2494
2495         /* the real guest size will always be smaller than msl */
2496         vcpu->arch.sie_block->mso = 0;
2497         vcpu->arch.sie_block->msl = sclp.hamax;
2498
2499         vcpu->arch.sie_block->icpua = id;
2500         spin_lock_init(&vcpu->arch.local_int.lock);
2501         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
2502         vcpu->arch.local_int.wq = &vcpu->wq;
2503         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
2504         seqcount_init(&vcpu->arch.cputm_seqcount);
2505
2506         rc = kvm_vcpu_init(vcpu, kvm, id);
2507         if (rc)
2508                 goto out_free_sie_block;
2509         VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
2510                  vcpu->arch.sie_block);
2511         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
2512
2513         return vcpu;
2514 out_free_sie_block:
2515         free_page((unsigned long)(vcpu->arch.sie_block));
2516 out_free_cpu:
2517         kmem_cache_free(kvm_vcpu_cache, vcpu);
2518 out:
2519         return ERR_PTR(rc);
2520 }
2521
2522 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2523 {
2524         return kvm_s390_vcpu_has_irq(vcpu, 0);
2525 }
2526
2527 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
2528 {
2529         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
2530 }
2531
2532 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
2533 {
2534         atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2535         exit_sie(vcpu);
2536 }
2537
2538 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
2539 {
2540         atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2541 }
2542
2543 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2544 {
2545         atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2546         exit_sie(vcpu);
2547 }
2548
2549 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2550 {
2551         atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2552 }
2553
2554 /*
2555  * Kick a guest cpu out of SIE and wait until SIE is not running.
2556  * If the CPU is not running (e.g. waiting as idle) the function will
2557  * return immediately. */
2558 void exit_sie(struct kvm_vcpu *vcpu)
2559 {
2560         atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
2561         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2562                 cpu_relax();
2563 }
2564
2565 /* Kick a guest cpu out of SIE to process a request synchronously */
2566 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
2567 {
2568         kvm_make_request(req, vcpu);
2569         kvm_s390_vcpu_request(vcpu);
2570 }
2571
2572 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2573                               unsigned long end)
2574 {
2575         struct kvm *kvm = gmap->private;
2576         struct kvm_vcpu *vcpu;
2577         unsigned long prefix;
2578         int i;
2579
2580         if (gmap_is_shadow(gmap))
2581                 return;
2582         if (start >= 1UL << 31)
2583                 /* We are only interested in prefix pages */
2584                 return;
2585         kvm_for_each_vcpu(i, vcpu, kvm) {
2586                 /* match against both prefix pages */
2587                 prefix = kvm_s390_get_prefix(vcpu);
2588                 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2589                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2590                                    start, end);
2591                         kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2592                 }
2593         }
2594 }
2595
2596 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2597 {
2598         /* kvm common code refers to this, but never calls it */
2599         BUG();
2600         return 0;
2601 }
2602
2603 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2604                                            struct kvm_one_reg *reg)
2605 {
2606         int r = -EINVAL;
2607
2608         switch (reg->id) {
2609         case KVM_REG_S390_TODPR:
2610                 r = put_user(vcpu->arch.sie_block->todpr,
2611                              (u32 __user *)reg->addr);
2612                 break;
2613         case KVM_REG_S390_EPOCHDIFF:
2614                 r = put_user(vcpu->arch.sie_block->epoch,
2615                              (u64 __user *)reg->addr);
2616                 break;
2617         case KVM_REG_S390_CPU_TIMER:
2618                 r = put_user(kvm_s390_get_cpu_timer(vcpu),
2619                              (u64 __user *)reg->addr);
2620                 break;
2621         case KVM_REG_S390_CLOCK_COMP:
2622                 r = put_user(vcpu->arch.sie_block->ckc,
2623                              (u64 __user *)reg->addr);
2624                 break;
2625         case KVM_REG_S390_PFTOKEN:
2626                 r = put_user(vcpu->arch.pfault_token,
2627                              (u64 __user *)reg->addr);
2628                 break;
2629         case KVM_REG_S390_PFCOMPARE:
2630                 r = put_user(vcpu->arch.pfault_compare,
2631                              (u64 __user *)reg->addr);
2632                 break;
2633         case KVM_REG_S390_PFSELECT:
2634                 r = put_user(vcpu->arch.pfault_select,
2635                              (u64 __user *)reg->addr);
2636                 break;
2637         case KVM_REG_S390_PP:
2638                 r = put_user(vcpu->arch.sie_block->pp,
2639                              (u64 __user *)reg->addr);
2640                 break;
2641         case KVM_REG_S390_GBEA:
2642                 r = put_user(vcpu->arch.sie_block->gbea,
2643                              (u64 __user *)reg->addr);
2644                 break;
2645         default:
2646                 break;
2647         }
2648
2649         return r;
2650 }
2651
2652 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2653                                            struct kvm_one_reg *reg)
2654 {
2655         int r = -EINVAL;
2656         __u64 val;
2657
2658         switch (reg->id) {
2659         case KVM_REG_S390_TODPR:
2660                 r = get_user(vcpu->arch.sie_block->todpr,
2661                              (u32 __user *)reg->addr);
2662                 break;
2663         case KVM_REG_S390_EPOCHDIFF:
2664                 r = get_user(vcpu->arch.sie_block->epoch,
2665                              (u64 __user *)reg->addr);
2666                 break;
2667         case KVM_REG_S390_CPU_TIMER:
2668                 r = get_user(val, (u64 __user *)reg->addr);
2669                 if (!r)
2670                         kvm_s390_set_cpu_timer(vcpu, val);
2671                 break;
2672         case KVM_REG_S390_CLOCK_COMP:
2673                 r = get_user(vcpu->arch.sie_block->ckc,
2674                              (u64 __user *)reg->addr);
2675                 break;
2676         case KVM_REG_S390_PFTOKEN:
2677                 r = get_user(vcpu->arch.pfault_token,
2678                              (u64 __user *)reg->addr);
2679                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2680                         kvm_clear_async_pf_completion_queue(vcpu);
2681                 break;
2682         case KVM_REG_S390_PFCOMPARE:
2683                 r = get_user(vcpu->arch.pfault_compare,
2684                              (u64 __user *)reg->addr);
2685                 break;
2686         case KVM_REG_S390_PFSELECT:
2687                 r = get_user(vcpu->arch.pfault_select,
2688                              (u64 __user *)reg->addr);
2689                 break;
2690         case KVM_REG_S390_PP:
2691                 r = get_user(vcpu->arch.sie_block->pp,
2692                              (u64 __user *)reg->addr);
2693                 break;
2694         case KVM_REG_S390_GBEA:
2695                 r = get_user(vcpu->arch.sie_block->gbea,
2696                              (u64 __user *)reg->addr);
2697                 break;
2698         default:
2699                 break;
2700         }
2701
2702         return r;
2703 }
2704
2705 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2706 {
2707         kvm_s390_vcpu_initial_reset(vcpu);
2708         return 0;
2709 }
2710
2711 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2712 {
2713         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
2714         return 0;
2715 }
2716
2717 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2718 {
2719         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
2720         return 0;
2721 }
2722
2723 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2724                                   struct kvm_sregs *sregs)
2725 {
2726         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
2727         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
2728         return 0;
2729 }
2730
2731 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2732                                   struct kvm_sregs *sregs)
2733 {
2734         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
2735         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
2736         return 0;
2737 }
2738
2739 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2740 {
2741         if (test_fp_ctl(fpu->fpc))
2742                 return -EINVAL;
2743         vcpu->run->s.regs.fpc = fpu->fpc;
2744         if (MACHINE_HAS_VX)
2745                 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
2746                                  (freg_t *) fpu->fprs);
2747         else
2748                 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
2749         return 0;
2750 }
2751
2752 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2753 {
2754         /* make sure we have the latest values */
2755         save_fpu_regs();
2756         if (MACHINE_HAS_VX)
2757                 convert_vx_to_fp((freg_t *) fpu->fprs,
2758                                  (__vector128 *) vcpu->run->s.regs.vrs);
2759         else
2760                 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
2761         fpu->fpc = vcpu->run->s.regs.fpc;
2762         return 0;
2763 }
2764
2765 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2766 {
2767         int rc = 0;
2768
2769         if (!is_vcpu_stopped(vcpu))
2770                 rc = -EBUSY;
2771         else {
2772                 vcpu->run->psw_mask = psw.mask;
2773                 vcpu->run->psw_addr = psw.addr;
2774         }
2775         return rc;
2776 }
2777
2778 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2779                                   struct kvm_translation *tr)
2780 {
2781         return -EINVAL; /* not implemented yet */
2782 }
2783
2784 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2785                               KVM_GUESTDBG_USE_HW_BP | \
2786                               KVM_GUESTDBG_ENABLE)
2787
2788 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2789                                         struct kvm_guest_debug *dbg)
2790 {
2791         int rc = 0;
2792
2793         vcpu->guest_debug = 0;
2794         kvm_s390_clear_bp_data(vcpu);
2795
2796         if (dbg->control & ~VALID_GUESTDBG_FLAGS)
2797                 return -EINVAL;
2798         if (!sclp.has_gpere)
2799                 return -EINVAL;
2800
2801         if (dbg->control & KVM_GUESTDBG_ENABLE) {
2802                 vcpu->guest_debug = dbg->control;
2803                 /* enforce guest PER */
2804                 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2805
2806                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2807                         rc = kvm_s390_import_bp_data(vcpu, dbg);
2808         } else {
2809                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2810                 vcpu->arch.guestdbg.last_bp = 0;
2811         }
2812
2813         if (rc) {
2814                 vcpu->guest_debug = 0;
2815                 kvm_s390_clear_bp_data(vcpu);
2816                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2817         }
2818
2819         return rc;
2820 }
2821
2822 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2823                                     struct kvm_mp_state *mp_state)
2824 {
2825         /* CHECK_STOP and LOAD are not supported yet */
2826         return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2827                                        KVM_MP_STATE_OPERATING;
2828 }
2829
2830 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2831                                     struct kvm_mp_state *mp_state)
2832 {
2833         int rc = 0;
2834
2835         /* user space knows about this interface - let it control the state */
2836         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2837
2838         switch (mp_state->mp_state) {
2839         case KVM_MP_STATE_STOPPED:
2840                 kvm_s390_vcpu_stop(vcpu);
2841                 break;
2842         case KVM_MP_STATE_OPERATING:
2843                 kvm_s390_vcpu_start(vcpu);
2844                 break;
2845         case KVM_MP_STATE_LOAD:
2846         case KVM_MP_STATE_CHECK_STOP:
2847                 /* fall through - CHECK_STOP and LOAD are not supported yet */
2848         default:
2849                 rc = -ENXIO;
2850         }
2851
2852         return rc;
2853 }
2854
2855 static bool ibs_enabled(struct kvm_vcpu *vcpu)
2856 {
2857         return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2858 }
2859
2860 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2861 {
2862 retry:
2863         kvm_s390_vcpu_request_handled(vcpu);
2864         if (!kvm_request_pending(vcpu))
2865                 return 0;
2866         /*
2867          * We use MMU_RELOAD just to re-arm the ipte notifier for the
2868          * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2869          * This ensures that the ipte instruction for this request has
2870          * already finished. We might race against a second unmapper that
2871          * wants to set the blocking bit. Lets just retry the request loop.
2872          */
2873         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2874                 int rc;
2875                 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2876                                           kvm_s390_get_prefix(vcpu),
2877                                           PAGE_SIZE * 2, PROT_WRITE);
2878                 if (rc) {
2879                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2880                         return rc;
2881                 }
2882                 goto retry;
2883         }
2884
2885         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2886                 vcpu->arch.sie_block->ihcpu = 0xffff;
2887                 goto retry;
2888         }
2889
2890         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2891                 if (!ibs_enabled(vcpu)) {
2892                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
2893                         atomic_or(CPUSTAT_IBS,
2894                                         &vcpu->arch.sie_block->cpuflags);
2895                 }
2896                 goto retry;
2897         }
2898
2899         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2900                 if (ibs_enabled(vcpu)) {
2901                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
2902                         atomic_andnot(CPUSTAT_IBS,
2903                                           &vcpu->arch.sie_block->cpuflags);
2904                 }
2905                 goto retry;
2906         }
2907
2908         if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
2909                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2910                 goto retry;
2911         }
2912
2913         if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
2914                 /*
2915                  * Disable CMMA virtualization; we will emulate the ESSA
2916                  * instruction manually, in order to provide additional
2917                  * functionalities needed for live migration.
2918                  */
2919                 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
2920                 goto retry;
2921         }
2922
2923         if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
2924                 /*
2925                  * Re-enable CMMA virtualization if CMMA is available and
2926                  * was used.
2927                  */
2928                 if ((vcpu->kvm->arch.use_cmma) &&
2929                     (vcpu->kvm->mm->context.use_cmma))
2930                         vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
2931                 goto retry;
2932         }
2933
2934         /* nothing to do, just clear the request */
2935         kvm_clear_request(KVM_REQ_UNHALT, vcpu);
2936
2937         return 0;
2938 }
2939
2940 void kvm_s390_set_tod_clock_ext(struct kvm *kvm,
2941                                  const struct kvm_s390_vm_tod_clock *gtod)
2942 {
2943         struct kvm_vcpu *vcpu;
2944         struct kvm_s390_tod_clock_ext htod;
2945         int i;
2946
2947         mutex_lock(&kvm->lock);
2948         preempt_disable();
2949
2950         get_tod_clock_ext((char *)&htod);
2951
2952         kvm->arch.epoch = gtod->tod - htod.tod;
2953         kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
2954
2955         if (kvm->arch.epoch > gtod->tod)
2956                 kvm->arch.epdx -= 1;
2957
2958         kvm_s390_vcpu_block_all(kvm);
2959         kvm_for_each_vcpu(i, vcpu, kvm) {
2960                 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2961                 vcpu->arch.sie_block->epdx  = kvm->arch.epdx;
2962         }
2963
2964         kvm_s390_vcpu_unblock_all(kvm);
2965         preempt_enable();
2966         mutex_unlock(&kvm->lock);
2967 }
2968
2969 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2970 {
2971         struct kvm_vcpu *vcpu;
2972         int i;
2973
2974         mutex_lock(&kvm->lock);
2975         preempt_disable();
2976         kvm->arch.epoch = tod - get_tod_clock();
2977         kvm_s390_vcpu_block_all(kvm);
2978         kvm_for_each_vcpu(i, vcpu, kvm)
2979                 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2980         kvm_s390_vcpu_unblock_all(kvm);
2981         preempt_enable();
2982         mutex_unlock(&kvm->lock);
2983 }
2984
2985 /**
2986  * kvm_arch_fault_in_page - fault-in guest page if necessary
2987  * @vcpu: The corresponding virtual cpu
2988  * @gpa: Guest physical address
2989  * @writable: Whether the page should be writable or not
2990  *
2991  * Make sure that a guest page has been faulted-in on the host.
2992  *
2993  * Return: Zero on success, negative error code otherwise.
2994  */
2995 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
2996 {
2997         return gmap_fault(vcpu->arch.gmap, gpa,
2998                           writable ? FAULT_FLAG_WRITE : 0);
2999 }
3000
3001 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3002                                       unsigned long token)
3003 {
3004         struct kvm_s390_interrupt inti;
3005         struct kvm_s390_irq irq;
3006
3007         if (start_token) {
3008                 irq.u.ext.ext_params2 = token;
3009                 irq.type = KVM_S390_INT_PFAULT_INIT;
3010                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3011         } else {
3012                 inti.type = KVM_S390_INT_PFAULT_DONE;
3013                 inti.parm64 = token;
3014                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3015         }
3016 }
3017
3018 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3019                                      struct kvm_async_pf *work)
3020 {
3021         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3022         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
3023 }
3024
3025 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
3026                                  struct kvm_async_pf *work)
3027 {
3028         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
3029         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
3030 }
3031
3032 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
3033                                struct kvm_async_pf *work)
3034 {
3035         /* s390 will always inject the page directly */
3036 }
3037
3038 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
3039 {
3040         /*
3041          * s390 will always inject the page directly,
3042          * but we still want check_async_completion to cleanup
3043          */
3044         return true;
3045 }
3046
3047 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3048 {
3049         hva_t hva;
3050         struct kvm_arch_async_pf arch;
3051         int rc;
3052
3053         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3054                 return 0;
3055         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3056             vcpu->arch.pfault_compare)
3057                 return 0;
3058         if (psw_extint_disabled(vcpu))
3059                 return 0;
3060         if (kvm_s390_vcpu_has_irq(vcpu, 0))
3061                 return 0;
3062         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
3063                 return 0;
3064         if (!vcpu->arch.gmap->pfault_enabled)
3065                 return 0;
3066
3067         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3068         hva += current->thread.gmap_addr & ~PAGE_MASK;
3069         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3070                 return 0;
3071
3072         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3073         return rc;
3074 }
3075
3076 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
3077 {
3078         int rc, cpuflags;
3079
3080         /*
3081          * On s390 notifications for arriving pages will be delivered directly
3082          * to the guest but the house keeping for completed pfaults is
3083          * handled outside the worker.
3084          */
3085         kvm_check_async_pf_completion(vcpu);
3086
3087         vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
3088         vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
3089
3090         if (need_resched())
3091                 schedule();
3092
3093         if (test_cpu_flag(CIF_MCCK_PENDING))
3094                 s390_handle_mcck();
3095
3096         if (!kvm_is_ucontrol(vcpu->kvm)) {
3097                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
3098                 if (rc)
3099                         return rc;
3100         }
3101
3102         rc = kvm_s390_handle_requests(vcpu);
3103         if (rc)
3104                 return rc;
3105
3106         if (guestdbg_enabled(vcpu)) {
3107                 kvm_s390_backup_guest_per_regs(vcpu);
3108                 kvm_s390_patch_guest_per_regs(vcpu);
3109         }
3110
3111         vcpu->arch.sie_block->icptcode = 0;
3112         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
3113         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
3114         trace_kvm_s390_sie_enter(vcpu, cpuflags);
3115
3116         return 0;
3117 }
3118
3119 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
3120 {
3121         struct kvm_s390_pgm_info pgm_info = {
3122                 .code = PGM_ADDRESSING,
3123         };
3124         u8 opcode, ilen;
3125         int rc;
3126
3127         VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
3128         trace_kvm_s390_sie_fault(vcpu);
3129
3130         /*
3131          * We want to inject an addressing exception, which is defined as a
3132          * suppressing or terminating exception. However, since we came here
3133          * by a DAT access exception, the PSW still points to the faulting
3134          * instruction since DAT exceptions are nullifying. So we've got
3135          * to look up the current opcode to get the length of the instruction
3136          * to be able to forward the PSW.
3137          */
3138         rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
3139         ilen = insn_length(opcode);
3140         if (rc < 0) {
3141                 return rc;
3142         } else if (rc) {
3143                 /* Instruction-Fetching Exceptions - we can't detect the ilen.
3144                  * Forward by arbitrary ilc, injection will take care of
3145                  * nullification if necessary.
3146                  */
3147                 pgm_info = vcpu->arch.pgm;
3148                 ilen = 4;
3149         }
3150         pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
3151         kvm_s390_forward_psw(vcpu, ilen);
3152         return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
3153 }
3154
3155 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
3156 {
3157         struct mcck_volatile_info *mcck_info;
3158         struct sie_page *sie_page;
3159
3160         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
3161                    vcpu->arch.sie_block->icptcode);
3162         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
3163
3164         if (guestdbg_enabled(vcpu))
3165                 kvm_s390_restore_guest_per_regs(vcpu);
3166
3167         vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
3168         vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
3169
3170         if (exit_reason == -EINTR) {
3171                 VCPU_EVENT(vcpu, 3, "%s", "machine check");
3172                 sie_page = container_of(vcpu->arch.sie_block,
3173                                         struct sie_page, sie_block);
3174                 mcck_info = &sie_page->mcck_info;
3175                 kvm_s390_reinject_machine_check(vcpu, mcck_info);
3176                 return 0;
3177         }
3178
3179         if (vcpu->arch.sie_block->icptcode > 0) {
3180                 int rc = kvm_handle_sie_intercept(vcpu);
3181
3182                 if (rc != -EOPNOTSUPP)
3183                         return rc;
3184                 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
3185                 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
3186                 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
3187                 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
3188                 return -EREMOTE;
3189         } else if (exit_reason != -EFAULT) {
3190                 vcpu->stat.exit_null++;
3191                 return 0;
3192         } else if (kvm_is_ucontrol(vcpu->kvm)) {
3193                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
3194                 vcpu->run->s390_ucontrol.trans_exc_code =
3195                                                 current->thread.gmap_addr;
3196                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
3197                 return -EREMOTE;
3198         } else if (current->thread.gmap_pfault) {
3199                 trace_kvm_s390_major_guest_pfault(vcpu);
3200                 current->thread.gmap_pfault = 0;
3201                 if (kvm_arch_setup_async_pf(vcpu))
3202                         return 0;
3203                 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
3204         }
3205         return vcpu_post_run_fault_in_sie(vcpu);
3206 }
3207
3208 static int __vcpu_run(struct kvm_vcpu *vcpu)
3209 {
3210         int rc, exit_reason;
3211
3212         /*
3213          * We try to hold kvm->srcu during most of vcpu_run (except when run-
3214          * ning the guest), so that memslots (and other stuff) are protected
3215          */
3216         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3217
3218         do {
3219                 rc = vcpu_pre_run(vcpu);
3220                 if (rc)
3221                         break;
3222
3223                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3224                 /*
3225                  * As PF_VCPU will be used in fault handler, between
3226                  * guest_enter and guest_exit should be no uaccess.
3227                  */
3228                 local_irq_disable();
3229                 guest_enter_irqoff();
3230                 __disable_cpu_timer_accounting(vcpu);
3231                 local_irq_enable();
3232                 exit_reason = sie64a(vcpu->arch.sie_block,
3233                                      vcpu->run->s.regs.gprs);
3234                 local_irq_disable();
3235                 __enable_cpu_timer_accounting(vcpu);
3236                 guest_exit_irqoff();
3237                 local_irq_enable();
3238                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3239
3240                 rc = vcpu_post_run(vcpu, exit_reason);
3241         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3242
3243         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3244         return rc;
3245 }
3246
3247 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3248 {
3249         struct runtime_instr_cb *riccb;
3250         struct gs_cb *gscb;
3251
3252         riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
3253         gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
3254         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
3255         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
3256         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
3257                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
3258         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
3259                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
3260                 /* some control register changes require a tlb flush */
3261                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3262         }
3263         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
3264                 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
3265                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
3266                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
3267                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
3268                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
3269         }
3270         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
3271                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
3272                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
3273                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
3274                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3275                         kvm_clear_async_pf_completion_queue(vcpu);
3276         }
3277         /*
3278          * If userspace sets the riccb (e.g. after migration) to a valid state,
3279          * we should enable RI here instead of doing the lazy enablement.
3280          */
3281         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
3282             test_kvm_facility(vcpu->kvm, 64) &&
3283             riccb->v &&
3284             !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
3285                 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
3286                 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
3287         }
3288         /*
3289          * If userspace sets the gscb (e.g. after migration) to non-zero,
3290          * we should enable GS here instead of doing the lazy enablement.
3291          */
3292         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
3293             test_kvm_facility(vcpu->kvm, 133) &&
3294             gscb->gssm &&
3295             !vcpu->arch.gs_enabled) {
3296                 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
3297                 vcpu->arch.sie_block->ecb |= ECB_GS;
3298                 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3299                 vcpu->arch.gs_enabled = 1;
3300         }
3301         save_access_regs(vcpu->arch.host_acrs);
3302         restore_access_regs(vcpu->run->s.regs.acrs);
3303         /* save host (userspace) fprs/vrs */
3304         save_fpu_regs();
3305         vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
3306         vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
3307         if (MACHINE_HAS_VX)
3308                 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
3309         else
3310                 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
3311         current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
3312         if (test_fp_ctl(current->thread.fpu.fpc))
3313                 /* User space provided an invalid FPC, let's clear it */
3314                 current->thread.fpu.fpc = 0;
3315         if (MACHINE_HAS_GS) {
3316                 preempt_disable();
3317                 __ctl_set_bit(2, 4);
3318                 if (current->thread.gs_cb) {
3319                         vcpu->arch.host_gscb = current->thread.gs_cb;
3320                         save_gs_cb(vcpu->arch.host_gscb);
3321                 }
3322                 if (vcpu->arch.gs_enabled) {
3323                         current->thread.gs_cb = (struct gs_cb *)
3324                                                 &vcpu->run->s.regs.gscb;
3325                         restore_gs_cb(current->thread.gs_cb);
3326                 }
3327                 preempt_enable();
3328         }
3329
3330         kvm_run->kvm_dirty_regs = 0;
3331 }
3332
3333 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3334 {
3335         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
3336         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
3337         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
3338         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
3339         kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
3340         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
3341         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
3342         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
3343         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
3344         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
3345         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
3346         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
3347         save_access_regs(vcpu->run->s.regs.acrs);
3348         restore_access_regs(vcpu->arch.host_acrs);
3349         /* Save guest register state */
3350         save_fpu_regs();
3351         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3352         /* Restore will be done lazily at return */
3353         current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
3354         current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
3355         if (MACHINE_HAS_GS) {
3356                 __ctl_set_bit(2, 4);
3357                 if (vcpu->arch.gs_enabled)
3358                         save_gs_cb(current->thread.gs_cb);
3359                 preempt_disable();
3360                 current->thread.gs_cb = vcpu->arch.host_gscb;
3361                 restore_gs_cb(vcpu->arch.host_gscb);
3362                 preempt_enable();
3363                 if (!vcpu->arch.host_gscb)
3364                         __ctl_clear_bit(2, 4);
3365                 vcpu->arch.host_gscb = NULL;
3366         }
3367
3368 }
3369
3370 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3371 {
3372         int rc;
3373
3374         if (kvm_run->immediate_exit)
3375                 return -EINTR;
3376
3377         if (guestdbg_exit_pending(vcpu)) {
3378                 kvm_s390_prepare_debug_exit(vcpu);
3379                 return 0;
3380         }
3381
3382         kvm_sigset_activate(vcpu);
3383
3384         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
3385                 kvm_s390_vcpu_start(vcpu);
3386         } else if (is_vcpu_stopped(vcpu)) {
3387                 pr_err_ratelimited("can't run stopped vcpu %d\n",
3388                                    vcpu->vcpu_id);
3389                 return -EINVAL;
3390         }
3391
3392         sync_regs(vcpu, kvm_run);
3393         enable_cpu_timer_accounting(vcpu);
3394
3395         might_fault();
3396         rc = __vcpu_run(vcpu);
3397
3398         if (signal_pending(current) && !rc) {
3399                 kvm_run->exit_reason = KVM_EXIT_INTR;
3400                 rc = -EINTR;
3401         }
3402
3403         if (guestdbg_exit_pending(vcpu) && !rc)  {
3404                 kvm_s390_prepare_debug_exit(vcpu);
3405                 rc = 0;
3406         }
3407
3408         if (rc == -EREMOTE) {
3409                 /* userspace support is needed, kvm_run has been prepared */
3410                 rc = 0;
3411         }
3412
3413         disable_cpu_timer_accounting(vcpu);
3414         store_regs(vcpu, kvm_run);
3415
3416         kvm_sigset_deactivate(vcpu);
3417
3418         vcpu->stat.exit_userspace++;
3419         return rc;
3420 }
3421
3422 /*
3423  * store status at address
3424  * we use have two special cases:
3425  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
3426  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
3427  */
3428 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
3429 {
3430         unsigned char archmode = 1;
3431         freg_t fprs[NUM_FPRS];
3432         unsigned int px;
3433         u64 clkcomp, cputm;
3434         int rc;
3435
3436         px = kvm_s390_get_prefix(vcpu);
3437         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
3438                 if (write_guest_abs(vcpu, 163, &archmode, 1))
3439                         return -EFAULT;
3440                 gpa = 0;
3441         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
3442                 if (write_guest_real(vcpu, 163, &archmode, 1))
3443                         return -EFAULT;
3444                 gpa = px;
3445         } else
3446                 gpa -= __LC_FPREGS_SAVE_AREA;
3447
3448         /* manually convert vector registers if necessary */
3449         if (MACHINE_HAS_VX) {
3450                 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
3451                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3452                                      fprs, 128);
3453         } else {
3454                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3455                                      vcpu->run->s.regs.fprs, 128);
3456         }
3457         rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
3458                               vcpu->run->s.regs.gprs, 128);
3459         rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
3460                               &vcpu->arch.sie_block->gpsw, 16);
3461         rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
3462                               &px, 4);
3463         rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
3464                               &vcpu->run->s.regs.fpc, 4);
3465         rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
3466                               &vcpu->arch.sie_block->todpr, 4);
3467         cputm = kvm_s390_get_cpu_timer(vcpu);
3468         rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
3469                               &cputm, 8);
3470         clkcomp = vcpu->arch.sie_block->ckc >> 8;
3471         rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
3472                               &clkcomp, 8);
3473         rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
3474                               &vcpu->run->s.regs.acrs, 64);
3475         rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
3476                               &vcpu->arch.sie_block->gcr, 128);
3477         return rc ? -EFAULT : 0;
3478 }
3479
3480 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
3481 {
3482         /*
3483          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
3484          * switch in the run ioctl. Let's update our copies before we save
3485          * it into the save area
3486          */
3487         save_fpu_regs();
3488         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3489         save_access_regs(vcpu->run->s.regs.acrs);
3490
3491         return kvm_s390_store_status_unloaded(vcpu, addr);
3492 }
3493
3494 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3495 {
3496         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
3497         kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
3498 }
3499
3500 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
3501 {
3502         unsigned int i;
3503         struct kvm_vcpu *vcpu;
3504
3505         kvm_for_each_vcpu(i, vcpu, kvm) {
3506                 __disable_ibs_on_vcpu(vcpu);
3507         }
3508 }
3509
3510 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3511 {
3512         if (!sclp.has_ibs)
3513                 return;
3514         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
3515         kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
3516 }
3517
3518 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
3519 {
3520         int i, online_vcpus, started_vcpus = 0;
3521
3522         if (!is_vcpu_stopped(vcpu))
3523                 return;
3524
3525         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
3526         /* Only one cpu at a time may enter/leave the STOPPED state. */
3527         spin_lock(&vcpu->kvm->arch.start_stop_lock);
3528         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3529
3530         for (i = 0; i < online_vcpus; i++) {
3531                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
3532                         started_vcpus++;
3533         }
3534
3535         if (started_vcpus == 0) {
3536                 /* we're the only active VCPU -> speed it up */
3537                 __enable_ibs_on_vcpu(vcpu);
3538         } else if (started_vcpus == 1) {
3539                 /*
3540                  * As we are starting a second VCPU, we have to disable
3541                  * the IBS facility on all VCPUs to remove potentially
3542                  * oustanding ENABLE requests.
3543                  */
3544                 __disable_ibs_on_all_vcpus(vcpu->kvm);
3545         }
3546
3547         atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
3548         /*
3549          * Another VCPU might have used IBS while we were offline.
3550          * Let's play safe and flush the VCPU at startup.
3551          */
3552         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3553         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3554         return;
3555 }
3556
3557 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
3558 {
3559         int i, online_vcpus, started_vcpus = 0;
3560         struct kvm_vcpu *started_vcpu = NULL;
3561
3562         if (is_vcpu_stopped(vcpu))
3563                 return;
3564
3565         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
3566         /* Only one cpu at a time may enter/leave the STOPPED state. */
3567         spin_lock(&vcpu->kvm->arch.start_stop_lock);
3568         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3569
3570         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
3571         kvm_s390_clear_stop_irq(vcpu);
3572
3573         atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
3574         __disable_ibs_on_vcpu(vcpu);
3575
3576         for (i = 0; i < online_vcpus; i++) {
3577                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
3578                         started_vcpus++;
3579                         started_vcpu = vcpu->kvm->vcpus[i];
3580                 }
3581         }
3582
3583         if (started_vcpus == 1) {
3584                 /*
3585                  * As we only have one VCPU left, we want to enable the
3586                  * IBS facility for that VCPU to speed it up.
3587                  */
3588                 __enable_ibs_on_vcpu(started_vcpu);
3589         }
3590
3591         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3592         return;
3593 }
3594
3595 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
3596                                      struct kvm_enable_cap *cap)
3597 {
3598         int r;
3599
3600         if (cap->flags)
3601                 return -EINVAL;
3602
3603         switch (cap->cap) {
3604         case KVM_CAP_S390_CSS_SUPPORT:
3605                 if (!vcpu->kvm->arch.css_support) {
3606                         vcpu->kvm->arch.css_support = 1;
3607                         VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
3608                         trace_kvm_s390_enable_css(vcpu->kvm);
3609                 }
3610                 r = 0;
3611                 break;
3612         default:
3613                 r = -EINVAL;
3614                 break;
3615         }
3616         return r;
3617 }
3618
3619 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3620                                   struct kvm_s390_mem_op *mop)
3621 {
3622         void __user *uaddr = (void __user *)mop->buf;
3623         void *tmpbuf = NULL;
3624         int r, srcu_idx;
3625         const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3626                                     | KVM_S390_MEMOP_F_CHECK_ONLY;
3627
3628         if (mop->flags & ~supported_flags)
3629                 return -EINVAL;
3630
3631         if (mop->size > MEM_OP_MAX_SIZE)
3632                 return -E2BIG;
3633
3634         if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3635                 tmpbuf = vmalloc(mop->size);
3636                 if (!tmpbuf)
3637                         return -ENOMEM;
3638         }
3639
3640         srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3641
3642         switch (mop->op) {
3643         case KVM_S390_MEMOP_LOGICAL_READ:
3644                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3645                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3646                                             mop->size, GACC_FETCH);
3647                         break;
3648                 }
3649                 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3650                 if (r == 0) {
3651                         if (copy_to_user(uaddr, tmpbuf, mop->size))
3652                                 r = -EFAULT;
3653                 }
3654                 break;
3655         case KVM_S390_MEMOP_LOGICAL_WRITE:
3656                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3657                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3658                                             mop->size, GACC_STORE);
3659                         break;
3660                 }
3661                 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3662                         r = -EFAULT;
3663                         break;
3664                 }
3665                 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3666                 break;
3667         default:
3668                 r = -EINVAL;
3669         }
3670
3671         srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3672
3673         if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3674                 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3675
3676         vfree(tmpbuf);
3677         return r;
3678 }
3679
3680 long kvm_arch_vcpu_ioctl(struct file *filp,
3681                          unsigned int ioctl, unsigned long arg)
3682 {
3683         struct kvm_vcpu *vcpu = filp->private_data;
3684         void __user *argp = (void __user *)arg;
3685         int idx;
3686         long r;
3687
3688         switch (ioctl) {
3689         case KVM_S390_IRQ: {
3690                 struct kvm_s390_irq s390irq;
3691
3692                 r = -EFAULT;
3693                 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3694                         break;
3695                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3696                 break;
3697         }
3698         case KVM_S390_INTERRUPT: {
3699                 struct kvm_s390_interrupt s390int;
3700                 struct kvm_s390_irq s390irq;
3701
3702                 r = -EFAULT;
3703                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
3704                         break;
3705                 if (s390int_to_s390irq(&s390int, &s390irq))
3706                         return -EINVAL;
3707                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3708                 break;
3709         }
3710         case KVM_S390_STORE_STATUS:
3711                 idx = srcu_read_lock(&vcpu->kvm->srcu);
3712                 r = kvm_s390_vcpu_store_status(vcpu, arg);
3713                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
3714                 break;
3715         case KVM_S390_SET_INITIAL_PSW: {
3716                 psw_t psw;
3717
3718                 r = -EFAULT;
3719                 if (copy_from_user(&psw, argp, sizeof(psw)))
3720                         break;
3721                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3722                 break;
3723         }
3724         case KVM_S390_INITIAL_RESET:
3725                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3726                 break;
3727         case KVM_SET_ONE_REG:
3728         case KVM_GET_ONE_REG: {
3729                 struct kvm_one_reg reg;
3730                 r = -EFAULT;
3731                 if (copy_from_user(&reg, argp, sizeof(reg)))
3732                         break;
3733                 if (ioctl == KVM_SET_ONE_REG)
3734                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3735                 else
3736                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3737                 break;
3738         }
3739 #ifdef CONFIG_KVM_S390_UCONTROL
3740         case KVM_S390_UCAS_MAP: {
3741                 struct kvm_s390_ucas_mapping ucasmap;
3742
3743                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3744                         r = -EFAULT;
3745                         break;
3746                 }
3747
3748                 if (!kvm_is_ucontrol(vcpu->kvm)) {
3749                         r = -EINVAL;
3750                         break;
3751                 }
3752
3753                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3754                                      ucasmap.vcpu_addr, ucasmap.length);
3755                 break;
3756         }
3757         case KVM_S390_UCAS_UNMAP: {
3758                 struct kvm_s390_ucas_mapping ucasmap;
3759
3760                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3761                         r = -EFAULT;
3762                         break;
3763                 }
3764
3765                 if (!kvm_is_ucontrol(vcpu->kvm)) {
3766                         r = -EINVAL;
3767                         break;
3768                 }
3769
3770                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3771                         ucasmap.length);
3772                 break;
3773         }
3774 #endif
3775         case KVM_S390_VCPU_FAULT: {
3776                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
3777                 break;
3778         }
3779         case KVM_ENABLE_CAP:
3780         {
3781                 struct kvm_enable_cap cap;
3782                 r = -EFAULT;
3783                 if (copy_from_user(&cap, argp, sizeof(cap)))
3784                         break;
3785                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3786                 break;
3787         }
3788         case KVM_S390_MEM_OP: {
3789                 struct kvm_s390_mem_op mem_op;
3790
3791                 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3792                         r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3793                 else
3794                         r = -EFAULT;
3795                 break;
3796         }
3797         case KVM_S390_SET_IRQ_STATE: {
3798                 struct kvm_s390_irq_state irq_state;
3799
3800                 r = -EFAULT;
3801                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3802                         break;
3803                 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3804                     irq_state.len == 0 ||
3805                     irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3806                         r = -EINVAL;
3807                         break;
3808                 }
3809                 /* do not use irq_state.flags, it will break old QEMUs */
3810                 r = kvm_s390_set_irq_state(vcpu,
3811                                            (void __user *) irq_state.buf,
3812                                            irq_state.len);
3813                 break;
3814         }
3815         case KVM_S390_GET_IRQ_STATE: {
3816                 struct kvm_s390_irq_state irq_state;
3817
3818                 r = -EFAULT;
3819                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3820                         break;
3821                 if (irq_state.len == 0) {
3822                         r = -EINVAL;
3823                         break;
3824                 }
3825                 /* do not use irq_state.flags, it will break old QEMUs */
3826                 r = kvm_s390_get_irq_state(vcpu,
3827                                            (__u8 __user *)  irq_state.buf,
3828                                            irq_state.len);
3829                 break;
3830         }
3831         default:
3832                 r = -ENOTTY;
3833         }
3834         return r;
3835 }
3836
3837 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3838 {
3839 #ifdef CONFIG_KVM_S390_UCONTROL
3840         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3841                  && (kvm_is_ucontrol(vcpu->kvm))) {
3842                 vmf->page = virt_to_page(vcpu->arch.sie_block);
3843                 get_page(vmf->page);
3844                 return 0;
3845         }
3846 #endif
3847         return VM_FAULT_SIGBUS;
3848 }
3849
3850 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3851                             unsigned long npages)
3852 {
3853         return 0;
3854 }
3855
3856 /* Section: memory related */
3857 int kvm_arch_prepare_memory_region(struct kvm *kvm,
3858                                    struct kvm_memory_slot *memslot,
3859                                    const struct kvm_userspace_memory_region *mem,
3860                                    enum kvm_mr_change change)
3861 {
3862         /* A few sanity checks. We can have memory slots which have to be
3863            located/ended at a segment boundary (1MB). The memory in userland is
3864            ok to be fragmented into various different vmas. It is okay to mmap()
3865            and munmap() stuff in this slot after doing this call at any time */
3866
3867         if (mem->userspace_addr & 0xffffful)
3868                 return -EINVAL;
3869
3870         if (mem->memory_size & 0xffffful)
3871                 return -EINVAL;
3872
3873         if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3874                 return -EINVAL;
3875
3876         return 0;
3877 }
3878
3879 void kvm_arch_commit_memory_region(struct kvm *kvm,
3880                                 const struct kvm_userspace_memory_region *mem,
3881                                 const struct kvm_memory_slot *old,
3882                                 const struct kvm_memory_slot *new,
3883                                 enum kvm_mr_change change)
3884 {
3885         int rc;
3886
3887         /* If the basics of the memslot do not change, we do not want
3888          * to update the gmap. Every update causes several unnecessary
3889          * segment translation exceptions. This is usually handled just
3890          * fine by the normal fault handler + gmap, but it will also
3891          * cause faults on the prefix page of running guest CPUs.
3892          */
3893         if (old->userspace_addr == mem->userspace_addr &&
3894             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3895             old->npages * PAGE_SIZE == mem->memory_size)
3896                 return;
3897
3898         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3899                 mem->guest_phys_addr, mem->memory_size);
3900         if (rc)
3901                 pr_warn("failed to commit memory region\n");
3902         return;
3903 }
3904
3905 static inline unsigned long nonhyp_mask(int i)
3906 {
3907         unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3908
3909         return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3910 }
3911
3912 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3913 {
3914         vcpu->valid_wakeup = false;
3915 }
3916
3917 static int __init kvm_s390_init(void)
3918 {
3919         int i;
3920
3921         if (!sclp.has_sief2) {
3922                 pr_info("SIE not available\n");
3923                 return -ENODEV;
3924         }
3925
3926         for (i = 0; i < 16; i++)
3927                 kvm_s390_fac_list_mask[i] |=
3928                         S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3929
3930         return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
3931 }
3932
3933 static void __exit kvm_s390_exit(void)
3934 {
3935         kvm_exit();
3936 }
3937
3938 module_init(kvm_s390_init);
3939 module_exit(kvm_s390_exit);
3940
3941 /*
3942  * Enable autoloading of the kvm module.
3943  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3944  * since x86 takes a different approach.
3945  */
3946 #include <linux/miscdevice.h>
3947 MODULE_ALIAS_MISCDEV(KVM_MINOR);
3948 MODULE_ALIAS("devname:kvm");