Merge tag 'kvm-s390-next-5.6-1' of git://git.kernel.org/pub/scm/linux/kernel/git...
[sfrench/cifs-2.6.git] / arch / s390 / kvm / kvm-s390.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * hosting IBM Z kernel virtual machines (s390x)
4  *
5  * Copyright IBM Corp. 2008, 2018
6  *
7  *    Author(s): Carsten Otte <cotte@de.ibm.com>
8  *               Christian Borntraeger <borntraeger@de.ibm.com>
9  *               Heiko Carstens <heiko.carstens@de.ibm.com>
10  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
11  *               Jason J. Herne <jjherne@us.ibm.com>
12  */
13
14 #define KMSG_COMPONENT "kvm-s390"
15 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
16
17 #include <linux/compiler.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/mman.h>
25 #include <linux/module.h>
26 #include <linux/moduleparam.h>
27 #include <linux/random.h>
28 #include <linux/slab.h>
29 #include <linux/timer.h>
30 #include <linux/vmalloc.h>
31 #include <linux/bitmap.h>
32 #include <linux/sched/signal.h>
33 #include <linux/string.h>
34
35 #include <asm/asm-offsets.h>
36 #include <asm/lowcore.h>
37 #include <asm/stp.h>
38 #include <asm/pgtable.h>
39 #include <asm/gmap.h>
40 #include <asm/nmi.h>
41 #include <asm/switch_to.h>
42 #include <asm/isc.h>
43 #include <asm/sclp.h>
44 #include <asm/cpacf.h>
45 #include <asm/timex.h>
46 #include <asm/ap.h>
47 #include "kvm-s390.h"
48 #include "gaccess.h"
49
50 #define CREATE_TRACE_POINTS
51 #include "trace.h"
52 #include "trace-s390.h"
53
54 #define MEM_OP_MAX_SIZE 65536   /* Maximum transfer size for KVM_S390_MEM_OP */
55 #define LOCAL_IRQS 32
56 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
57                            (KVM_MAX_VCPUS + LOCAL_IRQS))
58
59 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
60 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
61
62 struct kvm_stats_debugfs_item debugfs_entries[] = {
63         { "userspace_handled", VCPU_STAT(exit_userspace) },
64         { "exit_null", VCPU_STAT(exit_null) },
65         { "exit_validity", VCPU_STAT(exit_validity) },
66         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
67         { "exit_external_request", VCPU_STAT(exit_external_request) },
68         { "exit_io_request", VCPU_STAT(exit_io_request) },
69         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
70         { "exit_instruction", VCPU_STAT(exit_instruction) },
71         { "exit_pei", VCPU_STAT(exit_pei) },
72         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
73         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
74         { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
75         { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
76         { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
77         { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
78         { "halt_no_poll_steal", VCPU_STAT(halt_no_poll_steal) },
79         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
80         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
81         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
82         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
83         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
84         { "deliver_ckc", VCPU_STAT(deliver_ckc) },
85         { "deliver_cputm", VCPU_STAT(deliver_cputm) },
86         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
87         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
88         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
89         { "deliver_virtio", VCPU_STAT(deliver_virtio) },
90         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
91         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
92         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
93         { "deliver_program", VCPU_STAT(deliver_program) },
94         { "deliver_io", VCPU_STAT(deliver_io) },
95         { "deliver_machine_check", VCPU_STAT(deliver_machine_check) },
96         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
97         { "inject_ckc", VCPU_STAT(inject_ckc) },
98         { "inject_cputm", VCPU_STAT(inject_cputm) },
99         { "inject_external_call", VCPU_STAT(inject_external_call) },
100         { "inject_float_mchk", VM_STAT(inject_float_mchk) },
101         { "inject_emergency_signal", VCPU_STAT(inject_emergency_signal) },
102         { "inject_io", VM_STAT(inject_io) },
103         { "inject_mchk", VCPU_STAT(inject_mchk) },
104         { "inject_pfault_done", VM_STAT(inject_pfault_done) },
105         { "inject_program", VCPU_STAT(inject_program) },
106         { "inject_restart", VCPU_STAT(inject_restart) },
107         { "inject_service_signal", VM_STAT(inject_service_signal) },
108         { "inject_set_prefix", VCPU_STAT(inject_set_prefix) },
109         { "inject_stop_signal", VCPU_STAT(inject_stop_signal) },
110         { "inject_pfault_init", VCPU_STAT(inject_pfault_init) },
111         { "inject_virtio", VM_STAT(inject_virtio) },
112         { "instruction_epsw", VCPU_STAT(instruction_epsw) },
113         { "instruction_gs", VCPU_STAT(instruction_gs) },
114         { "instruction_io_other", VCPU_STAT(instruction_io_other) },
115         { "instruction_lpsw", VCPU_STAT(instruction_lpsw) },
116         { "instruction_lpswe", VCPU_STAT(instruction_lpswe) },
117         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
118         { "instruction_ptff", VCPU_STAT(instruction_ptff) },
119         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
120         { "instruction_sck", VCPU_STAT(instruction_sck) },
121         { "instruction_sckpf", VCPU_STAT(instruction_sckpf) },
122         { "instruction_spx", VCPU_STAT(instruction_spx) },
123         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
124         { "instruction_stap", VCPU_STAT(instruction_stap) },
125         { "instruction_iske", VCPU_STAT(instruction_iske) },
126         { "instruction_ri", VCPU_STAT(instruction_ri) },
127         { "instruction_rrbe", VCPU_STAT(instruction_rrbe) },
128         { "instruction_sske", VCPU_STAT(instruction_sske) },
129         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
130         { "instruction_essa", VCPU_STAT(instruction_essa) },
131         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
132         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
133         { "instruction_tb", VCPU_STAT(instruction_tb) },
134         { "instruction_tpi", VCPU_STAT(instruction_tpi) },
135         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
136         { "instruction_tsch", VCPU_STAT(instruction_tsch) },
137         { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
138         { "instruction_sie", VCPU_STAT(instruction_sie) },
139         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
140         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
141         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
142         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
143         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
144         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
145         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
146         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
147         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
148         { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
149         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
150         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
151         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
152         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
153         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
154         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
155         { "instruction_diag_10", VCPU_STAT(diagnose_10) },
156         { "instruction_diag_44", VCPU_STAT(diagnose_44) },
157         { "instruction_diag_9c", VCPU_STAT(diagnose_9c) },
158         { "diag_9c_ignored", VCPU_STAT(diagnose_9c_ignored) },
159         { "instruction_diag_258", VCPU_STAT(diagnose_258) },
160         { "instruction_diag_308", VCPU_STAT(diagnose_308) },
161         { "instruction_diag_500", VCPU_STAT(diagnose_500) },
162         { "instruction_diag_other", VCPU_STAT(diagnose_other) },
163         { NULL }
164 };
165
166 struct kvm_s390_tod_clock_ext {
167         __u8 epoch_idx;
168         __u64 tod;
169         __u8 reserved[7];
170 } __packed;
171
172 /* allow nested virtualization in KVM (if enabled by user space) */
173 static int nested;
174 module_param(nested, int, S_IRUGO);
175 MODULE_PARM_DESC(nested, "Nested virtualization support");
176
177 /* allow 1m huge page guest backing, if !nested */
178 static int hpage;
179 module_param(hpage, int, 0444);
180 MODULE_PARM_DESC(hpage, "1m huge page backing support");
181
182 /* maximum percentage of steal time for polling.  >100 is treated like 100 */
183 static u8 halt_poll_max_steal = 10;
184 module_param(halt_poll_max_steal, byte, 0644);
185 MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
186
187 /*
188  * For now we handle at most 16 double words as this is what the s390 base
189  * kernel handles and stores in the prefix page. If we ever need to go beyond
190  * this, this requires changes to code, but the external uapi can stay.
191  */
192 #define SIZE_INTERNAL 16
193
194 /*
195  * Base feature mask that defines default mask for facilities. Consists of the
196  * defines in FACILITIES_KVM and the non-hypervisor managed bits.
197  */
198 static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
199 /*
200  * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
201  * and defines the facilities that can be enabled via a cpu model.
202  */
203 static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
204
205 static unsigned long kvm_s390_fac_size(void)
206 {
207         BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
208         BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
209         BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
210                 sizeof(S390_lowcore.stfle_fac_list));
211
212         return SIZE_INTERNAL;
213 }
214
215 /* available cpu features supported by kvm */
216 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
217 /* available subfunctions indicated via query / "test bit" */
218 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
219
220 static struct gmap_notifier gmap_notifier;
221 static struct gmap_notifier vsie_gmap_notifier;
222 debug_info_t *kvm_s390_dbf;
223
224 /* Section: not file related */
225 int kvm_arch_hardware_enable(void)
226 {
227         /* every s390 is virtualization enabled ;-) */
228         return 0;
229 }
230
231 int kvm_arch_check_processor_compat(void)
232 {
233         return 0;
234 }
235
236 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
237                               unsigned long end);
238
239 static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
240 {
241         u8 delta_idx = 0;
242
243         /*
244          * The TOD jumps by delta, we have to compensate this by adding
245          * -delta to the epoch.
246          */
247         delta = -delta;
248
249         /* sign-extension - we're adding to signed values below */
250         if ((s64)delta < 0)
251                 delta_idx = -1;
252
253         scb->epoch += delta;
254         if (scb->ecd & ECD_MEF) {
255                 scb->epdx += delta_idx;
256                 if (scb->epoch < delta)
257                         scb->epdx += 1;
258         }
259 }
260
261 /*
262  * This callback is executed during stop_machine(). All CPUs are therefore
263  * temporarily stopped. In order not to change guest behavior, we have to
264  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
265  * so a CPU won't be stopped while calculating with the epoch.
266  */
267 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
268                           void *v)
269 {
270         struct kvm *kvm;
271         struct kvm_vcpu *vcpu;
272         int i;
273         unsigned long long *delta = v;
274
275         list_for_each_entry(kvm, &vm_list, vm_list) {
276                 kvm_for_each_vcpu(i, vcpu, kvm) {
277                         kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
278                         if (i == 0) {
279                                 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
280                                 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
281                         }
282                         if (vcpu->arch.cputm_enabled)
283                                 vcpu->arch.cputm_start += *delta;
284                         if (vcpu->arch.vsie_block)
285                                 kvm_clock_sync_scb(vcpu->arch.vsie_block,
286                                                    *delta);
287                 }
288         }
289         return NOTIFY_OK;
290 }
291
292 static struct notifier_block kvm_clock_notifier = {
293         .notifier_call = kvm_clock_sync,
294 };
295
296 int kvm_arch_hardware_setup(void)
297 {
298         gmap_notifier.notifier_call = kvm_gmap_notifier;
299         gmap_register_pte_notifier(&gmap_notifier);
300         vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
301         gmap_register_pte_notifier(&vsie_gmap_notifier);
302         atomic_notifier_chain_register(&s390_epoch_delta_notifier,
303                                        &kvm_clock_notifier);
304         return 0;
305 }
306
307 void kvm_arch_hardware_unsetup(void)
308 {
309         gmap_unregister_pte_notifier(&gmap_notifier);
310         gmap_unregister_pte_notifier(&vsie_gmap_notifier);
311         atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
312                                          &kvm_clock_notifier);
313 }
314
315 static void allow_cpu_feat(unsigned long nr)
316 {
317         set_bit_inv(nr, kvm_s390_available_cpu_feat);
318 }
319
320 static inline int plo_test_bit(unsigned char nr)
321 {
322         register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
323         int cc;
324
325         asm volatile(
326                 /* Parameter registers are ignored for "test bit" */
327                 "       plo     0,0,0,0(0)\n"
328                 "       ipm     %0\n"
329                 "       srl     %0,28\n"
330                 : "=d" (cc)
331                 : "d" (r0)
332                 : "cc");
333         return cc == 0;
334 }
335
336 static __always_inline void __insn32_query(unsigned int opcode, u8 *query)
337 {
338         register unsigned long r0 asm("0") = 0; /* query function */
339         register unsigned long r1 asm("1") = (unsigned long) query;
340
341         asm volatile(
342                 /* Parameter regs are ignored */
343                 "       .insn   rrf,%[opc] << 16,2,4,6,0\n"
344                 :
345                 : "d" (r0), "a" (r1), [opc] "i" (opcode)
346                 : "cc", "memory");
347 }
348
349 #define INSN_SORTL 0xb938
350 #define INSN_DFLTCC 0xb939
351
352 static void kvm_s390_cpu_feat_init(void)
353 {
354         int i;
355
356         for (i = 0; i < 256; ++i) {
357                 if (plo_test_bit(i))
358                         kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
359         }
360
361         if (test_facility(28)) /* TOD-clock steering */
362                 ptff(kvm_s390_available_subfunc.ptff,
363                      sizeof(kvm_s390_available_subfunc.ptff),
364                      PTFF_QAF);
365
366         if (test_facility(17)) { /* MSA */
367                 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
368                               kvm_s390_available_subfunc.kmac);
369                 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
370                               kvm_s390_available_subfunc.kmc);
371                 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
372                               kvm_s390_available_subfunc.km);
373                 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
374                               kvm_s390_available_subfunc.kimd);
375                 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
376                               kvm_s390_available_subfunc.klmd);
377         }
378         if (test_facility(76)) /* MSA3 */
379                 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
380                               kvm_s390_available_subfunc.pckmo);
381         if (test_facility(77)) { /* MSA4 */
382                 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
383                               kvm_s390_available_subfunc.kmctr);
384                 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
385                               kvm_s390_available_subfunc.kmf);
386                 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
387                               kvm_s390_available_subfunc.kmo);
388                 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
389                               kvm_s390_available_subfunc.pcc);
390         }
391         if (test_facility(57)) /* MSA5 */
392                 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
393                               kvm_s390_available_subfunc.ppno);
394
395         if (test_facility(146)) /* MSA8 */
396                 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
397                               kvm_s390_available_subfunc.kma);
398
399         if (test_facility(155)) /* MSA9 */
400                 __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
401                               kvm_s390_available_subfunc.kdsa);
402
403         if (test_facility(150)) /* SORTL */
404                 __insn32_query(INSN_SORTL, kvm_s390_available_subfunc.sortl);
405
406         if (test_facility(151)) /* DFLTCC */
407                 __insn32_query(INSN_DFLTCC, kvm_s390_available_subfunc.dfltcc);
408
409         if (MACHINE_HAS_ESOP)
410                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
411         /*
412          * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
413          * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
414          */
415         if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
416             !test_facility(3) || !nested)
417                 return;
418         allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
419         if (sclp.has_64bscao)
420                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
421         if (sclp.has_siif)
422                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
423         if (sclp.has_gpere)
424                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
425         if (sclp.has_gsls)
426                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
427         if (sclp.has_ib)
428                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
429         if (sclp.has_cei)
430                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
431         if (sclp.has_ibs)
432                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
433         if (sclp.has_kss)
434                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
435         /*
436          * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
437          * all skey handling functions read/set the skey from the PGSTE
438          * instead of the real storage key.
439          *
440          * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
441          * pages being detected as preserved although they are resident.
442          *
443          * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
444          * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
445          *
446          * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
447          * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
448          * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
449          *
450          * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
451          * cannot easily shadow the SCA because of the ipte lock.
452          */
453 }
454
455 int kvm_arch_init(void *opaque)
456 {
457         int rc = -ENOMEM;
458
459         kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
460         if (!kvm_s390_dbf)
461                 return -ENOMEM;
462
463         if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view))
464                 goto out;
465
466         kvm_s390_cpu_feat_init();
467
468         /* Register floating interrupt controller interface. */
469         rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
470         if (rc) {
471                 pr_err("A FLIC registration call failed with rc=%d\n", rc);
472                 goto out;
473         }
474
475         rc = kvm_s390_gib_init(GAL_ISC);
476         if (rc)
477                 goto out;
478
479         return 0;
480
481 out:
482         kvm_arch_exit();
483         return rc;
484 }
485
486 void kvm_arch_exit(void)
487 {
488         kvm_s390_gib_destroy();
489         debug_unregister(kvm_s390_dbf);
490 }
491
492 /* Section: device related */
493 long kvm_arch_dev_ioctl(struct file *filp,
494                         unsigned int ioctl, unsigned long arg)
495 {
496         if (ioctl == KVM_S390_ENABLE_SIE)
497                 return s390_enable_sie();
498         return -EINVAL;
499 }
500
501 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
502 {
503         int r;
504
505         switch (ext) {
506         case KVM_CAP_S390_PSW:
507         case KVM_CAP_S390_GMAP:
508         case KVM_CAP_SYNC_MMU:
509 #ifdef CONFIG_KVM_S390_UCONTROL
510         case KVM_CAP_S390_UCONTROL:
511 #endif
512         case KVM_CAP_ASYNC_PF:
513         case KVM_CAP_SYNC_REGS:
514         case KVM_CAP_ONE_REG:
515         case KVM_CAP_ENABLE_CAP:
516         case KVM_CAP_S390_CSS_SUPPORT:
517         case KVM_CAP_IOEVENTFD:
518         case KVM_CAP_DEVICE_CTRL:
519         case KVM_CAP_S390_IRQCHIP:
520         case KVM_CAP_VM_ATTRIBUTES:
521         case KVM_CAP_MP_STATE:
522         case KVM_CAP_IMMEDIATE_EXIT:
523         case KVM_CAP_S390_INJECT_IRQ:
524         case KVM_CAP_S390_USER_SIGP:
525         case KVM_CAP_S390_USER_STSI:
526         case KVM_CAP_S390_SKEYS:
527         case KVM_CAP_S390_IRQ_STATE:
528         case KVM_CAP_S390_USER_INSTR0:
529         case KVM_CAP_S390_CMMA_MIGRATION:
530         case KVM_CAP_S390_AIS:
531         case KVM_CAP_S390_AIS_MIGRATION:
532         case KVM_CAP_S390_VCPU_RESETS:
533                 r = 1;
534                 break;
535         case KVM_CAP_S390_HPAGE_1M:
536                 r = 0;
537                 if (hpage && !kvm_is_ucontrol(kvm))
538                         r = 1;
539                 break;
540         case KVM_CAP_S390_MEM_OP:
541                 r = MEM_OP_MAX_SIZE;
542                 break;
543         case KVM_CAP_NR_VCPUS:
544         case KVM_CAP_MAX_VCPUS:
545         case KVM_CAP_MAX_VCPU_ID:
546                 r = KVM_S390_BSCA_CPU_SLOTS;
547                 if (!kvm_s390_use_sca_entries())
548                         r = KVM_MAX_VCPUS;
549                 else if (sclp.has_esca && sclp.has_64bscao)
550                         r = KVM_S390_ESCA_CPU_SLOTS;
551                 break;
552         case KVM_CAP_S390_COW:
553                 r = MACHINE_HAS_ESOP;
554                 break;
555         case KVM_CAP_S390_VECTOR_REGISTERS:
556                 r = MACHINE_HAS_VX;
557                 break;
558         case KVM_CAP_S390_RI:
559                 r = test_facility(64);
560                 break;
561         case KVM_CAP_S390_GS:
562                 r = test_facility(133);
563                 break;
564         case KVM_CAP_S390_BPB:
565                 r = test_facility(82);
566                 break;
567         default:
568                 r = 0;
569         }
570         return r;
571 }
572
573 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
574                                     struct kvm_memory_slot *memslot)
575 {
576         int i;
577         gfn_t cur_gfn, last_gfn;
578         unsigned long gaddr, vmaddr;
579         struct gmap *gmap = kvm->arch.gmap;
580         DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
581
582         /* Loop over all guest segments */
583         cur_gfn = memslot->base_gfn;
584         last_gfn = memslot->base_gfn + memslot->npages;
585         for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
586                 gaddr = gfn_to_gpa(cur_gfn);
587                 vmaddr = gfn_to_hva_memslot(memslot, cur_gfn);
588                 if (kvm_is_error_hva(vmaddr))
589                         continue;
590
591                 bitmap_zero(bitmap, _PAGE_ENTRIES);
592                 gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
593                 for (i = 0; i < _PAGE_ENTRIES; i++) {
594                         if (test_bit(i, bitmap))
595                                 mark_page_dirty(kvm, cur_gfn + i);
596                 }
597
598                 if (fatal_signal_pending(current))
599                         return;
600                 cond_resched();
601         }
602 }
603
604 /* Section: vm related */
605 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
606
607 /*
608  * Get (and clear) the dirty memory log for a memory slot.
609  */
610 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
611                                struct kvm_dirty_log *log)
612 {
613         int r;
614         unsigned long n;
615         struct kvm_memslots *slots;
616         struct kvm_memory_slot *memslot;
617         int is_dirty = 0;
618
619         if (kvm_is_ucontrol(kvm))
620                 return -EINVAL;
621
622         mutex_lock(&kvm->slots_lock);
623
624         r = -EINVAL;
625         if (log->slot >= KVM_USER_MEM_SLOTS)
626                 goto out;
627
628         slots = kvm_memslots(kvm);
629         memslot = id_to_memslot(slots, log->slot);
630         r = -ENOENT;
631         if (!memslot->dirty_bitmap)
632                 goto out;
633
634         kvm_s390_sync_dirty_log(kvm, memslot);
635         r = kvm_get_dirty_log(kvm, log, &is_dirty);
636         if (r)
637                 goto out;
638
639         /* Clear the dirty log */
640         if (is_dirty) {
641                 n = kvm_dirty_bitmap_bytes(memslot);
642                 memset(memslot->dirty_bitmap, 0, n);
643         }
644         r = 0;
645 out:
646         mutex_unlock(&kvm->slots_lock);
647         return r;
648 }
649
650 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
651 {
652         unsigned int i;
653         struct kvm_vcpu *vcpu;
654
655         kvm_for_each_vcpu(i, vcpu, kvm) {
656                 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
657         }
658 }
659
660 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
661 {
662         int r;
663
664         if (cap->flags)
665                 return -EINVAL;
666
667         switch (cap->cap) {
668         case KVM_CAP_S390_IRQCHIP:
669                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
670                 kvm->arch.use_irqchip = 1;
671                 r = 0;
672                 break;
673         case KVM_CAP_S390_USER_SIGP:
674                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
675                 kvm->arch.user_sigp = 1;
676                 r = 0;
677                 break;
678         case KVM_CAP_S390_VECTOR_REGISTERS:
679                 mutex_lock(&kvm->lock);
680                 if (kvm->created_vcpus) {
681                         r = -EBUSY;
682                 } else if (MACHINE_HAS_VX) {
683                         set_kvm_facility(kvm->arch.model.fac_mask, 129);
684                         set_kvm_facility(kvm->arch.model.fac_list, 129);
685                         if (test_facility(134)) {
686                                 set_kvm_facility(kvm->arch.model.fac_mask, 134);
687                                 set_kvm_facility(kvm->arch.model.fac_list, 134);
688                         }
689                         if (test_facility(135)) {
690                                 set_kvm_facility(kvm->arch.model.fac_mask, 135);
691                                 set_kvm_facility(kvm->arch.model.fac_list, 135);
692                         }
693                         if (test_facility(148)) {
694                                 set_kvm_facility(kvm->arch.model.fac_mask, 148);
695                                 set_kvm_facility(kvm->arch.model.fac_list, 148);
696                         }
697                         if (test_facility(152)) {
698                                 set_kvm_facility(kvm->arch.model.fac_mask, 152);
699                                 set_kvm_facility(kvm->arch.model.fac_list, 152);
700                         }
701                         r = 0;
702                 } else
703                         r = -EINVAL;
704                 mutex_unlock(&kvm->lock);
705                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
706                          r ? "(not available)" : "(success)");
707                 break;
708         case KVM_CAP_S390_RI:
709                 r = -EINVAL;
710                 mutex_lock(&kvm->lock);
711                 if (kvm->created_vcpus) {
712                         r = -EBUSY;
713                 } else if (test_facility(64)) {
714                         set_kvm_facility(kvm->arch.model.fac_mask, 64);
715                         set_kvm_facility(kvm->arch.model.fac_list, 64);
716                         r = 0;
717                 }
718                 mutex_unlock(&kvm->lock);
719                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
720                          r ? "(not available)" : "(success)");
721                 break;
722         case KVM_CAP_S390_AIS:
723                 mutex_lock(&kvm->lock);
724                 if (kvm->created_vcpus) {
725                         r = -EBUSY;
726                 } else {
727                         set_kvm_facility(kvm->arch.model.fac_mask, 72);
728                         set_kvm_facility(kvm->arch.model.fac_list, 72);
729                         r = 0;
730                 }
731                 mutex_unlock(&kvm->lock);
732                 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
733                          r ? "(not available)" : "(success)");
734                 break;
735         case KVM_CAP_S390_GS:
736                 r = -EINVAL;
737                 mutex_lock(&kvm->lock);
738                 if (kvm->created_vcpus) {
739                         r = -EBUSY;
740                 } else if (test_facility(133)) {
741                         set_kvm_facility(kvm->arch.model.fac_mask, 133);
742                         set_kvm_facility(kvm->arch.model.fac_list, 133);
743                         r = 0;
744                 }
745                 mutex_unlock(&kvm->lock);
746                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
747                          r ? "(not available)" : "(success)");
748                 break;
749         case KVM_CAP_S390_HPAGE_1M:
750                 mutex_lock(&kvm->lock);
751                 if (kvm->created_vcpus)
752                         r = -EBUSY;
753                 else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
754                         r = -EINVAL;
755                 else {
756                         r = 0;
757                         down_write(&kvm->mm->mmap_sem);
758                         kvm->mm->context.allow_gmap_hpage_1m = 1;
759                         up_write(&kvm->mm->mmap_sem);
760                         /*
761                          * We might have to create fake 4k page
762                          * tables. To avoid that the hardware works on
763                          * stale PGSTEs, we emulate these instructions.
764                          */
765                         kvm->arch.use_skf = 0;
766                         kvm->arch.use_pfmfi = 0;
767                 }
768                 mutex_unlock(&kvm->lock);
769                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
770                          r ? "(not available)" : "(success)");
771                 break;
772         case KVM_CAP_S390_USER_STSI:
773                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
774                 kvm->arch.user_stsi = 1;
775                 r = 0;
776                 break;
777         case KVM_CAP_S390_USER_INSTR0:
778                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
779                 kvm->arch.user_instr0 = 1;
780                 icpt_operexc_on_all_vcpus(kvm);
781                 r = 0;
782                 break;
783         default:
784                 r = -EINVAL;
785                 break;
786         }
787         return r;
788 }
789
790 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
791 {
792         int ret;
793
794         switch (attr->attr) {
795         case KVM_S390_VM_MEM_LIMIT_SIZE:
796                 ret = 0;
797                 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
798                          kvm->arch.mem_limit);
799                 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
800                         ret = -EFAULT;
801                 break;
802         default:
803                 ret = -ENXIO;
804                 break;
805         }
806         return ret;
807 }
808
809 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
810 {
811         int ret;
812         unsigned int idx;
813         switch (attr->attr) {
814         case KVM_S390_VM_MEM_ENABLE_CMMA:
815                 ret = -ENXIO;
816                 if (!sclp.has_cmma)
817                         break;
818
819                 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
820                 mutex_lock(&kvm->lock);
821                 if (kvm->created_vcpus)
822                         ret = -EBUSY;
823                 else if (kvm->mm->context.allow_gmap_hpage_1m)
824                         ret = -EINVAL;
825                 else {
826                         kvm->arch.use_cmma = 1;
827                         /* Not compatible with cmma. */
828                         kvm->arch.use_pfmfi = 0;
829                         ret = 0;
830                 }
831                 mutex_unlock(&kvm->lock);
832                 break;
833         case KVM_S390_VM_MEM_CLR_CMMA:
834                 ret = -ENXIO;
835                 if (!sclp.has_cmma)
836                         break;
837                 ret = -EINVAL;
838                 if (!kvm->arch.use_cmma)
839                         break;
840
841                 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
842                 mutex_lock(&kvm->lock);
843                 idx = srcu_read_lock(&kvm->srcu);
844                 s390_reset_cmma(kvm->arch.gmap->mm);
845                 srcu_read_unlock(&kvm->srcu, idx);
846                 mutex_unlock(&kvm->lock);
847                 ret = 0;
848                 break;
849         case KVM_S390_VM_MEM_LIMIT_SIZE: {
850                 unsigned long new_limit;
851
852                 if (kvm_is_ucontrol(kvm))
853                         return -EINVAL;
854
855                 if (get_user(new_limit, (u64 __user *)attr->addr))
856                         return -EFAULT;
857
858                 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
859                     new_limit > kvm->arch.mem_limit)
860                         return -E2BIG;
861
862                 if (!new_limit)
863                         return -EINVAL;
864
865                 /* gmap_create takes last usable address */
866                 if (new_limit != KVM_S390_NO_MEM_LIMIT)
867                         new_limit -= 1;
868
869                 ret = -EBUSY;
870                 mutex_lock(&kvm->lock);
871                 if (!kvm->created_vcpus) {
872                         /* gmap_create will round the limit up */
873                         struct gmap *new = gmap_create(current->mm, new_limit);
874
875                         if (!new) {
876                                 ret = -ENOMEM;
877                         } else {
878                                 gmap_remove(kvm->arch.gmap);
879                                 new->private = kvm;
880                                 kvm->arch.gmap = new;
881                                 ret = 0;
882                         }
883                 }
884                 mutex_unlock(&kvm->lock);
885                 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
886                 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
887                          (void *) kvm->arch.gmap->asce);
888                 break;
889         }
890         default:
891                 ret = -ENXIO;
892                 break;
893         }
894         return ret;
895 }
896
897 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
898
899 void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
900 {
901         struct kvm_vcpu *vcpu;
902         int i;
903
904         kvm_s390_vcpu_block_all(kvm);
905
906         kvm_for_each_vcpu(i, vcpu, kvm) {
907                 kvm_s390_vcpu_crypto_setup(vcpu);
908                 /* recreate the shadow crycb by leaving the VSIE handler */
909                 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
910         }
911
912         kvm_s390_vcpu_unblock_all(kvm);
913 }
914
915 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
916 {
917         mutex_lock(&kvm->lock);
918         switch (attr->attr) {
919         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
920                 if (!test_kvm_facility(kvm, 76)) {
921                         mutex_unlock(&kvm->lock);
922                         return -EINVAL;
923                 }
924                 get_random_bytes(
925                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
926                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
927                 kvm->arch.crypto.aes_kw = 1;
928                 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
929                 break;
930         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
931                 if (!test_kvm_facility(kvm, 76)) {
932                         mutex_unlock(&kvm->lock);
933                         return -EINVAL;
934                 }
935                 get_random_bytes(
936                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
937                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
938                 kvm->arch.crypto.dea_kw = 1;
939                 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
940                 break;
941         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
942                 if (!test_kvm_facility(kvm, 76)) {
943                         mutex_unlock(&kvm->lock);
944                         return -EINVAL;
945                 }
946                 kvm->arch.crypto.aes_kw = 0;
947                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
948                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
949                 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
950                 break;
951         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
952                 if (!test_kvm_facility(kvm, 76)) {
953                         mutex_unlock(&kvm->lock);
954                         return -EINVAL;
955                 }
956                 kvm->arch.crypto.dea_kw = 0;
957                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
958                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
959                 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
960                 break;
961         case KVM_S390_VM_CRYPTO_ENABLE_APIE:
962                 if (!ap_instructions_available()) {
963                         mutex_unlock(&kvm->lock);
964                         return -EOPNOTSUPP;
965                 }
966                 kvm->arch.crypto.apie = 1;
967                 break;
968         case KVM_S390_VM_CRYPTO_DISABLE_APIE:
969                 if (!ap_instructions_available()) {
970                         mutex_unlock(&kvm->lock);
971                         return -EOPNOTSUPP;
972                 }
973                 kvm->arch.crypto.apie = 0;
974                 break;
975         default:
976                 mutex_unlock(&kvm->lock);
977                 return -ENXIO;
978         }
979
980         kvm_s390_vcpu_crypto_reset_all(kvm);
981         mutex_unlock(&kvm->lock);
982         return 0;
983 }
984
985 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
986 {
987         int cx;
988         struct kvm_vcpu *vcpu;
989
990         kvm_for_each_vcpu(cx, vcpu, kvm)
991                 kvm_s390_sync_request(req, vcpu);
992 }
993
994 /*
995  * Must be called with kvm->srcu held to avoid races on memslots, and with
996  * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
997  */
998 static int kvm_s390_vm_start_migration(struct kvm *kvm)
999 {
1000         struct kvm_memory_slot *ms;
1001         struct kvm_memslots *slots;
1002         unsigned long ram_pages = 0;
1003         int slotnr;
1004
1005         /* migration mode already enabled */
1006         if (kvm->arch.migration_mode)
1007                 return 0;
1008         slots = kvm_memslots(kvm);
1009         if (!slots || !slots->used_slots)
1010                 return -EINVAL;
1011
1012         if (!kvm->arch.use_cmma) {
1013                 kvm->arch.migration_mode = 1;
1014                 return 0;
1015         }
1016         /* mark all the pages in active slots as dirty */
1017         for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
1018                 ms = slots->memslots + slotnr;
1019                 if (!ms->dirty_bitmap)
1020                         return -EINVAL;
1021                 /*
1022                  * The second half of the bitmap is only used on x86,
1023                  * and would be wasted otherwise, so we put it to good
1024                  * use here to keep track of the state of the storage
1025                  * attributes.
1026                  */
1027                 memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
1028                 ram_pages += ms->npages;
1029         }
1030         atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
1031         kvm->arch.migration_mode = 1;
1032         kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
1033         return 0;
1034 }
1035
1036 /*
1037  * Must be called with kvm->slots_lock to avoid races with ourselves and
1038  * kvm_s390_vm_start_migration.
1039  */
1040 static int kvm_s390_vm_stop_migration(struct kvm *kvm)
1041 {
1042         /* migration mode already disabled */
1043         if (!kvm->arch.migration_mode)
1044                 return 0;
1045         kvm->arch.migration_mode = 0;
1046         if (kvm->arch.use_cmma)
1047                 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
1048         return 0;
1049 }
1050
1051 static int kvm_s390_vm_set_migration(struct kvm *kvm,
1052                                      struct kvm_device_attr *attr)
1053 {
1054         int res = -ENXIO;
1055
1056         mutex_lock(&kvm->slots_lock);
1057         switch (attr->attr) {
1058         case KVM_S390_VM_MIGRATION_START:
1059                 res = kvm_s390_vm_start_migration(kvm);
1060                 break;
1061         case KVM_S390_VM_MIGRATION_STOP:
1062                 res = kvm_s390_vm_stop_migration(kvm);
1063                 break;
1064         default:
1065                 break;
1066         }
1067         mutex_unlock(&kvm->slots_lock);
1068
1069         return res;
1070 }
1071
1072 static int kvm_s390_vm_get_migration(struct kvm *kvm,
1073                                      struct kvm_device_attr *attr)
1074 {
1075         u64 mig = kvm->arch.migration_mode;
1076
1077         if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1078                 return -ENXIO;
1079
1080         if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1081                 return -EFAULT;
1082         return 0;
1083 }
1084
1085 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1086 {
1087         struct kvm_s390_vm_tod_clock gtod;
1088
1089         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
1090                 return -EFAULT;
1091
1092         if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
1093                 return -EINVAL;
1094         kvm_s390_set_tod_clock(kvm, &gtod);
1095
1096         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1097                 gtod.epoch_idx, gtod.tod);
1098
1099         return 0;
1100 }
1101
1102 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1103 {
1104         u8 gtod_high;
1105
1106         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
1107                                            sizeof(gtod_high)))
1108                 return -EFAULT;
1109
1110         if (gtod_high != 0)
1111                 return -EINVAL;
1112         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
1113
1114         return 0;
1115 }
1116
1117 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1118 {
1119         struct kvm_s390_vm_tod_clock gtod = { 0 };
1120
1121         if (copy_from_user(&gtod.tod, (void __user *)attr->addr,
1122                            sizeof(gtod.tod)))
1123                 return -EFAULT;
1124
1125         kvm_s390_set_tod_clock(kvm, &gtod);
1126         VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1127         return 0;
1128 }
1129
1130 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1131 {
1132         int ret;
1133
1134         if (attr->flags)
1135                 return -EINVAL;
1136
1137         switch (attr->attr) {
1138         case KVM_S390_VM_TOD_EXT:
1139                 ret = kvm_s390_set_tod_ext(kvm, attr);
1140                 break;
1141         case KVM_S390_VM_TOD_HIGH:
1142                 ret = kvm_s390_set_tod_high(kvm, attr);
1143                 break;
1144         case KVM_S390_VM_TOD_LOW:
1145                 ret = kvm_s390_set_tod_low(kvm, attr);
1146                 break;
1147         default:
1148                 ret = -ENXIO;
1149                 break;
1150         }
1151         return ret;
1152 }
1153
1154 static void kvm_s390_get_tod_clock(struct kvm *kvm,
1155                                    struct kvm_s390_vm_tod_clock *gtod)
1156 {
1157         struct kvm_s390_tod_clock_ext htod;
1158
1159         preempt_disable();
1160
1161         get_tod_clock_ext((char *)&htod);
1162
1163         gtod->tod = htod.tod + kvm->arch.epoch;
1164         gtod->epoch_idx = 0;
1165         if (test_kvm_facility(kvm, 139)) {
1166                 gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
1167                 if (gtod->tod < htod.tod)
1168                         gtod->epoch_idx += 1;
1169         }
1170
1171         preempt_enable();
1172 }
1173
1174 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1175 {
1176         struct kvm_s390_vm_tod_clock gtod;
1177
1178         memset(&gtod, 0, sizeof(gtod));
1179         kvm_s390_get_tod_clock(kvm, &gtod);
1180         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1181                 return -EFAULT;
1182
1183         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1184                 gtod.epoch_idx, gtod.tod);
1185         return 0;
1186 }
1187
1188 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1189 {
1190         u8 gtod_high = 0;
1191
1192         if (copy_to_user((void __user *)attr->addr, &gtod_high,
1193                                          sizeof(gtod_high)))
1194                 return -EFAULT;
1195         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1196
1197         return 0;
1198 }
1199
1200 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1201 {
1202         u64 gtod;
1203
1204         gtod = kvm_s390_get_tod_clock_fast(kvm);
1205         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1206                 return -EFAULT;
1207         VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1208
1209         return 0;
1210 }
1211
1212 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1213 {
1214         int ret;
1215
1216         if (attr->flags)
1217                 return -EINVAL;
1218
1219         switch (attr->attr) {
1220         case KVM_S390_VM_TOD_EXT:
1221                 ret = kvm_s390_get_tod_ext(kvm, attr);
1222                 break;
1223         case KVM_S390_VM_TOD_HIGH:
1224                 ret = kvm_s390_get_tod_high(kvm, attr);
1225                 break;
1226         case KVM_S390_VM_TOD_LOW:
1227                 ret = kvm_s390_get_tod_low(kvm, attr);
1228                 break;
1229         default:
1230                 ret = -ENXIO;
1231                 break;
1232         }
1233         return ret;
1234 }
1235
1236 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1237 {
1238         struct kvm_s390_vm_cpu_processor *proc;
1239         u16 lowest_ibc, unblocked_ibc;
1240         int ret = 0;
1241
1242         mutex_lock(&kvm->lock);
1243         if (kvm->created_vcpus) {
1244                 ret = -EBUSY;
1245                 goto out;
1246         }
1247         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1248         if (!proc) {
1249                 ret = -ENOMEM;
1250                 goto out;
1251         }
1252         if (!copy_from_user(proc, (void __user *)attr->addr,
1253                             sizeof(*proc))) {
1254                 kvm->arch.model.cpuid = proc->cpuid;
1255                 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1256                 unblocked_ibc = sclp.ibc & 0xfff;
1257                 if (lowest_ibc && proc->ibc) {
1258                         if (proc->ibc > unblocked_ibc)
1259                                 kvm->arch.model.ibc = unblocked_ibc;
1260                         else if (proc->ibc < lowest_ibc)
1261                                 kvm->arch.model.ibc = lowest_ibc;
1262                         else
1263                                 kvm->arch.model.ibc = proc->ibc;
1264                 }
1265                 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1266                        S390_ARCH_FAC_LIST_SIZE_BYTE);
1267                 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1268                          kvm->arch.model.ibc,
1269                          kvm->arch.model.cpuid);
1270                 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1271                          kvm->arch.model.fac_list[0],
1272                          kvm->arch.model.fac_list[1],
1273                          kvm->arch.model.fac_list[2]);
1274         } else
1275                 ret = -EFAULT;
1276         kfree(proc);
1277 out:
1278         mutex_unlock(&kvm->lock);
1279         return ret;
1280 }
1281
1282 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1283                                        struct kvm_device_attr *attr)
1284 {
1285         struct kvm_s390_vm_cpu_feat data;
1286
1287         if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1288                 return -EFAULT;
1289         if (!bitmap_subset((unsigned long *) data.feat,
1290                            kvm_s390_available_cpu_feat,
1291                            KVM_S390_VM_CPU_FEAT_NR_BITS))
1292                 return -EINVAL;
1293
1294         mutex_lock(&kvm->lock);
1295         if (kvm->created_vcpus) {
1296                 mutex_unlock(&kvm->lock);
1297                 return -EBUSY;
1298         }
1299         bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1300                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1301         mutex_unlock(&kvm->lock);
1302         VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1303                          data.feat[0],
1304                          data.feat[1],
1305                          data.feat[2]);
1306         return 0;
1307 }
1308
1309 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1310                                           struct kvm_device_attr *attr)
1311 {
1312         mutex_lock(&kvm->lock);
1313         if (kvm->created_vcpus) {
1314                 mutex_unlock(&kvm->lock);
1315                 return -EBUSY;
1316         }
1317
1318         if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
1319                            sizeof(struct kvm_s390_vm_cpu_subfunc))) {
1320                 mutex_unlock(&kvm->lock);
1321                 return -EFAULT;
1322         }
1323         mutex_unlock(&kvm->lock);
1324
1325         VM_EVENT(kvm, 3, "SET: guest PLO    subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1326                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1327                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1328                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1329                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1330         VM_EVENT(kvm, 3, "SET: guest PTFF   subfunc 0x%16.16lx.%16.16lx",
1331                  ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1332                  ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1333         VM_EVENT(kvm, 3, "SET: guest KMAC   subfunc 0x%16.16lx.%16.16lx",
1334                  ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1335                  ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1336         VM_EVENT(kvm, 3, "SET: guest KMC    subfunc 0x%16.16lx.%16.16lx",
1337                  ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1338                  ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1339         VM_EVENT(kvm, 3, "SET: guest KM     subfunc 0x%16.16lx.%16.16lx",
1340                  ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1341                  ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1342         VM_EVENT(kvm, 3, "SET: guest KIMD   subfunc 0x%16.16lx.%16.16lx",
1343                  ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1344                  ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1345         VM_EVENT(kvm, 3, "SET: guest KLMD   subfunc 0x%16.16lx.%16.16lx",
1346                  ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1347                  ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1348         VM_EVENT(kvm, 3, "SET: guest PCKMO  subfunc 0x%16.16lx.%16.16lx",
1349                  ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1350                  ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1351         VM_EVENT(kvm, 3, "SET: guest KMCTR  subfunc 0x%16.16lx.%16.16lx",
1352                  ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1353                  ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1354         VM_EVENT(kvm, 3, "SET: guest KMF    subfunc 0x%16.16lx.%16.16lx",
1355                  ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1356                  ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1357         VM_EVENT(kvm, 3, "SET: guest KMO    subfunc 0x%16.16lx.%16.16lx",
1358                  ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1359                  ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1360         VM_EVENT(kvm, 3, "SET: guest PCC    subfunc 0x%16.16lx.%16.16lx",
1361                  ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1362                  ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1363         VM_EVENT(kvm, 3, "SET: guest PPNO   subfunc 0x%16.16lx.%16.16lx",
1364                  ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1365                  ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1366         VM_EVENT(kvm, 3, "SET: guest KMA    subfunc 0x%16.16lx.%16.16lx",
1367                  ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1368                  ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1369         VM_EVENT(kvm, 3, "SET: guest KDSA   subfunc 0x%16.16lx.%16.16lx",
1370                  ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1371                  ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1372         VM_EVENT(kvm, 3, "SET: guest SORTL  subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1373                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1374                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1375                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1376                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1377         VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1378                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1379                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1380                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1381                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1382
1383         return 0;
1384 }
1385
1386 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1387 {
1388         int ret = -ENXIO;
1389
1390         switch (attr->attr) {
1391         case KVM_S390_VM_CPU_PROCESSOR:
1392                 ret = kvm_s390_set_processor(kvm, attr);
1393                 break;
1394         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1395                 ret = kvm_s390_set_processor_feat(kvm, attr);
1396                 break;
1397         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1398                 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1399                 break;
1400         }
1401         return ret;
1402 }
1403
1404 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1405 {
1406         struct kvm_s390_vm_cpu_processor *proc;
1407         int ret = 0;
1408
1409         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1410         if (!proc) {
1411                 ret = -ENOMEM;
1412                 goto out;
1413         }
1414         proc->cpuid = kvm->arch.model.cpuid;
1415         proc->ibc = kvm->arch.model.ibc;
1416         memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1417                S390_ARCH_FAC_LIST_SIZE_BYTE);
1418         VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1419                  kvm->arch.model.ibc,
1420                  kvm->arch.model.cpuid);
1421         VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1422                  kvm->arch.model.fac_list[0],
1423                  kvm->arch.model.fac_list[1],
1424                  kvm->arch.model.fac_list[2]);
1425         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1426                 ret = -EFAULT;
1427         kfree(proc);
1428 out:
1429         return ret;
1430 }
1431
1432 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1433 {
1434         struct kvm_s390_vm_cpu_machine *mach;
1435         int ret = 0;
1436
1437         mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1438         if (!mach) {
1439                 ret = -ENOMEM;
1440                 goto out;
1441         }
1442         get_cpu_id((struct cpuid *) &mach->cpuid);
1443         mach->ibc = sclp.ibc;
1444         memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1445                S390_ARCH_FAC_LIST_SIZE_BYTE);
1446         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
1447                sizeof(S390_lowcore.stfle_fac_list));
1448         VM_EVENT(kvm, 3, "GET: host ibc:  0x%4.4x, host cpuid:  0x%16.16llx",
1449                  kvm->arch.model.ibc,
1450                  kvm->arch.model.cpuid);
1451         VM_EVENT(kvm, 3, "GET: host facmask:  0x%16.16llx.%16.16llx.%16.16llx",
1452                  mach->fac_mask[0],
1453                  mach->fac_mask[1],
1454                  mach->fac_mask[2]);
1455         VM_EVENT(kvm, 3, "GET: host faclist:  0x%16.16llx.%16.16llx.%16.16llx",
1456                  mach->fac_list[0],
1457                  mach->fac_list[1],
1458                  mach->fac_list[2]);
1459         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1460                 ret = -EFAULT;
1461         kfree(mach);
1462 out:
1463         return ret;
1464 }
1465
1466 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1467                                        struct kvm_device_attr *attr)
1468 {
1469         struct kvm_s390_vm_cpu_feat data;
1470
1471         bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1472                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1473         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1474                 return -EFAULT;
1475         VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1476                          data.feat[0],
1477                          data.feat[1],
1478                          data.feat[2]);
1479         return 0;
1480 }
1481
1482 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1483                                      struct kvm_device_attr *attr)
1484 {
1485         struct kvm_s390_vm_cpu_feat data;
1486
1487         bitmap_copy((unsigned long *) data.feat,
1488                     kvm_s390_available_cpu_feat,
1489                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1490         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1491                 return -EFAULT;
1492         VM_EVENT(kvm, 3, "GET: host feat:  0x%16.16llx.0x%16.16llx.0x%16.16llx",
1493                          data.feat[0],
1494                          data.feat[1],
1495                          data.feat[2]);
1496         return 0;
1497 }
1498
1499 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1500                                           struct kvm_device_attr *attr)
1501 {
1502         if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
1503             sizeof(struct kvm_s390_vm_cpu_subfunc)))
1504                 return -EFAULT;
1505
1506         VM_EVENT(kvm, 3, "GET: guest PLO    subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1507                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1508                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1509                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1510                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1511         VM_EVENT(kvm, 3, "GET: guest PTFF   subfunc 0x%16.16lx.%16.16lx",
1512                  ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1513                  ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1514         VM_EVENT(kvm, 3, "GET: guest KMAC   subfunc 0x%16.16lx.%16.16lx",
1515                  ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1516                  ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1517         VM_EVENT(kvm, 3, "GET: guest KMC    subfunc 0x%16.16lx.%16.16lx",
1518                  ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1519                  ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1520         VM_EVENT(kvm, 3, "GET: guest KM     subfunc 0x%16.16lx.%16.16lx",
1521                  ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1522                  ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1523         VM_EVENT(kvm, 3, "GET: guest KIMD   subfunc 0x%16.16lx.%16.16lx",
1524                  ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1525                  ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1526         VM_EVENT(kvm, 3, "GET: guest KLMD   subfunc 0x%16.16lx.%16.16lx",
1527                  ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1528                  ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1529         VM_EVENT(kvm, 3, "GET: guest PCKMO  subfunc 0x%16.16lx.%16.16lx",
1530                  ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1531                  ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1532         VM_EVENT(kvm, 3, "GET: guest KMCTR  subfunc 0x%16.16lx.%16.16lx",
1533                  ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1534                  ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1535         VM_EVENT(kvm, 3, "GET: guest KMF    subfunc 0x%16.16lx.%16.16lx",
1536                  ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1537                  ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1538         VM_EVENT(kvm, 3, "GET: guest KMO    subfunc 0x%16.16lx.%16.16lx",
1539                  ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1540                  ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1541         VM_EVENT(kvm, 3, "GET: guest PCC    subfunc 0x%16.16lx.%16.16lx",
1542                  ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1543                  ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1544         VM_EVENT(kvm, 3, "GET: guest PPNO   subfunc 0x%16.16lx.%16.16lx",
1545                  ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1546                  ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1547         VM_EVENT(kvm, 3, "GET: guest KMA    subfunc 0x%16.16lx.%16.16lx",
1548                  ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1549                  ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1550         VM_EVENT(kvm, 3, "GET: guest KDSA   subfunc 0x%16.16lx.%16.16lx",
1551                  ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1552                  ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1553         VM_EVENT(kvm, 3, "GET: guest SORTL  subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1554                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1555                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1556                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1557                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1558         VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1559                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1560                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1561                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1562                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1563
1564         return 0;
1565 }
1566
1567 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1568                                         struct kvm_device_attr *attr)
1569 {
1570         if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1571             sizeof(struct kvm_s390_vm_cpu_subfunc)))
1572                 return -EFAULT;
1573
1574         VM_EVENT(kvm, 3, "GET: host  PLO    subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1575                  ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
1576                  ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
1577                  ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
1578                  ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
1579         VM_EVENT(kvm, 3, "GET: host  PTFF   subfunc 0x%16.16lx.%16.16lx",
1580                  ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
1581                  ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
1582         VM_EVENT(kvm, 3, "GET: host  KMAC   subfunc 0x%16.16lx.%16.16lx",
1583                  ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
1584                  ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
1585         VM_EVENT(kvm, 3, "GET: host  KMC    subfunc 0x%16.16lx.%16.16lx",
1586                  ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
1587                  ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
1588         VM_EVENT(kvm, 3, "GET: host  KM     subfunc 0x%16.16lx.%16.16lx",
1589                  ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
1590                  ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
1591         VM_EVENT(kvm, 3, "GET: host  KIMD   subfunc 0x%16.16lx.%16.16lx",
1592                  ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
1593                  ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
1594         VM_EVENT(kvm, 3, "GET: host  KLMD   subfunc 0x%16.16lx.%16.16lx",
1595                  ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
1596                  ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
1597         VM_EVENT(kvm, 3, "GET: host  PCKMO  subfunc 0x%16.16lx.%16.16lx",
1598                  ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
1599                  ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
1600         VM_EVENT(kvm, 3, "GET: host  KMCTR  subfunc 0x%16.16lx.%16.16lx",
1601                  ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
1602                  ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
1603         VM_EVENT(kvm, 3, "GET: host  KMF    subfunc 0x%16.16lx.%16.16lx",
1604                  ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
1605                  ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
1606         VM_EVENT(kvm, 3, "GET: host  KMO    subfunc 0x%16.16lx.%16.16lx",
1607                  ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
1608                  ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
1609         VM_EVENT(kvm, 3, "GET: host  PCC    subfunc 0x%16.16lx.%16.16lx",
1610                  ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
1611                  ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
1612         VM_EVENT(kvm, 3, "GET: host  PPNO   subfunc 0x%16.16lx.%16.16lx",
1613                  ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
1614                  ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
1615         VM_EVENT(kvm, 3, "GET: host  KMA    subfunc 0x%16.16lx.%16.16lx",
1616                  ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
1617                  ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
1618         VM_EVENT(kvm, 3, "GET: host  KDSA   subfunc 0x%16.16lx.%16.16lx",
1619                  ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
1620                  ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
1621         VM_EVENT(kvm, 3, "GET: host  SORTL  subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1622                  ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
1623                  ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
1624                  ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
1625                  ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
1626         VM_EVENT(kvm, 3, "GET: host  DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1627                  ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
1628                  ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
1629                  ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
1630                  ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
1631
1632         return 0;
1633 }
1634
1635 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1636 {
1637         int ret = -ENXIO;
1638
1639         switch (attr->attr) {
1640         case KVM_S390_VM_CPU_PROCESSOR:
1641                 ret = kvm_s390_get_processor(kvm, attr);
1642                 break;
1643         case KVM_S390_VM_CPU_MACHINE:
1644                 ret = kvm_s390_get_machine(kvm, attr);
1645                 break;
1646         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1647                 ret = kvm_s390_get_processor_feat(kvm, attr);
1648                 break;
1649         case KVM_S390_VM_CPU_MACHINE_FEAT:
1650                 ret = kvm_s390_get_machine_feat(kvm, attr);
1651                 break;
1652         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1653                 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1654                 break;
1655         case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1656                 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1657                 break;
1658         }
1659         return ret;
1660 }
1661
1662 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1663 {
1664         int ret;
1665
1666         switch (attr->group) {
1667         case KVM_S390_VM_MEM_CTRL:
1668                 ret = kvm_s390_set_mem_control(kvm, attr);
1669                 break;
1670         case KVM_S390_VM_TOD:
1671                 ret = kvm_s390_set_tod(kvm, attr);
1672                 break;
1673         case KVM_S390_VM_CPU_MODEL:
1674                 ret = kvm_s390_set_cpu_model(kvm, attr);
1675                 break;
1676         case KVM_S390_VM_CRYPTO:
1677                 ret = kvm_s390_vm_set_crypto(kvm, attr);
1678                 break;
1679         case KVM_S390_VM_MIGRATION:
1680                 ret = kvm_s390_vm_set_migration(kvm, attr);
1681                 break;
1682         default:
1683                 ret = -ENXIO;
1684                 break;
1685         }
1686
1687         return ret;
1688 }
1689
1690 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1691 {
1692         int ret;
1693
1694         switch (attr->group) {
1695         case KVM_S390_VM_MEM_CTRL:
1696                 ret = kvm_s390_get_mem_control(kvm, attr);
1697                 break;
1698         case KVM_S390_VM_TOD:
1699                 ret = kvm_s390_get_tod(kvm, attr);
1700                 break;
1701         case KVM_S390_VM_CPU_MODEL:
1702                 ret = kvm_s390_get_cpu_model(kvm, attr);
1703                 break;
1704         case KVM_S390_VM_MIGRATION:
1705                 ret = kvm_s390_vm_get_migration(kvm, attr);
1706                 break;
1707         default:
1708                 ret = -ENXIO;
1709                 break;
1710         }
1711
1712         return ret;
1713 }
1714
1715 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1716 {
1717         int ret;
1718
1719         switch (attr->group) {
1720         case KVM_S390_VM_MEM_CTRL:
1721                 switch (attr->attr) {
1722                 case KVM_S390_VM_MEM_ENABLE_CMMA:
1723                 case KVM_S390_VM_MEM_CLR_CMMA:
1724                         ret = sclp.has_cmma ? 0 : -ENXIO;
1725                         break;
1726                 case KVM_S390_VM_MEM_LIMIT_SIZE:
1727                         ret = 0;
1728                         break;
1729                 default:
1730                         ret = -ENXIO;
1731                         break;
1732                 }
1733                 break;
1734         case KVM_S390_VM_TOD:
1735                 switch (attr->attr) {
1736                 case KVM_S390_VM_TOD_LOW:
1737                 case KVM_S390_VM_TOD_HIGH:
1738                         ret = 0;
1739                         break;
1740                 default:
1741                         ret = -ENXIO;
1742                         break;
1743                 }
1744                 break;
1745         case KVM_S390_VM_CPU_MODEL:
1746                 switch (attr->attr) {
1747                 case KVM_S390_VM_CPU_PROCESSOR:
1748                 case KVM_S390_VM_CPU_MACHINE:
1749                 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1750                 case KVM_S390_VM_CPU_MACHINE_FEAT:
1751                 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1752                 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1753                         ret = 0;
1754                         break;
1755                 default:
1756                         ret = -ENXIO;
1757                         break;
1758                 }
1759                 break;
1760         case KVM_S390_VM_CRYPTO:
1761                 switch (attr->attr) {
1762                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1763                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1764                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1765                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1766                         ret = 0;
1767                         break;
1768                 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1769                 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1770                         ret = ap_instructions_available() ? 0 : -ENXIO;
1771                         break;
1772                 default:
1773                         ret = -ENXIO;
1774                         break;
1775                 }
1776                 break;
1777         case KVM_S390_VM_MIGRATION:
1778                 ret = 0;
1779                 break;
1780         default:
1781                 ret = -ENXIO;
1782                 break;
1783         }
1784
1785         return ret;
1786 }
1787
1788 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1789 {
1790         uint8_t *keys;
1791         uint64_t hva;
1792         int srcu_idx, i, r = 0;
1793
1794         if (args->flags != 0)
1795                 return -EINVAL;
1796
1797         /* Is this guest using storage keys? */
1798         if (!mm_uses_skeys(current->mm))
1799                 return KVM_S390_GET_SKEYS_NONE;
1800
1801         /* Enforce sane limit on memory allocation */
1802         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1803                 return -EINVAL;
1804
1805         keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1806         if (!keys)
1807                 return -ENOMEM;
1808
1809         down_read(&current->mm->mmap_sem);
1810         srcu_idx = srcu_read_lock(&kvm->srcu);
1811         for (i = 0; i < args->count; i++) {
1812                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1813                 if (kvm_is_error_hva(hva)) {
1814                         r = -EFAULT;
1815                         break;
1816                 }
1817
1818                 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1819                 if (r)
1820                         break;
1821         }
1822         srcu_read_unlock(&kvm->srcu, srcu_idx);
1823         up_read(&current->mm->mmap_sem);
1824
1825         if (!r) {
1826                 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1827                                  sizeof(uint8_t) * args->count);
1828                 if (r)
1829                         r = -EFAULT;
1830         }
1831
1832         kvfree(keys);
1833         return r;
1834 }
1835
1836 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1837 {
1838         uint8_t *keys;
1839         uint64_t hva;
1840         int srcu_idx, i, r = 0;
1841         bool unlocked;
1842
1843         if (args->flags != 0)
1844                 return -EINVAL;
1845
1846         /* Enforce sane limit on memory allocation */
1847         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1848                 return -EINVAL;
1849
1850         keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1851         if (!keys)
1852                 return -ENOMEM;
1853
1854         r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1855                            sizeof(uint8_t) * args->count);
1856         if (r) {
1857                 r = -EFAULT;
1858                 goto out;
1859         }
1860
1861         /* Enable storage key handling for the guest */
1862         r = s390_enable_skey();
1863         if (r)
1864                 goto out;
1865
1866         i = 0;
1867         down_read(&current->mm->mmap_sem);
1868         srcu_idx = srcu_read_lock(&kvm->srcu);
1869         while (i < args->count) {
1870                 unlocked = false;
1871                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1872                 if (kvm_is_error_hva(hva)) {
1873                         r = -EFAULT;
1874                         break;
1875                 }
1876
1877                 /* Lowest order bit is reserved */
1878                 if (keys[i] & 0x01) {
1879                         r = -EINVAL;
1880                         break;
1881                 }
1882
1883                 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1884                 if (r) {
1885                         r = fixup_user_fault(current, current->mm, hva,
1886                                              FAULT_FLAG_WRITE, &unlocked);
1887                         if (r)
1888                                 break;
1889                 }
1890                 if (!r)
1891                         i++;
1892         }
1893         srcu_read_unlock(&kvm->srcu, srcu_idx);
1894         up_read(&current->mm->mmap_sem);
1895 out:
1896         kvfree(keys);
1897         return r;
1898 }
1899
1900 /*
1901  * Base address and length must be sent at the start of each block, therefore
1902  * it's cheaper to send some clean data, as long as it's less than the size of
1903  * two longs.
1904  */
1905 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1906 /* for consistency */
1907 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1908
1909 /*
1910  * Similar to gfn_to_memslot, but returns the index of a memslot also when the
1911  * address falls in a hole. In that case the index of one of the memslots
1912  * bordering the hole is returned.
1913  */
1914 static int gfn_to_memslot_approx(struct kvm_memslots *slots, gfn_t gfn)
1915 {
1916         int start = 0, end = slots->used_slots;
1917         int slot = atomic_read(&slots->lru_slot);
1918         struct kvm_memory_slot *memslots = slots->memslots;
1919
1920         if (gfn >= memslots[slot].base_gfn &&
1921             gfn < memslots[slot].base_gfn + memslots[slot].npages)
1922                 return slot;
1923
1924         while (start < end) {
1925                 slot = start + (end - start) / 2;
1926
1927                 if (gfn >= memslots[slot].base_gfn)
1928                         end = slot;
1929                 else
1930                         start = slot + 1;
1931         }
1932
1933         if (gfn >= memslots[start].base_gfn &&
1934             gfn < memslots[start].base_gfn + memslots[start].npages) {
1935                 atomic_set(&slots->lru_slot, start);
1936         }
1937
1938         return start;
1939 }
1940
1941 static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1942                               u8 *res, unsigned long bufsize)
1943 {
1944         unsigned long pgstev, hva, cur_gfn = args->start_gfn;
1945
1946         args->count = 0;
1947         while (args->count < bufsize) {
1948                 hva = gfn_to_hva(kvm, cur_gfn);
1949                 /*
1950                  * We return an error if the first value was invalid, but we
1951                  * return successfully if at least one value was copied.
1952                  */
1953                 if (kvm_is_error_hva(hva))
1954                         return args->count ? 0 : -EFAULT;
1955                 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
1956                         pgstev = 0;
1957                 res[args->count++] = (pgstev >> 24) & 0x43;
1958                 cur_gfn++;
1959         }
1960
1961         return 0;
1962 }
1963
1964 static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
1965                                               unsigned long cur_gfn)
1966 {
1967         int slotidx = gfn_to_memslot_approx(slots, cur_gfn);
1968         struct kvm_memory_slot *ms = slots->memslots + slotidx;
1969         unsigned long ofs = cur_gfn - ms->base_gfn;
1970
1971         if (ms->base_gfn + ms->npages <= cur_gfn) {
1972                 slotidx--;
1973                 /* If we are above the highest slot, wrap around */
1974                 if (slotidx < 0)
1975                         slotidx = slots->used_slots - 1;
1976
1977                 ms = slots->memslots + slotidx;
1978                 ofs = 0;
1979         }
1980         ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs);
1981         while ((slotidx > 0) && (ofs >= ms->npages)) {
1982                 slotidx--;
1983                 ms = slots->memslots + slotidx;
1984                 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, 0);
1985         }
1986         return ms->base_gfn + ofs;
1987 }
1988
1989 static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1990                              u8 *res, unsigned long bufsize)
1991 {
1992         unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
1993         struct kvm_memslots *slots = kvm_memslots(kvm);
1994         struct kvm_memory_slot *ms;
1995
1996         cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
1997         ms = gfn_to_memslot(kvm, cur_gfn);
1998         args->count = 0;
1999         args->start_gfn = cur_gfn;
2000         if (!ms)
2001                 return 0;
2002         next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2003         mem_end = slots->memslots[0].base_gfn + slots->memslots[0].npages;
2004
2005         while (args->count < bufsize) {
2006                 hva = gfn_to_hva(kvm, cur_gfn);
2007                 if (kvm_is_error_hva(hva))
2008                         return 0;
2009                 /* Decrement only if we actually flipped the bit to 0 */
2010                 if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
2011                         atomic64_dec(&kvm->arch.cmma_dirty_pages);
2012                 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
2013                         pgstev = 0;
2014                 /* Save the value */
2015                 res[args->count++] = (pgstev >> 24) & 0x43;
2016                 /* If the next bit is too far away, stop. */
2017                 if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
2018                         return 0;
2019                 /* If we reached the previous "next", find the next one */
2020                 if (cur_gfn == next_gfn)
2021                         next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2022                 /* Reached the end of memory or of the buffer, stop */
2023                 if ((next_gfn >= mem_end) ||
2024                     (next_gfn - args->start_gfn >= bufsize))
2025                         return 0;
2026                 cur_gfn++;
2027                 /* Reached the end of the current memslot, take the next one. */
2028                 if (cur_gfn - ms->base_gfn >= ms->npages) {
2029                         ms = gfn_to_memslot(kvm, cur_gfn);
2030                         if (!ms)
2031                                 return 0;
2032                 }
2033         }
2034         return 0;
2035 }
2036
2037 /*
2038  * This function searches for the next page with dirty CMMA attributes, and
2039  * saves the attributes in the buffer up to either the end of the buffer or
2040  * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
2041  * no trailing clean bytes are saved.
2042  * In case no dirty bits were found, or if CMMA was not enabled or used, the
2043  * output buffer will indicate 0 as length.
2044  */
2045 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
2046                                   struct kvm_s390_cmma_log *args)
2047 {
2048         unsigned long bufsize;
2049         int srcu_idx, peek, ret;
2050         u8 *values;
2051
2052         if (!kvm->arch.use_cmma)
2053                 return -ENXIO;
2054         /* Invalid/unsupported flags were specified */
2055         if (args->flags & ~KVM_S390_CMMA_PEEK)
2056                 return -EINVAL;
2057         /* Migration mode query, and we are not doing a migration */
2058         peek = !!(args->flags & KVM_S390_CMMA_PEEK);
2059         if (!peek && !kvm->arch.migration_mode)
2060                 return -EINVAL;
2061         /* CMMA is disabled or was not used, or the buffer has length zero */
2062         bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
2063         if (!bufsize || !kvm->mm->context.uses_cmm) {
2064                 memset(args, 0, sizeof(*args));
2065                 return 0;
2066         }
2067         /* We are not peeking, and there are no dirty pages */
2068         if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
2069                 memset(args, 0, sizeof(*args));
2070                 return 0;
2071         }
2072
2073         values = vmalloc(bufsize);
2074         if (!values)
2075                 return -ENOMEM;
2076
2077         down_read(&kvm->mm->mmap_sem);
2078         srcu_idx = srcu_read_lock(&kvm->srcu);
2079         if (peek)
2080                 ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
2081         else
2082                 ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
2083         srcu_read_unlock(&kvm->srcu, srcu_idx);
2084         up_read(&kvm->mm->mmap_sem);
2085
2086         if (kvm->arch.migration_mode)
2087                 args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
2088         else
2089                 args->remaining = 0;
2090
2091         if (copy_to_user((void __user *)args->values, values, args->count))
2092                 ret = -EFAULT;
2093
2094         vfree(values);
2095         return ret;
2096 }
2097
2098 /*
2099  * This function sets the CMMA attributes for the given pages. If the input
2100  * buffer has zero length, no action is taken, otherwise the attributes are
2101  * set and the mm->context.uses_cmm flag is set.
2102  */
2103 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
2104                                   const struct kvm_s390_cmma_log *args)
2105 {
2106         unsigned long hva, mask, pgstev, i;
2107         uint8_t *bits;
2108         int srcu_idx, r = 0;
2109
2110         mask = args->mask;
2111
2112         if (!kvm->arch.use_cmma)
2113                 return -ENXIO;
2114         /* invalid/unsupported flags */
2115         if (args->flags != 0)
2116                 return -EINVAL;
2117         /* Enforce sane limit on memory allocation */
2118         if (args->count > KVM_S390_CMMA_SIZE_MAX)
2119                 return -EINVAL;
2120         /* Nothing to do */
2121         if (args->count == 0)
2122                 return 0;
2123
2124         bits = vmalloc(array_size(sizeof(*bits), args->count));
2125         if (!bits)
2126                 return -ENOMEM;
2127
2128         r = copy_from_user(bits, (void __user *)args->values, args->count);
2129         if (r) {
2130                 r = -EFAULT;
2131                 goto out;
2132         }
2133
2134         down_read(&kvm->mm->mmap_sem);
2135         srcu_idx = srcu_read_lock(&kvm->srcu);
2136         for (i = 0; i < args->count; i++) {
2137                 hva = gfn_to_hva(kvm, args->start_gfn + i);
2138                 if (kvm_is_error_hva(hva)) {
2139                         r = -EFAULT;
2140                         break;
2141                 }
2142
2143                 pgstev = bits[i];
2144                 pgstev = pgstev << 24;
2145                 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
2146                 set_pgste_bits(kvm->mm, hva, mask, pgstev);
2147         }
2148         srcu_read_unlock(&kvm->srcu, srcu_idx);
2149         up_read(&kvm->mm->mmap_sem);
2150
2151         if (!kvm->mm->context.uses_cmm) {
2152                 down_write(&kvm->mm->mmap_sem);
2153                 kvm->mm->context.uses_cmm = 1;
2154                 up_write(&kvm->mm->mmap_sem);
2155         }
2156 out:
2157         vfree(bits);
2158         return r;
2159 }
2160
2161 long kvm_arch_vm_ioctl(struct file *filp,
2162                        unsigned int ioctl, unsigned long arg)
2163 {
2164         struct kvm *kvm = filp->private_data;
2165         void __user *argp = (void __user *)arg;
2166         struct kvm_device_attr attr;
2167         int r;
2168
2169         switch (ioctl) {
2170         case KVM_S390_INTERRUPT: {
2171                 struct kvm_s390_interrupt s390int;
2172
2173                 r = -EFAULT;
2174                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2175                         break;
2176                 r = kvm_s390_inject_vm(kvm, &s390int);
2177                 break;
2178         }
2179         case KVM_CREATE_IRQCHIP: {
2180                 struct kvm_irq_routing_entry routing;
2181
2182                 r = -EINVAL;
2183                 if (kvm->arch.use_irqchip) {
2184                         /* Set up dummy routing. */
2185                         memset(&routing, 0, sizeof(routing));
2186                         r = kvm_set_irq_routing(kvm, &routing, 0, 0);
2187                 }
2188                 break;
2189         }
2190         case KVM_SET_DEVICE_ATTR: {
2191                 r = -EFAULT;
2192                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2193                         break;
2194                 r = kvm_s390_vm_set_attr(kvm, &attr);
2195                 break;
2196         }
2197         case KVM_GET_DEVICE_ATTR: {
2198                 r = -EFAULT;
2199                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2200                         break;
2201                 r = kvm_s390_vm_get_attr(kvm, &attr);
2202                 break;
2203         }
2204         case KVM_HAS_DEVICE_ATTR: {
2205                 r = -EFAULT;
2206                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2207                         break;
2208                 r = kvm_s390_vm_has_attr(kvm, &attr);
2209                 break;
2210         }
2211         case KVM_S390_GET_SKEYS: {
2212                 struct kvm_s390_skeys args;
2213
2214                 r = -EFAULT;
2215                 if (copy_from_user(&args, argp,
2216                                    sizeof(struct kvm_s390_skeys)))
2217                         break;
2218                 r = kvm_s390_get_skeys(kvm, &args);
2219                 break;
2220         }
2221         case KVM_S390_SET_SKEYS: {
2222                 struct kvm_s390_skeys args;
2223
2224                 r = -EFAULT;
2225                 if (copy_from_user(&args, argp,
2226                                    sizeof(struct kvm_s390_skeys)))
2227                         break;
2228                 r = kvm_s390_set_skeys(kvm, &args);
2229                 break;
2230         }
2231         case KVM_S390_GET_CMMA_BITS: {
2232                 struct kvm_s390_cmma_log args;
2233
2234                 r = -EFAULT;
2235                 if (copy_from_user(&args, argp, sizeof(args)))
2236                         break;
2237                 mutex_lock(&kvm->slots_lock);
2238                 r = kvm_s390_get_cmma_bits(kvm, &args);
2239                 mutex_unlock(&kvm->slots_lock);
2240                 if (!r) {
2241                         r = copy_to_user(argp, &args, sizeof(args));
2242                         if (r)
2243                                 r = -EFAULT;
2244                 }
2245                 break;
2246         }
2247         case KVM_S390_SET_CMMA_BITS: {
2248                 struct kvm_s390_cmma_log args;
2249
2250                 r = -EFAULT;
2251                 if (copy_from_user(&args, argp, sizeof(args)))
2252                         break;
2253                 mutex_lock(&kvm->slots_lock);
2254                 r = kvm_s390_set_cmma_bits(kvm, &args);
2255                 mutex_unlock(&kvm->slots_lock);
2256                 break;
2257         }
2258         default:
2259                 r = -ENOTTY;
2260         }
2261
2262         return r;
2263 }
2264
2265 static int kvm_s390_apxa_installed(void)
2266 {
2267         struct ap_config_info info;
2268
2269         if (ap_instructions_available()) {
2270                 if (ap_qci(&info) == 0)
2271                         return info.apxa;
2272         }
2273
2274         return 0;
2275 }
2276
2277 /*
2278  * The format of the crypto control block (CRYCB) is specified in the 3 low
2279  * order bits of the CRYCB designation (CRYCBD) field as follows:
2280  * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
2281  *           AP extended addressing (APXA) facility are installed.
2282  * Format 1: The APXA facility is not installed but the MSAX3 facility is.
2283  * Format 2: Both the APXA and MSAX3 facilities are installed
2284  */
2285 static void kvm_s390_set_crycb_format(struct kvm *kvm)
2286 {
2287         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
2288
2289         /* Clear the CRYCB format bits - i.e., set format 0 by default */
2290         kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
2291
2292         /* Check whether MSAX3 is installed */
2293         if (!test_kvm_facility(kvm, 76))
2294                 return;
2295
2296         if (kvm_s390_apxa_installed())
2297                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
2298         else
2299                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
2300 }
2301
2302 void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
2303                                unsigned long *aqm, unsigned long *adm)
2304 {
2305         struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
2306
2307         mutex_lock(&kvm->lock);
2308         kvm_s390_vcpu_block_all(kvm);
2309
2310         switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
2311         case CRYCB_FORMAT2: /* APCB1 use 256 bits */
2312                 memcpy(crycb->apcb1.apm, apm, 32);
2313                 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
2314                          apm[0], apm[1], apm[2], apm[3]);
2315                 memcpy(crycb->apcb1.aqm, aqm, 32);
2316                 VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
2317                          aqm[0], aqm[1], aqm[2], aqm[3]);
2318                 memcpy(crycb->apcb1.adm, adm, 32);
2319                 VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
2320                          adm[0], adm[1], adm[2], adm[3]);
2321                 break;
2322         case CRYCB_FORMAT1:
2323         case CRYCB_FORMAT0: /* Fall through both use APCB0 */
2324                 memcpy(crycb->apcb0.apm, apm, 8);
2325                 memcpy(crycb->apcb0.aqm, aqm, 2);
2326                 memcpy(crycb->apcb0.adm, adm, 2);
2327                 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
2328                          apm[0], *((unsigned short *)aqm),
2329                          *((unsigned short *)adm));
2330                 break;
2331         default:        /* Can not happen */
2332                 break;
2333         }
2334
2335         /* recreate the shadow crycb for each vcpu */
2336         kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2337         kvm_s390_vcpu_unblock_all(kvm);
2338         mutex_unlock(&kvm->lock);
2339 }
2340 EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
2341
2342 void kvm_arch_crypto_clear_masks(struct kvm *kvm)
2343 {
2344         mutex_lock(&kvm->lock);
2345         kvm_s390_vcpu_block_all(kvm);
2346
2347         memset(&kvm->arch.crypto.crycb->apcb0, 0,
2348                sizeof(kvm->arch.crypto.crycb->apcb0));
2349         memset(&kvm->arch.crypto.crycb->apcb1, 0,
2350                sizeof(kvm->arch.crypto.crycb->apcb1));
2351
2352         VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
2353         /* recreate the shadow crycb for each vcpu */
2354         kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2355         kvm_s390_vcpu_unblock_all(kvm);
2356         mutex_unlock(&kvm->lock);
2357 }
2358 EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
2359
2360 static u64 kvm_s390_get_initial_cpuid(void)
2361 {
2362         struct cpuid cpuid;
2363
2364         get_cpu_id(&cpuid);
2365         cpuid.version = 0xff;
2366         return *((u64 *) &cpuid);
2367 }
2368
2369 static void kvm_s390_crypto_init(struct kvm *kvm)
2370 {
2371         kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
2372         kvm_s390_set_crycb_format(kvm);
2373
2374         if (!test_kvm_facility(kvm, 76))
2375                 return;
2376
2377         /* Enable AES/DEA protected key functions by default */
2378         kvm->arch.crypto.aes_kw = 1;
2379         kvm->arch.crypto.dea_kw = 1;
2380         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
2381                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
2382         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
2383                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
2384 }
2385
2386 static void sca_dispose(struct kvm *kvm)
2387 {
2388         if (kvm->arch.use_esca)
2389                 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
2390         else
2391                 free_page((unsigned long)(kvm->arch.sca));
2392         kvm->arch.sca = NULL;
2393 }
2394
2395 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
2396 {
2397         gfp_t alloc_flags = GFP_KERNEL;
2398         int i, rc;
2399         char debug_name[16];
2400         static unsigned long sca_offset;
2401
2402         rc = -EINVAL;
2403 #ifdef CONFIG_KVM_S390_UCONTROL
2404         if (type & ~KVM_VM_S390_UCONTROL)
2405                 goto out_err;
2406         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
2407                 goto out_err;
2408 #else
2409         if (type)
2410                 goto out_err;
2411 #endif
2412
2413         rc = s390_enable_sie();
2414         if (rc)
2415                 goto out_err;
2416
2417         rc = -ENOMEM;
2418
2419         if (!sclp.has_64bscao)
2420                 alloc_flags |= GFP_DMA;
2421         rwlock_init(&kvm->arch.sca_lock);
2422         /* start with basic SCA */
2423         kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
2424         if (!kvm->arch.sca)
2425                 goto out_err;
2426         mutex_lock(&kvm_lock);
2427         sca_offset += 16;
2428         if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
2429                 sca_offset = 0;
2430         kvm->arch.sca = (struct bsca_block *)
2431                         ((char *) kvm->arch.sca + sca_offset);
2432         mutex_unlock(&kvm_lock);
2433
2434         sprintf(debug_name, "kvm-%u", current->pid);
2435
2436         kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
2437         if (!kvm->arch.dbf)
2438                 goto out_err;
2439
2440         BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
2441         kvm->arch.sie_page2 =
2442              (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
2443         if (!kvm->arch.sie_page2)
2444                 goto out_err;
2445
2446         kvm->arch.sie_page2->kvm = kvm;
2447         kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
2448
2449         for (i = 0; i < kvm_s390_fac_size(); i++) {
2450                 kvm->arch.model.fac_mask[i] = S390_lowcore.stfle_fac_list[i] &
2451                                               (kvm_s390_fac_base[i] |
2452                                                kvm_s390_fac_ext[i]);
2453                 kvm->arch.model.fac_list[i] = S390_lowcore.stfle_fac_list[i] &
2454                                               kvm_s390_fac_base[i];
2455         }
2456         kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
2457
2458         /* we are always in czam mode - even on pre z14 machines */
2459         set_kvm_facility(kvm->arch.model.fac_mask, 138);
2460         set_kvm_facility(kvm->arch.model.fac_list, 138);
2461         /* we emulate STHYI in kvm */
2462         set_kvm_facility(kvm->arch.model.fac_mask, 74);
2463         set_kvm_facility(kvm->arch.model.fac_list, 74);
2464         if (MACHINE_HAS_TLB_GUEST) {
2465                 set_kvm_facility(kvm->arch.model.fac_mask, 147);
2466                 set_kvm_facility(kvm->arch.model.fac_list, 147);
2467         }
2468
2469         if (css_general_characteristics.aiv && test_facility(65))
2470                 set_kvm_facility(kvm->arch.model.fac_mask, 65);
2471
2472         kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
2473         kvm->arch.model.ibc = sclp.ibc & 0x0fff;
2474
2475         kvm_s390_crypto_init(kvm);
2476
2477         mutex_init(&kvm->arch.float_int.ais_lock);
2478         spin_lock_init(&kvm->arch.float_int.lock);
2479         for (i = 0; i < FIRQ_LIST_COUNT; i++)
2480                 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
2481         init_waitqueue_head(&kvm->arch.ipte_wq);
2482         mutex_init(&kvm->arch.ipte_mutex);
2483
2484         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
2485         VM_EVENT(kvm, 3, "vm created with type %lu", type);
2486
2487         if (type & KVM_VM_S390_UCONTROL) {
2488                 kvm->arch.gmap = NULL;
2489                 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
2490         } else {
2491                 if (sclp.hamax == U64_MAX)
2492                         kvm->arch.mem_limit = TASK_SIZE_MAX;
2493                 else
2494                         kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
2495                                                     sclp.hamax + 1);
2496                 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
2497                 if (!kvm->arch.gmap)
2498                         goto out_err;
2499                 kvm->arch.gmap->private = kvm;
2500                 kvm->arch.gmap->pfault_enabled = 0;
2501         }
2502
2503         kvm->arch.use_pfmfi = sclp.has_pfmfi;
2504         kvm->arch.use_skf = sclp.has_skey;
2505         spin_lock_init(&kvm->arch.start_stop_lock);
2506         kvm_s390_vsie_init(kvm);
2507         kvm_s390_gisa_init(kvm);
2508         KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
2509
2510         return 0;
2511 out_err:
2512         free_page((unsigned long)kvm->arch.sie_page2);
2513         debug_unregister(kvm->arch.dbf);
2514         sca_dispose(kvm);
2515         KVM_EVENT(3, "creation of vm failed: %d", rc);
2516         return rc;
2517 }
2518
2519 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2520 {
2521         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
2522         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
2523         kvm_s390_clear_local_irqs(vcpu);
2524         kvm_clear_async_pf_completion_queue(vcpu);
2525         if (!kvm_is_ucontrol(vcpu->kvm))
2526                 sca_del_vcpu(vcpu);
2527
2528         if (kvm_is_ucontrol(vcpu->kvm))
2529                 gmap_remove(vcpu->arch.gmap);
2530
2531         if (vcpu->kvm->arch.use_cmma)
2532                 kvm_s390_vcpu_unsetup_cmma(vcpu);
2533         free_page((unsigned long)(vcpu->arch.sie_block));
2534 }
2535
2536 static void kvm_free_vcpus(struct kvm *kvm)
2537 {
2538         unsigned int i;
2539         struct kvm_vcpu *vcpu;
2540
2541         kvm_for_each_vcpu(i, vcpu, kvm)
2542                 kvm_vcpu_destroy(vcpu);
2543
2544         mutex_lock(&kvm->lock);
2545         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2546                 kvm->vcpus[i] = NULL;
2547
2548         atomic_set(&kvm->online_vcpus, 0);
2549         mutex_unlock(&kvm->lock);
2550 }
2551
2552 void kvm_arch_destroy_vm(struct kvm *kvm)
2553 {
2554         kvm_free_vcpus(kvm);
2555         sca_dispose(kvm);
2556         debug_unregister(kvm->arch.dbf);
2557         kvm_s390_gisa_destroy(kvm);
2558         free_page((unsigned long)kvm->arch.sie_page2);
2559         if (!kvm_is_ucontrol(kvm))
2560                 gmap_remove(kvm->arch.gmap);
2561         kvm_s390_destroy_adapters(kvm);
2562         kvm_s390_clear_float_irqs(kvm);
2563         kvm_s390_vsie_destroy(kvm);
2564         KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
2565 }
2566
2567 /* Section: vcpu related */
2568 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2569 {
2570         vcpu->arch.gmap = gmap_create(current->mm, -1UL);
2571         if (!vcpu->arch.gmap)
2572                 return -ENOMEM;
2573         vcpu->arch.gmap->private = vcpu->kvm;
2574
2575         return 0;
2576 }
2577
2578 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2579 {
2580         if (!kvm_s390_use_sca_entries())
2581                 return;
2582         read_lock(&vcpu->kvm->arch.sca_lock);
2583         if (vcpu->kvm->arch.use_esca) {
2584                 struct esca_block *sca = vcpu->kvm->arch.sca;
2585
2586                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2587                 sca->cpu[vcpu->vcpu_id].sda = 0;
2588         } else {
2589                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2590
2591                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2592                 sca->cpu[vcpu->vcpu_id].sda = 0;
2593         }
2594         read_unlock(&vcpu->kvm->arch.sca_lock);
2595 }
2596
2597 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
2598 {
2599         if (!kvm_s390_use_sca_entries()) {
2600                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2601
2602                 /* we still need the basic sca for the ipte control */
2603                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2604                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2605                 return;
2606         }
2607         read_lock(&vcpu->kvm->arch.sca_lock);
2608         if (vcpu->kvm->arch.use_esca) {
2609                 struct esca_block *sca = vcpu->kvm->arch.sca;
2610
2611                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2612                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2613                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
2614                 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2615                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2616         } else {
2617                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2618
2619                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2620                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2621                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2622                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2623         }
2624         read_unlock(&vcpu->kvm->arch.sca_lock);
2625 }
2626
2627 /* Basic SCA to Extended SCA data copy routines */
2628 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2629 {
2630         d->sda = s->sda;
2631         d->sigp_ctrl.c = s->sigp_ctrl.c;
2632         d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2633 }
2634
2635 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2636 {
2637         int i;
2638
2639         d->ipte_control = s->ipte_control;
2640         d->mcn[0] = s->mcn;
2641         for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2642                 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2643 }
2644
2645 static int sca_switch_to_extended(struct kvm *kvm)
2646 {
2647         struct bsca_block *old_sca = kvm->arch.sca;
2648         struct esca_block *new_sca;
2649         struct kvm_vcpu *vcpu;
2650         unsigned int vcpu_idx;
2651         u32 scaol, scaoh;
2652
2653         new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2654         if (!new_sca)
2655                 return -ENOMEM;
2656
2657         scaoh = (u32)((u64)(new_sca) >> 32);
2658         scaol = (u32)(u64)(new_sca) & ~0x3fU;
2659
2660         kvm_s390_vcpu_block_all(kvm);
2661         write_lock(&kvm->arch.sca_lock);
2662
2663         sca_copy_b_to_e(new_sca, old_sca);
2664
2665         kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2666                 vcpu->arch.sie_block->scaoh = scaoh;
2667                 vcpu->arch.sie_block->scaol = scaol;
2668                 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2669         }
2670         kvm->arch.sca = new_sca;
2671         kvm->arch.use_esca = 1;
2672
2673         write_unlock(&kvm->arch.sca_lock);
2674         kvm_s390_vcpu_unblock_all(kvm);
2675
2676         free_page((unsigned long)old_sca);
2677
2678         VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2679                  old_sca, kvm->arch.sca);
2680         return 0;
2681 }
2682
2683 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2684 {
2685         int rc;
2686
2687         if (!kvm_s390_use_sca_entries()) {
2688                 if (id < KVM_MAX_VCPUS)
2689                         return true;
2690                 return false;
2691         }
2692         if (id < KVM_S390_BSCA_CPU_SLOTS)
2693                 return true;
2694         if (!sclp.has_esca || !sclp.has_64bscao)
2695                 return false;
2696
2697         mutex_lock(&kvm->lock);
2698         rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2699         mutex_unlock(&kvm->lock);
2700
2701         return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
2702 }
2703
2704 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2705 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2706 {
2707         WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
2708         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2709         vcpu->arch.cputm_start = get_tod_clock_fast();
2710         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2711 }
2712
2713 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2714 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2715 {
2716         WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
2717         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2718         vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2719         vcpu->arch.cputm_start = 0;
2720         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2721 }
2722
2723 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2724 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2725 {
2726         WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2727         vcpu->arch.cputm_enabled = true;
2728         __start_cpu_timer_accounting(vcpu);
2729 }
2730
2731 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2732 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2733 {
2734         WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2735         __stop_cpu_timer_accounting(vcpu);
2736         vcpu->arch.cputm_enabled = false;
2737 }
2738
2739 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2740 {
2741         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2742         __enable_cpu_timer_accounting(vcpu);
2743         preempt_enable();
2744 }
2745
2746 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2747 {
2748         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2749         __disable_cpu_timer_accounting(vcpu);
2750         preempt_enable();
2751 }
2752
2753 /* set the cpu timer - may only be called from the VCPU thread itself */
2754 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
2755 {
2756         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2757         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2758         if (vcpu->arch.cputm_enabled)
2759                 vcpu->arch.cputm_start = get_tod_clock_fast();
2760         vcpu->arch.sie_block->cputm = cputm;
2761         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2762         preempt_enable();
2763 }
2764
2765 /* update and get the cpu timer - can also be called from other VCPU threads */
2766 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
2767 {
2768         unsigned int seq;
2769         __u64 value;
2770
2771         if (unlikely(!vcpu->arch.cputm_enabled))
2772                 return vcpu->arch.sie_block->cputm;
2773
2774         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2775         do {
2776                 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
2777                 /*
2778                  * If the writer would ever execute a read in the critical
2779                  * section, e.g. in irq context, we have a deadlock.
2780                  */
2781                 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
2782                 value = vcpu->arch.sie_block->cputm;
2783                 /* if cputm_start is 0, accounting is being started/stopped */
2784                 if (likely(vcpu->arch.cputm_start))
2785                         value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2786         } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
2787         preempt_enable();
2788         return value;
2789 }
2790
2791 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2792 {
2793
2794         gmap_enable(vcpu->arch.enabled_gmap);
2795         kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
2796         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2797                 __start_cpu_timer_accounting(vcpu);
2798         vcpu->cpu = cpu;
2799 }
2800
2801 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2802 {
2803         vcpu->cpu = -1;
2804         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2805                 __stop_cpu_timer_accounting(vcpu);
2806         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
2807         vcpu->arch.enabled_gmap = gmap_get_enabled();
2808         gmap_disable(vcpu->arch.enabled_gmap);
2809
2810 }
2811
2812 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
2813 {
2814         mutex_lock(&vcpu->kvm->lock);
2815         preempt_disable();
2816         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
2817         vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
2818         preempt_enable();
2819         mutex_unlock(&vcpu->kvm->lock);
2820         if (!kvm_is_ucontrol(vcpu->kvm)) {
2821                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
2822                 sca_add_vcpu(vcpu);
2823         }
2824         if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
2825                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2826         /* make vcpu_load load the right gmap on the first trigger */
2827         vcpu->arch.enabled_gmap = vcpu->arch.gmap;
2828 }
2829
2830 static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
2831 {
2832         if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
2833             test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
2834                 return true;
2835         return false;
2836 }
2837
2838 static bool kvm_has_pckmo_ecc(struct kvm *kvm)
2839 {
2840         /* At least one ECC subfunction must be present */
2841         return kvm_has_pckmo_subfunc(kvm, 32) ||
2842                kvm_has_pckmo_subfunc(kvm, 33) ||
2843                kvm_has_pckmo_subfunc(kvm, 34) ||
2844                kvm_has_pckmo_subfunc(kvm, 40) ||
2845                kvm_has_pckmo_subfunc(kvm, 41);
2846
2847 }
2848
2849 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
2850 {
2851         /*
2852          * If the AP instructions are not being interpreted and the MSAX3
2853          * facility is not configured for the guest, there is nothing to set up.
2854          */
2855         if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
2856                 return;
2857
2858         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
2859         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
2860         vcpu->arch.sie_block->eca &= ~ECA_APIE;
2861         vcpu->arch.sie_block->ecd &= ~ECD_ECC;
2862
2863         if (vcpu->kvm->arch.crypto.apie)
2864                 vcpu->arch.sie_block->eca |= ECA_APIE;
2865
2866         /* Set up protected key support */
2867         if (vcpu->kvm->arch.crypto.aes_kw) {
2868                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
2869                 /* ecc is also wrapped with AES key */
2870                 if (kvm_has_pckmo_ecc(vcpu->kvm))
2871                         vcpu->arch.sie_block->ecd |= ECD_ECC;
2872         }
2873
2874         if (vcpu->kvm->arch.crypto.dea_kw)
2875                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
2876 }
2877
2878 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
2879 {
2880         free_page(vcpu->arch.sie_block->cbrlo);
2881         vcpu->arch.sie_block->cbrlo = 0;
2882 }
2883
2884 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
2885 {
2886         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
2887         if (!vcpu->arch.sie_block->cbrlo)
2888                 return -ENOMEM;
2889         return 0;
2890 }
2891
2892 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
2893 {
2894         struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
2895
2896         vcpu->arch.sie_block->ibc = model->ibc;
2897         if (test_kvm_facility(vcpu->kvm, 7))
2898                 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
2899 }
2900
2901 static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu)
2902 {
2903         int rc = 0;
2904
2905         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
2906                                                     CPUSTAT_SM |
2907                                                     CPUSTAT_STOPPED);
2908
2909         if (test_kvm_facility(vcpu->kvm, 78))
2910                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
2911         else if (test_kvm_facility(vcpu->kvm, 8))
2912                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
2913
2914         kvm_s390_vcpu_setup_model(vcpu);
2915
2916         /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2917         if (MACHINE_HAS_ESOP)
2918                 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
2919         if (test_kvm_facility(vcpu->kvm, 9))
2920                 vcpu->arch.sie_block->ecb |= ECB_SRSI;
2921         if (test_kvm_facility(vcpu->kvm, 73))
2922                 vcpu->arch.sie_block->ecb |= ECB_TE;
2923
2924         if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
2925                 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
2926         if (test_kvm_facility(vcpu->kvm, 130))
2927                 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
2928         vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
2929         if (sclp.has_cei)
2930                 vcpu->arch.sie_block->eca |= ECA_CEI;
2931         if (sclp.has_ib)
2932                 vcpu->arch.sie_block->eca |= ECA_IB;
2933         if (sclp.has_siif)
2934                 vcpu->arch.sie_block->eca |= ECA_SII;
2935         if (sclp.has_sigpif)
2936                 vcpu->arch.sie_block->eca |= ECA_SIGPI;
2937         if (test_kvm_facility(vcpu->kvm, 129)) {
2938                 vcpu->arch.sie_block->eca |= ECA_VX;
2939                 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
2940         }
2941         if (test_kvm_facility(vcpu->kvm, 139))
2942                 vcpu->arch.sie_block->ecd |= ECD_MEF;
2943         if (test_kvm_facility(vcpu->kvm, 156))
2944                 vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
2945         if (vcpu->arch.sie_block->gd) {
2946                 vcpu->arch.sie_block->eca |= ECA_AIV;
2947                 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
2948                            vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
2949         }
2950         vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2951                                         | SDNXC;
2952         vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
2953
2954         if (sclp.has_kss)
2955                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
2956         else
2957                 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
2958
2959         if (vcpu->kvm->arch.use_cmma) {
2960                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
2961                 if (rc)
2962                         return rc;
2963         }
2964         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2965         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
2966
2967         vcpu->arch.sie_block->hpid = HPID_KVM;
2968
2969         kvm_s390_vcpu_crypto_setup(vcpu);
2970
2971         return rc;
2972 }
2973
2974 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
2975 {
2976         if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
2977                 return -EINVAL;
2978         return 0;
2979 }
2980
2981 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
2982 {
2983         struct sie_page *sie_page;
2984         int rc;
2985
2986         BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
2987         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2988         if (!sie_page)
2989                 return -ENOMEM;
2990
2991         vcpu->arch.sie_block = &sie_page->sie_block;
2992         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2993
2994         /* the real guest size will always be smaller than msl */
2995         vcpu->arch.sie_block->mso = 0;
2996         vcpu->arch.sie_block->msl = sclp.hamax;
2997
2998         vcpu->arch.sie_block->icpua = vcpu->vcpu_id;
2999         spin_lock_init(&vcpu->arch.local_int.lock);
3000         vcpu->arch.sie_block->gd = (u32)(u64)vcpu->kvm->arch.gisa_int.origin;
3001         if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
3002                 vcpu->arch.sie_block->gd |= GISA_FORMAT1;
3003         seqcount_init(&vcpu->arch.cputm_seqcount);
3004
3005         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3006         kvm_clear_async_pf_completion_queue(vcpu);
3007         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
3008                                     KVM_SYNC_GPRS |
3009                                     KVM_SYNC_ACRS |
3010                                     KVM_SYNC_CRS |
3011                                     KVM_SYNC_ARCH0 |
3012                                     KVM_SYNC_PFAULT;
3013         kvm_s390_set_prefix(vcpu, 0);
3014         if (test_kvm_facility(vcpu->kvm, 64))
3015                 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
3016         if (test_kvm_facility(vcpu->kvm, 82))
3017                 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
3018         if (test_kvm_facility(vcpu->kvm, 133))
3019                 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
3020         if (test_kvm_facility(vcpu->kvm, 156))
3021                 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
3022         /* fprs can be synchronized via vrs, even if the guest has no vx. With
3023          * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
3024          */
3025         if (MACHINE_HAS_VX)
3026                 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
3027         else
3028                 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
3029
3030         if (kvm_is_ucontrol(vcpu->kvm)) {
3031                 rc = __kvm_ucontrol_vcpu_init(vcpu);
3032                 if (rc)
3033                         goto out_free_sie_block;
3034         }
3035
3036         VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK",
3037                  vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3038         trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3039
3040         rc = kvm_s390_vcpu_setup(vcpu);
3041         if (rc)
3042                 goto out_ucontrol_uninit;
3043         return 0;
3044
3045 out_ucontrol_uninit:
3046         if (kvm_is_ucontrol(vcpu->kvm))
3047                 gmap_remove(vcpu->arch.gmap);
3048 out_free_sie_block:
3049         free_page((unsigned long)(vcpu->arch.sie_block));
3050         return rc;
3051 }
3052
3053 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
3054 {
3055         return kvm_s390_vcpu_has_irq(vcpu, 0);
3056 }
3057
3058 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
3059 {
3060         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
3061 }
3062
3063 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
3064 {
3065         atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3066         exit_sie(vcpu);
3067 }
3068
3069 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
3070 {
3071         atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3072 }
3073
3074 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
3075 {
3076         atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3077         exit_sie(vcpu);
3078 }
3079
3080 bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
3081 {
3082         return atomic_read(&vcpu->arch.sie_block->prog20) &
3083                (PROG_BLOCK_SIE | PROG_REQUEST);
3084 }
3085
3086 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
3087 {
3088         atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3089 }
3090
3091 /*
3092  * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
3093  * If the CPU is not running (e.g. waiting as idle) the function will
3094  * return immediately. */
3095 void exit_sie(struct kvm_vcpu *vcpu)
3096 {
3097         kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
3098         kvm_s390_vsie_kick(vcpu);
3099         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
3100                 cpu_relax();
3101 }
3102
3103 /* Kick a guest cpu out of SIE to process a request synchronously */
3104 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
3105 {
3106         kvm_make_request(req, vcpu);
3107         kvm_s390_vcpu_request(vcpu);
3108 }
3109
3110 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
3111                               unsigned long end)
3112 {
3113         struct kvm *kvm = gmap->private;
3114         struct kvm_vcpu *vcpu;
3115         unsigned long prefix;
3116         int i;
3117
3118         if (gmap_is_shadow(gmap))
3119                 return;
3120         if (start >= 1UL << 31)
3121                 /* We are only interested in prefix pages */
3122                 return;
3123         kvm_for_each_vcpu(i, vcpu, kvm) {
3124                 /* match against both prefix pages */
3125                 prefix = kvm_s390_get_prefix(vcpu);
3126                 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
3127                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
3128                                    start, end);
3129                         kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
3130                 }
3131         }
3132 }
3133
3134 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
3135 {
3136         /* do not poll with more than halt_poll_max_steal percent of steal time */
3137         if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >=
3138             halt_poll_max_steal) {
3139                 vcpu->stat.halt_no_poll_steal++;
3140                 return true;
3141         }
3142         return false;
3143 }
3144
3145 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
3146 {
3147         /* kvm common code refers to this, but never calls it */
3148         BUG();
3149         return 0;
3150 }
3151
3152 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
3153                                            struct kvm_one_reg *reg)
3154 {
3155         int r = -EINVAL;
3156
3157         switch (reg->id) {
3158         case KVM_REG_S390_TODPR:
3159                 r = put_user(vcpu->arch.sie_block->todpr,
3160                              (u32 __user *)reg->addr);
3161                 break;
3162         case KVM_REG_S390_EPOCHDIFF:
3163                 r = put_user(vcpu->arch.sie_block->epoch,
3164                              (u64 __user *)reg->addr);
3165                 break;
3166         case KVM_REG_S390_CPU_TIMER:
3167                 r = put_user(kvm_s390_get_cpu_timer(vcpu),
3168                              (u64 __user *)reg->addr);
3169                 break;
3170         case KVM_REG_S390_CLOCK_COMP:
3171                 r = put_user(vcpu->arch.sie_block->ckc,
3172                              (u64 __user *)reg->addr);
3173                 break;
3174         case KVM_REG_S390_PFTOKEN:
3175                 r = put_user(vcpu->arch.pfault_token,
3176                              (u64 __user *)reg->addr);
3177                 break;
3178         case KVM_REG_S390_PFCOMPARE: