Merge tag 'powerpc-4.20-2' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc...
[sfrench/cifs-2.6.git] / arch / powerpc / kvm / book3s_hv.c
1 /*
2  * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
3  * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
4  *
5  * Authors:
6  *    Paul Mackerras <paulus@au1.ibm.com>
7  *    Alexander Graf <agraf@suse.de>
8  *    Kevin Wolf <mail@kevin-wolf.de>
9  *
10  * Description: KVM functions specific to running on Book 3S
11  * processors in hypervisor mode (specifically POWER7 and later).
12  *
13  * This file is derived from arch/powerpc/kvm/book3s.c,
14  * by Alexander Graf <agraf@suse.de>.
15  *
16  * This program is free software; you can redistribute it and/or modify
17  * it under the terms of the GNU General Public License, version 2, as
18  * published by the Free Software Foundation.
19  */
20
21 #include <linux/kvm_host.h>
22 #include <linux/kernel.h>
23 #include <linux/err.h>
24 #include <linux/slab.h>
25 #include <linux/preempt.h>
26 #include <linux/sched/signal.h>
27 #include <linux/sched/stat.h>
28 #include <linux/delay.h>
29 #include <linux/export.h>
30 #include <linux/fs.h>
31 #include <linux/anon_inodes.h>
32 #include <linux/cpu.h>
33 #include <linux/cpumask.h>
34 #include <linux/spinlock.h>
35 #include <linux/page-flags.h>
36 #include <linux/srcu.h>
37 #include <linux/miscdevice.h>
38 #include <linux/debugfs.h>
39 #include <linux/gfp.h>
40 #include <linux/vmalloc.h>
41 #include <linux/highmem.h>
42 #include <linux/hugetlb.h>
43 #include <linux/kvm_irqfd.h>
44 #include <linux/irqbypass.h>
45 #include <linux/module.h>
46 #include <linux/compiler.h>
47 #include <linux/of.h>
48
49 #include <asm/ftrace.h>
50 #include <asm/reg.h>
51 #include <asm/ppc-opcode.h>
52 #include <asm/asm-prototypes.h>
53 #include <asm/archrandom.h>
54 #include <asm/debug.h>
55 #include <asm/disassemble.h>
56 #include <asm/cputable.h>
57 #include <asm/cacheflush.h>
58 #include <linux/uaccess.h>
59 #include <asm/io.h>
60 #include <asm/kvm_ppc.h>
61 #include <asm/kvm_book3s.h>
62 #include <asm/mmu_context.h>
63 #include <asm/lppaca.h>
64 #include <asm/processor.h>
65 #include <asm/cputhreads.h>
66 #include <asm/page.h>
67 #include <asm/hvcall.h>
68 #include <asm/switch_to.h>
69 #include <asm/smp.h>
70 #include <asm/dbell.h>
71 #include <asm/hmi.h>
72 #include <asm/pnv-pci.h>
73 #include <asm/mmu.h>
74 #include <asm/opal.h>
75 #include <asm/xics.h>
76 #include <asm/xive.h>
77
78 #include "book3s.h"
79
80 #define CREATE_TRACE_POINTS
81 #include "trace_hv.h"
82
83 /* #define EXIT_DEBUG */
84 /* #define EXIT_DEBUG_SIMPLE */
85 /* #define EXIT_DEBUG_INT */
86
87 /* Used to indicate that a guest page fault needs to be handled */
88 #define RESUME_PAGE_FAULT       (RESUME_GUEST | RESUME_FLAG_ARCH1)
89 /* Used to indicate that a guest passthrough interrupt needs to be handled */
90 #define RESUME_PASSTHROUGH      (RESUME_GUEST | RESUME_FLAG_ARCH2)
91
92 /* Used as a "null" value for timebase values */
93 #define TB_NIL  (~(u64)0)
94
95 static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1);
96
97 static int dynamic_mt_modes = 6;
98 module_param(dynamic_mt_modes, int, 0644);
99 MODULE_PARM_DESC(dynamic_mt_modes, "Set of allowed dynamic micro-threading modes: 0 (= none), 2, 4, or 6 (= 2 or 4)");
100 static int target_smt_mode;
101 module_param(target_smt_mode, int, 0644);
102 MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)");
103
104 static bool indep_threads_mode = true;
105 module_param(indep_threads_mode, bool, S_IRUGO | S_IWUSR);
106 MODULE_PARM_DESC(indep_threads_mode, "Independent-threads mode (only on POWER9)");
107
108 static bool one_vm_per_core;
109 module_param(one_vm_per_core, bool, S_IRUGO | S_IWUSR);
110 MODULE_PARM_DESC(one_vm_per_core, "Only run vCPUs from the same VM on a core (requires indep_threads_mode=N)");
111
112 #ifdef CONFIG_KVM_XICS
113 static struct kernel_param_ops module_param_ops = {
114         .set = param_set_int,
115         .get = param_get_int,
116 };
117
118 module_param_cb(kvm_irq_bypass, &module_param_ops, &kvm_irq_bypass, 0644);
119 MODULE_PARM_DESC(kvm_irq_bypass, "Bypass passthrough interrupt optimization");
120
121 module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect, 0644);
122 MODULE_PARM_DESC(h_ipi_redirect, "Redirect H_IPI wakeup to a free host core");
123 #endif
124
125 /* If set, guests are allowed to create and control nested guests */
126 static bool nested = true;
127 module_param(nested, bool, S_IRUGO | S_IWUSR);
128 MODULE_PARM_DESC(nested, "Enable nested virtualization (only on POWER9)");
129
130 static inline bool nesting_enabled(struct kvm *kvm)
131 {
132         return kvm->arch.nested_enable && kvm_is_radix(kvm);
133 }
134
135 /* If set, the threads on each CPU core have to be in the same MMU mode */
136 static bool no_mixing_hpt_and_radix;
137
138 static void kvmppc_end_cede(struct kvm_vcpu *vcpu);
139 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
140
141 /*
142  * RWMR values for POWER8.  These control the rate at which PURR
143  * and SPURR count and should be set according to the number of
144  * online threads in the vcore being run.
145  */
146 #define RWMR_RPA_P8_1THREAD     0x164520C62609AECAUL
147 #define RWMR_RPA_P8_2THREAD     0x7FFF2908450D8DA9UL
148 #define RWMR_RPA_P8_3THREAD     0x164520C62609AECAUL
149 #define RWMR_RPA_P8_4THREAD     0x199A421245058DA9UL
150 #define RWMR_RPA_P8_5THREAD     0x164520C62609AECAUL
151 #define RWMR_RPA_P8_6THREAD     0x164520C62609AECAUL
152 #define RWMR_RPA_P8_7THREAD     0x164520C62609AECAUL
153 #define RWMR_RPA_P8_8THREAD     0x164520C62609AECAUL
154
155 static unsigned long p8_rwmr_values[MAX_SMT_THREADS + 1] = {
156         RWMR_RPA_P8_1THREAD,
157         RWMR_RPA_P8_1THREAD,
158         RWMR_RPA_P8_2THREAD,
159         RWMR_RPA_P8_3THREAD,
160         RWMR_RPA_P8_4THREAD,
161         RWMR_RPA_P8_5THREAD,
162         RWMR_RPA_P8_6THREAD,
163         RWMR_RPA_P8_7THREAD,
164         RWMR_RPA_P8_8THREAD,
165 };
166
167 static inline struct kvm_vcpu *next_runnable_thread(struct kvmppc_vcore *vc,
168                 int *ip)
169 {
170         int i = *ip;
171         struct kvm_vcpu *vcpu;
172
173         while (++i < MAX_SMT_THREADS) {
174                 vcpu = READ_ONCE(vc->runnable_threads[i]);
175                 if (vcpu) {
176                         *ip = i;
177                         return vcpu;
178                 }
179         }
180         return NULL;
181 }
182
183 /* Used to traverse the list of runnable threads for a given vcore */
184 #define for_each_runnable_thread(i, vcpu, vc) \
185         for (i = -1; (vcpu = next_runnable_thread(vc, &i)); )
186
187 static bool kvmppc_ipi_thread(int cpu)
188 {
189         unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
190
191         /* If we're a nested hypervisor, fall back to ordinary IPIs for now */
192         if (kvmhv_on_pseries())
193                 return false;
194
195         /* On POWER9 we can use msgsnd to IPI any cpu */
196         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
197                 msg |= get_hard_smp_processor_id(cpu);
198                 smp_mb();
199                 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
200                 return true;
201         }
202
203         /* On POWER8 for IPIs to threads in the same core, use msgsnd */
204         if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
205                 preempt_disable();
206                 if (cpu_first_thread_sibling(cpu) ==
207                     cpu_first_thread_sibling(smp_processor_id())) {
208                         msg |= cpu_thread_in_core(cpu);
209                         smp_mb();
210                         __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
211                         preempt_enable();
212                         return true;
213                 }
214                 preempt_enable();
215         }
216
217 #if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP)
218         if (cpu >= 0 && cpu < nr_cpu_ids) {
219                 if (paca_ptrs[cpu]->kvm_hstate.xics_phys) {
220                         xics_wake_cpu(cpu);
221                         return true;
222                 }
223                 opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY);
224                 return true;
225         }
226 #endif
227
228         return false;
229 }
230
231 static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
232 {
233         int cpu;
234         struct swait_queue_head *wqp;
235
236         wqp = kvm_arch_vcpu_wq(vcpu);
237         if (swq_has_sleeper(wqp)) {
238                 swake_up_one(wqp);
239                 ++vcpu->stat.halt_wakeup;
240         }
241
242         cpu = READ_ONCE(vcpu->arch.thread_cpu);
243         if (cpu >= 0 && kvmppc_ipi_thread(cpu))
244                 return;
245
246         /* CPU points to the first thread of the core */
247         cpu = vcpu->cpu;
248         if (cpu >= 0 && cpu < nr_cpu_ids && cpu_online(cpu))
249                 smp_send_reschedule(cpu);
250 }
251
252 /*
253  * We use the vcpu_load/put functions to measure stolen time.
254  * Stolen time is counted as time when either the vcpu is able to
255  * run as part of a virtual core, but the task running the vcore
256  * is preempted or sleeping, or when the vcpu needs something done
257  * in the kernel by the task running the vcpu, but that task is
258  * preempted or sleeping.  Those two things have to be counted
259  * separately, since one of the vcpu tasks will take on the job
260  * of running the core, and the other vcpu tasks in the vcore will
261  * sleep waiting for it to do that, but that sleep shouldn't count
262  * as stolen time.
263  *
264  * Hence we accumulate stolen time when the vcpu can run as part of
265  * a vcore using vc->stolen_tb, and the stolen time when the vcpu
266  * needs its task to do other things in the kernel (for example,
267  * service a page fault) in busy_stolen.  We don't accumulate
268  * stolen time for a vcore when it is inactive, or for a vcpu
269  * when it is in state RUNNING or NOTREADY.  NOTREADY is a bit of
270  * a misnomer; it means that the vcpu task is not executing in
271  * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in
272  * the kernel.  We don't have any way of dividing up that time
273  * between time that the vcpu is genuinely stopped, time that
274  * the task is actively working on behalf of the vcpu, and time
275  * that the task is preempted, so we don't count any of it as
276  * stolen.
277  *
278  * Updates to busy_stolen are protected by arch.tbacct_lock;
279  * updates to vc->stolen_tb are protected by the vcore->stoltb_lock
280  * lock.  The stolen times are measured in units of timebase ticks.
281  * (Note that the != TB_NIL checks below are purely defensive;
282  * they should never fail.)
283  */
284
285 static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc)
286 {
287         unsigned long flags;
288
289         spin_lock_irqsave(&vc->stoltb_lock, flags);
290         vc->preempt_tb = mftb();
291         spin_unlock_irqrestore(&vc->stoltb_lock, flags);
292 }
293
294 static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc)
295 {
296         unsigned long flags;
297
298         spin_lock_irqsave(&vc->stoltb_lock, flags);
299         if (vc->preempt_tb != TB_NIL) {
300                 vc->stolen_tb += mftb() - vc->preempt_tb;
301                 vc->preempt_tb = TB_NIL;
302         }
303         spin_unlock_irqrestore(&vc->stoltb_lock, flags);
304 }
305
306 static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu)
307 {
308         struct kvmppc_vcore *vc = vcpu->arch.vcore;
309         unsigned long flags;
310
311         /*
312          * We can test vc->runner without taking the vcore lock,
313          * because only this task ever sets vc->runner to this
314          * vcpu, and once it is set to this vcpu, only this task
315          * ever sets it to NULL.
316          */
317         if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
318                 kvmppc_core_end_stolen(vc);
319
320         spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
321         if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST &&
322             vcpu->arch.busy_preempt != TB_NIL) {
323                 vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt;
324                 vcpu->arch.busy_preempt = TB_NIL;
325         }
326         spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
327 }
328
329 static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu)
330 {
331         struct kvmppc_vcore *vc = vcpu->arch.vcore;
332         unsigned long flags;
333
334         if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
335                 kvmppc_core_start_stolen(vc);
336
337         spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
338         if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST)
339                 vcpu->arch.busy_preempt = mftb();
340         spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
341 }
342
343 static void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr)
344 {
345         /*
346          * Check for illegal transactional state bit combination
347          * and if we find it, force the TS field to a safe state.
348          */
349         if ((msr & MSR_TS_MASK) == MSR_TS_MASK)
350                 msr &= ~MSR_TS_MASK;
351         vcpu->arch.shregs.msr = msr;
352         kvmppc_end_cede(vcpu);
353 }
354
355 static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr)
356 {
357         vcpu->arch.pvr = pvr;
358 }
359
360 /* Dummy value used in computing PCR value below */
361 #define PCR_ARCH_300    (PCR_ARCH_207 << 1)
362
363 static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat)
364 {
365         unsigned long host_pcr_bit = 0, guest_pcr_bit = 0;
366         struct kvmppc_vcore *vc = vcpu->arch.vcore;
367
368         /* We can (emulate) our own architecture version and anything older */
369         if (cpu_has_feature(CPU_FTR_ARCH_300))
370                 host_pcr_bit = PCR_ARCH_300;
371         else if (cpu_has_feature(CPU_FTR_ARCH_207S))
372                 host_pcr_bit = PCR_ARCH_207;
373         else if (cpu_has_feature(CPU_FTR_ARCH_206))
374                 host_pcr_bit = PCR_ARCH_206;
375         else
376                 host_pcr_bit = PCR_ARCH_205;
377
378         /* Determine lowest PCR bit needed to run guest in given PVR level */
379         guest_pcr_bit = host_pcr_bit;
380         if (arch_compat) {
381                 switch (arch_compat) {
382                 case PVR_ARCH_205:
383                         guest_pcr_bit = PCR_ARCH_205;
384                         break;
385                 case PVR_ARCH_206:
386                 case PVR_ARCH_206p:
387                         guest_pcr_bit = PCR_ARCH_206;
388                         break;
389                 case PVR_ARCH_207:
390                         guest_pcr_bit = PCR_ARCH_207;
391                         break;
392                 case PVR_ARCH_300:
393                         guest_pcr_bit = PCR_ARCH_300;
394                         break;
395                 default:
396                         return -EINVAL;
397                 }
398         }
399
400         /* Check requested PCR bits don't exceed our capabilities */
401         if (guest_pcr_bit > host_pcr_bit)
402                 return -EINVAL;
403
404         spin_lock(&vc->lock);
405         vc->arch_compat = arch_compat;
406         /* Set all PCR bits for which guest_pcr_bit <= bit < host_pcr_bit */
407         vc->pcr = host_pcr_bit - guest_pcr_bit;
408         spin_unlock(&vc->lock);
409
410         return 0;
411 }
412
413 static void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
414 {
415         int r;
416
417         pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
418         pr_err("pc  = %.16lx  msr = %.16llx  trap = %x\n",
419                vcpu->arch.regs.nip, vcpu->arch.shregs.msr, vcpu->arch.trap);
420         for (r = 0; r < 16; ++r)
421                 pr_err("r%2d = %.16lx  r%d = %.16lx\n",
422                        r, kvmppc_get_gpr(vcpu, r),
423                        r+16, kvmppc_get_gpr(vcpu, r+16));
424         pr_err("ctr = %.16lx  lr  = %.16lx\n",
425                vcpu->arch.regs.ctr, vcpu->arch.regs.link);
426         pr_err("srr0 = %.16llx srr1 = %.16llx\n",
427                vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
428         pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
429                vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1);
430         pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
431                vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
432         pr_err("cr = %.8lx  xer = %.16lx  dsisr = %.8x\n",
433                vcpu->arch.regs.ccr, vcpu->arch.regs.xer, vcpu->arch.shregs.dsisr);
434         pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
435         pr_err("fault dar = %.16lx dsisr = %.8x\n",
436                vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
437         pr_err("SLB (%d entries):\n", vcpu->arch.slb_max);
438         for (r = 0; r < vcpu->arch.slb_max; ++r)
439                 pr_err("  ESID = %.16llx VSID = %.16llx\n",
440                        vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv);
441         pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n",
442                vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1,
443                vcpu->arch.last_inst);
444 }
445
446 static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
447 {
448         struct kvm_vcpu *ret;
449
450         mutex_lock(&kvm->lock);
451         ret = kvm_get_vcpu_by_id(kvm, id);
452         mutex_unlock(&kvm->lock);
453         return ret;
454 }
455
456 static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa)
457 {
458         vpa->__old_status |= LPPACA_OLD_SHARED_PROC;
459         vpa->yield_count = cpu_to_be32(1);
460 }
461
462 static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v,
463                    unsigned long addr, unsigned long len)
464 {
465         /* check address is cacheline aligned */
466         if (addr & (L1_CACHE_BYTES - 1))
467                 return -EINVAL;
468         spin_lock(&vcpu->arch.vpa_update_lock);
469         if (v->next_gpa != addr || v->len != len) {
470                 v->next_gpa = addr;
471                 v->len = addr ? len : 0;
472                 v->update_pending = 1;
473         }
474         spin_unlock(&vcpu->arch.vpa_update_lock);
475         return 0;
476 }
477
478 /* Length for a per-processor buffer is passed in at offset 4 in the buffer */
479 struct reg_vpa {
480         u32 dummy;
481         union {
482                 __be16 hword;
483                 __be32 word;
484         } length;
485 };
486
487 static int vpa_is_registered(struct kvmppc_vpa *vpap)
488 {
489         if (vpap->update_pending)
490                 return vpap->next_gpa != 0;
491         return vpap->pinned_addr != NULL;
492 }
493
494 static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu,
495                                        unsigned long flags,
496                                        unsigned long vcpuid, unsigned long vpa)
497 {
498         struct kvm *kvm = vcpu->kvm;
499         unsigned long len, nb;
500         void *va;
501         struct kvm_vcpu *tvcpu;
502         int err;
503         int subfunc;
504         struct kvmppc_vpa *vpap;
505
506         tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
507         if (!tvcpu)
508                 return H_PARAMETER;
509
510         subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK;
511         if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL ||
512             subfunc == H_VPA_REG_SLB) {
513                 /* Registering new area - address must be cache-line aligned */
514                 if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa)
515                         return H_PARAMETER;
516
517                 /* convert logical addr to kernel addr and read length */
518                 va = kvmppc_pin_guest_page(kvm, vpa, &nb);
519                 if (va == NULL)
520                         return H_PARAMETER;
521                 if (subfunc == H_VPA_REG_VPA)
522                         len = be16_to_cpu(((struct reg_vpa *)va)->length.hword);
523                 else
524                         len = be32_to_cpu(((struct reg_vpa *)va)->length.word);
525                 kvmppc_unpin_guest_page(kvm, va, vpa, false);
526
527                 /* Check length */
528                 if (len > nb || len < sizeof(struct reg_vpa))
529                         return H_PARAMETER;
530         } else {
531                 vpa = 0;
532                 len = 0;
533         }
534
535         err = H_PARAMETER;
536         vpap = NULL;
537         spin_lock(&tvcpu->arch.vpa_update_lock);
538
539         switch (subfunc) {
540         case H_VPA_REG_VPA:             /* register VPA */
541                 /*
542                  * The size of our lppaca is 1kB because of the way we align
543                  * it for the guest to avoid crossing a 4kB boundary. We only
544                  * use 640 bytes of the structure though, so we should accept
545                  * clients that set a size of 640.
546                  */
547                 BUILD_BUG_ON(sizeof(struct lppaca) != 640);
548                 if (len < sizeof(struct lppaca))
549                         break;
550                 vpap = &tvcpu->arch.vpa;
551                 err = 0;
552                 break;
553
554         case H_VPA_REG_DTL:             /* register DTL */
555                 if (len < sizeof(struct dtl_entry))
556                         break;
557                 len -= len % sizeof(struct dtl_entry);
558
559                 /* Check that they have previously registered a VPA */
560                 err = H_RESOURCE;
561                 if (!vpa_is_registered(&tvcpu->arch.vpa))
562                         break;
563
564                 vpap = &tvcpu->arch.dtl;
565                 err = 0;
566                 break;
567
568         case H_VPA_REG_SLB:             /* register SLB shadow buffer */
569                 /* Check that they have previously registered a VPA */
570                 err = H_RESOURCE;
571                 if (!vpa_is_registered(&tvcpu->arch.vpa))
572                         break;
573
574                 vpap = &tvcpu->arch.slb_shadow;
575                 err = 0;
576                 break;
577
578         case H_VPA_DEREG_VPA:           /* deregister VPA */
579                 /* Check they don't still have a DTL or SLB buf registered */
580                 err = H_RESOURCE;
581                 if (vpa_is_registered(&tvcpu->arch.dtl) ||
582                     vpa_is_registered(&tvcpu->arch.slb_shadow))
583                         break;
584
585                 vpap = &tvcpu->arch.vpa;
586                 err = 0;
587                 break;
588
589         case H_VPA_DEREG_DTL:           /* deregister DTL */
590                 vpap = &tvcpu->arch.dtl;
591                 err = 0;
592                 break;
593
594         case H_VPA_DEREG_SLB:           /* deregister SLB shadow buffer */
595                 vpap = &tvcpu->arch.slb_shadow;
596                 err = 0;
597                 break;
598         }
599
600         if (vpap) {
601                 vpap->next_gpa = vpa;
602                 vpap->len = len;
603                 vpap->update_pending = 1;
604         }
605
606         spin_unlock(&tvcpu->arch.vpa_update_lock);
607
608         return err;
609 }
610
611 static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap)
612 {
613         struct kvm *kvm = vcpu->kvm;
614         void *va;
615         unsigned long nb;
616         unsigned long gpa;
617
618         /*
619          * We need to pin the page pointed to by vpap->next_gpa,
620          * but we can't call kvmppc_pin_guest_page under the lock
621          * as it does get_user_pages() and down_read().  So we
622          * have to drop the lock, pin the page, then get the lock
623          * again and check that a new area didn't get registered
624          * in the meantime.
625          */
626         for (;;) {
627                 gpa = vpap->next_gpa;
628                 spin_unlock(&vcpu->arch.vpa_update_lock);
629                 va = NULL;
630                 nb = 0;
631                 if (gpa)
632                         va = kvmppc_pin_guest_page(kvm, gpa, &nb);
633                 spin_lock(&vcpu->arch.vpa_update_lock);
634                 if (gpa == vpap->next_gpa)
635                         break;
636                 /* sigh... unpin that one and try again */
637                 if (va)
638                         kvmppc_unpin_guest_page(kvm, va, gpa, false);
639         }
640
641         vpap->update_pending = 0;
642         if (va && nb < vpap->len) {
643                 /*
644                  * If it's now too short, it must be that userspace
645                  * has changed the mappings underlying guest memory,
646                  * so unregister the region.
647                  */
648                 kvmppc_unpin_guest_page(kvm, va, gpa, false);
649                 va = NULL;
650         }
651         if (vpap->pinned_addr)
652                 kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa,
653                                         vpap->dirty);
654         vpap->gpa = gpa;
655         vpap->pinned_addr = va;
656         vpap->dirty = false;
657         if (va)
658                 vpap->pinned_end = va + vpap->len;
659 }
660
661 static void kvmppc_update_vpas(struct kvm_vcpu *vcpu)
662 {
663         if (!(vcpu->arch.vpa.update_pending ||
664               vcpu->arch.slb_shadow.update_pending ||
665               vcpu->arch.dtl.update_pending))
666                 return;
667
668         spin_lock(&vcpu->arch.vpa_update_lock);
669         if (vcpu->arch.vpa.update_pending) {
670                 kvmppc_update_vpa(vcpu, &vcpu->arch.vpa);
671                 if (vcpu->arch.vpa.pinned_addr)
672                         init_vpa(vcpu, vcpu->arch.vpa.pinned_addr);
673         }
674         if (vcpu->arch.dtl.update_pending) {
675                 kvmppc_update_vpa(vcpu, &vcpu->arch.dtl);
676                 vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr;
677                 vcpu->arch.dtl_index = 0;
678         }
679         if (vcpu->arch.slb_shadow.update_pending)
680                 kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow);
681         spin_unlock(&vcpu->arch.vpa_update_lock);
682 }
683
684 /*
685  * Return the accumulated stolen time for the vcore up until `now'.
686  * The caller should hold the vcore lock.
687  */
688 static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now)
689 {
690         u64 p;
691         unsigned long flags;
692
693         spin_lock_irqsave(&vc->stoltb_lock, flags);
694         p = vc->stolen_tb;
695         if (vc->vcore_state != VCORE_INACTIVE &&
696             vc->preempt_tb != TB_NIL)
697                 p += now - vc->preempt_tb;
698         spin_unlock_irqrestore(&vc->stoltb_lock, flags);
699         return p;
700 }
701
702 static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu,
703                                     struct kvmppc_vcore *vc)
704 {
705         struct dtl_entry *dt;
706         struct lppaca *vpa;
707         unsigned long stolen;
708         unsigned long core_stolen;
709         u64 now;
710         unsigned long flags;
711
712         dt = vcpu->arch.dtl_ptr;
713         vpa = vcpu->arch.vpa.pinned_addr;
714         now = mftb();
715         core_stolen = vcore_stolen_time(vc, now);
716         stolen = core_stolen - vcpu->arch.stolen_logged;
717         vcpu->arch.stolen_logged = core_stolen;
718         spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
719         stolen += vcpu->arch.busy_stolen;
720         vcpu->arch.busy_stolen = 0;
721         spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
722         if (!dt || !vpa)
723                 return;
724         memset(dt, 0, sizeof(struct dtl_entry));
725         dt->dispatch_reason = 7;
726         dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid);
727         dt->timebase = cpu_to_be64(now + vc->tb_offset);
728         dt->enqueue_to_dispatch_time = cpu_to_be32(stolen);
729         dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu));
730         dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr);
731         ++dt;
732         if (dt == vcpu->arch.dtl.pinned_end)
733                 dt = vcpu->arch.dtl.pinned_addr;
734         vcpu->arch.dtl_ptr = dt;
735         /* order writing *dt vs. writing vpa->dtl_idx */
736         smp_wmb();
737         vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index);
738         vcpu->arch.dtl.dirty = true;
739 }
740
741 /* See if there is a doorbell interrupt pending for a vcpu */
742 static bool kvmppc_doorbell_pending(struct kvm_vcpu *vcpu)
743 {
744         int thr;
745         struct kvmppc_vcore *vc;
746
747         if (vcpu->arch.doorbell_request)
748                 return true;
749         /*
750          * Ensure that the read of vcore->dpdes comes after the read
751          * of vcpu->doorbell_request.  This barrier matches the
752          * smb_wmb() in kvmppc_guest_entry_inject().
753          */
754         smp_rmb();
755         vc = vcpu->arch.vcore;
756         thr = vcpu->vcpu_id - vc->first_vcpuid;
757         return !!(vc->dpdes & (1 << thr));
758 }
759
760 static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu)
761 {
762         if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207)
763                 return true;
764         if ((!vcpu->arch.vcore->arch_compat) &&
765             cpu_has_feature(CPU_FTR_ARCH_207S))
766                 return true;
767         return false;
768 }
769
770 static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags,
771                              unsigned long resource, unsigned long value1,
772                              unsigned long value2)
773 {
774         switch (resource) {
775         case H_SET_MODE_RESOURCE_SET_CIABR:
776                 if (!kvmppc_power8_compatible(vcpu))
777                         return H_P2;
778                 if (value2)
779                         return H_P4;
780                 if (mflags)
781                         return H_UNSUPPORTED_FLAG_START;
782                 /* Guests can't breakpoint the hypervisor */
783                 if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER)
784                         return H_P3;
785                 vcpu->arch.ciabr  = value1;
786                 return H_SUCCESS;
787         case H_SET_MODE_RESOURCE_SET_DAWR:
788                 if (!kvmppc_power8_compatible(vcpu))
789                         return H_P2;
790                 if (!ppc_breakpoint_available())
791                         return H_P2;
792                 if (mflags)
793                         return H_UNSUPPORTED_FLAG_START;
794                 if (value2 & DABRX_HYP)
795                         return H_P4;
796                 vcpu->arch.dawr  = value1;
797                 vcpu->arch.dawrx = value2;
798                 return H_SUCCESS;
799         default:
800                 return H_TOO_HARD;
801         }
802 }
803
804 static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target)
805 {
806         struct kvmppc_vcore *vcore = target->arch.vcore;
807
808         /*
809          * We expect to have been called by the real mode handler
810          * (kvmppc_rm_h_confer()) which would have directly returned
811          * H_SUCCESS if the source vcore wasn't idle (e.g. if it may
812          * have useful work to do and should not confer) so we don't
813          * recheck that here.
814          */
815
816         spin_lock(&vcore->lock);
817         if (target->arch.state == KVMPPC_VCPU_RUNNABLE &&
818             vcore->vcore_state != VCORE_INACTIVE &&
819             vcore->runner)
820                 target = vcore->runner;
821         spin_unlock(&vcore->lock);
822
823         return kvm_vcpu_yield_to(target);
824 }
825
826 static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu)
827 {
828         int yield_count = 0;
829         struct lppaca *lppaca;
830
831         spin_lock(&vcpu->arch.vpa_update_lock);
832         lppaca = (struct lppaca *)vcpu->arch.vpa.pinned_addr;
833         if (lppaca)
834                 yield_count = be32_to_cpu(lppaca->yield_count);
835         spin_unlock(&vcpu->arch.vpa_update_lock);
836         return yield_count;
837 }
838
839 int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
840 {
841         unsigned long req = kvmppc_get_gpr(vcpu, 3);
842         unsigned long target, ret = H_SUCCESS;
843         int yield_count;
844         struct kvm_vcpu *tvcpu;
845         int idx, rc;
846
847         if (req <= MAX_HCALL_OPCODE &&
848             !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls))
849                 return RESUME_HOST;
850
851         switch (req) {
852         case H_CEDE:
853                 break;
854         case H_PROD:
855                 target = kvmppc_get_gpr(vcpu, 4);
856                 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
857                 if (!tvcpu) {
858                         ret = H_PARAMETER;
859                         break;
860                 }
861                 tvcpu->arch.prodded = 1;
862                 smp_mb();
863                 if (tvcpu->arch.ceded)
864                         kvmppc_fast_vcpu_kick_hv(tvcpu);
865                 break;
866         case H_CONFER:
867                 target = kvmppc_get_gpr(vcpu, 4);
868                 if (target == -1)
869                         break;
870                 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
871                 if (!tvcpu) {
872                         ret = H_PARAMETER;
873                         break;
874                 }
875                 yield_count = kvmppc_get_gpr(vcpu, 5);
876                 if (kvmppc_get_yield_count(tvcpu) != yield_count)
877                         break;
878                 kvm_arch_vcpu_yield_to(tvcpu);
879                 break;
880         case H_REGISTER_VPA:
881                 ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4),
882                                         kvmppc_get_gpr(vcpu, 5),
883                                         kvmppc_get_gpr(vcpu, 6));
884                 break;
885         case H_RTAS:
886                 if (list_empty(&vcpu->kvm->arch.rtas_tokens))
887                         return RESUME_HOST;
888
889                 idx = srcu_read_lock(&vcpu->kvm->srcu);
890                 rc = kvmppc_rtas_hcall(vcpu);
891                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
892
893                 if (rc == -ENOENT)
894                         return RESUME_HOST;
895                 else if (rc == 0)
896                         break;
897
898                 /* Send the error out to userspace via KVM_RUN */
899                 return rc;
900         case H_LOGICAL_CI_LOAD:
901                 ret = kvmppc_h_logical_ci_load(vcpu);
902                 if (ret == H_TOO_HARD)
903                         return RESUME_HOST;
904                 break;
905         case H_LOGICAL_CI_STORE:
906                 ret = kvmppc_h_logical_ci_store(vcpu);
907                 if (ret == H_TOO_HARD)
908                         return RESUME_HOST;
909                 break;
910         case H_SET_MODE:
911                 ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4),
912                                         kvmppc_get_gpr(vcpu, 5),
913                                         kvmppc_get_gpr(vcpu, 6),
914                                         kvmppc_get_gpr(vcpu, 7));
915                 if (ret == H_TOO_HARD)
916                         return RESUME_HOST;
917                 break;
918         case H_XIRR:
919         case H_CPPR:
920         case H_EOI:
921         case H_IPI:
922         case H_IPOLL:
923         case H_XIRR_X:
924                 if (kvmppc_xics_enabled(vcpu)) {
925                         if (xive_enabled()) {
926                                 ret = H_NOT_AVAILABLE;
927                                 return RESUME_GUEST;
928                         }
929                         ret = kvmppc_xics_hcall(vcpu, req);
930                         break;
931                 }
932                 return RESUME_HOST;
933         case H_SET_DABR:
934                 ret = kvmppc_h_set_dabr(vcpu, kvmppc_get_gpr(vcpu, 4));
935                 break;
936         case H_SET_XDABR:
937                 ret = kvmppc_h_set_xdabr(vcpu, kvmppc_get_gpr(vcpu, 4),
938                                                 kvmppc_get_gpr(vcpu, 5));
939                 break;
940         case H_GET_TCE:
941                 ret = kvmppc_h_get_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
942                                                 kvmppc_get_gpr(vcpu, 5));
943                 if (ret == H_TOO_HARD)
944                         return RESUME_HOST;
945                 break;
946         case H_PUT_TCE:
947                 ret = kvmppc_h_put_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
948                                                 kvmppc_get_gpr(vcpu, 5),
949                                                 kvmppc_get_gpr(vcpu, 6));
950                 if (ret == H_TOO_HARD)
951                         return RESUME_HOST;
952                 break;
953         case H_PUT_TCE_INDIRECT:
954                 ret = kvmppc_h_put_tce_indirect(vcpu, kvmppc_get_gpr(vcpu, 4),
955                                                 kvmppc_get_gpr(vcpu, 5),
956                                                 kvmppc_get_gpr(vcpu, 6),
957                                                 kvmppc_get_gpr(vcpu, 7));
958                 if (ret == H_TOO_HARD)
959                         return RESUME_HOST;
960                 break;
961         case H_STUFF_TCE:
962                 ret = kvmppc_h_stuff_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
963                                                 kvmppc_get_gpr(vcpu, 5),
964                                                 kvmppc_get_gpr(vcpu, 6),
965                                                 kvmppc_get_gpr(vcpu, 7));
966                 if (ret == H_TOO_HARD)
967                         return RESUME_HOST;
968                 break;
969         case H_RANDOM:
970                 if (!powernv_get_random_long(&vcpu->arch.regs.gpr[4]))
971                         ret = H_HARDWARE;
972                 break;
973
974         case H_SET_PARTITION_TABLE:
975                 ret = H_FUNCTION;
976                 if (nesting_enabled(vcpu->kvm))
977                         ret = kvmhv_set_partition_table(vcpu);
978                 break;
979         case H_ENTER_NESTED:
980                 ret = H_FUNCTION;
981                 if (!nesting_enabled(vcpu->kvm))
982                         break;
983                 ret = kvmhv_enter_nested_guest(vcpu);
984                 if (ret == H_INTERRUPT) {
985                         kvmppc_set_gpr(vcpu, 3, 0);
986                         return -EINTR;
987                 }
988                 break;
989         case H_TLB_INVALIDATE:
990                 ret = H_FUNCTION;
991                 if (nesting_enabled(vcpu->kvm))
992                         ret = kvmhv_do_nested_tlbie(vcpu);
993                 break;
994
995         default:
996                 return RESUME_HOST;
997         }
998         kvmppc_set_gpr(vcpu, 3, ret);
999         vcpu->arch.hcall_needed = 0;
1000         return RESUME_GUEST;
1001 }
1002
1003 /*
1004  * Handle H_CEDE in the nested virtualization case where we haven't
1005  * called the real-mode hcall handlers in book3s_hv_rmhandlers.S.
1006  * This has to be done early, not in kvmppc_pseries_do_hcall(), so
1007  * that the cede logic in kvmppc_run_single_vcpu() works properly.
1008  */
1009 static void kvmppc_nested_cede(struct kvm_vcpu *vcpu)
1010 {
1011         vcpu->arch.shregs.msr |= MSR_EE;
1012         vcpu->arch.ceded = 1;
1013         smp_mb();
1014         if (vcpu->arch.prodded) {
1015                 vcpu->arch.prodded = 0;
1016                 smp_mb();
1017                 vcpu->arch.ceded = 0;
1018         }
1019 }
1020
1021 static int kvmppc_hcall_impl_hv(unsigned long cmd)
1022 {
1023         switch (cmd) {
1024         case H_CEDE:
1025         case H_PROD:
1026         case H_CONFER:
1027         case H_REGISTER_VPA:
1028         case H_SET_MODE:
1029         case H_LOGICAL_CI_LOAD:
1030         case H_LOGICAL_CI_STORE:
1031 #ifdef CONFIG_KVM_XICS
1032         case H_XIRR:
1033         case H_CPPR:
1034         case H_EOI:
1035         case H_IPI:
1036         case H_IPOLL:
1037         case H_XIRR_X:
1038 #endif
1039                 return 1;
1040         }
1041
1042         /* See if it's in the real-mode table */
1043         return kvmppc_hcall_impl_hv_realmode(cmd);
1044 }
1045
1046 static int kvmppc_emulate_debug_inst(struct kvm_run *run,
1047                                         struct kvm_vcpu *vcpu)
1048 {
1049         u32 last_inst;
1050
1051         if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
1052                                         EMULATE_DONE) {
1053                 /*
1054                  * Fetch failed, so return to guest and
1055                  * try executing it again.
1056                  */
1057                 return RESUME_GUEST;
1058         }
1059
1060         if (last_inst == KVMPPC_INST_SW_BREAKPOINT) {
1061                 run->exit_reason = KVM_EXIT_DEBUG;
1062                 run->debug.arch.address = kvmppc_get_pc(vcpu);
1063                 return RESUME_HOST;
1064         } else {
1065                 kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1066                 return RESUME_GUEST;
1067         }
1068 }
1069
1070 static void do_nothing(void *x)
1071 {
1072 }
1073
1074 static unsigned long kvmppc_read_dpdes(struct kvm_vcpu *vcpu)
1075 {
1076         int thr, cpu, pcpu, nthreads;
1077         struct kvm_vcpu *v;
1078         unsigned long dpdes;
1079
1080         nthreads = vcpu->kvm->arch.emul_smt_mode;
1081         dpdes = 0;
1082         cpu = vcpu->vcpu_id & ~(nthreads - 1);
1083         for (thr = 0; thr < nthreads; ++thr, ++cpu) {
1084                 v = kvmppc_find_vcpu(vcpu->kvm, cpu);
1085                 if (!v)
1086                         continue;
1087                 /*
1088                  * If the vcpu is currently running on a physical cpu thread,
1089                  * interrupt it in order to pull it out of the guest briefly,
1090                  * which will update its vcore->dpdes value.
1091                  */
1092                 pcpu = READ_ONCE(v->cpu);
1093                 if (pcpu >= 0)
1094                         smp_call_function_single(pcpu, do_nothing, NULL, 1);
1095                 if (kvmppc_doorbell_pending(v))
1096                         dpdes |= 1 << thr;
1097         }
1098         return dpdes;
1099 }
1100
1101 /*
1102  * On POWER9, emulate doorbell-related instructions in order to
1103  * give the guest the illusion of running on a multi-threaded core.
1104  * The instructions emulated are msgsndp, msgclrp, mfspr TIR,
1105  * and mfspr DPDES.
1106  */
1107 static int kvmppc_emulate_doorbell_instr(struct kvm_vcpu *vcpu)
1108 {
1109         u32 inst, rb, thr;
1110         unsigned long arg;
1111         struct kvm *kvm = vcpu->kvm;
1112         struct kvm_vcpu *tvcpu;
1113
1114         if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &inst) != EMULATE_DONE)
1115                 return RESUME_GUEST;
1116         if (get_op(inst) != 31)
1117                 return EMULATE_FAIL;
1118         rb = get_rb(inst);
1119         thr = vcpu->vcpu_id & (kvm->arch.emul_smt_mode - 1);
1120         switch (get_xop(inst)) {
1121         case OP_31_XOP_MSGSNDP:
1122                 arg = kvmppc_get_gpr(vcpu, rb);
1123                 if (((arg >> 27) & 0xf) != PPC_DBELL_SERVER)
1124                         break;
1125                 arg &= 0x3f;
1126                 if (arg >= kvm->arch.emul_smt_mode)
1127                         break;
1128                 tvcpu = kvmppc_find_vcpu(kvm, vcpu->vcpu_id - thr + arg);
1129                 if (!tvcpu)
1130                         break;
1131                 if (!tvcpu->arch.doorbell_request) {
1132                         tvcpu->arch.doorbell_request = 1;
1133                         kvmppc_fast_vcpu_kick_hv(tvcpu);
1134                 }
1135                 break;
1136         case OP_31_XOP_MSGCLRP:
1137                 arg = kvmppc_get_gpr(vcpu, rb);
1138                 if (((arg >> 27) & 0xf) != PPC_DBELL_SERVER)
1139                         break;
1140                 vcpu->arch.vcore->dpdes = 0;
1141                 vcpu->arch.doorbell_request = 0;
1142                 break;
1143         case OP_31_XOP_MFSPR:
1144                 switch (get_sprn(inst)) {
1145                 case SPRN_TIR:
1146                         arg = thr;
1147                         break;
1148                 case SPRN_DPDES:
1149                         arg = kvmppc_read_dpdes(vcpu);
1150                         break;
1151                 default:
1152                         return EMULATE_FAIL;
1153                 }
1154                 kvmppc_set_gpr(vcpu, get_rt(inst), arg);
1155                 break;
1156         default:
1157                 return EMULATE_FAIL;
1158         }
1159         kvmppc_set_pc(vcpu, kvmppc_get_pc(vcpu) + 4);
1160         return RESUME_GUEST;
1161 }
1162
1163 static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
1164                                  struct task_struct *tsk)
1165 {
1166         int r = RESUME_HOST;
1167
1168         vcpu->stat.sum_exits++;
1169
1170         /*
1171          * This can happen if an interrupt occurs in the last stages
1172          * of guest entry or the first stages of guest exit (i.e. after
1173          * setting paca->kvm_hstate.in_guest to KVM_GUEST_MODE_GUEST_HV
1174          * and before setting it to KVM_GUEST_MODE_HOST_HV).
1175          * That can happen due to a bug, or due to a machine check
1176          * occurring at just the wrong time.
1177          */
1178         if (vcpu->arch.shregs.msr & MSR_HV) {
1179                 printk(KERN_EMERG "KVM trap in HV mode!\n");
1180                 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1181                         vcpu->arch.trap, kvmppc_get_pc(vcpu),
1182                         vcpu->arch.shregs.msr);
1183                 kvmppc_dump_regs(vcpu);
1184                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1185                 run->hw.hardware_exit_reason = vcpu->arch.trap;
1186                 return RESUME_HOST;
1187         }
1188         run->exit_reason = KVM_EXIT_UNKNOWN;
1189         run->ready_for_interrupt_injection = 1;
1190         switch (vcpu->arch.trap) {
1191         /* We're good on these - the host merely wanted to get our attention */
1192         case BOOK3S_INTERRUPT_HV_DECREMENTER:
1193                 vcpu->stat.dec_exits++;
1194                 r = RESUME_GUEST;
1195                 break;
1196         case BOOK3S_INTERRUPT_EXTERNAL:
1197         case BOOK3S_INTERRUPT_H_DOORBELL:
1198         case BOOK3S_INTERRUPT_H_VIRT:
1199                 vcpu->stat.ext_intr_exits++;
1200                 r = RESUME_GUEST;
1201                 break;
1202         /* SR/HMI/PMI are HV interrupts that host has handled. Resume guest.*/
1203         case BOOK3S_INTERRUPT_HMI:
1204         case BOOK3S_INTERRUPT_PERFMON:
1205         case BOOK3S_INTERRUPT_SYSTEM_RESET:
1206                 r = RESUME_GUEST;
1207                 break;
1208         case BOOK3S_INTERRUPT_MACHINE_CHECK:
1209                 /* Exit to guest with KVM_EXIT_NMI as exit reason */
1210                 run->exit_reason = KVM_EXIT_NMI;
1211                 run->hw.hardware_exit_reason = vcpu->arch.trap;
1212                 /* Clear out the old NMI status from run->flags */
1213                 run->flags &= ~KVM_RUN_PPC_NMI_DISP_MASK;
1214                 /* Now set the NMI status */
1215                 if (vcpu->arch.mce_evt.disposition == MCE_DISPOSITION_RECOVERED)
1216                         run->flags |= KVM_RUN_PPC_NMI_DISP_FULLY_RECOV;
1217                 else
1218                         run->flags |= KVM_RUN_PPC_NMI_DISP_NOT_RECOV;
1219
1220                 r = RESUME_HOST;
1221                 /* Print the MCE event to host console. */
1222                 machine_check_print_event_info(&vcpu->arch.mce_evt, false);
1223                 break;
1224         case BOOK3S_INTERRUPT_PROGRAM:
1225         {
1226                 ulong flags;
1227                 /*
1228                  * Normally program interrupts are delivered directly
1229                  * to the guest by the hardware, but we can get here
1230                  * as a result of a hypervisor emulation interrupt
1231                  * (e40) getting turned into a 700 by BML RTAS.
1232                  */
1233                 flags = vcpu->arch.shregs.msr & 0x1f0000ull;
1234                 kvmppc_core_queue_program(vcpu, flags);
1235                 r = RESUME_GUEST;
1236                 break;
1237         }
1238         case BOOK3S_INTERRUPT_SYSCALL:
1239         {
1240                 /* hcall - punt to userspace */
1241                 int i;
1242
1243                 /* hypercall with MSR_PR has already been handled in rmode,
1244                  * and never reaches here.
1245                  */
1246
1247                 run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3);
1248                 for (i = 0; i < 9; ++i)
1249                         run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i);
1250                 run->exit_reason = KVM_EXIT_PAPR_HCALL;
1251                 vcpu->arch.hcall_needed = 1;
1252                 r = RESUME_HOST;
1253                 break;
1254         }
1255         /*
1256          * We get these next two if the guest accesses a page which it thinks
1257          * it has mapped but which is not actually present, either because
1258          * it is for an emulated I/O device or because the corresonding
1259          * host page has been paged out.  Any other HDSI/HISI interrupts
1260          * have been handled already.
1261          */
1262         case BOOK3S_INTERRUPT_H_DATA_STORAGE:
1263                 r = RESUME_PAGE_FAULT;
1264                 break;
1265         case BOOK3S_INTERRUPT_H_INST_STORAGE:
1266                 vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
1267                 vcpu->arch.fault_dsisr = vcpu->arch.shregs.msr &
1268                         DSISR_SRR1_MATCH_64S;
1269                 if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE)
1270                         vcpu->arch.fault_dsisr |= DSISR_ISSTORE;
1271                 r = RESUME_PAGE_FAULT;
1272                 break;
1273         /*
1274          * This occurs if the guest executes an illegal instruction.
1275          * If the guest debug is disabled, generate a program interrupt
1276          * to the guest. If guest debug is enabled, we need to check
1277          * whether the instruction is a software breakpoint instruction.
1278          * Accordingly return to Guest or Host.
1279          */
1280         case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
1281                 if (vcpu->arch.emul_inst != KVM_INST_FETCH_FAILED)
1282                         vcpu->arch.last_inst = kvmppc_need_byteswap(vcpu) ?
1283                                 swab32(vcpu->arch.emul_inst) :
1284                                 vcpu->arch.emul_inst;
1285                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) {
1286                         r = kvmppc_emulate_debug_inst(run, vcpu);
1287                 } else {
1288                         kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1289                         r = RESUME_GUEST;
1290                 }
1291                 break;
1292         /*
1293          * This occurs if the guest (kernel or userspace), does something that
1294          * is prohibited by HFSCR.
1295          * On POWER9, this could be a doorbell instruction that we need
1296          * to emulate.
1297          * Otherwise, we just generate a program interrupt to the guest.
1298          */
1299         case BOOK3S_INTERRUPT_H_FAC_UNAVAIL:
1300                 r = EMULATE_FAIL;
1301                 if (((vcpu->arch.hfscr >> 56) == FSCR_MSGP_LG) &&
1302                     cpu_has_feature(CPU_FTR_ARCH_300))
1303                         r = kvmppc_emulate_doorbell_instr(vcpu);
1304                 if (r == EMULATE_FAIL) {
1305                         kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1306                         r = RESUME_GUEST;
1307                 }
1308                 break;
1309
1310 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1311         case BOOK3S_INTERRUPT_HV_SOFTPATCH:
1312                 /*
1313                  * This occurs for various TM-related instructions that
1314                  * we need to emulate on POWER9 DD2.2.  We have already
1315                  * handled the cases where the guest was in real-suspend
1316                  * mode and was transitioning to transactional state.
1317                  */
1318                 r = kvmhv_p9_tm_emulation(vcpu);
1319                 break;
1320 #endif
1321
1322         case BOOK3S_INTERRUPT_HV_RM_HARD:
1323                 r = RESUME_PASSTHROUGH;
1324                 break;
1325         default:
1326                 kvmppc_dump_regs(vcpu);
1327                 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1328                         vcpu->arch.trap, kvmppc_get_pc(vcpu),
1329                         vcpu->arch.shregs.msr);
1330                 run->hw.hardware_exit_reason = vcpu->arch.trap;
1331                 r = RESUME_HOST;
1332                 break;
1333         }
1334
1335         return r;
1336 }
1337
1338 static int kvmppc_handle_nested_exit(struct kvm_vcpu *vcpu)
1339 {
1340         int r;
1341         int srcu_idx;
1342
1343         vcpu->stat.sum_exits++;
1344
1345         /*
1346          * This can happen if an interrupt occurs in the last stages
1347          * of guest entry or the first stages of guest exit (i.e. after
1348          * setting paca->kvm_hstate.in_guest to KVM_GUEST_MODE_GUEST_HV
1349          * and before setting it to KVM_GUEST_MODE_HOST_HV).
1350          * That can happen due to a bug, or due to a machine check
1351          * occurring at just the wrong time.
1352          */
1353         if (vcpu->arch.shregs.msr & MSR_HV) {
1354                 pr_emerg("KVM trap in HV mode while nested!\n");
1355                 pr_emerg("trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1356                          vcpu->arch.trap, kvmppc_get_pc(vcpu),
1357                          vcpu->arch.shregs.msr);
1358                 kvmppc_dump_regs(vcpu);
1359                 return RESUME_HOST;
1360         }
1361         switch (vcpu->arch.trap) {
1362         /* We're good on these - the host merely wanted to get our attention */
1363         case BOOK3S_INTERRUPT_HV_DECREMENTER:
1364                 vcpu->stat.dec_exits++;
1365                 r = RESUME_GUEST;
1366                 break;
1367         case BOOK3S_INTERRUPT_EXTERNAL:
1368                 vcpu->stat.ext_intr_exits++;
1369                 r = RESUME_HOST;
1370                 break;
1371         case BOOK3S_INTERRUPT_H_DOORBELL:
1372         case BOOK3S_INTERRUPT_H_VIRT:
1373                 vcpu->stat.ext_intr_exits++;
1374                 r = RESUME_GUEST;
1375                 break;
1376         /* SR/HMI/PMI are HV interrupts that host has handled. Resume guest.*/
1377         case BOOK3S_INTERRUPT_HMI:
1378         case BOOK3S_INTERRUPT_PERFMON:
1379         case BOOK3S_INTERRUPT_SYSTEM_RESET:
1380                 r = RESUME_GUEST;
1381                 break;
1382         case BOOK3S_INTERRUPT_MACHINE_CHECK:
1383                 /* Pass the machine check to the L1 guest */
1384                 r = RESUME_HOST;
1385                 /* Print the MCE event to host console. */
1386                 machine_check_print_event_info(&vcpu->arch.mce_evt, false);
1387                 break;
1388         /*
1389          * We get these next two if the guest accesses a page which it thinks
1390          * it has mapped but which is not actually present, either because
1391          * it is for an emulated I/O device or because the corresonding
1392          * host page has been paged out.
1393          */
1394         case BOOK3S_INTERRUPT_H_DATA_STORAGE:
1395                 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1396                 r = kvmhv_nested_page_fault(vcpu);
1397                 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
1398                 break;
1399         case BOOK3S_INTERRUPT_H_INST_STORAGE:
1400                 vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
1401                 vcpu->arch.fault_dsisr = kvmppc_get_msr(vcpu) &
1402                                          DSISR_SRR1_MATCH_64S;
1403                 if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE)
1404                         vcpu->arch.fault_dsisr |= DSISR_ISSTORE;
1405                 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1406                 r = kvmhv_nested_page_fault(vcpu);
1407                 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
1408                 break;
1409
1410 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1411         case BOOK3S_INTERRUPT_HV_SOFTPATCH:
1412                 /*
1413                  * This occurs for various TM-related instructions that
1414                  * we need to emulate on POWER9 DD2.2.  We have already
1415                  * handled the cases where the guest was in real-suspend
1416                  * mode and was transitioning to transactional state.
1417                  */
1418                 r = kvmhv_p9_tm_emulation(vcpu);
1419                 break;
1420 #endif
1421
1422         case BOOK3S_INTERRUPT_HV_RM_HARD:
1423                 vcpu->arch.trap = 0;
1424                 r = RESUME_GUEST;
1425                 if (!xive_enabled())
1426                         kvmppc_xics_rm_complete(vcpu, 0);
1427                 break;
1428         default:
1429                 r = RESUME_HOST;
1430                 break;
1431         }
1432
1433         return r;
1434 }
1435
1436 static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu,
1437                                             struct kvm_sregs *sregs)
1438 {
1439         int i;
1440
1441         memset(sregs, 0, sizeof(struct kvm_sregs));
1442         sregs->pvr = vcpu->arch.pvr;
1443         for (i = 0; i < vcpu->arch.slb_max; i++) {
1444                 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige;
1445                 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
1446         }
1447
1448         return 0;
1449 }
1450
1451 static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu,
1452                                             struct kvm_sregs *sregs)
1453 {
1454         int i, j;
1455
1456         /* Only accept the same PVR as the host's, since we can't spoof it */
1457         if (sregs->pvr != vcpu->arch.pvr)
1458                 return -EINVAL;
1459
1460         j = 0;
1461         for (i = 0; i < vcpu->arch.slb_nr; i++) {
1462                 if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) {
1463                         vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe;
1464                         vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv;
1465                         ++j;
1466                 }
1467         }
1468         vcpu->arch.slb_max = j;
1469
1470         return 0;
1471 }
1472
1473 static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr,
1474                 bool preserve_top32)
1475 {
1476         struct kvm *kvm = vcpu->kvm;
1477         struct kvmppc_vcore *vc = vcpu->arch.vcore;
1478         u64 mask;
1479
1480         mutex_lock(&kvm->lock);
1481         spin_lock(&vc->lock);
1482         /*
1483          * If ILE (interrupt little-endian) has changed, update the
1484          * MSR_LE bit in the intr_msr for each vcpu in this vcore.
1485          */
1486         if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) {
1487                 struct kvm_vcpu *vcpu;
1488                 int i;
1489
1490                 kvm_for_each_vcpu(i, vcpu, kvm) {
1491                         if (vcpu->arch.vcore != vc)
1492                                 continue;
1493                         if (new_lpcr & LPCR_ILE)
1494                                 vcpu->arch.intr_msr |= MSR_LE;
1495                         else
1496                                 vcpu->arch.intr_msr &= ~MSR_LE;
1497                 }
1498         }
1499
1500         /*
1501          * Userspace can only modify DPFD (default prefetch depth),
1502          * ILE (interrupt little-endian) and TC (translation control).
1503          * On POWER8 and POWER9 userspace can also modify AIL (alt. interrupt loc.).
1504          */
1505         mask = LPCR_DPFD | LPCR_ILE | LPCR_TC;
1506         if (cpu_has_feature(CPU_FTR_ARCH_207S))
1507                 mask |= LPCR_AIL;
1508         /*
1509          * On POWER9, allow userspace to enable large decrementer for the
1510          * guest, whether or not the host has it enabled.
1511          */
1512         if (cpu_has_feature(CPU_FTR_ARCH_300))
1513                 mask |= LPCR_LD;
1514
1515         /* Broken 32-bit version of LPCR must not clear top bits */
1516         if (preserve_top32)
1517                 mask &= 0xFFFFFFFF;
1518         vc->lpcr = (vc->lpcr & ~mask) | (new_lpcr & mask);
1519         spin_unlock(&vc->lock);
1520         mutex_unlock(&kvm->lock);
1521 }
1522
1523 static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
1524                                  union kvmppc_one_reg *val)
1525 {
1526         int r = 0;
1527         long int i;
1528
1529         switch (id) {
1530         case KVM_REG_PPC_DEBUG_INST:
1531                 *val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT);
1532                 break;
1533         case KVM_REG_PPC_HIOR:
1534                 *val = get_reg_val(id, 0);
1535                 break;
1536         case KVM_REG_PPC_DABR:
1537                 *val = get_reg_val(id, vcpu->arch.dabr);
1538                 break;
1539         case KVM_REG_PPC_DABRX:
1540                 *val = get_reg_val(id, vcpu->arch.dabrx);
1541                 break;
1542         case KVM_REG_PPC_DSCR:
1543                 *val = get_reg_val(id, vcpu->arch.dscr);
1544                 break;
1545         case KVM_REG_PPC_PURR:
1546                 *val = get_reg_val(id, vcpu->arch.purr);
1547                 break;
1548         case KVM_REG_PPC_SPURR:
1549                 *val = get_reg_val(id, vcpu->arch.spurr);
1550                 break;
1551         case KVM_REG_PPC_AMR:
1552                 *val = get_reg_val(id, vcpu->arch.amr);
1553                 break;
1554         case KVM_REG_PPC_UAMOR:
1555                 *val = get_reg_val(id, vcpu->arch.uamor);
1556                 break;
1557         case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
1558                 i = id - KVM_REG_PPC_MMCR0;
1559                 *val = get_reg_val(id, vcpu->arch.mmcr[i]);
1560                 break;
1561         case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
1562                 i = id - KVM_REG_PPC_PMC1;
1563                 *val = get_reg_val(id, vcpu->arch.pmc[i]);
1564                 break;
1565         case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
1566                 i = id - KVM_REG_PPC_SPMC1;
1567                 *val = get_reg_val(id, vcpu->arch.spmc[i]);
1568                 break;
1569         case KVM_REG_PPC_SIAR:
1570                 *val = get_reg_val(id, vcpu->arch.siar);
1571                 break;
1572         case KVM_REG_PPC_SDAR:
1573                 *val = get_reg_val(id, vcpu->arch.sdar);
1574                 break;
1575         case KVM_REG_PPC_SIER:
1576                 *val = get_reg_val(id, vcpu->arch.sier);
1577                 break;
1578         case KVM_REG_PPC_IAMR:
1579                 *val = get_reg_val(id, vcpu->arch.iamr);
1580                 break;
1581         case KVM_REG_PPC_PSPB:
1582                 *val = get_reg_val(id, vcpu->arch.pspb);
1583                 break;
1584         case KVM_REG_PPC_DPDES:
1585                 *val = get_reg_val(id, vcpu->arch.vcore->dpdes);
1586                 break;
1587         case KVM_REG_PPC_VTB:
1588                 *val = get_reg_val(id, vcpu->arch.vcore->vtb);
1589                 break;
1590         case KVM_REG_PPC_DAWR:
1591                 *val = get_reg_val(id, vcpu->arch.dawr);
1592                 break;
1593         case KVM_REG_PPC_DAWRX:
1594                 *val = get_reg_val(id, vcpu->arch.dawrx);
1595                 break;
1596         case KVM_REG_PPC_CIABR:
1597                 *val = get_reg_val(id, vcpu->arch.ciabr);
1598                 break;
1599         case KVM_REG_PPC_CSIGR:
1600                 *val = get_reg_val(id, vcpu->arch.csigr);
1601                 break;
1602         case KVM_REG_PPC_TACR:
1603                 *val = get_reg_val(id, vcpu->arch.tacr);
1604                 break;
1605         case KVM_REG_PPC_TCSCR:
1606                 *val = get_reg_val(id, vcpu->arch.tcscr);
1607                 break;
1608         case KVM_REG_PPC_PID:
1609                 *val = get_reg_val(id, vcpu->arch.pid);
1610                 break;
1611         case KVM_REG_PPC_ACOP:
1612                 *val = get_reg_val(id, vcpu->arch.acop);
1613                 break;
1614         case KVM_REG_PPC_WORT:
1615                 *val = get_reg_val(id, vcpu->arch.wort);
1616                 break;
1617         case KVM_REG_PPC_TIDR:
1618                 *val = get_reg_val(id, vcpu->arch.tid);
1619                 break;
1620         case KVM_REG_PPC_PSSCR:
1621                 *val = get_reg_val(id, vcpu->arch.psscr);
1622                 break;
1623         case KVM_REG_PPC_VPA_ADDR:
1624                 spin_lock(&vcpu->arch.vpa_update_lock);
1625                 *val = get_reg_val(id, vcpu->arch.vpa.next_gpa);
1626                 spin_unlock(&vcpu->arch.vpa_update_lock);
1627                 break;
1628         case KVM_REG_PPC_VPA_SLB:
1629                 spin_lock(&vcpu->arch.vpa_update_lock);
1630                 val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa;
1631                 val->vpaval.length = vcpu->arch.slb_shadow.len;
1632                 spin_unlock(&vcpu->arch.vpa_update_lock);
1633                 break;
1634         case KVM_REG_PPC_VPA_DTL:
1635                 spin_lock(&vcpu->arch.vpa_update_lock);
1636                 val->vpaval.addr = vcpu->arch.dtl.next_gpa;
1637                 val->vpaval.length = vcpu->arch.dtl.len;
1638                 spin_unlock(&vcpu->arch.vpa_update_lock);
1639                 break;
1640         case KVM_REG_PPC_TB_OFFSET:
1641                 *val = get_reg_val(id, vcpu->arch.vcore->tb_offset);
1642                 break;
1643         case KVM_REG_PPC_LPCR:
1644         case KVM_REG_PPC_LPCR_64:
1645                 *val = get_reg_val(id, vcpu->arch.vcore->lpcr);
1646                 break;
1647         case KVM_REG_PPC_PPR:
1648                 *val = get_reg_val(id, vcpu->arch.ppr);
1649                 break;
1650 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1651         case KVM_REG_PPC_TFHAR:
1652                 *val = get_reg_val(id, vcpu->arch.tfhar);
1653                 break;
1654         case KVM_REG_PPC_TFIAR:
1655                 *val = get_reg_val(id, vcpu->arch.tfiar);
1656                 break;
1657         case KVM_REG_PPC_TEXASR:
1658                 *val = get_reg_val(id, vcpu->arch.texasr);
1659                 break;
1660         case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
1661                 i = id - KVM_REG_PPC_TM_GPR0;
1662                 *val = get_reg_val(id, vcpu->arch.gpr_tm[i]);
1663                 break;
1664         case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
1665         {
1666                 int j;
1667                 i = id - KVM_REG_PPC_TM_VSR0;
1668                 if (i < 32)
1669                         for (j = 0; j < TS_FPRWIDTH; j++)
1670                                 val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j];
1671                 else {
1672                         if (cpu_has_feature(CPU_FTR_ALTIVEC))
1673                                 val->vval = vcpu->arch.vr_tm.vr[i-32];
1674                         else
1675                                 r = -ENXIO;
1676                 }
1677                 break;
1678         }
1679         case KVM_REG_PPC_TM_CR:
1680                 *val = get_reg_val(id, vcpu->arch.cr_tm);
1681                 break;
1682         case KVM_REG_PPC_TM_XER:
1683                 *val = get_reg_val(id, vcpu->arch.xer_tm);
1684                 break;
1685         case KVM_REG_PPC_TM_LR:
1686                 *val = get_reg_val(id, vcpu->arch.lr_tm);
1687                 break;
1688         case KVM_REG_PPC_TM_CTR:
1689                 *val = get_reg_val(id, vcpu->arch.ctr_tm);
1690                 break;
1691         case KVM_REG_PPC_TM_FPSCR:
1692                 *val = get_reg_val(id, vcpu->arch.fp_tm.fpscr);
1693                 break;
1694         case KVM_REG_PPC_TM_AMR:
1695                 *val = get_reg_val(id, vcpu->arch.amr_tm);
1696                 break;
1697         case KVM_REG_PPC_TM_PPR:
1698                 *val = get_reg_val(id, vcpu->arch.ppr_tm);
1699                 break;
1700         case KVM_REG_PPC_TM_VRSAVE:
1701                 *val = get_reg_val(id, vcpu->arch.vrsave_tm);
1702                 break;
1703         case KVM_REG_PPC_TM_VSCR:
1704                 if (cpu_has_feature(CPU_FTR_ALTIVEC))
1705                         *val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]);
1706                 else
1707                         r = -ENXIO;
1708                 break;
1709         case KVM_REG_PPC_TM_DSCR:
1710                 *val = get_reg_val(id, vcpu->arch.dscr_tm);
1711                 break;
1712         case KVM_REG_PPC_TM_TAR:
1713                 *val = get_reg_val(id, vcpu->arch.tar_tm);
1714                 break;
1715 #endif
1716         case KVM_REG_PPC_ARCH_COMPAT:
1717                 *val = get_reg_val(id, vcpu->arch.vcore->arch_compat);
1718                 break;
1719         case KVM_REG_PPC_DEC_EXPIRY:
1720                 *val = get_reg_val(id, vcpu->arch.dec_expires +
1721                                    vcpu->arch.vcore->tb_offset);
1722                 break;
1723         case KVM_REG_PPC_ONLINE:
1724                 *val = get_reg_val(id, vcpu->arch.online);
1725                 break;
1726         case KVM_REG_PPC_PTCR:
1727                 *val = get_reg_val(id, vcpu->kvm->arch.l1_ptcr);
1728                 break;
1729         default:
1730                 r = -EINVAL;
1731                 break;
1732         }
1733
1734         return r;
1735 }
1736
1737 static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
1738                                  union kvmppc_one_reg *val)
1739 {
1740         int r = 0;
1741         long int i;
1742         unsigned long addr, len;
1743
1744         switch (id) {
1745         case KVM_REG_PPC_HIOR:
1746                 /* Only allow this to be set to zero */
1747                 if (set_reg_val(id, *val))
1748                         r = -EINVAL;
1749                 break;
1750         case KVM_REG_PPC_DABR:
1751                 vcpu->arch.dabr = set_reg_val(id, *val);
1752                 break;
1753         case KVM_REG_PPC_DABRX:
1754                 vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP;
1755                 break;
1756         case KVM_REG_PPC_DSCR:
1757                 vcpu->arch.dscr = set_reg_val(id, *val);
1758                 break;
1759         case KVM_REG_PPC_PURR:
1760                 vcpu->arch.purr = set_reg_val(id, *val);
1761                 break;
1762         case KVM_REG_PPC_SPURR:
1763                 vcpu->arch.spurr = set_reg_val(id, *val);
1764                 break;
1765         case KVM_REG_PPC_AMR:
1766                 vcpu->arch.amr = set_reg_val(id, *val);
1767                 break;
1768         case KVM_REG_PPC_UAMOR:
1769                 vcpu->arch.uamor = set_reg_val(id, *val);
1770                 break;
1771         case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
1772                 i = id - KVM_REG_PPC_MMCR0;
1773                 vcpu->arch.mmcr[i] = set_reg_val(id, *val);
1774                 break;
1775         case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
1776                 i = id - KVM_REG_PPC_PMC1;
1777                 vcpu->arch.pmc[i] = set_reg_val(id, *val);
1778                 break;
1779         case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
1780                 i = id - KVM_REG_PPC_SPMC1;
1781                 vcpu->arch.spmc[i] = set_reg_val(id, *val);
1782                 break;
1783         case KVM_REG_PPC_SIAR:
1784                 vcpu->arch.siar = set_reg_val(id, *val);
1785                 break;
1786         case KVM_REG_PPC_SDAR:
1787                 vcpu->arch.sdar = set_reg_val(id, *val);
1788                 break;
1789         case KVM_REG_PPC_SIER:
1790                 vcpu->arch.sier = set_reg_val(id, *val);
1791                 break;
1792         case KVM_REG_PPC_IAMR:
1793                 vcpu->arch.iamr = set_reg_val(id, *val);
1794                 break;
1795         case KVM_REG_PPC_PSPB:
1796                 vcpu->arch.pspb = set_reg_val(id, *val);
1797                 break;
1798         case KVM_REG_PPC_DPDES:
1799                 vcpu->arch.vcore->dpdes = set_reg_val(id, *val);
1800                 break;
1801         case KVM_REG_PPC_VTB:
1802                 vcpu->arch.vcore->vtb = set_reg_val(id, *val);
1803                 break;
1804         case KVM_REG_PPC_DAWR:
1805                 vcpu->arch.dawr = set_reg_val(id, *val);
1806                 break;
1807         case KVM_REG_PPC_DAWRX:
1808                 vcpu->arch.dawrx = set_reg_val(id, *val) & ~DAWRX_HYP;
1809                 break;
1810         case KVM_REG_PPC_CIABR:
1811                 vcpu->arch.ciabr = set_reg_val(id, *val);
1812                 /* Don't allow setting breakpoints in hypervisor code */
1813                 if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER)
1814                         vcpu->arch.ciabr &= ~CIABR_PRIV;        /* disable */
1815                 break;
1816         case KVM_REG_PPC_CSIGR:
1817                 vcpu->arch.csigr = set_reg_val(id, *val);
1818                 break;
1819         case KVM_REG_PPC_TACR:
1820                 vcpu->arch.tacr = set_reg_val(id, *val);
1821                 break;
1822         case KVM_REG_PPC_TCSCR:
1823                 vcpu->arch.tcscr = set_reg_val(id, *val);
1824                 break;
1825         case KVM_REG_PPC_PID:
1826                 vcpu->arch.pid = set_reg_val(id, *val);
1827                 break;
1828         case KVM_REG_PPC_ACOP:
1829                 vcpu->arch.acop = set_reg_val(id, *val);
1830                 break;
1831         case KVM_REG_PPC_WORT:
1832                 vcpu->arch.wort = set_reg_val(id, *val);
1833                 break;
1834         case KVM_REG_PPC_TIDR:
1835                 vcpu->arch.tid = set_reg_val(id, *val);
1836                 break;
1837         case KVM_REG_PPC_PSSCR:
1838                 vcpu->arch.psscr = set_reg_val(id, *val) & PSSCR_GUEST_VIS;
1839                 break;
1840         case KVM_REG_PPC_VPA_ADDR:
1841                 addr = set_reg_val(id, *val);
1842                 r = -EINVAL;
1843                 if (!addr && (vcpu->arch.slb_shadow.next_gpa ||
1844                               vcpu->arch.dtl.next_gpa))
1845                         break;
1846                 r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca));
1847                 break;
1848         case KVM_REG_PPC_VPA_SLB:
1849                 addr = val->vpaval.addr;
1850                 len = val->vpaval.length;
1851                 r = -EINVAL;
1852                 if (addr && !vcpu->arch.vpa.next_gpa)
1853                         break;
1854                 r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len);
1855                 break;
1856         case KVM_REG_PPC_VPA_DTL:
1857                 addr = val->vpaval.addr;
1858                 len = val->vpaval.length;
1859                 r = -EINVAL;
1860                 if (addr && (len < sizeof(struct dtl_entry) ||
1861                              !vcpu->arch.vpa.next_gpa))
1862                         break;
1863                 len -= len % sizeof(struct dtl_entry);
1864                 r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len);
1865                 break;
1866         case KVM_REG_PPC_TB_OFFSET:
1867                 /* round up to multiple of 2^24 */
1868                 vcpu->arch.vcore->tb_offset =
1869                         ALIGN(set_reg_val(id, *val), 1UL << 24);
1870                 break;
1871         case KVM_REG_PPC_LPCR:
1872                 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true);
1873                 break;
1874         case KVM_REG_PPC_LPCR_64:
1875                 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false);
1876                 break;
1877         case KVM_REG_PPC_PPR:
1878                 vcpu->arch.ppr = set_reg_val(id, *val);
1879                 break;
1880 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1881         case KVM_REG_PPC_TFHAR:
1882                 vcpu->arch.tfhar = set_reg_val(id, *val);
1883                 break;
1884         case KVM_REG_PPC_TFIAR:
1885                 vcpu->arch.tfiar = set_reg_val(id, *val);
1886                 break;
1887         case KVM_REG_PPC_TEXASR:
1888                 vcpu->arch.texasr = set_reg_val(id, *val);
1889                 break;
1890         case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
1891                 i = id - KVM_REG_PPC_TM_GPR0;
1892                 vcpu->arch.gpr_tm[i] = set_reg_val(id, *val);
1893                 break;
1894         case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
1895         {
1896                 int j;
1897                 i = id - KVM_REG_PPC_TM_VSR0;
1898                 if (i < 32)
1899                         for (j = 0; j < TS_FPRWIDTH; j++)
1900                                 vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j];
1901                 else
1902                         if (cpu_has_feature(CPU_FTR_ALTIVEC))
1903                                 vcpu->arch.vr_tm.vr[i-32] = val->vval;
1904                         else
1905                                 r = -ENXIO;
1906                 break;
1907         }
1908         case KVM_REG_PPC_TM_CR:
1909                 vcpu->arch.cr_tm = set_reg_val(id, *val);
1910                 break;
1911         case KVM_REG_PPC_TM_XER:
1912                 vcpu->arch.xer_tm = set_reg_val(id, *val);
1913                 break;
1914         case KVM_REG_PPC_TM_LR:
1915                 vcpu->arch.lr_tm = set_reg_val(id, *val);
1916                 break;
1917         case KVM_REG_PPC_TM_CTR:
1918                 vcpu->arch.ctr_tm = set_reg_val(id, *val);
1919                 break;
1920         case KVM_REG_PPC_TM_FPSCR:
1921                 vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val);
1922                 break;
1923         case KVM_REG_PPC_TM_AMR:
1924                 vcpu->arch.amr_tm = set_reg_val(id, *val);
1925                 break;
1926         case KVM_REG_PPC_TM_PPR:
1927                 vcpu->arch.ppr_tm = set_reg_val(id, *val);
1928                 break;
1929         case KVM_REG_PPC_TM_VRSAVE:
1930                 vcpu->arch.vrsave_tm = set_reg_val(id, *val);
1931                 break;
1932         case KVM_REG_PPC_TM_VSCR:
1933                 if (cpu_has_feature(CPU_FTR_ALTIVEC))
1934                         vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val);
1935                 else
1936                         r = - ENXIO;
1937                 break;
1938         case KVM_REG_PPC_TM_DSCR:
1939                 vcpu->arch.dscr_tm = set_reg_val(id, *val);
1940                 break;
1941         case KVM_REG_PPC_TM_TAR:
1942                 vcpu->arch.tar_tm = set_reg_val(id, *val);
1943                 break;
1944 #endif
1945         case KVM_REG_PPC_ARCH_COMPAT:
1946                 r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val));
1947                 break;
1948         case KVM_REG_PPC_DEC_EXPIRY:
1949                 vcpu->arch.dec_expires = set_reg_val(id, *val) -
1950                         vcpu->arch.vcore->tb_offset;
1951                 break;
1952         case KVM_REG_PPC_ONLINE:
1953                 i = set_reg_val(id, *val);
1954                 if (i && !vcpu->arch.online)
1955                         atomic_inc(&vcpu->arch.vcore->online_count);
1956                 else if (!i && vcpu->arch.online)
1957                         atomic_dec(&vcpu->arch.vcore->online_count);
1958                 vcpu->arch.online = i;
1959                 break;
1960         case KVM_REG_PPC_PTCR:
1961                 vcpu->kvm->arch.l1_ptcr = set_reg_val(id, *val);
1962                 break;
1963         default:
1964                 r = -EINVAL;
1965                 break;
1966         }
1967
1968         return r;
1969 }
1970
1971 /*
1972  * On POWER9, threads are independent and can be in different partitions.
1973  * Therefore we consider each thread to be a subcore.
1974  * There is a restriction that all threads have to be in the same
1975  * MMU mode (radix or HPT), unfortunately, but since we only support
1976  * HPT guests on a HPT host so far, that isn't an impediment yet.
1977  */
1978 static int threads_per_vcore(struct kvm *kvm)
1979 {
1980         if (kvm->arch.threads_indep)
1981                 return 1;
1982         return threads_per_subcore;
1983 }
1984
1985 static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int id)
1986 {
1987         struct kvmppc_vcore *vcore;
1988
1989         vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL);
1990
1991         if (vcore == NULL)
1992                 return NULL;
1993
1994         spin_lock_init(&vcore->lock);
1995         spin_lock_init(&vcore->stoltb_lock);
1996         init_swait_queue_head(&vcore->wq);
1997         vcore->preempt_tb = TB_NIL;
1998         vcore->lpcr = kvm->arch.lpcr;
1999         vcore->first_vcpuid = id;
2000         vcore->kvm = kvm;
2001         INIT_LIST_HEAD(&vcore->preempt_list);
2002
2003         return vcore;
2004 }
2005
2006 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2007 static struct debugfs_timings_element {
2008         const char *name;
2009         size_t offset;
2010 } timings[] = {
2011         {"rm_entry",    offsetof(struct kvm_vcpu, arch.rm_entry)},
2012         {"rm_intr",     offsetof(struct kvm_vcpu, arch.rm_intr)},
2013         {"rm_exit",     offsetof(struct kvm_vcpu, arch.rm_exit)},
2014         {"guest",       offsetof(struct kvm_vcpu, arch.guest_time)},
2015         {"cede",        offsetof(struct kvm_vcpu, arch.cede_time)},
2016 };
2017
2018 #define N_TIMINGS       (ARRAY_SIZE(timings))
2019
2020 struct debugfs_timings_state {
2021         struct kvm_vcpu *vcpu;
2022         unsigned int    buflen;
2023         char            buf[N_TIMINGS * 100];
2024 };
2025
2026 static int debugfs_timings_open(struct inode *inode, struct file *file)
2027 {
2028         struct kvm_vcpu *vcpu = inode->i_private;
2029         struct debugfs_timings_state *p;
2030
2031         p = kzalloc(sizeof(*p), GFP_KERNEL);
2032         if (!p)
2033                 return -ENOMEM;
2034
2035         kvm_get_kvm(vcpu->kvm);
2036         p->vcpu = vcpu;
2037         file->private_data = p;
2038
2039         return nonseekable_open(inode, file);
2040 }
2041
2042 static int debugfs_timings_release(struct inode *inode, struct file *file)
2043 {
2044         struct debugfs_timings_state *p = file->private_data;
2045
2046         kvm_put_kvm(p->vcpu->kvm);
2047         kfree(p);
2048         return 0;
2049 }
2050
2051 static ssize_t debugfs_timings_read(struct file *file, char __user *buf,
2052                                     size_t len, loff_t *ppos)
2053 {
2054         struct debugfs_timings_state *p = file->private_data;
2055         struct kvm_vcpu *vcpu = p->vcpu;
2056         char *s, *buf_end;
2057         struct kvmhv_tb_accumulator tb;
2058         u64 count;
2059         loff_t pos;
2060         ssize_t n;
2061         int i, loops;
2062         bool ok;
2063
2064         if (!p->buflen) {
2065                 s = p->buf;
2066                 buf_end = s + sizeof(p->buf);
2067                 for (i = 0; i < N_TIMINGS; ++i) {
2068                         struct kvmhv_tb_accumulator *acc;
2069
2070                         acc = (struct kvmhv_tb_accumulator *)
2071                                 ((unsigned long)vcpu + timings[i].offset);
2072                         ok = false;
2073                         for (loops = 0; loops < 1000; ++loops) {
2074                                 count = acc->seqcount;
2075                                 if (!(count & 1)) {
2076                                         smp_rmb();
2077                                         tb = *acc;
2078                                         smp_rmb();
2079                                         if (count == acc->seqcount) {
2080                                                 ok = true;
2081                                                 break;
2082                                         }
2083                                 }
2084                                 udelay(1);
2085                         }
2086                         if (!ok)
2087                                 snprintf(s, buf_end - s, "%s: stuck\n",
2088                                         timings[i].name);
2089                         else
2090                                 snprintf(s, buf_end - s,
2091                                         "%s: %llu %llu %llu %llu\n",
2092                                         timings[i].name, count / 2,
2093                                         tb_to_ns(tb.tb_total),
2094                                         tb_to_ns(tb.tb_min),
2095                                         tb_to_ns(tb.tb_max));
2096                         s += strlen(s);
2097                 }
2098                 p->buflen = s - p->buf;
2099         }
2100
2101         pos = *ppos;
2102         if (pos >= p->buflen)
2103                 return 0;
2104         if (len > p->buflen - pos)
2105                 len = p->buflen - pos;
2106         n = copy_to_user(buf, p->buf + pos, len);
2107         if (n) {
2108                 if (n == len)
2109                         return -EFAULT;
2110                 len -= n;
2111         }
2112         *ppos = pos + len;
2113         return len;
2114 }
2115
2116 static ssize_t debugfs_timings_write(struct file *file, const char __user *buf,
2117                                      size_t len, loff_t *ppos)
2118 {
2119         return -EACCES;
2120 }
2121
2122 static const struct file_operations debugfs_timings_ops = {
2123         .owner   = THIS_MODULE,
2124         .open    = debugfs_timings_open,
2125         .release = debugfs_timings_release,
2126         .read    = debugfs_timings_read,
2127         .write   = debugfs_timings_write,
2128         .llseek  = generic_file_llseek,
2129 };
2130
2131 /* Create a debugfs directory for the vcpu */
2132 static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
2133 {
2134         char buf[16];
2135         struct kvm *kvm = vcpu->kvm;
2136
2137         snprintf(buf, sizeof(buf), "vcpu%u", id);
2138         if (IS_ERR_OR_NULL(kvm->arch.debugfs_dir))
2139                 return;
2140         vcpu->arch.debugfs_dir = debugfs_create_dir(buf, kvm->arch.debugfs_dir);
2141         if (IS_ERR_OR_NULL(vcpu->arch.debugfs_dir))
2142                 return;
2143         vcpu->arch.debugfs_timings =
2144                 debugfs_create_file("timings", 0444, vcpu->arch.debugfs_dir,
2145                                     vcpu, &debugfs_timings_ops);
2146 }
2147
2148 #else /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */
2149 static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
2150 {
2151 }
2152 #endif /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */
2153
2154 static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm,
2155                                                    unsigned int id)
2156 {
2157         struct kvm_vcpu *vcpu;
2158         int err;
2159         int core;
2160         struct kvmppc_vcore *vcore;
2161
2162         err = -ENOMEM;
2163         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
2164         if (!vcpu)
2165                 goto out;
2166
2167         err = kvm_vcpu_init(vcpu, kvm, id);
2168         if (err)
2169                 goto free_vcpu;
2170
2171         vcpu->arch.shared = &vcpu->arch.shregs;
2172 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2173         /*
2174          * The shared struct is never shared on HV,
2175          * so we can always use host endianness
2176          */
2177 #ifdef __BIG_ENDIAN__
2178         vcpu->arch.shared_big_endian = true;
2179 #else
2180         vcpu->arch.shared_big_endian = false;
2181 #endif
2182 #endif
2183         vcpu->arch.mmcr[0] = MMCR0_FC;
2184         vcpu->arch.ctrl = CTRL_RUNLATCH;
2185         /* default to host PVR, since we can't spoof it */
2186         kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR));
2187         spin_lock_init(&vcpu->arch.vpa_update_lock);
2188         spin_lock_init(&vcpu->arch.tbacct_lock);
2189         vcpu->arch.busy_preempt = TB_NIL;
2190         vcpu->arch.intr_msr = MSR_SF | MSR_ME;
2191
2192         /*
2193          * Set the default HFSCR for the guest from the host value.
2194          * This value is only used on POWER9.
2195          * On POWER9, we want to virtualize the doorbell facility, so we
2196          * don't set the HFSCR_MSGP bit, and that causes those instructions
2197          * to trap and then we emulate them.
2198          */
2199         vcpu->arch.hfscr = HFSCR_TAR | HFSCR_EBB | HFSCR_PM | HFSCR_BHRB |
2200                 HFSCR_DSCR | HFSCR_VECVSX | HFSCR_FP;
2201         if (cpu_has_feature(CPU_FTR_HVMODE)) {
2202                 vcpu->arch.hfscr &= mfspr(SPRN_HFSCR);
2203                 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
2204                         vcpu->arch.hfscr |= HFSCR_TM;
2205         }
2206         if (cpu_has_feature(CPU_FTR_TM_COMP))
2207                 vcpu->arch.hfscr |= HFSCR_TM;
2208
2209         kvmppc_mmu_book3s_hv_init(vcpu);
2210
2211         vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
2212
2213         init_waitqueue_head(&vcpu->arch.cpu_run);
2214
2215         mutex_lock(&kvm->lock);
2216         vcore = NULL;
2217         err = -EINVAL;
2218         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
2219                 if (id >= (KVM_MAX_VCPUS * kvm->arch.emul_smt_mode)) {
2220                         pr_devel("KVM: VCPU ID too high\n");
2221                         core = KVM_MAX_VCORES;
2222                 } else {
2223                         BUG_ON(kvm->arch.smt_mode != 1);
2224                         core = kvmppc_pack_vcpu_id(kvm, id);
2225                 }
2226         } else {
2227                 core = id / kvm->arch.smt_mode;
2228         }
2229         if (core < KVM_MAX_VCORES) {
2230                 vcore = kvm->arch.vcores[core];
2231                 if (vcore && cpu_has_feature(CPU_FTR_ARCH_300)) {
2232                         pr_devel("KVM: collision on id %u", id);
2233                         vcore = NULL;
2234                 } else if (!vcore) {
2235                         err = -ENOMEM;
2236                         vcore = kvmppc_vcore_create(kvm,
2237                                         id & ~(kvm->arch.smt_mode - 1));
2238                         kvm->arch.vcores[core] = vcore;
2239                         kvm->arch.online_vcores++;
2240                 }
2241         }
2242         mutex_unlock(&kvm->lock);
2243
2244         if (!vcore)
2245                 goto free_vcpu;
2246
2247         spin_lock(&vcore->lock);
2248         ++vcore->num_threads;
2249         spin_unlock(&vcore->lock);
2250         vcpu->arch.vcore = vcore;
2251         vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid;
2252         vcpu->arch.thread_cpu = -1;
2253         vcpu->arch.prev_cpu = -1;
2254
2255         vcpu->arch.cpu_type = KVM_CPU_3S_64;
2256         kvmppc_sanity_check(vcpu);
2257
2258         debugfs_vcpu_init(vcpu, id);
2259
2260         return vcpu;
2261
2262 free_vcpu:
2263         kmem_cache_free(kvm_vcpu_cache, vcpu);
2264 out:
2265         return ERR_PTR(err);
2266 }
2267
2268 static int kvmhv_set_smt_mode(struct kvm *kvm, unsigned long smt_mode,
2269                               unsigned long flags)
2270 {
2271         int err;
2272         int esmt = 0;
2273
2274         if (flags)
2275                 return -EINVAL;
2276         if (smt_mode > MAX_SMT_THREADS || !is_power_of_2(smt_mode))
2277                 return -EINVAL;
2278         if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
2279                 /*
2280                  * On POWER8 (or POWER7), the threading mode is "strict",
2281                  * so we pack smt_mode vcpus per vcore.
2282                  */
2283                 if (smt_mode > threads_per_subcore)
2284                         return -EINVAL;
2285         } else {
2286                 /*
2287                  * On POWER9, the threading mode is "loose",
2288                  * so each vcpu gets its own vcore.
2289                  */
2290                 esmt = smt_mode;
2291                 smt_mode = 1;
2292         }
2293         mutex_lock(&kvm->lock);
2294         err = -EBUSY;
2295         if (!kvm->arch.online_vcores) {
2296                 kvm->arch.smt_mode = smt_mode;
2297                 kvm->arch.emul_smt_mode = esmt;
2298                 err = 0;
2299         }
2300         mutex_unlock(&kvm->lock);
2301
2302         return err;
2303 }
2304
2305 static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa)
2306 {
2307         if (vpa->pinned_addr)
2308                 kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa,
2309                                         vpa->dirty);
2310 }
2311
2312 static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu)
2313 {
2314         spin_lock(&vcpu->arch.vpa_update_lock);
2315         unpin_vpa(vcpu->kvm, &vcpu->arch.dtl);
2316         unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow);
2317         unpin_vpa(vcpu->kvm, &vcpu->arch.vpa);
2318         spin_unlock(&vcpu->arch.vpa_update_lock);
2319         kvm_vcpu_uninit(vcpu);
2320         kmem_cache_free(kvm_vcpu_cache, vcpu);
2321 }
2322
2323 static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu)
2324 {
2325         /* Indicate we want to get back into the guest */
2326         return 1;
2327 }
2328
2329 static void kvmppc_set_timer(struct kvm_vcpu *vcpu)
2330 {
2331         unsigned long dec_nsec, now;
2332
2333         now = get_tb();
2334         if (now > vcpu->arch.dec_expires) {
2335                 /* decrementer has already gone negative */
2336                 kvmppc_core_queue_dec(vcpu);
2337                 kvmppc_core_prepare_to_enter(vcpu);
2338                 return;
2339         }
2340         dec_nsec = tb_to_ns(vcpu->arch.dec_expires - now);
2341         hrtimer_start(&vcpu->arch.dec_timer, dec_nsec, HRTIMER_MODE_REL);
2342         vcpu->arch.timer_running = 1;
2343 }
2344
2345 static void kvmppc_end_cede(struct kvm_vcpu *vcpu)
2346 {
2347         vcpu->arch.ceded = 0;
2348         if (vcpu->arch.timer_running) {
2349                 hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
2350                 vcpu->arch.timer_running = 0;
2351         }
2352 }
2353
2354 extern int __kvmppc_vcore_entry(void);
2355
2356 static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
2357                                    struct kvm_vcpu *vcpu)
2358 {
2359         u64 now;
2360
2361         if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
2362                 return;
2363         spin_lock_irq(&vcpu->arch.tbacct_lock);
2364         now = mftb();
2365         vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) -
2366                 vcpu->arch.stolen_logged;
2367         vcpu->arch.busy_preempt = now;
2368         vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
2369         spin_unlock_irq(&vcpu->arch.tbacct_lock);
2370         --vc->n_runnable;
2371         WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], NULL);
2372 }
2373
2374 static int kvmppc_grab_hwthread(int cpu)
2375 {
2376         struct paca_struct *tpaca;
2377         long timeout = 10000;
2378
2379         tpaca = paca_ptrs[cpu];
2380
2381         /* Ensure the thread won't go into the kernel if it wakes */
2382         tpaca->kvm_hstate.kvm_vcpu = NULL;
2383         tpaca->kvm_hstate.kvm_vcore = NULL;
2384         tpaca->kvm_hstate.napping = 0;
2385         smp_wmb();
2386         tpaca->kvm_hstate.hwthread_req = 1;
2387
2388         /*
2389          * If the thread is already executing in the kernel (e.g. handling
2390          * a stray interrupt), wait for it to get back to nap mode.
2391          * The smp_mb() is to ensure that our setting of hwthread_req
2392          * is visible before we look at hwthread_state, so if this
2393          * races with the code at system_reset_pSeries and the thread
2394          * misses our setting of hwthread_req, we are sure to see its
2395          * setting of hwthread_state, and vice versa.
2396          */
2397         smp_mb();
2398         while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) {
2399                 if (--timeout <= 0) {
2400                         pr_err("KVM: couldn't grab cpu %d\n", cpu);
2401                         return -EBUSY;
2402                 }
2403                 udelay(1);
2404         }
2405         return 0;
2406 }
2407
2408 static void kvmppc_release_hwthread(int cpu)
2409 {
2410         struct paca_struct *tpaca;
2411
2412         tpaca = paca_ptrs[cpu];
2413         tpaca->kvm_hstate.hwthread_req = 0;
2414         tpaca->kvm_hstate.kvm_vcpu = NULL;
2415         tpaca->kvm_hstate.kvm_vcore = NULL;
2416         tpaca->kvm_hstate.kvm_split_mode = NULL;
2417 }
2418
2419 static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu)
2420 {
2421         struct kvm_nested_guest *nested = vcpu->arch.nested;
2422         cpumask_t *cpu_in_guest;
2423         int i;
2424
2425         cpu = cpu_first_thread_sibling(cpu);
2426         if (nested) {
2427                 cpumask_set_cpu(cpu, &nested->need_tlb_flush);
2428                 cpu_in_guest = &nested->cpu_in_guest;
2429         } else {
2430                 cpumask_set_cpu(cpu, &kvm->arch.need_tlb_flush);
2431                 cpu_in_guest = &kvm->arch.cpu_in_guest;
2432         }
2433         /*
2434          * Make sure setting of bit in need_tlb_flush precedes
2435          * testing of cpu_in_guest bits.  The matching barrier on
2436          * the other side is the first smp_mb() in kvmppc_run_core().
2437          */
2438         smp_mb();
2439         for (i = 0; i < threads_per_core; ++i)
2440                 if (cpumask_test_cpu(cpu + i, cpu_in_guest))
2441                         smp_call_function_single(cpu + i, do_nothing, NULL, 1);
2442 }
2443
2444 static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu)
2445 {
2446         struct kvm_nested_guest *nested = vcpu->arch.nested;
2447         struct kvm *kvm = vcpu->kvm;
2448         int prev_cpu;
2449
2450         if (!cpu_has_feature(CPU_FTR_HVMODE))
2451                 return;
2452
2453         if (nested)
2454                 prev_cpu = nested->prev_cpu[vcpu->arch.nested_vcpu_id];
2455         else
2456                 prev_cpu = vcpu->arch.prev_cpu;
2457
2458         /*
2459          * With radix, the guest can do TLB invalidations itself,
2460          * and it could choose to use the local form (tlbiel) if
2461          * it is invalidating a translation that has only ever been
2462          * used on one vcpu.  However, that doesn't mean it has
2463          * only ever been used on one physical cpu, since vcpus
2464          * can move around between pcpus.  To cope with this, when
2465          * a vcpu moves from one pcpu to another, we need to tell
2466          * any vcpus running on the same core as this vcpu previously
2467          * ran to flush the TLB.  The TLB is shared between threads,
2468          * so we use a single bit in .need_tlb_flush for all 4 threads.
2469          */
2470         if (prev_cpu != pcpu) {
2471                 if (prev_cpu >= 0 &&
2472                     cpu_first_thread_sibling(prev_cpu) !=
2473                     cpu_first_thread_sibling(pcpu))
2474                         radix_flush_cpu(kvm, prev_cpu, vcpu);
2475                 if (nested)
2476                         nested->prev_cpu[vcpu->arch.nested_vcpu_id] = pcpu;
2477                 else
2478                         vcpu->arch.prev_cpu = pcpu;
2479         }
2480 }
2481
2482 static void kvmppc_radix_check_need_tlb_flush(struct kvm *kvm, int pcpu,
2483                                               struct kvm_nested_guest *nested)
2484 {
2485         cpumask_t *need_tlb_flush;
2486         int lpid;
2487
2488         if (!cpu_has_feature(CPU_FTR_HVMODE))
2489                 return;
2490
2491         if (cpu_has_feature(CPU_FTR_ARCH_300))
2492                 pcpu &= ~0x3UL;
2493
2494         if (nested) {
2495                 lpid = nested->shadow_lpid;
2496                 need_tlb_flush = &nested->need_tlb_flush;
2497         } else {
2498                 lpid = kvm->arch.lpid;
2499                 need_tlb_flush = &kvm->arch.need_tlb_flush;
2500         }
2501
2502         mtspr(SPRN_LPID, lpid);
2503         isync();
2504         smp_mb();
2505
2506         if (cpumask_test_cpu(pcpu, need_tlb_flush)) {
2507                 radix__local_flush_tlb_lpid_guest(lpid);
2508                 /* Clear the bit after the TLB flush */
2509                 cpumask_clear_cpu(pcpu, need_tlb_flush);
2510         }
2511 }
2512
2513 static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc)
2514 {
2515         int cpu;
2516         struct paca_struct *tpaca;
2517         struct kvm *kvm = vc->kvm;
2518
2519         cpu = vc->pcpu;
2520         if (vcpu) {
2521                 if (vcpu->arch.timer_running) {
2522                         hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
2523                         vcpu->arch.timer_running = 0;
2524                 }
2525                 cpu += vcpu->arch.ptid;
2526                 vcpu->cpu = vc->pcpu;
2527                 vcpu->arch.thread_cpu = cpu;
2528                 cpumask_set_cpu(cpu, &kvm->arch.cpu_in_guest);
2529         }
2530         tpaca = paca_ptrs[cpu];
2531         tpaca->kvm_hstate.kvm_vcpu = vcpu;
2532         tpaca->kvm_hstate.ptid = cpu - vc->pcpu;
2533         tpaca->kvm_hstate.fake_suspend = 0;
2534         /* Order stores to hstate.kvm_vcpu etc. before store to kvm_vcore */
2535         smp_wmb();
2536         tpaca->kvm_hstate.kvm_vcore = vc;
2537         if (cpu != smp_processor_id())
2538                 kvmppc_ipi_thread(cpu);
2539 }
2540
2541 static void kvmppc_wait_for_nap(int n_threads)
2542 {
2543         int cpu = smp_processor_id();
2544         int i, loops;
2545
2546         if (n_threads <= 1)
2547                 return;
2548         for (loops = 0; loops < 1000000; ++loops) {
2549                 /*
2550                  * Check if all threads are finished.
2551                  * We set the vcore pointer when starting a thread
2552                  * and the thread clears it when finished, so we look
2553                  * for any threads that still have a non-NULL vcore ptr.
2554                  */
2555                 for (i = 1; i < n_threads; ++i)
2556                         if (paca_ptrs[cpu + i]->kvm_hstate.kvm_vcore)
2557                                 break;
2558                 if (i == n_threads) {
2559                         HMT_medium();
2560                         return;
2561                 }
2562                 HMT_low();
2563         }
2564         HMT_medium();
2565         for (i = 1; i < n_threads; ++i)
2566                 if (paca_ptrs[cpu + i]->kvm_hstate.kvm_vcore)
2567                         pr_err("KVM: CPU %d seems to be stuck\n", cpu + i);
2568 }
2569
2570 /*
2571  * Check that we are on thread 0 and that any other threads in
2572  * this core are off-line.  Then grab the threads so they can't
2573  * enter the kernel.
2574  */
2575 static int on_primary_thread(void)
2576 {
2577         int cpu = smp_processor_id();
2578         int thr;
2579
2580         /* Are we on a primary subcore? */
2581         if (cpu_thread_in_subcore(cpu))
2582                 return 0;
2583
2584         thr = 0;
2585         while (++thr < threads_per_subcore)
2586                 if (cpu_online(cpu + thr))
2587                         return 0;
2588
2589         /* Grab all hw threads so they can't go into the kernel */
2590         for (thr = 1; thr < threads_per_subcore; ++thr) {
2591                 if (kvmppc_grab_hwthread(cpu + thr)) {
2592                         /* Couldn't grab one; let the others go */
2593                         do {
2594                                 kvmppc_release_hwthread(cpu + thr);
2595                         } while (--thr > 0);
2596                         return 0;
2597                 }
2598         }
2599         return 1;
2600 }
2601
2602 /*
2603  * A list of virtual cores for each physical CPU.
2604  * These are vcores that could run but their runner VCPU tasks are
2605  * (or may be) preempted.
2606  */
2607 struct preempted_vcore_list {
2608         struct list_head        list;
2609         spinlock_t              lock;
2610 };
2611
2612 static DEFINE_PER_CPU(struct preempted_vcore_list, preempted_vcores);
2613
2614 static void init_vcore_lists(void)
2615 {
2616         int cpu;
2617
2618         for_each_possible_cpu(cpu) {
2619                 struct preempted_vcore_list *lp = &per_cpu(preempted_vcores, cpu);
2620                 spin_lock_init(&lp->lock);
2621                 INIT_LIST_HEAD(&lp->list);
2622         }
2623 }
2624
2625 static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc)
2626 {
2627         struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
2628
2629         vc->vcore_state = VCORE_PREEMPT;
2630         vc->pcpu = smp_processor_id();
2631         if (vc->num_threads < threads_per_vcore(vc->kvm)) {
2632                 spin_lock(&lp->lock);
2633                 list_add_tail(&vc->preempt_list, &lp->list);
2634                 spin_unlock(&lp->lock);
2635         }
2636
2637         /* Start accumulating stolen time */
2638         kvmppc_core_start_stolen(vc);
2639 }
2640
2641 static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc)
2642 {
2643         struct preempted_vcore_list *lp;
2644
2645         kvmppc_core_end_stolen(vc);
2646         if (!list_empty(&vc->preempt_list)) {
2647                 lp = &per_cpu(preempted_vcores, vc->pcpu);
2648                 spin_lock(&lp->lock);
2649                 list_del_init(&vc->preempt_list);
2650                 spin_unlock(&lp->lock);
2651         }
2652         vc->vcore_state = VCORE_INACTIVE;
2653 }
2654
2655 /*
2656  * This stores information about the virtual cores currently
2657  * assigned to a physical core.
2658  */
2659 struct core_info {
2660         int             n_subcores;
2661         int             max_subcore_threads;
2662         int             total_threads;
2663         int             subcore_threads[MAX_SUBCORES];
2664         struct kvmppc_vcore *vc[MAX_SUBCORES];
2665 };
2666
2667 /*
2668  * This mapping means subcores 0 and 1 can use threads 0-3 and 4-7
2669  * respectively in 2-way micro-threading (split-core) mode on POWER8.
2670  */
2671 static int subcore_thread_map[MAX_SUBCORES] = { 0, 4, 2, 6 };
2672
2673 static void init_core_info(struct core_info *cip, struct kvmppc_vcore *vc)
2674 {
2675         memset(cip, 0, sizeof(*cip));
2676         cip->n_subcores = 1;
2677         cip->max_subcore_threads = vc->num_threads;
2678         cip->total_threads = vc->num_threads;
2679         cip->subcore_threads[0] = vc->num_threads;
2680         cip->vc[0] = vc;
2681 }
2682
2683 static bool subcore_config_ok(int n_subcores, int n_threads)
2684 {
2685         /*
2686          * POWER9 "SMT4" cores are permanently in what is effectively a 4-way
2687          * split-core mode, with one thread per subcore.
2688          */
2689         if (cpu_has_feature(CPU_FTR_ARCH_300))
2690                 return n_subcores <= 4 && n_threads == 1;
2691
2692         /* On POWER8, can only dynamically split if unsplit to begin with */
2693         if (n_subcores > 1 && threads_per_subcore < MAX_SMT_THREADS)
2694                 return false;
2695         if (n_subcores > MAX_SUBCORES)
2696                 return false;
2697         if (n_subcores > 1) {
2698                 if (!(dynamic_mt_modes & 2))
2699                         n_subcores = 4;
2700                 if (n_subcores > 2 && !(dynamic_mt_modes & 4))
2701                         return false;
2702         }
2703
2704         return n_subcores * roundup_pow_of_two(n_threads) <= MAX_SMT_THREADS;
2705 }
2706
2707 static void init_vcore_to_run(struct kvmppc_vcore *vc)
2708 {
2709         vc->entry_exit_map = 0;
2710         vc->in_guest = 0;
2711         vc->napping_threads = 0;
2712         vc->conferring_threads = 0;
2713         vc->tb_offset_applied = 0;
2714 }
2715
2716 static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip)
2717 {
2718         int n_threads = vc->num_threads;
2719         int sub;
2720
2721         if (!cpu_has_feature(CPU_FTR_ARCH_207S))
2722                 return false;
2723
2724         /* In one_vm_per_core mode, require all vcores to be from the same vm */
2725         if (one_vm_per_core && vc->kvm != cip->vc[0]->kvm)
2726                 return false;
2727
2728         /* Some POWER9 chips require all threads to be in the same MMU mode */
2729         if (no_mixing_hpt_and_radix &&
2730             kvm_is_radix(vc->kvm) != kvm_is_radix(cip->vc[0]->kvm))
2731                 return false;
2732
2733         if (n_threads < cip->max_subcore_threads)
2734                 n_threads = cip->max_subcore_threads;
2735         if (!subcore_config_ok(cip->n_subcores + 1, n_threads))
2736                 return false;
2737         cip->max_subcore_threads = n_threads;
2738
2739         sub = cip->n_subcores;
2740         ++cip->n_subcores;
2741         cip->total_threads += vc->num_threads;
2742         cip->subcore_threads[sub] = vc->num_threads;
2743         cip->vc[sub] = vc;
2744         init_vcore_to_run(vc);
2745         list_del_init(&vc->preempt_list);
2746
2747         return true;
2748 }
2749
2750 /*
2751  * Work out whether it is possible to piggyback the execution of
2752  * vcore *pvc onto the execution of the other vcores described in *cip.
2753  */
2754 static bool can_piggyback(struct kvmppc_vcore *pvc, struct core_info *cip,
2755                           int target_threads)
2756 {
2757         if (cip->total_threads + pvc->num_threads > target_threads)
2758                 return false;
2759
2760         return can_dynamic_split(pvc, cip);
2761 }
2762
2763 static void prepare_threads(struct kvmppc_vcore *vc)
2764 {
2765         int i;
2766         struct kvm_vcpu *vcpu;
2767
2768         for_each_runnable_thread(i, vcpu, vc) {
2769                 if (signal_pending(vcpu->arch.run_task))
2770                         vcpu->arch.ret = -EINTR;
2771                 else if (vcpu->arch.vpa.update_pending ||
2772                          vcpu->arch.slb_shadow.update_pending ||
2773                          vcpu->arch.dtl.update_pending)
2774                         vcpu->arch.ret = RESUME_GUEST;
2775                 else
2776                         continue;
2777                 kvmppc_remove_runnable(vc, vcpu);
2778                 wake_up(&vcpu->arch.cpu_run);
2779         }
2780 }
2781
2782 static void collect_piggybacks(struct core_info *cip, int target_threads)
2783 {
2784         struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
2785         struct kvmppc_vcore *pvc, *vcnext;
2786
2787         spin_lock(&lp->lock);
2788         list_for_each_entry_safe(pvc, vcnext, &lp->list, preempt_list) {
2789                 if (!spin_trylock(&pvc->lock))
2790                         continue;
2791                 prepare_threads(pvc);
2792                 if (!pvc->n_runnable) {
2793                         list_del_init(&pvc->preempt_list);
2794                         if (pvc->runner == NULL) {
2795                                 pvc->vcore_state = VCORE_INACTIVE;
2796                                 kvmppc_core_end_stolen(pvc);
2797                         }
2798                         spin_unlock(&pvc->lock);
2799                         continue;
2800                 }
2801                 if (!can_piggyback(pvc, cip, target_threads)) {
2802                         spin_unlock(&pvc->lock);
2803                         continue;
2804                 }
2805                 kvmppc_core_end_stolen(pvc);
2806                 pvc->vcore_state = VCORE_PIGGYBACK;
2807                 if (cip->total_threads >= target_threads)
2808                         break;
2809         }
2810         spin_unlock(&lp->lock);
2811 }
2812
2813 static bool recheck_signals(struct core_info *cip)
2814 {
2815         int sub, i;
2816         struct kvm_vcpu *vcpu;
2817
2818         for (sub = 0; sub < cip->n_subcores; ++sub)
2819                 for_each_runnable_thread(i, vcpu, cip->vc[sub])
2820                         if (signal_pending(vcpu->arch.run_task))
2821                                 return true;
2822         return false;
2823 }
2824
2825 static void post_guest_process(struct kvmppc_vcore *vc, bool is_master)
2826 {
2827         int still_running = 0, i;
2828         u64 now;
2829         long ret;
2830         struct kvm_vcpu *vcpu;
2831
2832         spin_lock(&vc->lock);
2833         now = get_tb();
2834         for_each_runnable_thread(i, vcpu, vc) {
2835                 /*
2836                  * It's safe to unlock the vcore in the loop here, because
2837                  * for_each_runnable_thread() is safe against removal of
2838                  * the vcpu, and the vcore state is VCORE_EXITING here,
2839                  * so any vcpus becoming runnable will have their arch.trap
2840                  * set to zero and can't actually run in the guest.
2841                  */
2842                 spin_unlock(&vc->lock);
2843                 /* cancel pending dec exception if dec is positive */
2844                 if (now < vcpu->arch.dec_expires &&
2845                     kvmppc_core_pending_dec(vcpu))
2846                         kvmppc_core_dequeue_dec(vcpu);
2847
2848                 trace_kvm_guest_exit(vcpu);
2849
2850                 ret = RESUME_GUEST;
2851                 if (vcpu->arch.trap)
2852                         ret = kvmppc_handle_exit_hv(vcpu->arch.kvm_run, vcpu,
2853                                                     vcpu->arch.run_task);
2854
2855                 vcpu->arch.ret = ret;
2856                 vcpu->arch.trap = 0;
2857
2858                 spin_lock(&vc->lock);
2859                 if (is_kvmppc_resume_guest(vcpu->arch.ret)) {
2860                         if (vcpu->arch.pending_exceptions)
2861                                 kvmppc_core_prepare_to_enter(vcpu);
2862                         if (vcpu->arch.ceded)
2863                                 kvmppc_set_timer(vcpu);
2864                         else
2865                                 ++still_running;
2866                 } else {
2867                         kvmppc_remove_runnable(vc, vcpu);
2868                         wake_up(&vcpu->arch.cpu_run);
2869                 }
2870         }
2871         if (!is_master) {
2872                 if (still_running > 0) {
2873                         kvmppc_vcore_preempt(vc);
2874                 } else if (vc->runner) {
2875                         vc->vcore_state = VCORE_PREEMPT;
2876                         kvmppc_core_start_stolen(vc);
2877                 } else {
2878                         vc->vcore_state = VCORE_INACTIVE;
2879                 }
2880                 if (vc->n_runnable > 0 && vc->runner == NULL) {
2881                         /* make sure there's a candidate runner awake */
2882                         i = -1;
2883                         vcpu = next_runnable_thread(vc, &i);
2884                         wake_up(&vcpu->arch.cpu_run);
2885                 }
2886         }
2887         spin_unlock(&vc->lock);
2888 }
2889
2890 /*
2891  * Clear core from the list of active host cores as we are about to
2892  * enter the guest. Only do this if it is the primary thread of the
2893  * core (not if a subcore) that is entering the guest.
2894  */
2895 static inline int kvmppc_clear_host_core(unsigned int cpu)
2896 {
2897         int core;
2898
2899         if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
2900                 return 0;
2901         /*
2902          * Memory barrier can be omitted here as we will do a smp_wmb()
2903          * later in kvmppc_start_thread and we need ensure that state is
2904          * visible to other CPUs only after we enter guest.
2905          */
2906         core = cpu >> threads_shift;
2907         kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 0;
2908         return 0;
2909 }
2910
2911 /*
2912  * Advertise this core as an active host core since we exited the guest
2913  * Only need to do this if it is the primary thread of the core that is
2914  * exiting.
2915  */
2916 static inline int kvmppc_set_host_core(unsigned int cpu)
2917 {
2918         int core;
2919
2920         if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
2921                 return 0;
2922
2923         /*
2924          * Memory barrier can be omitted here because we do a spin_unlock
2925          * immediately after this which provides the memory barrier.
2926          */
2927         core = cpu >> threads_shift;
2928         kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 1;
2929         return 0;
2930 }
2931
2932 static void set_irq_happened(int trap)
2933 {
2934         switch (trap) {
2935         case BOOK3S_INTERRUPT_EXTERNAL:
2936                 local_paca->irq_happened |= PACA_IRQ_EE;
2937                 break;
2938         case BOOK3S_INTERRUPT_H_DOORBELL:
2939                 local_paca->irq_happened |= PACA_IRQ_DBELL;
2940                 break;
2941         case BOOK3S_INTERRUPT_HMI:
2942                 local_paca->irq_happened |= PACA_IRQ_HMI;
2943                 break;
2944         case BOOK3S_INTERRUPT_SYSTEM_RESET:
2945                 replay_system_reset();
2946                 break;
2947         }
2948 }
2949
2950 /*
2951  * Run a set of guest threads on a physical core.
2952  * Called with vc->lock held.
2953  */
2954 static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
2955 {
2956         struct kvm_vcpu *vcpu;
2957         int i;
2958         int srcu_idx;
2959         struct core_info core_info;
2960         struct kvmppc_vcore *pvc;
2961         struct kvm_split_mode split_info, *sip;
2962         int split, subcore_size, active;
2963         int sub;
2964         bool thr0_done;
2965         unsigned long cmd_bit, stat_bit;
2966         int pcpu, thr;
2967         int target_threads;
2968         int controlled_threads;
2969         int trap;
2970         bool is_power8;
2971         bool hpt_on_radix;
2972
2973         /*
2974          * Remove from the list any threads that have a signal pending
2975          * or need a VPA update done
2976          */
2977         prepare_threads(vc);
2978
2979         /* if the runner is no longer runnable, let the caller pick a new one */
2980         if (vc->runner->arch.state != KVMPPC_VCPU_RUNNABLE)
2981                 return;
2982
2983         /*
2984          * Initialize *vc.
2985          */
2986         init_vcore_to_run(vc);
2987         vc->preempt_tb = TB_NIL;
2988
2989         /*
2990          * Number of threads that we will be controlling: the same as
2991          * the number of threads per subcore, except on POWER9,
2992          * where it's 1 because the threads are (mostly) independent.
2993          */
2994         controlled_threads = threads_per_vcore(vc->kvm);
2995
2996         /*
2997          * Make sure we are running on primary threads, and that secondary
2998          * threads are offline.  Also check if the number of threads in this
2999          * guest are greater than the current system threads per guest.
3000          * On POWER9, we need to be not in independent-threads mode if
3001          * this is a HPT guest on a radix host machine where the
3002          * CPU threads may not be in different MMU modes.
3003          */
3004         hpt_on_radix = no_mixing_hpt_and_radix && radix_enabled() &&
3005                 !kvm_is_radix(vc->kvm);
3006         if (((controlled_threads > 1) &&
3007              ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) ||
3008             (hpt_on_radix && vc->kvm->arch.threads_indep)) {
3009                 for_each_runnable_thread(i, vcpu, vc) {
3010                         vcpu->arch.ret = -EBUSY;
3011                         kvmppc_remove_runnable(vc, vcpu);
3012                         wake_up(&vcpu->arch.cpu_run);
3013                 }
3014                 goto out;
3015         }
3016
3017         /*
3018          * See if we could run any other vcores on the physical core
3019          * along with this one.
3020          */
3021         init_core_info(&core_info, vc);
3022         pcpu = smp_processor_id();
3023         target_threads = controlled_threads;
3024         if (target_smt_mode && target_smt_mode < target_threads)
3025                 target_threads = target_smt_mode;
3026         if (vc->num_threads < target_threads)
3027                 collect_piggybacks(&core_info, target_threads);
3028
3029         /*
3030          * On radix, arrange for TLB flushing if necessary.
3031          * This has to be done before disabling interrupts since
3032          * it uses smp_call_function().
3033          */
3034         pcpu = smp_processor_id();
3035         if (kvm_is_radix(vc->kvm)) {
3036                 for (sub = 0; sub < core_info.n_subcores; ++sub)
3037                         for_each_runnable_thread(i, vcpu, core_info.vc[sub])
3038                                 kvmppc_prepare_radix_vcpu(vcpu, pcpu);
3039         }
3040
3041         /*
3042          * Hard-disable interrupts, and check resched flag and signals.
3043          * If we need to reschedule or deliver a signal, clean up
3044          * and return without going into the guest(s).
3045          * If the mmu_ready flag has been cleared, don't go into the
3046          * guest because that means a HPT resize operation is in progress.
3047          */
3048         local_irq_disable();
3049         hard_irq_disable();
3050         if (lazy_irq_pending() || need_resched() ||
3051             recheck_signals(&core_info) || !vc->kvm->arch.mmu_ready) {
3052                 local_irq_enable();
3053                 vc->vcore_state = VCORE_INACTIVE;
3054                 /* Unlock all except the primary vcore */
3055                 for (sub = 1; sub < core_info.n_subcores; ++sub) {
3056                         pvc = core_info.vc[sub];
3057                         /* Put back on to the preempted vcores list */
3058                         kvmppc_vcore_preempt(pvc);
3059                         spin_unlock(&pvc->lock);
3060                 }
3061                 for (i = 0; i < controlled_threads; ++i)
3062                         kvmppc_release_hwthread(pcpu + i);
3063                 return;
3064         }
3065
3066         kvmppc_clear_host_core(pcpu);
3067
3068         /* Decide on micro-threading (split-core) mode */
3069         subcore_size = threads_per_subcore;
3070         cmd_bit = stat_bit = 0;
3071         split = core_info.n_subcores;
3072         sip = NULL;
3073         is_power8 = cpu_has_feature(CPU_FTR_ARCH_207S)
3074                 && !cpu_has_feature(CPU_FTR_ARCH_300);
3075
3076         if (split > 1 || hpt_on_radix) {
3077                 sip = &split_info;
3078                 memset(&split_info, 0, sizeof(split_info));
3079                 for (sub = 0; sub < core_info.n_subcores; ++sub)
3080                         split_info.vc[sub] = core_info.vc[sub];
3081
3082                 if (is_power8) {
3083                         if (split == 2 && (dynamic_mt_modes & 2)) {
3084                                 cmd_bit = HID0_POWER8_1TO2LPAR;
3085                                 stat_bit = HID0_POWER8_2LPARMODE;
3086                         } else {
3087                                 split = 4;
3088                                 cmd_bit = HID0_POWER8_1TO4LPAR;
3089                                 stat_bit = HID0_POWER8_4LPARMODE;
3090                         }
3091                         subcore_size = MAX_SMT_THREADS / split;
3092                         split_info.rpr = mfspr(SPRN_RPR);
3093                         split_info.pmmar = mfspr(SPRN_PMMAR);
3094                         split_info.ldbar = mfspr(SPRN_LDBAR);
3095                         split_info.subcore_size = subcore_size;
3096                 } else {
3097                         split_info.subcore_size = 1;
3098                         if (hpt_on_radix) {
3099                                 /* Use the split_info for LPCR/LPIDR changes */
3100                                 split_info.lpcr_req = vc->lpcr;
3101                                 split_info.lpidr_req = vc->kvm->arch.lpid;
3102                                 split_info.host_lpcr = vc->kvm->arch.host_lpcr;
3103                                 split_info.do_set = 1;
3104                         }
3105                 }
3106
3107                 /* order writes to split_info before kvm_split_mode pointer */
3108                 smp_wmb();
3109         }
3110
3111         for (thr = 0; thr < controlled_threads; ++thr) {
3112                 struct paca_struct *paca = paca_ptrs[pcpu + thr];
3113
3114                 paca->kvm_hstate.tid = thr;
3115                 paca->kvm_hstate.napping = 0;
3116                 paca->kvm_hstate.kvm_split_mode = sip;
3117         }
3118
3119         /* Initiate micro-threading (split-core) on POWER8 if required */
3120         if (cmd_bit) {
3121                 unsigned long hid0 = mfspr(SPRN_HID0);