2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
13 * Derived from book3s_rmhandlers.S and other files, which are:
15 * Copyright SUSE Linux Products GmbH 2009
17 * Authors: Alexander Graf <agraf@suse.de>
20 #include <asm/ppc_asm.h>
21 #include <asm/kvm_asm.h>
25 #include <asm/ptrace.h>
26 #include <asm/hvcall.h>
27 #include <asm/asm-offsets.h>
28 #include <asm/exception-64s.h>
29 #include <asm/kvm_book3s_asm.h>
30 #include <asm/mmu-hash64.h>
33 #define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM)
35 /* Values in HSTATE_NAPPING(r13) */
36 #define NAPPING_CEDE 1
37 #define NAPPING_NOVCPU 2
40 * Call kvmppc_hv_entry in real mode.
41 * Must be called with interrupts hard-disabled.
45 * LR = return address to continue at after eventually re-enabling MMU
47 _GLOBAL_TOC(kvmppc_hv_entry_trampoline)
49 std r0, PPC_LR_STKOFF(r1)
52 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry)
57 mtmsrd r0,1 /* clear RI in MSR */
63 ld r4, HSTATE_KVM_VCPU(r13)
66 /* Back from guest - restore host state and return to caller */
69 /* Restore host DABR and DABRX */
70 ld r5,HSTATE_DABR(r13)
74 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
77 ld r3,PACA_SPRG_VDSO(r13)
78 mtspr SPRN_SPRG_VDSO_WRITE,r3
80 /* Reload the host's PMU registers */
81 ld r3, PACALPPACAPTR(r13) /* is the host using the PMU? */
82 lbz r4, LPPACA_PMCINUSE(r3)
84 beq 23f /* skip if not */
86 ld r3, HSTATE_MMCR(r13)
87 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
90 END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
91 lwz r3, HSTATE_PMC(r13)
92 lwz r4, HSTATE_PMC + 4(r13)
93 lwz r5, HSTATE_PMC + 8(r13)
94 lwz r6, HSTATE_PMC + 12(r13)
95 lwz r8, HSTATE_PMC + 16(r13)
96 lwz r9, HSTATE_PMC + 20(r13)
98 lwz r10, HSTATE_PMC + 24(r13)
99 lwz r11, HSTATE_PMC + 28(r13)
100 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
110 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
111 ld r3, HSTATE_MMCR(r13)
112 ld r4, HSTATE_MMCR + 8(r13)
113 ld r5, HSTATE_MMCR + 16(r13)
114 ld r6, HSTATE_MMCR + 24(r13)
115 ld r7, HSTATE_MMCR + 32(r13)
121 ld r8, HSTATE_MMCR + 40(r13)
122 ld r9, HSTATE_MMCR + 48(r13)
125 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
131 * Reload DEC. HDEC interrupts were disabled when
132 * we reloaded the host's LPCR value.
134 ld r3, HSTATE_DECEXP(r13)
140 * For external and machine check interrupts, we need
141 * to call the Linux handler to process the interrupt.
142 * We do that by jumping to absolute address 0x500 for
143 * external interrupts, or the machine_check_fwnmi label
144 * for machine checks (since firmware might have patched
145 * the vector area at 0x200). The [h]rfid at the end of the
146 * handler will return to the book3s_hv_interrupts.S code.
147 * For other interrupts we do the rfid to get back
148 * to the book3s_hv_interrupts.S code here.
150 ld r8, 112+PPC_LR_STKOFF(r1)
152 ld r7, HSTATE_HOST_MSR(r13)
154 cmpwi cr1, r12, BOOK3S_INTERRUPT_MACHINE_CHECK
155 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
158 cmpwi cr2, r12, BOOK3S_INTERRUPT_HMI
159 beq cr2, 14f /* HMI check */
160 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
162 /* RFI into the highmem handler, or branch to interrupt handler */
166 mtmsrd r6, 1 /* Clear RI in MSR */
169 beqa 0x500 /* external interrupt (PPC970) */
170 beq cr1, 13f /* machine check */
173 /* On POWER7, we have external interrupts set to use HSRR0/1 */
174 11: mtspr SPRN_HSRR0, r8
178 13: b machine_check_fwnmi
180 14: mtspr SPRN_HSRR0, r8
182 b hmi_exception_after_realmode
184 kvmppc_primary_no_guest:
185 /* We handle this much like a ceded vcpu */
186 /* set our bit in napping_threads */
187 ld r5, HSTATE_KVM_VCORE(r13)
188 lbz r7, HSTATE_PTID(r13)
191 addi r6, r5, VCORE_NAPPING_THREADS
196 /* order napping_threads update vs testing entry_exit_count */
199 lwz r7, VCORE_ENTRY_EXIT(r5)
201 bge kvm_novcpu_exit /* another thread already exiting */
202 li r3, NAPPING_NOVCPU
203 stb r3, HSTATE_NAPPING(r13)
205 stb r3, HSTATE_HWTHREAD_REQ(r13)
210 ld r1, HSTATE_HOST_R1(r13)
211 ld r5, HSTATE_KVM_VCORE(r13)
213 stb r0, HSTATE_NAPPING(r13)
214 stb r0, HSTATE_HWTHREAD_REQ(r13)
216 /* check the wake reason */
217 bl kvmppc_check_wake_reason
219 /* see if any other thread is already exiting */
220 lwz r0, VCORE_ENTRY_EXIT(r5)
224 /* clear our bit in napping_threads */
225 lbz r7, HSTATE_PTID(r13)
228 addi r6, r5, VCORE_NAPPING_THREADS
234 /* See if the wake reason means we need to exit */
238 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */
239 ld r4, HSTATE_KVM_VCPU(r13)
247 * We come in here when wakened from nap mode.
248 * Relocation is off and most register values are lost.
249 * r13 points to the PACA.
251 .globl kvm_start_guest
254 /* Set runlatch bit the minute you wake up from nap */
261 li r0,KVM_HWTHREAD_IN_KVM
262 stb r0,HSTATE_HWTHREAD_STATE(r13)
264 /* NV GPR values from power7_idle() will no longer be valid */
266 stb r0,PACA_NAPSTATELOST(r13)
268 /* were we napping due to cede? */
269 lbz r0,HSTATE_NAPPING(r13)
270 cmpwi r0,NAPPING_CEDE
272 cmpwi r0,NAPPING_NOVCPU
273 beq kvm_novcpu_wakeup
275 ld r1,PACAEMERGSP(r13)
276 subi r1,r1,STACK_FRAME_OVERHEAD
279 * We weren't napping due to cede, so this must be a secondary
280 * thread being woken up to run a guest, or being woken up due
281 * to a stray IPI. (Or due to some machine check or hypervisor
282 * maintenance interrupt while the core is in KVM.)
285 /* Check the wake reason in SRR1 to see why we got here */
286 bl kvmppc_check_wake_reason
290 /* get vcpu pointer, NULL if we have no vcpu to run */
291 ld r4,HSTATE_KVM_VCPU(r13)
293 /* if we have no vcpu to run, go back to sleep */
296 /* Set HSTATE_DSCR(r13) to something sensible */
297 ld r6, PACA_DSCR(r13)
298 std r6, HSTATE_DSCR(r13)
302 /* Back from the guest, go back to nap */
303 /* Clear our vcpu pointer so we don't come back in early */
305 std r0, HSTATE_KVM_VCPU(r13)
307 * Make sure we clear HSTATE_KVM_VCPU(r13) before incrementing
308 * the nap_count, because once the increment to nap_count is
309 * visible we could be given another vcpu.
313 /* increment the nap count and then go to nap mode */
314 ld r4, HSTATE_KVM_VCORE(r13)
315 addi r4, r4, VCORE_NAP_COUNT
322 li r0, KVM_HWTHREAD_IN_NAP
323 stb r0, HSTATE_HWTHREAD_STATE(r13)
325 /* Clear the runlatch bit before napping */
332 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
335 std r0, HSTATE_SCRATCH0(r13)
337 ld r0, HSTATE_SCRATCH0(r13)
343 /******************************************************************************
347 *****************************************************************************/
349 .global kvmppc_hv_entry
354 * R4 = vcpu pointer (or NULL)
359 * all other volatile GPRS = free
362 std r0, PPC_LR_STKOFF(r1)
365 /* Save R1 in the PACA */
366 std r1, HSTATE_HOST_R1(r13)
368 li r6, KVM_GUEST_MODE_HOST_HV
369 stb r6, HSTATE_IN_GUEST(r13)
379 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
381 * POWER7 host -> guest partition switch code.
382 * We don't have to lock against concurrent tlbies,
383 * but we do have to coordinate across hardware threads.
385 /* Increment entry count iff exit count is zero. */
386 ld r5,HSTATE_KVM_VCORE(r13)
387 addi r9,r5,VCORE_ENTRY_EXIT
389 cmpwi r3,0x100 /* any threads starting to exit? */
390 bge secondary_too_late /* if so we're too late to the party */
395 /* Primary thread switches to guest partition. */
396 ld r9,VCORE_KVM(r5) /* pointer to struct kvm */
397 lbz r6,HSTATE_PTID(r13)
402 li r0,LPID_RSVD /* switch to reserved LPID */
405 mtspr SPRN_SDR1,r6 /* switch to partition page table */
409 /* See if we need to flush the TLB */
410 lhz r6,PACAPACAINDEX(r13) /* test_bit(cpu, need_tlb_flush) */
411 clrldi r7,r6,64-6 /* extract bit number (6 bits) */
412 srdi r6,r6,6 /* doubleword number */
413 sldi r6,r6,3 /* address offset */
415 addi r6,r6,KVM_NEED_FLUSH /* dword in kvm->arch.need_tlb_flush */
421 23: ldarx r7,0,r6 /* if set, clear the bit */
425 /* Flush the TLB of any entries for this LPID */
426 /* use arch 2.07S as a proxy for POWER8 */
428 li r6,512 /* POWER8 has 512 sets */
430 li r6,128 /* POWER7 has 128 sets */
431 ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
433 li r7,0x800 /* IS field = 0b10 */
440 /* Add timebase offset onto timebase */
441 22: ld r8,VCORE_TB_OFFSET(r5)
444 mftb r6 /* current host timebase */
446 mtspr SPRN_TBU40,r8 /* update upper 40 bits */
447 mftb r7 /* check if lower 24 bits overflowed */
452 addis r8,r8,0x100 /* if so, increment upper 40 bits */
455 /* Load guest PCR value to select appropriate compat mode */
456 37: ld r7, VCORE_PCR(r5)
463 /* DPDES is shared between threads */
464 ld r8, VCORE_DPDES(r5)
466 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
469 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */
472 /* Secondary threads wait for primary to have done partition switch */
473 20: lbz r0,VCORE_IN_GUEST(r5)
477 /* Set LPCR and RMOR. */
478 10: ld r8,VCORE_LPCR(r5)
484 /* Check if HDEC expires soon */
486 cmpwi r3,512 /* 1 microsecond */
487 li r12,BOOK3S_INTERRUPT_HV_DECREMENTER
492 * PPC970 host -> guest partition switch code.
493 * We have to lock against concurrent tlbies,
494 * using native_tlbie_lock to lock against host tlbies
495 * and kvm->arch.tlbie_lock to lock against guest tlbies.
496 * We also have to invalidate the TLB since its
497 * entries aren't tagged with the LPID.
499 30: ld r5,HSTATE_KVM_VCORE(r13)
500 ld r9,VCORE_KVM(r5) /* pointer to struct kvm */
502 /* first take native_tlbie_lock */
505 .tc native_tlbie_lock[TC],native_tlbie_lock
507 ld r3,toc_tlbie_lock@toc(r2)
508 #ifdef __BIG_ENDIAN__
509 lwz r8,PACA_LOCK_TOKEN(r13)
511 lwz r8,PACAPACAINDEX(r13)
520 ld r5,HSTATE_KVM_VCORE(r13)
521 ld r7,VCORE_LPCR(r5) /* use vcore->lpcr to store HID4 */
523 rotldi r0,r0,HID4_LPID5_SH /* all lpid bits in HID4 = 1 */
527 mtspr SPRN_HID4,r0 /* switch to reserved LPID */
530 stw r0,0(r3) /* drop native_tlbie_lock */
532 /* invalidate the whole TLB */
541 /* Take the guest's tlbie_lock */
542 addi r3,r9,KVM_TLBIE_LOCK
550 mtspr SPRN_SDR1,r6 /* switch to partition page table */
552 /* Set up HID4 with the guest's LPID etc. */
557 /* drop the guest's tlbie_lock */
561 /* Check if HDEC expires soon */
564 li r12,BOOK3S_INTERRUPT_HV_DECREMENTER
567 /* Enable HDEC interrupts */
570 rldimi r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
580 /* Do we have a guest vcpu to run? */
582 beq kvmppc_primary_no_guest
585 /* Load up guest SLB entries */
586 lwz r5,VCPU_SLB_MAX(r4)
591 1: ld r8,VCPU_SLB_E(r6)
594 addi r6,r6,VCPU_SLB_SIZE
597 /* Increment yield count if they have a VPA */
601 li r6, LPPACA_YIELDCOUNT
606 stb r6, VCPU_VPA_DIRTY(r4)
610 /* Save purr/spurr */
613 std r5,HSTATE_PURR(r13)
614 std r6,HSTATE_SPURR(r13)
619 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
622 /* Set partition DABR */
623 /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
624 lwz r5,VCPU_DABRX(r4)
628 BEGIN_FTR_SECTION_NESTED(89)
630 END_FTR_SECTION_NESTED(CPU_FTR_ARCH_206, CPU_FTR_ARCH_206, 89)
631 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
633 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
636 END_FTR_SECTION_IFCLR(CPU_FTR_TM)
638 /* Turn on TM/FP/VSX/VMX so we can restore them. */
644 oris r5, r5, (MSR_VEC | MSR_VSX)@h
648 * The user may change these outside of a transaction, so they must
649 * always be context switched.
651 ld r5, VCPU_TFHAR(r4)
652 ld r6, VCPU_TFIAR(r4)
653 ld r7, VCPU_TEXASR(r4)
656 mtspr SPRN_TEXASR, r7
659 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
660 beq skip_tm /* TM not active in guest */
662 /* Make sure the failure summary is set, otherwise we'll program check
663 * when we trechkpt. It's possible that this might have been not set
664 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the
667 oris r7, r7, (TEXASR_FS)@h
668 mtspr SPRN_TEXASR, r7
671 * We need to load up the checkpointed state for the guest.
672 * We need to do this early as it will blow away any GPRs, VSRs and
677 addi r3, r31, VCPU_FPRS_TM
679 addi r3, r31, VCPU_VRS_TM
682 lwz r7, VCPU_VRSAVE_TM(r4)
683 mtspr SPRN_VRSAVE, r7
685 ld r5, VCPU_LR_TM(r4)
686 lwz r6, VCPU_CR_TM(r4)
687 ld r7, VCPU_CTR_TM(r4)
688 ld r8, VCPU_AMR_TM(r4)
689 ld r9, VCPU_TAR_TM(r4)
697 * Load up PPR and DSCR values but don't put them in the actual SPRs
698 * till the last moment to avoid running with userspace PPR and DSCR for
701 ld r29, VCPU_DSCR_TM(r4)
702 ld r30, VCPU_PPR_TM(r4)
704 std r2, PACATMSCRATCH(r13) /* Save TOC */
706 /* Clear the MSR RI since r1, r13 are all going to be foobar. */
710 /* Load GPRs r0-r28 */
713 ld reg, VCPU_GPRS_TM(reg)(r31)
720 /* Load final GPRs */
721 ld 29, VCPU_GPRS_TM(29)(r31)
722 ld 30, VCPU_GPRS_TM(30)(r31)
723 ld 31, VCPU_GPRS_TM(31)(r31)
725 /* TM checkpointed state is now setup. All GPRs are now volatile. */
728 /* Now let's get back the state we need. */
731 ld r29, HSTATE_DSCR(r13)
733 ld r4, HSTATE_KVM_VCPU(r13)
734 ld r1, HSTATE_HOST_R1(r13)
735 ld r2, PACATMSCRATCH(r13)
737 /* Set the MSR RI since we have our registers back. */
743 /* Load guest PMU registers */
744 /* R4 is live here (vcpu pointer) */
746 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
747 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
751 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
754 END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
755 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */
756 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */
757 lwz r6, VCPU_PMC + 8(r4)
758 lwz r7, VCPU_PMC + 12(r4)
759 lwz r8, VCPU_PMC + 16(r4)
760 lwz r9, VCPU_PMC + 20(r4)
762 lwz r10, VCPU_PMC + 24(r4)
763 lwz r11, VCPU_PMC + 28(r4)
764 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
774 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
776 ld r5, VCPU_MMCR + 8(r4)
777 ld r6, VCPU_MMCR + 16(r4)
785 ld r5, VCPU_MMCR + 24(r4)
787 lwz r7, VCPU_PMC + 24(r4)
788 lwz r8, VCPU_PMC + 28(r4)
789 ld r9, VCPU_MMCR + 32(r4)
795 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
799 /* Load up FP, VMX and VSX registers */
802 ld r14, VCPU_GPR(R14)(r4)
803 ld r15, VCPU_GPR(R15)(r4)
804 ld r16, VCPU_GPR(R16)(r4)
805 ld r17, VCPU_GPR(R17)(r4)
806 ld r18, VCPU_GPR(R18)(r4)
807 ld r19, VCPU_GPR(R19)(r4)
808 ld r20, VCPU_GPR(R20)(r4)
809 ld r21, VCPU_GPR(R21)(r4)
810 ld r22, VCPU_GPR(R22)(r4)
811 ld r23, VCPU_GPR(R23)(r4)
812 ld r24, VCPU_GPR(R24)(r4)
813 ld r25, VCPU_GPR(R25)(r4)
814 ld r26, VCPU_GPR(R26)(r4)
815 ld r27, VCPU_GPR(R27)(r4)
816 ld r28, VCPU_GPR(R28)(r4)
817 ld r29, VCPU_GPR(R29)(r4)
818 ld r30, VCPU_GPR(R30)(r4)
819 ld r31, VCPU_GPR(R31)(r4)
822 /* Switch DSCR to guest value */
825 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
828 /* Skip next section on POWER7 or PPC970 */
830 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
831 /* Turn on TM so we can access TFHAR/TFIAR/TEXASR */
834 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG
837 /* Load up POWER8-specific registers */
839 lwz r6, VCPU_PSPB(r4)
845 ld r6, VCPU_DAWRX(r4)
846 ld r7, VCPU_CIABR(r4)
856 ld r8, VCPU_EBBHR(r4)
858 ld r5, VCPU_EBBRR(r4)
859 ld r6, VCPU_BESCR(r4)
860 ld r7, VCPU_CSIGR(r4)
866 ld r5, VCPU_TCSCR(r4)
868 lwz r7, VCPU_GUEST_PID(r4)
877 * Set the decrementer to the guest decrementer.
879 ld r8,VCPU_DEC_EXPIRES(r4)
880 /* r8 is a host timebase value here, convert to guest TB */
881 ld r5,HSTATE_KVM_VCORE(r13)
882 ld r6,VCORE_TB_OFFSET(r5)
889 ld r5, VCPU_SPRG0(r4)
890 ld r6, VCPU_SPRG1(r4)
891 ld r7, VCPU_SPRG2(r4)
892 ld r8, VCPU_SPRG3(r4)
898 /* Load up DAR and DSISR */
900 lwz r6, VCPU_DSISR(r4)
905 /* Restore AMR and UAMOR, set AMOR to all 1s */
912 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
914 /* Restore state of CTRL run bit; assume 1 on entry */
928 kvmppc_cede_reentry: /* r4 = vcpu, r13 = paca */
936 deliver_guest_interrupt:
937 /* r11 = vcpu->arch.msr & ~MSR_HV */
938 rldicl r11, r11, 63 - MSR_HV_LG, 1
939 rotldi r11, r11, 1 + MSR_HV_LG
942 /* Check if we can deliver an external or decrementer interrupt now */
943 ld r0, VCPU_PENDING_EXC(r4)
944 rldicl r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63
946 andi. r8, r11, MSR_EE
949 /* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */
950 rldimi r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH
953 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
955 li r0, BOOK3S_INTERRUPT_EXTERNAL
959 li r0, BOOK3S_INTERRUPT_DECREMENTER
962 12: mtspr SPRN_SRR0, r10
966 bl kvmppc_msr_interrupt
972 * R10: value for HSRR0
973 * R11: value for HSRR1
978 stb r0,VCPU_CEDED(r4) /* cancel cede */
982 /* Activate guest mode, so faults get handled by KVM */
983 li r9, KVM_GUEST_MODE_GUEST_HV
984 stb r9, HSTATE_IN_GUEST(r13)
991 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
994 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1001 ld r1, VCPU_GPR(R1)(r4)
1002 ld r2, VCPU_GPR(R2)(r4)
1003 ld r3, VCPU_GPR(R3)(r4)
1004 ld r5, VCPU_GPR(R5)(r4)
1005 ld r6, VCPU_GPR(R6)(r4)
1006 ld r7, VCPU_GPR(R7)(r4)
1007 ld r8, VCPU_GPR(R8)(r4)
1008 ld r9, VCPU_GPR(R9)(r4)
1009 ld r10, VCPU_GPR(R10)(r4)
1010 ld r11, VCPU_GPR(R11)(r4)
1011 ld r12, VCPU_GPR(R12)(r4)
1012 ld r13, VCPU_GPR(R13)(r4)
1016 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1017 ld r0, VCPU_GPR(R0)(r4)
1018 ld r4, VCPU_GPR(R4)(r4)
1023 /******************************************************************************
1027 *****************************************************************************/
1030 * We come here from the first-level interrupt handlers.
1032 .globl kvmppc_interrupt_hv
1033 kvmppc_interrupt_hv:
1035 * Register contents:
1036 * R12 = interrupt vector
1038 * guest CR, R12 saved in shadow VCPU SCRATCH1/0
1039 * guest R13 saved in SPRN_SCRATCH0
1041 std r9, HSTATE_SCRATCH2(r13)
1043 lbz r9, HSTATE_IN_GUEST(r13)
1044 cmpwi r9, KVM_GUEST_MODE_HOST_HV
1045 beq kvmppc_bad_host_intr
1046 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1047 cmpwi r9, KVM_GUEST_MODE_GUEST
1048 ld r9, HSTATE_SCRATCH2(r13)
1049 beq kvmppc_interrupt_pr
1051 /* We're now back in the host but in guest MMU context */
1052 li r9, KVM_GUEST_MODE_HOST_HV
1053 stb r9, HSTATE_IN_GUEST(r13)
1055 ld r9, HSTATE_KVM_VCPU(r13)
1057 /* Save registers */
1059 std r0, VCPU_GPR(R0)(r9)
1060 std r1, VCPU_GPR(R1)(r9)
1061 std r2, VCPU_GPR(R2)(r9)
1062 std r3, VCPU_GPR(R3)(r9)
1063 std r4, VCPU_GPR(R4)(r9)
1064 std r5, VCPU_GPR(R5)(r9)
1065 std r6, VCPU_GPR(R6)(r9)
1066 std r7, VCPU_GPR(R7)(r9)
1067 std r8, VCPU_GPR(R8)(r9)
1068 ld r0, HSTATE_SCRATCH2(r13)
1069 std r0, VCPU_GPR(R9)(r9)
1070 std r10, VCPU_GPR(R10)(r9)
1071 std r11, VCPU_GPR(R11)(r9)
1072 ld r3, HSTATE_SCRATCH0(r13)
1073 lwz r4, HSTATE_SCRATCH1(r13)
1074 std r3, VCPU_GPR(R12)(r9)
1077 ld r3, HSTATE_CFAR(r13)
1078 std r3, VCPU_CFAR(r9)
1079 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1081 ld r4, HSTATE_PPR(r13)
1082 std r4, VCPU_PPR(r9)
1083 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1085 /* Restore R1/R2 so we can handle faults */
1086 ld r1, HSTATE_HOST_R1(r13)
1089 mfspr r10, SPRN_SRR0
1090 mfspr r11, SPRN_SRR1
1091 std r10, VCPU_SRR0(r9)
1092 std r11, VCPU_SRR1(r9)
1093 andi. r0, r12, 2 /* need to read HSRR0/1? */
1095 mfspr r10, SPRN_HSRR0
1096 mfspr r11, SPRN_HSRR1
1098 1: std r10, VCPU_PC(r9)
1099 std r11, VCPU_MSR(r9)
1103 std r3, VCPU_GPR(R13)(r9)
1106 stw r12,VCPU_TRAP(r9)
1108 /* Save HEIR (HV emulation assist reg) in last_inst
1109 if this is an HEI (HV emulation interrupt, e40) */
1110 li r3,KVM_INST_FETCH_FAILED
1112 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
1115 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1116 11: stw r3,VCPU_LAST_INST(r9)
1118 /* these are volatile across C function calls */
1121 std r3, VCPU_CTR(r9)
1122 stw r4, VCPU_XER(r9)
1125 /* If this is a page table miss then see if it's theirs or ours */
1126 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1128 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1130 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1132 /* See if this is a leftover HDEC interrupt */
1133 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
1139 /* See if this is an hcall we can handle in real mode */
1140 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL
1141 beq hcall_try_real_mode
1143 /* Only handle external interrupts here on arch 206 and later */
1145 b ext_interrupt_to_host
1146 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_206)
1148 /* External interrupt ? */
1149 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
1150 bne+ ext_interrupt_to_host
1152 /* External interrupt, first check for host_ipi. If this is
1153 * set, we know the host wants us out so let's do it now
1157 bgt ext_interrupt_to_host
1159 /* Check if any CPU is heading out to the host, if so head out too */
1160 ld r5, HSTATE_KVM_VCORE(r13)
1161 lwz r0, VCORE_ENTRY_EXIT(r5)
1163 bge ext_interrupt_to_host
1165 /* Return to guest after delivering any pending interrupt */
1167 b deliver_guest_interrupt
1169 ext_interrupt_to_host:
1171 guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */
1172 /* Save more register state */
1175 std r6, VCPU_DAR(r9)
1176 stw r7, VCPU_DSISR(r9)
1178 /* don't overwrite fault_dar/fault_dsisr if HDSI */
1179 cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
1181 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1182 std r6, VCPU_FAULT_DAR(r9)
1183 stw r7, VCPU_FAULT_DSISR(r9)
1185 /* See if it is a machine check */
1186 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK
1187 beq machine_check_realmode
1190 /* Save guest CTRL register, set runlatch to 1 */
1191 6: mfspr r6,SPRN_CTRLF
1192 stw r6,VCPU_CTRL(r9)
1198 /* Read the guest SLB and save it away */
1199 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */
1205 andis. r0,r8,SLB_ESID_V@h
1207 add r8,r8,r6 /* put index in */
1209 std r8,VCPU_SLB_E(r7)
1210 std r3,VCPU_SLB_V(r7)
1211 addi r7,r7,VCPU_SLB_SIZE
1215 stw r5,VCPU_SLB_MAX(r9)
1218 * Save the guest PURR/SPURR
1224 ld r8,VCPU_SPURR(r9)
1225 std r5,VCPU_PURR(r9)
1226 std r6,VCPU_SPURR(r9)
1231 * Restore host PURR/SPURR and add guest times
1232 * so that the time in the guest gets accounted.
1234 ld r3,HSTATE_PURR(r13)
1235 ld r4,HSTATE_SPURR(r13)
1240 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_201)
1247 /* r5 is a guest timebase value here, convert to host TB */
1248 ld r3,HSTATE_KVM_VCORE(r13)
1249 ld r4,VCORE_TB_OFFSET(r3)
1251 std r5,VCPU_DEC_EXPIRES(r9)
1255 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
1256 /* Save POWER8-specific registers */
1260 std r5, VCPU_IAMR(r9)
1261 stw r6, VCPU_PSPB(r9)
1262 std r7, VCPU_FSCR(r9)
1267 std r6, VCPU_VTB(r9)
1268 std r7, VCPU_TAR(r9)
1269 mfspr r8, SPRN_EBBHR
1270 std r8, VCPU_EBBHR(r9)
1271 mfspr r5, SPRN_EBBRR
1272 mfspr r6, SPRN_BESCR
1273 mfspr r7, SPRN_CSIGR
1275 std r5, VCPU_EBBRR(r9)
1276 std r6, VCPU_BESCR(r9)
1277 std r7, VCPU_CSIGR(r9)
1278 std r8, VCPU_TACR(r9)
1279 mfspr r5, SPRN_TCSCR
1283 std r5, VCPU_TCSCR(r9)
1284 std r6, VCPU_ACOP(r9)
1285 stw r7, VCPU_GUEST_PID(r9)
1286 std r8, VCPU_WORT(r9)
1289 /* Save and reset AMR and UAMOR before turning on the MMU */
1294 std r6,VCPU_UAMOR(r9)
1297 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1299 /* Switch DSCR back to host value */
1302 ld r7, HSTATE_DSCR(r13)
1303 std r8, VCPU_DSCR(r9)
1305 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1307 /* Save non-volatile GPRs */
1308 std r14, VCPU_GPR(R14)(r9)
1309 std r15, VCPU_GPR(R15)(r9)
1310 std r16, VCPU_GPR(R16)(r9)
1311 std r17, VCPU_GPR(R17)(r9)
1312 std r18, VCPU_GPR(R18)(r9)
1313 std r19, VCPU_GPR(R19)(r9)
1314 std r20, VCPU_GPR(R20)(r9)
1315 std r21, VCPU_GPR(R21)(r9)
1316 std r22, VCPU_GPR(R22)(r9)
1317 std r23, VCPU_GPR(R23)(r9)
1318 std r24, VCPU_GPR(R24)(r9)
1319 std r25, VCPU_GPR(R25)(r9)
1320 std r26, VCPU_GPR(R26)(r9)
1321 std r27, VCPU_GPR(R27)(r9)
1322 std r28, VCPU_GPR(R28)(r9)
1323 std r29, VCPU_GPR(R29)(r9)
1324 std r30, VCPU_GPR(R30)(r9)
1325 std r31, VCPU_GPR(R31)(r9)
1328 mfspr r3, SPRN_SPRG0
1329 mfspr r4, SPRN_SPRG1
1330 mfspr r5, SPRN_SPRG2
1331 mfspr r6, SPRN_SPRG3
1332 std r3, VCPU_SPRG0(r9)
1333 std r4, VCPU_SPRG1(r9)
1334 std r5, VCPU_SPRG2(r9)
1335 std r6, VCPU_SPRG3(r9)
1341 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1344 END_FTR_SECTION_IFCLR(CPU_FTR_TM)
1348 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG
1352 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
1353 beq 1f /* TM not active in guest. */
1355 li r3, TM_CAUSE_KVM_RESCHED
1357 /* Clear the MSR RI since r1, r13 are all going to be foobar. */
1361 /* All GPRs are volatile at this point. */
1364 /* Temporarily store r13 and r9 so we have some regs to play with */
1367 std r9, PACATMSCRATCH(r13)
1368 ld r9, HSTATE_KVM_VCPU(r13)
1370 /* Get a few more GPRs free. */
1371 std r29, VCPU_GPRS_TM(29)(r9)
1372 std r30, VCPU_GPRS_TM(30)(r9)
1373 std r31, VCPU_GPRS_TM(31)(r9)
1375 /* Save away PPR and DSCR soon so don't run with user values. */
1378 mfspr r30, SPRN_DSCR
1379 ld r29, HSTATE_DSCR(r13)
1380 mtspr SPRN_DSCR, r29
1382 /* Save all but r9, r13 & r29-r31 */
1385 .if (reg != 9) && (reg != 13)
1386 std reg, VCPU_GPRS_TM(reg)(r9)
1390 /* ... now save r13 */
1392 std r4, VCPU_GPRS_TM(13)(r9)
1393 /* ... and save r9 */
1394 ld r4, PACATMSCRATCH(r13)
1395 std r4, VCPU_GPRS_TM(9)(r9)
1397 /* Reload stack pointer and TOC. */
1398 ld r1, HSTATE_HOST_R1(r13)
1401 /* Set MSR RI now we have r1 and r13 back. */
1405 /* Save away checkpinted SPRs. */
1406 std r31, VCPU_PPR_TM(r9)
1407 std r30, VCPU_DSCR_TM(r9)
1413 std r5, VCPU_LR_TM(r9)
1414 stw r6, VCPU_CR_TM(r9)
1415 std r7, VCPU_CTR_TM(r9)
1416 std r8, VCPU_AMR_TM(r9)
1417 std r10, VCPU_TAR_TM(r9)
1419 /* Restore r12 as trap number. */
1420 lwz r12, VCPU_TRAP(r9)
1423 addi r3, r9, VCPU_FPRS_TM
1425 addi r3, r9, VCPU_VRS_TM
1427 mfspr r6, SPRN_VRSAVE
1428 stw r6, VCPU_VRSAVE_TM(r9)
1431 * We need to save these SPRs after the treclaim so that the software
1432 * error code is recorded correctly in the TEXASR. Also the user may
1433 * change these outside of a transaction, so they must always be
1436 mfspr r5, SPRN_TFHAR
1437 mfspr r6, SPRN_TFIAR
1438 mfspr r7, SPRN_TEXASR
1439 std r5, VCPU_TFHAR(r9)
1440 std r6, VCPU_TFIAR(r9)
1441 std r7, VCPU_TEXASR(r9)
1445 /* Increment yield count if they have a VPA */
1446 ld r8, VCPU_VPA(r9) /* do they have a VPA? */
1449 li r4, LPPACA_YIELDCOUNT
1454 stb r3, VCPU_VPA_DIRTY(r9)
1456 /* Save PMU registers if requested */
1457 /* r8 and cr0.eq are live here */
1460 * POWER8 seems to have a hardware bug where setting
1461 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE]
1462 * when some counters are already negative doesn't seem
1463 * to cause a performance monitor alert (and hence interrupt).
1464 * The effect of this is that when saving the PMU state,
1465 * if there is no PMU alert pending when we read MMCR0
1466 * before freezing the counters, but one becomes pending
1467 * before we read the counters, we lose it.
1468 * To work around this, we need a way to freeze the counters
1469 * before reading MMCR0. Normally, freezing the counters
1470 * is done by writing MMCR0 (to set MMCR0[FC]) which
1471 * unavoidably writes MMCR0[PMA0] as well. On POWER8,
1472 * we can also freeze the counters using MMCR2, by writing
1473 * 1s to all the counter freeze condition bits (there are
1474 * 9 bits each for 6 counters).
1476 li r3, -1 /* set all freeze bits */
1478 mfspr r10, SPRN_MMCR2
1479 mtspr SPRN_MMCR2, r3
1481 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1483 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
1484 mfspr r4, SPRN_MMCR0 /* save MMCR0 */
1485 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
1486 mfspr r6, SPRN_MMCRA
1488 /* On P7, clear MMCRA in order to disable SDAR updates */
1490 mtspr SPRN_MMCRA, r7
1491 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1493 beq 21f /* if no VPA, save PMU stuff anyway */
1494 lbz r7, LPPACA_PMCINUSE(r8)
1495 cmpwi r7, 0 /* did they ask for PMU stuff to be saved? */
1497 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */
1499 21: mfspr r5, SPRN_MMCR1
1502 std r4, VCPU_MMCR(r9)
1503 std r5, VCPU_MMCR + 8(r9)
1504 std r6, VCPU_MMCR + 16(r9)
1506 std r10, VCPU_MMCR + 24(r9)
1507 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1508 std r7, VCPU_SIAR(r9)
1509 std r8, VCPU_SDAR(r9)
1517 mfspr r10, SPRN_PMC7
1518 mfspr r11, SPRN_PMC8
1519 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
1520 stw r3, VCPU_PMC(r9)
1521 stw r4, VCPU_PMC + 4(r9)
1522 stw r5, VCPU_PMC + 8(r9)
1523 stw r6, VCPU_PMC + 12(r9)
1524 stw r7, VCPU_PMC + 16(r9)
1525 stw r8, VCPU_PMC + 20(r9)
1527 stw r10, VCPU_PMC + 24(r9)
1528 stw r11, VCPU_PMC + 28(r9)
1529 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
1532 mfspr r6, SPRN_SPMC1
1533 mfspr r7, SPRN_SPMC2
1534 mfspr r8, SPRN_MMCRS
1535 std r5, VCPU_SIER(r9)
1536 stw r6, VCPU_PMC + 24(r9)
1537 stw r7, VCPU_PMC + 28(r9)
1538 std r8, VCPU_MMCR + 32(r9)
1540 mtspr SPRN_MMCRS, r4
1541 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1549 hdec_soon: /* r12 = trap, r13 = paca */
1552 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
1554 * POWER7 guest -> host partition switch code.
1555 * We don't have to lock against tlbies but we do
1556 * have to coordinate the hardware threads.
1558 /* Increment the threads-exiting-guest count in the 0xff00
1559 bits of vcore->entry_exit_count */
1560 ld r5,HSTATE_KVM_VCORE(r13)
1561 addi r6,r5,VCORE_ENTRY_EXIT
1566 isync /* order stwcx. vs. reading napping_threads */
1569 * At this point we have an interrupt that we have to pass
1570 * up to the kernel or qemu; we can't handle it in real mode.
1571 * Thus we have to do a partition switch, so we have to
1572 * collect the other threads, if we are the first thread
1573 * to take an interrupt. To do this, we set the HDEC to 0,
1574 * which causes an HDEC interrupt in all threads within 2ns
1575 * because the HDEC register is shared between all 4 threads.
1576 * However, we don't need to bother if this is an HDEC
1577 * interrupt, since the other threads will already be on their
1578 * way here in that case.
1580 cmpwi r3,0x100 /* Are we the first here? */
1582 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
1588 * Send an IPI to any napping threads, since an HDEC interrupt
1589 * doesn't wake CPUs up from nap.
1591 lwz r3,VCORE_NAPPING_THREADS(r5)
1592 lbz r4,HSTATE_PTID(r13)
1595 andc. r3,r3,r0 /* no sense IPI'ing ourselves */
1597 /* Order entry/exit update vs. IPIs */
1599 mulli r4,r4,PACA_SIZE /* get paca for thread 0 */
1603 ld r8,HSTATE_XICS_PHYS(r6) /* get thread's XICS reg addr */
1606 stbcix r0,r7,r8 /* trigger the IPI */
1608 addi r6,r6,PACA_SIZE
1612 /* Secondary threads wait for primary to do partition switch */
1613 43: ld r5,HSTATE_KVM_VCORE(r13)
1614 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */
1615 lbz r3,HSTATE_PTID(r13)
1619 13: lbz r3,VCORE_IN_GUEST(r5)
1625 /* Primary thread waits for all the secondaries to exit guest */
1626 15: lwz r3,VCORE_ENTRY_EXIT(r5)
1633 /* Primary thread switches back to host partition */
1634 ld r6,KVM_HOST_SDR1(r4)
1635 lwz r7,KVM_HOST_LPID(r4)
1636 li r8,LPID_RSVD /* switch to reserved LPID */
1639 mtspr SPRN_SDR1,r6 /* switch to partition page table */
1644 /* DPDES is shared between threads */
1645 mfspr r7, SPRN_DPDES
1646 std r7, VCORE_DPDES(r5)
1647 /* clear DPDES so we don't get guest doorbells in the host */
1649 mtspr SPRN_DPDES, r8
1650 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1652 /* Subtract timebase offset from timebase */
1653 ld r8,VCORE_TB_OFFSET(r5)
1656 mftb r6 /* current guest timebase */
1658 mtspr SPRN_TBU40,r8 /* update upper 40 bits */
1659 mftb r7 /* check if lower 24 bits overflowed */
1664 addis r8,r8,0x100 /* if so, increment upper 40 bits */
1668 17: ld r0, VCORE_PCR(r5)
1674 /* Signal secondary CPUs to continue */
1675 stb r0,VCORE_IN_GUEST(r5)
1676 lis r8,0x7fff /* MAX_INT@h */
1679 16: ld r8,KVM_HOST_LPCR(r4)
1685 * PPC970 guest -> host partition switch code.
1686 * We have to lock against concurrent tlbies, and
1687 * we have to flush the whole TLB.
1689 32: ld r5,HSTATE_KVM_VCORE(r13)
1690 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */
1692 /* Take the guest's tlbie_lock */
1693 #ifdef __BIG_ENDIAN__
1694 lwz r8,PACA_LOCK_TOKEN(r13)
1696 lwz r8,PACAPACAINDEX(r13)
1698 addi r3,r4,KVM_TLBIE_LOCK
1706 ld r7,KVM_HOST_LPCR(r4) /* use kvm->arch.host_lpcr for HID4 */
1708 rotldi r0,r0,HID4_LPID5_SH /* all lpid bits in HID4 = 1 */
1712 mtspr SPRN_HID4,r0 /* switch to reserved LPID */
1715 stw r0,0(r3) /* drop guest tlbie_lock */
1717 /* invalidate the whole TLB */
1726 /* take native_tlbie_lock */
1727 ld r3,toc_tlbie_lock@toc(2)
1735 ld r6,KVM_HOST_SDR1(r4)
1736 mtspr SPRN_SDR1,r6 /* switch to host page table */
1738 /* Set up host HID4 value */
1743 stw r0,0(r3) /* drop native_tlbie_lock */
1745 lis r8,0x7fff /* MAX_INT@h */
1748 /* Disable HDEC interrupts */
1751 rldimi r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
1761 /* load host SLB entries */
1762 33: ld r8,PACA_SLBSHADOWPTR(r13)
1764 .rept SLB_NUM_BOLTED
1765 li r3, SLBSHADOW_SAVEAREA
1769 andis. r7,r5,SLB_ESID_V@h
1775 /* Unset guest mode */
1776 li r0, KVM_GUEST_MODE_NONE
1777 stb r0, HSTATE_IN_GUEST(r13)
1779 ld r0, 112+PPC_LR_STKOFF(r1)
1785 * Check whether an HDSI is an HPTE not found fault or something else.
1786 * If it is an HPTE not found fault that is due to the guest accessing
1787 * a page that they have mapped but which we have paged out, then
1788 * we continue on with the guest exit path. In all other cases,
1789 * reflect the HDSI to the guest as a DSI.
1793 mfspr r6, SPRN_HDSISR
1794 /* HPTE not found fault or protection fault? */
1795 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
1796 beq 1f /* if not, send it to the guest */
1797 andi. r0, r11, MSR_DR /* data relocation enabled? */
1800 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
1801 bne 1f /* if no SLB entry found */
1802 4: std r4, VCPU_FAULT_DAR(r9)
1803 stw r6, VCPU_FAULT_DSISR(r9)
1805 /* Search the hash table. */
1806 mr r3, r9 /* vcpu pointer */
1807 li r7, 1 /* data fault */
1808 bl kvmppc_hpte_hv_fault
1809 ld r9, HSTATE_KVM_VCPU(r13)
1811 ld r11, VCPU_MSR(r9)
1812 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1813 cmpdi r3, 0 /* retry the instruction */
1815 cmpdi r3, -1 /* handle in kernel mode */
1817 cmpdi r3, -2 /* MMIO emulation; need instr word */
1820 /* Synthesize a DSI for the guest */
1821 ld r4, VCPU_FAULT_DAR(r9)
1823 1: mtspr SPRN_DAR, r4
1824 mtspr SPRN_DSISR, r6
1825 mtspr SPRN_SRR0, r10
1826 mtspr SPRN_SRR1, r11
1827 li r10, BOOK3S_INTERRUPT_DATA_STORAGE
1828 bl kvmppc_msr_interrupt
1829 fast_interrupt_c_return:
1830 6: ld r7, VCPU_CTR(r9)
1831 lwz r8, VCPU_XER(r9)
1837 3: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */
1838 ld r5, KVM_VRMA_SLB_V(r5)
1841 /* If this is for emulated MMIO, load the instruction word */
1842 2: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */
1844 /* Set guest mode to 'jump over instruction' so if lwz faults
1845 * we'll just continue at the next IP. */
1846 li r0, KVM_GUEST_MODE_SKIP
1847 stb r0, HSTATE_IN_GUEST(r13)
1849 /* Do the access with MSR:DR enabled */
1851 ori r4, r3, MSR_DR /* Enable paging for data */
1856 /* Store the result */
1857 stw r8, VCPU_LAST_INST(r9)
1859 /* Unset guest mode. */
1860 li r0, KVM_GUEST_MODE_HOST_HV
1861 stb r0, HSTATE_IN_GUEST(r13)
1865 * Similarly for an HISI, reflect it to the guest as an ISI unless
1866 * it is an HPTE not found fault for a page that we have paged out.
1869 andis. r0, r11, SRR1_ISI_NOPT@h
1871 andi. r0, r11, MSR_IR /* instruction relocation enabled? */
1874 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
1875 bne 1f /* if no SLB entry found */
1877 /* Search the hash table. */
1878 mr r3, r9 /* vcpu pointer */
1881 li r7, 0 /* instruction fault */
1882 bl kvmppc_hpte_hv_fault
1883 ld r9, HSTATE_KVM_VCPU(r13)
1885 ld r11, VCPU_MSR(r9)
1886 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1887 cmpdi r3, 0 /* retry the instruction */
1888 beq fast_interrupt_c_return
1889 cmpdi r3, -1 /* handle in kernel mode */
1892 /* Synthesize an ISI for the guest */
1894 1: mtspr SPRN_SRR0, r10
1895 mtspr SPRN_SRR1, r11
1896 li r10, BOOK3S_INTERRUPT_INST_STORAGE
1897 bl kvmppc_msr_interrupt
1898 b fast_interrupt_c_return
1900 3: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */
1901 ld r5, KVM_VRMA_SLB_V(r6)
1905 * Try to handle an hcall in real mode.
1906 * Returns to the guest if we handle it, or continues on up to
1907 * the kernel if we can't (i.e. if we don't have a handler for
1908 * it, or if the handler returns H_TOO_HARD).
1910 .globl hcall_try_real_mode
1911 hcall_try_real_mode:
1912 ld r3,VCPU_GPR(R3)(r9)
1914 /* sc 1 from userspace - reflect to guest syscall */
1915 bne sc_1_fast_return
1917 cmpldi r3,hcall_real_table_end - hcall_real_table
1919 /* See if this hcall is enabled for in-kernel handling */
1921 srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */
1922 sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */
1924 ld r0, KVM_ENABLED_HCALLS(r4)
1925 rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */
1929 /* Get pointer to handler, if any, and call it */
1930 LOAD_REG_ADDR(r4, hcall_real_table)
1936 mr r3,r9 /* get vcpu pointer */
1937 ld r4,VCPU_GPR(R4)(r9)
1940 beq hcall_real_fallback
1941 ld r4,HSTATE_KVM_VCPU(r13)
1942 std r3,VCPU_GPR(R3)(r4)
1950 li r10, BOOK3S_INTERRUPT_SYSCALL
1951 bl kvmppc_msr_interrupt
1955 /* We've attempted a real mode hcall, but it's punted it back
1956 * to userspace. We need to restore some clobbered volatiles
1957 * before resuming the pass-it-to-qemu path */
1958 hcall_real_fallback:
1959 li r12,BOOK3S_INTERRUPT_SYSCALL
1960 ld r9, HSTATE_KVM_VCPU(r13)
1964 .globl hcall_real_table
1966 .long 0 /* 0 - unused */
1967 .long DOTSYM(kvmppc_h_remove) - hcall_real_table
1968 .long DOTSYM(kvmppc_h_enter) - hcall_real_table
1969 .long DOTSYM(kvmppc_h_read) - hcall_real_table
1970 .long 0 /* 0x10 - H_CLEAR_MOD */
1971 .long 0 /* 0x14 - H_CLEAR_REF */
1972 .long DOTSYM(kvmppc_h_protect) - hcall_real_table
1973 .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table
1974 .long DOTSYM(kvmppc_h_put_tce) - hcall_real_table
1975 .long 0 /* 0x24 - H_SET_SPRG0 */
1976 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table
1991 #ifdef CONFIG_KVM_XICS
1992 .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table
1993 .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table
1994 .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table
1995 .long 0 /* 0x70 - H_IPOLL */
1996 .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table
1998 .long 0 /* 0x64 - H_EOI */
1999 .long 0 /* 0x68 - H_CPPR */
2000 .long 0 /* 0x6c - H_IPI */
2001 .long 0 /* 0x70 - H_IPOLL */
2002 .long 0 /* 0x74 - H_XIRR */
2030 .long DOTSYM(kvmppc_h_cede) - hcall_real_table
2047 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table
2051 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table
2052 .globl hcall_real_table_end
2053 hcall_real_table_end:
2059 _GLOBAL(kvmppc_h_set_xdabr)
2060 andi. r0, r5, DABRX_USER | DABRX_KERNEL
2062 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI
2065 6: li r3, H_PARAMETER
2068 _GLOBAL(kvmppc_h_set_dabr)
2069 li r5, DABRX_USER | DABRX_KERNEL
2073 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2074 std r4,VCPU_DABR(r3)
2075 stw r5, VCPU_DABRX(r3)
2076 mtspr SPRN_DABRX, r5
2077 /* Work around P7 bug where DABR can get corrupted on mtspr */
2078 1: mtspr SPRN_DABR,r4
2086 /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */
2087 2: rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW
2088 rlwimi r5, r4, 1, DAWRX_WT
2090 std r4, VCPU_DAWR(r3)
2091 std r5, VCPU_DAWRX(r3)
2093 mtspr SPRN_DAWRX, r5
2097 _GLOBAL(kvmppc_h_cede)
2099 std r11,VCPU_MSR(r3)
2101 stb r0,VCPU_CEDED(r3)
2102 sync /* order setting ceded vs. testing prodded */
2103 lbz r5,VCPU_PRODDED(r3)
2105 bne kvm_cede_prodded
2106 li r0,0 /* set trap to 0 to say hcall is handled */
2107 stw r0,VCPU_TRAP(r3)
2109 std r0,VCPU_GPR(R3)(r3)
2111 b kvm_cede_exit /* just send it up to host on 970 */
2112 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_206)
2115 * Set our bit in the bitmask of napping threads unless all the
2116 * other threads are already napping, in which case we send this
2119 ld r5,HSTATE_KVM_VCORE(r13)
2120 lbz r6,HSTATE_PTID(r13)
2121 lwz r8,VCORE_ENTRY_EXIT(r5)
2125 addi r6,r5,VCORE_NAPPING_THREADS
2133 /* order napping_threads update vs testing entry_exit_count */
2136 stb r0,HSTATE_NAPPING(r13)
2137 lwz r7,VCORE_ENTRY_EXIT(r5)
2139 bge 33f /* another thread already exiting */
2142 * Although not specifically required by the architecture, POWER7
2143 * preserves the following registers in nap mode, even if an SMT mode
2144 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
2145 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
2147 /* Save non-volatile GPRs */
2148 std r14, VCPU_GPR(R14)(r3)
2149 std r15, VCPU_GPR(R15)(r3)
2150 std r16, VCPU_GPR(R16)(r3)
2151 std r17, VCPU_GPR(R17)(r3)
2152 std r18, VCPU_GPR(R18)(r3)
2153 std r19, VCPU_GPR(R19)(r3)
2154 std r20, VCPU_GPR(R20)(r3)
2155 std r21, VCPU_GPR(R21)(r3)
2156 std r22, VCPU_GPR(R22)(r3)
2157 std r23, VCPU_GPR(R23)(r3)
2158 std r24, VCPU_GPR(R24)(r3)
2159 std r25, VCPU_GPR(R25)(r3)
2160 std r26, VCPU_GPR(R26)(r3)
2161 std r27, VCPU_GPR(R27)(r3)
2162 std r28, VCPU_GPR(R28)(r3)
2163 std r29, VCPU_GPR(R29)(r3)
2164 std r30, VCPU_GPR(R30)(r3)
2165 std r31, VCPU_GPR(R31)(r3)
2171 * Take a nap until a decrementer or external or doobell interrupt
2172 * occurs, with PECE1, PECE0 and PECEDP set in LPCR. Also clear the
2173 * runlatch bit before napping.
2175 mfspr r2, SPRN_CTRLF
2177 mtspr SPRN_CTRLT, r2
2180 stb r0,HSTATE_HWTHREAD_REQ(r13)
2182 ori r5,r5,LPCR_PECE0 | LPCR_PECE1
2184 oris r5,r5,LPCR_PECEDP@h
2185 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2189 std r0, HSTATE_SCRATCH0(r13)
2191 ld r0, HSTATE_SCRATCH0(r13)
2203 /* get vcpu pointer */
2204 ld r4, HSTATE_KVM_VCPU(r13)
2206 /* Woken by external or decrementer interrupt */
2207 ld r1, HSTATE_HOST_R1(r13)
2209 /* load up FP state */
2213 ld r14, VCPU_GPR(R14)(r4)
2214 ld r15, VCPU_GPR(R15)(r4)
2215 ld r16, VCPU_GPR(R16)(r4)
2216 ld r17, VCPU_GPR(R17)(r4)
2217 ld r18, VCPU_GPR(R18)(r4)
2218 ld r19, VCPU_GPR(R19)(r4)
2219 ld r20, VCPU_GPR(R20)(r4)
2220 ld r21, VCPU_GPR(R21)(r4)
2221 ld r22, VCPU_GPR(R22)(r4)
2222 ld r23, VCPU_GPR(R23)(r4)
2223 ld r24, VCPU_GPR(R24)(r4)
2224 ld r25, VCPU_GPR(R25)(r4)
2225 ld r26, VCPU_GPR(R26)(r4)
2226 ld r27, VCPU_GPR(R27)(r4)
2227 ld r28, VCPU_GPR(R28)(r4)
2228 ld r29, VCPU_GPR(R29)(r4)
2229 ld r30, VCPU_GPR(R30)(r4)
2230 ld r31, VCPU_GPR(R31)(r4)
2232 /* Check the wake reason in SRR1 to see why we got here */
2233 bl kvmppc_check_wake_reason
2235 /* clear our bit in vcore->napping_threads */
2236 34: ld r5,HSTATE_KVM_VCORE(r13)
2237 lbz r7,HSTATE_PTID(r13)
2240 addi r6,r5,VCORE_NAPPING_THREADS
2246 stb r0,HSTATE_NAPPING(r13)
2248 /* See if the wake reason means we need to exit */
2249 stw r12, VCPU_TRAP(r4)
2254 /* see if any other thread is already exiting */
2255 lwz r0,VCORE_ENTRY_EXIT(r5)
2259 b kvmppc_cede_reentry /* if not go back to guest */
2261 /* cede when already previously prodded case */
2264 stb r0,VCPU_PRODDED(r3)
2265 sync /* order testing prodded vs. clearing ceded */
2266 stb r0,VCPU_CEDED(r3)
2270 /* we've ceded but we want to give control to the host */
2272 b hcall_real_fallback
2274 /* Try to handle a machine check in real mode */
2275 machine_check_realmode:
2276 mr r3, r9 /* get vcpu pointer */
2277 bl kvmppc_realmode_machine_check
2279 cmpdi r3, 0 /* Did we handle MCE ? */
2280 ld r9, HSTATE_KVM_VCPU(r13)
2281 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK
2283 * Deliver unhandled/fatal (e.g. UE) MCE errors to guest through
2284 * machine check interrupt (set HSRR0 to 0x200). And for handled
2285 * errors (no-fatal), just go back to guest execution with current
2286 * HSRR0 instead of exiting guest. This new approach will inject
2287 * machine check to guest for fatal error causing guest to crash.
2289 * The old code used to return to host for unhandled errors which
2290 * was causing guest to hang with soft lockups inside guest and
2291 * makes it difficult to recover guest instance.
2294 ld r11, VCPU_MSR(r9)
2295 bne 2f /* Continue guest execution. */
2296 /* If not, deliver a machine check. SRR0/1 are already set */
2297 li r10, BOOK3S_INTERRUPT_MACHINE_CHECK
2298 ld r11, VCPU_MSR(r9)
2299 bl kvmppc_msr_interrupt
2300 2: b fast_interrupt_c_return
2303 * Check the reason we woke from nap, and take appropriate action.
2305 * 0 if nothing needs to be done
2306 * 1 if something happened that needs to be handled by the host
2307 * -1 if there was a guest wakeup (IPI)
2309 * Also sets r12 to the interrupt vector for any interrupt that needs
2310 * to be handled now by the host (0x500 for external interrupt), or zero.
2312 kvmppc_check_wake_reason:
2315 rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */
2317 rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */
2318 ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
2319 cmpwi r6, 8 /* was it an external interrupt? */
2320 li r12, BOOK3S_INTERRUPT_EXTERNAL
2321 beq kvmppc_read_intr /* if so, see what it was */
2324 cmpwi r6, 6 /* was it the decrementer? */
2327 cmpwi r6, 5 /* privileged doorbell? */
2329 cmpwi r6, 3 /* hypervisor doorbell? */
2331 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2332 li r3, 1 /* anything else, return 1 */
2335 /* hypervisor doorbell */
2336 3: li r12, BOOK3S_INTERRUPT_H_DOORBELL
2341 * Determine what sort of external interrupt is pending (if any).
2343 * 0 if no interrupt is pending
2344 * 1 if an interrupt is pending that needs to be handled by the host
2345 * -1 if there was a guest wakeup IPI (which has now been cleared)
2348 /* see if a host IPI is pending */
2350 lbz r0, HSTATE_HOST_IPI(r13)
2354 /* Now read the interrupt from the ICP */
2355 ld r6, HSTATE_XICS_PHYS(r13)
2361 * Save XIRR for later. Since we get in in reverse endian on LE
2362 * systems, save it byte reversed and fetch it back in host endian.
2364 li r3, HSTATE_SAVED_XIRR
2366 #ifdef __LITTLE_ENDIAN__
2367 lwz r3, HSTATE_SAVED_XIRR(r13)
2371 rlwinm. r3, r3, 0, 0xffffff
2373 beq 1f /* if nothing pending in the ICP */
2375 /* We found something in the ICP...
2377 * If it's not an IPI, stash it in the PACA and return to
2378 * the host, we don't (yet) handle directing real external
2379 * interrupts directly to the guest
2381 cmpwi r3, XICS_IPI /* if there is, is it an IPI? */
2384 /* It's an IPI, clear the MFRR and EOI it */
2387 stbcix r3, r6, r8 /* clear the IPI */
2388 stwcix r0, r6, r7 /* EOI it */
2391 /* We need to re-check host IPI now in case it got set in the
2392 * meantime. If it's clear, we bounce the interrupt to the
2395 lbz r0, HSTATE_HOST_IPI(r13)
2399 /* OK, it's an IPI for us */
2403 42: /* It's not an IPI and it's for the host. We saved a copy of XIRR in
2404 * the PACA earlier, it will be picked up by the host ICP driver
2409 43: /* We raced with the host, we need to resend that IPI, bummer */
2411 stbcix r0, r6, r8 /* set the IPI */
2417 * Save away FP, VMX and VSX registers.
2419 * N.B. r30 and r31 are volatile across this function,
2420 * thus it is not callable from C.
2427 #ifdef CONFIG_ALTIVEC
2429 oris r8,r8,MSR_VEC@h
2430 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2434 oris r8,r8,MSR_VSX@h
2435 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2439 addi r3,r3,VCPU_FPRS
2441 #ifdef CONFIG_ALTIVEC
2443 addi r3,r31,VCPU_VRS
2445 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2447 mfspr r6,SPRN_VRSAVE
2448 stw r6,VCPU_VRSAVE(r31)
2453 * Load up FP, VMX and VSX registers
2455 * N.B. r30 and r31 are volatile across this function,
2456 * thus it is not callable from C.
2463 #ifdef CONFIG_ALTIVEC
2465 oris r8,r8,MSR_VEC@h
2466 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2470 oris r8,r8,MSR_VSX@h
2471 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2475 addi r3,r4,VCPU_FPRS
2477 #ifdef CONFIG_ALTIVEC
2479 addi r3,r31,VCPU_VRS
2481 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2483 lwz r7,VCPU_VRSAVE(r31)
2484 mtspr SPRN_VRSAVE,r7
2490 * We come here if we get any exception or interrupt while we are
2491 * executing host real mode code while in guest MMU context.
2492 * For now just spin, but we should do something better.
2494 kvmppc_bad_host_intr:
2498 * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken
2499 * from VCPU_INTR_MSR and is modified based on the required TM state changes.
2500 * r11 has the guest MSR value (in/out)
2501 * r9 has a vcpu pointer (in)
2502 * r0 is used as a scratch register
2504 kvmppc_msr_interrupt:
2505 rldicl r0, r11, 64 - MSR_TS_S_LG, 62
2506 cmpwi r0, 2 /* Check if we are in transactional state.. */
2507 ld r11, VCPU_INTR_MSR(r9)
2509 /* ... if transactional, change to suspended */
2511 1: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
2515 * This works around a hardware bug on POWER8E processors, where
2516 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a
2517 * performance monitor interrupt. Instead, when we need to have
2518 * an interrupt pending, we have to arrange for a counter to overflow.
2522 mtspr SPRN_MMCR2, r3
2523 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h
2524 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN
2525 mtspr SPRN_MMCR0, r3