Merge branches 'edac-spr', 'edac-igen6' and 'edac-misc' into edac-updates-for-v5.11

[sfrench/cifs-2.6.git] / arch / powerpc / kernel / exceptions-64s.S

diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 07d64883c0b55be4f237cafd168b86576d277a31..4d01f09ecf80819e1275264bd0b06ea8fce1977d 100644 (file)
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -1410,6 +1410,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
  *   If none is found, do a Linux page fault. Linux page faults can happen in
  *   kernel mode due to user copy operations of course.
  *
+ *   KVM: The KVM HDSI handler may perform a load with MSR[DR]=1 in guest
+ *   MMU context, which may cause a DSI in the host, which must go to the
+ *   KVM handler. MSR[IR] is not enabled, so the real-mode handler will
+ *   always be used regardless of AIL setting.
+ *
  * - Radix MMU
  *   The hardware loads from the Linux page table directly, so a fault goes
  *   immediately to Linux page fault.
@@ -1420,10 +1425,8 @@ INT_DEFINE_BEGIN(data_access)
        IVEC=0x300
        IDAR=1
        IDSISR=1
-#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
        IKVM_SKIP=1
        IKVM_REAL=1
-#endif
 INT_DEFINE_END(data_access)
 
 EXC_REAL_BEGIN(data_access, 0x300, 0x80)
@@ -1462,6 +1465,8 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
  *   ppc64_bolted_size (first segment). The kernel handler must avoid stomping
  *   on user-handler data structures.
  *
+ *   KVM: Same as 0x300, DSLB must test for KVM guest.
+ *
  * A dedicated save area EXSLB is used (XXX: but it actually need not be
  * these days, we could use EXGEN).
  */
@@ -1470,10 +1475,8 @@ INT_DEFINE_BEGIN(data_access_slb)
        IAREA=PACA_EXSLB
        IRECONCILE=0
        IDAR=1
-#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
        IKVM_SKIP=1
        IKVM_REAL=1
-#endif
 INT_DEFINE_END(data_access_slb)
 
 EXC_REAL_BEGIN(data_access_slb, 0x380, 0x80)
@@ -2949,15 +2952,8 @@ TRAMP_REAL_BEGIN(stf_barrier_fallback)
        .endr
        blr
 
-TRAMP_REAL_BEGIN(rfi_flush_fallback)
-       SET_SCRATCH0(r13);
-       GET_PACA(r13);
-       std     r1,PACA_EXRFI+EX_R12(r13)
-       ld      r1,PACAKSAVE(r13)
-       std     r9,PACA_EXRFI+EX_R9(r13)
-       std     r10,PACA_EXRFI+EX_R10(r13)
-       std     r11,PACA_EXRFI+EX_R11(r13)
-       mfctr   r9
+/* Clobbers r10, r11, ctr */
+.macro L1D_DISPLACEMENT_FLUSH
        ld      r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
        ld      r11,PACA_L1D_FLUSH_SIZE(r13)
        srdi    r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
@@ -2968,7 +2964,7 @@ TRAMP_REAL_BEGIN(rfi_flush_fallback)
        sync
 
        /*
-        * The load adresses are at staggered offsets within cachelines,
+        * The load addresses are at staggered offsets within cachelines,
         * which suits some pipelines better (on others it should not
         * hurt).
         */
@@ -2983,7 +2979,30 @@ TRAMP_REAL_BEGIN(rfi_flush_fallback)
        ld      r11,(0x80 + 8)*7(r10)
        addi    r10,r10,0x80*8
        bdnz    1b
+.endm
 
+TRAMP_REAL_BEGIN(entry_flush_fallback)
+       std     r9,PACA_EXRFI+EX_R9(r13)
+       std     r10,PACA_EXRFI+EX_R10(r13)
+       std     r11,PACA_EXRFI+EX_R11(r13)
+       mfctr   r9
+       L1D_DISPLACEMENT_FLUSH
+       mtctr   r9
+       ld      r9,PACA_EXRFI+EX_R9(r13)
+       ld      r10,PACA_EXRFI+EX_R10(r13)
+       ld      r11,PACA_EXRFI+EX_R11(r13)
+       blr
+
+TRAMP_REAL_BEGIN(rfi_flush_fallback)
+       SET_SCRATCH0(r13);
+       GET_PACA(r13);
+       std     r1,PACA_EXRFI+EX_R12(r13)
+       ld      r1,PACAKSAVE(r13)
+       std     r9,PACA_EXRFI+EX_R9(r13)
+       std     r10,PACA_EXRFI+EX_R10(r13)
+       std     r11,PACA_EXRFI+EX_R11(r13)
+       mfctr   r9
+       L1D_DISPLACEMENT_FLUSH
        mtctr   r9
        ld      r9,PACA_EXRFI+EX_R9(r13)
        ld      r10,PACA_EXRFI+EX_R10(r13)
@@ -3001,32 +3020,7 @@ TRAMP_REAL_BEGIN(hrfi_flush_fallback)
        std     r10,PACA_EXRFI+EX_R10(r13)
        std     r11,PACA_EXRFI+EX_R11(r13)
        mfctr   r9
-       ld      r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
-       ld      r11,PACA_L1D_FLUSH_SIZE(r13)
-       srdi    r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
-       mtctr   r11
-       DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
-
-       /* order ld/st prior to dcbt stop all streams with flushing */
-       sync
-
-       /*
-        * The load adresses are at staggered offsets within cachelines,
-        * which suits some pipelines better (on others it should not
-        * hurt).
-        */
-1:
-       ld      r11,(0x80 + 8)*0(r10)
-       ld      r11,(0x80 + 8)*1(r10)
-       ld      r11,(0x80 + 8)*2(r10)
-       ld      r11,(0x80 + 8)*3(r10)
-       ld      r11,(0x80 + 8)*4(r10)
-       ld      r11,(0x80 + 8)*5(r10)
-       ld      r11,(0x80 + 8)*6(r10)
-       ld      r11,(0x80 + 8)*7(r10)
-       addi    r10,r10,0x80*8
-       bdnz    1b
-
+       L1D_DISPLACEMENT_FLUSH
        mtctr   r9
        ld      r9,PACA_EXRFI+EX_R9(r13)
        ld      r10,PACA_EXRFI+EX_R10(r13)
@@ -3077,8 +3071,21 @@ TRAMP_REAL_BEGIN(rfscv_flush_fallback)
        RFSCV
 
 USE_TEXT_SECTION()
-       MASKED_INTERRUPT
-       MASKED_INTERRUPT hsrr=1
+
+_GLOBAL(do_uaccess_flush)
+       UACCESS_FLUSH_FIXUP_SECTION
+       nop
+       nop
+       nop
+       blr
+       L1D_DISPLACEMENT_FLUSH
+       blr
+_ASM_NOKPROBE_SYMBOL(do_uaccess_flush)
+EXPORT_SYMBOL(do_uaccess_flush)
+
+
+MASKED_INTERRUPT
+MASKED_INTERRUPT hsrr=1
 
 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
 kvmppc_skip_interrupt: