powerpc/64s/interrupt: handle MSR EE and RI in interrupt entry wrapper

The mtmsrd to enable MSR[RI] can be combined with the mtmsrd to enable
MSR[EE] in interrupt entry code, for those interrupts which enable EE.
This helps performance of important synchronous interrupts (e.g., page
faults).

This is similar to what commit dd152f70bdc1 ("powerpc/64s: system call
avoid setting MSR[RI] until we set MSR[EE]") does for system calls.

Do this by enabling EE and RI together at the beginning of the entry
wrapper if PACA_IRQ_HARD_DIS is clear, and only enabling RI if it is
set.

Asynchronous interrupts set PACA_IRQ_HARD_DIS, but synchronous ones
leave it unchanged, so by default they always get EE=1 unless they have
interrupted a caller that is hard disabled. When the sync interrupt
later calls interrupt_cond_local_irq_enable(), it will not require
another mtmsrd because MSR[EE] was already enabled here.

This avoids one mtmsrd L=1 for synchronous interrupts on 64s, which
saves about 20 cycles on POWER9. And for kernel-mode interrupts, both
synchronous and asynchronous, this saves an additional 40 cycles due to
the mtmsrd being moved ahead of mfspr SPRN_AMR, which prevents a SPR
scoreboard stall.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20210922145452.352571-3-npiggin@gmail.com

diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index 2ab7e31c823fdcc4a20cfe0439579a5c592ef0b1..aa65bb774cdb062ad4ae50ba87e2940ac54fda70 100644 (file)
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -149,8 +149,14 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup
 #endif
 
 #ifdef CONFIG_PPC64
-       if (irq_soft_mask_set_return(IRQS_ALL_DISABLED) == IRQS_ENABLED)
-               trace_hardirqs_off();
+       bool trace_enable = false;
+
+       if (IS_ENABLED(CONFIG_TRACE_IRQFLAGS)) {
+               if (irq_soft_mask_set_return(IRQS_ALL_DISABLED) == IRQS_ENABLED)
+                       trace_enable = true;
+       } else {
+               irq_soft_mask_set(IRQS_ALL_DISABLED);
+       }
 
        /*
         * If the interrupt was taken with HARD_DIS clear, then enable MSR[EE].
@@ -164,8 +170,14 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup
                if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
                        BUG_ON(!(regs->msr & MSR_EE));
                __hard_irq_enable();
+       } else {
+               __hard_RI_enable();
        }
 
+       /* Do this when RI=1 because it can cause SLB faults */
+       if (trace_enable)
+               trace_hardirqs_off();
+
        if (user_mode(regs)) {
                kuap_lock();
                CT_WARN_ON(ct_state() != CONTEXT_USER);
@@ -220,13 +232,16 @@ static inline void interrupt_async_enter_prepare(struct pt_regs *regs, struct in
        /* Ensure interrupt_enter_prepare does not enable MSR[EE] */
        local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
 #endif
+       interrupt_enter_prepare(regs, state);
 #ifdef CONFIG_PPC_BOOK3S_64
+       /*
+        * RI=1 is set by interrupt_enter_prepare, so this thread flags access
+        * has to come afterward (it can cause SLB faults).
+        */
        if (cpu_has_feature(CPU_FTR_CTRL) &&
            !test_thread_local_flags(_TLF_RUNLATCH))
                __ppc64_runlatch_on();
 #endif
-
-       interrupt_enter_prepare(regs, state);
        irq_enter();
 }
 
@@ -296,6 +311,8 @@ static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inte
                regs->softe = IRQS_ALL_DISABLED;
        }
 
+       __hard_RI_enable();
+
        /* Don't do any per-CPU operations until interrupt state is fixed */
 
        if (nmi_disables_ftrace(regs)) {
@@ -393,6 +410,8 @@ interrupt_handler long func(struct pt_regs *regs)                   \
 {                                                                      \
        long ret;                                                       \
                                                                        \
+       __hard_RI_enable();                                             \
+                                                                       \
        ret = ____##func (regs);                                        \
                                                                        \
        return ret;                                                     \

diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index a30f563bc7a8396fab1839128ab0d949eda24ac2..4545b7a28aad094262662bf938605885682c80cb 100644 (file)
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -113,7 +113,6 @@ name:
 #define IISIDE         .L_IISIDE_\name\()      /* Uses SRR0/1 not DAR/DSISR */
 #define IDAR           .L_IDAR_\name\()        /* Uses DAR (or SRR0) */
 #define IDSISR         .L_IDSISR_\name\()      /* Uses DSISR (or SRR1) */
-#define ISET_RI                .L_ISET_RI_\name\()     /* Run common code w/ MSR[RI]=1 */
 #define IBRANCH_TO_COMMON      .L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */
 #define IREALMODE_COMMON       .L_IREALMODE_COMMON_\name\() /* Common runs in realmode */
 #define IMASK          .L_IMASK_\name\()       /* IRQ soft-mask bit */
@@ -157,9 +156,6 @@ do_define_int n
        .ifndef IDSISR
                IDSISR=0
        .endif
-       .ifndef ISET_RI
-               ISET_RI=1
-       .endif
        .ifndef IBRANCH_TO_COMMON
                IBRANCH_TO_COMMON=1
        .endif
@@ -512,11 +508,6 @@ DEFINE_FIXED_SYMBOL(\name\()_common_real)
        stb     r10,PACASRR_VALID(r13)
        .endif
 
-       .if ISET_RI
-       li      r10,MSR_RI
-       mtmsrd  r10,1                   /* Set MSR_RI */
-       .endif
-
        .if ISTACK
        .if IKUAP
        kuap_save_amr_and_lock r9, r10, cr1, cr0
@@ -900,11 +891,6 @@ INT_DEFINE_BEGIN(system_reset)
        IVEC=0x100
        IAREA=PACA_EXNMI
        IVIRT=0 /* no virt entry point */
-       /*
-        * MSR_RI is not enabled, because PACA_EXNMI and nmi stack is
-        * being used, so a nested NMI exception would corrupt it.
-        */
-       ISET_RI=0
        ISTACK=0
        IKVM_REAL=1
 INT_DEFINE_END(system_reset)
@@ -977,16 +963,14 @@ TRAMP_REAL_BEGIN(system_reset_fwnmi)
 EXC_COMMON_BEGIN(system_reset_common)
        __GEN_COMMON_ENTRY system_reset
        /*
-        * Increment paca->in_nmi then enable MSR_RI. SLB or MCE will be able
-        * to recover, but nested NMI will notice in_nmi and not recover
-        * because of the use of the NMI stack. in_nmi reentrancy is tested in
-        * system_reset_exception.
+        * Increment paca->in_nmi. When the interrupt entry wrapper later
+        * enable MSR_RI, then SLB or MCE will be able to recover, but a nested
+        * NMI will notice in_nmi and not recover because of the use of the NMI
+        * stack. in_nmi reentrancy is tested in system_reset_exception.
         */
        lhz     r10,PACA_IN_NMI(r13)
        addi    r10,r10,1
        sth     r10,PACA_IN_NMI(r13)
-       li      r10,MSR_RI
-       mtmsrd  r10,1
 
        mr      r10,r1
        ld      r1,PACA_NMI_EMERG_SP(r13)
@@ -1060,12 +1044,6 @@ INT_DEFINE_BEGIN(machine_check_early)
        IAREA=PACA_EXMC
        IVIRT=0 /* no virt entry point */
        IREALMODE_COMMON=1
-       /*
-        * MSR_RI is not enabled, because PACA_EXMC is being used, so a
-        * nested machine check corrupts it. machine_check_common enables
-        * MSR_RI.
-        */
-       ISET_RI=0
        ISTACK=0
        IDAR=1
        IDSISR=1
@@ -1076,7 +1054,6 @@ INT_DEFINE_BEGIN(machine_check)
        IVEC=0x200
        IAREA=PACA_EXMC
        IVIRT=0 /* no virt entry point */
-       ISET_RI=0
        IDAR=1
        IDSISR=1
        IKVM_REAL=1
@@ -1146,9 +1123,6 @@ EXC_COMMON_BEGIN(machine_check_early_common)
 BEGIN_FTR_SECTION
        bl      enable_machine_check
 END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
-       li      r10,MSR_RI
-       mtmsrd  r10,1
-
        addi    r3,r1,STACK_FRAME_OVERHEAD
        bl      machine_check_early
        std     r3,RESULT(r1)   /* Save result */
@@ -1236,10 +1210,6 @@ EXC_COMMON_BEGIN(machine_check_common)
         * save area: PACA_EXMC instead of PACA_EXGEN.
         */
        GEN_COMMON machine_check
-
-       /* Enable MSR_RI when finished with PACA_EXMC */
-       li      r10,MSR_RI
-       mtmsrd  r10,1
        addi    r3,r1,STACK_FRAME_OVERHEAD
        bl      machine_check_exception_async
        b       interrupt_return_srr

diff --git a/arch/powerpc/kernel/fpu.S b/arch/powerpc/kernel/fpu.S
index ba4afe3b5a9cc761b66e12cddc2e7672dbad23a5..f71f2bbd4de646ad8736e024a810883db6ee8037 100644 (file)
--- a/arch/powerpc/kernel/fpu.S
+++ b/arch/powerpc/kernel/fpu.S
@@ -81,7 +81,12 @@ EXPORT_SYMBOL(store_fp_state)
  */
 _GLOBAL(load_up_fpu)
        mfmsr   r5
+#ifdef CONFIG_PPC_BOOK3S_64
+       /* interrupt doesn't set MSR[RI] and HPT can fault on current access */
+       ori     r5,r5,MSR_FP|MSR_RI
+#else
        ori     r5,r5,MSR_FP
+#endif
 #ifdef CONFIG_VSX
 BEGIN_FTR_SECTION
        oris    r5,r5,MSR_VSX@h

diff --git a/arch/powerpc/kernel/vector.S b/arch/powerpc/kernel/vector.S
index ba03eedfdcd89f5d1208a8a085e5c69af9a5c1b8..5cc24d8cce94e2ff2ad514297b07ee9674336c26 100644 (file)
--- a/arch/powerpc/kernel/vector.S
+++ b/arch/powerpc/kernel/vector.S
@@ -47,6 +47,10 @@ EXPORT_SYMBOL(store_vr_state)
  */
 _GLOBAL(load_up_altivec)
        mfmsr   r5                      /* grab the current MSR */
+#ifdef CONFIG_PPC_BOOK3S_64
+       /* interrupt doesn't set MSR[RI] and HPT can fault on current access */
+       ori     r5,r5,MSR_RI
+#endif
        oris    r5,r5,MSR_VEC@h
        MTMSRD(r5)                      /* enable use of AltiVec now */
        isync
@@ -126,6 +130,12 @@ _GLOBAL(load_up_vsx)
        andis.  r5,r12,MSR_VEC@h
        beql+   load_up_altivec         /* skip if already loaded */
 
+#ifdef CONFIG_PPC_BOOK3S_64
+       /* interrupt doesn't set MSR[RI] and HPT can fault on current access */
+       li      r5,MSR_RI
+       mtmsrd  r5,1
+#endif
+
        ld      r4,PACACURRENT(r13)
        addi    r4,r4,THREAD            /* Get THREAD */
        li      r6,1