Merge tag 'v6.8-rc4' into x86/percpu, to resolve conflicts and refresh the branch

[sfrench/cifs-2.6.git] / arch / x86 / entry / entry_64.S

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 6d236652fceb6df8f09f3291378dea7311b3da12..567d973eed0381810ed276a5b8c8561573a13374 100644 (file)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -18,6 +18,7 @@
  * - SYM_FUNC_START/END:Define functions in the symbol table.
  * - idtentry:         Define exception entry points.
  */
+#include <linux/export.h>
 #include <linux/linkage.h>
 #include <asm/segment.h>
 #include <asm/cache.h>
@@ -34,7 +35,6 @@
 #include <asm/asm.h>
 #include <asm/smap.h>
 #include <asm/pgtable_types.h>
-#include <asm/export.h>
 #include <asm/frame.h>
 #include <asm/trapnr.h>
 #include <asm/nospec-branch.h>
@@ -126,70 +126,8 @@ SYM_INNER_LABEL(entry_SYSCALL_64_after_hwframe, SYM_L_GLOBAL)
         * In the Xen PV case we must use iret anyway.
         */
 
-       ALTERNATIVE "", "jmp    swapgs_restore_regs_and_return_to_usermode", \
-               X86_FEATURE_XENPV
-
-       movq    RCX(%rsp), %rcx
-       movq    RIP(%rsp), %r11
-
-       cmpq    %rcx, %r11      /* SYSRET requires RCX == RIP */
-       jne     swapgs_restore_regs_and_return_to_usermode
-
-       /*
-        * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
-        * in kernel space.  This essentially lets the user take over
-        * the kernel, since userspace controls RSP.
-        *
-        * If width of "canonical tail" ever becomes variable, this will need
-        * to be updated to remain correct on both old and new CPUs.
-        *
-        * Change top bits to match most significant bit (47th or 56th bit
-        * depending on paging mode) in the address.
-        */
-#ifdef CONFIG_X86_5LEVEL
-       ALTERNATIVE "shl $(64 - 48), %rcx; sar $(64 - 48), %rcx", \
-               "shl $(64 - 57), %rcx; sar $(64 - 57), %rcx", X86_FEATURE_LA57
-#else
-       shl     $(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
-       sar     $(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
-#endif
-
-       /* If this changed %rcx, it was not canonical */
-       cmpq    %rcx, %r11
-       jne     swapgs_restore_regs_and_return_to_usermode
-
-       cmpq    $__USER_CS, CS(%rsp)            /* CS must match SYSRET */
-       jne     swapgs_restore_regs_and_return_to_usermode
-
-       movq    R11(%rsp), %r11
-       cmpq    %r11, EFLAGS(%rsp)              /* R11 == RFLAGS */
-       jne     swapgs_restore_regs_and_return_to_usermode
-
-       /*
-        * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot
-        * restore RF properly. If the slowpath sets it for whatever reason, we
-        * need to restore it correctly.
-        *
-        * SYSRET can restore TF, but unlike IRET, restoring TF results in a
-        * trap from userspace immediately after SYSRET.  This would cause an
-        * infinite loop whenever #DB happens with register state that satisfies
-        * the opportunistic SYSRET conditions.  For example, single-stepping
-        * this user code:
-        *
-        *           movq       $stuck_here, %rcx
-        *           pushfq
-        *           popq %r11
-        *   stuck_here:
-        *
-        * would never get past 'stuck_here'.
-        */
-       testq   $(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
-       jnz     swapgs_restore_regs_and_return_to_usermode
-
-       /* nothing to check for RSP */
-
-       cmpq    $__USER_DS, SS(%rsp)            /* SS must match SYSRET */
-       jne     swapgs_restore_regs_and_return_to_usermode
+       ALTERNATIVE "testb %al, %al; jz swapgs_restore_regs_and_return_to_usermode", \
+               "jmp swapgs_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV
 
        /*
         * We win! This label is here just for ease of understanding
@@ -621,17 +559,27 @@ __irqentry_text_end:
 SYM_CODE_START_LOCAL(common_interrupt_return)
 SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
        IBRS_EXIT
-#ifdef CONFIG_DEBUG_ENTRY
-       /* Assert that pt_regs indicates user mode. */
-       testb   $3, CS(%rsp)
-       jnz     1f
-       ud2
-1:
-#endif
 #ifdef CONFIG_XEN_PV
        ALTERNATIVE "", "jmp xenpv_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV
 #endif
+#ifdef CONFIG_PAGE_TABLE_ISOLATION
+       ALTERNATIVE "", "jmp .Lpti_restore_regs_and_return_to_usermode", X86_FEATURE_PTI
+#endif
+
+       STACKLEAK_ERASE
+       POP_REGS
+       add     $8, %rsp        /* orig_ax */
+       UNWIND_HINT_IRET_REGS
 
+.Lswapgs_and_iret:
+       swapgs
+       /* Assert that the IRET frame indicates user mode. */
+       testb   $3, 8(%rsp)
+       jnz     .Lnative_iret
+       ud2
+
+#ifdef CONFIG_PAGE_TABLE_ISOLATION
+.Lpti_restore_regs_and_return_to_usermode:
        POP_REGS pop_rdi=0
 
        /*
@@ -658,13 +606,14 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
         */
        STACKLEAK_ERASE_NOCLOBBER
 
-       SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
+       push    %rax
+       SWITCH_TO_USER_CR3 scratch_reg=%rdi scratch_reg2=%rax
+       pop     %rax
 
        /* Restore RDI. */
        popq    %rdi
-       swapgs
-       jmp     .Lnative_iret
-
+       jmp     .Lswapgs_and_iret
+#endif
 
 SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
 #ifdef CONFIG_DEBUG_ENTRY
@@ -1509,18 +1458,16 @@ nmi_restore:
        iretq
 SYM_CODE_END(asm_exc_nmi)
 
-#ifndef CONFIG_IA32_EMULATION
 /*
  * This handles SYSCALL from 32-bit code.  There is no way to program
  * MSRs to fully disable 32-bit SYSCALL.
  */
-SYM_CODE_START(ignore_sysret)
+SYM_CODE_START(entry_SYSCALL32_ignore)
        UNWIND_HINT_END_OF_STACK
        ENDBR
        mov     $-ENOSYS, %eax
        sysretl
-SYM_CODE_END(ignore_sysret)
-#endif
+SYM_CODE_END(entry_SYSCALL32_ignore)
 
 .pushsection .text, "ax"
        __FUNC_ALIGN