Merge tag 'tag-chrome-platform-fixes-for-v5.3-rc6' of git://git.kernel.org/pub/scm...

[sfrench/cifs-2.6.git] / arch / x86 / entry / entry_64.S

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 3f5a978a02a7d0906237e517580b6d5440a7fcba..be9ca198c581aea7ed29f4417aae9c1c1b835473 100644 (file)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -519,7 +519,7 @@ ENTRY(interrupt_entry)
        testb   $3, CS-ORIG_RAX+8(%rsp)
        jz      1f
        SWAPGS
-
+       FENCE_SWAPGS_USER_ENTRY
        /*
         * Switch to the thread stack. The IRET frame and orig_ax are
         * on the stack, as well as the return address. RDI..R12 are
@@ -549,8 +549,10 @@ ENTRY(interrupt_entry)
        UNWIND_HINT_FUNC
 
        movq    (%rdi), %rdi
+       jmp     2f
 1:
-
+       FENCE_SWAPGS_KERNEL_ENTRY
+2:
        PUSH_AND_CLEAR_REGS save_ret=1
        ENCODE_FRAME_POINTER 8
 
@@ -1238,6 +1240,13 @@ ENTRY(paranoid_entry)
         */
        SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
 
+       /*
+        * The above SAVE_AND_SWITCH_TO_KERNEL_CR3 macro doesn't do an
+        * unconditional CR3 write, even in the PTI case.  So do an lfence
+        * to prevent GS speculation, regardless of whether PTI is enabled.
+        */
+       FENCE_SWAPGS_KERNEL_ENTRY
+
        ret
 END(paranoid_entry)
 
@@ -1288,6 +1297,7 @@ ENTRY(error_entry)
         * from user mode due to an IRET fault.
         */
        SWAPGS
+       FENCE_SWAPGS_USER_ENTRY
        /* We have user CR3.  Change to kernel CR3. */
        SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
 
@@ -1301,6 +1311,8 @@ ENTRY(error_entry)
        pushq   %r12
        ret
 
+.Lerror_entry_done_lfence:
+       FENCE_SWAPGS_KERNEL_ENTRY
 .Lerror_entry_done:
        ret
 
@@ -1318,7 +1330,7 @@ ENTRY(error_entry)
        cmpq    %rax, RIP+8(%rsp)
        je      .Lbstep_iret
        cmpq    $.Lgs_change, RIP+8(%rsp)
-       jne     .Lerror_entry_done
+       jne     .Lerror_entry_done_lfence
 
        /*
         * hack: .Lgs_change can fail with user gsbase.  If this happens, fix up
@@ -1326,6 +1338,7 @@ ENTRY(error_entry)
         * .Lgs_change's error handler with kernel gsbase.
         */
        SWAPGS
+       FENCE_SWAPGS_USER_ENTRY
        SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
        jmp .Lerror_entry_done
 
@@ -1340,6 +1353,7 @@ ENTRY(error_entry)
         * gsbase and CR3.  Switch to kernel gsbase and CR3:
         */
        SWAPGS
+       FENCE_SWAPGS_USER_ENTRY
        SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
 
        /*
@@ -1431,6 +1445,7 @@ ENTRY(nmi)
 
        swapgs
        cld
+       FENCE_SWAPGS_USER_ENTRY
        SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
        movq    %rsp, %rdx
        movq    PER_CPU_VAR(cpu_current_top_of_stack), %rsp