x86: fixup more paravirt fallout

[sfrench/cifs-2.6.git] / arch / x86 / kernel / entry_64.S

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 3a058bb16409329f9ee255c57e81ff1cefb8cf2a..6be39a387c5ae66780b0b5f946a71c336980fea8 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -50,6 +50,7 @@
 #include <asm/hw_irq.h>
 #include <asm/page.h>
 #include <asm/irqflags.h>
+#include <asm/paravirt.h>
 
        .code64
 
@@ -57,6 +58,13 @@
 #define retint_kernel retint_restore_args
 #endif 
 
+#ifdef CONFIG_PARAVIRT
+ENTRY(native_irq_enable_syscall_ret)
+       movq    %gs:pda_oldrsp,%rsp
+       swapgs
+       sysretq
+#endif /* CONFIG_PARAVIRT */
+
 
 .macro TRACE_IRQS_IRETQ offset=ARGOFFSET
 #ifdef CONFIG_TRACE_IRQFLAGS
@@ -216,14 +224,21 @@ ENTRY(system_call)
        CFI_DEF_CFA     rsp,PDA_STACKOFFSET
        CFI_REGISTER    rip,rcx
        /*CFI_REGISTER  rflags,r11*/
-       swapgs
+       SWAPGS_UNSAFE_STACK
+       /*
+        * A hypervisor implementation might want to use a label
+        * after the swapgs, so that it can do the swapgs
+        * for the guest and jump here on syscall.
+        */
+ENTRY(system_call_after_swapgs)
+
        movq    %rsp,%gs:pda_oldrsp 
        movq    %gs:pda_kernelstack,%rsp
        /*
         * No need to follow this irqs off/on section - it's straight
         * and short:
         */
-       sti                                     
+       ENABLE_INTERRUPTS(CLBR_NONE)
        SAVE_ARGS 8,1
        movq  %rax,ORIG_RAX-ARGOFFSET(%rsp) 
        movq  %rcx,RIP-ARGOFFSET(%rsp)
@@ -246,7 +261,7 @@ ret_from_sys_call:
 sysret_check:          
        LOCKDEP_SYS_EXIT
        GET_THREAD_INFO(%rcx)
-       cli
+       DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        movl threadinfo_flags(%rcx),%edx
        andl %edi,%edx
@@ -260,9 +275,7 @@ sysret_check:
        CFI_REGISTER    rip,rcx
        RESTORE_ARGS 0,-ARG_SKIP,1
        /*CFI_REGISTER  rflags,r11*/
-       movq    %gs:pda_oldrsp,%rsp
-       swapgs
-       sysretq
+       ENABLE_INTERRUPTS_SYSCALL_RET
 
        CFI_RESTORE_STATE
        /* Handle reschedules */
@@ -271,7 +284,7 @@ sysret_careful:
        bt $TIF_NEED_RESCHED,%edx
        jnc sysret_signal
        TRACE_IRQS_ON
-       sti
+       ENABLE_INTERRUPTS(CLBR_NONE)
        pushq %rdi
        CFI_ADJUST_CFA_OFFSET 8
        call schedule
@@ -282,8 +295,8 @@ sysret_careful:
        /* Handle a signal */ 
 sysret_signal:
        TRACE_IRQS_ON
-       sti
-       testl $(_TIF_SIGPENDING|_TIF_SINGLESTEP|_TIF_MCE_NOTIFY),%edx
+       ENABLE_INTERRUPTS(CLBR_NONE)
+       testl $_TIF_DO_NOTIFY_MASK,%edx
        jz    1f
 
        /* Really a signal */
@@ -295,7 +308,7 @@ sysret_signal:
 1:     movl $_TIF_NEED_RESCHED,%edi
        /* Use IRET because user could have changed frame. This
           works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
-       cli
+       DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        jmp int_with_check
        
@@ -327,7 +340,7 @@ tracesys:
  */
        .globl int_ret_from_sys_call
 int_ret_from_sys_call:
-       cli
+       DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        testl $3,CS-ARGOFFSET(%rsp)
        je retint_restore_args
@@ -349,20 +362,20 @@ int_careful:
        bt $TIF_NEED_RESCHED,%edx
        jnc  int_very_careful
        TRACE_IRQS_ON
-       sti
+       ENABLE_INTERRUPTS(CLBR_NONE)
        pushq %rdi
        CFI_ADJUST_CFA_OFFSET 8
        call schedule
        popq %rdi
        CFI_ADJUST_CFA_OFFSET -8
-       cli
+       DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        jmp int_with_check
 
        /* handle signals and tracing -- both require a full stack frame */
 int_very_careful:
        TRACE_IRQS_ON
-       sti
+       ENABLE_INTERRUPTS(CLBR_NONE)
        SAVE_REST
        /* Check for syscall exit trace */      
        testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
@@ -377,7 +390,7 @@ int_very_careful:
        jmp int_restore_rest
        
 int_signal:
-       testl $(_TIF_SIGPENDING|_TIF_SINGLESTEP|_TIF_MCE_NOTIFY),%edx
+       testl $_TIF_DO_NOTIFY_MASK,%edx
        jz 1f
        movq %rsp,%rdi          # &ptregs -> arg1
        xorl %esi,%esi          # oldset -> arg2
@@ -385,7 +398,7 @@ int_signal:
 1:     movl $_TIF_NEED_RESCHED,%edi    
 int_restore_rest:
        RESTORE_REST
-       cli
+       DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        jmp int_with_check
        CFI_ENDPROC
@@ -506,7 +519,7 @@ END(stub_rt_sigreturn)
        CFI_DEF_CFA_REGISTER    rbp
        testl $3,CS(%rdi)
        je 1f
-       swapgs  
+       SWAPGS
        /* irqcount is used to check if a CPU is already on an interrupt
           stack or not. While this is essentially redundant with preempt_count
           it is a little cheaper to use a separate counter in the PDA
@@ -527,7 +540,7 @@ ENTRY(common_interrupt)
        interrupt do_IRQ
        /* 0(%rsp): oldrsp-ARGOFFSET */
 ret_from_intr:
-       cli     
+       DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        decl %gs:pda_irqcount
        leaveq
@@ -556,64 +569,84 @@ retint_swapgs:            /* return to user-space */
        /*
         * The iretq could re-enable interrupts:
         */
-       cli
+       DISABLE_INTERRUPTS(CLBR_ANY)
        TRACE_IRQS_IRETQ
-       swapgs 
+       SWAPGS
        jmp restore_args
 
 retint_restore_args:   /* return to kernel space */
-       cli
+       DISABLE_INTERRUPTS(CLBR_ANY)
        /*
         * The iretq could re-enable interrupts:
         */
        TRACE_IRQS_IRETQ
 restore_args:
-       RESTORE_ARGS 0,8,0                                              
-iret_label:    
+       RESTORE_ARGS 0,8,0
+
+ENTRY(irq_return)
+       INTERRUPT_RETURN
+
+       .section __ex_table, "a"
+       .quad irq_return, bad_iret
+       .previous
+
+#ifdef CONFIG_PARAVIRT
+ENTRY(native_iret)
        iretq
 
        .section __ex_table,"a"
-       .quad iret_label,bad_iret       
+       .quad native_iret, bad_iret
        .previous
+#endif
+
        .section .fixup,"ax"
-       /* force a signal here? this matches i386 behaviour */
-       /* running with kernel gs */
 bad_iret:
-       movq $11,%rdi   /* SIGSEGV */
-       TRACE_IRQS_ON
-       sti
-       jmp do_exit                     
-       .previous       
-       
+       /*
+        * The iret traps when the %cs or %ss being restored is bogus.
+        * We've lost the original trap vector and error code.
+        * #GPF is the most likely one to get for an invalid selector.
+        * So pretend we completed the iret and took the #GPF in user mode.
+        *
+        * We are now running with the kernel GS after exception recovery.
+        * But error_entry expects us to have user GS to match the user %cs,
+        * so swap back.
+        */
+       pushq $0
+
+       SWAPGS
+       jmp general_protection
+
+       .previous
+
        /* edi: workmask, edx: work */
 retint_careful:
        CFI_RESTORE_STATE
        bt    $TIF_NEED_RESCHED,%edx
        jnc   retint_signal
        TRACE_IRQS_ON
-       sti
+       ENABLE_INTERRUPTS(CLBR_NONE)
        pushq %rdi
        CFI_ADJUST_CFA_OFFSET   8
        call  schedule
        popq %rdi               
        CFI_ADJUST_CFA_OFFSET   -8
        GET_THREAD_INFO(%rcx)
-       cli
+       DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        jmp retint_check
        
 retint_signal:
-       testl $(_TIF_SIGPENDING|_TIF_SINGLESTEP|_TIF_MCE_NOTIFY),%edx
+       testl $_TIF_DO_NOTIFY_MASK,%edx
        jz    retint_swapgs
        TRACE_IRQS_ON
-       sti
+       ENABLE_INTERRUPTS(CLBR_NONE)
        SAVE_REST
        movq $-1,ORIG_RAX(%rsp)                         
        xorl %esi,%esi          # oldset
        movq %rsp,%rdi          # &pt_regs
        call do_notify_resume
        RESTORE_REST
-       cli
+       DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        movl $_TIF_NEED_RESCHED,%edi
        GET_THREAD_INFO(%rcx)
@@ -731,7 +764,7 @@ END(spurious_interrupt)
        rdmsr
        testl %edx,%edx
        js    1f
-       swapgs
+       SWAPGS
        xorl  %ebx,%ebx
 1:
        .if \ist
@@ -747,7 +780,7 @@ END(spurious_interrupt)
        .if \ist
        addq    $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
        .endif
-       cli
+       DISABLE_INTERRUPTS(CLBR_NONE)
        .if \irqtrace
        TRACE_IRQS_OFF
        .endif
@@ -776,10 +809,10 @@ paranoid_swapgs\trace:
        .if \trace
        TRACE_IRQS_IRETQ 0
        .endif
-       swapgs
+       SWAPGS_UNSAFE_STACK
 paranoid_restore\trace:
        RESTORE_ALL 8
-       iretq
+       jmp irq_return
 paranoid_userspace\trace:
        GET_THREAD_INFO(%rcx)
        movl threadinfo_flags(%rcx),%ebx
@@ -794,11 +827,11 @@ paranoid_userspace\trace:
        .if \trace
        TRACE_IRQS_ON
        .endif
-       sti
+       ENABLE_INTERRUPTS(CLBR_NONE)
        xorl %esi,%esi                  /* arg2: oldset */
        movq %rsp,%rdi                  /* arg1: &pt_regs */
        call do_notify_resume
-       cli
+       DISABLE_INTERRUPTS(CLBR_NONE)
        .if \trace
        TRACE_IRQS_OFF
        .endif
@@ -807,9 +840,9 @@ paranoid_schedule\trace:
        .if \trace
        TRACE_IRQS_ON
        .endif
-       sti
+       ENABLE_INTERRUPTS(CLBR_ANY)
        call schedule
-       cli
+       DISABLE_INTERRUPTS(CLBR_ANY)
        .if \trace
        TRACE_IRQS_OFF
        .endif
@@ -862,7 +895,7 @@ KPROBE_ENTRY(error_entry)
        testl $3,CS(%rsp)
        je  error_kernelspace
 error_swapgs:  
-       swapgs
+       SWAPGS
 error_sti:     
        movq %rdi,RDI(%rsp)     
        CFI_REL_OFFSET  rdi,RDI
@@ -874,7 +907,7 @@ error_sti:
 error_exit:
        movl %ebx,%eax
        RESTORE_REST
-       cli
+       DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        GET_THREAD_INFO(%rcx)   
        testl %eax,%eax
@@ -894,7 +927,7 @@ error_kernelspace:
           iret run with kernel gs again, so don't set the user space flag.
           B stepping K8s sometimes report an truncated RIP for IRET 
           exceptions returning to compat mode. Check for these here too. */
-       leaq iret_label(%rip),%rbp
+       leaq irq_return(%rip),%rbp
        cmpq %rbp,RIP(%rsp) 
        je   error_swapgs
        movl %ebp,%ebp  /* zero extend */
@@ -911,12 +944,12 @@ ENTRY(load_gs_index)
        CFI_STARTPROC
        pushf
        CFI_ADJUST_CFA_OFFSET 8
-       cli
-        swapgs
+       DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
+        SWAPGS
 gs_change:     
         movl %edi,%gs   
 2:     mfence          /* workaround */
-       swapgs
+       SWAPGS
         popf
        CFI_ADJUST_CFA_OFFSET -8
         ret
@@ -930,7 +963,7 @@ ENDPROC(load_gs_index)
         .section .fixup,"ax"
        /* running with kernelgs */
 bad_gs: 
-       swapgs                  /* switch back to user gs */
+       SWAPGS                  /* switch back to user gs */
        xorl %eax,%eax
         movl %eax,%gs
         jmp  2b