Merge tag 'pr-20150114-x86-entry' of git://git.kernel.org/pub/scm/linux/kernel/git...

Pull x86/entry enhancements from Andy Lutomirski:

" This is my accumulated x86 entry work, part 1, for 3.20.  The meat
  of this is an IST rework.  When an IST exception interrupts user
  space, we will handle it on the per-thread kernel stack instead of
  on the IST stack.  This sounds messy, but it actually simplifies the
  IST entry/exit code, because it eliminates some ugly games we used
  to play in order to handle rescheduling, signal delivery, etc on the
  way out of an IST exception.

  The IST rework introduces proper context tracking to IST exception
  handlers.  I haven't seen any bug reports, but the old code could
  have incorrectly treated an IST exception handler as an RCU extended
  quiescent state.

  The memory failure change (included in this pull request with
  Borislav and Tony's permission) eliminates a bunch of code that
  is no longer needed now that user memory failure handlers are
  called in process context.

  Finally, this includes a few on Denys' uncontroversial and Obviously
  Correct (tm) cleanups.

  The IST and memory failure changes have been in -next for a while.

  LKML references:

  IST rework:
  http://lkml.kernel.org/r/cover.1416604491.git.luto@amacapital.net

  Memory failure change:
  http://lkml.kernel.org/r/54ab2ffa301102cd6e@agluck-desk.sc.intel.com

  Denys' cleanups:
  http://lkml.kernel.org/r/1420927210-19738-1-git-send-email-dvlasenk@redhat.com
"

This tree semantically depends on and is based on the following RCU commit:

  734d16801349 ("rcu: Make rcu_nmi_enter() handle nesting")

... and for that reason won't be pushed upstream before the RCU bits hit Linus's tree.

Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --combined arch/x86/kernel/entry_64.S
index c653dc437e6b85d26232aae355925c608c8dcae2,7d59df23e5bbfb75e3fc64ec016a21e0944f4c44..501212f14c871db1fe6dd5113d278374dfe05c4f
--- 1/arch/x86/kernel/entry_64.S
--- 2/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@@ -143,8 -143,7 +143,8 @@@ ENDPROC(native_usergs_sysret64
        movq \tmp,RSP+\offset(%rsp)
        movq $__USER_DS,SS+\offset(%rsp)
        movq $__USER_CS,CS+\offset(%rsp)
 -      movq $-1,RCX+\offset(%rsp)
 +      movq RIP+\offset(%rsp),\tmp  /* get rip */
 +      movq \tmp,RCX+\offset(%rsp)  /* copy it to rcx as sysret would do */
        movq R11+\offset(%rsp),\tmp  /* get eflags */
        movq \tmp,EFLAGS+\offset(%rsp)
        .endm
@@@ -156,27 -155,6 +156,6 @@@
        movq \tmp,R11+\offset(%rsp)
        .endm
  
-       .macro FAKE_STACK_FRAME child_rip
-       /* push in order ss, rsp, eflags, cs, rip */
-       xorl %eax, %eax
-       pushq_cfi $__KERNEL_DS /* ss */
-       /*CFI_REL_OFFSET        ss,0*/
-       pushq_cfi %rax /* rsp */
-       CFI_REL_OFFSET  rsp,0
-       pushq_cfi $(X86_EFLAGS_IF|X86_EFLAGS_FIXED) /* eflags - interrupts on */
-       /*CFI_REL_OFFSET        rflags,0*/
-       pushq_cfi $__KERNEL_CS /* cs */
-       /*CFI_REL_OFFSET        cs,0*/
-       pushq_cfi \child_rip /* rip */
-       CFI_REL_OFFSET  rip,0
-       pushq_cfi %rax /* orig rax */
-       .endm
- 
-       .macro UNFAKE_STACK_FRAME
-       addq $8*6, %rsp
-       CFI_ADJUST_CFA_OFFSET   -(6*8)
-       .endm
- 
  /*
   * initial frame state for interrupts (and exceptions without error code)
   */
@@@ -239,51 -217,6 +218,6 @@@
        CFI_REL_OFFSET r15, R15+\offset
        .endm
  
- /* save partial stack frame */
-       .macro SAVE_ARGS_IRQ
-       cld
-       /* start from rbp in pt_regs and jump over */
-       movq_cfi rdi, (RDI-RBP)
-       movq_cfi rsi, (RSI-RBP)
-       movq_cfi rdx, (RDX-RBP)
-       movq_cfi rcx, (RCX-RBP)
-       movq_cfi rax, (RAX-RBP)
-       movq_cfi  r8,  (R8-RBP)
-       movq_cfi  r9,  (R9-RBP)
-       movq_cfi r10, (R10-RBP)
-       movq_cfi r11, (R11-RBP)
- 
-       /* Save rbp so that we can unwind from get_irq_regs() */
-       movq_cfi rbp, 0
- 
-       /* Save previous stack value */
-       movq %rsp, %rsi
- 
-       leaq -RBP(%rsp),%rdi    /* arg1 for handler */
-       testl $3, CS-RBP(%rsi)
-       je 1f
-       SWAPGS
-       /*
-        * irq_count is used to check if a CPU is already on an interrupt stack
-        * or not. While this is essentially redundant with preempt_count it is
-        * a little cheaper to use a separate counter in the PDA (short of
-        * moving irq_enter into assembly, which would be too much work)
-        */
- 1:    incl PER_CPU_VAR(irq_count)
-       cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp
-       CFI_DEF_CFA_REGISTER    rsi
- 
-       /* Store previous stack value */
-       pushq %rsi
-       CFI_ESCAPE      0x0f /* DW_CFA_def_cfa_expression */, 6, \
-                       0x77 /* DW_OP_breg7 */, 0, \
-                       0x06 /* DW_OP_deref */, \
-                       0x08 /* DW_OP_const1u */, SS+8-RBP, \
-                       0x22 /* DW_OP_plus */
-       /* We entered an interrupt context - irqs are off: */
-       TRACE_IRQS_OFF
-       .endm
- 
  ENTRY(save_paranoid)
        XCPT_FRAME 1 RDI+8
        cld
@@@ -627,19 -560,6 +561,6 @@@ END(\label
        FORK_LIKE  vfork
        FIXED_FRAME stub_iopl, sys_iopl
  
- ENTRY(ptregscall_common)
-       DEFAULT_FRAME 1 8       /* offset 8: return address */
-       RESTORE_TOP_OF_STACK %r11, 8
-       movq_cfi_restore R15+8, r15
-       movq_cfi_restore R14+8, r14
-       movq_cfi_restore R13+8, r13
-       movq_cfi_restore R12+8, r12
-       movq_cfi_restore RBP+8, rbp
-       movq_cfi_restore RBX+8, rbx
-       ret $REST_SKIP          /* pop extended registers */
-       CFI_ENDPROC
- END(ptregscall_common)
- 
  ENTRY(stub_execve)
        CFI_STARTPROC
        addq $8, %rsp
@@@ -780,7 -700,48 +701,48 @@@ END(interrupt
        /* reserve pt_regs for scratch regs and rbp */
        subq $ORIG_RAX-RBP, %rsp
        CFI_ADJUST_CFA_OFFSET ORIG_RAX-RBP
-       SAVE_ARGS_IRQ
+       cld
+       /* start from rbp in pt_regs and jump over */
+       movq_cfi rdi, (RDI-RBP)
+       movq_cfi rsi, (RSI-RBP)
+       movq_cfi rdx, (RDX-RBP)
+       movq_cfi rcx, (RCX-RBP)
+       movq_cfi rax, (RAX-RBP)
+       movq_cfi  r8,  (R8-RBP)
+       movq_cfi  r9,  (R9-RBP)
+       movq_cfi r10, (R10-RBP)
+       movq_cfi r11, (R11-RBP)
+ 
+       /* Save rbp so that we can unwind from get_irq_regs() */
+       movq_cfi rbp, 0
+ 
+       /* Save previous stack value */
+       movq %rsp, %rsi
+ 
+       leaq -RBP(%rsp),%rdi    /* arg1 for handler */
+       testl $3, CS-RBP(%rsi)
+       je 1f
+       SWAPGS
+       /*
+        * irq_count is used to check if a CPU is already on an interrupt stack
+        * or not. While this is essentially redundant with preempt_count it is
+        * a little cheaper to use a separate counter in the PDA (short of
+        * moving irq_enter into assembly, which would be too much work)
+        */
+ 1:    incl PER_CPU_VAR(irq_count)
+       cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp
+       CFI_DEF_CFA_REGISTER    rsi
+ 
+       /* Store previous stack value */
+       pushq %rsi
+       CFI_ESCAPE      0x0f /* DW_CFA_def_cfa_expression */, 6, \
+                       0x77 /* DW_OP_breg7 */, 0, \
+                       0x06 /* DW_OP_deref */, \
+                       0x08 /* DW_OP_const1u */, SS+8-RBP, \
+                       0x22 /* DW_OP_plus */
+       /* We entered an interrupt context - irqs are off: */
+       TRACE_IRQS_OFF
+ 
        call \func
        .endm
  
@@@ -1049,6 -1010,11 +1011,11 @@@ ENTRY(\sym
        CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
  
        .if \paranoid
+       .if \paranoid == 1
+       CFI_REMEMBER_STATE
+       testl $3, CS(%rsp)              /* If coming from userspace, switch */
+       jnz 1f                          /* stacks. */
+       .endif
        call save_paranoid
        .else
        call error_entry
@@@ -1089,6 -1055,36 +1056,36 @@@
        jmp error_exit                  /* %ebx: no swapgs flag */
        .endif
  
+       .if \paranoid == 1
+       CFI_RESTORE_STATE
+       /*
+        * Paranoid entry from userspace.  Switch stacks and treat it
+        * as a normal entry.  This means that paranoid handlers
+        * run in real process context if user_mode(regs).
+        */
+ 1:
+       call error_entry
+ 
+       DEFAULT_FRAME 0
+ 
+       movq %rsp,%rdi                  /* pt_regs pointer */
+       call sync_regs
+       movq %rax,%rsp                  /* switch stack */
+ 
+       movq %rsp,%rdi                  /* pt_regs pointer */
+ 
+       .if \has_error_code
+       movq ORIG_RAX(%rsp),%rsi        /* get error code */
+       movq $-1,ORIG_RAX(%rsp)         /* no syscall to restart */
+       .else
+       xorl %esi,%esi                  /* no error code */
+       .endif
+ 
+       call \do_sym
+ 
+       jmp error_exit                  /* %ebx: no swapgs flag */
+       .endif
+ 
        CFI_ENDPROC
  END(\sym)
  .endm
@@@ -1109,7 -1105,7 +1106,7 @@@ idtentry overflow do_overflow has_error
  idtentry bounds do_bounds has_error_code=0
  idtentry invalid_op do_invalid_op has_error_code=0
  idtentry device_not_available do_device_not_available has_error_code=0
- idtentry double_fault do_double_fault has_error_code=1 paranoid=1
+ idtentry double_fault do_double_fault has_error_code=1 paranoid=2
  idtentry coprocessor_segment_overrun do_coprocessor_segment_overrun has_error_code=0
  idtentry invalid_TSS do_invalid_TSS has_error_code=1
  idtentry segment_not_present do_segment_not_present has_error_code=1
@@@ -1290,16 -1286,14 +1287,14 @@@ idtentry machine_check has_error_code=
  #endif
  
        /*
-        * "Paranoid" exit path from exception stack.
-        * Paranoid because this is used by NMIs and cannot take
-        * any kernel state for granted.
-        * We don't do kernel preemption checks here, because only
-        * NMI should be common and it does not enable IRQs and
-        * cannot get reschedule ticks.
+        * "Paranoid" exit path from exception stack.  This is invoked
+        * only on return from non-NMI IST interrupts that came
+        * from kernel space.
         *
-        * "trace" is 0 for the NMI handler only, because irq-tracing
-        * is fundamentally NMI-unsafe. (we cannot change the soft and
-        * hard flags at once, atomically)
+        * We may be returning to very strange contexts (e.g. very early
+        * in syscall entry), so checking for preemption here would
+        * be complicated.  Fortunately, we there's no good reason
+        * to try to handle preemption here.
         */
  
        /* ebx: no swapgs flag */
@@@ -1309,43 -1303,14 +1304,14 @@@ ENTRY(paranoid_exit
        TRACE_IRQS_OFF_DEBUG
        testl %ebx,%ebx                         /* swapgs needed? */
        jnz paranoid_restore
-       testl $3,CS(%rsp)
-       jnz   paranoid_userspace
- paranoid_swapgs:
        TRACE_IRQS_IRETQ 0
        SWAPGS_UNSAFE_STACK
        RESTORE_ALL 8
-       jmp irq_return
+       INTERRUPT_RETURN
  paranoid_restore:
        TRACE_IRQS_IRETQ_DEBUG 0
        RESTORE_ALL 8
-       jmp irq_return
- paranoid_userspace:
-       GET_THREAD_INFO(%rcx)
-       movl TI_flags(%rcx),%ebx
-       andl $_TIF_WORK_MASK,%ebx
-       jz paranoid_swapgs
-       movq %rsp,%rdi                  /* &pt_regs */
-       call sync_regs
-       movq %rax,%rsp                  /* switch stack for scheduling */
-       testl $_TIF_NEED_RESCHED,%ebx
-       jnz paranoid_schedule
-       movl %ebx,%edx                  /* arg3: thread flags */
-       TRACE_IRQS_ON
-       ENABLE_INTERRUPTS(CLBR_NONE)
-       xorl %esi,%esi                  /* arg2: oldset */
-       movq %rsp,%rdi                  /* arg1: &pt_regs */
-       call do_notify_resume
-       DISABLE_INTERRUPTS(CLBR_NONE)
-       TRACE_IRQS_OFF
-       jmp paranoid_userspace
- paranoid_schedule:
-       TRACE_IRQS_ON
-       ENABLE_INTERRUPTS(CLBR_ANY)
-       SCHEDULE_USER
-       DISABLE_INTERRUPTS(CLBR_ANY)
-       TRACE_IRQS_OFF
-       jmp paranoid_userspace
+       INTERRUPT_RETURN
        CFI_ENDPROC
  END(paranoid_exit)