kernel/entry/common.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <linux/context_tracking.h>
   4 #include <linux/entry-common.h>
   5 #include <linux/livepatch.h>
   6 #include <linux/audit.h>
   7
   8 #define CREATE_TRACE_POINTS
   9 #include <trace/events/syscalls.h>
  10
  11 /**
  12  * enter_from_user_mode - Establish state when coming from user mode
  13  *
  14  * Syscall/interrupt entry disables interrupts, but user mode is traced as
  15  * interrupts enabled. Also with NO_HZ_FULL RCU might be idle.
  16  *
  17  * 1) Tell lockdep that interrupts are disabled
  18  * 2) Invoke context tracking if enabled to reactivate RCU
  19  * 3) Trace interrupts off state
  20  */
  21 static __always_inline void enter_from_user_mode(struct pt_regs *regs)
  22 {
  23         arch_check_user_regs(regs);
  24         lockdep_hardirqs_off(CALLER_ADDR0);
  25
  26         CT_WARN_ON(ct_state() != CONTEXT_USER);
  27         user_exit_irqoff();
  28
  29         instrumentation_begin();
  30         trace_hardirqs_off_finish();
  31         instrumentation_end();
  32 }
  33
  34 static inline void syscall_enter_audit(struct pt_regs *regs, long syscall)
  35 {
  36         if (unlikely(audit_context())) {
  37                 unsigned long args[6];
  38
  39                 syscall_get_arguments(current, regs, args);
  40                 audit_syscall_entry(syscall, args[0], args[1], args[2], args[3]);
  41         }
  42 }
  43
  44 static long syscall_trace_enter(struct pt_regs *regs, long syscall,
  45                                 unsigned long ti_work)
  46 {
  47         long ret = 0;
  48
  49         /* Handle ptrace */
  50         if (ti_work & (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_EMU)) {
  51                 ret = arch_syscall_enter_tracehook(regs);
  52                 if (ret || (ti_work & _TIF_SYSCALL_EMU))
  53                         return -1L;
  54         }
  55
  56         /* Do seccomp after ptrace, to catch any tracer changes. */
  57         if (ti_work & _TIF_SECCOMP) {
  58                 ret = __secure_computing(NULL);
  59                 if (ret == -1L)
  60                         return ret;
  61         }
  62
  63         /* Either of the above might have changed the syscall number */
  64         syscall = syscall_get_nr(current, regs);
  65
  66         if (unlikely(ti_work & _TIF_SYSCALL_TRACEPOINT))
  67                 trace_sys_enter(regs, syscall);
  68
  69         syscall_enter_audit(regs, syscall);
  70
  71         return ret ? : syscall;
  72 }
  73
  74 static __always_inline long
  75 __syscall_enter_from_user_work(struct pt_regs *regs, long syscall)
  76 {
  77         unsigned long ti_work;
  78
  79         ti_work = READ_ONCE(current_thread_info()->flags);
  80         if (ti_work & SYSCALL_ENTER_WORK)
  81                 syscall = syscall_trace_enter(regs, syscall, ti_work);
  82
  83         return syscall;
  84 }
  85
  86 long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall)
  87 {
  88         return __syscall_enter_from_user_work(regs, syscall);
  89 }
  90
  91 noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall)
  92 {
  93         long ret;
  94
  95         enter_from_user_mode(regs);
  96
  97         instrumentation_begin();
  98         local_irq_enable();
  99         ret = __syscall_enter_from_user_work(regs, syscall);
 100         instrumentation_end();
 101
 102         return ret;
 103 }
 104
 105 noinstr void syscall_enter_from_user_mode_prepare(struct pt_regs *regs)
 106 {
 107         enter_from_user_mode(regs);
 108         instrumentation_begin();
 109         local_irq_enable();
 110         instrumentation_end();
 111 }
 112
 113 /**
 114  * exit_to_user_mode - Fixup state when exiting to user mode
 115  *
 116  * Syscall/interupt exit enables interrupts, but the kernel state is
 117  * interrupts disabled when this is invoked. Also tell RCU about it.
 118  *
 119  * 1) Trace interrupts on state
 120  * 2) Invoke context tracking if enabled to adjust RCU state
 121  * 3) Invoke architecture specific last minute exit code, e.g. speculation
 122  *    mitigations, etc.
 123  * 4) Tell lockdep that interrupts are enabled
 124  */
 125 static __always_inline void exit_to_user_mode(void)
 126 {
 127         instrumentation_begin();
 128         trace_hardirqs_on_prepare();
 129         lockdep_hardirqs_on_prepare(CALLER_ADDR0);
 130         instrumentation_end();
 131
 132         user_enter_irqoff();
 133         arch_exit_to_user_mode();
 134         lockdep_hardirqs_on(CALLER_ADDR0);
 135 }
 136
 137 /* Workaround to allow gradual conversion of architecture code */
 138 void __weak arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal) { }
 139
 140 static void handle_signal_work(struct pt_regs *regs, unsigned long ti_work)
 141 {
 142         if (ti_work & _TIF_NOTIFY_SIGNAL)
 143                 tracehook_notify_signal();
 144
 145         arch_do_signal_or_restart(regs, ti_work & _TIF_SIGPENDING);
 146 }
 147
 148 static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
 149                                             unsigned long ti_work)
 150 {
 151         /*
 152          * Before returning to user space ensure that all pending work
 153          * items have been completed.
 154          */
 155         while (ti_work & EXIT_TO_USER_MODE_WORK) {
 156
 157                 local_irq_enable_exit_to_user(ti_work);
 158
 159                 if (ti_work & _TIF_NEED_RESCHED)
 160                         schedule();
 161
 162                 if (ti_work & _TIF_UPROBE)
 163                         uprobe_notify_resume(regs);
 164
 165                 if (ti_work & _TIF_PATCH_PENDING)
 166                         klp_update_patch_state(current);
 167
 168                 if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL))
 169                         handle_signal_work(regs, ti_work);
 170
 171                 if (ti_work & _TIF_NOTIFY_RESUME) {
 172                         tracehook_notify_resume(regs);
 173                         rseq_handle_notify_resume(NULL, regs);
 174                 }
 175
 176                 /* Architecture specific TIF work */
 177                 arch_exit_to_user_mode_work(regs, ti_work);
 178
 179                 /*
 180                  * Disable interrupts and reevaluate the work flags as they
 181                  * might have changed while interrupts and preemption was
 182                  * enabled above.
 183                  */
 184                 local_irq_disable_exit_to_user();
 185                 ti_work = READ_ONCE(current_thread_info()->flags);
 186         }
 187
 188         /* Return the latest work state for arch_exit_to_user_mode() */
 189         return ti_work;
 190 }
 191
 192 static void exit_to_user_mode_prepare(struct pt_regs *regs)
 193 {
 194         unsigned long ti_work = READ_ONCE(current_thread_info()->flags);
 195
 196         lockdep_assert_irqs_disabled();
 197
 198         if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK))
 199                 ti_work = exit_to_user_mode_loop(regs, ti_work);
 200
 201         arch_exit_to_user_mode_prepare(regs, ti_work);
 202
 203         /* Ensure that the address limit is intact and no locks are held */
 204         addr_limit_user_check();
 205         lockdep_assert_irqs_disabled();
 206         lockdep_sys_exit();
 207 }
 208
 209 #ifndef _TIF_SINGLESTEP
 210 static inline bool report_single_step(unsigned long ti_work)
 211 {
 212         return false;
 213 }
 214 #else
 215 /*
 216  * If TIF_SYSCALL_EMU is set, then the only reason to report is when
 217  * TIF_SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP).  This syscall
 218  * instruction has been already reported in syscall_enter_from_user_mode().
 219  */
 220 #define SYSEMU_STEP     (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU)
 221
 222 static inline bool report_single_step(unsigned long ti_work)
 223 {
 224         return (ti_work & SYSEMU_STEP) == _TIF_SINGLESTEP;
 225 }
 226 #endif
 227
 228 static void syscall_exit_work(struct pt_regs *regs, unsigned long ti_work)
 229 {
 230         bool step;
 231
 232         audit_syscall_exit(regs);
 233
 234         if (ti_work & _TIF_SYSCALL_TRACEPOINT)
 235                 trace_sys_exit(regs, syscall_get_return_value(current, regs));
 236
 237         step = report_single_step(ti_work);
 238         if (step || ti_work & _TIF_SYSCALL_TRACE)
 239                 arch_syscall_exit_tracehook(regs, step);
 240 }
 241
 242 /*
 243  * Syscall specific exit to user mode preparation. Runs with interrupts
 244  * enabled.
 245  */
 246 static void syscall_exit_to_user_mode_prepare(struct pt_regs *regs)
 247 {
 248         u32 cached_flags = READ_ONCE(current_thread_info()->flags);
 249         unsigned long nr = syscall_get_nr(current, regs);
 250
 251         CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
 252
 253         if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
 254                 if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr))
 255                         local_irq_enable();
 256         }
 257
 258         rseq_syscall(regs);
 259
 260         /*
 261          * Do one-time syscall specific work. If these work items are
 262          * enabled, we want to run them exactly once per syscall exit with
 263          * interrupts enabled.
 264          */
 265         if (unlikely(cached_flags & SYSCALL_EXIT_WORK))
 266                 syscall_exit_work(regs, cached_flags);
 267 }
 268
 269 __visible noinstr void syscall_exit_to_user_mode(struct pt_regs *regs)
 270 {
 271         instrumentation_begin();
 272         syscall_exit_to_user_mode_prepare(regs);
 273         local_irq_disable_exit_to_user();
 274         exit_to_user_mode_prepare(regs);
 275         instrumentation_end();
 276         exit_to_user_mode();
 277 }
 278
 279 noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs)
 280 {
 281         enter_from_user_mode(regs);
 282 }
 283
 284 noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs)
 285 {
 286         instrumentation_begin();
 287         exit_to_user_mode_prepare(regs);
 288         instrumentation_end();
 289         exit_to_user_mode();
 290 }
 291
 292 noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)
 293 {
 294         irqentry_state_t ret = {
 295                 .exit_rcu = false,
 296         };
 297
 298         if (user_mode(regs)) {
 299                 irqentry_enter_from_user_mode(regs);
 300                 return ret;
 301         }
 302
 303         /*
 304          * If this entry hit the idle task invoke rcu_irq_enter() whether
 305          * RCU is watching or not.
 306          *
 307          * Interupts can nest when the first interrupt invokes softirq
 308          * processing on return which enables interrupts.
 309          *
 310          * Scheduler ticks in the idle task can mark quiescent state and
 311          * terminate a grace period, if and only if the timer interrupt is
 312          * not nested into another interrupt.
 313          *
 314          * Checking for rcu_is_watching() here would prevent the nesting
 315          * interrupt to invoke rcu_irq_enter(). If that nested interrupt is
 316          * the tick then rcu_flavor_sched_clock_irq() would wrongfully
 317          * assume that it is the first interupt and eventually claim
 318          * quiescient state and end grace periods prematurely.
 319          *
 320          * Unconditionally invoke rcu_irq_enter() so RCU state stays
 321          * consistent.
 322          *
 323          * TINY_RCU does not support EQS, so let the compiler eliminate
 324          * this part when enabled.
 325          */
 326         if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) {
 327                 /*
 328                  * If RCU is not watching then the same careful
 329                  * sequence vs. lockdep and tracing is required
 330                  * as in irqentry_enter_from_user_mode().
 331                  */
 332                 lockdep_hardirqs_off(CALLER_ADDR0);
 333                 rcu_irq_enter();
 334                 instrumentation_begin();
 335                 trace_hardirqs_off_finish();
 336                 instrumentation_end();
 337
 338                 ret.exit_rcu = true;
 339                 return ret;
 340         }
 341
 342         /*
 343          * If RCU is watching then RCU only wants to check whether it needs
 344          * to restart the tick in NOHZ mode. rcu_irq_enter_check_tick()
 345          * already contains a warning when RCU is not watching, so no point
 346          * in having another one here.
 347          */
 348         lockdep_hardirqs_off(CALLER_ADDR0);
 349         instrumentation_begin();
 350         rcu_irq_enter_check_tick();
 351         trace_hardirqs_off_finish();
 352         instrumentation_end();
 353
 354         return ret;
 355 }
 356
 357 void irqentry_exit_cond_resched(void)
 358 {
 359         if (!preempt_count()) {
 360                 /* Sanity check RCU and thread stack */
 361                 rcu_irq_exit_check_preempt();
 362                 if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
 363                         WARN_ON_ONCE(!on_thread_stack());
 364                 if (need_resched())
 365                         preempt_schedule_irq();
 366         }
 367 }
 368
 369 noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
 370 {
 371         lockdep_assert_irqs_disabled();
 372
 373         /* Check whether this returns to user mode */
 374         if (user_mode(regs)) {
 375                 irqentry_exit_to_user_mode(regs);
 376         } else if (!regs_irqs_disabled(regs)) {
 377                 /*
 378                  * If RCU was not watching on entry this needs to be done
 379                  * carefully and needs the same ordering of lockdep/tracing
 380                  * and RCU as the return to user mode path.
 381                  */
 382                 if (state.exit_rcu) {
 383                         instrumentation_begin();
 384                         /* Tell the tracer that IRET will enable interrupts */
 385                         trace_hardirqs_on_prepare();
 386                         lockdep_hardirqs_on_prepare(CALLER_ADDR0);
 387                         instrumentation_end();
 388                         rcu_irq_exit();
 389                         lockdep_hardirqs_on(CALLER_ADDR0);
 390                         return;
 391                 }
 392
 393                 instrumentation_begin();
 394                 if (IS_ENABLED(CONFIG_PREEMPTION))
 395                         irqentry_exit_cond_resched();
 396                 /* Covers both tracing and lockdep */
 397                 trace_hardirqs_on();
 398                 instrumentation_end();
 399         } else {
 400                 /*
 401                  * IRQ flags state is correct already. Just tell RCU if it
 402                  * was not watching on entry.
 403                  */
 404                 if (state.exit_rcu)
 405                         rcu_irq_exit();
 406         }
 407 }