arch/riscv/kernel/traps.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2012 Regents of the University of California
   4  */
   5
   6 #include <linux/cpu.h>
   7 #include <linux/kernel.h>
   8 #include <linux/init.h>
   9 #include <linux/sched.h>
  10 #include <linux/sched/debug.h>
  11 #include <linux/sched/signal.h>
  12 #include <linux/signal.h>
  13 #include <linux/kdebug.h>
  14 #include <linux/uaccess.h>
  15 #include <linux/kprobes.h>
  16 #include <linux/mm.h>
  17 #include <linux/module.h>
  18 #include <linux/irq.h>
  19 #include <linux/kexec.h>
  20 #include <linux/entry-common.h>
  21
  22 #include <asm/asm-prototypes.h>
  23 #include <asm/bug.h>
  24 #include <asm/csr.h>
  25 #include <asm/processor.h>
  26 #include <asm/ptrace.h>
  27 #include <asm/syscall.h>
  28 #include <asm/thread_info.h>
  29 #include <asm/vector.h>
  30 #include <asm/irq_stack.h>
  31
  32 int show_unhandled_signals = 1;
  33
  34 static DEFINE_SPINLOCK(die_lock);
  35
  36 static void dump_kernel_instr(const char *loglvl, struct pt_regs *regs)
  37 {
  38         char str[sizeof("0000 ") * 12 + 2 + 1], *p = str;
  39         const u16 *insns = (u16 *)instruction_pointer(regs);
  40         long bad;
  41         u16 val;
  42         int i;
  43
  44         for (i = -10; i < 2; i++) {
  45                 bad = get_kernel_nofault(val, &insns[i]);
  46                 if (!bad) {
  47                         p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val);
  48                 } else {
  49                         printk("%sCode: Unable to access instruction at 0x%px.\n",
  50                                loglvl, &insns[i]);
  51                         return;
  52                 }
  53         }
  54         printk("%sCode: %s\n", loglvl, str);
  55 }
  56
  57 void die(struct pt_regs *regs, const char *str)
  58 {
  59         static int die_counter;
  60         int ret;
  61         long cause;
  62         unsigned long flags;
  63
  64         oops_enter();
  65
  66         spin_lock_irqsave(&die_lock, flags);
  67         console_verbose();
  68         bust_spinlocks(1);
  69
  70         pr_emerg("%s [#%d]\n", str, ++die_counter);
  71         print_modules();
  72         if (regs) {
  73                 show_regs(regs);
  74                 dump_kernel_instr(KERN_EMERG, regs);
  75         }
  76
  77         cause = regs ? regs->cause : -1;
  78         ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV);
  79
  80         if (kexec_should_crash(current))
  81                 crash_kexec(regs);
  82
  83         bust_spinlocks(0);
  84         add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
  85         spin_unlock_irqrestore(&die_lock, flags);
  86         oops_exit();
  87
  88         if (in_interrupt())
  89                 panic("Fatal exception in interrupt");
  90         if (panic_on_oops)
  91                 panic("Fatal exception");
  92         if (ret != NOTIFY_STOP)
  93                 make_task_dead(SIGSEGV);
  94 }
  95
  96 void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr)
  97 {
  98         struct task_struct *tsk = current;
  99
 100         if (show_unhandled_signals && unhandled_signal(tsk, signo)
 101             && printk_ratelimit()) {
 102                 pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT,
 103                         tsk->comm, task_pid_nr(tsk), signo, code, addr);
 104                 print_vma_addr(KERN_CONT " in ", instruction_pointer(regs));
 105                 pr_cont("\n");
 106                 __show_regs(regs);
 107         }
 108
 109         force_sig_fault(signo, code, (void __user *)addr);
 110 }
 111
 112 static void do_trap_error(struct pt_regs *regs, int signo, int code,
 113         unsigned long addr, const char *str)
 114 {
 115         current->thread.bad_cause = regs->cause;
 116
 117         if (user_mode(regs)) {
 118                 do_trap(regs, signo, code, addr);
 119         } else {
 120                 if (!fixup_exception(regs))
 121                         die(regs, str);
 122         }
 123 }
 124
 125 #if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE)
 126 #define __trap_section __noinstr_section(".xip.traps")
 127 #else
 128 #define __trap_section noinstr
 129 #endif
 130 #define DO_ERROR_INFO(name, signo, code, str)                                   \
 131 asmlinkage __visible __trap_section void name(struct pt_regs *regs)             \
 132 {                                                                               \
 133         if (user_mode(regs)) {                                                  \
 134                 irqentry_enter_from_user_mode(regs);                            \
 135                 do_trap_error(regs, signo, code, regs->epc, "Oops - " str);     \
 136                 irqentry_exit_to_user_mode(regs);                               \
 137         } else {                                                                \
 138                 irqentry_state_t state = irqentry_nmi_enter(regs);              \
 139                 do_trap_error(regs, signo, code, regs->epc, "Oops - " str);     \
 140                 irqentry_nmi_exit(regs, state);                                 \
 141         }                                                                       \
 142 }
 143
 144 DO_ERROR_INFO(do_trap_unknown,
 145         SIGILL, ILL_ILLTRP, "unknown exception");
 146 DO_ERROR_INFO(do_trap_insn_misaligned,
 147         SIGBUS, BUS_ADRALN, "instruction address misaligned");
 148 DO_ERROR_INFO(do_trap_insn_fault,
 149         SIGSEGV, SEGV_ACCERR, "instruction access fault");
 150
 151 asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs)
 152 {
 153         bool handled;
 154
 155         if (user_mode(regs)) {
 156                 irqentry_enter_from_user_mode(regs);
 157
 158                 local_irq_enable();
 159
 160                 handled = riscv_v_first_use_handler(regs);
 161
 162                 local_irq_disable();
 163
 164                 if (!handled)
 165                         do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
 166                                       "Oops - illegal instruction");
 167
 168                 irqentry_exit_to_user_mode(regs);
 169         } else {
 170                 irqentry_state_t state = irqentry_nmi_enter(regs);
 171
 172                 do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
 173                               "Oops - illegal instruction");
 174
 175                 irqentry_nmi_exit(regs, state);
 176         }
 177 }
 178
 179 DO_ERROR_INFO(do_trap_load_fault,
 180         SIGSEGV, SEGV_ACCERR, "load access fault");
 181 #ifndef CONFIG_RISCV_M_MODE
 182 DO_ERROR_INFO(do_trap_load_misaligned,
 183         SIGBUS, BUS_ADRALN, "Oops - load address misaligned");
 184 DO_ERROR_INFO(do_trap_store_misaligned,
 185         SIGBUS, BUS_ADRALN, "Oops - store (or AMO) address misaligned");
 186 #else
 187 int handle_misaligned_load(struct pt_regs *regs);
 188 int handle_misaligned_store(struct pt_regs *regs);
 189
 190 asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs)
 191 {
 192         if (user_mode(regs)) {
 193                 irqentry_enter_from_user_mode(regs);
 194
 195                 if (handle_misaligned_load(regs))
 196                         do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
 197                               "Oops - load address misaligned");
 198
 199                 irqentry_exit_to_user_mode(regs);
 200         } else {
 201                 irqentry_state_t state = irqentry_nmi_enter(regs);
 202
 203                 if (handle_misaligned_load(regs))
 204                         do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
 205                               "Oops - load address misaligned");
 206
 207                 irqentry_nmi_exit(regs, state);
 208         }
 209 }
 210
 211 asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs)
 212 {
 213         if (user_mode(regs)) {
 214                 irqentry_enter_from_user_mode(regs);
 215
 216                 if (handle_misaligned_store(regs))
 217                         do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
 218                                 "Oops - store (or AMO) address misaligned");
 219
 220                 irqentry_exit_to_user_mode(regs);
 221         } else {
 222                 irqentry_state_t state = irqentry_nmi_enter(regs);
 223
 224                 if (handle_misaligned_store(regs))
 225                         do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
 226                                 "Oops - store (or AMO) address misaligned");
 227
 228                 irqentry_nmi_exit(regs, state);
 229         }
 230 }
 231 #endif
 232 DO_ERROR_INFO(do_trap_store_fault,
 233         SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault");
 234 DO_ERROR_INFO(do_trap_ecall_s,
 235         SIGILL, ILL_ILLTRP, "environment call from S-mode");
 236 DO_ERROR_INFO(do_trap_ecall_m,
 237         SIGILL, ILL_ILLTRP, "environment call from M-mode");
 238
 239 static inline unsigned long get_break_insn_length(unsigned long pc)
 240 {
 241         bug_insn_t insn;
 242
 243         if (get_kernel_nofault(insn, (bug_insn_t *)pc))
 244                 return 0;
 245
 246         return GET_INSN_LENGTH(insn);
 247 }
 248
 249 void handle_break(struct pt_regs *regs)
 250 {
 251 #ifdef CONFIG_KPROBES
 252         if (kprobe_single_step_handler(regs))
 253                 return;
 254
 255         if (kprobe_breakpoint_handler(regs))
 256                 return;
 257 #endif
 258 #ifdef CONFIG_UPROBES
 259         if (uprobe_single_step_handler(regs))
 260                 return;
 261
 262         if (uprobe_breakpoint_handler(regs))
 263                 return;
 264 #endif
 265         current->thread.bad_cause = regs->cause;
 266
 267         if (user_mode(regs))
 268                 force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc);
 269 #ifdef CONFIG_KGDB
 270         else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP)
 271                                                                 == NOTIFY_STOP)
 272                 return;
 273 #endif
 274         else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN)
 275                 regs->epc += get_break_insn_length(regs->epc);
 276         else
 277                 die(regs, "Kernel BUG");
 278 }
 279
 280 asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs)
 281 {
 282         if (user_mode(regs)) {
 283                 irqentry_enter_from_user_mode(regs);
 284
 285                 handle_break(regs);
 286
 287                 irqentry_exit_to_user_mode(regs);
 288         } else {
 289                 irqentry_state_t state = irqentry_nmi_enter(regs);
 290
 291                 handle_break(regs);
 292
 293                 irqentry_nmi_exit(regs, state);
 294         }
 295 }
 296
 297 asmlinkage __visible __trap_section void do_trap_ecall_u(struct pt_regs *regs)
 298 {
 299         if (user_mode(regs)) {
 300                 long syscall = regs->a7;
 301
 302                 regs->epc += 4;
 303                 regs->orig_a0 = regs->a0;
 304
 305                 riscv_v_vstate_discard(regs);
 306
 307                 syscall = syscall_enter_from_user_mode(regs, syscall);
 308
 309                 if (syscall >= 0 && syscall < NR_syscalls)
 310                         syscall_handler(regs, syscall);
 311                 else if (syscall != -1)
 312                         regs->a0 = -ENOSYS;
 313
 314                 syscall_exit_to_user_mode(regs);
 315         } else {
 316                 irqentry_state_t state = irqentry_nmi_enter(regs);
 317
 318                 do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc,
 319                         "Oops - environment call from U-mode");
 320
 321                 irqentry_nmi_exit(regs, state);
 322         }
 323
 324 }
 325
 326 #ifdef CONFIG_MMU
 327 asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
 328 {
 329         irqentry_state_t state = irqentry_enter(regs);
 330
 331         handle_page_fault(regs);
 332
 333         local_irq_disable();
 334
 335         irqentry_exit(regs, state);
 336 }
 337 #endif
 338
 339 static void noinstr handle_riscv_irq(struct pt_regs *regs)
 340 {
 341         struct pt_regs *old_regs;
 342
 343         irq_enter_rcu();
 344         old_regs = set_irq_regs(regs);
 345         handle_arch_irq(regs);
 346         set_irq_regs(old_regs);
 347         irq_exit_rcu();
 348 }
 349
 350 asmlinkage void noinstr do_irq(struct pt_regs *regs)
 351 {
 352         irqentry_state_t state = irqentry_enter(regs);
 353 #ifdef CONFIG_IRQ_STACKS
 354         if (on_thread_stack()) {
 355                 ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
 356                                         + IRQ_STACK_SIZE/sizeof(ulong);
 357                 __asm__ __volatile(
 358                 "addi   sp, sp, -"RISCV_SZPTR  "\n"
 359                 REG_S"  ra, (sp)                \n"
 360                 "addi   sp, sp, -"RISCV_SZPTR  "\n"
 361                 REG_S"  s0, (sp)                \n"
 362                 "addi   s0, sp, 2*"RISCV_SZPTR "\n"
 363                 "move   sp, %[sp]               \n"
 364                 "move   a0, %[regs]             \n"
 365                 "call   handle_riscv_irq        \n"
 366                 "addi   sp, s0, -2*"RISCV_SZPTR"\n"
 367                 REG_L"  s0, (sp)                \n"
 368                 "addi   sp, sp, "RISCV_SZPTR   "\n"
 369                 REG_L"  ra, (sp)                \n"
 370                 "addi   sp, sp, "RISCV_SZPTR   "\n"
 371                 :
 372                 : [sp] "r" (sp), [regs] "r" (regs)
 373                 : "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
 374                   "t0", "t1", "t2", "t3", "t4", "t5", "t6",
 375 #ifndef CONFIG_FRAME_POINTER
 376                   "s0",
 377 #endif
 378                   "memory");
 379         } else
 380 #endif
 381                 handle_riscv_irq(regs);
 382
 383         irqentry_exit(regs, state);
 384 }
 385
 386 #ifdef CONFIG_GENERIC_BUG
 387 int is_valid_bugaddr(unsigned long pc)
 388 {
 389         bug_insn_t insn;
 390
 391         if (pc < VMALLOC_START)
 392                 return 0;
 393         if (get_kernel_nofault(insn, (bug_insn_t *)pc))
 394                 return 0;
 395         if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32)
 396                 return (insn == __BUG_INSN_32);
 397         else
 398                 return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16);
 399 }
 400 #endif /* CONFIG_GENERIC_BUG */
 401
 402 #ifdef CONFIG_VMAP_STACK
 403 /*
 404  * Extra stack space that allows us to provide panic messages when the kernel
 405  * has overflowed its stack.
 406  */
 407 static DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],
 408                 overflow_stack)__aligned(16);
 409 /*
 410  * A temporary stack for use by handle_kernel_stack_overflow.  This is used so
 411  * we can call into C code to get the per-hart overflow stack.  Usage of this
 412  * stack must be protected by spin_shadow_stack.
 413  */
 414 long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE/sizeof(long)] __aligned(16);
 415
 416 /*
 417  * A pseudo spinlock to protect the shadow stack from being used by multiple
 418  * harts concurrently.  This isn't a real spinlock because the lock side must
 419  * be taken without a valid stack and only a single register, it's only taken
 420  * while in the process of panicing anyway so the performance and error
 421  * checking a proper spinlock gives us doesn't matter.
 422  */
 423 unsigned long spin_shadow_stack;
 424
 425 asmlinkage unsigned long get_overflow_stack(void)
 426 {
 427         return (unsigned long)this_cpu_ptr(overflow_stack) +
 428                 OVERFLOW_STACK_SIZE;
 429 }
 430
 431 asmlinkage void handle_bad_stack(struct pt_regs *regs)
 432 {
 433         unsigned long tsk_stk = (unsigned long)current->stack;
 434         unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
 435
 436         /*
 437          * We're done with the shadow stack by this point, as we're on the
 438          * overflow stack.  Tell any other concurrent overflowing harts that
 439          * they can proceed with panicing by releasing the pseudo-spinlock.
 440          *
 441          * This pairs with an amoswap.aq in handle_kernel_stack_overflow.
 442          */
 443         smp_store_release(&spin_shadow_stack, 0);
 444
 445         console_verbose();
 446
 447         pr_emerg("Insufficient stack space to handle exception!\n");
 448         pr_emerg("Task stack:     [0x%016lx..0x%016lx]\n",
 449                         tsk_stk, tsk_stk + THREAD_SIZE);
 450         pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
 451                         ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
 452
 453         __show_regs(regs);
 454         panic("Kernel stack overflow");
 455
 456         for (;;)
 457                 wait_for_interrupt();
 458 }
 459 #endif