arch/arm64/kernel/traps.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Based on arch/arm/kernel/traps.c
   4  *
   5  * Copyright (C) 1995-2009 Russell King
   6  * Copyright (C) 2012 ARM Ltd.
   7  */
   8
   9 #include <linux/bug.h>
  10 #include <linux/signal.h>
  11 #include <linux/personality.h>
  12 #include <linux/kallsyms.h>
  13 #include <linux/spinlock.h>
  14 #include <linux/uaccess.h>
  15 #include <linux/hardirq.h>
  16 #include <linux/kdebug.h>
  17 #include <linux/module.h>
  18 #include <linux/kexec.h>
  19 #include <linux/delay.h>
  20 #include <linux/init.h>
  21 #include <linux/sched/signal.h>
  22 #include <linux/sched/debug.h>
  23 #include <linux/sched/task_stack.h>
  24 #include <linux/sizes.h>
  25 #include <linux/syscalls.h>
  26 #include <linux/mm_types.h>
  27 #include <linux/kasan.h>
  28
  29 #include <asm/atomic.h>
  30 #include <asm/bug.h>
  31 #include <asm/cpufeature.h>
  32 #include <asm/daifflags.h>
  33 #include <asm/debug-monitors.h>
  34 #include <asm/esr.h>
  35 #include <asm/insn.h>
  36 #include <asm/traps.h>
  37 #include <asm/smp.h>
  38 #include <asm/stack_pointer.h>
  39 #include <asm/stacktrace.h>
  40 #include <asm/exception.h>
  41 #include <asm/system_misc.h>
  42 #include <asm/sysreg.h>
  43
  44 static const char *handler[]= {
  45         "Synchronous Abort",
  46         "IRQ",
  47         "FIQ",
  48         "Error"
  49 };
  50
  51 int show_unhandled_signals = 0;
  52
  53 static void dump_backtrace_entry(unsigned long where)
  54 {
  55         printk(" %pS\n", (void *)where);
  56 }
  57
  58 static void dump_kernel_instr(const char *lvl, struct pt_regs *regs)
  59 {
  60         unsigned long addr = instruction_pointer(regs);
  61         char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str;
  62         int i;
  63
  64         if (user_mode(regs))
  65                 return;
  66
  67         for (i = -4; i < 1; i++) {
  68                 unsigned int val, bad;
  69
  70                 bad = aarch64_insn_read(&((u32 *)addr)[i], &val);
  71
  72                 if (!bad)
  73                         p += sprintf(p, i == 0 ? "(%08x) " : "%08x ", val);
  74                 else {
  75                         p += sprintf(p, "bad PC value");
  76                         break;
  77                 }
  78         }
  79
  80         printk("%sCode: %s\n", lvl, str);
  81 }
  82
  83 void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
  84 {
  85         struct stackframe frame;
  86         int skip = 0;
  87
  88         pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk);
  89
  90         if (regs) {
  91                 if (user_mode(regs))
  92                         return;
  93                 skip = 1;
  94         }
  95
  96         if (!tsk)
  97                 tsk = current;
  98
  99         if (!try_get_task_stack(tsk))
 100                 return;
 101
 102         if (tsk == current) {
 103                 frame.fp = (unsigned long)__builtin_frame_address(0);
 104                 frame.pc = (unsigned long)dump_backtrace;
 105         } else {
 106                 /*
 107                  * task blocked in __switch_to
 108                  */
 109                 frame.fp = thread_saved_fp(tsk);
 110                 frame.pc = thread_saved_pc(tsk);
 111         }
 112 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 113         frame.graph = 0;
 114 #endif
 115
 116         printk("Call trace:\n");
 117         do {
 118                 /* skip until specified stack frame */
 119                 if (!skip) {
 120                         dump_backtrace_entry(frame.pc);
 121                 } else if (frame.fp == regs->regs[29]) {
 122                         skip = 0;
 123                         /*
 124                          * Mostly, this is the case where this function is
 125                          * called in panic/abort. As exception handler's
 126                          * stack frame does not contain the corresponding pc
 127                          * at which an exception has taken place, use regs->pc
 128                          * instead.
 129                          */
 130                         dump_backtrace_entry(regs->pc);
 131                 }
 132         } while (!unwind_frame(tsk, &frame));
 133
 134         put_task_stack(tsk);
 135 }
 136
 137 void show_stack(struct task_struct *tsk, unsigned long *sp)
 138 {
 139         dump_backtrace(NULL, tsk);
 140         barrier();
 141 }
 142
 143 #ifdef CONFIG_PREEMPT
 144 #define S_PREEMPT " PREEMPT"
 145 #else
 146 #define S_PREEMPT ""
 147 #endif
 148 #define S_SMP " SMP"
 149
 150 static int __die(const char *str, int err, struct pt_regs *regs)
 151 {
 152         static int die_counter;
 153         int ret;
 154
 155         pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP "\n",
 156                  str, err, ++die_counter);
 157
 158         /* trap and error numbers are mostly meaningless on ARM */
 159         ret = notify_die(DIE_OOPS, str, regs, err, 0, SIGSEGV);
 160         if (ret == NOTIFY_STOP)
 161                 return ret;
 162
 163         print_modules();
 164         show_regs(regs);
 165
 166         dump_kernel_instr(KERN_EMERG, regs);
 167
 168         return ret;
 169 }
 170
 171 static DEFINE_RAW_SPINLOCK(die_lock);
 172
 173 /*
 174  * This function is protected against re-entrancy.
 175  */
 176 void die(const char *str, struct pt_regs *regs, int err)
 177 {
 178         int ret;
 179         unsigned long flags;
 180
 181         raw_spin_lock_irqsave(&die_lock, flags);
 182
 183         oops_enter();
 184
 185         console_verbose();
 186         bust_spinlocks(1);
 187         ret = __die(str, err, regs);
 188
 189         if (regs && kexec_should_crash(current))
 190                 crash_kexec(regs);
 191
 192         bust_spinlocks(0);
 193         add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
 194         oops_exit();
 195
 196         if (in_interrupt())
 197                 panic("Fatal exception in interrupt");
 198         if (panic_on_oops)
 199                 panic("Fatal exception");
 200
 201         raw_spin_unlock_irqrestore(&die_lock, flags);
 202
 203         if (ret != NOTIFY_STOP)
 204                 do_exit(SIGSEGV);
 205 }
 206
 207 static void arm64_show_signal(int signo, const char *str)
 208 {
 209         static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
 210                                       DEFAULT_RATELIMIT_BURST);
 211         struct task_struct *tsk = current;
 212         unsigned int esr = tsk->thread.fault_code;
 213         struct pt_regs *regs = task_pt_regs(tsk);
 214
 215         /* Leave if the signal won't be shown */
 216         if (!show_unhandled_signals ||
 217             !unhandled_signal(tsk, signo) ||
 218             !__ratelimit(&rs))
 219                 return;
 220
 221         pr_info("%s[%d]: unhandled exception: ", tsk->comm, task_pid_nr(tsk));
 222         if (esr)
 223                 pr_cont("%s, ESR 0x%08x, ", esr_get_class_string(esr), esr);
 224
 225         pr_cont("%s", str);
 226         print_vma_addr(KERN_CONT " in ", regs->pc);
 227         pr_cont("\n");
 228         __show_regs(regs);
 229 }
 230
 231 void arm64_force_sig_fault(int signo, int code, void __user *addr,
 232                            const char *str)
 233 {
 234         arm64_show_signal(signo, str);
 235         if (signo == SIGKILL)
 236                 force_sig(SIGKILL);
 237         else
 238                 force_sig_fault(signo, code, addr);
 239 }
 240
 241 void arm64_force_sig_mceerr(int code, void __user *addr, short lsb,
 242                             const char *str)
 243 {
 244         arm64_show_signal(SIGBUS, str);
 245         force_sig_mceerr(code, addr, lsb);
 246 }
 247
 248 void arm64_force_sig_ptrace_errno_trap(int errno, void __user *addr,
 249                                        const char *str)
 250 {
 251         arm64_show_signal(SIGTRAP, str);
 252         force_sig_ptrace_errno_trap(errno, addr);
 253 }
 254
 255 void arm64_notify_die(const char *str, struct pt_regs *regs,
 256                       int signo, int sicode, void __user *addr,
 257                       int err)
 258 {
 259         if (user_mode(regs)) {
 260                 WARN_ON(regs != current_pt_regs());
 261                 current->thread.fault_address = 0;
 262                 current->thread.fault_code = err;
 263
 264                 arm64_force_sig_fault(signo, sicode, addr, str);
 265         } else {
 266                 die(str, regs, err);
 267         }
 268 }
 269
 270 void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size)
 271 {
 272         regs->pc += size;
 273
 274         /*
 275          * If we were single stepping, we want to get the step exception after
 276          * we return from the trap.
 277          */
 278         if (user_mode(regs))
 279                 user_fastforward_single_step(current);
 280 }
 281
 282 static LIST_HEAD(undef_hook);
 283 static DEFINE_RAW_SPINLOCK(undef_lock);
 284
 285 void register_undef_hook(struct undef_hook *hook)
 286 {
 287         unsigned long flags;
 288
 289         raw_spin_lock_irqsave(&undef_lock, flags);
 290         list_add(&hook->node, &undef_hook);
 291         raw_spin_unlock_irqrestore(&undef_lock, flags);
 292 }
 293
 294 void unregister_undef_hook(struct undef_hook *hook)
 295 {
 296         unsigned long flags;
 297
 298         raw_spin_lock_irqsave(&undef_lock, flags);
 299         list_del(&hook->node);
 300         raw_spin_unlock_irqrestore(&undef_lock, flags);
 301 }
 302
 303 static int call_undef_hook(struct pt_regs *regs)
 304 {
 305         struct undef_hook *hook;
 306         unsigned long flags;
 307         u32 instr;
 308         int (*fn)(struct pt_regs *regs, u32 instr) = NULL;
 309         void __user *pc = (void __user *)instruction_pointer(regs);
 310
 311         if (!user_mode(regs)) {
 312                 __le32 instr_le;
 313                 if (probe_kernel_address((__force __le32 *)pc, instr_le))
 314                         goto exit;
 315                 instr = le32_to_cpu(instr_le);
 316         } else if (compat_thumb_mode(regs)) {
 317                 /* 16-bit Thumb instruction */
 318                 __le16 instr_le;
 319                 if (get_user(instr_le, (__le16 __user *)pc))
 320                         goto exit;
 321                 instr = le16_to_cpu(instr_le);
 322                 if (aarch32_insn_is_wide(instr)) {
 323                         u32 instr2;
 324
 325                         if (get_user(instr_le, (__le16 __user *)(pc + 2)))
 326                                 goto exit;
 327                         instr2 = le16_to_cpu(instr_le);
 328                         instr = (instr << 16) | instr2;
 329                 }
 330         } else {
 331                 /* 32-bit ARM instruction */
 332                 __le32 instr_le;
 333                 if (get_user(instr_le, (__le32 __user *)pc))
 334                         goto exit;
 335                 instr = le32_to_cpu(instr_le);
 336         }
 337
 338         raw_spin_lock_irqsave(&undef_lock, flags);
 339         list_for_each_entry(hook, &undef_hook, node)
 340                 if ((instr & hook->instr_mask) == hook->instr_val &&
 341                         (regs->pstate & hook->pstate_mask) == hook->pstate_val)
 342                         fn = hook->fn;
 343
 344         raw_spin_unlock_irqrestore(&undef_lock, flags);
 345 exit:
 346         return fn ? fn(regs, instr) : 1;
 347 }
 348
 349 void force_signal_inject(int signal, int code, unsigned long address)
 350 {
 351         const char *desc;
 352         struct pt_regs *regs = current_pt_regs();
 353
 354         if (WARN_ON(!user_mode(regs)))
 355                 return;
 356
 357         switch (signal) {
 358         case SIGILL:
 359                 desc = "undefined instruction";
 360                 break;
 361         case SIGSEGV:
 362                 desc = "illegal memory access";
 363                 break;
 364         default:
 365                 desc = "unknown or unrecoverable error";
 366                 break;
 367         }
 368
 369         /* Force signals we don't understand to SIGKILL */
 370         if (WARN_ON(signal != SIGKILL &&
 371                     siginfo_layout(signal, code) != SIL_FAULT)) {
 372                 signal = SIGKILL;
 373         }
 374
 375         arm64_notify_die(desc, regs, signal, code, (void __user *)address, 0);
 376 }
 377
 378 /*
 379  * Set up process info to signal segmentation fault - called on access error.
 380  */
 381 void arm64_notify_segfault(unsigned long addr)
 382 {
 383         int code;
 384
 385         down_read(&current->mm->mmap_sem);
 386         if (find_vma(current->mm, addr) == NULL)
 387                 code = SEGV_MAPERR;
 388         else
 389                 code = SEGV_ACCERR;
 390         up_read(&current->mm->mmap_sem);
 391
 392         force_signal_inject(SIGSEGV, code, addr);
 393 }
 394
 395 asmlinkage void __exception do_undefinstr(struct pt_regs *regs)
 396 {
 397         /* check for AArch32 breakpoint instructions */
 398         if (!aarch32_break_handler(regs))
 399                 return;
 400
 401         if (call_undef_hook(regs) == 0)
 402                 return;
 403
 404         BUG_ON(!user_mode(regs));
 405         force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
 406 }
 407
 408 #define __user_cache_maint(insn, address, res)                  \
 409         if (address >= user_addr_max()) {                       \
 410                 res = -EFAULT;                                  \
 411         } else {                                                \
 412                 uaccess_ttbr0_enable();                         \
 413                 asm volatile (                                  \
 414                         "1:     " insn ", %1\n"                 \
 415                         "       mov     %w0, #0\n"              \
 416                         "2:\n"                                  \
 417                         "       .pushsection .fixup,\"ax\"\n"   \
 418                         "       .align  2\n"                    \
 419                         "3:     mov     %w0, %w2\n"             \
 420                         "       b       2b\n"                   \
 421                         "       .popsection\n"                  \
 422                         _ASM_EXTABLE(1b, 3b)                    \
 423                         : "=r" (res)                            \
 424                         : "r" (address), "i" (-EFAULT));        \
 425                 uaccess_ttbr0_disable();                        \
 426         }
 427
 428 static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs)
 429 {
 430         unsigned long address;
 431         int rt = ESR_ELx_SYS64_ISS_RT(esr);
 432         int crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
 433         int ret = 0;
 434
 435         address = untagged_addr(pt_regs_read_reg(regs, rt));
 436
 437         switch (crm) {
 438         case ESR_ELx_SYS64_ISS_CRM_DC_CVAU:     /* DC CVAU, gets promoted */
 439                 __user_cache_maint("dc civac", address, ret);
 440                 break;
 441         case ESR_ELx_SYS64_ISS_CRM_DC_CVAC:     /* DC CVAC, gets promoted */
 442                 __user_cache_maint("dc civac", address, ret);
 443                 break;
 444         case ESR_ELx_SYS64_ISS_CRM_DC_CVADP:    /* DC CVADP */
 445                 __user_cache_maint("sys 3, c7, c13, 1", address, ret);
 446                 break;
 447         case ESR_ELx_SYS64_ISS_CRM_DC_CVAP:     /* DC CVAP */
 448                 __user_cache_maint("sys 3, c7, c12, 1", address, ret);
 449                 break;
 450         case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC:    /* DC CIVAC */
 451                 __user_cache_maint("dc civac", address, ret);
 452                 break;
 453         case ESR_ELx_SYS64_ISS_CRM_IC_IVAU:     /* IC IVAU */
 454                 __user_cache_maint("ic ivau", address, ret);
 455                 break;
 456         default:
 457                 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
 458                 return;
 459         }
 460
 461         if (ret)
 462                 arm64_notify_segfault(address);
 463         else
 464                 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 465 }
 466
 467 static void ctr_read_handler(unsigned int esr, struct pt_regs *regs)
 468 {
 469         int rt = ESR_ELx_SYS64_ISS_RT(esr);
 470         unsigned long val = arm64_ftr_reg_user_value(&arm64_ftr_reg_ctrel0);
 471
 472         pt_regs_write_reg(regs, rt, val);
 473
 474         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 475 }
 476
 477 static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs)
 478 {
 479         int rt = ESR_ELx_SYS64_ISS_RT(esr);
 480
 481         pt_regs_write_reg(regs, rt, arch_timer_read_counter());
 482         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 483 }
 484
 485 static void cntfrq_read_handler(unsigned int esr, struct pt_regs *regs)
 486 {
 487         int rt = ESR_ELx_SYS64_ISS_RT(esr);
 488
 489         pt_regs_write_reg(regs, rt, arch_timer_get_rate());
 490         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 491 }
 492
 493 static void mrs_handler(unsigned int esr, struct pt_regs *regs)
 494 {
 495         u32 sysreg, rt;
 496
 497         rt = ESR_ELx_SYS64_ISS_RT(esr);
 498         sysreg = esr_sys64_to_sysreg(esr);
 499
 500         if (do_emulate_mrs(regs, sysreg, rt) != 0)
 501                 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
 502 }
 503
 504 static void wfi_handler(unsigned int esr, struct pt_regs *regs)
 505 {
 506         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 507 }
 508
 509 struct sys64_hook {
 510         unsigned int esr_mask;
 511         unsigned int esr_val;
 512         void (*handler)(unsigned int esr, struct pt_regs *regs);
 513 };
 514
 515 static struct sys64_hook sys64_hooks[] = {
 516         {
 517                 .esr_mask = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_MASK,
 518                 .esr_val = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL,
 519                 .handler = user_cache_maint_handler,
 520         },
 521         {
 522                 /* Trap read access to CTR_EL0 */
 523                 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
 524                 .esr_val = ESR_ELx_SYS64_ISS_SYS_CTR_READ,
 525                 .handler = ctr_read_handler,
 526         },
 527         {
 528                 /* Trap read access to CNTVCT_EL0 */
 529                 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
 530                 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCT,
 531                 .handler = cntvct_read_handler,
 532         },
 533         {
 534                 /* Trap read access to CNTFRQ_EL0 */
 535                 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
 536                 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTFRQ,
 537                 .handler = cntfrq_read_handler,
 538         },
 539         {
 540                 /* Trap read access to CPUID registers */
 541                 .esr_mask = ESR_ELx_SYS64_ISS_SYS_MRS_OP_MASK,
 542                 .esr_val = ESR_ELx_SYS64_ISS_SYS_MRS_OP_VAL,
 543                 .handler = mrs_handler,
 544         },
 545         {
 546                 /* Trap WFI instructions executed in userspace */
 547                 .esr_mask = ESR_ELx_WFx_MASK,
 548                 .esr_val = ESR_ELx_WFx_WFI_VAL,
 549                 .handler = wfi_handler,
 550         },
 551         {},
 552 };
 553
 554
 555 #ifdef CONFIG_COMPAT
 556 #define PSTATE_IT_1_0_SHIFT     25
 557 #define PSTATE_IT_1_0_MASK      (0x3 << PSTATE_IT_1_0_SHIFT)
 558 #define PSTATE_IT_7_2_SHIFT     10
 559 #define PSTATE_IT_7_2_MASK      (0x3f << PSTATE_IT_7_2_SHIFT)
 560
 561 static u32 compat_get_it_state(struct pt_regs *regs)
 562 {
 563         u32 it, pstate = regs->pstate;
 564
 565         it  = (pstate & PSTATE_IT_1_0_MASK) >> PSTATE_IT_1_0_SHIFT;
 566         it |= ((pstate & PSTATE_IT_7_2_MASK) >> PSTATE_IT_7_2_SHIFT) << 2;
 567
 568         return it;
 569 }
 570
 571 static void compat_set_it_state(struct pt_regs *regs, u32 it)
 572 {
 573         u32 pstate_it;
 574
 575         pstate_it  = (it << PSTATE_IT_1_0_SHIFT) & PSTATE_IT_1_0_MASK;
 576         pstate_it |= ((it >> 2) << PSTATE_IT_7_2_SHIFT) & PSTATE_IT_7_2_MASK;
 577
 578         regs->pstate &= ~PSR_AA32_IT_MASK;
 579         regs->pstate |= pstate_it;
 580 }
 581
 582 static bool cp15_cond_valid(unsigned int esr, struct pt_regs *regs)
 583 {
 584         int cond;
 585
 586         /* Only a T32 instruction can trap without CV being set */
 587         if (!(esr & ESR_ELx_CV)) {
 588                 u32 it;
 589
 590                 it = compat_get_it_state(regs);
 591                 if (!it)
 592                         return true;
 593
 594                 cond = it >> 4;
 595         } else {
 596                 cond = (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT;
 597         }
 598
 599         return aarch32_opcode_cond_checks[cond](regs->pstate);
 600 }
 601
 602 static void advance_itstate(struct pt_regs *regs)
 603 {
 604         u32 it;
 605
 606         /* ARM mode */
 607         if (!(regs->pstate & PSR_AA32_T_BIT) ||
 608             !(regs->pstate & PSR_AA32_IT_MASK))
 609                 return;
 610
 611         it  = compat_get_it_state(regs);
 612
 613         /*
 614          * If this is the last instruction of the block, wipe the IT
 615          * state. Otherwise advance it.
 616          */
 617         if (!(it & 7))
 618                 it = 0;
 619         else
 620                 it = (it & 0xe0) | ((it << 1) & 0x1f);
 621
 622         compat_set_it_state(regs, it);
 623 }
 624
 625 static void arm64_compat_skip_faulting_instruction(struct pt_regs *regs,
 626                                                    unsigned int sz)
 627 {
 628         advance_itstate(regs);
 629         arm64_skip_faulting_instruction(regs, sz);
 630 }
 631
 632 static void compat_cntfrq_read_handler(unsigned int esr, struct pt_regs *regs)
 633 {
 634         int reg = (esr & ESR_ELx_CP15_32_ISS_RT_MASK) >> ESR_ELx_CP15_32_ISS_RT_SHIFT;
 635
 636         pt_regs_write_reg(regs, reg, arch_timer_get_rate());
 637         arm64_compat_skip_faulting_instruction(regs, 4);
 638 }
 639
 640 static struct sys64_hook cp15_32_hooks[] = {
 641         {
 642                 .esr_mask = ESR_ELx_CP15_32_ISS_SYS_MASK,
 643                 .esr_val = ESR_ELx_CP15_32_ISS_SYS_CNTFRQ,
 644                 .handler = compat_cntfrq_read_handler,
 645         },
 646         {},
 647 };
 648
 649 static void compat_cntvct_read_handler(unsigned int esr, struct pt_regs *regs)
 650 {
 651         int rt = (esr & ESR_ELx_CP15_64_ISS_RT_MASK) >> ESR_ELx_CP15_64_ISS_RT_SHIFT;
 652         int rt2 = (esr & ESR_ELx_CP15_64_ISS_RT2_MASK) >> ESR_ELx_CP15_64_ISS_RT2_SHIFT;
 653         u64 val = arch_timer_read_counter();
 654
 655         pt_regs_write_reg(regs, rt, lower_32_bits(val));
 656         pt_regs_write_reg(regs, rt2, upper_32_bits(val));
 657         arm64_compat_skip_faulting_instruction(regs, 4);
 658 }
 659
 660 static struct sys64_hook cp15_64_hooks[] = {
 661         {
 662                 .esr_mask = ESR_ELx_CP15_64_ISS_SYS_MASK,
 663                 .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCT,
 664                 .handler = compat_cntvct_read_handler,
 665         },
 666         {},
 667 };
 668
 669 asmlinkage void __exception do_cp15instr(unsigned int esr, struct pt_regs *regs)
 670 {
 671         struct sys64_hook *hook, *hook_base;
 672
 673         if (!cp15_cond_valid(esr, regs)) {
 674                 /*
 675                  * There is no T16 variant of a CP access, so we
 676                  * always advance PC by 4 bytes.
 677                  */
 678                 arm64_compat_skip_faulting_instruction(regs, 4);
 679                 return;
 680         }
 681
 682         switch (ESR_ELx_EC(esr)) {
 683         case ESR_ELx_EC_CP15_32:
 684                 hook_base = cp15_32_hooks;
 685                 break;
 686         case ESR_ELx_EC_CP15_64:
 687                 hook_base = cp15_64_hooks;
 688                 break;
 689         default:
 690                 do_undefinstr(regs);
 691                 return;
 692         }
 693
 694         for (hook = hook_base; hook->handler; hook++)
 695                 if ((hook->esr_mask & esr) == hook->esr_val) {
 696                         hook->handler(esr, regs);
 697                         return;
 698                 }
 699
 700         /*
 701          * New cp15 instructions may previously have been undefined at
 702          * EL0. Fall back to our usual undefined instruction handler
 703          * so that we handle these consistently.
 704          */
 705         do_undefinstr(regs);
 706 }
 707 #endif
 708
 709 asmlinkage void __exception do_sysinstr(unsigned int esr, struct pt_regs *regs)
 710 {
 711         struct sys64_hook *hook;
 712
 713         for (hook = sys64_hooks; hook->handler; hook++)
 714                 if ((hook->esr_mask & esr) == hook->esr_val) {
 715                         hook->handler(esr, regs);
 716                         return;
 717                 }
 718
 719         /*
 720          * New SYS instructions may previously have been undefined at EL0. Fall
 721          * back to our usual undefined instruction handler so that we handle
 722          * these consistently.
 723          */
 724         do_undefinstr(regs);
 725 }
 726
 727 static const char *esr_class_str[] = {
 728         [0 ... ESR_ELx_EC_MAX]          = "UNRECOGNIZED EC",
 729         [ESR_ELx_EC_UNKNOWN]            = "Unknown/Uncategorized",
 730         [ESR_ELx_EC_WFx]                = "WFI/WFE",
 731         [ESR_ELx_EC_CP15_32]            = "CP15 MCR/MRC",
 732         [ESR_ELx_EC_CP15_64]            = "CP15 MCRR/MRRC",
 733         [ESR_ELx_EC_CP14_MR]            = "CP14 MCR/MRC",
 734         [ESR_ELx_EC_CP14_LS]            = "CP14 LDC/STC",
 735         [ESR_ELx_EC_FP_ASIMD]           = "ASIMD",
 736         [ESR_ELx_EC_CP10_ID]            = "CP10 MRC/VMRS",
 737         [ESR_ELx_EC_CP14_64]            = "CP14 MCRR/MRRC",
 738         [ESR_ELx_EC_ILL]                = "PSTATE.IL",
 739         [ESR_ELx_EC_SVC32]              = "SVC (AArch32)",
 740         [ESR_ELx_EC_HVC32]              = "HVC (AArch32)",
 741         [ESR_ELx_EC_SMC32]              = "SMC (AArch32)",
 742         [ESR_ELx_EC_SVC64]              = "SVC (AArch64)",
 743         [ESR_ELx_EC_HVC64]              = "HVC (AArch64)",
 744         [ESR_ELx_EC_SMC64]              = "SMC (AArch64)",
 745         [ESR_ELx_EC_SYS64]              = "MSR/MRS (AArch64)",
 746         [ESR_ELx_EC_SVE]                = "SVE",
 747         [ESR_ELx_EC_IMP_DEF]            = "EL3 IMP DEF",
 748         [ESR_ELx_EC_IABT_LOW]           = "IABT (lower EL)",
 749         [ESR_ELx_EC_IABT_CUR]           = "IABT (current EL)",
 750         [ESR_ELx_EC_PC_ALIGN]           = "PC Alignment",
 751         [ESR_ELx_EC_DABT_LOW]           = "DABT (lower EL)",
 752         [ESR_ELx_EC_DABT_CUR]           = "DABT (current EL)",
 753         [ESR_ELx_EC_SP_ALIGN]           = "SP Alignment",
 754         [ESR_ELx_EC_FP_EXC32]           = "FP (AArch32)",
 755         [ESR_ELx_EC_FP_EXC64]           = "FP (AArch64)",
 756         [ESR_ELx_EC_SERROR]             = "SError",
 757         [ESR_ELx_EC_BREAKPT_LOW]        = "Breakpoint (lower EL)",
 758         [ESR_ELx_EC_BREAKPT_CUR]        = "Breakpoint (current EL)",
 759         [ESR_ELx_EC_SOFTSTP_LOW]        = "Software Step (lower EL)",
 760         [ESR_ELx_EC_SOFTSTP_CUR]        = "Software Step (current EL)",
 761         [ESR_ELx_EC_WATCHPT_LOW]        = "Watchpoint (lower EL)",
 762         [ESR_ELx_EC_WATCHPT_CUR]        = "Watchpoint (current EL)",
 763         [ESR_ELx_EC_BKPT32]             = "BKPT (AArch32)",
 764         [ESR_ELx_EC_VECTOR32]           = "Vector catch (AArch32)",
 765         [ESR_ELx_EC_BRK64]              = "BRK (AArch64)",
 766 };
 767
 768 const char *esr_get_class_string(u32 esr)
 769 {
 770         return esr_class_str[ESR_ELx_EC(esr)];
 771 }
 772
 773 /*
 774  * bad_mode handles the impossible case in the exception vector. This is always
 775  * fatal.
 776  */
 777 asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
 778 {
 779         console_verbose();
 780
 781         pr_crit("Bad mode in %s handler detected on CPU%d, code 0x%08x -- %s\n",
 782                 handler[reason], smp_processor_id(), esr,
 783                 esr_get_class_string(esr));
 784
 785         local_daif_mask();
 786         panic("bad mode");
 787 }
 788
 789 /*
 790  * bad_el0_sync handles unexpected, but potentially recoverable synchronous
 791  * exceptions taken from EL0. Unlike bad_mode, this returns.
 792  */
 793 asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
 794 {
 795         void __user *pc = (void __user *)instruction_pointer(regs);
 796
 797         current->thread.fault_address = 0;
 798         current->thread.fault_code = esr;
 799
 800         arm64_force_sig_fault(SIGILL, ILL_ILLOPC, pc,
 801                               "Bad EL0 synchronous exception");
 802 }
 803
 804 #ifdef CONFIG_VMAP_STACK
 805
 806 DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack)
 807         __aligned(16);
 808
 809 asmlinkage void handle_bad_stack(struct pt_regs *regs)
 810 {
 811         unsigned long tsk_stk = (unsigned long)current->stack;
 812         unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
 813         unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
 814         unsigned int esr = read_sysreg(esr_el1);
 815         unsigned long far = read_sysreg(far_el1);
 816
 817         console_verbose();
 818         pr_emerg("Insufficient stack space to handle exception!");
 819
 820         pr_emerg("ESR: 0x%08x -- %s\n", esr, esr_get_class_string(esr));
 821         pr_emerg("FAR: 0x%016lx\n", far);
 822
 823         pr_emerg("Task stack:     [0x%016lx..0x%016lx]\n",
 824                  tsk_stk, tsk_stk + THREAD_SIZE);
 825         pr_emerg("IRQ stack:      [0x%016lx..0x%016lx]\n",
 826                  irq_stk, irq_stk + THREAD_SIZE);
 827         pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
 828                  ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
 829
 830         __show_regs(regs);
 831
 832         /*
 833          * We use nmi_panic to limit the potential for recusive overflows, and
 834          * to get a better stack trace.
 835          */
 836         nmi_panic(NULL, "kernel stack overflow");
 837         cpu_park_loop();
 838 }
 839 #endif
 840
 841 void __noreturn arm64_serror_panic(struct pt_regs *regs, u32 esr)
 842 {
 843         console_verbose();
 844
 845         pr_crit("SError Interrupt on CPU%d, code 0x%08x -- %s\n",
 846                 smp_processor_id(), esr, esr_get_class_string(esr));
 847         if (regs)
 848                 __show_regs(regs);
 849
 850         nmi_panic(regs, "Asynchronous SError Interrupt");
 851
 852         cpu_park_loop();
 853         unreachable();
 854 }
 855
 856 bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned int esr)
 857 {
 858         u32 aet = arm64_ras_serror_get_severity(esr);
 859
 860         switch (aet) {
 861         case ESR_ELx_AET_CE:    /* corrected error */
 862         case ESR_ELx_AET_UEO:   /* restartable, not yet consumed */
 863                 /*
 864                  * The CPU can make progress. We may take UEO again as
 865                  * a more severe error.
 866                  */
 867                 return false;
 868
 869         case ESR_ELx_AET_UEU:   /* Uncorrected Unrecoverable */
 870         case ESR_ELx_AET_UER:   /* Uncorrected Recoverable */
 871                 /*
 872                  * The CPU can't make progress. The exception may have
 873                  * been imprecise.
 874                  */
 875                 return true;
 876
 877         case ESR_ELx_AET_UC:    /* Uncontainable or Uncategorized error */
 878         default:
 879                 /* Error has been silently propagated */
 880                 arm64_serror_panic(regs, esr);
 881         }
 882 }
 883
 884 asmlinkage void do_serror(struct pt_regs *regs, unsigned int esr)
 885 {
 886         const bool was_in_nmi = in_nmi();
 887
 888         if (!was_in_nmi)
 889                 nmi_enter();
 890
 891         /* non-RAS errors are not containable */
 892         if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr))
 893                 arm64_serror_panic(regs, esr);
 894
 895         if (!was_in_nmi)
 896                 nmi_exit();
 897 }
 898
 899 void __pte_error(const char *file, int line, unsigned long val)
 900 {
 901         pr_err("%s:%d: bad pte %016lx.\n", file, line, val);
 902 }
 903
 904 void __pmd_error(const char *file, int line, unsigned long val)
 905 {
 906         pr_err("%s:%d: bad pmd %016lx.\n", file, line, val);
 907 }
 908
 909 void __pud_error(const char *file, int line, unsigned long val)
 910 {
 911         pr_err("%s:%d: bad pud %016lx.\n", file, line, val);
 912 }
 913
 914 void __pgd_error(const char *file, int line, unsigned long val)
 915 {
 916         pr_err("%s:%d: bad pgd %016lx.\n", file, line, val);
 917 }
 918
 919 /* GENERIC_BUG traps */
 920
 921 int is_valid_bugaddr(unsigned long addr)
 922 {
 923         /*
 924          * bug_handler() only called for BRK #BUG_BRK_IMM.
 925          * So the answer is trivial -- any spurious instances with no
 926          * bug table entry will be rejected by report_bug() and passed
 927          * back to the debug-monitors code and handled as a fatal
 928          * unexpected debug exception.
 929          */
 930         return 1;
 931 }
 932
 933 static int bug_handler(struct pt_regs *regs, unsigned int esr)
 934 {
 935         switch (report_bug(regs->pc, regs)) {
 936         case BUG_TRAP_TYPE_BUG:
 937                 die("Oops - BUG", regs, 0);
 938                 break;
 939
 940         case BUG_TRAP_TYPE_WARN:
 941                 break;
 942
 943         default:
 944                 /* unknown/unrecognised bug trap type */
 945                 return DBG_HOOK_ERROR;
 946         }
 947
 948         /* If thread survives, skip over the BUG instruction and continue: */
 949         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 950         return DBG_HOOK_HANDLED;
 951 }
 952
 953 static struct break_hook bug_break_hook = {
 954         .fn = bug_handler,
 955         .imm = BUG_BRK_IMM,
 956 };
 957
 958 #ifdef CONFIG_KASAN_SW_TAGS
 959
 960 #define KASAN_ESR_RECOVER       0x20
 961 #define KASAN_ESR_WRITE 0x10
 962 #define KASAN_ESR_SIZE_MASK     0x0f
 963 #define KASAN_ESR_SIZE(esr)     (1 << ((esr) & KASAN_ESR_SIZE_MASK))
 964
 965 static int kasan_handler(struct pt_regs *regs, unsigned int esr)
 966 {
 967         bool recover = esr & KASAN_ESR_RECOVER;
 968         bool write = esr & KASAN_ESR_WRITE;
 969         size_t size = KASAN_ESR_SIZE(esr);
 970         u64 addr = regs->regs[0];
 971         u64 pc = regs->pc;
 972
 973         kasan_report(addr, size, write, pc);
 974
 975         /*
 976          * The instrumentation allows to control whether we can proceed after
 977          * a crash was detected. This is done by passing the -recover flag to
 978          * the compiler. Disabling recovery allows to generate more compact
 979          * code.
 980          *
 981          * Unfortunately disabling recovery doesn't work for the kernel right
 982          * now. KASAN reporting is disabled in some contexts (for example when
 983          * the allocator accesses slab object metadata; this is controlled by
 984          * current->kasan_depth). All these accesses are detected by the tool,
 985          * even though the reports for them are not printed.
 986          *
 987          * This is something that might be fixed at some point in the future.
 988          */
 989         if (!recover)
 990                 die("Oops - KASAN", regs, 0);
 991
 992         /* If thread survives, skip over the brk instruction and continue: */
 993         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 994         return DBG_HOOK_HANDLED;
 995 }
 996
 997 static struct break_hook kasan_break_hook = {
 998         .fn     = kasan_handler,
 999         .imm    = KASAN_BRK_IMM,
1000         .mask   = KASAN_BRK_MASK,
1001 };
1002 #endif
1003
1004 /*
1005  * Initial handler for AArch64 BRK exceptions
1006  * This handler only used until debug_traps_init().
1007  */
1008 int __init early_brk64(unsigned long addr, unsigned int esr,
1009                 struct pt_regs *regs)
1010 {
1011 #ifdef CONFIG_KASAN_SW_TAGS
1012         unsigned int comment = esr & ESR_ELx_BRK64_ISS_COMMENT_MASK;
1013
1014         if ((comment & ~KASAN_BRK_MASK) == KASAN_BRK_IMM)
1015                 return kasan_handler(regs, esr) != DBG_HOOK_HANDLED;
1016 #endif
1017         return bug_handler(regs, esr) != DBG_HOOK_HANDLED;
1018 }
1019
1020 /* This registration must happen early, before debug_traps_init(). */
1021 void __init trap_init(void)
1022 {
1023         register_kernel_break_hook(&bug_break_hook);
1024 #ifdef CONFIG_KASAN_SW_TAGS
1025         register_kernel_break_hook(&kasan_break_hook);
1026 #endif
1027 }