arch/s390/kernel/kprobes.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  *  Kernel Probes (KProbes)
   4  *
   5  * Copyright IBM Corp. 2002, 2006
   6  *
   7  * s390 port, used ppc64 as template. Mike Grundy <grundym@us.ibm.com>
   8  */
   9
  10 #define pr_fmt(fmt) "kprobes: " fmt
  11
  12 #include <linux/moduleloader.h>
  13 #include <linux/kprobes.h>
  14 #include <linux/ptrace.h>
  15 #include <linux/preempt.h>
  16 #include <linux/stop_machine.h>
  17 #include <linux/kdebug.h>
  18 #include <linux/uaccess.h>
  19 #include <linux/extable.h>
  20 #include <linux/module.h>
  21 #include <linux/slab.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/ftrace.h>
  24 #include <asm/set_memory.h>
  25 #include <asm/sections.h>
  26 #include <asm/dis.h>
  27 #include "entry.h"
  28
  29 DEFINE_PER_CPU(struct kprobe *, current_kprobe);
  30 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
  31
  32 struct kretprobe_blackpoint kretprobe_blacklist[] = { };
  33
  34 DEFINE_INSN_CACHE_OPS(s390_insn);
  35
  36 static int insn_page_in_use;
  37
  38 void *alloc_insn_page(void)
  39 {
  40         void *page;
  41
  42         page = module_alloc(PAGE_SIZE);
  43         if (!page)
  44                 return NULL;
  45         __set_memory((unsigned long) page, 1, SET_MEMORY_RO | SET_MEMORY_X);
  46         return page;
  47 }
  48
  49 static void *alloc_s390_insn_page(void)
  50 {
  51         if (xchg(&insn_page_in_use, 1) == 1)
  52                 return NULL;
  53         return &kprobes_insn_page;
  54 }
  55
  56 static void free_s390_insn_page(void *page)
  57 {
  58         xchg(&insn_page_in_use, 0);
  59 }
  60
  61 struct kprobe_insn_cache kprobe_s390_insn_slots = {
  62         .mutex = __MUTEX_INITIALIZER(kprobe_s390_insn_slots.mutex),
  63         .alloc = alloc_s390_insn_page,
  64         .free = free_s390_insn_page,
  65         .pages = LIST_HEAD_INIT(kprobe_s390_insn_slots.pages),
  66         .insn_size = MAX_INSN_SIZE,
  67 };
  68
  69 static void copy_instruction(struct kprobe *p)
  70 {
  71         kprobe_opcode_t insn[MAX_INSN_SIZE];
  72         s64 disp, new_disp;
  73         u64 addr, new_addr;
  74         unsigned int len;
  75
  76         len = insn_length(*p->addr >> 8);
  77         memcpy(&insn, p->addr, len);
  78         p->opcode = insn[0];
  79         if (probe_is_insn_relative_long(&insn[0])) {
  80                 /*
  81                  * For pc-relative instructions in RIL-b or RIL-c format patch
  82                  * the RI2 displacement field. We have already made sure that
  83                  * the insn slot for the patched instruction is within the same
  84                  * 2GB area as the original instruction (either kernel image or
  85                  * module area). Therefore the new displacement will always fit.
  86                  */
  87                 disp = *(s32 *)&insn[1];
  88                 addr = (u64)(unsigned long)p->addr;
  89                 new_addr = (u64)(unsigned long)p->ainsn.insn;
  90                 new_disp = ((addr + (disp * 2)) - new_addr) / 2;
  91                 *(s32 *)&insn[1] = new_disp;
  92         }
  93         s390_kernel_write(p->ainsn.insn, &insn, len);
  94 }
  95 NOKPROBE_SYMBOL(copy_instruction);
  96
  97 static int s390_get_insn_slot(struct kprobe *p)
  98 {
  99         /*
 100          * Get an insn slot that is within the same 2GB area like the original
 101          * instruction. That way instructions with a 32bit signed displacement
 102          * field can be patched and executed within the insn slot.
 103          */
 104         p->ainsn.insn = NULL;
 105         if (is_kernel((unsigned long)p->addr))
 106                 p->ainsn.insn = get_s390_insn_slot();
 107         else if (is_module_addr(p->addr))
 108                 p->ainsn.insn = get_insn_slot();
 109         return p->ainsn.insn ? 0 : -ENOMEM;
 110 }
 111 NOKPROBE_SYMBOL(s390_get_insn_slot);
 112
 113 static void s390_free_insn_slot(struct kprobe *p)
 114 {
 115         if (!p->ainsn.insn)
 116                 return;
 117         if (is_kernel((unsigned long)p->addr))
 118                 free_s390_insn_slot(p->ainsn.insn, 0);
 119         else
 120                 free_insn_slot(p->ainsn.insn, 0);
 121         p->ainsn.insn = NULL;
 122 }
 123 NOKPROBE_SYMBOL(s390_free_insn_slot);
 124
 125 int arch_prepare_kprobe(struct kprobe *p)
 126 {
 127         if ((unsigned long) p->addr & 0x01)
 128                 return -EINVAL;
 129         /* Make sure the probe isn't going on a difficult instruction */
 130         if (probe_is_prohibited_opcode(p->addr))
 131                 return -EINVAL;
 132         if (s390_get_insn_slot(p))
 133                 return -ENOMEM;
 134         copy_instruction(p);
 135         return 0;
 136 }
 137 NOKPROBE_SYMBOL(arch_prepare_kprobe);
 138
 139 struct swap_insn_args {
 140         struct kprobe *p;
 141         unsigned int arm_kprobe : 1;
 142 };
 143
 144 static int swap_instruction(void *data)
 145 {
 146         struct swap_insn_args *args = data;
 147         struct kprobe *p = args->p;
 148         u16 opc;
 149
 150         opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode;
 151         s390_kernel_write(p->addr, &opc, sizeof(opc));
 152         return 0;
 153 }
 154 NOKPROBE_SYMBOL(swap_instruction);
 155
 156 void arch_arm_kprobe(struct kprobe *p)
 157 {
 158         struct swap_insn_args args = {.p = p, .arm_kprobe = 1};
 159
 160         stop_machine_cpuslocked(swap_instruction, &args, NULL);
 161 }
 162 NOKPROBE_SYMBOL(arch_arm_kprobe);
 163
 164 void arch_disarm_kprobe(struct kprobe *p)
 165 {
 166         struct swap_insn_args args = {.p = p, .arm_kprobe = 0};
 167
 168         stop_machine_cpuslocked(swap_instruction, &args, NULL);
 169 }
 170 NOKPROBE_SYMBOL(arch_disarm_kprobe);
 171
 172 void arch_remove_kprobe(struct kprobe *p)
 173 {
 174         s390_free_insn_slot(p);
 175 }
 176 NOKPROBE_SYMBOL(arch_remove_kprobe);
 177
 178 static void enable_singlestep(struct kprobe_ctlblk *kcb,
 179                               struct pt_regs *regs,
 180                               unsigned long ip)
 181 {
 182         struct per_regs per_kprobe;
 183
 184         /* Set up the PER control registers %cr9-%cr11 */
 185         per_kprobe.control = PER_EVENT_IFETCH;
 186         per_kprobe.start = ip;
 187         per_kprobe.end = ip;
 188
 189         /* Save control regs and psw mask */
 190         __ctl_store(kcb->kprobe_saved_ctl, 9, 11);
 191         kcb->kprobe_saved_imask = regs->psw.mask &
 192                 (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT);
 193
 194         /* Set PER control regs, turns on single step for the given address */
 195         __ctl_load(per_kprobe, 9, 11);
 196         regs->psw.mask |= PSW_MASK_PER;
 197         regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT);
 198         regs->psw.addr = ip;
 199 }
 200 NOKPROBE_SYMBOL(enable_singlestep);
 201
 202 static void disable_singlestep(struct kprobe_ctlblk *kcb,
 203                                struct pt_regs *regs,
 204                                unsigned long ip)
 205 {
 206         /* Restore control regs and psw mask, set new psw address */
 207         __ctl_load(kcb->kprobe_saved_ctl, 9, 11);
 208         regs->psw.mask &= ~PSW_MASK_PER;
 209         regs->psw.mask |= kcb->kprobe_saved_imask;
 210         regs->psw.addr = ip;
 211 }
 212 NOKPROBE_SYMBOL(disable_singlestep);
 213
 214 /*
 215  * Activate a kprobe by storing its pointer to current_kprobe. The
 216  * previous kprobe is stored in kcb->prev_kprobe. A stack of up to
 217  * two kprobes can be active, see KPROBE_REENTER.
 218  */
 219 static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p)
 220 {
 221         kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe);
 222         kcb->prev_kprobe.status = kcb->kprobe_status;
 223         __this_cpu_write(current_kprobe, p);
 224 }
 225 NOKPROBE_SYMBOL(push_kprobe);
 226
 227 /*
 228  * Deactivate a kprobe by backing up to the previous state. If the
 229  * current state is KPROBE_REENTER prev_kprobe.kp will be non-NULL,
 230  * for any other state prev_kprobe.kp will be NULL.
 231  */
 232 static void pop_kprobe(struct kprobe_ctlblk *kcb)
 233 {
 234         __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
 235         kcb->kprobe_status = kcb->prev_kprobe.status;
 236 }
 237 NOKPROBE_SYMBOL(pop_kprobe);
 238
 239 void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
 240 {
 241         ri->ret_addr = (kprobe_opcode_t *) regs->gprs[14];
 242         ri->fp = NULL;
 243
 244         /* Replace the return addr with trampoline addr */
 245         regs->gprs[14] = (unsigned long) &__kretprobe_trampoline;
 246 }
 247 NOKPROBE_SYMBOL(arch_prepare_kretprobe);
 248
 249 static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p)
 250 {
 251         switch (kcb->kprobe_status) {
 252         case KPROBE_HIT_SSDONE:
 253         case KPROBE_HIT_ACTIVE:
 254                 kprobes_inc_nmissed_count(p);
 255                 break;
 256         case KPROBE_HIT_SS:
 257         case KPROBE_REENTER:
 258         default:
 259                 /*
 260                  * A kprobe on the code path to single step an instruction
 261                  * is a BUG. The code path resides in the .kprobes.text
 262                  * section and is executed with interrupts disabled.
 263                  */
 264                 pr_err("Failed to recover from reentered kprobes.\n");
 265                 dump_kprobe(p);
 266                 BUG();
 267         }
 268 }
 269 NOKPROBE_SYMBOL(kprobe_reenter_check);
 270
 271 static int kprobe_handler(struct pt_regs *regs)
 272 {
 273         struct kprobe_ctlblk *kcb;
 274         struct kprobe *p;
 275
 276         /*
 277          * We want to disable preemption for the entire duration of kprobe
 278          * processing. That includes the calls to the pre/post handlers
 279          * and single stepping the kprobe instruction.
 280          */
 281         preempt_disable();
 282         kcb = get_kprobe_ctlblk();
 283         p = get_kprobe((void *)(regs->psw.addr - 2));
 284
 285         if (p) {
 286                 if (kprobe_running()) {
 287                         /*
 288                          * We have hit a kprobe while another is still
 289                          * active. This can happen in the pre and post
 290                          * handler. Single step the instruction of the
 291                          * new probe but do not call any handler function
 292                          * of this secondary kprobe.
 293                          * push_kprobe and pop_kprobe saves and restores
 294                          * the currently active kprobe.
 295                          */
 296                         kprobe_reenter_check(kcb, p);
 297                         push_kprobe(kcb, p);
 298                         kcb->kprobe_status = KPROBE_REENTER;
 299                 } else {
 300                         /*
 301                          * If we have no pre-handler or it returned 0, we
 302                          * continue with single stepping. If we have a
 303                          * pre-handler and it returned non-zero, it prepped
 304                          * for changing execution path, so get out doing
 305                          * nothing more here.
 306                          */
 307                         push_kprobe(kcb, p);
 308                         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
 309                         if (p->pre_handler && p->pre_handler(p, regs)) {
 310                                 pop_kprobe(kcb);
 311                                 preempt_enable_no_resched();
 312                                 return 1;
 313                         }
 314                         kcb->kprobe_status = KPROBE_HIT_SS;
 315                 }
 316                 enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn);
 317                 return 1;
 318         } /* else:
 319            * No kprobe at this address and no active kprobe. The trap has
 320            * not been caused by a kprobe breakpoint. The race of breakpoint
 321            * vs. kprobe remove does not exist because on s390 as we use
 322            * stop_machine to arm/disarm the breakpoints.
 323            */
 324         preempt_enable_no_resched();
 325         return 0;
 326 }
 327 NOKPROBE_SYMBOL(kprobe_handler);
 328
 329 /*
 330  * Function return probe trampoline:
 331  *      - init_kprobes() establishes a probepoint here
 332  *      - When the probed function returns, this probe
 333  *              causes the handlers to fire
 334  */
 335 static void __used kretprobe_trampoline_holder(void)
 336 {
 337         asm volatile(".global __kretprobe_trampoline\n"
 338                      "__kretprobe_trampoline: bcr 0,0\n");
 339 }
 340
 341 /*
 342  * Called when the probe at kretprobe trampoline is hit
 343  */
 344 static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
 345 {
 346         regs->psw.addr = __kretprobe_trampoline_handler(regs, NULL);
 347         /*
 348          * By returning a non-zero value, we are telling
 349          * kprobe_handler() that we don't want the post_handler
 350          * to run (and have re-enabled preemption)
 351          */
 352         return 1;
 353 }
 354 NOKPROBE_SYMBOL(trampoline_probe_handler);
 355
 356 /*
 357  * Called after single-stepping.  p->addr is the address of the
 358  * instruction whose first byte has been replaced by the "breakpoint"
 359  * instruction.  To avoid the SMP problems that can occur when we
 360  * temporarily put back the original opcode to single-step, we
 361  * single-stepped a copy of the instruction.  The address of this
 362  * copy is p->ainsn.insn.
 363  */
 364 static void resume_execution(struct kprobe *p, struct pt_regs *regs)
 365 {
 366         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 367         unsigned long ip = regs->psw.addr;
 368         int fixup = probe_get_fixup_type(p->ainsn.insn);
 369
 370         if (fixup & FIXUP_PSW_NORMAL)
 371                 ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn;
 372
 373         if (fixup & FIXUP_BRANCH_NOT_TAKEN) {
 374                 int ilen = insn_length(p->ainsn.insn[0] >> 8);
 375                 if (ip - (unsigned long) p->ainsn.insn == ilen)
 376                         ip = (unsigned long) p->addr + ilen;
 377         }
 378
 379         if (fixup & FIXUP_RETURN_REGISTER) {
 380                 int reg = (p->ainsn.insn[0] & 0xf0) >> 4;
 381                 regs->gprs[reg] += (unsigned long) p->addr -
 382                                    (unsigned long) p->ainsn.insn;
 383         }
 384
 385         disable_singlestep(kcb, regs, ip);
 386 }
 387 NOKPROBE_SYMBOL(resume_execution);
 388
 389 static int post_kprobe_handler(struct pt_regs *regs)
 390 {
 391         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 392         struct kprobe *p = kprobe_running();
 393
 394         if (!p)
 395                 return 0;
 396
 397         if (kcb->kprobe_status != KPROBE_REENTER && p->post_handler) {
 398                 kcb->kprobe_status = KPROBE_HIT_SSDONE;
 399                 p->post_handler(p, regs, 0);
 400         }
 401
 402         resume_execution(p, regs);
 403         pop_kprobe(kcb);
 404         preempt_enable_no_resched();
 405
 406         /*
 407          * if somebody else is singlestepping across a probe point, psw mask
 408          * will have PER set, in which case, continue the remaining processing
 409          * of do_single_step, as if this is not a probe hit.
 410          */
 411         if (regs->psw.mask & PSW_MASK_PER)
 412                 return 0;
 413
 414         return 1;
 415 }
 416 NOKPROBE_SYMBOL(post_kprobe_handler);
 417
 418 static int kprobe_trap_handler(struct pt_regs *regs, int trapnr)
 419 {
 420         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 421         struct kprobe *p = kprobe_running();
 422         const struct exception_table_entry *entry;
 423
 424         switch(kcb->kprobe_status) {
 425         case KPROBE_HIT_SS:
 426         case KPROBE_REENTER:
 427                 /*
 428                  * We are here because the instruction being single
 429                  * stepped caused a page fault. We reset the current
 430                  * kprobe and the nip points back to the probe address
 431                  * and allow the page fault handler to continue as a
 432                  * normal page fault.
 433                  */
 434                 disable_singlestep(kcb, regs, (unsigned long) p->addr);
 435                 pop_kprobe(kcb);
 436                 preempt_enable_no_resched();
 437                 break;
 438         case KPROBE_HIT_ACTIVE:
 439         case KPROBE_HIT_SSDONE:
 440                 /*
 441                  * In case the user-specified fault handler returned
 442                  * zero, try to fix up.
 443                  */
 444                 entry = s390_search_extables(regs->psw.addr);
 445                 if (entry && ex_handle(entry, regs))
 446                         return 1;
 447
 448                 /*
 449                  * fixup_exception() could not handle it,
 450                  * Let do_page_fault() fix it.
 451                  */
 452                 break;
 453         default:
 454                 break;
 455         }
 456         return 0;
 457 }
 458 NOKPROBE_SYMBOL(kprobe_trap_handler);
 459
 460 int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
 461 {
 462         int ret;
 463
 464         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 465                 local_irq_disable();
 466         ret = kprobe_trap_handler(regs, trapnr);
 467         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 468                 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
 469         return ret;
 470 }
 471 NOKPROBE_SYMBOL(kprobe_fault_handler);
 472
 473 /*
 474  * Wrapper routine to for handling exceptions.
 475  */
 476 int kprobe_exceptions_notify(struct notifier_block *self,
 477                              unsigned long val, void *data)
 478 {
 479         struct die_args *args = (struct die_args *) data;
 480         struct pt_regs *regs = args->regs;
 481         int ret = NOTIFY_DONE;
 482
 483         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 484                 local_irq_disable();
 485
 486         switch (val) {
 487         case DIE_BPT:
 488                 if (kprobe_handler(regs))
 489                         ret = NOTIFY_STOP;
 490                 break;
 491         case DIE_SSTEP:
 492                 if (post_kprobe_handler(regs))
 493                         ret = NOTIFY_STOP;
 494                 break;
 495         case DIE_TRAP:
 496                 if (!preemptible() && kprobe_running() &&
 497                     kprobe_trap_handler(regs, args->trapnr))
 498                         ret = NOTIFY_STOP;
 499                 break;
 500         default:
 501                 break;
 502         }
 503
 504         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 505                 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
 506
 507         return ret;
 508 }
 509 NOKPROBE_SYMBOL(kprobe_exceptions_notify);
 510
 511 static struct kprobe trampoline = {
 512         .addr = (kprobe_opcode_t *) &__kretprobe_trampoline,
 513         .pre_handler = trampoline_probe_handler
 514 };
 515
 516 int __init arch_init_kprobes(void)
 517 {
 518         return register_kprobe(&trampoline);
 519 }
 520
 521 int arch_trampoline_kprobe(struct kprobe *p)
 522 {
 523         return p->addr == (kprobe_opcode_t *) &__kretprobe_trampoline;
 524 }
 525 NOKPROBE_SYMBOL(arch_trampoline_kprobe);