arch/x86/kernel/unwind_frame.c

   1 #include <linux/sched.h>
   2 #include <linux/sched/task.h>
   3 #include <linux/sched/task_stack.h>
   4 #include <asm/ptrace.h>
   5 #include <asm/bitops.h>
   6 #include <asm/stacktrace.h>
   7 #include <asm/unwind.h>
   8
   9 #define FRAME_HEADER_SIZE (sizeof(long) * 2)
  10
  11 /*
  12  * This disables KASAN checking when reading a value from another task's stack,
  13  * since the other task could be running on another CPU and could have poisoned
  14  * the stack in the meantime.
  15  */
  16 #define READ_ONCE_TASK_STACK(task, x)                   \
  17 ({                                                      \
  18         unsigned long val;                              \
  19         if (task == current)                            \
  20                 val = READ_ONCE(x);                     \
  21         else                                            \
  22                 val = READ_ONCE_NOCHECK(x);             \
  23         val;                                            \
  24 })
  25
  26 static void unwind_dump(struct unwind_state *state, unsigned long *sp)
  27 {
  28         static bool dumped_before = false;
  29         bool prev_zero, zero = false;
  30         unsigned long word;
  31
  32         if (dumped_before)
  33                 return;
  34
  35         dumped_before = true;
  36
  37         printk_deferred("unwind stack type:%d next_sp:%p mask:%lx graph_idx:%d\n",
  38                         state->stack_info.type, state->stack_info.next_sp,
  39                         state->stack_mask, state->graph_idx);
  40
  41         for (sp = state->orig_sp; sp < state->stack_info.end; sp++) {
  42                 word = READ_ONCE_NOCHECK(*sp);
  43
  44                 prev_zero = zero;
  45                 zero = word == 0;
  46
  47                 if (zero) {
  48                         if (!prev_zero)
  49                                 printk_deferred("%p: %016x ...\n", sp, 0);
  50                         continue;
  51                 }
  52
  53                 printk_deferred("%p: %016lx (%pB)\n", sp, word, (void *)word);
  54         }
  55 }
  56
  57 unsigned long unwind_get_return_address(struct unwind_state *state)
  58 {
  59         unsigned long addr;
  60         unsigned long *addr_p = unwind_get_return_address_ptr(state);
  61
  62         if (unwind_done(state))
  63                 return 0;
  64
  65         if (state->regs && user_mode(state->regs))
  66                 return 0;
  67
  68         addr = READ_ONCE_TASK_STACK(state->task, *addr_p);
  69         addr = ftrace_graph_ret_addr(state->task, &state->graph_idx, addr,
  70                                      addr_p);
  71
  72         return __kernel_text_address(addr) ? addr : 0;
  73 }
  74 EXPORT_SYMBOL_GPL(unwind_get_return_address);
  75
  76 static size_t regs_size(struct pt_regs *regs)
  77 {
  78         /* x86_32 regs from kernel mode are two words shorter: */
  79         if (IS_ENABLED(CONFIG_X86_32) && !user_mode(regs))
  80                 return sizeof(*regs) - 2*sizeof(long);
  81
  82         return sizeof(*regs);
  83 }
  84
  85 #ifdef CONFIG_X86_32
  86 #define GCC_REALIGN_WORDS 3
  87 #else
  88 #define GCC_REALIGN_WORDS 1
  89 #endif
  90
  91 static bool is_last_task_frame(struct unwind_state *state)
  92 {
  93         unsigned long *last_bp = (unsigned long *)task_pt_regs(state->task) - 2;
  94         unsigned long *aligned_bp = last_bp - GCC_REALIGN_WORDS;
  95
  96         /*
  97          * We have to check for the last task frame at two different locations
  98          * because gcc can occasionally decide to realign the stack pointer and
  99          * change the offset of the stack frame in the prologue of a function
 100          * called by head/entry code.  Examples:
 101          *
 102          * <start_secondary>:
 103          *      push   %edi
 104          *      lea    0x8(%esp),%edi
 105          *      and    $0xfffffff8,%esp
 106          *      pushl  -0x4(%edi)
 107          *      push   %ebp
 108          *      mov    %esp,%ebp
 109          *
 110          * <x86_64_start_kernel>:
 111          *      lea    0x8(%rsp),%r10
 112          *      and    $0xfffffffffffffff0,%rsp
 113          *      pushq  -0x8(%r10)
 114          *      push   %rbp
 115          *      mov    %rsp,%rbp
 116          *
 117          * Note that after aligning the stack, it pushes a duplicate copy of
 118          * the return address before pushing the frame pointer.
 119          */
 120         return (state->bp == last_bp ||
 121                 (state->bp == aligned_bp && *(aligned_bp+1) == *(last_bp+1)));
 122 }
 123
 124 /*
 125  * This determines if the frame pointer actually contains an encoded pointer to
 126  * pt_regs on the stack.  See ENCODE_FRAME_POINTER.
 127  */
 128 static struct pt_regs *decode_frame_pointer(unsigned long *bp)
 129 {
 130         unsigned long regs = (unsigned long)bp;
 131
 132         if (!(regs & 0x1))
 133                 return NULL;
 134
 135         return (struct pt_regs *)(regs & ~0x1);
 136 }
 137
 138 static bool update_stack_state(struct unwind_state *state, void *addr,
 139                                size_t len)
 140 {
 141         struct stack_info *info = &state->stack_info;
 142         enum stack_type orig_type = info->type;
 143
 144         /*
 145          * If addr isn't on the current stack, switch to the next one.
 146          *
 147          * We may have to traverse multiple stacks to deal with the possibility
 148          * that 'info->next_sp' could point to an empty stack and 'addr' could
 149          * be on a subsequent stack.
 150          */
 151         while (!on_stack(info, addr, len))
 152                 if (get_stack_info(info->next_sp, state->task, info,
 153                                    &state->stack_mask))
 154                         return false;
 155
 156         if (!state->orig_sp || info->type != orig_type)
 157                 state->orig_sp = addr;
 158
 159         return true;
 160 }
 161
 162 bool unwind_next_frame(struct unwind_state *state)
 163 {
 164         struct pt_regs *regs;
 165         unsigned long *next_bp, *next_frame;
 166         size_t next_len;
 167         enum stack_type prev_type = state->stack_info.type;
 168
 169         if (unwind_done(state))
 170                 return false;
 171
 172         /* have we reached the end? */
 173         if (state->regs && user_mode(state->regs))
 174                 goto the_end;
 175
 176         if (is_last_task_frame(state)) {
 177                 regs = task_pt_regs(state->task);
 178
 179                 /*
 180                  * kthreads (other than the boot CPU's idle thread) have some
 181                  * partial regs at the end of their stack which were placed
 182                  * there by copy_thread_tls().  But the regs don't have any
 183                  * useful information, so we can skip them.
 184                  *
 185                  * This user_mode() check is slightly broader than a PF_KTHREAD
 186                  * check because it also catches the awkward situation where a
 187                  * newly forked kthread transitions into a user task by calling
 188                  * do_execve(), which eventually clears PF_KTHREAD.
 189                  */
 190                 if (!user_mode(regs))
 191                         goto the_end;
 192
 193                 /*
 194                  * We're almost at the end, but not quite: there's still the
 195                  * syscall regs frame.  Entry code doesn't encode the regs
 196                  * pointer for syscalls, so we have to set it manually.
 197                  */
 198                 state->regs = regs;
 199                 state->bp = NULL;
 200                 return true;
 201         }
 202
 203         /* get the next frame pointer */
 204         if (state->regs)
 205                 next_bp = (unsigned long *)state->regs->bp;
 206         else
 207                 next_bp = (unsigned long *)READ_ONCE_TASK_STACK(state->task,*state->bp);
 208
 209         /* is the next frame pointer an encoded pointer to pt_regs? */
 210         regs = decode_frame_pointer(next_bp);
 211         if (regs) {
 212                 next_frame = (unsigned long *)regs;
 213                 next_len = sizeof(*regs);
 214         } else {
 215                 next_frame = next_bp;
 216                 next_len = FRAME_HEADER_SIZE;
 217         }
 218
 219         /* make sure the next frame's data is accessible */
 220         if (!update_stack_state(state, next_frame, next_len)) {
 221                 /*
 222                  * Don't warn on bad regs->bp.  An interrupt in entry code
 223                  * might cause a false positive warning.
 224                  */
 225                 if (state->regs)
 226                         goto the_end;
 227
 228                 goto bad_address;
 229         }
 230
 231         /* Make sure it only unwinds up and doesn't overlap the last frame: */
 232         if (state->stack_info.type == prev_type) {
 233                 if (state->regs && (void *)next_frame < (void *)state->regs + regs_size(state->regs))
 234                         goto bad_address;
 235
 236                 if (state->bp && (void *)next_frame < (void *)state->bp + FRAME_HEADER_SIZE)
 237                         goto bad_address;
 238         }
 239
 240         /* move to the next frame */
 241         if (regs) {
 242                 state->regs = regs;
 243                 state->bp = NULL;
 244         } else {
 245                 state->bp = next_bp;
 246                 state->regs = NULL;
 247         }
 248
 249         return true;
 250
 251 bad_address:
 252         /*
 253          * When unwinding a non-current task, the task might actually be
 254          * running on another CPU, in which case it could be modifying its
 255          * stack while we're reading it.  This is generally not a problem and
 256          * can be ignored as long as the caller understands that unwinding
 257          * another task will not always succeed.
 258          */
 259         if (state->task != current)
 260                 goto the_end;
 261
 262         if (state->regs) {
 263                 printk_deferred_once(KERN_WARNING
 264                         "WARNING: kernel stack regs at %p in %s:%d has bad 'bp' value %p\n",
 265                         state->regs, state->task->comm,
 266                         state->task->pid, next_frame);
 267                 unwind_dump(state, (unsigned long *)state->regs);
 268         } else {
 269                 printk_deferred_once(KERN_WARNING
 270                         "WARNING: kernel stack frame pointer at %p in %s:%d has bad value %p\n",
 271                         state->bp, state->task->comm,
 272                         state->task->pid, next_frame);
 273                 unwind_dump(state, state->bp);
 274         }
 275 the_end:
 276         state->stack_info.type = STACK_TYPE_UNKNOWN;
 277         return false;
 278 }
 279 EXPORT_SYMBOL_GPL(unwind_next_frame);
 280
 281 void __unwind_start(struct unwind_state *state, struct task_struct *task,
 282                     struct pt_regs *regs, unsigned long *first_frame)
 283 {
 284         unsigned long *bp, *frame;
 285         size_t len;
 286
 287         memset(state, 0, sizeof(*state));
 288         state->task = task;
 289
 290         /* don't even attempt to start from user mode regs */
 291         if (regs && user_mode(regs)) {
 292                 state->stack_info.type = STACK_TYPE_UNKNOWN;
 293                 return;
 294         }
 295
 296         /* set up the starting stack frame */
 297         bp = get_frame_pointer(task, regs);
 298         regs = decode_frame_pointer(bp);
 299         if (regs) {
 300                 state->regs = regs;
 301                 frame = (unsigned long *)regs;
 302                 len = sizeof(*regs);
 303         } else {
 304                 state->bp = bp;
 305                 frame = bp;
 306                 len = FRAME_HEADER_SIZE;
 307         }
 308
 309         /* initialize stack info and make sure the frame data is accessible */
 310         get_stack_info(frame, state->task, &state->stack_info,
 311                        &state->stack_mask);
 312         update_stack_state(state, frame, len);
 313
 314         /*
 315          * The caller can provide the address of the first frame directly
 316          * (first_frame) or indirectly (regs->sp) to indicate which stack frame
 317          * to start unwinding at.  Skip ahead until we reach it.
 318          */
 319         while (!unwind_done(state) &&
 320                (!on_stack(&state->stack_info, first_frame, sizeof(long)) ||
 321                         state->bp < first_frame))
 322                 unwind_next_frame(state);
 323 }
 324 EXPORT_SYMBOL_GPL(__unwind_start);