1 #include <linux/sched.h> 2 #include <linux/sched/task.h> 3 #include <linux/sched/task_stack.h> 4 #include <asm/ptrace.h> 5 #include <asm/bitops.h> 6 #include <asm/stacktrace.h> 7 #include <asm/unwind.h> 8 9 #define FRAME_HEADER_SIZE (sizeof(long) * 2) 10 11 /* 12 * This disables KASAN checking when reading a value from another task's stack, 13 * since the other task could be running on another CPU and could have poisoned 14 * the stack in the meantime. 15 */ 16 #define READ_ONCE_TASK_STACK(task, x) \ 17 ({ \ 18 unsigned long val; \ 19 if (task == current) \ 20 val = READ_ONCE(x); \ 21 else \ 22 val = READ_ONCE_NOCHECK(x); \ 23 val; \ 24 }) 25 26 static void unwind_dump(struct unwind_state *state, unsigned long *sp) 27 { 28 static bool dumped_before = false; 29 bool prev_zero, zero = false; 30 unsigned long word; 31 32 if (dumped_before) 33 return; 34 35 dumped_before = true; 36 37 printk_deferred("unwind stack type:%d next_sp:%p mask:%lx graph_idx:%d\n", 38 state->stack_info.type, state->stack_info.next_sp, 39 state->stack_mask, state->graph_idx); 40 41 for (sp = state->orig_sp; sp < state->stack_info.end; sp++) { 42 word = READ_ONCE_NOCHECK(*sp); 43 44 prev_zero = zero; 45 zero = word == 0; 46 47 if (zero) { 48 if (!prev_zero) 49 printk_deferred("%p: %016x ...\n", sp, 0); 50 continue; 51 } 52 53 printk_deferred("%p: %016lx (%pB)\n", sp, word, (void *)word); 54 } 55 } 56 57 unsigned long unwind_get_return_address(struct unwind_state *state) 58 { 59 unsigned long addr; 60 unsigned long *addr_p = unwind_get_return_address_ptr(state); 61 62 if (unwind_done(state)) 63 return 0; 64 65 if (state->regs && user_mode(state->regs)) 66 return 0; 67 68 addr = READ_ONCE_TASK_STACK(state->task, *addr_p); 69 addr = ftrace_graph_ret_addr(state->task, &state->graph_idx, addr, 70 addr_p); 71 72 return __kernel_text_address(addr) ? addr : 0; 73 } 74 EXPORT_SYMBOL_GPL(unwind_get_return_address); 75 76 static size_t regs_size(struct pt_regs *regs) 77 { 78 /* x86_32 regs from kernel mode are two words shorter: */ 79 if (IS_ENABLED(CONFIG_X86_32) && !user_mode(regs)) 80 return sizeof(*regs) - 2*sizeof(long); 81 82 return sizeof(*regs); 83 } 84 85 #ifdef CONFIG_X86_32 86 #define GCC_REALIGN_WORDS 3 87 #else 88 #define GCC_REALIGN_WORDS 1 89 #endif 90 91 static bool is_last_task_frame(struct unwind_state *state) 92 { 93 unsigned long *last_bp = (unsigned long *)task_pt_regs(state->task) - 2; 94 unsigned long *aligned_bp = last_bp - GCC_REALIGN_WORDS; 95 96 /* 97 * We have to check for the last task frame at two different locations 98 * because gcc can occasionally decide to realign the stack pointer and 99 * change the offset of the stack frame in the prologue of a function 100 * called by head/entry code. Examples: 101 * 102 * <start_secondary>: 103 * push %edi 104 * lea 0x8(%esp),%edi 105 * and $0xfffffff8,%esp 106 * pushl -0x4(%edi) 107 * push %ebp 108 * mov %esp,%ebp 109 * 110 * <x86_64_start_kernel>: 111 * lea 0x8(%rsp),%r10 112 * and $0xfffffffffffffff0,%rsp 113 * pushq -0x8(%r10) 114 * push %rbp 115 * mov %rsp,%rbp 116 * 117 * Note that after aligning the stack, it pushes a duplicate copy of 118 * the return address before pushing the frame pointer. 119 */ 120 return (state->bp == last_bp || 121 (state->bp == aligned_bp && *(aligned_bp+1) == *(last_bp+1))); 122 } 123 124 /* 125 * This determines if the frame pointer actually contains an encoded pointer to 126 * pt_regs on the stack. See ENCODE_FRAME_POINTER. 127 */ 128 static struct pt_regs *decode_frame_pointer(unsigned long *bp) 129 { 130 unsigned long regs = (unsigned long)bp; 131 132 if (!(regs & 0x1)) 133 return NULL; 134 135 return (struct pt_regs *)(regs & ~0x1); 136 } 137 138 static bool update_stack_state(struct unwind_state *state, 139 unsigned long *next_bp) 140 { 141 struct stack_info *info = &state->stack_info; 142 enum stack_type prev_type = info->type; 143 struct pt_regs *regs; 144 unsigned long *frame, *prev_frame_end; 145 size_t len; 146 147 if (state->regs) 148 prev_frame_end = (void *)state->regs + regs_size(state->regs); 149 else 150 prev_frame_end = (void *)state->bp + FRAME_HEADER_SIZE; 151 152 /* Is the next frame pointer an encoded pointer to pt_regs? */ 153 regs = decode_frame_pointer(next_bp); 154 if (regs) { 155 frame = (unsigned long *)regs; 156 len = regs_size(regs); 157 } else { 158 frame = next_bp; 159 len = FRAME_HEADER_SIZE; 160 } 161 162 /* 163 * If the next bp isn't on the current stack, switch to the next one. 164 * 165 * We may have to traverse multiple stacks to deal with the possibility 166 * that info->next_sp could point to an empty stack and the next bp 167 * could be on a subsequent stack. 168 */ 169 while (!on_stack(info, frame, len)) 170 if (get_stack_info(info->next_sp, state->task, info, 171 &state->stack_mask)) 172 return false; 173 174 /* Make sure it only unwinds up and doesn't overlap the prev frame: */ 175 if (state->orig_sp && state->stack_info.type == prev_type && 176 frame < prev_frame_end) 177 return false; 178 179 /* Move state to the next frame: */ 180 if (regs) { 181 state->regs = regs; 182 state->bp = NULL; 183 } else { 184 state->bp = next_bp; 185 state->regs = NULL; 186 } 187 188 /* Save the original stack pointer for unwind_dump(): */ 189 if (!state->orig_sp || info->type != prev_type) 190 state->orig_sp = frame; 191 192 return true; 193 } 194 195 bool unwind_next_frame(struct unwind_state *state) 196 { 197 struct pt_regs *regs; 198 unsigned long *next_bp; 199 200 if (unwind_done(state)) 201 return false; 202 203 /* Have we reached the end? */ 204 if (state->regs && user_mode(state->regs)) 205 goto the_end; 206 207 if (is_last_task_frame(state)) { 208 regs = task_pt_regs(state->task); 209 210 /* 211 * kthreads (other than the boot CPU's idle thread) have some 212 * partial regs at the end of their stack which were placed 213 * there by copy_thread_tls(). But the regs don't have any 214 * useful information, so we can skip them. 215 * 216 * This user_mode() check is slightly broader than a PF_KTHREAD 217 * check because it also catches the awkward situation where a 218 * newly forked kthread transitions into a user task by calling 219 * do_execve(), which eventually clears PF_KTHREAD. 220 */ 221 if (!user_mode(regs)) 222 goto the_end; 223 224 /* 225 * We're almost at the end, but not quite: there's still the 226 * syscall regs frame. Entry code doesn't encode the regs 227 * pointer for syscalls, so we have to set it manually. 228 */ 229 state->regs = regs; 230 state->bp = NULL; 231 return true; 232 } 233 234 /* Get the next frame pointer: */ 235 if (state->regs) 236 next_bp = (unsigned long *)state->regs->bp; 237 else 238 next_bp = (unsigned long *)READ_ONCE_TASK_STACK(state->task, *state->bp); 239 240 /* Move to the next frame if it's safe: */ 241 if (!update_stack_state(state, next_bp)) { 242 /* 243 * Don't warn on bad regs->bp. An interrupt in entry code 244 * might cause a false positive warning. 245 */ 246 if (state->regs) 247 goto the_end; 248 249 goto bad_address; 250 } 251 252 return true; 253 254 bad_address: 255 /* 256 * When unwinding a non-current task, the task might actually be 257 * running on another CPU, in which case it could be modifying its 258 * stack while we're reading it. This is generally not a problem and 259 * can be ignored as long as the caller understands that unwinding 260 * another task will not always succeed. 261 */ 262 if (state->task != current) 263 goto the_end; 264 265 if (state->regs) { 266 printk_deferred_once(KERN_WARNING 267 "WARNING: kernel stack regs at %p in %s:%d has bad 'bp' value %p\n", 268 state->regs, state->task->comm, 269 state->task->pid, next_bp); 270 unwind_dump(state, (unsigned long *)state->regs); 271 } else { 272 printk_deferred_once(KERN_WARNING 273 "WARNING: kernel stack frame pointer at %p in %s:%d has bad value %p\n", 274 state->bp, state->task->comm, 275 state->task->pid, next_bp); 276 unwind_dump(state, state->bp); 277 } 278 the_end: 279 state->stack_info.type = STACK_TYPE_UNKNOWN; 280 return false; 281 } 282 EXPORT_SYMBOL_GPL(unwind_next_frame); 283 284 void __unwind_start(struct unwind_state *state, struct task_struct *task, 285 struct pt_regs *regs, unsigned long *first_frame) 286 { 287 unsigned long *bp; 288 289 memset(state, 0, sizeof(*state)); 290 state->task = task; 291 292 /* Don't even attempt to start from user mode regs: */ 293 if (regs && user_mode(regs)) { 294 state->stack_info.type = STACK_TYPE_UNKNOWN; 295 return; 296 } 297 298 bp = get_frame_pointer(task, regs); 299 300 /* Initialize stack info and make sure the frame data is accessible: */ 301 get_stack_info(bp, state->task, &state->stack_info, 302 &state->stack_mask); 303 update_stack_state(state, bp); 304 305 /* 306 * The caller can provide the address of the first frame directly 307 * (first_frame) or indirectly (regs->sp) to indicate which stack frame 308 * to start unwinding at. Skip ahead until we reach it. 309 */ 310 while (!unwind_done(state) && 311 (!on_stack(&state->stack_info, first_frame, sizeof(long)) || 312 state->bp < first_frame)) 313 unwind_next_frame(state); 314 } 315 EXPORT_SYMBOL_GPL(__unwind_start); 316