1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Kernel Probes (KProbes) 4 * 5 * Copyright IBM Corp. 2002, 2006 6 * 7 * s390 port, used ppc64 as template. Mike Grundy <grundym@us.ibm.com> 8 */ 9 10 #define pr_fmt(fmt) "kprobes: " fmt 11 12 #include <linux/moduleloader.h> 13 #include <linux/kprobes.h> 14 #include <linux/ptrace.h> 15 #include <linux/preempt.h> 16 #include <linux/stop_machine.h> 17 #include <linux/kdebug.h> 18 #include <linux/uaccess.h> 19 #include <linux/extable.h> 20 #include <linux/module.h> 21 #include <linux/slab.h> 22 #include <linux/hardirq.h> 23 #include <linux/ftrace.h> 24 #include <asm/set_memory.h> 25 #include <asm/sections.h> 26 #include <asm/dis.h> 27 #include "kprobes.h" 28 #include "entry.h" 29 30 DEFINE_PER_CPU(struct kprobe *, current_kprobe); 31 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); 32 33 struct kretprobe_blackpoint kretprobe_blacklist[] = { }; 34 35 static int insn_page_in_use; 36 37 void *alloc_insn_page(void) 38 { 39 void *page; 40 41 page = module_alloc(PAGE_SIZE); 42 if (!page) 43 return NULL; 44 set_memory_rox((unsigned long)page, 1); 45 return page; 46 } 47 48 static void *alloc_s390_insn_page(void) 49 { 50 if (xchg(&insn_page_in_use, 1) == 1) 51 return NULL; 52 return &kprobes_insn_page; 53 } 54 55 static void free_s390_insn_page(void *page) 56 { 57 xchg(&insn_page_in_use, 0); 58 } 59 60 struct kprobe_insn_cache kprobe_s390_insn_slots = { 61 .mutex = __MUTEX_INITIALIZER(kprobe_s390_insn_slots.mutex), 62 .alloc = alloc_s390_insn_page, 63 .free = free_s390_insn_page, 64 .pages = LIST_HEAD_INIT(kprobe_s390_insn_slots.pages), 65 .insn_size = MAX_INSN_SIZE, 66 }; 67 68 static void copy_instruction(struct kprobe *p) 69 { 70 kprobe_opcode_t insn[MAX_INSN_SIZE]; 71 s64 disp, new_disp; 72 u64 addr, new_addr; 73 unsigned int len; 74 75 len = insn_length(*p->addr >> 8); 76 memcpy(&insn, p->addr, len); 77 p->opcode = insn[0]; 78 if (probe_is_insn_relative_long(&insn[0])) { 79 /* 80 * For pc-relative instructions in RIL-b or RIL-c format patch 81 * the RI2 displacement field. We have already made sure that 82 * the insn slot for the patched instruction is within the same 83 * 2GB area as the original instruction (either kernel image or 84 * module area). Therefore the new displacement will always fit. 85 */ 86 disp = *(s32 *)&insn[1]; 87 addr = (u64)(unsigned long)p->addr; 88 new_addr = (u64)(unsigned long)p->ainsn.insn; 89 new_disp = ((addr + (disp * 2)) - new_addr) / 2; 90 *(s32 *)&insn[1] = new_disp; 91 } 92 s390_kernel_write(p->ainsn.insn, &insn, len); 93 } 94 NOKPROBE_SYMBOL(copy_instruction); 95 96 static int s390_get_insn_slot(struct kprobe *p) 97 { 98 /* 99 * Get an insn slot that is within the same 2GB area like the original 100 * instruction. That way instructions with a 32bit signed displacement 101 * field can be patched and executed within the insn slot. 102 */ 103 p->ainsn.insn = NULL; 104 if (is_kernel((unsigned long)p->addr)) 105 p->ainsn.insn = get_s390_insn_slot(); 106 else if (is_module_addr(p->addr)) 107 p->ainsn.insn = get_insn_slot(); 108 return p->ainsn.insn ? 0 : -ENOMEM; 109 } 110 NOKPROBE_SYMBOL(s390_get_insn_slot); 111 112 static void s390_free_insn_slot(struct kprobe *p) 113 { 114 if (!p->ainsn.insn) 115 return; 116 if (is_kernel((unsigned long)p->addr)) 117 free_s390_insn_slot(p->ainsn.insn, 0); 118 else 119 free_insn_slot(p->ainsn.insn, 0); 120 p->ainsn.insn = NULL; 121 } 122 NOKPROBE_SYMBOL(s390_free_insn_slot); 123 124 /* Check if paddr is at an instruction boundary */ 125 static bool can_probe(unsigned long paddr) 126 { 127 unsigned long addr, offset = 0; 128 kprobe_opcode_t insn; 129 struct kprobe *kp; 130 131 if (paddr & 0x01) 132 return false; 133 134 if (!kallsyms_lookup_size_offset(paddr, NULL, &offset)) 135 return false; 136 137 /* Decode instructions */ 138 addr = paddr - offset; 139 while (addr < paddr) { 140 if (copy_from_kernel_nofault(&insn, (void *)addr, sizeof(insn))) 141 return false; 142 143 if (insn >> 8 == 0) { 144 if (insn != BREAKPOINT_INSTRUCTION) { 145 /* 146 * Note that QEMU inserts opcode 0x0000 to implement 147 * software breakpoints for guests. Since the size of 148 * the original instruction is unknown, stop following 149 * instructions and prevent setting a kprobe. 150 */ 151 return false; 152 } 153 /* 154 * Check if the instruction has been modified by another 155 * kprobe, in which case the original instruction is 156 * decoded. 157 */ 158 kp = get_kprobe((void *)addr); 159 if (!kp) { 160 /* not a kprobe */ 161 return false; 162 } 163 insn = kp->opcode; 164 } 165 addr += insn_length(insn >> 8); 166 } 167 return addr == paddr; 168 } 169 170 int arch_prepare_kprobe(struct kprobe *p) 171 { 172 if (!can_probe((unsigned long)p->addr)) 173 return -EINVAL; 174 /* Make sure the probe isn't going on a difficult instruction */ 175 if (probe_is_prohibited_opcode(p->addr)) 176 return -EINVAL; 177 if (s390_get_insn_slot(p)) 178 return -ENOMEM; 179 copy_instruction(p); 180 return 0; 181 } 182 NOKPROBE_SYMBOL(arch_prepare_kprobe); 183 184 struct swap_insn_args { 185 struct kprobe *p; 186 unsigned int arm_kprobe : 1; 187 }; 188 189 static int swap_instruction(void *data) 190 { 191 struct swap_insn_args *args = data; 192 struct kprobe *p = args->p; 193 u16 opc; 194 195 opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode; 196 s390_kernel_write(p->addr, &opc, sizeof(opc)); 197 return 0; 198 } 199 NOKPROBE_SYMBOL(swap_instruction); 200 201 void arch_arm_kprobe(struct kprobe *p) 202 { 203 struct swap_insn_args args = {.p = p, .arm_kprobe = 1}; 204 205 stop_machine_cpuslocked(swap_instruction, &args, NULL); 206 } 207 NOKPROBE_SYMBOL(arch_arm_kprobe); 208 209 void arch_disarm_kprobe(struct kprobe *p) 210 { 211 struct swap_insn_args args = {.p = p, .arm_kprobe = 0}; 212 213 stop_machine_cpuslocked(swap_instruction, &args, NULL); 214 } 215 NOKPROBE_SYMBOL(arch_disarm_kprobe); 216 217 void arch_remove_kprobe(struct kprobe *p) 218 { 219 s390_free_insn_slot(p); 220 } 221 NOKPROBE_SYMBOL(arch_remove_kprobe); 222 223 static void enable_singlestep(struct kprobe_ctlblk *kcb, 224 struct pt_regs *regs, 225 unsigned long ip) 226 { 227 union { 228 struct ctlreg regs[3]; 229 struct { 230 struct ctlreg control; 231 struct ctlreg start; 232 struct ctlreg end; 233 }; 234 } per_kprobe; 235 236 /* Set up the PER control registers %cr9-%cr11 */ 237 per_kprobe.control.val = PER_EVENT_IFETCH; 238 per_kprobe.start.val = ip; 239 per_kprobe.end.val = ip; 240 241 /* Save control regs and psw mask */ 242 __local_ctl_store(9, 11, kcb->kprobe_saved_ctl); 243 kcb->kprobe_saved_imask = regs->psw.mask & 244 (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT); 245 246 /* Set PER control regs, turns on single step for the given address */ 247 __local_ctl_load(9, 11, per_kprobe.regs); 248 regs->psw.mask |= PSW_MASK_PER; 249 regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT); 250 regs->psw.addr = ip; 251 } 252 NOKPROBE_SYMBOL(enable_singlestep); 253 254 static void disable_singlestep(struct kprobe_ctlblk *kcb, 255 struct pt_regs *regs, 256 unsigned long ip) 257 { 258 /* Restore control regs and psw mask, set new psw address */ 259 __local_ctl_load(9, 11, kcb->kprobe_saved_ctl); 260 regs->psw.mask &= ~PSW_MASK_PER; 261 regs->psw.mask |= kcb->kprobe_saved_imask; 262 regs->psw.addr = ip; 263 } 264 NOKPROBE_SYMBOL(disable_singlestep); 265 266 /* 267 * Activate a kprobe by storing its pointer to current_kprobe. The 268 * previous kprobe is stored in kcb->prev_kprobe. A stack of up to 269 * two kprobes can be active, see KPROBE_REENTER. 270 */ 271 static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p) 272 { 273 kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe); 274 kcb->prev_kprobe.status = kcb->kprobe_status; 275 __this_cpu_write(current_kprobe, p); 276 } 277 NOKPROBE_SYMBOL(push_kprobe); 278 279 /* 280 * Deactivate a kprobe by backing up to the previous state. If the 281 * current state is KPROBE_REENTER prev_kprobe.kp will be non-NULL, 282 * for any other state prev_kprobe.kp will be NULL. 283 */ 284 static void pop_kprobe(struct kprobe_ctlblk *kcb) 285 { 286 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); 287 kcb->kprobe_status = kcb->prev_kprobe.status; 288 kcb->prev_kprobe.kp = NULL; 289 } 290 NOKPROBE_SYMBOL(pop_kprobe); 291 292 static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p) 293 { 294 switch (kcb->kprobe_status) { 295 case KPROBE_HIT_SSDONE: 296 case KPROBE_HIT_ACTIVE: 297 kprobes_inc_nmissed_count(p); 298 break; 299 case KPROBE_HIT_SS: 300 case KPROBE_REENTER: 301 default: 302 /* 303 * A kprobe on the code path to single step an instruction 304 * is a BUG. The code path resides in the .kprobes.text 305 * section and is executed with interrupts disabled. 306 */ 307 pr_err("Failed to recover from reentered kprobes.\n"); 308 dump_kprobe(p); 309 BUG(); 310 } 311 } 312 NOKPROBE_SYMBOL(kprobe_reenter_check); 313 314 static int kprobe_handler(struct pt_regs *regs) 315 { 316 struct kprobe_ctlblk *kcb; 317 struct kprobe *p; 318 319 /* 320 * We want to disable preemption for the entire duration of kprobe 321 * processing. That includes the calls to the pre/post handlers 322 * and single stepping the kprobe instruction. 323 */ 324 preempt_disable(); 325 kcb = get_kprobe_ctlblk(); 326 p = get_kprobe((void *)(regs->psw.addr - 2)); 327 328 if (p) { 329 if (kprobe_running()) { 330 /* 331 * We have hit a kprobe while another is still 332 * active. This can happen in the pre and post 333 * handler. Single step the instruction of the 334 * new probe but do not call any handler function 335 * of this secondary kprobe. 336 * push_kprobe and pop_kprobe saves and restores 337 * the currently active kprobe. 338 */ 339 kprobe_reenter_check(kcb, p); 340 push_kprobe(kcb, p); 341 kcb->kprobe_status = KPROBE_REENTER; 342 } else { 343 /* 344 * If we have no pre-handler or it returned 0, we 345 * continue with single stepping. If we have a 346 * pre-handler and it returned non-zero, it prepped 347 * for changing execution path, so get out doing 348 * nothing more here. 349 */ 350 push_kprobe(kcb, p); 351 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 352 if (p->pre_handler && p->pre_handler(p, regs)) { 353 pop_kprobe(kcb); 354 preempt_enable_no_resched(); 355 return 1; 356 } 357 kcb->kprobe_status = KPROBE_HIT_SS; 358 } 359 enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn); 360 return 1; 361 } /* else: 362 * No kprobe at this address and no active kprobe. The trap has 363 * not been caused by a kprobe breakpoint. The race of breakpoint 364 * vs. kprobe remove does not exist because on s390 as we use 365 * stop_machine to arm/disarm the breakpoints. 366 */ 367 preempt_enable_no_resched(); 368 return 0; 369 } 370 NOKPROBE_SYMBOL(kprobe_handler); 371 372 /* 373 * Called after single-stepping. p->addr is the address of the 374 * instruction whose first byte has been replaced by the "breakpoint" 375 * instruction. To avoid the SMP problems that can occur when we 376 * temporarily put back the original opcode to single-step, we 377 * single-stepped a copy of the instruction. The address of this 378 * copy is p->ainsn.insn. 379 */ 380 static void resume_execution(struct kprobe *p, struct pt_regs *regs) 381 { 382 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 383 unsigned long ip = regs->psw.addr; 384 int fixup = probe_get_fixup_type(p->ainsn.insn); 385 386 if (fixup & FIXUP_PSW_NORMAL) 387 ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn; 388 389 if (fixup & FIXUP_BRANCH_NOT_TAKEN) { 390 int ilen = insn_length(p->ainsn.insn[0] >> 8); 391 if (ip - (unsigned long) p->ainsn.insn == ilen) 392 ip = (unsigned long) p->addr + ilen; 393 } 394 395 if (fixup & FIXUP_RETURN_REGISTER) { 396 int reg = (p->ainsn.insn[0] & 0xf0) >> 4; 397 regs->gprs[reg] += (unsigned long) p->addr - 398 (unsigned long) p->ainsn.insn; 399 } 400 401 disable_singlestep(kcb, regs, ip); 402 } 403 NOKPROBE_SYMBOL(resume_execution); 404 405 static int post_kprobe_handler(struct pt_regs *regs) 406 { 407 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 408 struct kprobe *p = kprobe_running(); 409 410 if (!p) 411 return 0; 412 413 resume_execution(p, regs); 414 if (kcb->kprobe_status != KPROBE_REENTER && p->post_handler) { 415 kcb->kprobe_status = KPROBE_HIT_SSDONE; 416 p->post_handler(p, regs, 0); 417 } 418 pop_kprobe(kcb); 419 preempt_enable_no_resched(); 420 421 /* 422 * if somebody else is singlestepping across a probe point, psw mask 423 * will have PER set, in which case, continue the remaining processing 424 * of do_single_step, as if this is not a probe hit. 425 */ 426 if (regs->psw.mask & PSW_MASK_PER) 427 return 0; 428 429 return 1; 430 } 431 NOKPROBE_SYMBOL(post_kprobe_handler); 432 433 static int kprobe_trap_handler(struct pt_regs *regs, int trapnr) 434 { 435 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 436 struct kprobe *p = kprobe_running(); 437 438 switch(kcb->kprobe_status) { 439 case KPROBE_HIT_SS: 440 case KPROBE_REENTER: 441 /* 442 * We are here because the instruction being single 443 * stepped caused a page fault. We reset the current 444 * kprobe and the nip points back to the probe address 445 * and allow the page fault handler to continue as a 446 * normal page fault. 447 */ 448 disable_singlestep(kcb, regs, (unsigned long) p->addr); 449 pop_kprobe(kcb); 450 preempt_enable_no_resched(); 451 break; 452 case KPROBE_HIT_ACTIVE: 453 case KPROBE_HIT_SSDONE: 454 /* 455 * In case the user-specified fault handler returned 456 * zero, try to fix up. 457 */ 458 if (fixup_exception(regs)) 459 return 1; 460 /* 461 * fixup_exception() could not handle it, 462 * Let do_page_fault() fix it. 463 */ 464 break; 465 default: 466 break; 467 } 468 return 0; 469 } 470 NOKPROBE_SYMBOL(kprobe_trap_handler); 471 472 int kprobe_fault_handler(struct pt_regs *regs, int trapnr) 473 { 474 int ret; 475 476 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT)) 477 local_irq_disable(); 478 ret = kprobe_trap_handler(regs, trapnr); 479 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT)) 480 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER); 481 return ret; 482 } 483 NOKPROBE_SYMBOL(kprobe_fault_handler); 484 485 /* 486 * Wrapper routine to for handling exceptions. 487 */ 488 int kprobe_exceptions_notify(struct notifier_block *self, 489 unsigned long val, void *data) 490 { 491 struct die_args *args = (struct die_args *) data; 492 struct pt_regs *regs = args->regs; 493 int ret = NOTIFY_DONE; 494 495 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT)) 496 local_irq_disable(); 497 498 switch (val) { 499 case DIE_BPT: 500 if (kprobe_handler(regs)) 501 ret = NOTIFY_STOP; 502 break; 503 case DIE_SSTEP: 504 if (post_kprobe_handler(regs)) 505 ret = NOTIFY_STOP; 506 break; 507 case DIE_TRAP: 508 if (!preemptible() && kprobe_running() && 509 kprobe_trap_handler(regs, args->trapnr)) 510 ret = NOTIFY_STOP; 511 break; 512 default: 513 break; 514 } 515 516 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT)) 517 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER); 518 519 return ret; 520 } 521 NOKPROBE_SYMBOL(kprobe_exceptions_notify); 522 523 int __init arch_init_kprobes(void) 524 { 525 return 0; 526 } 527 528 int arch_trampoline_kprobe(struct kprobe *p) 529 { 530 return 0; 531 } 532 NOKPROBE_SYMBOL(arch_trampoline_kprobe); 533