1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Kernel Probes (KProbes) 4 * 5 * Copyright (C) IBM Corporation, 2002, 2004 6 * 7 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 8 * Probes initial implementation ( includes contributions from 9 * Rusty Russell). 10 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 11 * interface to access function arguments. 12 * 2004-Oct Jim Keniston <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 13 * <prasanna@in.ibm.com> adapted for x86_64 from i386. 14 * 2005-Mar Roland McGrath <roland@redhat.com> 15 * Fixed to handle %rip-relative addressing mode correctly. 16 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston 17 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 18 * <prasanna@in.ibm.com> added function-return probes. 19 * 2005-May Rusty Lynch <rusty.lynch@intel.com> 20 * Added function return probes functionality 21 * 2006-Feb Masami Hiramatsu <hiramatu@sdl.hitachi.co.jp> added 22 * kprobe-booster and kretprobe-booster for i386. 23 * 2007-Dec Masami Hiramatsu <mhiramat@redhat.com> added kprobe-booster 24 * and kretprobe-booster for x86-64 25 * 2007-Dec Masami Hiramatsu <mhiramat@redhat.com>, Arjan van de Ven 26 * <arjan@infradead.org> and Jim Keniston <jkenisto@us.ibm.com> 27 * unified x86 kprobes code. 28 */ 29 #include <linux/kprobes.h> 30 #include <linux/ptrace.h> 31 #include <linux/string.h> 32 #include <linux/slab.h> 33 #include <linux/hardirq.h> 34 #include <linux/preempt.h> 35 #include <linux/sched/debug.h> 36 #include <linux/perf_event.h> 37 #include <linux/extable.h> 38 #include <linux/kdebug.h> 39 #include <linux/kallsyms.h> 40 #include <linux/kgdb.h> 41 #include <linux/ftrace.h> 42 #include <linux/kasan.h> 43 #include <linux/moduleloader.h> 44 #include <linux/objtool.h> 45 #include <linux/vmalloc.h> 46 #include <linux/pgtable.h> 47 #include <linux/set_memory.h> 48 49 #include <asm/text-patching.h> 50 #include <asm/cacheflush.h> 51 #include <asm/desc.h> 52 #include <linux/uaccess.h> 53 #include <asm/alternative.h> 54 #include <asm/insn.h> 55 #include <asm/debugreg.h> 56 #include <asm/ibt.h> 57 58 #include "common.h" 59 60 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; 61 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); 62 63 #define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\ 64 (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \ 65 (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) | \ 66 (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) | \ 67 (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf)) \ 68 << (row % 32)) 69 /* 70 * Undefined/reserved opcodes, conditional jump, Opcode Extension 71 * Groups, and some special opcodes can not boost. 72 * This is non-const and volatile to keep gcc from statically 73 * optimizing it out, as variable_test_bit makes gcc think only 74 * *(unsigned long*) is used. 75 */ 76 static volatile u32 twobyte_is_boostable[256 / 32] = { 77 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ 78 /* ---------------------------------------------- */ 79 W(0x00, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0) | /* 00 */ 80 W(0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1) , /* 10 */ 81 W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 20 */ 82 W(0x30, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 30 */ 83 W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */ 84 W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 50 */ 85 W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1) | /* 60 */ 86 W(0x70, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) , /* 70 */ 87 W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 80 */ 88 W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */ 89 W(0xa0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* a0 */ 90 W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1) , /* b0 */ 91 W(0xc0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */ 92 W(0xd0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) , /* d0 */ 93 W(0xe0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* e0 */ 94 W(0xf0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0) /* f0 */ 95 /* ----------------------------------------------- */ 96 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ 97 }; 98 #undef W 99 100 struct kretprobe_blackpoint kretprobe_blacklist[] = { 101 {"__switch_to", }, /* This function switches only current task, but 102 doesn't switch kernel stack.*/ 103 {NULL, NULL} /* Terminator */ 104 }; 105 106 const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist); 107 108 static nokprobe_inline void 109 __synthesize_relative_insn(void *dest, void *from, void *to, u8 op) 110 { 111 struct __arch_relative_insn { 112 u8 op; 113 s32 raddr; 114 } __packed *insn; 115 116 insn = (struct __arch_relative_insn *)dest; 117 insn->raddr = (s32)((long)(to) - ((long)(from) + 5)); 118 insn->op = op; 119 } 120 121 /* Insert a jump instruction at address 'from', which jumps to address 'to'.*/ 122 void synthesize_reljump(void *dest, void *from, void *to) 123 { 124 __synthesize_relative_insn(dest, from, to, JMP32_INSN_OPCODE); 125 } 126 NOKPROBE_SYMBOL(synthesize_reljump); 127 128 /* Insert a call instruction at address 'from', which calls address 'to'.*/ 129 void synthesize_relcall(void *dest, void *from, void *to) 130 { 131 __synthesize_relative_insn(dest, from, to, CALL_INSN_OPCODE); 132 } 133 NOKPROBE_SYMBOL(synthesize_relcall); 134 135 /* 136 * Returns non-zero if INSN is boostable. 137 * RIP relative instructions are adjusted at copying time in 64 bits mode 138 */ 139 int can_boost(struct insn *insn, void *addr) 140 { 141 kprobe_opcode_t opcode; 142 insn_byte_t prefix; 143 int i; 144 145 if (search_exception_tables((unsigned long)addr)) 146 return 0; /* Page fault may occur on this address. */ 147 148 /* 2nd-byte opcode */ 149 if (insn->opcode.nbytes == 2) 150 return test_bit(insn->opcode.bytes[1], 151 (unsigned long *)twobyte_is_boostable); 152 153 if (insn->opcode.nbytes != 1) 154 return 0; 155 156 for_each_insn_prefix(insn, i, prefix) { 157 insn_attr_t attr; 158 159 attr = inat_get_opcode_attribute(prefix); 160 /* Can't boost Address-size override prefix and CS override prefix */ 161 if (prefix == 0x2e || inat_is_address_size_prefix(attr)) 162 return 0; 163 } 164 165 opcode = insn->opcode.bytes[0]; 166 167 switch (opcode) { 168 case 0x62: /* bound */ 169 case 0x70 ... 0x7f: /* Conditional jumps */ 170 case 0x9a: /* Call far */ 171 case 0xc0 ... 0xc1: /* Grp2 */ 172 case 0xcc ... 0xce: /* software exceptions */ 173 case 0xd0 ... 0xd3: /* Grp2 */ 174 case 0xd6: /* (UD) */ 175 case 0xd8 ... 0xdf: /* ESC */ 176 case 0xe0 ... 0xe3: /* LOOP*, JCXZ */ 177 case 0xe8 ... 0xe9: /* near Call, JMP */ 178 case 0xeb: /* Short JMP */ 179 case 0xf0 ... 0xf4: /* LOCK/REP, HLT */ 180 case 0xf6 ... 0xf7: /* Grp3 */ 181 case 0xfe: /* Grp4 */ 182 /* ... are not boostable */ 183 return 0; 184 case 0xff: /* Grp5 */ 185 /* Only indirect jmp is boostable */ 186 return X86_MODRM_REG(insn->modrm.bytes[0]) == 4; 187 default: 188 return 1; 189 } 190 } 191 192 static unsigned long 193 __recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr) 194 { 195 struct kprobe *kp; 196 bool faddr; 197 198 kp = get_kprobe((void *)addr); 199 faddr = ftrace_location(addr) == addr; 200 /* 201 * Use the current code if it is not modified by Kprobe 202 * and it cannot be modified by ftrace. 203 */ 204 if (!kp && !faddr) 205 return addr; 206 207 /* 208 * Basically, kp->ainsn.insn has an original instruction. 209 * However, RIP-relative instruction can not do single-stepping 210 * at different place, __copy_instruction() tweaks the displacement of 211 * that instruction. In that case, we can't recover the instruction 212 * from the kp->ainsn.insn. 213 * 214 * On the other hand, in case on normal Kprobe, kp->opcode has a copy 215 * of the first byte of the probed instruction, which is overwritten 216 * by int3. And the instruction at kp->addr is not modified by kprobes 217 * except for the first byte, we can recover the original instruction 218 * from it and kp->opcode. 219 * 220 * In case of Kprobes using ftrace, we do not have a copy of 221 * the original instruction. In fact, the ftrace location might 222 * be modified at anytime and even could be in an inconsistent state. 223 * Fortunately, we know that the original code is the ideal 5-byte 224 * long NOP. 225 */ 226 if (copy_from_kernel_nofault(buf, (void *)addr, 227 MAX_INSN_SIZE * sizeof(kprobe_opcode_t))) 228 return 0UL; 229 230 if (faddr) 231 memcpy(buf, x86_nops[5], 5); 232 else 233 buf[0] = kp->opcode; 234 return (unsigned long)buf; 235 } 236 237 /* 238 * Recover the probed instruction at addr for further analysis. 239 * Caller must lock kprobes by kprobe_mutex, or disable preemption 240 * for preventing to release referencing kprobes. 241 * Returns zero if the instruction can not get recovered (or access failed). 242 */ 243 unsigned long recover_probed_instruction(kprobe_opcode_t *buf, unsigned long addr) 244 { 245 unsigned long __addr; 246 247 __addr = __recover_optprobed_insn(buf, addr); 248 if (__addr != addr) 249 return __addr; 250 251 return __recover_probed_insn(buf, addr); 252 } 253 254 /* Check if paddr is at an instruction boundary */ 255 static int can_probe(unsigned long paddr) 256 { 257 unsigned long addr, __addr, offset = 0; 258 struct insn insn; 259 kprobe_opcode_t buf[MAX_INSN_SIZE]; 260 261 if (!kallsyms_lookup_size_offset(paddr, NULL, &offset)) 262 return 0; 263 264 /* Decode instructions */ 265 addr = paddr - offset; 266 while (addr < paddr) { 267 int ret; 268 269 /* 270 * Check if the instruction has been modified by another 271 * kprobe, in which case we replace the breakpoint by the 272 * original instruction in our buffer. 273 * Also, jump optimization will change the breakpoint to 274 * relative-jump. Since the relative-jump itself is 275 * normally used, we just go through if there is no kprobe. 276 */ 277 __addr = recover_probed_instruction(buf, addr); 278 if (!__addr) 279 return 0; 280 281 ret = insn_decode_kernel(&insn, (void *)__addr); 282 if (ret < 0) 283 return 0; 284 285 #ifdef CONFIG_KGDB 286 /* 287 * If there is a dynamically installed kgdb sw breakpoint, 288 * this function should not be probed. 289 */ 290 if (insn.opcode.bytes[0] == INT3_INSN_OPCODE && 291 kgdb_has_hit_break(addr)) 292 return 0; 293 #endif 294 addr += insn.length; 295 } 296 297 return (addr == paddr); 298 } 299 300 /* If x86 supports IBT (ENDBR) it must be skipped. */ 301 kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset, 302 bool *on_func_entry) 303 { 304 if (is_endbr(*(u32 *)addr)) { 305 *on_func_entry = !offset || offset == 4; 306 if (*on_func_entry) 307 offset = 4; 308 309 } else { 310 *on_func_entry = !offset; 311 } 312 313 return (kprobe_opcode_t *)(addr + offset); 314 } 315 316 /* 317 * Copy an instruction with recovering modified instruction by kprobes 318 * and adjust the displacement if the instruction uses the %rip-relative 319 * addressing mode. Note that since @real will be the final place of copied 320 * instruction, displacement must be adjust by @real, not @dest. 321 * This returns the length of copied instruction, or 0 if it has an error. 322 */ 323 int __copy_instruction(u8 *dest, u8 *src, u8 *real, struct insn *insn) 324 { 325 kprobe_opcode_t buf[MAX_INSN_SIZE]; 326 unsigned long recovered_insn = recover_probed_instruction(buf, (unsigned long)src); 327 int ret; 328 329 if (!recovered_insn || !insn) 330 return 0; 331 332 /* This can access kernel text if given address is not recovered */ 333 if (copy_from_kernel_nofault(dest, (void *)recovered_insn, 334 MAX_INSN_SIZE)) 335 return 0; 336 337 ret = insn_decode_kernel(insn, dest); 338 if (ret < 0) 339 return 0; 340 341 /* We can not probe force emulate prefixed instruction */ 342 if (insn_has_emulate_prefix(insn)) 343 return 0; 344 345 /* Another subsystem puts a breakpoint, failed to recover */ 346 if (insn->opcode.bytes[0] == INT3_INSN_OPCODE) 347 return 0; 348 349 /* We should not singlestep on the exception masking instructions */ 350 if (insn_masking_exception(insn)) 351 return 0; 352 353 #ifdef CONFIG_X86_64 354 /* Only x86_64 has RIP relative instructions */ 355 if (insn_rip_relative(insn)) { 356 s64 newdisp; 357 u8 *disp; 358 /* 359 * The copied instruction uses the %rip-relative addressing 360 * mode. Adjust the displacement for the difference between 361 * the original location of this instruction and the location 362 * of the copy that will actually be run. The tricky bit here 363 * is making sure that the sign extension happens correctly in 364 * this calculation, since we need a signed 32-bit result to 365 * be sign-extended to 64 bits when it's added to the %rip 366 * value and yield the same 64-bit result that the sign- 367 * extension of the original signed 32-bit displacement would 368 * have given. 369 */ 370 newdisp = (u8 *) src + (s64) insn->displacement.value 371 - (u8 *) real; 372 if ((s64) (s32) newdisp != newdisp) { 373 pr_err("Kprobes error: new displacement does not fit into s32 (%llx)\n", newdisp); 374 return 0; 375 } 376 disp = (u8 *) dest + insn_offset_displacement(insn); 377 *(s32 *) disp = (s32) newdisp; 378 } 379 #endif 380 return insn->length; 381 } 382 383 /* Prepare reljump or int3 right after instruction */ 384 static int prepare_singlestep(kprobe_opcode_t *buf, struct kprobe *p, 385 struct insn *insn) 386 { 387 int len = insn->length; 388 389 if (!IS_ENABLED(CONFIG_PREEMPTION) && 390 !p->post_handler && can_boost(insn, p->addr) && 391 MAX_INSN_SIZE - len >= JMP32_INSN_SIZE) { 392 /* 393 * These instructions can be executed directly if it 394 * jumps back to correct address. 395 */ 396 synthesize_reljump(buf + len, p->ainsn.insn + len, 397 p->addr + insn->length); 398 len += JMP32_INSN_SIZE; 399 p->ainsn.boostable = 1; 400 } else { 401 /* Otherwise, put an int3 for trapping singlestep */ 402 if (MAX_INSN_SIZE - len < INT3_INSN_SIZE) 403 return -ENOSPC; 404 405 buf[len] = INT3_INSN_OPCODE; 406 len += INT3_INSN_SIZE; 407 } 408 409 return len; 410 } 411 412 /* Make page to RO mode when allocate it */ 413 void *alloc_insn_page(void) 414 { 415 void *page; 416 417 page = module_alloc(PAGE_SIZE); 418 if (!page) 419 return NULL; 420 421 /* 422 * TODO: Once additional kernel code protection mechanisms are set, ensure 423 * that the page was not maliciously altered and it is still zeroed. 424 */ 425 set_memory_rox((unsigned long)page, 1); 426 427 return page; 428 } 429 430 /* Kprobe x86 instruction emulation - only regs->ip or IF flag modifiers */ 431 432 static void kprobe_emulate_ifmodifiers(struct kprobe *p, struct pt_regs *regs) 433 { 434 switch (p->ainsn.opcode) { 435 case 0xfa: /* cli */ 436 regs->flags &= ~(X86_EFLAGS_IF); 437 break; 438 case 0xfb: /* sti */ 439 regs->flags |= X86_EFLAGS_IF; 440 break; 441 case 0x9c: /* pushf */ 442 int3_emulate_push(regs, regs->flags); 443 break; 444 case 0x9d: /* popf */ 445 regs->flags = int3_emulate_pop(regs); 446 break; 447 } 448 regs->ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 449 } 450 NOKPROBE_SYMBOL(kprobe_emulate_ifmodifiers); 451 452 static void kprobe_emulate_ret(struct kprobe *p, struct pt_regs *regs) 453 { 454 int3_emulate_ret(regs); 455 } 456 NOKPROBE_SYMBOL(kprobe_emulate_ret); 457 458 static void kprobe_emulate_call(struct kprobe *p, struct pt_regs *regs) 459 { 460 unsigned long func = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 461 462 func += p->ainsn.rel32; 463 int3_emulate_call(regs, func); 464 } 465 NOKPROBE_SYMBOL(kprobe_emulate_call); 466 467 static void kprobe_emulate_jmp(struct kprobe *p, struct pt_regs *regs) 468 { 469 unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 470 471 ip += p->ainsn.rel32; 472 int3_emulate_jmp(regs, ip); 473 } 474 NOKPROBE_SYMBOL(kprobe_emulate_jmp); 475 476 static void kprobe_emulate_jcc(struct kprobe *p, struct pt_regs *regs) 477 { 478 unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 479 480 int3_emulate_jcc(regs, p->ainsn.jcc.type, ip, p->ainsn.rel32); 481 } 482 NOKPROBE_SYMBOL(kprobe_emulate_jcc); 483 484 static void kprobe_emulate_loop(struct kprobe *p, struct pt_regs *regs) 485 { 486 unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 487 bool match; 488 489 if (p->ainsn.loop.type != 3) { /* LOOP* */ 490 if (p->ainsn.loop.asize == 32) 491 match = ((*(u32 *)®s->cx)--) != 0; 492 #ifdef CONFIG_X86_64 493 else if (p->ainsn.loop.asize == 64) 494 match = ((*(u64 *)®s->cx)--) != 0; 495 #endif 496 else 497 match = ((*(u16 *)®s->cx)--) != 0; 498 } else { /* JCXZ */ 499 if (p->ainsn.loop.asize == 32) 500 match = *(u32 *)(®s->cx) == 0; 501 #ifdef CONFIG_X86_64 502 else if (p->ainsn.loop.asize == 64) 503 match = *(u64 *)(®s->cx) == 0; 504 #endif 505 else 506 match = *(u16 *)(®s->cx) == 0; 507 } 508 509 if (p->ainsn.loop.type == 0) /* LOOPNE */ 510 match = match && !(regs->flags & X86_EFLAGS_ZF); 511 else if (p->ainsn.loop.type == 1) /* LOOPE */ 512 match = match && (regs->flags & X86_EFLAGS_ZF); 513 514 if (match) 515 ip += p->ainsn.rel32; 516 int3_emulate_jmp(regs, ip); 517 } 518 NOKPROBE_SYMBOL(kprobe_emulate_loop); 519 520 static const int addrmode_regoffs[] = { 521 offsetof(struct pt_regs, ax), 522 offsetof(struct pt_regs, cx), 523 offsetof(struct pt_regs, dx), 524 offsetof(struct pt_regs, bx), 525 offsetof(struct pt_regs, sp), 526 offsetof(struct pt_regs, bp), 527 offsetof(struct pt_regs, si), 528 offsetof(struct pt_regs, di), 529 #ifdef CONFIG_X86_64 530 offsetof(struct pt_regs, r8), 531 offsetof(struct pt_regs, r9), 532 offsetof(struct pt_regs, r10), 533 offsetof(struct pt_regs, r11), 534 offsetof(struct pt_regs, r12), 535 offsetof(struct pt_regs, r13), 536 offsetof(struct pt_regs, r14), 537 offsetof(struct pt_regs, r15), 538 #endif 539 }; 540 541 static void kprobe_emulate_call_indirect(struct kprobe *p, struct pt_regs *regs) 542 { 543 unsigned long offs = addrmode_regoffs[p->ainsn.indirect.reg]; 544 545 int3_emulate_call(regs, regs_get_register(regs, offs)); 546 } 547 NOKPROBE_SYMBOL(kprobe_emulate_call_indirect); 548 549 static void kprobe_emulate_jmp_indirect(struct kprobe *p, struct pt_regs *regs) 550 { 551 unsigned long offs = addrmode_regoffs[p->ainsn.indirect.reg]; 552 553 int3_emulate_jmp(regs, regs_get_register(regs, offs)); 554 } 555 NOKPROBE_SYMBOL(kprobe_emulate_jmp_indirect); 556 557 static int prepare_emulation(struct kprobe *p, struct insn *insn) 558 { 559 insn_byte_t opcode = insn->opcode.bytes[0]; 560 561 switch (opcode) { 562 case 0xfa: /* cli */ 563 case 0xfb: /* sti */ 564 case 0x9c: /* pushfl */ 565 case 0x9d: /* popf/popfd */ 566 /* 567 * IF modifiers must be emulated since it will enable interrupt while 568 * int3 single stepping. 569 */ 570 p->ainsn.emulate_op = kprobe_emulate_ifmodifiers; 571 p->ainsn.opcode = opcode; 572 break; 573 case 0xc2: /* ret/lret */ 574 case 0xc3: 575 case 0xca: 576 case 0xcb: 577 p->ainsn.emulate_op = kprobe_emulate_ret; 578 break; 579 case 0x9a: /* far call absolute -- segment is not supported */ 580 case 0xea: /* far jmp absolute -- segment is not supported */ 581 case 0xcc: /* int3 */ 582 case 0xcf: /* iret -- in-kernel IRET is not supported */ 583 return -EOPNOTSUPP; 584 break; 585 case 0xe8: /* near call relative */ 586 p->ainsn.emulate_op = kprobe_emulate_call; 587 if (insn->immediate.nbytes == 2) 588 p->ainsn.rel32 = *(s16 *)&insn->immediate.value; 589 else 590 p->ainsn.rel32 = *(s32 *)&insn->immediate.value; 591 break; 592 case 0xeb: /* short jump relative */ 593 case 0xe9: /* near jump relative */ 594 p->ainsn.emulate_op = kprobe_emulate_jmp; 595 if (insn->immediate.nbytes == 1) 596 p->ainsn.rel32 = *(s8 *)&insn->immediate.value; 597 else if (insn->immediate.nbytes == 2) 598 p->ainsn.rel32 = *(s16 *)&insn->immediate.value; 599 else 600 p->ainsn.rel32 = *(s32 *)&insn->immediate.value; 601 break; 602 case 0x70 ... 0x7f: 603 /* 1 byte conditional jump */ 604 p->ainsn.emulate_op = kprobe_emulate_jcc; 605 p->ainsn.jcc.type = opcode & 0xf; 606 p->ainsn.rel32 = insn->immediate.value; 607 break; 608 case 0x0f: 609 opcode = insn->opcode.bytes[1]; 610 if ((opcode & 0xf0) == 0x80) { 611 /* 2 bytes Conditional Jump */ 612 p->ainsn.emulate_op = kprobe_emulate_jcc; 613 p->ainsn.jcc.type = opcode & 0xf; 614 if (insn->immediate.nbytes == 2) 615 p->ainsn.rel32 = *(s16 *)&insn->immediate.value; 616 else 617 p->ainsn.rel32 = *(s32 *)&insn->immediate.value; 618 } else if (opcode == 0x01 && 619 X86_MODRM_REG(insn->modrm.bytes[0]) == 0 && 620 X86_MODRM_MOD(insn->modrm.bytes[0]) == 3) { 621 /* VM extensions - not supported */ 622 return -EOPNOTSUPP; 623 } 624 break; 625 case 0xe0: /* Loop NZ */ 626 case 0xe1: /* Loop */ 627 case 0xe2: /* Loop */ 628 case 0xe3: /* J*CXZ */ 629 p->ainsn.emulate_op = kprobe_emulate_loop; 630 p->ainsn.loop.type = opcode & 0x3; 631 p->ainsn.loop.asize = insn->addr_bytes * 8; 632 p->ainsn.rel32 = *(s8 *)&insn->immediate.value; 633 break; 634 case 0xff: 635 /* 636 * Since the 0xff is an extended group opcode, the instruction 637 * is determined by the MOD/RM byte. 638 */ 639 opcode = insn->modrm.bytes[0]; 640 switch (X86_MODRM_REG(opcode)) { 641 case 0b010: /* FF /2, call near, absolute indirect */ 642 p->ainsn.emulate_op = kprobe_emulate_call_indirect; 643 break; 644 case 0b100: /* FF /4, jmp near, absolute indirect */ 645 p->ainsn.emulate_op = kprobe_emulate_jmp_indirect; 646 break; 647 case 0b011: /* FF /3, call far, absolute indirect */ 648 case 0b101: /* FF /5, jmp far, absolute indirect */ 649 return -EOPNOTSUPP; 650 } 651 652 if (!p->ainsn.emulate_op) 653 break; 654 655 if (insn->addr_bytes != sizeof(unsigned long)) 656 return -EOPNOTSUPP; /* Don't support different size */ 657 if (X86_MODRM_MOD(opcode) != 3) 658 return -EOPNOTSUPP; /* TODO: support memory addressing */ 659 660 p->ainsn.indirect.reg = X86_MODRM_RM(opcode); 661 #ifdef CONFIG_X86_64 662 if (X86_REX_B(insn->rex_prefix.value)) 663 p->ainsn.indirect.reg += 8; 664 #endif 665 break; 666 default: 667 break; 668 } 669 p->ainsn.size = insn->length; 670 671 return 0; 672 } 673 674 static int arch_copy_kprobe(struct kprobe *p) 675 { 676 struct insn insn; 677 kprobe_opcode_t buf[MAX_INSN_SIZE]; 678 int ret, len; 679 680 /* Copy an instruction with recovering if other optprobe modifies it.*/ 681 len = __copy_instruction(buf, p->addr, p->ainsn.insn, &insn); 682 if (!len) 683 return -EINVAL; 684 685 /* Analyze the opcode and setup emulate functions */ 686 ret = prepare_emulation(p, &insn); 687 if (ret < 0) 688 return ret; 689 690 /* Add int3 for single-step or booster jmp */ 691 len = prepare_singlestep(buf, p, &insn); 692 if (len < 0) 693 return len; 694 695 /* Also, displacement change doesn't affect the first byte */ 696 p->opcode = buf[0]; 697 698 p->ainsn.tp_len = len; 699 perf_event_text_poke(p->ainsn.insn, NULL, 0, buf, len); 700 701 /* OK, write back the instruction(s) into ROX insn buffer */ 702 text_poke(p->ainsn.insn, buf, len); 703 704 return 0; 705 } 706 707 int arch_prepare_kprobe(struct kprobe *p) 708 { 709 int ret; 710 711 if (alternatives_text_reserved(p->addr, p->addr)) 712 return -EINVAL; 713 714 if (!can_probe((unsigned long)p->addr)) 715 return -EILSEQ; 716 717 memset(&p->ainsn, 0, sizeof(p->ainsn)); 718 719 /* insn: must be on special executable page on x86. */ 720 p->ainsn.insn = get_insn_slot(); 721 if (!p->ainsn.insn) 722 return -ENOMEM; 723 724 ret = arch_copy_kprobe(p); 725 if (ret) { 726 free_insn_slot(p->ainsn.insn, 0); 727 p->ainsn.insn = NULL; 728 } 729 730 return ret; 731 } 732 733 void arch_arm_kprobe(struct kprobe *p) 734 { 735 u8 int3 = INT3_INSN_OPCODE; 736 737 text_poke(p->addr, &int3, 1); 738 text_poke_sync(); 739 perf_event_text_poke(p->addr, &p->opcode, 1, &int3, 1); 740 } 741 742 void arch_disarm_kprobe(struct kprobe *p) 743 { 744 u8 int3 = INT3_INSN_OPCODE; 745 746 perf_event_text_poke(p->addr, &int3, 1, &p->opcode, 1); 747 text_poke(p->addr, &p->opcode, 1); 748 text_poke_sync(); 749 } 750 751 void arch_remove_kprobe(struct kprobe *p) 752 { 753 if (p->ainsn.insn) { 754 /* Record the perf event before freeing the slot */ 755 perf_event_text_poke(p->ainsn.insn, p->ainsn.insn, 756 p->ainsn.tp_len, NULL, 0); 757 free_insn_slot(p->ainsn.insn, p->ainsn.boostable); 758 p->ainsn.insn = NULL; 759 } 760 } 761 762 static nokprobe_inline void 763 save_previous_kprobe(struct kprobe_ctlblk *kcb) 764 { 765 kcb->prev_kprobe.kp = kprobe_running(); 766 kcb->prev_kprobe.status = kcb->kprobe_status; 767 kcb->prev_kprobe.old_flags = kcb->kprobe_old_flags; 768 kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags; 769 } 770 771 static nokprobe_inline void 772 restore_previous_kprobe(struct kprobe_ctlblk *kcb) 773 { 774 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); 775 kcb->kprobe_status = kcb->prev_kprobe.status; 776 kcb->kprobe_old_flags = kcb->prev_kprobe.old_flags; 777 kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags; 778 } 779 780 static nokprobe_inline void 781 set_current_kprobe(struct kprobe *p, struct pt_regs *regs, 782 struct kprobe_ctlblk *kcb) 783 { 784 __this_cpu_write(current_kprobe, p); 785 kcb->kprobe_saved_flags = kcb->kprobe_old_flags 786 = (regs->flags & X86_EFLAGS_IF); 787 } 788 789 static void kprobe_post_process(struct kprobe *cur, struct pt_regs *regs, 790 struct kprobe_ctlblk *kcb) 791 { 792 /* Restore back the original saved kprobes variables and continue. */ 793 if (kcb->kprobe_status == KPROBE_REENTER) { 794 /* This will restore both kcb and current_kprobe */ 795 restore_previous_kprobe(kcb); 796 } else { 797 /* 798 * Always update the kcb status because 799 * reset_curent_kprobe() doesn't update kcb. 800 */ 801 kcb->kprobe_status = KPROBE_HIT_SSDONE; 802 if (cur->post_handler) 803 cur->post_handler(cur, regs, 0); 804 reset_current_kprobe(); 805 } 806 } 807 NOKPROBE_SYMBOL(kprobe_post_process); 808 809 static void setup_singlestep(struct kprobe *p, struct pt_regs *regs, 810 struct kprobe_ctlblk *kcb, int reenter) 811 { 812 if (setup_detour_execution(p, regs, reenter)) 813 return; 814 815 #if !defined(CONFIG_PREEMPTION) 816 if (p->ainsn.boostable) { 817 /* Boost up -- we can execute copied instructions directly */ 818 if (!reenter) 819 reset_current_kprobe(); 820 /* 821 * Reentering boosted probe doesn't reset current_kprobe, 822 * nor set current_kprobe, because it doesn't use single 823 * stepping. 824 */ 825 regs->ip = (unsigned long)p->ainsn.insn; 826 return; 827 } 828 #endif 829 if (reenter) { 830 save_previous_kprobe(kcb); 831 set_current_kprobe(p, regs, kcb); 832 kcb->kprobe_status = KPROBE_REENTER; 833 } else 834 kcb->kprobe_status = KPROBE_HIT_SS; 835 836 if (p->ainsn.emulate_op) { 837 p->ainsn.emulate_op(p, regs); 838 kprobe_post_process(p, regs, kcb); 839 return; 840 } 841 842 /* Disable interrupt, and set ip register on trampoline */ 843 regs->flags &= ~X86_EFLAGS_IF; 844 regs->ip = (unsigned long)p->ainsn.insn; 845 } 846 NOKPROBE_SYMBOL(setup_singlestep); 847 848 /* 849 * Called after single-stepping. p->addr is the address of the 850 * instruction whose first byte has been replaced by the "int3" 851 * instruction. To avoid the SMP problems that can occur when we 852 * temporarily put back the original opcode to single-step, we 853 * single-stepped a copy of the instruction. The address of this 854 * copy is p->ainsn.insn. We also doesn't use trap, but "int3" again 855 * right after the copied instruction. 856 * Different from the trap single-step, "int3" single-step can not 857 * handle the instruction which changes the ip register, e.g. jmp, 858 * call, conditional jmp, and the instructions which changes the IF 859 * flags because interrupt must be disabled around the single-stepping. 860 * Such instructions are software emulated, but others are single-stepped 861 * using "int3". 862 * 863 * When the 2nd "int3" handled, the regs->ip and regs->flags needs to 864 * be adjusted, so that we can resume execution on correct code. 865 */ 866 static void resume_singlestep(struct kprobe *p, struct pt_regs *regs, 867 struct kprobe_ctlblk *kcb) 868 { 869 unsigned long copy_ip = (unsigned long)p->ainsn.insn; 870 unsigned long orig_ip = (unsigned long)p->addr; 871 872 /* Restore saved interrupt flag and ip register */ 873 regs->flags |= kcb->kprobe_saved_flags; 874 /* Note that regs->ip is executed int3 so must be a step back */ 875 regs->ip += (orig_ip - copy_ip) - INT3_INSN_SIZE; 876 } 877 NOKPROBE_SYMBOL(resume_singlestep); 878 879 /* 880 * We have reentered the kprobe_handler(), since another probe was hit while 881 * within the handler. We save the original kprobes variables and just single 882 * step on the instruction of the new probe without calling any user handlers. 883 */ 884 static int reenter_kprobe(struct kprobe *p, struct pt_regs *regs, 885 struct kprobe_ctlblk *kcb) 886 { 887 switch (kcb->kprobe_status) { 888 case KPROBE_HIT_SSDONE: 889 case KPROBE_HIT_ACTIVE: 890 case KPROBE_HIT_SS: 891 kprobes_inc_nmissed_count(p); 892 setup_singlestep(p, regs, kcb, 1); 893 break; 894 case KPROBE_REENTER: 895 /* A probe has been hit in the codepath leading up to, or just 896 * after, single-stepping of a probed instruction. This entire 897 * codepath should strictly reside in .kprobes.text section. 898 * Raise a BUG or we'll continue in an endless reentering loop 899 * and eventually a stack overflow. 900 */ 901 pr_err("Unrecoverable kprobe detected.\n"); 902 dump_kprobe(p); 903 BUG(); 904 default: 905 /* impossible cases */ 906 WARN_ON(1); 907 return 0; 908 } 909 910 return 1; 911 } 912 NOKPROBE_SYMBOL(reenter_kprobe); 913 914 static nokprobe_inline int kprobe_is_ss(struct kprobe_ctlblk *kcb) 915 { 916 return (kcb->kprobe_status == KPROBE_HIT_SS || 917 kcb->kprobe_status == KPROBE_REENTER); 918 } 919 920 /* 921 * Interrupts are disabled on entry as trap3 is an interrupt gate and they 922 * remain disabled throughout this function. 923 */ 924 int kprobe_int3_handler(struct pt_regs *regs) 925 { 926 kprobe_opcode_t *addr; 927 struct kprobe *p; 928 struct kprobe_ctlblk *kcb; 929 930 if (user_mode(regs)) 931 return 0; 932 933 addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t)); 934 /* 935 * We don't want to be preempted for the entire duration of kprobe 936 * processing. Since int3 and debug trap disables irqs and we clear 937 * IF while singlestepping, it must be no preemptible. 938 */ 939 940 kcb = get_kprobe_ctlblk(); 941 p = get_kprobe(addr); 942 943 if (p) { 944 if (kprobe_running()) { 945 if (reenter_kprobe(p, regs, kcb)) 946 return 1; 947 } else { 948 set_current_kprobe(p, regs, kcb); 949 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 950 951 /* 952 * If we have no pre-handler or it returned 0, we 953 * continue with normal processing. If we have a 954 * pre-handler and it returned non-zero, that means 955 * user handler setup registers to exit to another 956 * instruction, we must skip the single stepping. 957 */ 958 if (!p->pre_handler || !p->pre_handler(p, regs)) 959 setup_singlestep(p, regs, kcb, 0); 960 else 961 reset_current_kprobe(); 962 return 1; 963 } 964 } else if (kprobe_is_ss(kcb)) { 965 p = kprobe_running(); 966 if ((unsigned long)p->ainsn.insn < regs->ip && 967 (unsigned long)p->ainsn.insn + MAX_INSN_SIZE > regs->ip) { 968 /* Most provably this is the second int3 for singlestep */ 969 resume_singlestep(p, regs, kcb); 970 kprobe_post_process(p, regs, kcb); 971 return 1; 972 } 973 } /* else: not a kprobe fault; let the kernel handle it */ 974 975 return 0; 976 } 977 NOKPROBE_SYMBOL(kprobe_int3_handler); 978 979 int kprobe_fault_handler(struct pt_regs *regs, int trapnr) 980 { 981 struct kprobe *cur = kprobe_running(); 982 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 983 984 if (unlikely(regs->ip == (unsigned long)cur->ainsn.insn)) { 985 /* This must happen on single-stepping */ 986 WARN_ON(kcb->kprobe_status != KPROBE_HIT_SS && 987 kcb->kprobe_status != KPROBE_REENTER); 988 /* 989 * We are here because the instruction being single 990 * stepped caused a page fault. We reset the current 991 * kprobe and the ip points back to the probe address 992 * and allow the page fault handler to continue as a 993 * normal page fault. 994 */ 995 regs->ip = (unsigned long)cur->addr; 996 997 /* 998 * If the IF flag was set before the kprobe hit, 999 * don't touch it: 1000 */ 1001 regs->flags |= kcb->kprobe_old_flags; 1002 1003 if (kcb->kprobe_status == KPROBE_REENTER) 1004 restore_previous_kprobe(kcb); 1005 else 1006 reset_current_kprobe(); 1007 } 1008 1009 return 0; 1010 } 1011 NOKPROBE_SYMBOL(kprobe_fault_handler); 1012 1013 int __init arch_populate_kprobe_blacklist(void) 1014 { 1015 return kprobe_add_area_blacklist((unsigned long)__entry_text_start, 1016 (unsigned long)__entry_text_end); 1017 } 1018 1019 int __init arch_init_kprobes(void) 1020 { 1021 return 0; 1022 } 1023 1024 int arch_trampoline_kprobe(struct kprobe *p) 1025 { 1026 return 0; 1027 } 1028