1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Kernel Probes (KProbes) 4 * 5 * Copyright (C) IBM Corporation, 2002, 2004 6 * 7 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 8 * Probes initial implementation ( includes contributions from 9 * Rusty Russell). 10 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 11 * interface to access function arguments. 12 * 2004-Oct Jim Keniston <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 13 * <prasanna@in.ibm.com> adapted for x86_64 from i386. 14 * 2005-Mar Roland McGrath <roland@redhat.com> 15 * Fixed to handle %rip-relative addressing mode correctly. 16 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston 17 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 18 * <prasanna@in.ibm.com> added function-return probes. 19 * 2005-May Rusty Lynch <rusty.lynch@intel.com> 20 * Added function return probes functionality 21 * 2006-Feb Masami Hiramatsu <hiramatu@sdl.hitachi.co.jp> added 22 * kprobe-booster and kretprobe-booster for i386. 23 * 2007-Dec Masami Hiramatsu <mhiramat@redhat.com> added kprobe-booster 24 * and kretprobe-booster for x86-64 25 * 2007-Dec Masami Hiramatsu <mhiramat@redhat.com>, Arjan van de Ven 26 * <arjan@infradead.org> and Jim Keniston <jkenisto@us.ibm.com> 27 * unified x86 kprobes code. 28 */ 29 #include <linux/kprobes.h> 30 #include <linux/ptrace.h> 31 #include <linux/string.h> 32 #include <linux/slab.h> 33 #include <linux/hardirq.h> 34 #include <linux/preempt.h> 35 #include <linux/sched/debug.h> 36 #include <linux/perf_event.h> 37 #include <linux/extable.h> 38 #include <linux/kdebug.h> 39 #include <linux/kallsyms.h> 40 #include <linux/kgdb.h> 41 #include <linux/ftrace.h> 42 #include <linux/kasan.h> 43 #include <linux/objtool.h> 44 #include <linux/vmalloc.h> 45 #include <linux/pgtable.h> 46 #include <linux/set_memory.h> 47 #include <linux/cfi.h> 48 #include <linux/execmem.h> 49 50 #include <asm/text-patching.h> 51 #include <asm/cacheflush.h> 52 #include <asm/desc.h> 53 #include <linux/uaccess.h> 54 #include <asm/alternative.h> 55 #include <asm/insn.h> 56 #include <asm/debugreg.h> 57 #include <asm/ibt.h> 58 59 #include "common.h" 60 61 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; 62 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); 63 64 #define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\ 65 (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \ 66 (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) | \ 67 (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) | \ 68 (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf)) \ 69 << (row % 32)) 70 /* 71 * Undefined/reserved opcodes, conditional jump, Opcode Extension 72 * Groups, and some special opcodes can not boost. 73 * This is non-const and volatile to keep gcc from statically 74 * optimizing it out, as variable_test_bit makes gcc think only 75 * *(unsigned long*) is used. 76 */ 77 static volatile u32 twobyte_is_boostable[256 / 32] = { 78 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ 79 /* ---------------------------------------------- */ 80 W(0x00, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0) | /* 00 */ 81 W(0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1) , /* 10 */ 82 W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 20 */ 83 W(0x30, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 30 */ 84 W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */ 85 W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 50 */ 86 W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1) | /* 60 */ 87 W(0x70, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) , /* 70 */ 88 W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 80 */ 89 W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */ 90 W(0xa0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* a0 */ 91 W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1) , /* b0 */ 92 W(0xc0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */ 93 W(0xd0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) , /* d0 */ 94 W(0xe0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* e0 */ 95 W(0xf0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0) /* f0 */ 96 /* ----------------------------------------------- */ 97 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ 98 }; 99 #undef W 100 101 struct kretprobe_blackpoint kretprobe_blacklist[] = { 102 {"__switch_to", }, /* This function switches only current task, but 103 doesn't switch kernel stack.*/ 104 {NULL, NULL} /* Terminator */ 105 }; 106 107 const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist); 108 109 static nokprobe_inline void 110 __synthesize_relative_insn(void *dest, void *from, void *to, u8 op) 111 { 112 struct __arch_relative_insn { 113 u8 op; 114 s32 raddr; 115 } __packed *insn; 116 117 insn = (struct __arch_relative_insn *)dest; 118 insn->raddr = (s32)((long)(to) - ((long)(from) + 5)); 119 insn->op = op; 120 } 121 122 /* Insert a jump instruction at address 'from', which jumps to address 'to'.*/ 123 void synthesize_reljump(void *dest, void *from, void *to) 124 { 125 __synthesize_relative_insn(dest, from, to, JMP32_INSN_OPCODE); 126 } 127 NOKPROBE_SYMBOL(synthesize_reljump); 128 129 /* Insert a call instruction at address 'from', which calls address 'to'.*/ 130 void synthesize_relcall(void *dest, void *from, void *to) 131 { 132 __synthesize_relative_insn(dest, from, to, CALL_INSN_OPCODE); 133 } 134 NOKPROBE_SYMBOL(synthesize_relcall); 135 136 /* 137 * Returns non-zero if INSN is boostable. 138 * RIP relative instructions are adjusted at copying time in 64 bits mode 139 */ 140 bool can_boost(struct insn *insn, void *addr) 141 { 142 kprobe_opcode_t opcode; 143 insn_byte_t prefix; 144 int i; 145 146 if (search_exception_tables((unsigned long)addr)) 147 return false; /* Page fault may occur on this address. */ 148 149 /* 2nd-byte opcode */ 150 if (insn->opcode.nbytes == 2) 151 return test_bit(insn->opcode.bytes[1], 152 (unsigned long *)twobyte_is_boostable); 153 154 if (insn->opcode.nbytes != 1) 155 return false; 156 157 for_each_insn_prefix(insn, i, prefix) { 158 insn_attr_t attr; 159 160 attr = inat_get_opcode_attribute(prefix); 161 /* Can't boost Address-size override prefix and CS override prefix */ 162 if (prefix == 0x2e || inat_is_address_size_prefix(attr)) 163 return false; 164 } 165 166 opcode = insn->opcode.bytes[0]; 167 168 switch (opcode) { 169 case 0x62: /* bound */ 170 case 0x70 ... 0x7f: /* Conditional jumps */ 171 case 0x9a: /* Call far */ 172 case 0xcc ... 0xce: /* software exceptions */ 173 case 0xd6: /* (UD) */ 174 case 0xd8 ... 0xdf: /* ESC */ 175 case 0xe0 ... 0xe3: /* LOOP*, JCXZ */ 176 case 0xe8 ... 0xe9: /* near Call, JMP */ 177 case 0xeb: /* Short JMP */ 178 case 0xf0 ... 0xf4: /* LOCK/REP, HLT */ 179 /* ... are not boostable */ 180 return false; 181 case 0xc0 ... 0xc1: /* Grp2 */ 182 case 0xd0 ... 0xd3: /* Grp2 */ 183 /* 184 * AMD uses nnn == 110 as SHL/SAL, but Intel makes it reserved. 185 */ 186 return X86_MODRM_REG(insn->modrm.bytes[0]) != 0b110; 187 case 0xf6 ... 0xf7: /* Grp3 */ 188 /* AMD uses nnn == 001 as TEST, but Intel makes it reserved. */ 189 return X86_MODRM_REG(insn->modrm.bytes[0]) != 0b001; 190 case 0xfe: /* Grp4 */ 191 /* Only INC and DEC are boostable */ 192 return X86_MODRM_REG(insn->modrm.bytes[0]) == 0b000 || 193 X86_MODRM_REG(insn->modrm.bytes[0]) == 0b001; 194 case 0xff: /* Grp5 */ 195 /* Only INC, DEC, and indirect JMP are boostable */ 196 return X86_MODRM_REG(insn->modrm.bytes[0]) == 0b000 || 197 X86_MODRM_REG(insn->modrm.bytes[0]) == 0b001 || 198 X86_MODRM_REG(insn->modrm.bytes[0]) == 0b100; 199 default: 200 return true; 201 } 202 } 203 204 static unsigned long 205 __recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr) 206 { 207 struct kprobe *kp; 208 bool faddr; 209 210 kp = get_kprobe((void *)addr); 211 faddr = ftrace_location(addr) == addr; 212 /* 213 * Use the current code if it is not modified by Kprobe 214 * and it cannot be modified by ftrace. 215 */ 216 if (!kp && !faddr) 217 return addr; 218 219 /* 220 * Basically, kp->ainsn.insn has an original instruction. 221 * However, RIP-relative instruction can not do single-stepping 222 * at different place, __copy_instruction() tweaks the displacement of 223 * that instruction. In that case, we can't recover the instruction 224 * from the kp->ainsn.insn. 225 * 226 * On the other hand, in case on normal Kprobe, kp->opcode has a copy 227 * of the first byte of the probed instruction, which is overwritten 228 * by int3. And the instruction at kp->addr is not modified by kprobes 229 * except for the first byte, we can recover the original instruction 230 * from it and kp->opcode. 231 * 232 * In case of Kprobes using ftrace, we do not have a copy of 233 * the original instruction. In fact, the ftrace location might 234 * be modified at anytime and even could be in an inconsistent state. 235 * Fortunately, we know that the original code is the ideal 5-byte 236 * long NOP. 237 */ 238 if (copy_from_kernel_nofault(buf, (void *)addr, 239 MAX_INSN_SIZE * sizeof(kprobe_opcode_t))) 240 return 0UL; 241 242 if (faddr) 243 memcpy(buf, x86_nops[5], 5); 244 else 245 buf[0] = kp->opcode; 246 return (unsigned long)buf; 247 } 248 249 /* 250 * Recover the probed instruction at addr for further analysis. 251 * Caller must lock kprobes by kprobe_mutex, or disable preemption 252 * for preventing to release referencing kprobes. 253 * Returns zero if the instruction can not get recovered (or access failed). 254 */ 255 unsigned long recover_probed_instruction(kprobe_opcode_t *buf, unsigned long addr) 256 { 257 unsigned long __addr; 258 259 __addr = __recover_optprobed_insn(buf, addr); 260 if (__addr != addr) 261 return __addr; 262 263 return __recover_probed_insn(buf, addr); 264 } 265 266 /* Check if insn is INT or UD */ 267 static inline bool is_exception_insn(struct insn *insn) 268 { 269 /* UD uses 0f escape */ 270 if (insn->opcode.bytes[0] == 0x0f) { 271 /* UD0 / UD1 / UD2 */ 272 return insn->opcode.bytes[1] == 0xff || 273 insn->opcode.bytes[1] == 0xb9 || 274 insn->opcode.bytes[1] == 0x0b; 275 } 276 277 /* INT3 / INT n / INTO / INT1 */ 278 return insn->opcode.bytes[0] == 0xcc || 279 insn->opcode.bytes[0] == 0xcd || 280 insn->opcode.bytes[0] == 0xce || 281 insn->opcode.bytes[0] == 0xf1; 282 } 283 284 /* 285 * Check if paddr is at an instruction boundary and that instruction can 286 * be probed 287 */ 288 static bool can_probe(unsigned long paddr) 289 { 290 unsigned long addr, __addr, offset = 0; 291 struct insn insn; 292 kprobe_opcode_t buf[MAX_INSN_SIZE]; 293 294 if (!kallsyms_lookup_size_offset(paddr, NULL, &offset)) 295 return false; 296 297 /* Decode instructions */ 298 addr = paddr - offset; 299 while (addr < paddr) { 300 /* 301 * Check if the instruction has been modified by another 302 * kprobe, in which case we replace the breakpoint by the 303 * original instruction in our buffer. 304 * Also, jump optimization will change the breakpoint to 305 * relative-jump. Since the relative-jump itself is 306 * normally used, we just go through if there is no kprobe. 307 */ 308 __addr = recover_probed_instruction(buf, addr); 309 if (!__addr) 310 return false; 311 312 if (insn_decode_kernel(&insn, (void *)__addr) < 0) 313 return false; 314 315 #ifdef CONFIG_KGDB 316 /* 317 * If there is a dynamically installed kgdb sw breakpoint, 318 * this function should not be probed. 319 */ 320 if (insn.opcode.bytes[0] == INT3_INSN_OPCODE && 321 kgdb_has_hit_break(addr)) 322 return false; 323 #endif 324 addr += insn.length; 325 } 326 327 /* Check if paddr is at an instruction boundary */ 328 if (addr != paddr) 329 return false; 330 331 __addr = recover_probed_instruction(buf, addr); 332 if (!__addr) 333 return false; 334 335 if (insn_decode_kernel(&insn, (void *)__addr) < 0) 336 return false; 337 338 /* INT and UD are special and should not be kprobed */ 339 if (is_exception_insn(&insn)) 340 return false; 341 342 if (IS_ENABLED(CONFIG_CFI_CLANG)) { 343 /* 344 * The compiler generates the following instruction sequence 345 * for indirect call checks and cfi.c decodes this; 346 * 347 * movl -<id>, %r10d ; 6 bytes 348 * addl -4(%reg), %r10d ; 4 bytes 349 * je .Ltmp1 ; 2 bytes 350 * ud2 ; <- regs->ip 351 * .Ltmp1: 352 * 353 * Also, these movl and addl are used for showing expected 354 * type. So those must not be touched. 355 */ 356 if (insn.opcode.value == 0xBA) 357 offset = 12; 358 else if (insn.opcode.value == 0x3) 359 offset = 6; 360 else 361 goto out; 362 363 /* This movl/addl is used for decoding CFI. */ 364 if (is_cfi_trap(addr + offset)) 365 return false; 366 } 367 368 out: 369 return true; 370 } 371 372 /* If x86 supports IBT (ENDBR) it must be skipped. */ 373 kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset, 374 bool *on_func_entry) 375 { 376 u32 insn; 377 378 /* 379 * Since 'addr' is not guaranteed to be safe to access, use 380 * copy_from_kernel_nofault() to read the instruction: 381 */ 382 if (copy_from_kernel_nofault(&insn, (void *)addr, sizeof(u32))) 383 return NULL; 384 385 if (is_endbr(insn)) { 386 *on_func_entry = !offset || offset == 4; 387 if (*on_func_entry) 388 offset = 4; 389 390 } else { 391 *on_func_entry = !offset; 392 } 393 394 return (kprobe_opcode_t *)(addr + offset); 395 } 396 397 /* 398 * Copy an instruction with recovering modified instruction by kprobes 399 * and adjust the displacement if the instruction uses the %rip-relative 400 * addressing mode. Note that since @real will be the final place of copied 401 * instruction, displacement must be adjust by @real, not @dest. 402 * This returns the length of copied instruction, or 0 if it has an error. 403 */ 404 int __copy_instruction(u8 *dest, u8 *src, u8 *real, struct insn *insn) 405 { 406 kprobe_opcode_t buf[MAX_INSN_SIZE]; 407 unsigned long recovered_insn = recover_probed_instruction(buf, (unsigned long)src); 408 int ret; 409 410 if (!recovered_insn || !insn) 411 return 0; 412 413 /* This can access kernel text if given address is not recovered */ 414 if (copy_from_kernel_nofault(dest, (void *)recovered_insn, 415 MAX_INSN_SIZE)) 416 return 0; 417 418 ret = insn_decode_kernel(insn, dest); 419 if (ret < 0) 420 return 0; 421 422 /* We can not probe force emulate prefixed instruction */ 423 if (insn_has_emulate_prefix(insn)) 424 return 0; 425 426 /* Another subsystem puts a breakpoint, failed to recover */ 427 if (insn->opcode.bytes[0] == INT3_INSN_OPCODE) 428 return 0; 429 430 /* We should not singlestep on the exception masking instructions */ 431 if (insn_masking_exception(insn)) 432 return 0; 433 434 #ifdef CONFIG_X86_64 435 /* Only x86_64 has RIP relative instructions */ 436 if (insn_rip_relative(insn)) { 437 s64 newdisp; 438 u8 *disp; 439 /* 440 * The copied instruction uses the %rip-relative addressing 441 * mode. Adjust the displacement for the difference between 442 * the original location of this instruction and the location 443 * of the copy that will actually be run. The tricky bit here 444 * is making sure that the sign extension happens correctly in 445 * this calculation, since we need a signed 32-bit result to 446 * be sign-extended to 64 bits when it's added to the %rip 447 * value and yield the same 64-bit result that the sign- 448 * extension of the original signed 32-bit displacement would 449 * have given. 450 */ 451 newdisp = (u8 *) src + (s64) insn->displacement.value 452 - (u8 *) real; 453 if ((s64) (s32) newdisp != newdisp) { 454 pr_err("Kprobes error: new displacement does not fit into s32 (%llx)\n", newdisp); 455 return 0; 456 } 457 disp = (u8 *) dest + insn_offset_displacement(insn); 458 *(s32 *) disp = (s32) newdisp; 459 } 460 #endif 461 return insn->length; 462 } 463 464 /* Prepare reljump or int3 right after instruction */ 465 static int prepare_singlestep(kprobe_opcode_t *buf, struct kprobe *p, 466 struct insn *insn) 467 { 468 int len = insn->length; 469 470 if (!IS_ENABLED(CONFIG_PREEMPTION) && 471 !p->post_handler && can_boost(insn, p->addr) && 472 MAX_INSN_SIZE - len >= JMP32_INSN_SIZE) { 473 /* 474 * These instructions can be executed directly if it 475 * jumps back to correct address. 476 */ 477 synthesize_reljump(buf + len, p->ainsn.insn + len, 478 p->addr + insn->length); 479 len += JMP32_INSN_SIZE; 480 p->ainsn.boostable = 1; 481 } else { 482 /* Otherwise, put an int3 for trapping singlestep */ 483 if (MAX_INSN_SIZE - len < INT3_INSN_SIZE) 484 return -ENOSPC; 485 486 buf[len] = INT3_INSN_OPCODE; 487 len += INT3_INSN_SIZE; 488 } 489 490 return len; 491 } 492 493 /* Make page to RO mode when allocate it */ 494 void *alloc_insn_page(void) 495 { 496 void *page; 497 498 page = execmem_alloc(EXECMEM_KPROBES, PAGE_SIZE); 499 if (!page) 500 return NULL; 501 502 /* 503 * TODO: Once additional kernel code protection mechanisms are set, ensure 504 * that the page was not maliciously altered and it is still zeroed. 505 */ 506 set_memory_rox((unsigned long)page, 1); 507 508 return page; 509 } 510 511 /* Kprobe x86 instruction emulation - only regs->ip or IF flag modifiers */ 512 513 static void kprobe_emulate_ifmodifiers(struct kprobe *p, struct pt_regs *regs) 514 { 515 switch (p->ainsn.opcode) { 516 case 0xfa: /* cli */ 517 regs->flags &= ~(X86_EFLAGS_IF); 518 break; 519 case 0xfb: /* sti */ 520 regs->flags |= X86_EFLAGS_IF; 521 break; 522 case 0x9c: /* pushf */ 523 int3_emulate_push(regs, regs->flags); 524 break; 525 case 0x9d: /* popf */ 526 regs->flags = int3_emulate_pop(regs); 527 break; 528 } 529 regs->ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 530 } 531 NOKPROBE_SYMBOL(kprobe_emulate_ifmodifiers); 532 533 static void kprobe_emulate_ret(struct kprobe *p, struct pt_regs *regs) 534 { 535 int3_emulate_ret(regs); 536 } 537 NOKPROBE_SYMBOL(kprobe_emulate_ret); 538 539 static void kprobe_emulate_call(struct kprobe *p, struct pt_regs *regs) 540 { 541 unsigned long func = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 542 543 func += p->ainsn.rel32; 544 int3_emulate_call(regs, func); 545 } 546 NOKPROBE_SYMBOL(kprobe_emulate_call); 547 548 static void kprobe_emulate_jmp(struct kprobe *p, struct pt_regs *regs) 549 { 550 unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 551 552 ip += p->ainsn.rel32; 553 int3_emulate_jmp(regs, ip); 554 } 555 NOKPROBE_SYMBOL(kprobe_emulate_jmp); 556 557 static void kprobe_emulate_jcc(struct kprobe *p, struct pt_regs *regs) 558 { 559 unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 560 561 int3_emulate_jcc(regs, p->ainsn.jcc.type, ip, p->ainsn.rel32); 562 } 563 NOKPROBE_SYMBOL(kprobe_emulate_jcc); 564 565 static void kprobe_emulate_loop(struct kprobe *p, struct pt_regs *regs) 566 { 567 unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 568 bool match; 569 570 if (p->ainsn.loop.type != 3) { /* LOOP* */ 571 if (p->ainsn.loop.asize == 32) 572 match = ((*(u32 *)®s->cx)--) != 0; 573 #ifdef CONFIG_X86_64 574 else if (p->ainsn.loop.asize == 64) 575 match = ((*(u64 *)®s->cx)--) != 0; 576 #endif 577 else 578 match = ((*(u16 *)®s->cx)--) != 0; 579 } else { /* JCXZ */ 580 if (p->ainsn.loop.asize == 32) 581 match = *(u32 *)(®s->cx) == 0; 582 #ifdef CONFIG_X86_64 583 else if (p->ainsn.loop.asize == 64) 584 match = *(u64 *)(®s->cx) == 0; 585 #endif 586 else 587 match = *(u16 *)(®s->cx) == 0; 588 } 589 590 if (p->ainsn.loop.type == 0) /* LOOPNE */ 591 match = match && !(regs->flags & X86_EFLAGS_ZF); 592 else if (p->ainsn.loop.type == 1) /* LOOPE */ 593 match = match && (regs->flags & X86_EFLAGS_ZF); 594 595 if (match) 596 ip += p->ainsn.rel32; 597 int3_emulate_jmp(regs, ip); 598 } 599 NOKPROBE_SYMBOL(kprobe_emulate_loop); 600 601 static const int addrmode_regoffs[] = { 602 offsetof(struct pt_regs, ax), 603 offsetof(struct pt_regs, cx), 604 offsetof(struct pt_regs, dx), 605 offsetof(struct pt_regs, bx), 606 offsetof(struct pt_regs, sp), 607 offsetof(struct pt_regs, bp), 608 offsetof(struct pt_regs, si), 609 offsetof(struct pt_regs, di), 610 #ifdef CONFIG_X86_64 611 offsetof(struct pt_regs, r8), 612 offsetof(struct pt_regs, r9), 613 offsetof(struct pt_regs, r10), 614 offsetof(struct pt_regs, r11), 615 offsetof(struct pt_regs, r12), 616 offsetof(struct pt_regs, r13), 617 offsetof(struct pt_regs, r14), 618 offsetof(struct pt_regs, r15), 619 #endif 620 }; 621 622 static void kprobe_emulate_call_indirect(struct kprobe *p, struct pt_regs *regs) 623 { 624 unsigned long offs = addrmode_regoffs[p->ainsn.indirect.reg]; 625 626 int3_emulate_push(regs, regs->ip - INT3_INSN_SIZE + p->ainsn.size); 627 int3_emulate_jmp(regs, regs_get_register(regs, offs)); 628 } 629 NOKPROBE_SYMBOL(kprobe_emulate_call_indirect); 630 631 static void kprobe_emulate_jmp_indirect(struct kprobe *p, struct pt_regs *regs) 632 { 633 unsigned long offs = addrmode_regoffs[p->ainsn.indirect.reg]; 634 635 int3_emulate_jmp(regs, regs_get_register(regs, offs)); 636 } 637 NOKPROBE_SYMBOL(kprobe_emulate_jmp_indirect); 638 639 static int prepare_emulation(struct kprobe *p, struct insn *insn) 640 { 641 insn_byte_t opcode = insn->opcode.bytes[0]; 642 643 switch (opcode) { 644 case 0xfa: /* cli */ 645 case 0xfb: /* sti */ 646 case 0x9c: /* pushfl */ 647 case 0x9d: /* popf/popfd */ 648 /* 649 * IF modifiers must be emulated since it will enable interrupt while 650 * int3 single stepping. 651 */ 652 p->ainsn.emulate_op = kprobe_emulate_ifmodifiers; 653 p->ainsn.opcode = opcode; 654 break; 655 case 0xc2: /* ret/lret */ 656 case 0xc3: 657 case 0xca: 658 case 0xcb: 659 p->ainsn.emulate_op = kprobe_emulate_ret; 660 break; 661 case 0x9a: /* far call absolute -- segment is not supported */ 662 case 0xea: /* far jmp absolute -- segment is not supported */ 663 case 0xcc: /* int3 */ 664 case 0xcf: /* iret -- in-kernel IRET is not supported */ 665 return -EOPNOTSUPP; 666 break; 667 case 0xe8: /* near call relative */ 668 p->ainsn.emulate_op = kprobe_emulate_call; 669 if (insn->immediate.nbytes == 2) 670 p->ainsn.rel32 = *(s16 *)&insn->immediate.value; 671 else 672 p->ainsn.rel32 = *(s32 *)&insn->immediate.value; 673 break; 674 case 0xeb: /* short jump relative */ 675 case 0xe9: /* near jump relative */ 676 p->ainsn.emulate_op = kprobe_emulate_jmp; 677 if (insn->immediate.nbytes == 1) 678 p->ainsn.rel32 = *(s8 *)&insn->immediate.value; 679 else if (insn->immediate.nbytes == 2) 680 p->ainsn.rel32 = *(s16 *)&insn->immediate.value; 681 else 682 p->ainsn.rel32 = *(s32 *)&insn->immediate.value; 683 break; 684 case 0x70 ... 0x7f: 685 /* 1 byte conditional jump */ 686 p->ainsn.emulate_op = kprobe_emulate_jcc; 687 p->ainsn.jcc.type = opcode & 0xf; 688 p->ainsn.rel32 = insn->immediate.value; 689 break; 690 case 0x0f: 691 opcode = insn->opcode.bytes[1]; 692 if ((opcode & 0xf0) == 0x80) { 693 /* 2 bytes Conditional Jump */ 694 p->ainsn.emulate_op = kprobe_emulate_jcc; 695 p->ainsn.jcc.type = opcode & 0xf; 696 if (insn->immediate.nbytes == 2) 697 p->ainsn.rel32 = *(s16 *)&insn->immediate.value; 698 else 699 p->ainsn.rel32 = *(s32 *)&insn->immediate.value; 700 } else if (opcode == 0x01 && 701 X86_MODRM_REG(insn->modrm.bytes[0]) == 0 && 702 X86_MODRM_MOD(insn->modrm.bytes[0]) == 3) { 703 /* VM extensions - not supported */ 704 return -EOPNOTSUPP; 705 } 706 break; 707 case 0xe0: /* Loop NZ */ 708 case 0xe1: /* Loop */ 709 case 0xe2: /* Loop */ 710 case 0xe3: /* J*CXZ */ 711 p->ainsn.emulate_op = kprobe_emulate_loop; 712 p->ainsn.loop.type = opcode & 0x3; 713 p->ainsn.loop.asize = insn->addr_bytes * 8; 714 p->ainsn.rel32 = *(s8 *)&insn->immediate.value; 715 break; 716 case 0xff: 717 /* 718 * Since the 0xff is an extended group opcode, the instruction 719 * is determined by the MOD/RM byte. 720 */ 721 opcode = insn->modrm.bytes[0]; 722 switch (X86_MODRM_REG(opcode)) { 723 case 0b010: /* FF /2, call near, absolute indirect */ 724 p->ainsn.emulate_op = kprobe_emulate_call_indirect; 725 break; 726 case 0b100: /* FF /4, jmp near, absolute indirect */ 727 p->ainsn.emulate_op = kprobe_emulate_jmp_indirect; 728 break; 729 case 0b011: /* FF /3, call far, absolute indirect */ 730 case 0b101: /* FF /5, jmp far, absolute indirect */ 731 return -EOPNOTSUPP; 732 } 733 734 if (!p->ainsn.emulate_op) 735 break; 736 737 if (insn->addr_bytes != sizeof(unsigned long)) 738 return -EOPNOTSUPP; /* Don't support different size */ 739 if (X86_MODRM_MOD(opcode) != 3) 740 return -EOPNOTSUPP; /* TODO: support memory addressing */ 741 742 p->ainsn.indirect.reg = X86_MODRM_RM(opcode); 743 #ifdef CONFIG_X86_64 744 if (X86_REX_B(insn->rex_prefix.value)) 745 p->ainsn.indirect.reg += 8; 746 #endif 747 break; 748 default: 749 break; 750 } 751 p->ainsn.size = insn->length; 752 753 return 0; 754 } 755 756 static int arch_copy_kprobe(struct kprobe *p) 757 { 758 struct insn insn; 759 kprobe_opcode_t buf[MAX_INSN_SIZE]; 760 int ret, len; 761 762 /* Copy an instruction with recovering if other optprobe modifies it.*/ 763 len = __copy_instruction(buf, p->addr, p->ainsn.insn, &insn); 764 if (!len) 765 return -EINVAL; 766 767 /* Analyze the opcode and setup emulate functions */ 768 ret = prepare_emulation(p, &insn); 769 if (ret < 0) 770 return ret; 771 772 /* Add int3 for single-step or booster jmp */ 773 len = prepare_singlestep(buf, p, &insn); 774 if (len < 0) 775 return len; 776 777 /* Also, displacement change doesn't affect the first byte */ 778 p->opcode = buf[0]; 779 780 p->ainsn.tp_len = len; 781 perf_event_text_poke(p->ainsn.insn, NULL, 0, buf, len); 782 783 /* OK, write back the instruction(s) into ROX insn buffer */ 784 text_poke(p->ainsn.insn, buf, len); 785 786 return 0; 787 } 788 789 int arch_prepare_kprobe(struct kprobe *p) 790 { 791 int ret; 792 793 if (alternatives_text_reserved(p->addr, p->addr)) 794 return -EINVAL; 795 796 if (!can_probe((unsigned long)p->addr)) 797 return -EILSEQ; 798 799 memset(&p->ainsn, 0, sizeof(p->ainsn)); 800 801 /* insn: must be on special executable page on x86. */ 802 p->ainsn.insn = get_insn_slot(); 803 if (!p->ainsn.insn) 804 return -ENOMEM; 805 806 ret = arch_copy_kprobe(p); 807 if (ret) { 808 free_insn_slot(p->ainsn.insn, 0); 809 p->ainsn.insn = NULL; 810 } 811 812 return ret; 813 } 814 815 void arch_arm_kprobe(struct kprobe *p) 816 { 817 u8 int3 = INT3_INSN_OPCODE; 818 819 text_poke(p->addr, &int3, 1); 820 text_poke_sync(); 821 perf_event_text_poke(p->addr, &p->opcode, 1, &int3, 1); 822 } 823 824 void arch_disarm_kprobe(struct kprobe *p) 825 { 826 u8 int3 = INT3_INSN_OPCODE; 827 828 perf_event_text_poke(p->addr, &int3, 1, &p->opcode, 1); 829 text_poke(p->addr, &p->opcode, 1); 830 text_poke_sync(); 831 } 832 833 void arch_remove_kprobe(struct kprobe *p) 834 { 835 if (p->ainsn.insn) { 836 /* Record the perf event before freeing the slot */ 837 perf_event_text_poke(p->ainsn.insn, p->ainsn.insn, 838 p->ainsn.tp_len, NULL, 0); 839 free_insn_slot(p->ainsn.insn, p->ainsn.boostable); 840 p->ainsn.insn = NULL; 841 } 842 } 843 844 static nokprobe_inline void 845 save_previous_kprobe(struct kprobe_ctlblk *kcb) 846 { 847 kcb->prev_kprobe.kp = kprobe_running(); 848 kcb->prev_kprobe.status = kcb->kprobe_status; 849 kcb->prev_kprobe.old_flags = kcb->kprobe_old_flags; 850 kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags; 851 } 852 853 static nokprobe_inline void 854 restore_previous_kprobe(struct kprobe_ctlblk *kcb) 855 { 856 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); 857 kcb->kprobe_status = kcb->prev_kprobe.status; 858 kcb->kprobe_old_flags = kcb->prev_kprobe.old_flags; 859 kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags; 860 } 861 862 static nokprobe_inline void 863 set_current_kprobe(struct kprobe *p, struct pt_regs *regs, 864 struct kprobe_ctlblk *kcb) 865 { 866 __this_cpu_write(current_kprobe, p); 867 kcb->kprobe_saved_flags = kcb->kprobe_old_flags 868 = (regs->flags & X86_EFLAGS_IF); 869 } 870 871 static void kprobe_post_process(struct kprobe *cur, struct pt_regs *regs, 872 struct kprobe_ctlblk *kcb) 873 { 874 /* Restore back the original saved kprobes variables and continue. */ 875 if (kcb->kprobe_status == KPROBE_REENTER) { 876 /* This will restore both kcb and current_kprobe */ 877 restore_previous_kprobe(kcb); 878 } else { 879 /* 880 * Always update the kcb status because 881 * reset_curent_kprobe() doesn't update kcb. 882 */ 883 kcb->kprobe_status = KPROBE_HIT_SSDONE; 884 if (cur->post_handler) 885 cur->post_handler(cur, regs, 0); 886 reset_current_kprobe(); 887 } 888 } 889 NOKPROBE_SYMBOL(kprobe_post_process); 890 891 static void setup_singlestep(struct kprobe *p, struct pt_regs *regs, 892 struct kprobe_ctlblk *kcb, int reenter) 893 { 894 if (setup_detour_execution(p, regs, reenter)) 895 return; 896 897 #if !defined(CONFIG_PREEMPTION) 898 if (p->ainsn.boostable) { 899 /* Boost up -- we can execute copied instructions directly */ 900 if (!reenter) 901 reset_current_kprobe(); 902 /* 903 * Reentering boosted probe doesn't reset current_kprobe, 904 * nor set current_kprobe, because it doesn't use single 905 * stepping. 906 */ 907 regs->ip = (unsigned long)p->ainsn.insn; 908 return; 909 } 910 #endif 911 if (reenter) { 912 save_previous_kprobe(kcb); 913 set_current_kprobe(p, regs, kcb); 914 kcb->kprobe_status = KPROBE_REENTER; 915 } else 916 kcb->kprobe_status = KPROBE_HIT_SS; 917 918 if (p->ainsn.emulate_op) { 919 p->ainsn.emulate_op(p, regs); 920 kprobe_post_process(p, regs, kcb); 921 return; 922 } 923 924 /* Disable interrupt, and set ip register on trampoline */ 925 regs->flags &= ~X86_EFLAGS_IF; 926 regs->ip = (unsigned long)p->ainsn.insn; 927 } 928 NOKPROBE_SYMBOL(setup_singlestep); 929 930 /* 931 * Called after single-stepping. p->addr is the address of the 932 * instruction whose first byte has been replaced by the "int3" 933 * instruction. To avoid the SMP problems that can occur when we 934 * temporarily put back the original opcode to single-step, we 935 * single-stepped a copy of the instruction. The address of this 936 * copy is p->ainsn.insn. We also doesn't use trap, but "int3" again 937 * right after the copied instruction. 938 * Different from the trap single-step, "int3" single-step can not 939 * handle the instruction which changes the ip register, e.g. jmp, 940 * call, conditional jmp, and the instructions which changes the IF 941 * flags because interrupt must be disabled around the single-stepping. 942 * Such instructions are software emulated, but others are single-stepped 943 * using "int3". 944 * 945 * When the 2nd "int3" handled, the regs->ip and regs->flags needs to 946 * be adjusted, so that we can resume execution on correct code. 947 */ 948 static void resume_singlestep(struct kprobe *p, struct pt_regs *regs, 949 struct kprobe_ctlblk *kcb) 950 { 951 unsigned long copy_ip = (unsigned long)p->ainsn.insn; 952 unsigned long orig_ip = (unsigned long)p->addr; 953 954 /* Restore saved interrupt flag and ip register */ 955 regs->flags |= kcb->kprobe_saved_flags; 956 /* Note that regs->ip is executed int3 so must be a step back */ 957 regs->ip += (orig_ip - copy_ip) - INT3_INSN_SIZE; 958 } 959 NOKPROBE_SYMBOL(resume_singlestep); 960 961 /* 962 * We have reentered the kprobe_handler(), since another probe was hit while 963 * within the handler. We save the original kprobes variables and just single 964 * step on the instruction of the new probe without calling any user handlers. 965 */ 966 static int reenter_kprobe(struct kprobe *p, struct pt_regs *regs, 967 struct kprobe_ctlblk *kcb) 968 { 969 switch (kcb->kprobe_status) { 970 case KPROBE_HIT_SSDONE: 971 case KPROBE_HIT_ACTIVE: 972 case KPROBE_HIT_SS: 973 kprobes_inc_nmissed_count(p); 974 setup_singlestep(p, regs, kcb, 1); 975 break; 976 case KPROBE_REENTER: 977 /* A probe has been hit in the codepath leading up to, or just 978 * after, single-stepping of a probed instruction. This entire 979 * codepath should strictly reside in .kprobes.text section. 980 * Raise a BUG or we'll continue in an endless reentering loop 981 * and eventually a stack overflow. 982 */ 983 pr_err("Unrecoverable kprobe detected.\n"); 984 dump_kprobe(p); 985 BUG(); 986 default: 987 /* impossible cases */ 988 WARN_ON(1); 989 return 0; 990 } 991 992 return 1; 993 } 994 NOKPROBE_SYMBOL(reenter_kprobe); 995 996 static nokprobe_inline int kprobe_is_ss(struct kprobe_ctlblk *kcb) 997 { 998 return (kcb->kprobe_status == KPROBE_HIT_SS || 999 kcb->kprobe_status == KPROBE_REENTER); 1000 } 1001 1002 /* 1003 * Interrupts are disabled on entry as trap3 is an interrupt gate and they 1004 * remain disabled throughout this function. 1005 */ 1006 int kprobe_int3_handler(struct pt_regs *regs) 1007 { 1008 kprobe_opcode_t *addr; 1009 struct kprobe *p; 1010 struct kprobe_ctlblk *kcb; 1011 1012 if (user_mode(regs)) 1013 return 0; 1014 1015 addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t)); 1016 /* 1017 * We don't want to be preempted for the entire duration of kprobe 1018 * processing. Since int3 and debug trap disables irqs and we clear 1019 * IF while singlestepping, it must be no preemptible. 1020 */ 1021 1022 kcb = get_kprobe_ctlblk(); 1023 p = get_kprobe(addr); 1024 1025 if (p) { 1026 if (kprobe_running()) { 1027 if (reenter_kprobe(p, regs, kcb)) 1028 return 1; 1029 } else { 1030 set_current_kprobe(p, regs, kcb); 1031 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 1032 1033 /* 1034 * If we have no pre-handler or it returned 0, we 1035 * continue with normal processing. If we have a 1036 * pre-handler and it returned non-zero, that means 1037 * user handler setup registers to exit to another 1038 * instruction, we must skip the single stepping. 1039 */ 1040 if (!p->pre_handler || !p->pre_handler(p, regs)) 1041 setup_singlestep(p, regs, kcb, 0); 1042 else 1043 reset_current_kprobe(); 1044 return 1; 1045 } 1046 } else if (kprobe_is_ss(kcb)) { 1047 p = kprobe_running(); 1048 if ((unsigned long)p->ainsn.insn < regs->ip && 1049 (unsigned long)p->ainsn.insn + MAX_INSN_SIZE > regs->ip) { 1050 /* Most provably this is the second int3 for singlestep */ 1051 resume_singlestep(p, regs, kcb); 1052 kprobe_post_process(p, regs, kcb); 1053 return 1; 1054 } 1055 } /* else: not a kprobe fault; let the kernel handle it */ 1056 1057 return 0; 1058 } 1059 NOKPROBE_SYMBOL(kprobe_int3_handler); 1060 1061 int kprobe_fault_handler(struct pt_regs *regs, int trapnr) 1062 { 1063 struct kprobe *cur = kprobe_running(); 1064 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 1065 1066 if (unlikely(regs->ip == (unsigned long)cur->ainsn.insn)) { 1067 /* This must happen on single-stepping */ 1068 WARN_ON(kcb->kprobe_status != KPROBE_HIT_SS && 1069 kcb->kprobe_status != KPROBE_REENTER); 1070 /* 1071 * We are here because the instruction being single 1072 * stepped caused a page fault. We reset the current 1073 * kprobe and the ip points back to the probe address 1074 * and allow the page fault handler to continue as a 1075 * normal page fault. 1076 */ 1077 regs->ip = (unsigned long)cur->addr; 1078 1079 /* 1080 * If the IF flag was set before the kprobe hit, 1081 * don't touch it: 1082 */ 1083 regs->flags |= kcb->kprobe_old_flags; 1084 1085 if (kcb->kprobe_status == KPROBE_REENTER) 1086 restore_previous_kprobe(kcb); 1087 else 1088 reset_current_kprobe(); 1089 } 1090 1091 return 0; 1092 } 1093 NOKPROBE_SYMBOL(kprobe_fault_handler); 1094 1095 int __init arch_populate_kprobe_blacklist(void) 1096 { 1097 return kprobe_add_area_blacklist((unsigned long)__entry_text_start, 1098 (unsigned long)__entry_text_end); 1099 } 1100 1101 int __init arch_init_kprobes(void) 1102 { 1103 return 0; 1104 } 1105 1106 int arch_trampoline_kprobe(struct kprobe *p) 1107 { 1108 return 0; 1109 } 1110