1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Code for replacing ftrace calls with jumps. 4 * 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 6 * 7 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box. 8 * 9 * Added function graph tracer code, taken from x86 that was written 10 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt. 11 * 12 */ 13 14 #define pr_fmt(fmt) "ftrace-powerpc: " fmt 15 16 #include <linux/spinlock.h> 17 #include <linux/hardirq.h> 18 #include <linux/uaccess.h> 19 #include <linux/module.h> 20 #include <linux/ftrace.h> 21 #include <linux/percpu.h> 22 #include <linux/init.h> 23 #include <linux/list.h> 24 25 #include <asm/asm-prototypes.h> 26 #include <asm/cacheflush.h> 27 #include <asm/code-patching.h> 28 #include <asm/ftrace.h> 29 #include <asm/syscall.h> 30 31 32 #ifdef CONFIG_DYNAMIC_FTRACE 33 static unsigned int 34 ftrace_call_replace(unsigned long ip, unsigned long addr, int link) 35 { 36 unsigned int op; 37 38 addr = ppc_function_entry((void *)addr); 39 40 /* if (link) set op to 'bl' else 'b' */ 41 op = create_branch((unsigned int *)ip, addr, link ? 1 : 0); 42 43 return op; 44 } 45 46 static int 47 ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new) 48 { 49 unsigned int replaced; 50 51 /* 52 * Note: 53 * We are paranoid about modifying text, as if a bug was to happen, it 54 * could cause us to read or write to someplace that could cause harm. 55 * Carefully read and modify the code with probe_kernel_*(), and make 56 * sure what we read is what we expected it to be before modifying it. 57 */ 58 59 /* read the text we want to modify */ 60 if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE)) 61 return -EFAULT; 62 63 /* Make sure it is what we expect it to be */ 64 if (replaced != old) { 65 pr_err("%p: replaced (%#x) != old (%#x)", 66 (void *)ip, replaced, old); 67 return -EINVAL; 68 } 69 70 /* replace the text with the new text */ 71 if (patch_instruction((unsigned int *)ip, new)) 72 return -EPERM; 73 74 return 0; 75 } 76 77 /* 78 * Helper functions that are the same for both PPC64 and PPC32. 79 */ 80 static int test_24bit_addr(unsigned long ip, unsigned long addr) 81 { 82 addr = ppc_function_entry((void *)addr); 83 84 /* use the create_branch to verify that this offset can be branched */ 85 return create_branch((unsigned int *)ip, addr, 0); 86 } 87 88 #ifdef CONFIG_MODULES 89 90 static int is_bl_op(unsigned int op) 91 { 92 return (op & 0xfc000003) == 0x48000001; 93 } 94 95 static unsigned long find_bl_target(unsigned long ip, unsigned int op) 96 { 97 static int offset; 98 99 offset = (op & 0x03fffffc); 100 /* make it signed */ 101 if (offset & 0x02000000) 102 offset |= 0xfe000000; 103 104 return ip + (long)offset; 105 } 106 107 #ifdef CONFIG_PPC64 108 static int 109 __ftrace_make_nop(struct module *mod, 110 struct dyn_ftrace *rec, unsigned long addr) 111 { 112 unsigned long entry, ptr, tramp; 113 unsigned long ip = rec->ip; 114 unsigned int op, pop; 115 116 /* read where this goes */ 117 if (probe_kernel_read(&op, (void *)ip, sizeof(int))) { 118 pr_err("Fetching opcode failed.\n"); 119 return -EFAULT; 120 } 121 122 /* Make sure that that this is still a 24bit jump */ 123 if (!is_bl_op(op)) { 124 pr_err("Not expected bl: opcode is %x\n", op); 125 return -EINVAL; 126 } 127 128 /* lets find where the pointer goes */ 129 tramp = find_bl_target(ip, op); 130 131 pr_devel("ip:%lx jumps to %lx", ip, tramp); 132 133 if (module_trampoline_target(mod, tramp, &ptr)) { 134 pr_err("Failed to get trampoline target\n"); 135 return -EFAULT; 136 } 137 138 pr_devel("trampoline target %lx", ptr); 139 140 entry = ppc_global_function_entry((void *)addr); 141 /* This should match what was called */ 142 if (ptr != entry) { 143 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 144 return -EINVAL; 145 } 146 147 #ifdef CONFIG_MPROFILE_KERNEL 148 /* When using -mkernel_profile there is no load to jump over */ 149 pop = PPC_INST_NOP; 150 151 if (probe_kernel_read(&op, (void *)(ip - 4), 4)) { 152 pr_err("Fetching instruction at %lx failed.\n", ip - 4); 153 return -EFAULT; 154 } 155 156 /* We expect either a mflr r0, or a std r0, LRSAVE(r1) */ 157 if (op != PPC_INST_MFLR && op != PPC_INST_STD_LR) { 158 pr_err("Unexpected instruction %08x around bl _mcount\n", op); 159 return -EINVAL; 160 } 161 #else 162 /* 163 * Our original call site looks like: 164 * 165 * bl <tramp> 166 * ld r2,XX(r1) 167 * 168 * Milton Miller pointed out that we can not simply nop the branch. 169 * If a task was preempted when calling a trace function, the nops 170 * will remove the way to restore the TOC in r2 and the r2 TOC will 171 * get corrupted. 172 * 173 * Use a b +8 to jump over the load. 174 */ 175 176 pop = PPC_INST_BRANCH | 8; /* b +8 */ 177 178 /* 179 * Check what is in the next instruction. We can see ld r2,40(r1), but 180 * on first pass after boot we will see mflr r0. 181 */ 182 if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE)) { 183 pr_err("Fetching op failed.\n"); 184 return -EFAULT; 185 } 186 187 if (op != PPC_INST_LD_TOC) { 188 pr_err("Expected %08x found %08x\n", PPC_INST_LD_TOC, op); 189 return -EINVAL; 190 } 191 #endif /* CONFIG_MPROFILE_KERNEL */ 192 193 if (patch_instruction((unsigned int *)ip, pop)) { 194 pr_err("Patching NOP failed.\n"); 195 return -EPERM; 196 } 197 198 return 0; 199 } 200 201 #else /* !PPC64 */ 202 static int 203 __ftrace_make_nop(struct module *mod, 204 struct dyn_ftrace *rec, unsigned long addr) 205 { 206 unsigned int op; 207 unsigned int jmp[4]; 208 unsigned long ip = rec->ip; 209 unsigned long tramp; 210 211 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE)) 212 return -EFAULT; 213 214 /* Make sure that that this is still a 24bit jump */ 215 if (!is_bl_op(op)) { 216 pr_err("Not expected bl: opcode is %x\n", op); 217 return -EINVAL; 218 } 219 220 /* lets find where the pointer goes */ 221 tramp = find_bl_target(ip, op); 222 223 /* 224 * On PPC32 the trampoline looks like: 225 * 0x3d, 0x80, 0x00, 0x00 lis r12,sym@ha 226 * 0x39, 0x8c, 0x00, 0x00 addi r12,r12,sym@l 227 * 0x7d, 0x89, 0x03, 0xa6 mtctr r12 228 * 0x4e, 0x80, 0x04, 0x20 bctr 229 */ 230 231 pr_devel("ip:%lx jumps to %lx", ip, tramp); 232 233 /* Find where the trampoline jumps to */ 234 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) { 235 pr_err("Failed to read %lx\n", tramp); 236 return -EFAULT; 237 } 238 239 pr_devel(" %08x %08x ", jmp[0], jmp[1]); 240 241 /* verify that this is what we expect it to be */ 242 if (((jmp[0] & 0xffff0000) != 0x3d800000) || 243 ((jmp[1] & 0xffff0000) != 0x398c0000) || 244 (jmp[2] != 0x7d8903a6) || 245 (jmp[3] != 0x4e800420)) { 246 pr_err("Not a trampoline\n"); 247 return -EINVAL; 248 } 249 250 tramp = (jmp[1] & 0xffff) | 251 ((jmp[0] & 0xffff) << 16); 252 if (tramp & 0x8000) 253 tramp -= 0x10000; 254 255 pr_devel(" %lx ", tramp); 256 257 if (tramp != addr) { 258 pr_err("Trampoline location %08lx does not match addr\n", 259 tramp); 260 return -EINVAL; 261 } 262 263 op = PPC_INST_NOP; 264 265 if (patch_instruction((unsigned int *)ip, op)) 266 return -EPERM; 267 268 return 0; 269 } 270 #endif /* PPC64 */ 271 #endif /* CONFIG_MODULES */ 272 273 int ftrace_make_nop(struct module *mod, 274 struct dyn_ftrace *rec, unsigned long addr) 275 { 276 unsigned long ip = rec->ip; 277 unsigned int old, new; 278 279 /* 280 * If the calling address is more that 24 bits away, 281 * then we had to use a trampoline to make the call. 282 * Otherwise just update the call site. 283 */ 284 if (test_24bit_addr(ip, addr)) { 285 /* within range */ 286 old = ftrace_call_replace(ip, addr, 1); 287 new = PPC_INST_NOP; 288 return ftrace_modify_code(ip, old, new); 289 } 290 291 #ifdef CONFIG_MODULES 292 /* 293 * Out of range jumps are called from modules. 294 * We should either already have a pointer to the module 295 * or it has been passed in. 296 */ 297 if (!rec->arch.mod) { 298 if (!mod) { 299 pr_err("No module loaded addr=%lx\n", addr); 300 return -EFAULT; 301 } 302 rec->arch.mod = mod; 303 } else if (mod) { 304 if (mod != rec->arch.mod) { 305 pr_err("Record mod %p not equal to passed in mod %p\n", 306 rec->arch.mod, mod); 307 return -EINVAL; 308 } 309 /* nothing to do if mod == rec->arch.mod */ 310 } else 311 mod = rec->arch.mod; 312 313 return __ftrace_make_nop(mod, rec, addr); 314 #else 315 /* We should not get here without modules */ 316 return -EINVAL; 317 #endif /* CONFIG_MODULES */ 318 } 319 320 #ifdef CONFIG_MODULES 321 #ifdef CONFIG_PPC64 322 /* 323 * Examine the existing instructions for __ftrace_make_call. 324 * They should effectively be a NOP, and follow formal constraints, 325 * depending on the ABI. Return false if they don't. 326 */ 327 #ifndef CONFIG_MPROFILE_KERNEL 328 static int 329 expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1) 330 { 331 /* 332 * We expect to see: 333 * 334 * b +8 335 * ld r2,XX(r1) 336 * 337 * The load offset is different depending on the ABI. For simplicity 338 * just mask it out when doing the compare. 339 */ 340 if ((op0 != 0x48000008) || ((op1 & 0xffff0000) != 0xe8410000)) 341 return 0; 342 return 1; 343 } 344 #else 345 static int 346 expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1) 347 { 348 /* look for patched "NOP" on ppc64 with -mprofile-kernel */ 349 if (op0 != PPC_INST_NOP) 350 return 0; 351 return 1; 352 } 353 #endif 354 355 static int 356 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 357 { 358 unsigned int op[2]; 359 void *ip = (void *)rec->ip; 360 unsigned long entry, ptr, tramp; 361 struct module *mod = rec->arch.mod; 362 363 /* read where this goes */ 364 if (probe_kernel_read(op, ip, sizeof(op))) 365 return -EFAULT; 366 367 if (!expected_nop_sequence(ip, op[0], op[1])) { 368 pr_err("Unexpected call sequence at %p: %x %x\n", 369 ip, op[0], op[1]); 370 return -EINVAL; 371 } 372 373 /* If we never set up ftrace trampoline(s), then bail */ 374 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 375 if (!mod->arch.tramp || !mod->arch.tramp_regs) { 376 #else 377 if (!mod->arch.tramp) { 378 #endif 379 pr_err("No ftrace trampoline\n"); 380 return -EINVAL; 381 } 382 383 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 384 if (rec->flags & FTRACE_FL_REGS) 385 tramp = mod->arch.tramp_regs; 386 else 387 #endif 388 tramp = mod->arch.tramp; 389 390 if (module_trampoline_target(mod, tramp, &ptr)) { 391 pr_err("Failed to get trampoline target\n"); 392 return -EFAULT; 393 } 394 395 pr_devel("trampoline target %lx", ptr); 396 397 entry = ppc_global_function_entry((void *)addr); 398 /* This should match what was called */ 399 if (ptr != entry) { 400 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 401 return -EINVAL; 402 } 403 404 /* Ensure branch is within 24 bits */ 405 if (!create_branch(ip, tramp, BRANCH_SET_LINK)) { 406 pr_err("Branch out of range\n"); 407 return -EINVAL; 408 } 409 410 if (patch_branch(ip, tramp, BRANCH_SET_LINK)) { 411 pr_err("REL24 out of range!\n"); 412 return -EINVAL; 413 } 414 415 return 0; 416 } 417 418 #else /* !CONFIG_PPC64: */ 419 static int 420 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 421 { 422 unsigned int op; 423 unsigned long ip = rec->ip; 424 425 /* read where this goes */ 426 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE)) 427 return -EFAULT; 428 429 /* It should be pointing to a nop */ 430 if (op != PPC_INST_NOP) { 431 pr_err("Expected NOP but have %x\n", op); 432 return -EINVAL; 433 } 434 435 /* If we never set up a trampoline to ftrace_caller, then bail */ 436 if (!rec->arch.mod->arch.tramp) { 437 pr_err("No ftrace trampoline\n"); 438 return -EINVAL; 439 } 440 441 /* create the branch to the trampoline */ 442 op = create_branch((unsigned int *)ip, 443 rec->arch.mod->arch.tramp, BRANCH_SET_LINK); 444 if (!op) { 445 pr_err("REL24 out of range!\n"); 446 return -EINVAL; 447 } 448 449 pr_devel("write to %lx\n", rec->ip); 450 451 if (patch_instruction((unsigned int *)ip, op)) 452 return -EPERM; 453 454 return 0; 455 } 456 #endif /* CONFIG_PPC64 */ 457 #endif /* CONFIG_MODULES */ 458 459 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 460 { 461 unsigned long ip = rec->ip; 462 unsigned int old, new; 463 464 /* 465 * If the calling address is more that 24 bits away, 466 * then we had to use a trampoline to make the call. 467 * Otherwise just update the call site. 468 */ 469 if (test_24bit_addr(ip, addr)) { 470 /* within range */ 471 old = PPC_INST_NOP; 472 new = ftrace_call_replace(ip, addr, 1); 473 return ftrace_modify_code(ip, old, new); 474 } 475 476 #ifdef CONFIG_MODULES 477 /* 478 * Out of range jumps are called from modules. 479 * Being that we are converting from nop, it had better 480 * already have a module defined. 481 */ 482 if (!rec->arch.mod) { 483 pr_err("No module loaded\n"); 484 return -EINVAL; 485 } 486 487 return __ftrace_make_call(rec, addr); 488 #else 489 /* We should not get here without modules */ 490 return -EINVAL; 491 #endif /* CONFIG_MODULES */ 492 } 493 494 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 495 #ifdef CONFIG_MODULES 496 static int 497 __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 498 unsigned long addr) 499 { 500 unsigned int op; 501 unsigned long ip = rec->ip; 502 unsigned long entry, ptr, tramp; 503 struct module *mod = rec->arch.mod; 504 505 /* If we never set up ftrace trampolines, then bail */ 506 if (!mod->arch.tramp || !mod->arch.tramp_regs) { 507 pr_err("No ftrace trampoline\n"); 508 return -EINVAL; 509 } 510 511 /* read where this goes */ 512 if (probe_kernel_read(&op, (void *)ip, sizeof(int))) { 513 pr_err("Fetching opcode failed.\n"); 514 return -EFAULT; 515 } 516 517 /* Make sure that that this is still a 24bit jump */ 518 if (!is_bl_op(op)) { 519 pr_err("Not expected bl: opcode is %x\n", op); 520 return -EINVAL; 521 } 522 523 /* lets find where the pointer goes */ 524 tramp = find_bl_target(ip, op); 525 entry = ppc_global_function_entry((void *)old_addr); 526 527 pr_devel("ip:%lx jumps to %lx", ip, tramp); 528 529 if (tramp != entry) { 530 /* old_addr is not within range, so we must have used a trampoline */ 531 if (module_trampoline_target(mod, tramp, &ptr)) { 532 pr_err("Failed to get trampoline target\n"); 533 return -EFAULT; 534 } 535 536 pr_devel("trampoline target %lx", ptr); 537 538 /* This should match what was called */ 539 if (ptr != entry) { 540 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 541 return -EINVAL; 542 } 543 } 544 545 /* The new target may be within range */ 546 if (test_24bit_addr(ip, addr)) { 547 /* within range */ 548 if (patch_branch((unsigned int *)ip, addr, BRANCH_SET_LINK)) { 549 pr_err("REL24 out of range!\n"); 550 return -EINVAL; 551 } 552 553 return 0; 554 } 555 556 if (rec->flags & FTRACE_FL_REGS) 557 tramp = mod->arch.tramp_regs; 558 else 559 tramp = mod->arch.tramp; 560 561 if (module_trampoline_target(mod, tramp, &ptr)) { 562 pr_err("Failed to get trampoline target\n"); 563 return -EFAULT; 564 } 565 566 pr_devel("trampoline target %lx", ptr); 567 568 entry = ppc_global_function_entry((void *)addr); 569 /* This should match what was called */ 570 if (ptr != entry) { 571 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 572 return -EINVAL; 573 } 574 575 /* Ensure branch is within 24 bits */ 576 if (!create_branch((unsigned int *)ip, tramp, BRANCH_SET_LINK)) { 577 pr_err("Branch out of range\n"); 578 return -EINVAL; 579 } 580 581 if (patch_branch((unsigned int *)ip, tramp, BRANCH_SET_LINK)) { 582 pr_err("REL24 out of range!\n"); 583 return -EINVAL; 584 } 585 586 return 0; 587 } 588 #endif 589 590 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 591 unsigned long addr) 592 { 593 unsigned long ip = rec->ip; 594 unsigned int old, new; 595 596 /* 597 * If the calling address is more that 24 bits away, 598 * then we had to use a trampoline to make the call. 599 * Otherwise just update the call site. 600 */ 601 if (test_24bit_addr(ip, addr) && test_24bit_addr(ip, old_addr)) { 602 /* within range */ 603 old = ftrace_call_replace(ip, old_addr, 1); 604 new = ftrace_call_replace(ip, addr, 1); 605 return ftrace_modify_code(ip, old, new); 606 } 607 608 #ifdef CONFIG_MODULES 609 /* 610 * Out of range jumps are called from modules. 611 */ 612 if (!rec->arch.mod) { 613 pr_err("No module loaded\n"); 614 return -EINVAL; 615 } 616 617 return __ftrace_modify_call(rec, old_addr, addr); 618 #else 619 /* We should not get here without modules */ 620 return -EINVAL; 621 #endif /* CONFIG_MODULES */ 622 } 623 #endif 624 625 int ftrace_update_ftrace_func(ftrace_func_t func) 626 { 627 unsigned long ip = (unsigned long)(&ftrace_call); 628 unsigned int old, new; 629 int ret; 630 631 old = *(unsigned int *)&ftrace_call; 632 new = ftrace_call_replace(ip, (unsigned long)func, 1); 633 ret = ftrace_modify_code(ip, old, new); 634 635 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 636 /* Also update the regs callback function */ 637 if (!ret) { 638 ip = (unsigned long)(&ftrace_regs_call); 639 old = *(unsigned int *)&ftrace_regs_call; 640 new = ftrace_call_replace(ip, (unsigned long)func, 1); 641 ret = ftrace_modify_code(ip, old, new); 642 } 643 #endif 644 645 return ret; 646 } 647 648 /* 649 * Use the default ftrace_modify_all_code, but without 650 * stop_machine(). 651 */ 652 void arch_ftrace_update_code(int command) 653 { 654 ftrace_modify_all_code(command); 655 } 656 657 int __init ftrace_dyn_arch_init(void) 658 { 659 return 0; 660 } 661 #endif /* CONFIG_DYNAMIC_FTRACE */ 662 663 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 664 665 extern void ftrace_graph_call(void); 666 extern void ftrace_graph_stub(void); 667 668 int ftrace_enable_ftrace_graph_caller(void) 669 { 670 unsigned long ip = (unsigned long)(&ftrace_graph_call); 671 unsigned long addr = (unsigned long)(&ftrace_graph_caller); 672 unsigned long stub = (unsigned long)(&ftrace_graph_stub); 673 unsigned int old, new; 674 675 old = ftrace_call_replace(ip, stub, 0); 676 new = ftrace_call_replace(ip, addr, 0); 677 678 return ftrace_modify_code(ip, old, new); 679 } 680 681 int ftrace_disable_ftrace_graph_caller(void) 682 { 683 unsigned long ip = (unsigned long)(&ftrace_graph_call); 684 unsigned long addr = (unsigned long)(&ftrace_graph_caller); 685 unsigned long stub = (unsigned long)(&ftrace_graph_stub); 686 unsigned int old, new; 687 688 old = ftrace_call_replace(ip, addr, 0); 689 new = ftrace_call_replace(ip, stub, 0); 690 691 return ftrace_modify_code(ip, old, new); 692 } 693 694 /* 695 * Hook the return address and push it in the stack of return addrs 696 * in current thread info. Return the address we want to divert to. 697 */ 698 unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip) 699 { 700 struct ftrace_graph_ent trace; 701 unsigned long return_hooker; 702 703 if (unlikely(ftrace_graph_is_dead())) 704 goto out; 705 706 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 707 goto out; 708 709 return_hooker = ppc_function_entry(return_to_handler); 710 711 trace.func = ip; 712 trace.depth = current->curr_ret_stack + 1; 713 714 /* Only trace if the calling function expects to */ 715 if (!ftrace_graph_entry(&trace)) 716 goto out; 717 718 if (ftrace_push_return_trace(parent, ip, &trace.depth, 0, 719 NULL) == -EBUSY) 720 goto out; 721 722 parent = return_hooker; 723 out: 724 return parent; 725 } 726 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 727 728 #if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64) 729 unsigned long __init arch_syscall_addr(int nr) 730 { 731 return sys_call_table[nr*2]; 732 } 733 #endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_PPC64 */ 734 735 #ifdef PPC64_ELF_ABI_v1 736 char *arch_ftrace_match_adjust(char *str, const char *search) 737 { 738 if (str[0] == '.' && search[0] != '.') 739 return str + 1; 740 else 741 return str; 742 } 743 #endif /* PPC64_ELF_ABI_v1 */ 744