1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Code for replacing ftrace calls with jumps. 4 * 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 6 * 7 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box. 8 * 9 * Added function graph tracer code, taken from x86 that was written 10 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt. 11 * 12 */ 13 14 #define pr_fmt(fmt) "ftrace-powerpc: " fmt 15 16 #include <linux/spinlock.h> 17 #include <linux/hardirq.h> 18 #include <linux/uaccess.h> 19 #include <linux/module.h> 20 #include <linux/ftrace.h> 21 #include <linux/percpu.h> 22 #include <linux/init.h> 23 #include <linux/list.h> 24 25 #include <asm/cacheflush.h> 26 #include <asm/text-patching.h> 27 #include <asm/ftrace.h> 28 #include <asm/syscall.h> 29 #include <asm/inst.h> 30 #include <asm/sections.h> 31 32 #define NUM_FTRACE_TRAMPS 2 33 static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS]; 34 35 unsigned long ftrace_call_adjust(unsigned long addr) 36 { 37 if (addr >= (unsigned long)__exittext_begin && addr < (unsigned long)__exittext_end) 38 return 0; 39 40 if (IS_ENABLED(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY) && 41 !IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { 42 addr += MCOUNT_INSN_SIZE; 43 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS)) 44 addr += MCOUNT_INSN_SIZE; 45 } 46 47 return addr; 48 } 49 50 static ppc_inst_t ftrace_create_branch_inst(unsigned long ip, unsigned long addr, int link) 51 { 52 ppc_inst_t op; 53 54 WARN_ON(!is_offset_in_branch_range(addr - ip)); 55 create_branch(&op, (u32 *)ip, addr, link ? BRANCH_SET_LINK : 0); 56 57 return op; 58 } 59 60 static inline int ftrace_read_inst(unsigned long ip, ppc_inst_t *op) 61 { 62 if (copy_inst_from_kernel_nofault(op, (void *)ip)) { 63 pr_err("0x%lx: fetching instruction failed\n", ip); 64 return -EFAULT; 65 } 66 67 return 0; 68 } 69 70 static inline int ftrace_validate_inst(unsigned long ip, ppc_inst_t inst) 71 { 72 ppc_inst_t op; 73 int ret; 74 75 ret = ftrace_read_inst(ip, &op); 76 if (!ret && !ppc_inst_equal(op, inst)) { 77 pr_err("0x%lx: expected (%08lx) != found (%08lx)\n", 78 ip, ppc_inst_as_ulong(inst), ppc_inst_as_ulong(op)); 79 ret = -EINVAL; 80 } 81 82 return ret; 83 } 84 85 static inline int ftrace_modify_code(unsigned long ip, ppc_inst_t old, ppc_inst_t new) 86 { 87 int ret = ftrace_validate_inst(ip, old); 88 89 if (!ret && !ppc_inst_equal(old, new)) 90 ret = patch_instruction((u32 *)ip, new); 91 92 return ret; 93 } 94 95 static int is_bl_op(ppc_inst_t op) 96 { 97 return (ppc_inst_val(op) & ~PPC_LI_MASK) == PPC_RAW_BL(0); 98 } 99 100 static unsigned long find_ftrace_tramp(unsigned long ip) 101 { 102 int i; 103 104 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 105 if (!ftrace_tramps[i]) 106 continue; 107 else if (is_offset_in_branch_range(ftrace_tramps[i] - ip)) 108 return ftrace_tramps[i]; 109 110 return 0; 111 } 112 113 #ifdef CONFIG_MODULES 114 static unsigned long ftrace_lookup_module_stub(unsigned long ip, unsigned long addr) 115 { 116 struct module *mod = NULL; 117 118 scoped_guard(rcu) 119 mod = __module_text_address(ip); 120 if (!mod) 121 pr_err("No module loaded at addr=%lx\n", ip); 122 123 return (addr == (unsigned long)ftrace_caller ? mod->arch.tramp : mod->arch.tramp_regs); 124 } 125 #else 126 static unsigned long ftrace_lookup_module_stub(unsigned long ip, unsigned long addr) 127 { 128 return 0; 129 } 130 #endif 131 132 static unsigned long ftrace_get_ool_stub(struct dyn_ftrace *rec) 133 { 134 #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE 135 return rec->arch.ool_stub; 136 #else 137 BUILD_BUG(); 138 #endif 139 } 140 141 static int ftrace_get_call_inst(struct dyn_ftrace *rec, unsigned long addr, ppc_inst_t *call_inst) 142 { 143 unsigned long ip; 144 unsigned long stub; 145 146 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) 147 ip = ftrace_get_ool_stub(rec) + MCOUNT_INSN_SIZE; /* second instruction in stub */ 148 else 149 ip = rec->ip; 150 151 if (!is_offset_in_branch_range(addr - ip) && addr != FTRACE_ADDR && 152 addr != FTRACE_REGS_ADDR) { 153 /* This can only happen with ftrace direct */ 154 if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS)) { 155 pr_err("0x%lx (0x%lx): Unexpected target address 0x%lx\n", 156 ip, rec->ip, addr); 157 return -EINVAL; 158 } 159 addr = FTRACE_ADDR; 160 } 161 162 if (is_offset_in_branch_range(addr - ip)) 163 /* Within range */ 164 stub = addr; 165 else if (core_kernel_text(ip)) 166 /* We would be branching to one of our ftrace stubs */ 167 stub = find_ftrace_tramp(ip); 168 else 169 stub = ftrace_lookup_module_stub(ip, addr); 170 171 if (!stub) { 172 pr_err("0x%lx (0x%lx): No ftrace stubs reachable\n", ip, rec->ip); 173 return -EINVAL; 174 } 175 176 *call_inst = ftrace_create_branch_inst(ip, stub, 1); 177 return 0; 178 } 179 180 static int ftrace_init_ool_stub(struct module *mod, struct dyn_ftrace *rec) 181 { 182 #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE 183 static int ool_stub_text_index, ool_stub_text_end_index, ool_stub_inittext_index; 184 int ret = 0, ool_stub_count, *ool_stub_index; 185 ppc_inst_t inst; 186 /* 187 * See ftrace_entry.S if changing the below instruction sequence, as we rely on 188 * decoding the last branch instruction here to recover the correct function ip. 189 */ 190 struct ftrace_ool_stub *ool_stub, ool_stub_template = { 191 .insn = { 192 PPC_RAW_MFLR(_R0), 193 PPC_RAW_NOP(), /* bl ftrace_caller */ 194 PPC_RAW_MTLR(_R0), 195 PPC_RAW_NOP() /* b rec->ip + 4 */ 196 } 197 }; 198 199 WARN_ON(rec->arch.ool_stub); 200 201 if (is_kernel_inittext(rec->ip)) { 202 ool_stub = ftrace_ool_stub_inittext; 203 ool_stub_index = &ool_stub_inittext_index; 204 ool_stub_count = ftrace_ool_stub_inittext_count; 205 } else if (is_kernel_text(rec->ip)) { 206 /* 207 * ftrace records are sorted, so we first use up the stub area within .text 208 * (ftrace_ool_stub_text) before using the area at the end of .text 209 * (ftrace_ool_stub_text_end), unless the stub is out of range of the record. 210 */ 211 if (ool_stub_text_index >= ftrace_ool_stub_text_count || 212 !is_offset_in_branch_range((long)rec->ip - 213 (long)&ftrace_ool_stub_text[ool_stub_text_index])) { 214 ool_stub = ftrace_ool_stub_text_end; 215 ool_stub_index = &ool_stub_text_end_index; 216 ool_stub_count = ftrace_ool_stub_text_end_count; 217 } else { 218 ool_stub = ftrace_ool_stub_text; 219 ool_stub_index = &ool_stub_text_index; 220 ool_stub_count = ftrace_ool_stub_text_count; 221 } 222 #ifdef CONFIG_MODULES 223 } else if (mod) { 224 ool_stub = mod->arch.ool_stubs; 225 ool_stub_index = &mod->arch.ool_stub_index; 226 ool_stub_count = mod->arch.ool_stub_count; 227 #endif 228 } else { 229 return -EINVAL; 230 } 231 232 ool_stub += (*ool_stub_index)++; 233 234 if (WARN_ON(*ool_stub_index > ool_stub_count)) 235 return -EINVAL; 236 237 if (!is_offset_in_branch_range((long)rec->ip - (long)&ool_stub->insn[0]) || 238 !is_offset_in_branch_range((long)(rec->ip + MCOUNT_INSN_SIZE) - 239 (long)&ool_stub->insn[3])) { 240 pr_err("%s: ftrace ool stub out of range (%p -> %p).\n", 241 __func__, (void *)rec->ip, (void *)&ool_stub->insn[0]); 242 return -EINVAL; 243 } 244 245 rec->arch.ool_stub = (unsigned long)&ool_stub->insn[0]; 246 247 /* bl ftrace_caller */ 248 if (!mod) 249 ret = ftrace_get_call_inst(rec, (unsigned long)ftrace_caller, &inst); 250 #ifdef CONFIG_MODULES 251 else 252 /* 253 * We can't use ftrace_get_call_inst() since that uses 254 * __module_text_address(rec->ip) to look up the module. 255 * But, since the module is not fully formed at this stage, 256 * the lookup fails. We know the target though, so generate 257 * the branch inst directly. 258 */ 259 inst = ftrace_create_branch_inst(ftrace_get_ool_stub(rec) + MCOUNT_INSN_SIZE, 260 mod->arch.tramp, 1); 261 #endif 262 ool_stub_template.insn[1] = ppc_inst_val(inst); 263 264 /* b rec->ip + 4 */ 265 if (!ret && create_branch(&inst, &ool_stub->insn[3], rec->ip + MCOUNT_INSN_SIZE, 0)) 266 return -EINVAL; 267 ool_stub_template.insn[3] = ppc_inst_val(inst); 268 269 if (!ret) 270 ret = patch_instructions((u32 *)ool_stub, (u32 *)&ool_stub_template, 271 sizeof(ool_stub_template), false); 272 273 return ret; 274 #else /* !CONFIG_PPC_FTRACE_OUT_OF_LINE */ 275 BUILD_BUG(); 276 #endif 277 } 278 279 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS 280 static const struct ftrace_ops *powerpc_rec_get_ops(struct dyn_ftrace *rec) 281 { 282 const struct ftrace_ops *ops = NULL; 283 284 if (rec->flags & FTRACE_FL_CALL_OPS_EN) { 285 ops = ftrace_find_unique_ops(rec); 286 WARN_ON_ONCE(!ops); 287 } 288 289 if (!ops) 290 ops = &ftrace_list_ops; 291 292 return ops; 293 } 294 295 static int ftrace_rec_set_ops(struct dyn_ftrace *rec, const struct ftrace_ops *ops) 296 { 297 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) 298 return patch_ulong((void *)(ftrace_get_ool_stub(rec) - sizeof(unsigned long)), 299 (unsigned long)ops); 300 else 301 return patch_ulong((void *)(rec->ip - MCOUNT_INSN_SIZE - sizeof(unsigned long)), 302 (unsigned long)ops); 303 } 304 305 static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec) 306 { 307 return ftrace_rec_set_ops(rec, &ftrace_nop_ops); 308 } 309 310 static int ftrace_rec_update_ops(struct dyn_ftrace *rec) 311 { 312 return ftrace_rec_set_ops(rec, powerpc_rec_get_ops(rec)); 313 } 314 #else 315 static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec) { return 0; } 316 static int ftrace_rec_update_ops(struct dyn_ftrace *rec) { return 0; } 317 #endif 318 319 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 320 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) 321 { 322 /* This should never be called since we override ftrace_replace_code() */ 323 WARN_ON(1); 324 return -EINVAL; 325 } 326 #endif 327 328 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 329 { 330 ppc_inst_t old, new; 331 unsigned long ip = rec->ip; 332 int ret = 0; 333 334 /* This can only ever be called during module load */ 335 if (WARN_ON(!IS_ENABLED(CONFIG_MODULES) || core_kernel_text(ip))) 336 return -EINVAL; 337 338 old = ppc_inst(PPC_RAW_NOP()); 339 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { 340 ip = ftrace_get_ool_stub(rec) + MCOUNT_INSN_SIZE; /* second instruction in stub */ 341 ret = ftrace_get_call_inst(rec, (unsigned long)ftrace_caller, &old); 342 } 343 344 ret |= ftrace_get_call_inst(rec, addr, &new); 345 346 if (!ret) 347 ret = ftrace_modify_code(ip, old, new); 348 349 ret = ftrace_rec_update_ops(rec); 350 if (ret) 351 return ret; 352 353 if (!ret && IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) 354 ret = ftrace_modify_code(rec->ip, ppc_inst(PPC_RAW_NOP()), 355 ppc_inst(PPC_RAW_BRANCH((long)ftrace_get_ool_stub(rec) - (long)rec->ip))); 356 357 return ret; 358 } 359 360 int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr) 361 { 362 /* 363 * This should never be called since we override ftrace_replace_code(), 364 * as well as ftrace_init_nop() 365 */ 366 WARN_ON(1); 367 return -EINVAL; 368 } 369 370 void ftrace_replace_code(int enable) 371 { 372 ppc_inst_t old, new, call_inst, new_call_inst; 373 ppc_inst_t nop_inst = ppc_inst(PPC_RAW_NOP()); 374 unsigned long ip, new_addr, addr; 375 struct ftrace_rec_iter *iter; 376 struct dyn_ftrace *rec; 377 int ret = 0, update; 378 379 for_ftrace_rec_iter(iter) { 380 rec = ftrace_rec_iter_record(iter); 381 ip = rec->ip; 382 383 if (rec->flags & FTRACE_FL_DISABLED && !(rec->flags & FTRACE_FL_ENABLED)) 384 continue; 385 386 addr = ftrace_get_addr_curr(rec); 387 new_addr = ftrace_get_addr_new(rec); 388 update = ftrace_update_record(rec, enable); 389 390 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) && update != FTRACE_UPDATE_IGNORE) { 391 ip = ftrace_get_ool_stub(rec) + MCOUNT_INSN_SIZE; 392 ret = ftrace_get_call_inst(rec, (unsigned long)ftrace_caller, &nop_inst); 393 if (ret) 394 goto out; 395 } 396 397 switch (update) { 398 case FTRACE_UPDATE_IGNORE: 399 default: 400 continue; 401 case FTRACE_UPDATE_MODIFY_CALL: 402 ret = ftrace_get_call_inst(rec, new_addr, &new_call_inst); 403 ret |= ftrace_get_call_inst(rec, addr, &call_inst); 404 ret |= ftrace_rec_update_ops(rec); 405 old = call_inst; 406 new = new_call_inst; 407 break; 408 case FTRACE_UPDATE_MAKE_NOP: 409 ret = ftrace_get_call_inst(rec, addr, &call_inst); 410 ret |= ftrace_rec_set_nop_ops(rec); 411 old = call_inst; 412 new = nop_inst; 413 break; 414 case FTRACE_UPDATE_MAKE_CALL: 415 ret = ftrace_get_call_inst(rec, new_addr, &call_inst); 416 ret |= ftrace_rec_update_ops(rec); 417 old = nop_inst; 418 new = call_inst; 419 break; 420 } 421 422 if (!ret) 423 ret = ftrace_modify_code(ip, old, new); 424 425 if (!ret && IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) && 426 (update == FTRACE_UPDATE_MAKE_NOP || update == FTRACE_UPDATE_MAKE_CALL)) { 427 /* Update the actual ftrace location */ 428 call_inst = ppc_inst(PPC_RAW_BRANCH((long)ftrace_get_ool_stub(rec) - 429 (long)rec->ip)); 430 nop_inst = ppc_inst(PPC_RAW_NOP()); 431 ip = rec->ip; 432 433 if (update == FTRACE_UPDATE_MAKE_NOP) 434 ret = ftrace_modify_code(ip, call_inst, nop_inst); 435 else 436 ret = ftrace_modify_code(ip, nop_inst, call_inst); 437 438 if (ret) 439 goto out; 440 } 441 442 if (ret) 443 goto out; 444 } 445 446 out: 447 if (ret) 448 ftrace_bug(ret, rec); 449 return; 450 } 451 452 int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) 453 { 454 unsigned long addr, ip = rec->ip; 455 ppc_inst_t old, new; 456 int ret = 0; 457 458 /* Verify instructions surrounding the ftrace location */ 459 if (IS_ENABLED(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY)) { 460 /* Expect nops */ 461 if (!IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) 462 ret = ftrace_validate_inst(ip - 4, ppc_inst(PPC_RAW_NOP())); 463 if (!ret) 464 ret = ftrace_validate_inst(ip, ppc_inst(PPC_RAW_NOP())); 465 } else if (IS_ENABLED(CONFIG_PPC32)) { 466 /* Expected sequence: 'mflr r0', 'stw r0,4(r1)', 'bl _mcount' */ 467 ret = ftrace_validate_inst(ip - 8, ppc_inst(PPC_RAW_MFLR(_R0))); 468 if (ret) 469 return ret; 470 ret = ftrace_modify_code(ip - 4, ppc_inst(PPC_RAW_STW(_R0, _R1, 4)), 471 ppc_inst(PPC_RAW_NOP())); 472 } else if (IS_ENABLED(CONFIG_MPROFILE_KERNEL)) { 473 /* Expected sequence: 'mflr r0', ['std r0,16(r1)'], 'bl _mcount' */ 474 ret = ftrace_read_inst(ip - 4, &old); 475 if (!ret && !ppc_inst_equal(old, ppc_inst(PPC_RAW_MFLR(_R0)))) { 476 /* Gcc v5.x emit the additional 'std' instruction, gcc v6.x don't */ 477 ret = ftrace_validate_inst(ip - 8, ppc_inst(PPC_RAW_MFLR(_R0))); 478 if (ret) 479 return ret; 480 ret = ftrace_modify_code(ip - 4, ppc_inst(PPC_RAW_STD(_R0, _R1, 16)), 481 ppc_inst(PPC_RAW_NOP())); 482 } 483 } else { 484 return -EINVAL; 485 } 486 487 if (ret) 488 return ret; 489 490 /* Set up out-of-line stub */ 491 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) 492 return ftrace_init_ool_stub(mod, rec); 493 494 /* Nop-out the ftrace location */ 495 new = ppc_inst(PPC_RAW_NOP()); 496 addr = MCOUNT_ADDR; 497 if (IS_ENABLED(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY)) { 498 /* we instead patch-in the 'mflr r0' */ 499 old = ppc_inst(PPC_RAW_NOP()); 500 new = ppc_inst(PPC_RAW_MFLR(_R0)); 501 ret = ftrace_modify_code(ip - 4, old, new); 502 } else if (is_offset_in_branch_range(addr - ip)) { 503 /* Within range */ 504 old = ftrace_create_branch_inst(ip, addr, 1); 505 ret = ftrace_modify_code(ip, old, new); 506 } else if (core_kernel_text(ip) || (IS_ENABLED(CONFIG_MODULES) && mod)) { 507 /* 508 * We would be branching to a linker-generated stub, or to the module _mcount 509 * stub. Let's just confirm we have a 'bl' here. 510 */ 511 ret = ftrace_read_inst(ip, &old); 512 if (ret) 513 return ret; 514 if (!is_bl_op(old)) { 515 pr_err("0x%lx: expected (bl) != found (%08lx)\n", ip, ppc_inst_as_ulong(old)); 516 return -EINVAL; 517 } 518 ret = patch_instruction((u32 *)ip, new); 519 } else { 520 return -EINVAL; 521 } 522 523 return ret; 524 } 525 526 int ftrace_update_ftrace_func(ftrace_func_t func) 527 { 528 unsigned long ip = (unsigned long)(&ftrace_call); 529 ppc_inst_t old, new; 530 int ret; 531 532 /* 533 * When using CALL_OPS, the function to call is associated with the 534 * call site, and we don't have a global function pointer to update. 535 */ 536 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS)) 537 return 0; 538 539 old = ppc_inst_read((u32 *)&ftrace_call); 540 new = ftrace_create_branch_inst(ip, ppc_function_entry(func), 1); 541 ret = ftrace_modify_code(ip, old, new); 542 543 /* Also update the regs callback function */ 544 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) && !ret) { 545 ip = (unsigned long)(&ftrace_regs_call); 546 old = ppc_inst_read((u32 *)&ftrace_regs_call); 547 new = ftrace_create_branch_inst(ip, ppc_function_entry(func), 1); 548 ret = ftrace_modify_code(ip, old, new); 549 } 550 551 return ret; 552 } 553 554 /* 555 * Use the default ftrace_modify_all_code, but without 556 * stop_machine(). 557 */ 558 void arch_ftrace_update_code(int command) 559 { 560 ftrace_modify_all_code(command); 561 } 562 563 void ftrace_free_init_tramp(void) 564 { 565 int i; 566 567 for (i = 0; i < NUM_FTRACE_TRAMPS && ftrace_tramps[i]; i++) 568 if (ftrace_tramps[i] == (unsigned long)ftrace_tramp_init) { 569 ftrace_tramps[i] = 0; 570 return; 571 } 572 } 573 574 static void __init add_ftrace_tramp(unsigned long tramp) 575 { 576 int i; 577 578 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 579 if (!ftrace_tramps[i]) { 580 ftrace_tramps[i] = tramp; 581 return; 582 } 583 } 584 585 int __init ftrace_dyn_arch_init(void) 586 { 587 unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init }; 588 unsigned long addr = FTRACE_REGS_ADDR; 589 long reladdr; 590 int i; 591 u32 stub_insns[] = { 592 #ifdef CONFIG_PPC_KERNEL_PCREL 593 /* pla r12,addr */ 594 PPC_PREFIX_MLS | __PPC_PRFX_R(1), 595 PPC_INST_PADDI | ___PPC_RT(_R12), 596 PPC_RAW_MTCTR(_R12), 597 PPC_RAW_BCTR() 598 #elif defined(CONFIG_PPC64) 599 PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernel_toc)), 600 PPC_RAW_ADDIS(_R12, _R12, 0), 601 PPC_RAW_ADDI(_R12, _R12, 0), 602 PPC_RAW_MTCTR(_R12), 603 PPC_RAW_BCTR() 604 #else 605 PPC_RAW_LIS(_R12, 0), 606 PPC_RAW_ADDI(_R12, _R12, 0), 607 PPC_RAW_MTCTR(_R12), 608 PPC_RAW_BCTR() 609 #endif 610 }; 611 612 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) { 613 for (i = 0; i < 2; i++) { 614 reladdr = addr - (unsigned long)tramp[i]; 615 616 if (reladdr >= (long)SZ_8G || reladdr < -(long)SZ_8G) { 617 pr_err("Address of %ps out of range of pcrel address.\n", 618 (void *)addr); 619 return -1; 620 } 621 622 memcpy(tramp[i], stub_insns, sizeof(stub_insns)); 623 tramp[i][0] |= IMM_H18(reladdr); 624 tramp[i][1] |= IMM_L(reladdr); 625 add_ftrace_tramp((unsigned long)tramp[i]); 626 } 627 } else if (IS_ENABLED(CONFIG_PPC64)) { 628 reladdr = addr - kernel_toc_addr(); 629 630 if (reladdr >= (long)SZ_2G || reladdr < -(long long)SZ_2G) { 631 pr_err("Address of %ps out of range of kernel_toc.\n", 632 (void *)addr); 633 return -1; 634 } 635 636 for (i = 0; i < 2; i++) { 637 memcpy(tramp[i], stub_insns, sizeof(stub_insns)); 638 tramp[i][1] |= PPC_HA(reladdr); 639 tramp[i][2] |= PPC_LO(reladdr); 640 add_ftrace_tramp((unsigned long)tramp[i]); 641 } 642 } else { 643 for (i = 0; i < 2; i++) { 644 memcpy(tramp[i], stub_insns, sizeof(stub_insns)); 645 tramp[i][0] |= PPC_HA(addr); 646 tramp[i][1] |= PPC_LO(addr); 647 add_ftrace_tramp((unsigned long)tramp[i]); 648 } 649 } 650 651 return 0; 652 } 653 654 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 655 void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, 656 struct ftrace_ops *op, struct ftrace_regs *fregs) 657 { 658 unsigned long sp = arch_ftrace_regs(fregs)->regs.gpr[1]; 659 660 if (unlikely(ftrace_graph_is_dead())) 661 goto out; 662 663 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 664 goto out; 665 666 if (!function_graph_enter_regs(parent_ip, ip, 0, (unsigned long *)sp, fregs)) 667 parent_ip = ppc_function_entry(return_to_handler); 668 669 out: 670 arch_ftrace_regs(fregs)->regs.link = parent_ip; 671 } 672 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 673