xref: /linux/arch/powerpc/kernel/trace/ftrace.c (revision 680e6ffa15103ab610c0fc1241d2f98c801b13e2)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Code for replacing ftrace calls with jumps.
4  *
5  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6  *
7  * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
8  *
9  * Added function graph tracer code, taken from x86 that was written
10  * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
11  *
12  */
13 
14 #define pr_fmt(fmt) "ftrace-powerpc: " fmt
15 
16 #include <linux/spinlock.h>
17 #include <linux/hardirq.h>
18 #include <linux/uaccess.h>
19 #include <linux/module.h>
20 #include <linux/ftrace.h>
21 #include <linux/percpu.h>
22 #include <linux/init.h>
23 #include <linux/list.h>
24 
25 #include <asm/asm-prototypes.h>
26 #include <asm/cacheflush.h>
27 #include <asm/code-patching.h>
28 #include <asm/ftrace.h>
29 #include <asm/syscall.h>
30 
31 
32 #ifdef CONFIG_DYNAMIC_FTRACE
33 
34 /*
35  * We generally only have a single long_branch tramp and at most 2 or 3 plt
36  * tramps generated. But, we don't use the plt tramps currently. We also allot
37  * 2 tramps after .text and .init.text. So, we only end up with around 3 usable
38  * tramps in total. Set aside 8 just to be sure.
39  */
40 #define	NUM_FTRACE_TRAMPS	8
41 static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS];
42 
43 static unsigned int
44 ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
45 {
46 	unsigned int op;
47 
48 	addr = ppc_function_entry((void *)addr);
49 
50 	/* if (link) set op to 'bl' else 'b' */
51 	op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
52 
53 	return op;
54 }
55 
56 static int
57 ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
58 {
59 	unsigned int replaced;
60 
61 	/*
62 	 * Note:
63 	 * We are paranoid about modifying text, as if a bug was to happen, it
64 	 * could cause us to read or write to someplace that could cause harm.
65 	 * Carefully read and modify the code with probe_kernel_*(), and make
66 	 * sure what we read is what we expected it to be before modifying it.
67 	 */
68 
69 	/* read the text we want to modify */
70 	if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
71 		return -EFAULT;
72 
73 	/* Make sure it is what we expect it to be */
74 	if (replaced != old) {
75 		pr_err("%p: replaced (%#x) != old (%#x)",
76 		(void *)ip, replaced, old);
77 		return -EINVAL;
78 	}
79 
80 	/* replace the text with the new text */
81 	if (patch_instruction((unsigned int *)ip, new))
82 		return -EPERM;
83 
84 	return 0;
85 }
86 
87 /*
88  * Helper functions that are the same for both PPC64 and PPC32.
89  */
90 static int test_24bit_addr(unsigned long ip, unsigned long addr)
91 {
92 	addr = ppc_function_entry((void *)addr);
93 
94 	/* use the create_branch to verify that this offset can be branched */
95 	return create_branch((unsigned int *)ip, addr, 0);
96 }
97 
98 static int is_bl_op(unsigned int op)
99 {
100 	return (op & 0xfc000003) == 0x48000001;
101 }
102 
103 static int is_b_op(unsigned int op)
104 {
105 	return (op & 0xfc000003) == 0x48000000;
106 }
107 
108 static unsigned long find_bl_target(unsigned long ip, unsigned int op)
109 {
110 	int offset;
111 
112 	offset = (op & 0x03fffffc);
113 	/* make it signed */
114 	if (offset & 0x02000000)
115 		offset |= 0xfe000000;
116 
117 	return ip + (long)offset;
118 }
119 
120 #ifdef CONFIG_MODULES
121 #ifdef CONFIG_PPC64
122 static int
123 __ftrace_make_nop(struct module *mod,
124 		  struct dyn_ftrace *rec, unsigned long addr)
125 {
126 	unsigned long entry, ptr, tramp;
127 	unsigned long ip = rec->ip;
128 	unsigned int op, pop;
129 
130 	/* read where this goes */
131 	if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
132 		pr_err("Fetching opcode failed.\n");
133 		return -EFAULT;
134 	}
135 
136 	/* Make sure that that this is still a 24bit jump */
137 	if (!is_bl_op(op)) {
138 		pr_err("Not expected bl: opcode is %x\n", op);
139 		return -EINVAL;
140 	}
141 
142 	/* lets find where the pointer goes */
143 	tramp = find_bl_target(ip, op);
144 
145 	pr_devel("ip:%lx jumps to %lx", ip, tramp);
146 
147 	if (module_trampoline_target(mod, tramp, &ptr)) {
148 		pr_err("Failed to get trampoline target\n");
149 		return -EFAULT;
150 	}
151 
152 	pr_devel("trampoline target %lx", ptr);
153 
154 	entry = ppc_global_function_entry((void *)addr);
155 	/* This should match what was called */
156 	if (ptr != entry) {
157 		pr_err("addr %lx does not match expected %lx\n", ptr, entry);
158 		return -EINVAL;
159 	}
160 
161 #ifdef CONFIG_MPROFILE_KERNEL
162 	/* When using -mkernel_profile there is no load to jump over */
163 	pop = PPC_INST_NOP;
164 
165 	if (probe_kernel_read(&op, (void *)(ip - 4), 4)) {
166 		pr_err("Fetching instruction at %lx failed.\n", ip - 4);
167 		return -EFAULT;
168 	}
169 
170 	/* We expect either a mflr r0, or a std r0, LRSAVE(r1) */
171 	if (op != PPC_INST_MFLR && op != PPC_INST_STD_LR) {
172 		pr_err("Unexpected instruction %08x around bl _mcount\n", op);
173 		return -EINVAL;
174 	}
175 #else
176 	/*
177 	 * Our original call site looks like:
178 	 *
179 	 * bl <tramp>
180 	 * ld r2,XX(r1)
181 	 *
182 	 * Milton Miller pointed out that we can not simply nop the branch.
183 	 * If a task was preempted when calling a trace function, the nops
184 	 * will remove the way to restore the TOC in r2 and the r2 TOC will
185 	 * get corrupted.
186 	 *
187 	 * Use a b +8 to jump over the load.
188 	 */
189 
190 	pop = PPC_INST_BRANCH | 8;	/* b +8 */
191 
192 	/*
193 	 * Check what is in the next instruction. We can see ld r2,40(r1), but
194 	 * on first pass after boot we will see mflr r0.
195 	 */
196 	if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE)) {
197 		pr_err("Fetching op failed.\n");
198 		return -EFAULT;
199 	}
200 
201 	if (op != PPC_INST_LD_TOC) {
202 		pr_err("Expected %08x found %08x\n", PPC_INST_LD_TOC, op);
203 		return -EINVAL;
204 	}
205 #endif /* CONFIG_MPROFILE_KERNEL */
206 
207 	if (patch_instruction((unsigned int *)ip, pop)) {
208 		pr_err("Patching NOP failed.\n");
209 		return -EPERM;
210 	}
211 
212 	return 0;
213 }
214 
215 #else /* !PPC64 */
216 static int
217 __ftrace_make_nop(struct module *mod,
218 		  struct dyn_ftrace *rec, unsigned long addr)
219 {
220 	unsigned int op;
221 	unsigned int jmp[4];
222 	unsigned long ip = rec->ip;
223 	unsigned long tramp;
224 
225 	if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
226 		return -EFAULT;
227 
228 	/* Make sure that that this is still a 24bit jump */
229 	if (!is_bl_op(op)) {
230 		pr_err("Not expected bl: opcode is %x\n", op);
231 		return -EINVAL;
232 	}
233 
234 	/* lets find where the pointer goes */
235 	tramp = find_bl_target(ip, op);
236 
237 	/*
238 	 * On PPC32 the trampoline looks like:
239 	 *  0x3d, 0x80, 0x00, 0x00  lis r12,sym@ha
240 	 *  0x39, 0x8c, 0x00, 0x00  addi r12,r12,sym@l
241 	 *  0x7d, 0x89, 0x03, 0xa6  mtctr r12
242 	 *  0x4e, 0x80, 0x04, 0x20  bctr
243 	 */
244 
245 	pr_devel("ip:%lx jumps to %lx", ip, tramp);
246 
247 	/* Find where the trampoline jumps to */
248 	if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
249 		pr_err("Failed to read %lx\n", tramp);
250 		return -EFAULT;
251 	}
252 
253 	pr_devel(" %08x %08x ", jmp[0], jmp[1]);
254 
255 	/* verify that this is what we expect it to be */
256 	if (((jmp[0] & 0xffff0000) != 0x3d800000) ||
257 	    ((jmp[1] & 0xffff0000) != 0x398c0000) ||
258 	    (jmp[2] != 0x7d8903a6) ||
259 	    (jmp[3] != 0x4e800420)) {
260 		pr_err("Not a trampoline\n");
261 		return -EINVAL;
262 	}
263 
264 	tramp = (jmp[1] & 0xffff) |
265 		((jmp[0] & 0xffff) << 16);
266 	if (tramp & 0x8000)
267 		tramp -= 0x10000;
268 
269 	pr_devel(" %lx ", tramp);
270 
271 	if (tramp != addr) {
272 		pr_err("Trampoline location %08lx does not match addr\n",
273 		       tramp);
274 		return -EINVAL;
275 	}
276 
277 	op = PPC_INST_NOP;
278 
279 	if (patch_instruction((unsigned int *)ip, op))
280 		return -EPERM;
281 
282 	return 0;
283 }
284 #endif /* PPC64 */
285 #endif /* CONFIG_MODULES */
286 
287 static unsigned long find_ftrace_tramp(unsigned long ip)
288 {
289 	int i;
290 
291 	/*
292 	 * We have the compiler generated long_branch tramps at the end
293 	 * and we prefer those
294 	 */
295 	for (i = NUM_FTRACE_TRAMPS - 1; i >= 0; i--)
296 		if (!ftrace_tramps[i])
297 			continue;
298 		else if (create_branch((void *)ip, ftrace_tramps[i], 0))
299 			return ftrace_tramps[i];
300 
301 	return 0;
302 }
303 
304 static int add_ftrace_tramp(unsigned long tramp)
305 {
306 	int i;
307 
308 	for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
309 		if (!ftrace_tramps[i]) {
310 			ftrace_tramps[i] = tramp;
311 			return 0;
312 		}
313 
314 	return -1;
315 }
316 
317 /*
318  * If this is a compiler generated long_branch trampoline (essentially, a
319  * trampoline that has a branch to _mcount()), we re-write the branch to
320  * instead go to ftrace_[regs_]caller() and note down the location of this
321  * trampoline.
322  */
323 static int setup_mcount_compiler_tramp(unsigned long tramp)
324 {
325 	int i, op;
326 	unsigned long ptr;
327 	static unsigned long ftrace_plt_tramps[NUM_FTRACE_TRAMPS];
328 
329 	/* Is this a known long jump tramp? */
330 	for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
331 		if (!ftrace_tramps[i])
332 			break;
333 		else if (ftrace_tramps[i] == tramp)
334 			return 0;
335 
336 	/* Is this a known plt tramp? */
337 	for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
338 		if (!ftrace_plt_tramps[i])
339 			break;
340 		else if (ftrace_plt_tramps[i] == tramp)
341 			return -1;
342 
343 	/* New trampoline -- read where this goes */
344 	if (probe_kernel_read(&op, (void *)tramp, sizeof(int))) {
345 		pr_debug("Fetching opcode failed.\n");
346 		return -1;
347 	}
348 
349 	/* Is this a 24 bit branch? */
350 	if (!is_b_op(op)) {
351 		pr_debug("Trampoline is not a long branch tramp.\n");
352 		return -1;
353 	}
354 
355 	/* lets find where the pointer goes */
356 	ptr = find_bl_target(tramp, op);
357 
358 	if (ptr != ppc_global_function_entry((void *)_mcount)) {
359 		pr_debug("Trampoline target %p is not _mcount\n", (void *)ptr);
360 		return -1;
361 	}
362 
363 	/* Let's re-write the tramp to go to ftrace_[regs_]caller */
364 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
365 	ptr = ppc_global_function_entry((void *)ftrace_regs_caller);
366 #else
367 	ptr = ppc_global_function_entry((void *)ftrace_caller);
368 #endif
369 	if (!create_branch((void *)tramp, ptr, 0)) {
370 		pr_debug("%ps is not reachable from existing mcount tramp\n",
371 				(void *)ptr);
372 		return -1;
373 	}
374 
375 	if (patch_branch((unsigned int *)tramp, ptr, 0)) {
376 		pr_debug("REL24 out of range!\n");
377 		return -1;
378 	}
379 
380 	if (add_ftrace_tramp(tramp)) {
381 		pr_debug("No tramp locations left\n");
382 		return -1;
383 	}
384 
385 	return 0;
386 }
387 
388 static int __ftrace_make_nop_kernel(struct dyn_ftrace *rec, unsigned long addr)
389 {
390 	unsigned long tramp, ip = rec->ip;
391 	unsigned int op;
392 
393 	/* Read where this goes */
394 	if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
395 		pr_err("Fetching opcode failed.\n");
396 		return -EFAULT;
397 	}
398 
399 	/* Make sure that that this is still a 24bit jump */
400 	if (!is_bl_op(op)) {
401 		pr_err("Not expected bl: opcode is %x\n", op);
402 		return -EINVAL;
403 	}
404 
405 	/* Let's find where the pointer goes */
406 	tramp = find_bl_target(ip, op);
407 
408 	pr_devel("ip:%lx jumps to %lx", ip, tramp);
409 
410 	if (setup_mcount_compiler_tramp(tramp)) {
411 		/* Are other trampolines reachable? */
412 		if (!find_ftrace_tramp(ip)) {
413 			pr_err("No ftrace trampolines reachable from %ps\n",
414 					(void *)ip);
415 			return -EINVAL;
416 		}
417 	}
418 
419 	if (patch_instruction((unsigned int *)ip, PPC_INST_NOP)) {
420 		pr_err("Patching NOP failed.\n");
421 		return -EPERM;
422 	}
423 
424 	return 0;
425 }
426 
427 int ftrace_make_nop(struct module *mod,
428 		    struct dyn_ftrace *rec, unsigned long addr)
429 {
430 	unsigned long ip = rec->ip;
431 	unsigned int old, new;
432 
433 	/*
434 	 * If the calling address is more that 24 bits away,
435 	 * then we had to use a trampoline to make the call.
436 	 * Otherwise just update the call site.
437 	 */
438 	if (test_24bit_addr(ip, addr)) {
439 		/* within range */
440 		old = ftrace_call_replace(ip, addr, 1);
441 		new = PPC_INST_NOP;
442 		return ftrace_modify_code(ip, old, new);
443 	} else if (core_kernel_text(ip))
444 		return __ftrace_make_nop_kernel(rec, addr);
445 
446 #ifdef CONFIG_MODULES
447 	/*
448 	 * Out of range jumps are called from modules.
449 	 * We should either already have a pointer to the module
450 	 * or it has been passed in.
451 	 */
452 	if (!rec->arch.mod) {
453 		if (!mod) {
454 			pr_err("No module loaded addr=%lx\n", addr);
455 			return -EFAULT;
456 		}
457 		rec->arch.mod = mod;
458 	} else if (mod) {
459 		if (mod != rec->arch.mod) {
460 			pr_err("Record mod %p not equal to passed in mod %p\n",
461 			       rec->arch.mod, mod);
462 			return -EINVAL;
463 		}
464 		/* nothing to do if mod == rec->arch.mod */
465 	} else
466 		mod = rec->arch.mod;
467 
468 	return __ftrace_make_nop(mod, rec, addr);
469 #else
470 	/* We should not get here without modules */
471 	return -EINVAL;
472 #endif /* CONFIG_MODULES */
473 }
474 
475 #ifdef CONFIG_MODULES
476 #ifdef CONFIG_PPC64
477 /*
478  * Examine the existing instructions for __ftrace_make_call.
479  * They should effectively be a NOP, and follow formal constraints,
480  * depending on the ABI. Return false if they don't.
481  */
482 #ifndef CONFIG_MPROFILE_KERNEL
483 static int
484 expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
485 {
486 	/*
487 	 * We expect to see:
488 	 *
489 	 * b +8
490 	 * ld r2,XX(r1)
491 	 *
492 	 * The load offset is different depending on the ABI. For simplicity
493 	 * just mask it out when doing the compare.
494 	 */
495 	if ((op0 != 0x48000008) || ((op1 & 0xffff0000) != 0xe8410000))
496 		return 0;
497 	return 1;
498 }
499 #else
500 static int
501 expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
502 {
503 	/* look for patched "NOP" on ppc64 with -mprofile-kernel */
504 	if (op0 != PPC_INST_NOP)
505 		return 0;
506 	return 1;
507 }
508 #endif
509 
510 static int
511 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
512 {
513 	unsigned int op[2];
514 	void *ip = (void *)rec->ip;
515 	unsigned long entry, ptr, tramp;
516 	struct module *mod = rec->arch.mod;
517 
518 	/* read where this goes */
519 	if (probe_kernel_read(op, ip, sizeof(op)))
520 		return -EFAULT;
521 
522 	if (!expected_nop_sequence(ip, op[0], op[1])) {
523 		pr_err("Unexpected call sequence at %p: %x %x\n",
524 		ip, op[0], op[1]);
525 		return -EINVAL;
526 	}
527 
528 	/* If we never set up ftrace trampoline(s), then bail */
529 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
530 	if (!mod->arch.tramp || !mod->arch.tramp_regs) {
531 #else
532 	if (!mod->arch.tramp) {
533 #endif
534 		pr_err("No ftrace trampoline\n");
535 		return -EINVAL;
536 	}
537 
538 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
539 	if (rec->flags & FTRACE_FL_REGS)
540 		tramp = mod->arch.tramp_regs;
541 	else
542 #endif
543 		tramp = mod->arch.tramp;
544 
545 	if (module_trampoline_target(mod, tramp, &ptr)) {
546 		pr_err("Failed to get trampoline target\n");
547 		return -EFAULT;
548 	}
549 
550 	pr_devel("trampoline target %lx", ptr);
551 
552 	entry = ppc_global_function_entry((void *)addr);
553 	/* This should match what was called */
554 	if (ptr != entry) {
555 		pr_err("addr %lx does not match expected %lx\n", ptr, entry);
556 		return -EINVAL;
557 	}
558 
559 	/* Ensure branch is within 24 bits */
560 	if (!create_branch(ip, tramp, BRANCH_SET_LINK)) {
561 		pr_err("Branch out of range\n");
562 		return -EINVAL;
563 	}
564 
565 	if (patch_branch(ip, tramp, BRANCH_SET_LINK)) {
566 		pr_err("REL24 out of range!\n");
567 		return -EINVAL;
568 	}
569 
570 	return 0;
571 }
572 
573 #else  /* !CONFIG_PPC64: */
574 static int
575 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
576 {
577 	unsigned int op;
578 	unsigned long ip = rec->ip;
579 
580 	/* read where this goes */
581 	if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
582 		return -EFAULT;
583 
584 	/* It should be pointing to a nop */
585 	if (op != PPC_INST_NOP) {
586 		pr_err("Expected NOP but have %x\n", op);
587 		return -EINVAL;
588 	}
589 
590 	/* If we never set up a trampoline to ftrace_caller, then bail */
591 	if (!rec->arch.mod->arch.tramp) {
592 		pr_err("No ftrace trampoline\n");
593 		return -EINVAL;
594 	}
595 
596 	/* create the branch to the trampoline */
597 	op = create_branch((unsigned int *)ip,
598 			   rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
599 	if (!op) {
600 		pr_err("REL24 out of range!\n");
601 		return -EINVAL;
602 	}
603 
604 	pr_devel("write to %lx\n", rec->ip);
605 
606 	if (patch_instruction((unsigned int *)ip, op))
607 		return -EPERM;
608 
609 	return 0;
610 }
611 #endif /* CONFIG_PPC64 */
612 #endif /* CONFIG_MODULES */
613 
614 static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long addr)
615 {
616 	unsigned int op;
617 	void *ip = (void *)rec->ip;
618 	unsigned long tramp, entry, ptr;
619 
620 	/* Make sure we're being asked to patch branch to a known ftrace addr */
621 	entry = ppc_global_function_entry((void *)ftrace_caller);
622 	ptr = ppc_global_function_entry((void *)addr);
623 
624 	if (ptr != entry) {
625 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
626 		entry = ppc_global_function_entry((void *)ftrace_regs_caller);
627 		if (ptr != entry) {
628 #endif
629 			pr_err("Unknown ftrace addr to patch: %ps\n", (void *)ptr);
630 			return -EINVAL;
631 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
632 		}
633 #endif
634 	}
635 
636 	/* Make sure we have a nop */
637 	if (probe_kernel_read(&op, ip, sizeof(op))) {
638 		pr_err("Unable to read ftrace location %p\n", ip);
639 		return -EFAULT;
640 	}
641 
642 	if (op != PPC_INST_NOP) {
643 		pr_err("Unexpected call sequence at %p: %x\n", ip, op);
644 		return -EINVAL;
645 	}
646 
647 	tramp = find_ftrace_tramp((unsigned long)ip);
648 	if (!tramp) {
649 		pr_err("No ftrace trampolines reachable from %ps\n", ip);
650 		return -EINVAL;
651 	}
652 
653 	if (patch_branch(ip, tramp, BRANCH_SET_LINK)) {
654 		pr_err("Error patching branch to ftrace tramp!\n");
655 		return -EINVAL;
656 	}
657 
658 	return 0;
659 }
660 
661 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
662 {
663 	unsigned long ip = rec->ip;
664 	unsigned int old, new;
665 
666 	/*
667 	 * If the calling address is more that 24 bits away,
668 	 * then we had to use a trampoline to make the call.
669 	 * Otherwise just update the call site.
670 	 */
671 	if (test_24bit_addr(ip, addr)) {
672 		/* within range */
673 		old = PPC_INST_NOP;
674 		new = ftrace_call_replace(ip, addr, 1);
675 		return ftrace_modify_code(ip, old, new);
676 	} else if (core_kernel_text(ip))
677 		return __ftrace_make_call_kernel(rec, addr);
678 
679 #ifdef CONFIG_MODULES
680 	/*
681 	 * Out of range jumps are called from modules.
682 	 * Being that we are converting from nop, it had better
683 	 * already have a module defined.
684 	 */
685 	if (!rec->arch.mod) {
686 		pr_err("No module loaded\n");
687 		return -EINVAL;
688 	}
689 
690 	return __ftrace_make_call(rec, addr);
691 #else
692 	/* We should not get here without modules */
693 	return -EINVAL;
694 #endif /* CONFIG_MODULES */
695 }
696 
697 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
698 #ifdef CONFIG_MODULES
699 static int
700 __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
701 					unsigned long addr)
702 {
703 	unsigned int op;
704 	unsigned long ip = rec->ip;
705 	unsigned long entry, ptr, tramp;
706 	struct module *mod = rec->arch.mod;
707 
708 	/* If we never set up ftrace trampolines, then bail */
709 	if (!mod->arch.tramp || !mod->arch.tramp_regs) {
710 		pr_err("No ftrace trampoline\n");
711 		return -EINVAL;
712 	}
713 
714 	/* read where this goes */
715 	if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
716 		pr_err("Fetching opcode failed.\n");
717 		return -EFAULT;
718 	}
719 
720 	/* Make sure that that this is still a 24bit jump */
721 	if (!is_bl_op(op)) {
722 		pr_err("Not expected bl: opcode is %x\n", op);
723 		return -EINVAL;
724 	}
725 
726 	/* lets find where the pointer goes */
727 	tramp = find_bl_target(ip, op);
728 	entry = ppc_global_function_entry((void *)old_addr);
729 
730 	pr_devel("ip:%lx jumps to %lx", ip, tramp);
731 
732 	if (tramp != entry) {
733 		/* old_addr is not within range, so we must have used a trampoline */
734 		if (module_trampoline_target(mod, tramp, &ptr)) {
735 			pr_err("Failed to get trampoline target\n");
736 			return -EFAULT;
737 		}
738 
739 		pr_devel("trampoline target %lx", ptr);
740 
741 		/* This should match what was called */
742 		if (ptr != entry) {
743 			pr_err("addr %lx does not match expected %lx\n", ptr, entry);
744 			return -EINVAL;
745 		}
746 	}
747 
748 	/* The new target may be within range */
749 	if (test_24bit_addr(ip, addr)) {
750 		/* within range */
751 		if (patch_branch((unsigned int *)ip, addr, BRANCH_SET_LINK)) {
752 			pr_err("REL24 out of range!\n");
753 			return -EINVAL;
754 		}
755 
756 		return 0;
757 	}
758 
759 	if (rec->flags & FTRACE_FL_REGS)
760 		tramp = mod->arch.tramp_regs;
761 	else
762 		tramp = mod->arch.tramp;
763 
764 	if (module_trampoline_target(mod, tramp, &ptr)) {
765 		pr_err("Failed to get trampoline target\n");
766 		return -EFAULT;
767 	}
768 
769 	pr_devel("trampoline target %lx", ptr);
770 
771 	entry = ppc_global_function_entry((void *)addr);
772 	/* This should match what was called */
773 	if (ptr != entry) {
774 		pr_err("addr %lx does not match expected %lx\n", ptr, entry);
775 		return -EINVAL;
776 	}
777 
778 	/* Ensure branch is within 24 bits */
779 	if (!create_branch((unsigned int *)ip, tramp, BRANCH_SET_LINK)) {
780 		pr_err("Branch out of range\n");
781 		return -EINVAL;
782 	}
783 
784 	if (patch_branch((unsigned int *)ip, tramp, BRANCH_SET_LINK)) {
785 		pr_err("REL24 out of range!\n");
786 		return -EINVAL;
787 	}
788 
789 	return 0;
790 }
791 #endif
792 
793 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
794 			unsigned long addr)
795 {
796 	unsigned long ip = rec->ip;
797 	unsigned int old, new;
798 
799 	/*
800 	 * If the calling address is more that 24 bits away,
801 	 * then we had to use a trampoline to make the call.
802 	 * Otherwise just update the call site.
803 	 */
804 	if (test_24bit_addr(ip, addr) && test_24bit_addr(ip, old_addr)) {
805 		/* within range */
806 		old = ftrace_call_replace(ip, old_addr, 1);
807 		new = ftrace_call_replace(ip, addr, 1);
808 		return ftrace_modify_code(ip, old, new);
809 	} else if (core_kernel_text(ip)) {
810 		/*
811 		 * We always patch out of range locations to go to the regs
812 		 * variant, so there is nothing to do here
813 		 */
814 		return 0;
815 	}
816 
817 #ifdef CONFIG_MODULES
818 	/*
819 	 * Out of range jumps are called from modules.
820 	 */
821 	if (!rec->arch.mod) {
822 		pr_err("No module loaded\n");
823 		return -EINVAL;
824 	}
825 
826 	return __ftrace_modify_call(rec, old_addr, addr);
827 #else
828 	/* We should not get here without modules */
829 	return -EINVAL;
830 #endif /* CONFIG_MODULES */
831 }
832 #endif
833 
834 int ftrace_update_ftrace_func(ftrace_func_t func)
835 {
836 	unsigned long ip = (unsigned long)(&ftrace_call);
837 	unsigned int old, new;
838 	int ret;
839 
840 	old = *(unsigned int *)&ftrace_call;
841 	new = ftrace_call_replace(ip, (unsigned long)func, 1);
842 	ret = ftrace_modify_code(ip, old, new);
843 
844 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
845 	/* Also update the regs callback function */
846 	if (!ret) {
847 		ip = (unsigned long)(&ftrace_regs_call);
848 		old = *(unsigned int *)&ftrace_regs_call;
849 		new = ftrace_call_replace(ip, (unsigned long)func, 1);
850 		ret = ftrace_modify_code(ip, old, new);
851 	}
852 #endif
853 
854 	return ret;
855 }
856 
857 /*
858  * Use the default ftrace_modify_all_code, but without
859  * stop_machine().
860  */
861 void arch_ftrace_update_code(int command)
862 {
863 	ftrace_modify_all_code(command);
864 }
865 
866 #ifdef CONFIG_PPC64
867 #define PACATOC offsetof(struct paca_struct, kernel_toc)
868 
869 #define PPC_LO(v) ((v) & 0xffff)
870 #define PPC_HI(v) (((v) >> 16) & 0xffff)
871 #define PPC_HA(v) PPC_HI ((v) + 0x8000)
872 
873 extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[];
874 
875 int __init ftrace_dyn_arch_init(void)
876 {
877 	int i;
878 	unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init };
879 	u32 stub_insns[] = {
880 		0xe98d0000 | PACATOC,	/* ld      r12,PACATOC(r13)	*/
881 		0x3d8c0000,		/* addis   r12,r12,<high>	*/
882 		0x398c0000,		/* addi    r12,r12,<low>	*/
883 		0x7d8903a6,		/* mtctr   r12			*/
884 		0x4e800420,		/* bctr				*/
885 	};
886 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
887 	unsigned long addr = ppc_global_function_entry((void *)ftrace_regs_caller);
888 #else
889 	unsigned long addr = ppc_global_function_entry((void *)ftrace_caller);
890 #endif
891 	long reladdr = addr - kernel_toc_addr();
892 
893 	if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
894 		pr_err("Address of %ps out of range of kernel_toc.\n",
895 				(void *)addr);
896 		return -1;
897 	}
898 
899 	for (i = 0; i < 2; i++) {
900 		memcpy(tramp[i], stub_insns, sizeof(stub_insns));
901 		tramp[i][1] |= PPC_HA(reladdr);
902 		tramp[i][2] |= PPC_LO(reladdr);
903 		add_ftrace_tramp((unsigned long)tramp[i]);
904 	}
905 
906 	return 0;
907 }
908 #else
909 int __init ftrace_dyn_arch_init(void)
910 {
911 	return 0;
912 }
913 #endif
914 #endif /* CONFIG_DYNAMIC_FTRACE */
915 
916 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
917 
918 extern void ftrace_graph_call(void);
919 extern void ftrace_graph_stub(void);
920 
921 int ftrace_enable_ftrace_graph_caller(void)
922 {
923 	unsigned long ip = (unsigned long)(&ftrace_graph_call);
924 	unsigned long addr = (unsigned long)(&ftrace_graph_caller);
925 	unsigned long stub = (unsigned long)(&ftrace_graph_stub);
926 	unsigned int old, new;
927 
928 	old = ftrace_call_replace(ip, stub, 0);
929 	new = ftrace_call_replace(ip, addr, 0);
930 
931 	return ftrace_modify_code(ip, old, new);
932 }
933 
934 int ftrace_disable_ftrace_graph_caller(void)
935 {
936 	unsigned long ip = (unsigned long)(&ftrace_graph_call);
937 	unsigned long addr = (unsigned long)(&ftrace_graph_caller);
938 	unsigned long stub = (unsigned long)(&ftrace_graph_stub);
939 	unsigned int old, new;
940 
941 	old = ftrace_call_replace(ip, addr, 0);
942 	new = ftrace_call_replace(ip, stub, 0);
943 
944 	return ftrace_modify_code(ip, old, new);
945 }
946 
947 /*
948  * Hook the return address and push it in the stack of return addrs
949  * in current thread info. Return the address we want to divert to.
950  */
951 unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip)
952 {
953 	unsigned long return_hooker;
954 
955 	if (unlikely(ftrace_graph_is_dead()))
956 		goto out;
957 
958 	if (unlikely(atomic_read(&current->tracing_graph_pause)))
959 		goto out;
960 
961 	return_hooker = ppc_function_entry(return_to_handler);
962 
963 	if (!function_graph_enter(parent, ip, 0, NULL))
964 		parent = return_hooker;
965 out:
966 	return parent;
967 }
968 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
969 
970 #ifdef PPC64_ELF_ABI_v1
971 char *arch_ftrace_match_adjust(char *str, const char *search)
972 {
973 	if (str[0] == '.' && search[0] != '.')
974 		return str + 1;
975 	else
976 		return str;
977 }
978 #endif /* PPC64_ELF_ABI_v1 */
979