xref: /linux/arch/x86/kernel/ftrace.c (revision dd5b2498d845f925904cb2afabb6ba11bfc317c5)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Dynamic function tracing support.
4  *
5  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6  *
7  * Thanks goes to Ingo Molnar, for suggesting the idea.
8  * Mathieu Desnoyers, for suggesting postponing the modifications.
9  * Arjan van de Ven, for keeping me straight, and explaining to me
10  * the dangers of modifying code on the run.
11  */
12 
13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14 
15 #include <linux/spinlock.h>
16 #include <linux/hardirq.h>
17 #include <linux/uaccess.h>
18 #include <linux/ftrace.h>
19 #include <linux/percpu.h>
20 #include <linux/sched.h>
21 #include <linux/slab.h>
22 #include <linux/init.h>
23 #include <linux/list.h>
24 #include <linux/module.h>
25 
26 #include <trace/syscall.h>
27 
28 #include <asm/set_memory.h>
29 #include <asm/kprobes.h>
30 #include <asm/ftrace.h>
31 #include <asm/nops.h>
32 
33 #ifdef CONFIG_DYNAMIC_FTRACE
34 
35 int ftrace_arch_code_modify_prepare(void)
36 {
37 	set_kernel_text_rw();
38 	set_all_modules_text_rw();
39 	return 0;
40 }
41 
42 int ftrace_arch_code_modify_post_process(void)
43 {
44 	set_all_modules_text_ro();
45 	set_kernel_text_ro();
46 	return 0;
47 }
48 
49 union ftrace_code_union {
50 	char code[MCOUNT_INSN_SIZE];
51 	struct {
52 		unsigned char op;
53 		int offset;
54 	} __attribute__((packed));
55 };
56 
57 static int ftrace_calc_offset(long ip, long addr)
58 {
59 	return (int)(addr - ip);
60 }
61 
62 static unsigned char *
63 ftrace_text_replace(unsigned char op, unsigned long ip, unsigned long addr)
64 {
65 	static union ftrace_code_union calc;
66 
67 	calc.op		= op;
68 	calc.offset	= ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
69 
70 	return calc.code;
71 }
72 
73 static unsigned char *
74 ftrace_call_replace(unsigned long ip, unsigned long addr)
75 {
76 	return ftrace_text_replace(0xe8, ip, addr);
77 }
78 
79 static inline int
80 within(unsigned long addr, unsigned long start, unsigned long end)
81 {
82 	return addr >= start && addr < end;
83 }
84 
85 static unsigned long text_ip_addr(unsigned long ip)
86 {
87 	/*
88 	 * On x86_64, kernel text mappings are mapped read-only, so we use
89 	 * the kernel identity mapping instead of the kernel text mapping
90 	 * to modify the kernel text.
91 	 *
92 	 * For 32bit kernels, these mappings are same and we can use
93 	 * kernel identity mapping to modify code.
94 	 */
95 	if (within(ip, (unsigned long)_text, (unsigned long)_etext))
96 		ip = (unsigned long)__va(__pa_symbol(ip));
97 
98 	return ip;
99 }
100 
101 static const unsigned char *ftrace_nop_replace(void)
102 {
103 	return ideal_nops[NOP_ATOMIC5];
104 }
105 
106 static int
107 ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
108 		   unsigned const char *new_code)
109 {
110 	unsigned char replaced[MCOUNT_INSN_SIZE];
111 
112 	ftrace_expected = old_code;
113 
114 	/*
115 	 * Note:
116 	 * We are paranoid about modifying text, as if a bug was to happen, it
117 	 * could cause us to read or write to someplace that could cause harm.
118 	 * Carefully read and modify the code with probe_kernel_*(), and make
119 	 * sure what we read is what we expected it to be before modifying it.
120 	 */
121 
122 	/* read the text we want to modify */
123 	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
124 		return -EFAULT;
125 
126 	/* Make sure it is what we expect it to be */
127 	if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
128 		return -EINVAL;
129 
130 	ip = text_ip_addr(ip);
131 
132 	/* replace the text with the new text */
133 	if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
134 		return -EPERM;
135 
136 	sync_core();
137 
138 	return 0;
139 }
140 
141 int ftrace_make_nop(struct module *mod,
142 		    struct dyn_ftrace *rec, unsigned long addr)
143 {
144 	unsigned const char *new, *old;
145 	unsigned long ip = rec->ip;
146 
147 	old = ftrace_call_replace(ip, addr);
148 	new = ftrace_nop_replace();
149 
150 	/*
151 	 * On boot up, and when modules are loaded, the MCOUNT_ADDR
152 	 * is converted to a nop, and will never become MCOUNT_ADDR
153 	 * again. This code is either running before SMP (on boot up)
154 	 * or before the code will ever be executed (module load).
155 	 * We do not want to use the breakpoint version in this case,
156 	 * just modify the code directly.
157 	 */
158 	if (addr == MCOUNT_ADDR)
159 		return ftrace_modify_code_direct(rec->ip, old, new);
160 
161 	ftrace_expected = NULL;
162 
163 	/* Normal cases use add_brk_on_nop */
164 	WARN_ONCE(1, "invalid use of ftrace_make_nop");
165 	return -EINVAL;
166 }
167 
168 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
169 {
170 	unsigned const char *new, *old;
171 	unsigned long ip = rec->ip;
172 
173 	old = ftrace_nop_replace();
174 	new = ftrace_call_replace(ip, addr);
175 
176 	/* Should only be called when module is loaded */
177 	return ftrace_modify_code_direct(rec->ip, old, new);
178 }
179 
180 /*
181  * The modifying_ftrace_code is used to tell the breakpoint
182  * handler to call ftrace_int3_handler(). If it fails to
183  * call this handler for a breakpoint added by ftrace, then
184  * the kernel may crash.
185  *
186  * As atomic_writes on x86 do not need a barrier, we do not
187  * need to add smp_mb()s for this to work. It is also considered
188  * that we can not read the modifying_ftrace_code before
189  * executing the breakpoint. That would be quite remarkable if
190  * it could do that. Here's the flow that is required:
191  *
192  *   CPU-0                          CPU-1
193  *
194  * atomic_inc(mfc);
195  * write int3s
196  *				<trap-int3> // implicit (r)mb
197  *				if (atomic_read(mfc))
198  *					call ftrace_int3_handler()
199  *
200  * Then when we are finished:
201  *
202  * atomic_dec(mfc);
203  *
204  * If we hit a breakpoint that was not set by ftrace, it does not
205  * matter if ftrace_int3_handler() is called or not. It will
206  * simply be ignored. But it is crucial that a ftrace nop/caller
207  * breakpoint is handled. No other user should ever place a
208  * breakpoint on an ftrace nop/caller location. It must only
209  * be done by this code.
210  */
211 atomic_t modifying_ftrace_code __read_mostly;
212 
213 static int
214 ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
215 		   unsigned const char *new_code);
216 
217 /*
218  * Should never be called:
219  *  As it is only called by __ftrace_replace_code() which is called by
220  *  ftrace_replace_code() that x86 overrides, and by ftrace_update_code()
221  *  which is called to turn mcount into nops or nops into function calls
222  *  but not to convert a function from not using regs to one that uses
223  *  regs, which ftrace_modify_call() is for.
224  */
225 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
226 				 unsigned long addr)
227 {
228 	WARN_ON(1);
229 	ftrace_expected = NULL;
230 	return -EINVAL;
231 }
232 
233 static unsigned long ftrace_update_func;
234 
235 static int update_ftrace_func(unsigned long ip, void *new)
236 {
237 	unsigned char old[MCOUNT_INSN_SIZE];
238 	int ret;
239 
240 	memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);
241 
242 	ftrace_update_func = ip;
243 	/* Make sure the breakpoints see the ftrace_update_func update */
244 	smp_wmb();
245 
246 	/* See comment above by declaration of modifying_ftrace_code */
247 	atomic_inc(&modifying_ftrace_code);
248 
249 	ret = ftrace_modify_code(ip, old, new);
250 
251 	atomic_dec(&modifying_ftrace_code);
252 
253 	return ret;
254 }
255 
256 int ftrace_update_ftrace_func(ftrace_func_t func)
257 {
258 	unsigned long ip = (unsigned long)(&ftrace_call);
259 	unsigned char *new;
260 	int ret;
261 
262 	new = ftrace_call_replace(ip, (unsigned long)func);
263 	ret = update_ftrace_func(ip, new);
264 
265 	/* Also update the regs callback function */
266 	if (!ret) {
267 		ip = (unsigned long)(&ftrace_regs_call);
268 		new = ftrace_call_replace(ip, (unsigned long)func);
269 		ret = update_ftrace_func(ip, new);
270 	}
271 
272 	return ret;
273 }
274 
275 static nokprobe_inline int is_ftrace_caller(unsigned long ip)
276 {
277 	if (ip == ftrace_update_func)
278 		return 1;
279 
280 	return 0;
281 }
282 
283 /*
284  * A breakpoint was added to the code address we are about to
285  * modify, and this is the handle that will just skip over it.
286  * We are either changing a nop into a trace call, or a trace
287  * call to a nop. While the change is taking place, we treat
288  * it just like it was a nop.
289  */
290 int ftrace_int3_handler(struct pt_regs *regs)
291 {
292 	unsigned long ip;
293 
294 	if (WARN_ON_ONCE(!regs))
295 		return 0;
296 
297 	ip = regs->ip - 1;
298 	if (!ftrace_location(ip) && !is_ftrace_caller(ip))
299 		return 0;
300 
301 	regs->ip += MCOUNT_INSN_SIZE - 1;
302 
303 	return 1;
304 }
305 NOKPROBE_SYMBOL(ftrace_int3_handler);
306 
307 static int ftrace_write(unsigned long ip, const char *val, int size)
308 {
309 	ip = text_ip_addr(ip);
310 
311 	if (probe_kernel_write((void *)ip, val, size))
312 		return -EPERM;
313 
314 	return 0;
315 }
316 
317 static int add_break(unsigned long ip, const char *old)
318 {
319 	unsigned char replaced[MCOUNT_INSN_SIZE];
320 	unsigned char brk = BREAKPOINT_INSTRUCTION;
321 
322 	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
323 		return -EFAULT;
324 
325 	ftrace_expected = old;
326 
327 	/* Make sure it is what we expect it to be */
328 	if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
329 		return -EINVAL;
330 
331 	return ftrace_write(ip, &brk, 1);
332 }
333 
334 static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
335 {
336 	unsigned const char *old;
337 	unsigned long ip = rec->ip;
338 
339 	old = ftrace_call_replace(ip, addr);
340 
341 	return add_break(rec->ip, old);
342 }
343 
344 
345 static int add_brk_on_nop(struct dyn_ftrace *rec)
346 {
347 	unsigned const char *old;
348 
349 	old = ftrace_nop_replace();
350 
351 	return add_break(rec->ip, old);
352 }
353 
354 static int add_breakpoints(struct dyn_ftrace *rec, int enable)
355 {
356 	unsigned long ftrace_addr;
357 	int ret;
358 
359 	ftrace_addr = ftrace_get_addr_curr(rec);
360 
361 	ret = ftrace_test_record(rec, enable);
362 
363 	switch (ret) {
364 	case FTRACE_UPDATE_IGNORE:
365 		return 0;
366 
367 	case FTRACE_UPDATE_MAKE_CALL:
368 		/* converting nop to call */
369 		return add_brk_on_nop(rec);
370 
371 	case FTRACE_UPDATE_MODIFY_CALL:
372 	case FTRACE_UPDATE_MAKE_NOP:
373 		/* converting a call to a nop */
374 		return add_brk_on_call(rec, ftrace_addr);
375 	}
376 	return 0;
377 }
378 
379 /*
380  * On error, we need to remove breakpoints. This needs to
381  * be done caefully. If the address does not currently have a
382  * breakpoint, we know we are done. Otherwise, we look at the
383  * remaining 4 bytes of the instruction. If it matches a nop
384  * we replace the breakpoint with the nop. Otherwise we replace
385  * it with the call instruction.
386  */
387 static int remove_breakpoint(struct dyn_ftrace *rec)
388 {
389 	unsigned char ins[MCOUNT_INSN_SIZE];
390 	unsigned char brk = BREAKPOINT_INSTRUCTION;
391 	const unsigned char *nop;
392 	unsigned long ftrace_addr;
393 	unsigned long ip = rec->ip;
394 
395 	/* If we fail the read, just give up */
396 	if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
397 		return -EFAULT;
398 
399 	/* If this does not have a breakpoint, we are done */
400 	if (ins[0] != brk)
401 		return 0;
402 
403 	nop = ftrace_nop_replace();
404 
405 	/*
406 	 * If the last 4 bytes of the instruction do not match
407 	 * a nop, then we assume that this is a call to ftrace_addr.
408 	 */
409 	if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
410 		/*
411 		 * For extra paranoidism, we check if the breakpoint is on
412 		 * a call that would actually jump to the ftrace_addr.
413 		 * If not, don't touch the breakpoint, we make just create
414 		 * a disaster.
415 		 */
416 		ftrace_addr = ftrace_get_addr_new(rec);
417 		nop = ftrace_call_replace(ip, ftrace_addr);
418 
419 		if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)
420 			goto update;
421 
422 		/* Check both ftrace_addr and ftrace_old_addr */
423 		ftrace_addr = ftrace_get_addr_curr(rec);
424 		nop = ftrace_call_replace(ip, ftrace_addr);
425 
426 		ftrace_expected = nop;
427 
428 		if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
429 			return -EINVAL;
430 	}
431 
432  update:
433 	return ftrace_write(ip, nop, 1);
434 }
435 
436 static int add_update_code(unsigned long ip, unsigned const char *new)
437 {
438 	/* skip breakpoint */
439 	ip++;
440 	new++;
441 	return ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1);
442 }
443 
444 static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
445 {
446 	unsigned long ip = rec->ip;
447 	unsigned const char *new;
448 
449 	new = ftrace_call_replace(ip, addr);
450 	return add_update_code(ip, new);
451 }
452 
453 static int add_update_nop(struct dyn_ftrace *rec)
454 {
455 	unsigned long ip = rec->ip;
456 	unsigned const char *new;
457 
458 	new = ftrace_nop_replace();
459 	return add_update_code(ip, new);
460 }
461 
462 static int add_update(struct dyn_ftrace *rec, int enable)
463 {
464 	unsigned long ftrace_addr;
465 	int ret;
466 
467 	ret = ftrace_test_record(rec, enable);
468 
469 	ftrace_addr  = ftrace_get_addr_new(rec);
470 
471 	switch (ret) {
472 	case FTRACE_UPDATE_IGNORE:
473 		return 0;
474 
475 	case FTRACE_UPDATE_MODIFY_CALL:
476 	case FTRACE_UPDATE_MAKE_CALL:
477 		/* converting nop to call */
478 		return add_update_call(rec, ftrace_addr);
479 
480 	case FTRACE_UPDATE_MAKE_NOP:
481 		/* converting a call to a nop */
482 		return add_update_nop(rec);
483 	}
484 
485 	return 0;
486 }
487 
488 static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
489 {
490 	unsigned long ip = rec->ip;
491 	unsigned const char *new;
492 
493 	new = ftrace_call_replace(ip, addr);
494 
495 	return ftrace_write(ip, new, 1);
496 }
497 
498 static int finish_update_nop(struct dyn_ftrace *rec)
499 {
500 	unsigned long ip = rec->ip;
501 	unsigned const char *new;
502 
503 	new = ftrace_nop_replace();
504 
505 	return ftrace_write(ip, new, 1);
506 }
507 
508 static int finish_update(struct dyn_ftrace *rec, int enable)
509 {
510 	unsigned long ftrace_addr;
511 	int ret;
512 
513 	ret = ftrace_update_record(rec, enable);
514 
515 	ftrace_addr = ftrace_get_addr_new(rec);
516 
517 	switch (ret) {
518 	case FTRACE_UPDATE_IGNORE:
519 		return 0;
520 
521 	case FTRACE_UPDATE_MODIFY_CALL:
522 	case FTRACE_UPDATE_MAKE_CALL:
523 		/* converting nop to call */
524 		return finish_update_call(rec, ftrace_addr);
525 
526 	case FTRACE_UPDATE_MAKE_NOP:
527 		/* converting a call to a nop */
528 		return finish_update_nop(rec);
529 	}
530 
531 	return 0;
532 }
533 
534 static void do_sync_core(void *data)
535 {
536 	sync_core();
537 }
538 
539 static void run_sync(void)
540 {
541 	int enable_irqs;
542 
543 	/* No need to sync if there's only one CPU */
544 	if (num_online_cpus() == 1)
545 		return;
546 
547 	enable_irqs = irqs_disabled();
548 
549 	/* We may be called with interrupts disabled (on bootup). */
550 	if (enable_irqs)
551 		local_irq_enable();
552 	on_each_cpu(do_sync_core, NULL, 1);
553 	if (enable_irqs)
554 		local_irq_disable();
555 }
556 
557 void ftrace_replace_code(int enable)
558 {
559 	struct ftrace_rec_iter *iter;
560 	struct dyn_ftrace *rec;
561 	const char *report = "adding breakpoints";
562 	int count = 0;
563 	int ret;
564 
565 	for_ftrace_rec_iter(iter) {
566 		rec = ftrace_rec_iter_record(iter);
567 
568 		ret = add_breakpoints(rec, enable);
569 		if (ret)
570 			goto remove_breakpoints;
571 		count++;
572 	}
573 
574 	run_sync();
575 
576 	report = "updating code";
577 	count = 0;
578 
579 	for_ftrace_rec_iter(iter) {
580 		rec = ftrace_rec_iter_record(iter);
581 
582 		ret = add_update(rec, enable);
583 		if (ret)
584 			goto remove_breakpoints;
585 		count++;
586 	}
587 
588 	run_sync();
589 
590 	report = "removing breakpoints";
591 	count = 0;
592 
593 	for_ftrace_rec_iter(iter) {
594 		rec = ftrace_rec_iter_record(iter);
595 
596 		ret = finish_update(rec, enable);
597 		if (ret)
598 			goto remove_breakpoints;
599 		count++;
600 	}
601 
602 	run_sync();
603 
604 	return;
605 
606  remove_breakpoints:
607 	pr_warn("Failed on %s (%d):\n", report, count);
608 	ftrace_bug(ret, rec);
609 	for_ftrace_rec_iter(iter) {
610 		rec = ftrace_rec_iter_record(iter);
611 		/*
612 		 * Breakpoints are handled only when this function is in
613 		 * progress. The system could not work with them.
614 		 */
615 		if (remove_breakpoint(rec))
616 			BUG();
617 	}
618 	run_sync();
619 }
620 
621 static int
622 ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
623 		   unsigned const char *new_code)
624 {
625 	int ret;
626 
627 	ret = add_break(ip, old_code);
628 	if (ret)
629 		goto out;
630 
631 	run_sync();
632 
633 	ret = add_update_code(ip, new_code);
634 	if (ret)
635 		goto fail_update;
636 
637 	run_sync();
638 
639 	ret = ftrace_write(ip, new_code, 1);
640 	/*
641 	 * The breakpoint is handled only when this function is in progress.
642 	 * The system could not work if we could not remove it.
643 	 */
644 	BUG_ON(ret);
645  out:
646 	run_sync();
647 	return ret;
648 
649  fail_update:
650 	/* Also here the system could not work with the breakpoint */
651 	if (ftrace_write(ip, old_code, 1))
652 		BUG();
653 	goto out;
654 }
655 
656 void arch_ftrace_update_code(int command)
657 {
658 	/* See comment above by declaration of modifying_ftrace_code */
659 	atomic_inc(&modifying_ftrace_code);
660 
661 	ftrace_modify_all_code(command);
662 
663 	atomic_dec(&modifying_ftrace_code);
664 }
665 
666 int __init ftrace_dyn_arch_init(void)
667 {
668 	return 0;
669 }
670 
671 /* Currently only x86_64 supports dynamic trampolines */
672 #ifdef CONFIG_X86_64
673 
674 #ifdef CONFIG_MODULES
675 #include <linux/moduleloader.h>
676 /* Module allocation simplifies allocating memory for code */
677 static inline void *alloc_tramp(unsigned long size)
678 {
679 	return module_alloc(size);
680 }
681 static inline void tramp_free(void *tramp, int size)
682 {
683 	int npages = PAGE_ALIGN(size) >> PAGE_SHIFT;
684 
685 	set_memory_nx((unsigned long)tramp, npages);
686 	set_memory_rw((unsigned long)tramp, npages);
687 	module_memfree(tramp);
688 }
689 #else
690 /* Trampolines can only be created if modules are supported */
691 static inline void *alloc_tramp(unsigned long size)
692 {
693 	return NULL;
694 }
695 static inline void tramp_free(void *tramp, int size) { }
696 #endif
697 
698 /* Defined as markers to the end of the ftrace default trampolines */
699 extern void ftrace_regs_caller_end(void);
700 extern void ftrace_epilogue(void);
701 extern void ftrace_caller_op_ptr(void);
702 extern void ftrace_regs_caller_op_ptr(void);
703 
704 /* movq function_trace_op(%rip), %rdx */
705 /* 0x48 0x8b 0x15 <offset-to-ftrace_trace_op (4 bytes)> */
706 #define OP_REF_SIZE	7
707 
708 /*
709  * The ftrace_ops is passed to the function callback. Since the
710  * trampoline only services a single ftrace_ops, we can pass in
711  * that ops directly.
712  *
713  * The ftrace_op_code_union is used to create a pointer to the
714  * ftrace_ops that will be passed to the callback function.
715  */
716 union ftrace_op_code_union {
717 	char code[OP_REF_SIZE];
718 	struct {
719 		char op[3];
720 		int offset;
721 	} __attribute__((packed));
722 };
723 
724 #define RET_SIZE		1
725 
726 static unsigned long
727 create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
728 {
729 	unsigned long start_offset;
730 	unsigned long end_offset;
731 	unsigned long op_offset;
732 	unsigned long offset;
733 	unsigned long size;
734 	unsigned long retq;
735 	unsigned long *ptr;
736 	void *trampoline;
737 	void *ip;
738 	/* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */
739 	unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 };
740 	union ftrace_op_code_union op_ptr;
741 	int ret;
742 
743 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
744 		start_offset = (unsigned long)ftrace_regs_caller;
745 		end_offset = (unsigned long)ftrace_regs_caller_end;
746 		op_offset = (unsigned long)ftrace_regs_caller_op_ptr;
747 	} else {
748 		start_offset = (unsigned long)ftrace_caller;
749 		end_offset = (unsigned long)ftrace_epilogue;
750 		op_offset = (unsigned long)ftrace_caller_op_ptr;
751 	}
752 
753 	size = end_offset - start_offset;
754 
755 	/*
756 	 * Allocate enough size to store the ftrace_caller code,
757 	 * the iret , as well as the address of the ftrace_ops this
758 	 * trampoline is used for.
759 	 */
760 	trampoline = alloc_tramp(size + RET_SIZE + sizeof(void *));
761 	if (!trampoline)
762 		return 0;
763 
764 	*tramp_size = size + RET_SIZE + sizeof(void *);
765 
766 	/* Copy ftrace_caller onto the trampoline memory */
767 	ret = probe_kernel_read(trampoline, (void *)start_offset, size);
768 	if (WARN_ON(ret < 0))
769 		goto fail;
770 
771 	ip = trampoline + size;
772 
773 	/* The trampoline ends with ret(q) */
774 	retq = (unsigned long)ftrace_stub;
775 	ret = probe_kernel_read(ip, (void *)retq, RET_SIZE);
776 	if (WARN_ON(ret < 0))
777 		goto fail;
778 
779 	/*
780 	 * The address of the ftrace_ops that is used for this trampoline
781 	 * is stored at the end of the trampoline. This will be used to
782 	 * load the third parameter for the callback. Basically, that
783 	 * location at the end of the trampoline takes the place of
784 	 * the global function_trace_op variable.
785 	 */
786 
787 	ptr = (unsigned long *)(trampoline + size + RET_SIZE);
788 	*ptr = (unsigned long)ops;
789 
790 	op_offset -= start_offset;
791 	memcpy(&op_ptr, trampoline + op_offset, OP_REF_SIZE);
792 
793 	/* Are we pointing to the reference? */
794 	if (WARN_ON(memcmp(op_ptr.op, op_ref, 3) != 0))
795 		goto fail;
796 
797 	/* Load the contents of ptr into the callback parameter */
798 	offset = (unsigned long)ptr;
799 	offset -= (unsigned long)trampoline + op_offset + OP_REF_SIZE;
800 
801 	op_ptr.offset = offset;
802 
803 	/* put in the new offset to the ftrace_ops */
804 	memcpy(trampoline + op_offset, &op_ptr, OP_REF_SIZE);
805 
806 	/* ALLOC_TRAMP flags lets us know we created it */
807 	ops->flags |= FTRACE_OPS_FL_ALLOC_TRAMP;
808 
809 	return (unsigned long)trampoline;
810 fail:
811 	tramp_free(trampoline, *tramp_size);
812 	return 0;
813 }
814 
815 static unsigned long calc_trampoline_call_offset(bool save_regs)
816 {
817 	unsigned long start_offset;
818 	unsigned long call_offset;
819 
820 	if (save_regs) {
821 		start_offset = (unsigned long)ftrace_regs_caller;
822 		call_offset = (unsigned long)ftrace_regs_call;
823 	} else {
824 		start_offset = (unsigned long)ftrace_caller;
825 		call_offset = (unsigned long)ftrace_call;
826 	}
827 
828 	return call_offset - start_offset;
829 }
830 
831 void arch_ftrace_update_trampoline(struct ftrace_ops *ops)
832 {
833 	ftrace_func_t func;
834 	unsigned char *new;
835 	unsigned long offset;
836 	unsigned long ip;
837 	unsigned int size;
838 	int ret, npages;
839 
840 	if (ops->trampoline) {
841 		/*
842 		 * The ftrace_ops caller may set up its own trampoline.
843 		 * In such a case, this code must not modify it.
844 		 */
845 		if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
846 			return;
847 		npages = PAGE_ALIGN(ops->trampoline_size) >> PAGE_SHIFT;
848 		set_memory_rw(ops->trampoline, npages);
849 	} else {
850 		ops->trampoline = create_trampoline(ops, &size);
851 		if (!ops->trampoline)
852 			return;
853 		ops->trampoline_size = size;
854 		npages = PAGE_ALIGN(size) >> PAGE_SHIFT;
855 	}
856 
857 	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);
858 	ip = ops->trampoline + offset;
859 
860 	func = ftrace_ops_get_func(ops);
861 
862 	/* Do a safe modify in case the trampoline is executing */
863 	new = ftrace_call_replace(ip, (unsigned long)func);
864 	ret = update_ftrace_func(ip, new);
865 	set_memory_ro(ops->trampoline, npages);
866 
867 	/* The update should never fail */
868 	WARN_ON(ret);
869 }
870 
871 /* Return the address of the function the trampoline calls */
872 static void *addr_from_call(void *ptr)
873 {
874 	union ftrace_code_union calc;
875 	int ret;
876 
877 	ret = probe_kernel_read(&calc, ptr, MCOUNT_INSN_SIZE);
878 	if (WARN_ON_ONCE(ret < 0))
879 		return NULL;
880 
881 	/* Make sure this is a call */
882 	if (WARN_ON_ONCE(calc.op != 0xe8)) {
883 		pr_warn("Expected e8, got %x\n", calc.op);
884 		return NULL;
885 	}
886 
887 	return ptr + MCOUNT_INSN_SIZE + calc.offset;
888 }
889 
890 void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
891 			   unsigned long frame_pointer);
892 
893 /*
894  * If the ops->trampoline was not allocated, then it probably
895  * has a static trampoline func, or is the ftrace caller itself.
896  */
897 static void *static_tramp_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
898 {
899 	unsigned long offset;
900 	bool save_regs = rec->flags & FTRACE_FL_REGS_EN;
901 	void *ptr;
902 
903 	if (ops && ops->trampoline) {
904 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
905 		/*
906 		 * We only know about function graph tracer setting as static
907 		 * trampoline.
908 		 */
909 		if (ops->trampoline == FTRACE_GRAPH_ADDR)
910 			return (void *)prepare_ftrace_return;
911 #endif
912 		return NULL;
913 	}
914 
915 	offset = calc_trampoline_call_offset(save_regs);
916 
917 	if (save_regs)
918 		ptr = (void *)FTRACE_REGS_ADDR + offset;
919 	else
920 		ptr = (void *)FTRACE_ADDR + offset;
921 
922 	return addr_from_call(ptr);
923 }
924 
925 void *arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
926 {
927 	unsigned long offset;
928 
929 	/* If we didn't allocate this trampoline, consider it static */
930 	if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
931 		return static_tramp_func(ops, rec);
932 
933 	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);
934 	return addr_from_call((void *)ops->trampoline + offset);
935 }
936 
937 void arch_ftrace_trampoline_free(struct ftrace_ops *ops)
938 {
939 	if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
940 		return;
941 
942 	tramp_free((void *)ops->trampoline, ops->trampoline_size);
943 	ops->trampoline = 0;
944 }
945 
946 #endif /* CONFIG_X86_64 */
947 #endif /* CONFIG_DYNAMIC_FTRACE */
948 
949 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
950 
951 #ifdef CONFIG_DYNAMIC_FTRACE
952 extern void ftrace_graph_call(void);
953 
954 static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
955 {
956 	return ftrace_text_replace(0xe9, ip, addr);
957 }
958 
959 static int ftrace_mod_jmp(unsigned long ip, void *func)
960 {
961 	unsigned char *new;
962 
963 	new = ftrace_jmp_replace(ip, (unsigned long)func);
964 
965 	return update_ftrace_func(ip, new);
966 }
967 
968 int ftrace_enable_ftrace_graph_caller(void)
969 {
970 	unsigned long ip = (unsigned long)(&ftrace_graph_call);
971 
972 	return ftrace_mod_jmp(ip, &ftrace_graph_caller);
973 }
974 
975 int ftrace_disable_ftrace_graph_caller(void)
976 {
977 	unsigned long ip = (unsigned long)(&ftrace_graph_call);
978 
979 	return ftrace_mod_jmp(ip, &ftrace_stub);
980 }
981 
982 #endif /* !CONFIG_DYNAMIC_FTRACE */
983 
984 /*
985  * Hook the return address and push it in the stack of return addrs
986  * in current thread info.
987  */
988 void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
989 			   unsigned long frame_pointer)
990 {
991 	unsigned long old;
992 	int faulted;
993 	unsigned long return_hooker = (unsigned long)
994 				&return_to_handler;
995 
996 	/*
997 	 * When resuming from suspend-to-ram, this function can be indirectly
998 	 * called from early CPU startup code while the CPU is in real mode,
999 	 * which would fail miserably.  Make sure the stack pointer is a
1000 	 * virtual address.
1001 	 *
1002 	 * This check isn't as accurate as virt_addr_valid(), but it should be
1003 	 * good enough for this purpose, and it's fast.
1004 	 */
1005 	if (unlikely((long)__builtin_frame_address(0) >= 0))
1006 		return;
1007 
1008 	if (unlikely(ftrace_graph_is_dead()))
1009 		return;
1010 
1011 	if (unlikely(atomic_read(&current->tracing_graph_pause)))
1012 		return;
1013 
1014 	/*
1015 	 * Protect against fault, even if it shouldn't
1016 	 * happen. This tool is too much intrusive to
1017 	 * ignore such a protection.
1018 	 */
1019 	asm volatile(
1020 		"1: " _ASM_MOV " (%[parent]), %[old]\n"
1021 		"2: " _ASM_MOV " %[return_hooker], (%[parent])\n"
1022 		"   movl $0, %[faulted]\n"
1023 		"3:\n"
1024 
1025 		".section .fixup, \"ax\"\n"
1026 		"4: movl $1, %[faulted]\n"
1027 		"   jmp 3b\n"
1028 		".previous\n"
1029 
1030 		_ASM_EXTABLE(1b, 4b)
1031 		_ASM_EXTABLE(2b, 4b)
1032 
1033 		: [old] "=&r" (old), [faulted] "=r" (faulted)
1034 		: [parent] "r" (parent), [return_hooker] "r" (return_hooker)
1035 		: "memory"
1036 	);
1037 
1038 	if (unlikely(faulted)) {
1039 		ftrace_graph_stop();
1040 		WARN_ON(1);
1041 		return;
1042 	}
1043 
1044 	if (function_graph_enter(old, self_addr, frame_pointer, parent))
1045 		*parent = old;
1046 }
1047 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1048