1 // SPDX-License-Identifier: GPL-2.0-only
15 #include <asm/debug-monitors.h>
18 #include <asm/text-patching.h>
23 	int offset;  member
29 	.offset = offsetof(struct __arch_ftrace_regs, field),	\
55 		if (!strcmp(roff->name, name))  in ftrace_regs_query_register_offset()
56 			return roff->offset;  in ftrace_regs_query_register_offset()
59 	return -EINVAL;  in ftrace_regs_query_register_offset()
73 	 * When using patchable-function-entry without pre-function NOPS, addr  in ftrace_call_adjust()
79 	 * addr+04:		NOP		// To be patched to BL <caller>  in ftrace_call_adjust()
83 	 * addr-04:		BTI	C  in ftrace_call_adjust()
85 	 * addr+04:		NOP		// To be patched to BL <caller>  in ftrace_call_adjust()
88 	 * to `BL <caller>`, which is at `addr + 4` bytes in either case.  in ftrace_call_adjust()
95 	 * When using patchable-function-entry with pre-function NOPs, addr is  in ftrace_call_adjust()
96 	 * the address of the first pre-function NOP.  in ftrace_call_adjust()
98 	 * Starting from an 8-byte aligned base, the compiler has either  in ftrace_call_adjust()
104 	 * addr+12:		NOP		// To be patched to BL <caller>  in ftrace_call_adjust()
112 	 * addr+16:		NOP		// To be patched to BL <caller>  in ftrace_call_adjust()
115 	 * to `BL <caller>`, which is at either addr+12 or addr+16 depending on  in ftrace_call_adjust()
120 		WARN_RATELIMIT(1, "Misaligned patch-site %pS\n",  in ftrace_call_adjust()
135 			WARN_RATELIMIT(1, "unexpected insn in patch-site %pS: 0x%08x\n",  in ftrace_call_adjust()
152 	 * When using patchable-function-entry without pre-function NOPS, ftrace  in arch_ftrace_get_symaddr()
158 	 * func+04:		NOP		// To be patched to BL <caller>  in arch_ftrace_get_symaddr()
162 	 * func-04:		BTI	C  in arch_ftrace_get_symaddr()
164 	 * func+04:		NOP		// To be patched to BL <caller>  in arch_ftrace_get_symaddr()
166 	 * The fentry_ip is the address of `BL <caller>` which is at `func + 4`  in arch_ftrace_get_symaddr()
170 		return fentry_ip - AARCH64_INSN_SIZE;  in arch_ftrace_get_symaddr()
173 	 * When using patchable-function-entry with pre-function NOPs, BTI is  in arch_ftrace_get_symaddr()
177 	 * func+04:		NOP		// To be patched to BL <caller>  in arch_ftrace_get_symaddr()
183 	 * func+08:		NOP		// To be patched to BL <caller>  in arch_ftrace_get_symaddr()
185 	 * The fentry_ip is the address of `BL <caller>` which is at either  in arch_ftrace_get_symaddr()
191 		return fentry_ip - AARCH64_INSN_SIZE;  in arch_ftrace_get_symaddr()
197 		if (get_kernel_nofault(insn, (u32 *)(fentry_ip - AARCH64_INSN_SIZE * 2)))  in arch_ftrace_get_symaddr()
200 		insn = *(u32 *)(fentry_ip - AARCH64_INSN_SIZE * 2);  in arch_ftrace_get_symaddr()
204 		return fentry_ip - AARCH64_INSN_SIZE * 2;  in arch_ftrace_get_symaddr()
206 	return fentry_ip - AARCH64_INSN_SIZE;  in arch_ftrace_get_symaddr()
222 	 * Carefully read and modify the code with aarch64_insn_*() which uses  in ftrace_modify_code()
228 			return -EFAULT;  in ftrace_modify_code()
231 			return -EINVAL;  in ftrace_modify_code()
234 		return -EPERM;  in ftrace_modify_code()
267 		plt = mod->arch.init_ftrace_trampolines;  in get_ftrace_plt()
269 		plt = mod->arch.ftrace_trampolines;  in get_ftrace_plt()
281 	long offset = (long)addr - (long)pc;  in reachable_by_bl()  local
283 	return offset >= -SZ_128M && offset < SZ_128M;  in reachable_by_bl()
289  * Due to the limited range of 'BL' instructions, modules may be placed too far
299 	unsigned long pc = rec->ip;  in ftrace_find_callable_addr()
305 	 * through ops->direct_call.  in ftrace_find_callable_addr()
311 	 * When the target is within range of the 'BL' instruction, use 'addr'  in ftrace_find_callable_addr()
312 	 * as-is and branch to that directly.  in ftrace_find_callable_addr()
318 	 * When the target is outside of the range of a 'BL' instruction, we  in ftrace_find_callable_addr()
320 	 * only when module PLT support is built-in.  in ftrace_find_callable_addr()
327 	 * dealing with an out-of-range condition, we can assume it  in ftrace_find_callable_addr()
332 	 * retains its validity throughout the remainder of this code.  in ftrace_find_callable_addr()
357 	if (rec->flags & FTRACE_FL_CALL_OPS_EN) {  in arm64_rec_get_ops()
371 	unsigned long literal = ALIGN_DOWN(rec->ip - 12, 8);  in ftrace_rec_set_ops()
395 	unsigned long pc = rec->ip;  in ftrace_make_call()
404 		return -EINVAL;  in ftrace_make_call()
416 	unsigned long pc = rec->ip;  in ftrace_modify_call()
425 		return -EINVAL;  in ftrace_modify_call()
427 		return -EINVAL;  in ftrace_modify_call()
440  * All instrumented functions follow the AAPCS, so x0-x8 and x19-x30 are live,
441  * and x9-x18 are free for our use.
443  * At runtime we want to be able to swing a single NOP <-> BL to enable or
444  * disable the ftrace call. The BL requires us to save the original LR value,
449  * +----------+------------+------------+
451  * | NOP      | NOP        | BL <entry> |
457  * Note: ftrace_process_locs() has pre-adjusted rec->ip to be the address of
458  * the BL.
462 	unsigned long pc = rec->ip - AARCH64_INSN_SIZE;  in ftrace_init_nop()
484 	unsigned long pc = rec->ip;  in ftrace_make_nop()
507 		return -EINVAL;  in ftrace_make_nop()
533 	if (unlikely(atomic_read(&current->tracing_graph_pause)))  in prepare_ftrace_return()
554 	unsigned long frame_pointer = arch_ftrace_regs(fregs)->fp;  in ftrace_graph_func()
555 	unsigned long *parent = &arch_ftrace_regs(fregs)->lr;  in ftrace_graph_func()
558 	if (unlikely(atomic_read(&current->tracing_graph_pause)))  in ftrace_graph_func()