1014c257cSAbhishek Sagar /* 2014c257cSAbhishek Sagar * Dynamic function tracing support. 3014c257cSAbhishek Sagar * 4014c257cSAbhishek Sagar * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com> 53b6c223bSRabin Vincent * Copyright (C) 2010 Rabin Vincent <rabin@rab.in> 6014c257cSAbhishek Sagar * 7014c257cSAbhishek Sagar * For licencing details, see COPYING. 8014c257cSAbhishek Sagar * 9014c257cSAbhishek Sagar * Defines low-level handling of mcount calls when the kernel 10014c257cSAbhishek Sagar * is compiled with the -pg flag. When using dynamic ftrace, the 113b6c223bSRabin Vincent * mcount call-sites get patched with NOP till they are enabled. 123b6c223bSRabin Vincent * All code mutation routines here are called under stop_machine(). 13014c257cSAbhishek Sagar */ 14014c257cSAbhishek Sagar 15014c257cSAbhishek Sagar #include <linux/ftrace.h> 163b6c223bSRabin Vincent #include <linux/uaccess.h> 17a672917aSRabin Vincent #include <linux/module.h> 1880d6b0c2SKees Cook #include <linux/stop_machine.h> 19014c257cSAbhishek Sagar 20395a59d0SAbhishek Sagar #include <asm/cacheflush.h> 214394e282SRabin Vincent #include <asm/opcodes.h> 22395a59d0SAbhishek Sagar #include <asm/ftrace.h> 230dc016dbSWang Nan #include <asm/insn.h> 2474d86a70SLaura Abbott #include <asm/set_memory.h> 2541918ec8SArd Biesheuvel #include <asm/stacktrace.h> 265a735583SPeter Zijlstra #include <asm/patch.h> 27d82227cfSRabin Vincent 28ad1c2f39SArd Biesheuvel /* 29ad1c2f39SArd Biesheuvel * The compiler emitted profiling hook consists of 30ad1c2f39SArd Biesheuvel * 31ad1c2f39SArd Biesheuvel * PUSH {LR} 32ad1c2f39SArd Biesheuvel * BL __gnu_mcount_nc 33ad1c2f39SArd Biesheuvel * 34ad1c2f39SArd Biesheuvel * To turn this combined sequence into a NOP, we need to restore the value of 35ad1c2f39SArd Biesheuvel * SP before the PUSH. Let's use an ADD rather than a POP into LR, as LR is not 36ad1c2f39SArd Biesheuvel * modified anyway, and reloading LR from memory is highly likely to be less 37ad1c2f39SArd Biesheuvel * efficient. 38ad1c2f39SArd Biesheuvel */ 3972dc43a9SRabin Vincent #ifdef CONFIG_THUMB2_KERNEL 40ad1c2f39SArd Biesheuvel #define NOP 0xf10d0d04 /* add.w sp, sp, #4 */ 4172dc43a9SRabin Vincent #else 42ad1c2f39SArd Biesheuvel #define NOP 0xe28dd004 /* add sp, sp, #4 */ 4372dc43a9SRabin Vincent #endif 44014c257cSAbhishek Sagar 45376cfa87STim Bird #ifdef CONFIG_DYNAMIC_FTRACE 463b6c223bSRabin Vincent 4780d6b0c2SKees Cook static int __ftrace_modify_code(void *data) 4880d6b0c2SKees Cook { 4980d6b0c2SKees Cook int *command = data; 5080d6b0c2SKees Cook 5180d6b0c2SKees Cook ftrace_modify_all_code(*command); 5280d6b0c2SKees Cook 5380d6b0c2SKees Cook return 0; 5480d6b0c2SKees Cook } 5580d6b0c2SKees Cook 5680d6b0c2SKees Cook void arch_ftrace_update_code(int command) 5780d6b0c2SKees Cook { 5880d6b0c2SKees Cook stop_machine(__ftrace_modify_code, &command, NULL); 5980d6b0c2SKees Cook } 6080d6b0c2SKees Cook 613b6c223bSRabin Vincent static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) 623b6c223bSRabin Vincent { 633b6c223bSRabin Vincent return NOP; 643b6c223bSRabin Vincent } 653b6c223bSRabin Vincent 66dc438db5SArd Biesheuvel void ftrace_caller_from_init(void); 67dc438db5SArd Biesheuvel void ftrace_regs_caller_from_init(void); 68dc438db5SArd Biesheuvel 69dc438db5SArd Biesheuvel static unsigned long __ref adjust_address(struct dyn_ftrace *rec, 70dc438db5SArd Biesheuvel unsigned long addr) 713b6c223bSRabin Vincent { 72dc438db5SArd Biesheuvel if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE) || 73dc438db5SArd Biesheuvel system_state >= SYSTEM_FREEING_INITMEM || 74dc438db5SArd Biesheuvel likely(!is_kernel_inittext(rec->ip))) 753b6c223bSRabin Vincent return addr; 76dc438db5SArd Biesheuvel if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) || 77dc438db5SArd Biesheuvel addr == (unsigned long)&ftrace_caller) 78dc438db5SArd Biesheuvel return (unsigned long)&ftrace_caller_from_init; 79dc438db5SArd Biesheuvel return (unsigned long)&ftrace_regs_caller_from_init; 803b6c223bSRabin Vincent } 813b6c223bSRabin Vincent 823a2bfec0SLi kunyu void ftrace_arch_code_modify_prepare(void) 83a672917aSRabin Vincent { 84a672917aSRabin Vincent } 85a672917aSRabin Vincent 863a2bfec0SLi kunyu void ftrace_arch_code_modify_post_process(void) 87a672917aSRabin Vincent { 8880d6b0c2SKees Cook /* Make sure any TLB misses during machine stop are cleared. */ 8980d6b0c2SKees Cook flush_tlb_all(); 90a672917aSRabin Vincent } 91a672917aSRabin Vincent 9279f32b22SAlex Sverdlin static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr, 9379f32b22SAlex Sverdlin bool warn) 94dd686eb1SRabin Vincent { 9579f32b22SAlex Sverdlin return arm_gen_branch_link(pc, addr, warn); 96dd686eb1SRabin Vincent } 97dd686eb1SRabin Vincent 983b6c223bSRabin Vincent static int ftrace_modify_code(unsigned long pc, unsigned long old, 99dc283d70SRabin Vincent unsigned long new, bool validate) 100014c257cSAbhishek Sagar { 1013b6c223bSRabin Vincent unsigned long replaced; 102014c257cSAbhishek Sagar 103be993e44SPeter Zijlstra if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) 1044394e282SRabin Vincent old = __opcode_to_mem_thumb32(old); 105be993e44SPeter Zijlstra else 1064394e282SRabin Vincent old = __opcode_to_mem_arm(old); 1074394e282SRabin Vincent 108dc283d70SRabin Vincent if (validate) { 109fe557319SChristoph Hellwig if (copy_from_kernel_nofault(&replaced, (void *)pc, 110fe557319SChristoph Hellwig MCOUNT_INSN_SIZE)) 1113b6c223bSRabin Vincent return -EFAULT; 112014c257cSAbhishek Sagar 1133b6c223bSRabin Vincent if (replaced != old) 1143b6c223bSRabin Vincent return -EINVAL; 115dc283d70SRabin Vincent } 116014c257cSAbhishek Sagar 1175a735583SPeter Zijlstra __patch_text((void *)pc, new); 118014c257cSAbhishek Sagar 1193b6c223bSRabin Vincent return 0; 120014c257cSAbhishek Sagar } 121014c257cSAbhishek Sagar 122014c257cSAbhishek Sagar int ftrace_update_ftrace_func(ftrace_func_t func) 123014c257cSAbhishek Sagar { 124dc283d70SRabin Vincent unsigned long pc; 1253b6c223bSRabin Vincent unsigned long new; 1263b6c223bSRabin Vincent int ret; 127014c257cSAbhishek Sagar 128014c257cSAbhishek Sagar pc = (unsigned long)&ftrace_call; 12979f32b22SAlex Sverdlin new = ftrace_call_replace(pc, (unsigned long)func, true); 1303b6c223bSRabin Vincent 131dc283d70SRabin Vincent ret = ftrace_modify_code(pc, 0, new, false); 1323b6c223bSRabin Vincent 133620176f3SAbel Vesa #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 134620176f3SAbel Vesa if (!ret) { 135620176f3SAbel Vesa pc = (unsigned long)&ftrace_regs_call; 13679f32b22SAlex Sverdlin new = ftrace_call_replace(pc, (unsigned long)func, true); 137620176f3SAbel Vesa 138620176f3SAbel Vesa ret = ftrace_modify_code(pc, 0, new, false); 139620176f3SAbel Vesa } 140620176f3SAbel Vesa #endif 141620176f3SAbel Vesa 142014c257cSAbhishek Sagar return ret; 143014c257cSAbhishek Sagar } 144014c257cSAbhishek Sagar 1453b6c223bSRabin Vincent int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 1463b6c223bSRabin Vincent { 1473b6c223bSRabin Vincent unsigned long new, old; 1483b6c223bSRabin Vincent unsigned long ip = rec->ip; 14979f32b22SAlex Sverdlin unsigned long aaddr = adjust_address(rec, addr); 15079f32b22SAlex Sverdlin struct module *mod = NULL; 15179f32b22SAlex Sverdlin 15279f32b22SAlex Sverdlin #ifdef CONFIG_ARM_MODULE_PLTS 15379f32b22SAlex Sverdlin mod = rec->arch.mod; 15479f32b22SAlex Sverdlin #endif 1553b6c223bSRabin Vincent 1563b6c223bSRabin Vincent old = ftrace_nop_replace(rec); 157620176f3SAbel Vesa 15879f32b22SAlex Sverdlin new = ftrace_call_replace(ip, aaddr, !mod); 15979f32b22SAlex Sverdlin #ifdef CONFIG_ARM_MODULE_PLTS 16079f32b22SAlex Sverdlin if (!new && mod) { 16179f32b22SAlex Sverdlin aaddr = get_module_plt(mod, ip, aaddr); 16279f32b22SAlex Sverdlin new = ftrace_call_replace(ip, aaddr, true); 16379f32b22SAlex Sverdlin } 16479f32b22SAlex Sverdlin #endif 1653b6c223bSRabin Vincent 166dc283d70SRabin Vincent return ftrace_modify_code(rec->ip, old, new, true); 1673b6c223bSRabin Vincent } 1683b6c223bSRabin Vincent 169620176f3SAbel Vesa #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 170620176f3SAbel Vesa 171620176f3SAbel Vesa int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 172620176f3SAbel Vesa unsigned long addr) 173620176f3SAbel Vesa { 174620176f3SAbel Vesa unsigned long new, old; 175620176f3SAbel Vesa unsigned long ip = rec->ip; 176620176f3SAbel Vesa 17779f32b22SAlex Sverdlin old = ftrace_call_replace(ip, adjust_address(rec, old_addr), true); 178620176f3SAbel Vesa 17979f32b22SAlex Sverdlin new = ftrace_call_replace(ip, adjust_address(rec, addr), true); 180620176f3SAbel Vesa 181620176f3SAbel Vesa return ftrace_modify_code(rec->ip, old, new, true); 182620176f3SAbel Vesa } 183620176f3SAbel Vesa 184620176f3SAbel Vesa #endif 185620176f3SAbel Vesa 1863b6c223bSRabin Vincent int ftrace_make_nop(struct module *mod, 1873b6c223bSRabin Vincent struct dyn_ftrace *rec, unsigned long addr) 1883b6c223bSRabin Vincent { 18979f32b22SAlex Sverdlin unsigned long aaddr = adjust_address(rec, addr); 1903b6c223bSRabin Vincent unsigned long ip = rec->ip; 1913b6c223bSRabin Vincent unsigned long old; 1923b6c223bSRabin Vincent unsigned long new; 1933b6c223bSRabin Vincent int ret; 1943b6c223bSRabin Vincent 19579f32b22SAlex Sverdlin #ifdef CONFIG_ARM_MODULE_PLTS 19679f32b22SAlex Sverdlin /* mod is only supplied during module loading */ 19779f32b22SAlex Sverdlin if (!mod) 19879f32b22SAlex Sverdlin mod = rec->arch.mod; 19979f32b22SAlex Sverdlin else 20079f32b22SAlex Sverdlin rec->arch.mod = mod; 20179f32b22SAlex Sverdlin #endif 20279f32b22SAlex Sverdlin 20379f32b22SAlex Sverdlin old = ftrace_call_replace(ip, aaddr, 20479f32b22SAlex Sverdlin !IS_ENABLED(CONFIG_ARM_MODULE_PLTS) || !mod); 20579f32b22SAlex Sverdlin #ifdef CONFIG_ARM_MODULE_PLTS 20679f32b22SAlex Sverdlin if (!old && mod) { 20779f32b22SAlex Sverdlin aaddr = get_module_plt(mod, ip, aaddr); 20879f32b22SAlex Sverdlin old = ftrace_call_replace(ip, aaddr, true); 20979f32b22SAlex Sverdlin } 21079f32b22SAlex Sverdlin #endif 21179f32b22SAlex Sverdlin 2123b6c223bSRabin Vincent new = ftrace_nop_replace(rec); 213dc438db5SArd Biesheuvel /* 214dc438db5SArd Biesheuvel * Locations in .init.text may call __gnu_mcount_mc via a linker 215dc438db5SArd Biesheuvel * emitted veneer if they are too far away from its implementation, and 216dc438db5SArd Biesheuvel * so validation may fail spuriously in such cases. Let's work around 217dc438db5SArd Biesheuvel * this by omitting those from validation. 218dc438db5SArd Biesheuvel */ 219dc438db5SArd Biesheuvel ret = ftrace_modify_code(ip, old, new, !is_kernel_inittext(ip)); 2203b6c223bSRabin Vincent 2213b6c223bSRabin Vincent return ret; 2223b6c223bSRabin Vincent } 223376cfa87STim Bird #endif /* CONFIG_DYNAMIC_FTRACE */ 224376cfa87STim Bird 225376cfa87STim Bird #ifdef CONFIG_FUNCTION_GRAPH_TRACER 22641918ec8SArd Biesheuvel asmlinkage 227376cfa87STim Bird void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, 22841918ec8SArd Biesheuvel unsigned long frame_pointer, 22941918ec8SArd Biesheuvel unsigned long stack_pointer) 230376cfa87STim Bird { 231376cfa87STim Bird unsigned long return_hooker = (unsigned long) &return_to_handler; 232376cfa87STim Bird unsigned long old; 233376cfa87STim Bird 234376cfa87STim Bird if (unlikely(atomic_read(¤t->tracing_graph_pause))) 235*e3cf20e5SArd Biesheuvel err_out: 236376cfa87STim Bird return; 237376cfa87STim Bird 238953f534aSArd Biesheuvel if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER)) { 239*e3cf20e5SArd Biesheuvel /* 240*e3cf20e5SArd Biesheuvel * Usually, the stack frames are contiguous in memory but cases 241*e3cf20e5SArd Biesheuvel * have been observed where the next stack frame does not live 242*e3cf20e5SArd Biesheuvel * at 'frame_pointer + 4' as this code used to assume. 243*e3cf20e5SArd Biesheuvel * 244*e3cf20e5SArd Biesheuvel * Instead, dereference the field in the stack frame that 245*e3cf20e5SArd Biesheuvel * stores the SP of the calling frame: to avoid unbounded 246*e3cf20e5SArd Biesheuvel * recursion, this cannot involve any ftrace instrumented 247*e3cf20e5SArd Biesheuvel * functions, so use the __get_kernel_nofault() primitive 248*e3cf20e5SArd Biesheuvel * directly. 249*e3cf20e5SArd Biesheuvel */ 250*e3cf20e5SArd Biesheuvel __get_kernel_nofault(&frame_pointer, 251*e3cf20e5SArd Biesheuvel (unsigned long *)(frame_pointer - 8), 252*e3cf20e5SArd Biesheuvel unsigned long, err_out); 25341918ec8SArd Biesheuvel } else { 25441918ec8SArd Biesheuvel struct stackframe frame = { 25541918ec8SArd Biesheuvel .fp = frame_pointer, 25641918ec8SArd Biesheuvel .sp = stack_pointer, 25741918ec8SArd Biesheuvel .lr = self_addr, 25841918ec8SArd Biesheuvel .pc = self_addr, 25941918ec8SArd Biesheuvel }; 26041918ec8SArd Biesheuvel if (unwind_frame(&frame) < 0) 26141918ec8SArd Biesheuvel return; 26241918ec8SArd Biesheuvel if (frame.lr != self_addr) 26341918ec8SArd Biesheuvel parent = frame.lr_addr; 26441918ec8SArd Biesheuvel frame_pointer = frame.sp; 265953f534aSArd Biesheuvel } 266953f534aSArd Biesheuvel 267376cfa87STim Bird old = *parent; 268376cfa87STim Bird *parent = return_hooker; 269376cfa87STim Bird 270f1f5b14aSSteven Rostedt (VMware) if (function_graph_enter(old, self_addr, frame_pointer, NULL)) 2714c36595eSColin Cross *parent = old; 272376cfa87STim Bird } 273dd686eb1SRabin Vincent 274dd686eb1SRabin Vincent #ifdef CONFIG_DYNAMIC_FTRACE 275dd686eb1SRabin Vincent extern unsigned long ftrace_graph_call; 276dd686eb1SRabin Vincent extern unsigned long ftrace_graph_call_old; 277dd686eb1SRabin Vincent extern void ftrace_graph_caller_old(void); 278620176f3SAbel Vesa extern unsigned long ftrace_graph_regs_call; 279620176f3SAbel Vesa extern void ftrace_graph_regs_caller(void); 280dd686eb1SRabin Vincent 281dd686eb1SRabin Vincent static int __ftrace_modify_caller(unsigned long *callsite, 282dd686eb1SRabin Vincent void (*func) (void), bool enable) 283dd686eb1SRabin Vincent { 284dd686eb1SRabin Vincent unsigned long caller_fn = (unsigned long) func; 285dd686eb1SRabin Vincent unsigned long pc = (unsigned long) callsite; 286d82227cfSRabin Vincent unsigned long branch = arm_gen_branch(pc, caller_fn); 28741918ec8SArd Biesheuvel unsigned long nop = arm_gen_nop(); 288dd686eb1SRabin Vincent unsigned long old = enable ? nop : branch; 289dd686eb1SRabin Vincent unsigned long new = enable ? branch : nop; 290dd686eb1SRabin Vincent 291dc283d70SRabin Vincent return ftrace_modify_code(pc, old, new, true); 292dd686eb1SRabin Vincent } 293dd686eb1SRabin Vincent 294dd686eb1SRabin Vincent static int ftrace_modify_graph_caller(bool enable) 295dd686eb1SRabin Vincent { 296dd686eb1SRabin Vincent int ret; 297dd686eb1SRabin Vincent 298dd686eb1SRabin Vincent ret = __ftrace_modify_caller(&ftrace_graph_call, 299dd686eb1SRabin Vincent ftrace_graph_caller, 300dd686eb1SRabin Vincent enable); 301dd686eb1SRabin Vincent 302620176f3SAbel Vesa #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 303620176f3SAbel Vesa if (!ret) 304620176f3SAbel Vesa ret = __ftrace_modify_caller(&ftrace_graph_regs_call, 305620176f3SAbel Vesa ftrace_graph_regs_caller, 306620176f3SAbel Vesa enable); 307620176f3SAbel Vesa #endif 308620176f3SAbel Vesa 309620176f3SAbel Vesa 310dd686eb1SRabin Vincent return ret; 311dd686eb1SRabin Vincent } 312dd686eb1SRabin Vincent 313dd686eb1SRabin Vincent int ftrace_enable_ftrace_graph_caller(void) 314dd686eb1SRabin Vincent { 315dd686eb1SRabin Vincent return ftrace_modify_graph_caller(true); 316dd686eb1SRabin Vincent } 317dd686eb1SRabin Vincent 318dd686eb1SRabin Vincent int ftrace_disable_ftrace_graph_caller(void) 319dd686eb1SRabin Vincent { 320dd686eb1SRabin Vincent return ftrace_modify_graph_caller(false); 321dd686eb1SRabin Vincent } 322dd686eb1SRabin Vincent #endif /* CONFIG_DYNAMIC_FTRACE */ 323376cfa87STim Bird #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 324