1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Code for replacing ftrace calls with jumps. 4 * 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 6 * Copyright (C) 2009, 2010 DSLab, Lanzhou University, China 7 * Author: Wu Zhangjin <wuzhangjin@gmail.com> 8 * 9 * Thanks goes to Steven Rostedt for writing the original x86 version. 10 */ 11 12 #include <linux/uaccess.h> 13 #include <linux/init.h> 14 #include <linux/ftrace.h> 15 #include <linux/syscalls.h> 16 17 #include <asm/asm.h> 18 #include <asm/asm-offsets.h> 19 #include <asm/cacheflush.h> 20 #include <asm/syscall.h> 21 #include <asm/uasm.h> 22 #include <asm/unistd.h> 23 24 #include <asm-generic/sections.h> 25 26 #if defined(KBUILD_MCOUNT_RA_ADDRESS) && defined(CONFIG_32BIT) 27 #define MCOUNT_OFFSET_INSNS 5 28 #else 29 #define MCOUNT_OFFSET_INSNS 4 30 #endif 31 32 #ifdef CONFIG_DYNAMIC_FTRACE 33 34 /* Arch override because MIPS doesn't need to run this from stop_machine() */ 35 void arch_ftrace_update_code(int command) 36 { 37 ftrace_modify_all_code(command); 38 } 39 40 #define JAL 0x0c000000 /* jump & link: ip --> ra, jump to target */ 41 #define ADDR_MASK 0x03ffffff /* op_code|addr : 31...26|25 ....0 */ 42 #define JUMP_RANGE_MASK ((1UL << 28) - 1) 43 44 #define INSN_NOP 0x00000000 /* nop */ 45 #define INSN_JAL(addr) \ 46 ((unsigned int)(JAL | (((addr) >> 2) & ADDR_MASK))) 47 48 static unsigned int insn_jal_ftrace_caller __read_mostly; 49 static unsigned int insn_la_mcount[2] __read_mostly; 50 static unsigned int insn_j_ftrace_graph_caller __maybe_unused __read_mostly; 51 52 static inline void ftrace_dyn_arch_init_insns(void) 53 { 54 u32 *buf; 55 unsigned int v1; 56 57 /* la v1, _mcount */ 58 v1 = 3; 59 buf = (u32 *)&insn_la_mcount[0]; 60 UASM_i_LA(&buf, v1, MCOUNT_ADDR); 61 62 /* jal (ftrace_caller + 8), jump over the first two instruction */ 63 buf = (u32 *)&insn_jal_ftrace_caller; 64 uasm_i_jal(&buf, (FTRACE_ADDR + 8) & JUMP_RANGE_MASK); 65 66 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 67 /* j ftrace_graph_caller */ 68 buf = (u32 *)&insn_j_ftrace_graph_caller; 69 uasm_i_j(&buf, (unsigned long)ftrace_graph_caller & JUMP_RANGE_MASK); 70 #endif 71 } 72 73 static int ftrace_modify_code(unsigned long ip, unsigned int new_code) 74 { 75 int faulted; 76 mm_segment_t old_fs; 77 78 /* *(unsigned int *)ip = new_code; */ 79 safe_store_code(new_code, ip, faulted); 80 81 if (unlikely(faulted)) 82 return -EFAULT; 83 84 old_fs = get_fs(); 85 set_fs(KERNEL_DS); 86 flush_icache_range(ip, ip + 8); 87 set_fs(old_fs); 88 89 return 0; 90 } 91 92 #ifndef CONFIG_64BIT 93 static int ftrace_modify_code_2(unsigned long ip, unsigned int new_code1, 94 unsigned int new_code2) 95 { 96 int faulted; 97 mm_segment_t old_fs; 98 99 safe_store_code(new_code1, ip, faulted); 100 if (unlikely(faulted)) 101 return -EFAULT; 102 103 ip += 4; 104 safe_store_code(new_code2, ip, faulted); 105 if (unlikely(faulted)) 106 return -EFAULT; 107 108 ip -= 4; 109 old_fs = get_fs(); 110 set_fs(KERNEL_DS); 111 flush_icache_range(ip, ip + 8); 112 set_fs(old_fs); 113 114 return 0; 115 } 116 117 static int ftrace_modify_code_2r(unsigned long ip, unsigned int new_code1, 118 unsigned int new_code2) 119 { 120 int faulted; 121 mm_segment_t old_fs; 122 123 ip += 4; 124 safe_store_code(new_code2, ip, faulted); 125 if (unlikely(faulted)) 126 return -EFAULT; 127 128 ip -= 4; 129 safe_store_code(new_code1, ip, faulted); 130 if (unlikely(faulted)) 131 return -EFAULT; 132 133 old_fs = get_fs(); 134 set_fs(KERNEL_DS); 135 flush_icache_range(ip, ip + 8); 136 set_fs(old_fs); 137 138 return 0; 139 } 140 #endif 141 142 /* 143 * The details about the calling site of mcount on MIPS 144 * 145 * 1. For kernel: 146 * 147 * move at, ra 148 * jal _mcount --> nop 149 * sub sp, sp, 8 --> nop (CONFIG_32BIT) 150 * 151 * 2. For modules: 152 * 153 * 2.1 For KBUILD_MCOUNT_RA_ADDRESS and CONFIG_32BIT 154 * 155 * lui v1, hi_16bit_of_mcount --> b 1f (0x10000005) 156 * addiu v1, v1, low_16bit_of_mcount --> nop (CONFIG_32BIT) 157 * move at, ra 158 * move $12, ra_address 159 * jalr v1 160 * sub sp, sp, 8 161 * 1: offset = 5 instructions 162 * 2.2 For the Other situations 163 * 164 * lui v1, hi_16bit_of_mcount --> b 1f (0x10000004) 165 * addiu v1, v1, low_16bit_of_mcount --> nop (CONFIG_32BIT) 166 * move at, ra 167 * jalr v1 168 * nop | move $12, ra_address | sub sp, sp, 8 169 * 1: offset = 4 instructions 170 */ 171 172 #define INSN_B_1F (0x10000000 | MCOUNT_OFFSET_INSNS) 173 174 int ftrace_make_nop(struct module *mod, 175 struct dyn_ftrace *rec, unsigned long addr) 176 { 177 unsigned int new; 178 unsigned long ip = rec->ip; 179 180 /* 181 * If ip is in kernel space, no long call, otherwise, long call is 182 * needed. 183 */ 184 new = core_kernel_text(ip) ? INSN_NOP : INSN_B_1F; 185 #ifdef CONFIG_64BIT 186 return ftrace_modify_code(ip, new); 187 #else 188 /* 189 * On 32 bit MIPS platforms, gcc adds a stack adjust 190 * instruction in the delay slot after the branch to 191 * mcount and expects mcount to restore the sp on return. 192 * This is based on a legacy API and does nothing but 193 * waste instructions so it's being removed at runtime. 194 */ 195 return ftrace_modify_code_2(ip, new, INSN_NOP); 196 #endif 197 } 198 199 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 200 { 201 unsigned int new; 202 unsigned long ip = rec->ip; 203 204 new = core_kernel_text(ip) ? insn_jal_ftrace_caller : insn_la_mcount[0]; 205 206 #ifdef CONFIG_64BIT 207 return ftrace_modify_code(ip, new); 208 #else 209 return ftrace_modify_code_2r(ip, new, core_kernel_text(ip) ? 210 INSN_NOP : insn_la_mcount[1]); 211 #endif 212 } 213 214 #define FTRACE_CALL_IP ((unsigned long)(&ftrace_call)) 215 216 int ftrace_update_ftrace_func(ftrace_func_t func) 217 { 218 unsigned int new; 219 220 new = INSN_JAL((unsigned long)func); 221 222 return ftrace_modify_code(FTRACE_CALL_IP, new); 223 } 224 225 int __init ftrace_dyn_arch_init(void) 226 { 227 /* Encode the instructions when booting */ 228 ftrace_dyn_arch_init_insns(); 229 230 /* Remove "b ftrace_stub" to ensure ftrace_caller() is executed */ 231 ftrace_modify_code(MCOUNT_ADDR, INSN_NOP); 232 233 return 0; 234 } 235 #endif /* CONFIG_DYNAMIC_FTRACE */ 236 237 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 238 239 #ifdef CONFIG_DYNAMIC_FTRACE 240 241 extern void ftrace_graph_call(void); 242 #define FTRACE_GRAPH_CALL_IP ((unsigned long)(&ftrace_graph_call)) 243 244 int ftrace_enable_ftrace_graph_caller(void) 245 { 246 return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, 247 insn_j_ftrace_graph_caller); 248 } 249 250 int ftrace_disable_ftrace_graph_caller(void) 251 { 252 return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, INSN_NOP); 253 } 254 255 #endif /* CONFIG_DYNAMIC_FTRACE */ 256 257 #ifndef KBUILD_MCOUNT_RA_ADDRESS 258 259 #define S_RA_SP (0xafbf << 16) /* s{d,w} ra, offset(sp) */ 260 #define S_R_SP (0xafb0 << 16) /* s{d,w} R, offset(sp) */ 261 #define OFFSET_MASK 0xffff /* stack offset range: 0 ~ PT_SIZE */ 262 263 unsigned long ftrace_get_parent_ra_addr(unsigned long self_ra, unsigned long 264 old_parent_ra, unsigned long parent_ra_addr, unsigned long fp) 265 { 266 unsigned long sp, ip, tmp; 267 unsigned int code; 268 int faulted; 269 270 /* 271 * For module, move the ip from the return address after the 272 * instruction "lui v1, hi_16bit_of_mcount"(offset is 24), but for 273 * kernel, move after the instruction "move ra, at"(offset is 16) 274 */ 275 ip = self_ra - (core_kernel_text(self_ra) ? 16 : 24); 276 277 /* 278 * search the text until finding the non-store instruction or "s{d,w} 279 * ra, offset(sp)" instruction 280 */ 281 do { 282 /* get the code at "ip": code = *(unsigned int *)ip; */ 283 safe_load_code(code, ip, faulted); 284 285 if (unlikely(faulted)) 286 return 0; 287 /* 288 * If we hit the non-store instruction before finding where the 289 * ra is stored, then this is a leaf function and it does not 290 * store the ra on the stack 291 */ 292 if ((code & S_R_SP) != S_R_SP) 293 return parent_ra_addr; 294 295 /* Move to the next instruction */ 296 ip -= 4; 297 } while ((code & S_RA_SP) != S_RA_SP); 298 299 sp = fp + (code & OFFSET_MASK); 300 301 /* tmp = *(unsigned long *)sp; */ 302 safe_load_stack(tmp, sp, faulted); 303 if (unlikely(faulted)) 304 return 0; 305 306 if (tmp == old_parent_ra) 307 return sp; 308 return 0; 309 } 310 311 #endif /* !KBUILD_MCOUNT_RA_ADDRESS */ 312 313 /* 314 * Hook the return address and push it in the stack of return addrs 315 * in current thread info. 316 */ 317 void prepare_ftrace_return(unsigned long *parent_ra_addr, unsigned long self_ra, 318 unsigned long fp) 319 { 320 unsigned long old_parent_ra; 321 unsigned long return_hooker = (unsigned long) 322 &return_to_handler; 323 int faulted, insns; 324 325 if (unlikely(ftrace_graph_is_dead())) 326 return; 327 328 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 329 return; 330 331 /* 332 * "parent_ra_addr" is the stack address where the return address of 333 * the caller of _mcount is saved. 334 * 335 * If gcc < 4.5, a leaf function does not save the return address 336 * in the stack address, so we "emulate" one in _mcount's stack space, 337 * and hijack it directly. 338 * For a non-leaf function, it does save the return address to its own 339 * stack space, so we can not hijack it directly, but need to find the 340 * real stack address, which is done by ftrace_get_parent_addr(). 341 * 342 * If gcc >= 4.5, with the new -mmcount-ra-address option, for a 343 * non-leaf function, the location of the return address will be saved 344 * to $12 for us. 345 * For a leaf function, it just puts a zero into $12, so we handle 346 * it in ftrace_graph_caller() of mcount.S. 347 */ 348 349 /* old_parent_ra = *parent_ra_addr; */ 350 safe_load_stack(old_parent_ra, parent_ra_addr, faulted); 351 if (unlikely(faulted)) 352 goto out; 353 #ifndef KBUILD_MCOUNT_RA_ADDRESS 354 parent_ra_addr = (unsigned long *)ftrace_get_parent_ra_addr(self_ra, 355 old_parent_ra, (unsigned long)parent_ra_addr, fp); 356 /* 357 * If fails when getting the stack address of the non-leaf function's 358 * ra, stop function graph tracer and return 359 */ 360 if (parent_ra_addr == NULL) 361 goto out; 362 #endif 363 /* *parent_ra_addr = return_hooker; */ 364 safe_store_stack(return_hooker, parent_ra_addr, faulted); 365 if (unlikely(faulted)) 366 goto out; 367 368 /* 369 * Get the recorded ip of the current mcount calling site in the 370 * __mcount_loc section, which will be used to filter the function 371 * entries configured through the tracing/set_graph_function interface. 372 */ 373 374 insns = core_kernel_text(self_ra) ? 2 : MCOUNT_OFFSET_INSNS + 1; 375 self_ra -= (MCOUNT_INSN_SIZE * insns); 376 377 if (function_graph_enter(old_parent_ra, self_ra, fp, NULL)) 378 *parent_ra_addr = old_parent_ra; 379 return; 380 out: 381 ftrace_graph_stop(); 382 WARN_ON(1); 383 } 384 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 385 386 #ifdef CONFIG_FTRACE_SYSCALLS 387 388 #ifdef CONFIG_32BIT 389 unsigned long __init arch_syscall_addr(int nr) 390 { 391 return (unsigned long)sys_call_table[nr - __NR_O32_Linux]; 392 } 393 #endif 394 395 #ifdef CONFIG_64BIT 396 397 unsigned long __init arch_syscall_addr(int nr) 398 { 399 #ifdef CONFIG_MIPS32_N32 400 if (nr >= __NR_N32_Linux && nr < __NR_N32_Linux + __NR_N32_Linux_syscalls) 401 return (unsigned long)sysn32_call_table[nr - __NR_N32_Linux]; 402 #endif 403 if (nr >= __NR_64_Linux && nr < __NR_64_Linux + __NR_64_Linux_syscalls) 404 return (unsigned long)sys_call_table[nr - __NR_64_Linux]; 405 #ifdef CONFIG_MIPS32_O32 406 if (nr >= __NR_O32_Linux && nr < __NR_O32_Linux + __NR_O32_Linux_syscalls) 407 return (unsigned long)sys32_call_table[nr - __NR_O32_Linux]; 408 #endif 409 410 return (unsigned long) &sys_ni_syscall; 411 } 412 #endif 413 414 #endif /* CONFIG_FTRACE_SYSCALLS */ 415