1 /* 2 * Code for replacing ftrace calls with jumps. 3 * 4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 5 * Copyright (C) 2009 DSLab, Lanzhou University, China 6 * Author: Wu Zhangjin <wuzj@lemote.com> 7 * 8 * Thanks goes to Steven Rostedt for writing the original x86 version. 9 */ 10 11 #include <linux/uaccess.h> 12 #include <linux/init.h> 13 #include <linux/ftrace.h> 14 15 #include <asm/cacheflush.h> 16 #include <asm/asm.h> 17 #include <asm/asm-offsets.h> 18 19 #ifdef CONFIG_DYNAMIC_FTRACE 20 21 #define JAL 0x0c000000 /* jump & link: ip --> ra, jump to target */ 22 #define ADDR_MASK 0x03ffffff /* op_code|addr : 31...26|25 ....0 */ 23 #define jump_insn_encode(op_code, addr) \ 24 ((unsigned int)((op_code) | (((addr) >> 2) & ADDR_MASK))) 25 26 static unsigned int ftrace_nop = 0x00000000; 27 28 static int ftrace_modify_code(unsigned long ip, unsigned int new_code) 29 { 30 int faulted; 31 32 /* *(unsigned int *)ip = new_code; */ 33 safe_store_code(new_code, ip, faulted); 34 35 if (unlikely(faulted)) 36 return -EFAULT; 37 38 flush_icache_range(ip, ip + 8); 39 40 return 0; 41 } 42 43 static int lui_v1; 44 static int jal_mcount; 45 46 int ftrace_make_nop(struct module *mod, 47 struct dyn_ftrace *rec, unsigned long addr) 48 { 49 unsigned int new; 50 int faulted; 51 unsigned long ip = rec->ip; 52 53 /* We have compiled module with -mlong-calls, but compiled the kernel 54 * without it, we need to cope with them respectively. */ 55 if (ip & 0x40000000) { 56 /* record it for ftrace_make_call */ 57 if (lui_v1 == 0) { 58 /* lui_v1 = *(unsigned int *)ip; */ 59 safe_load_code(lui_v1, ip, faulted); 60 61 if (unlikely(faulted)) 62 return -EFAULT; 63 } 64 65 /* lui v1, hi_16bit_of_mcount --> b 1f (0x10000004) 66 * addiu v1, v1, low_16bit_of_mcount 67 * move at, ra 68 * jalr v1 69 * nop 70 * 1f: (ip + 12) 71 */ 72 new = 0x10000004; 73 } else { 74 /* record/calculate it for ftrace_make_call */ 75 if (jal_mcount == 0) { 76 /* We can record it directly like this: 77 * jal_mcount = *(unsigned int *)ip; 78 * Herein, jump over the first two nop instructions */ 79 jal_mcount = jump_insn_encode(JAL, (MCOUNT_ADDR + 8)); 80 } 81 82 /* move at, ra 83 * jalr v1 --> nop 84 */ 85 new = ftrace_nop; 86 } 87 return ftrace_modify_code(ip, new); 88 } 89 90 static int modified; /* initialized as 0 by default */ 91 92 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 93 { 94 unsigned int new; 95 unsigned long ip = rec->ip; 96 97 /* We just need to remove the "b ftrace_stub" at the fist time! */ 98 if (modified == 0) { 99 modified = 1; 100 ftrace_modify_code(addr, ftrace_nop); 101 } 102 /* ip, module: 0xc0000000, kernel: 0x80000000 */ 103 new = (ip & 0x40000000) ? lui_v1 : jal_mcount; 104 105 return ftrace_modify_code(ip, new); 106 } 107 108 #define FTRACE_CALL_IP ((unsigned long)(&ftrace_call)) 109 110 int ftrace_update_ftrace_func(ftrace_func_t func) 111 { 112 unsigned int new; 113 114 new = jump_insn_encode(JAL, (unsigned long)func); 115 116 return ftrace_modify_code(FTRACE_CALL_IP, new); 117 } 118 119 int __init ftrace_dyn_arch_init(void *data) 120 { 121 /* The return code is retured via data */ 122 *(unsigned long *)data = 0; 123 124 return 0; 125 } 126 #endif /* CONFIG_DYNAMIC_FTRACE */ 127 128 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 129 130 #ifdef CONFIG_DYNAMIC_FTRACE 131 132 extern void ftrace_graph_call(void); 133 #define JMP 0x08000000 /* jump to target directly */ 134 #define CALL_FTRACE_GRAPH_CALLER \ 135 jump_insn_encode(JMP, (unsigned long)(&ftrace_graph_caller)) 136 #define FTRACE_GRAPH_CALL_IP ((unsigned long)(&ftrace_graph_call)) 137 138 int ftrace_enable_ftrace_graph_caller(void) 139 { 140 return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, 141 CALL_FTRACE_GRAPH_CALLER); 142 } 143 144 int ftrace_disable_ftrace_graph_caller(void) 145 { 146 return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, ftrace_nop); 147 } 148 149 #endif /* !CONFIG_DYNAMIC_FTRACE */ 150 151 #ifndef KBUILD_MCOUNT_RA_ADDRESS 152 #define S_RA_SP (0xafbf << 16) /* s{d,w} ra, offset(sp) */ 153 #define S_R_SP (0xafb0 << 16) /* s{d,w} R, offset(sp) */ 154 #define OFFSET_MASK 0xffff /* stack offset range: 0 ~ PT_SIZE */ 155 156 unsigned long ftrace_get_parent_addr(unsigned long self_addr, 157 unsigned long parent, 158 unsigned long parent_addr, 159 unsigned long fp) 160 { 161 unsigned long sp, ip, ra; 162 unsigned int code; 163 int faulted; 164 165 /* in module or kernel? */ 166 if (self_addr & 0x40000000) { 167 /* module: move to the instruction "lui v1, HI_16BIT_OF_MCOUNT" */ 168 ip = self_addr - 20; 169 } else { 170 /* kernel: move to the instruction "move ra, at" */ 171 ip = self_addr - 12; 172 } 173 174 /* search the text until finding the non-store instruction or "s{d,w} 175 * ra, offset(sp)" instruction */ 176 do { 177 ip -= 4; 178 179 /* get the code at "ip": code = *(unsigned int *)ip; */ 180 safe_load_code(code, ip, faulted); 181 182 if (unlikely(faulted)) 183 return 0; 184 185 /* If we hit the non-store instruction before finding where the 186 * ra is stored, then this is a leaf function and it does not 187 * store the ra on the stack. */ 188 if ((code & S_R_SP) != S_R_SP) 189 return parent_addr; 190 191 } while (((code & S_RA_SP) != S_RA_SP)); 192 193 sp = fp + (code & OFFSET_MASK); 194 195 /* ra = *(unsigned long *)sp; */ 196 safe_load_stack(ra, sp, faulted); 197 if (unlikely(faulted)) 198 return 0; 199 200 if (ra == parent) 201 return sp; 202 return 0; 203 } 204 205 #endif 206 207 /* 208 * Hook the return address and push it in the stack of return addrs 209 * in current thread info. 210 */ 211 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, 212 unsigned long fp) 213 { 214 unsigned long old; 215 struct ftrace_graph_ent trace; 216 unsigned long return_hooker = (unsigned long) 217 &return_to_handler; 218 int faulted; 219 220 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 221 return; 222 223 /* "parent" is the stack address saved the return address of the caller 224 * of _mcount. 225 * 226 * if the gcc < 4.5, a leaf function does not save the return address 227 * in the stack address, so, we "emulate" one in _mcount's stack space, 228 * and hijack it directly, but for a non-leaf function, it save the 229 * return address to the its own stack space, we can not hijack it 230 * directly, but need to find the real stack address, 231 * ftrace_get_parent_addr() does it! 232 * 233 * if gcc>= 4.5, with the new -mmcount-ra-address option, for a 234 * non-leaf function, the location of the return address will be saved 235 * to $12 for us, and for a leaf function, only put a zero into $12. we 236 * do it in ftrace_graph_caller of mcount.S. 237 */ 238 239 /* old = *parent; */ 240 safe_load_stack(old, parent, faulted); 241 if (unlikely(faulted)) 242 goto out; 243 #ifndef KBUILD_MCOUNT_RA_ADDRESS 244 parent = (unsigned long *)ftrace_get_parent_addr(self_addr, old, 245 (unsigned long)parent, 246 fp); 247 /* If fails when getting the stack address of the non-leaf function's 248 * ra, stop function graph tracer and return */ 249 if (parent == 0) 250 goto out; 251 #endif 252 /* *parent = return_hooker; */ 253 safe_store_stack(return_hooker, parent, faulted); 254 if (unlikely(faulted)) 255 goto out; 256 257 if (ftrace_push_return_trace(old, self_addr, &trace.depth, fp) == 258 -EBUSY) { 259 *parent = old; 260 return; 261 } 262 263 trace.func = self_addr; 264 265 /* Only trace if the calling function expects to */ 266 if (!ftrace_graph_entry(&trace)) { 267 current->curr_ret_stack--; 268 *parent = old; 269 } 270 return; 271 out: 272 ftrace_graph_stop(); 273 WARN_ON(1); 274 } 275 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 276