1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2013 Linaro Limited 4 * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> 5 * Copyright (C) 2017 Andes Technology Corporation 6 */ 7 8 #include <linux/ftrace.h> 9 #include <linux/uaccess.h> 10 #include <linux/memory.h> 11 #include <linux/irqflags.h> 12 #include <linux/stop_machine.h> 13 #include <asm/cacheflush.h> 14 #include <asm/text-patching.h> 15 #include <asm/insn.h> 16 17 #ifdef CONFIG_DYNAMIC_FTRACE 18 void ftrace_arch_code_modify_prepare(void) 19 __acquires(&text_mutex) 20 { 21 mutex_lock(&text_mutex); 22 } 23 24 void ftrace_arch_code_modify_post_process(void) 25 __releases(&text_mutex) 26 { 27 mutex_unlock(&text_mutex); 28 } 29 30 unsigned long ftrace_call_adjust(unsigned long addr) 31 { 32 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS)) 33 return addr + 8 + MCOUNT_AUIPC_SIZE; 34 35 return addr + MCOUNT_AUIPC_SIZE; 36 } 37 38 unsigned long arch_ftrace_get_symaddr(unsigned long fentry_ip) 39 { 40 return fentry_ip - MCOUNT_AUIPC_SIZE; 41 } 42 43 void arch_ftrace_update_code(int command) 44 { 45 command |= FTRACE_MAY_SLEEP; 46 ftrace_modify_all_code(command); 47 flush_icache_all(); 48 } 49 50 static int __ftrace_modify_call(unsigned long source, unsigned long target, bool validate) 51 { 52 unsigned int call[2], offset; 53 unsigned int replaced[2]; 54 55 offset = target - source; 56 call[1] = to_jalr_t0(offset); 57 58 if (validate) { 59 call[0] = to_auipc_t0(offset); 60 /* 61 * Read the text we want to modify; 62 * return must be -EFAULT on read error 63 */ 64 if (copy_from_kernel_nofault(replaced, (void *)source, 2 * MCOUNT_INSN_SIZE)) 65 return -EFAULT; 66 67 /* Bypass the check if the auipc insn is a kprobe breakpoint */ 68 if (replaced[0] != call[0] && 69 !(riscv_insn_is_ebreak(replaced[0]) || riscv_insn_is_c_ebreak(replaced[0]))) { 70 pr_err("%p: expected (%08x) but got (%08x)\n", 71 (void *)source, call[0], replaced[0]); 72 return -EINVAL; 73 } 74 } 75 76 /* Replace the jalr at once. Return -EPERM on write error. */ 77 if (patch_insn_write((void *)(source + MCOUNT_AUIPC_SIZE), call + 1, MCOUNT_JALR_SIZE)) 78 return -EPERM; 79 80 return 0; 81 } 82 83 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS 84 static const struct ftrace_ops *riscv64_rec_get_ops(struct dyn_ftrace *rec) 85 { 86 const struct ftrace_ops *ops = NULL; 87 88 if (rec->flags & FTRACE_FL_CALL_OPS_EN) { 89 ops = ftrace_find_unique_ops(rec); 90 WARN_ON_ONCE(!ops); 91 } 92 93 if (!ops) 94 ops = &ftrace_list_ops; 95 96 return ops; 97 } 98 99 static int ftrace_rec_set_ops(const struct dyn_ftrace *rec, const struct ftrace_ops *ops) 100 { 101 unsigned long literal = ALIGN_DOWN(rec->ip - 12, 8); 102 103 return patch_text_nosync((void *)literal, &ops, sizeof(ops)); 104 } 105 106 static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec) 107 { 108 return ftrace_rec_set_ops(rec, &ftrace_nop_ops); 109 } 110 111 static int ftrace_rec_update_ops(struct dyn_ftrace *rec) 112 { 113 return ftrace_rec_set_ops(rec, riscv64_rec_get_ops(rec)); 114 } 115 #else 116 static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec) { return 0; } 117 static int ftrace_rec_update_ops(struct dyn_ftrace *rec) { return 0; } 118 #endif 119 120 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 121 { 122 unsigned long distance, orig_addr, pc = rec->ip - MCOUNT_AUIPC_SIZE; 123 int ret; 124 125 ret = ftrace_rec_update_ops(rec); 126 if (ret) 127 return ret; 128 129 orig_addr = (unsigned long)&ftrace_caller; 130 distance = addr > orig_addr ? addr - orig_addr : orig_addr - addr; 131 if (distance > JALR_RANGE) 132 addr = FTRACE_ADDR; 133 134 return __ftrace_modify_call(pc, addr, false); 135 } 136 137 int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr) 138 { 139 u32 nop4 = RISCV_INSN_NOP4; 140 int ret; 141 142 ret = ftrace_rec_set_nop_ops(rec); 143 if (ret) 144 return ret; 145 146 if (patch_insn_write((void *)rec->ip, &nop4, MCOUNT_NOP4_SIZE)) 147 return -EPERM; 148 149 return 0; 150 } 151 152 /* 153 * This is called early on, and isn't wrapped by 154 * ftrace_arch_code_modify_{prepare,post_process}() and therefore doesn't hold 155 * text_mutex, which triggers a lockdep failure. SMP isn't running so we could 156 * just directly poke the text, but it's simpler to just take the lock 157 * ourselves. 158 */ 159 int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) 160 { 161 unsigned long pc = rec->ip - MCOUNT_AUIPC_SIZE; 162 unsigned int nops[2], offset; 163 int ret; 164 165 guard(mutex)(&text_mutex); 166 167 ret = ftrace_rec_set_nop_ops(rec); 168 if (ret) 169 return ret; 170 171 offset = (unsigned long) &ftrace_caller - pc; 172 nops[0] = to_auipc_t0(offset); 173 nops[1] = RISCV_INSN_NOP4; 174 175 ret = patch_insn_write((void *)pc, nops, 2 * MCOUNT_INSN_SIZE); 176 177 return ret; 178 } 179 180 ftrace_func_t ftrace_call_dest = ftrace_stub; 181 int ftrace_update_ftrace_func(ftrace_func_t func) 182 { 183 /* 184 * When using CALL_OPS, the function to call is associated with the 185 * call site, and we don't have a global function pointer to update. 186 */ 187 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS)) 188 return 0; 189 190 WRITE_ONCE(ftrace_call_dest, func); 191 /* 192 * The data fence ensure that the update to ftrace_call_dest happens 193 * before the write to function_trace_op later in the generic ftrace. 194 * If the sequence is not enforced, then an old ftrace_call_dest may 195 * race loading a new function_trace_op set in ftrace_modify_all_code 196 */ 197 smp_wmb(); 198 /* 199 * Updating ftrace dpes not take stop_machine path, so irqs should not 200 * be disabled. 201 */ 202 WARN_ON(irqs_disabled()); 203 smp_call_function(ftrace_sync_ipi, NULL, 1); 204 return 0; 205 } 206 207 #else /* CONFIG_DYNAMIC_FTRACE */ 208 unsigned long ftrace_call_adjust(unsigned long addr) 209 { 210 return addr; 211 } 212 #endif /* CONFIG_DYNAMIC_FTRACE */ 213 214 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS 215 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 216 unsigned long addr) 217 { 218 unsigned long caller = rec->ip - MCOUNT_AUIPC_SIZE; 219 int ret; 220 221 ret = ftrace_rec_update_ops(rec); 222 if (ret) 223 return ret; 224 225 return __ftrace_modify_call(caller, FTRACE_ADDR, true); 226 } 227 #endif 228 229 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 230 /* 231 * Most of this function is copied from arm64. 232 */ 233 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, 234 unsigned long frame_pointer) 235 { 236 unsigned long return_hooker = (unsigned long)&return_to_handler; 237 unsigned long old; 238 239 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 240 return; 241 242 /* 243 * We don't suffer access faults, so no extra fault-recovery assembly 244 * is needed here. 245 */ 246 old = *parent; 247 248 if (!function_graph_enter(old, self_addr, frame_pointer, parent)) 249 *parent = return_hooker; 250 } 251 252 #ifdef CONFIG_DYNAMIC_FTRACE 253 void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, 254 struct ftrace_ops *op, struct ftrace_regs *fregs) 255 { 256 unsigned long return_hooker = (unsigned long)&return_to_handler; 257 unsigned long frame_pointer = arch_ftrace_regs(fregs)->s0; 258 unsigned long *parent = &arch_ftrace_regs(fregs)->ra; 259 unsigned long old; 260 261 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 262 return; 263 264 /* 265 * We don't suffer access faults, so no extra fault-recovery assembly 266 * is needed here. 267 */ 268 old = *parent; 269 270 if (!function_graph_enter_regs(old, ip, frame_pointer, parent, fregs)) 271 *parent = return_hooker; 272 } 273 #endif /* CONFIG_DYNAMIC_FTRACE */ 274 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 275