1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * fprobe - Simple ftrace probe wrapper for function entry. 4 */ 5 #define pr_fmt(fmt) "fprobe: " fmt 6 7 #include <linux/err.h> 8 #include <linux/fprobe.h> 9 #include <linux/kallsyms.h> 10 #include <linux/kprobes.h> 11 #include <linux/rethook.h> 12 #include <linux/slab.h> 13 #include <linux/sort.h> 14 15 #include "trace.h" 16 17 struct fprobe_rethook_node { 18 struct rethook_node node; 19 unsigned long entry_ip; 20 }; 21 22 static void fprobe_handler(unsigned long ip, unsigned long parent_ip, 23 struct ftrace_ops *ops, struct ftrace_regs *fregs) 24 { 25 struct fprobe_rethook_node *fpr; 26 struct rethook_node *rh; 27 struct fprobe *fp; 28 int bit; 29 30 fp = container_of(ops, struct fprobe, ops); 31 if (fprobe_disabled(fp)) 32 return; 33 34 bit = ftrace_test_recursion_trylock(ip, parent_ip); 35 if (bit < 0) { 36 fp->nmissed++; 37 return; 38 } 39 40 if (fp->entry_handler) 41 fp->entry_handler(fp, ip, ftrace_get_regs(fregs)); 42 43 if (fp->exit_handler) { 44 rh = rethook_try_get(fp->rethook); 45 if (!rh) { 46 fp->nmissed++; 47 goto out; 48 } 49 fpr = container_of(rh, struct fprobe_rethook_node, node); 50 fpr->entry_ip = ip; 51 rethook_hook(rh, ftrace_get_regs(fregs), true); 52 } 53 54 out: 55 ftrace_test_recursion_unlock(bit); 56 } 57 NOKPROBE_SYMBOL(fprobe_handler); 58 59 static void fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip, 60 struct ftrace_ops *ops, struct ftrace_regs *fregs) 61 { 62 struct fprobe *fp = container_of(ops, struct fprobe, ops); 63 64 if (unlikely(kprobe_running())) { 65 fp->nmissed++; 66 return; 67 } 68 kprobe_busy_begin(); 69 fprobe_handler(ip, parent_ip, ops, fregs); 70 kprobe_busy_end(); 71 } 72 73 static void fprobe_exit_handler(struct rethook_node *rh, void *data, 74 struct pt_regs *regs) 75 { 76 struct fprobe *fp = (struct fprobe *)data; 77 struct fprobe_rethook_node *fpr; 78 79 if (!fp || fprobe_disabled(fp)) 80 return; 81 82 fpr = container_of(rh, struct fprobe_rethook_node, node); 83 84 fp->exit_handler(fp, fpr->entry_ip, regs); 85 } 86 NOKPROBE_SYMBOL(fprobe_exit_handler); 87 88 static int symbols_cmp(const void *a, const void *b) 89 { 90 const char **str_a = (const char **) a; 91 const char **str_b = (const char **) b; 92 93 return strcmp(*str_a, *str_b); 94 } 95 96 /* Convert ftrace location address from symbols */ 97 static unsigned long *get_ftrace_locations(const char **syms, int num) 98 { 99 unsigned long *addrs; 100 101 /* Convert symbols to symbol address */ 102 addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL); 103 if (!addrs) 104 return ERR_PTR(-ENOMEM); 105 106 /* ftrace_lookup_symbols expects sorted symbols */ 107 sort(syms, num, sizeof(*syms), symbols_cmp, NULL); 108 109 if (!ftrace_lookup_symbols(syms, num, addrs)) 110 return addrs; 111 112 kfree(addrs); 113 return ERR_PTR(-ENOENT); 114 } 115 116 static void fprobe_init(struct fprobe *fp) 117 { 118 fp->nmissed = 0; 119 if (fprobe_shared_with_kprobes(fp)) 120 fp->ops.func = fprobe_kprobe_handler; 121 else 122 fp->ops.func = fprobe_handler; 123 fp->ops.flags |= FTRACE_OPS_FL_SAVE_REGS; 124 } 125 126 static int fprobe_init_rethook(struct fprobe *fp, int num) 127 { 128 int i, size; 129 130 if (num < 0) 131 return -EINVAL; 132 133 if (!fp->exit_handler) { 134 fp->rethook = NULL; 135 return 0; 136 } 137 138 /* Initialize rethook if needed */ 139 size = num * num_possible_cpus() * 2; 140 if (size < 0) 141 return -E2BIG; 142 143 fp->rethook = rethook_alloc((void *)fp, fprobe_exit_handler); 144 for (i = 0; i < size; i++) { 145 struct fprobe_rethook_node *node; 146 147 node = kzalloc(sizeof(*node), GFP_KERNEL); 148 if (!node) { 149 rethook_free(fp->rethook); 150 fp->rethook = NULL; 151 return -ENOMEM; 152 } 153 rethook_add_node(fp->rethook, &node->node); 154 } 155 return 0; 156 } 157 158 static void fprobe_fail_cleanup(struct fprobe *fp) 159 { 160 if (fp->rethook) { 161 /* Don't need to cleanup rethook->handler because this is not used. */ 162 rethook_free(fp->rethook); 163 fp->rethook = NULL; 164 } 165 ftrace_free_filter(&fp->ops); 166 } 167 168 /** 169 * register_fprobe() - Register fprobe to ftrace by pattern. 170 * @fp: A fprobe data structure to be registered. 171 * @filter: A wildcard pattern of probed symbols. 172 * @notfilter: A wildcard pattern of NOT probed symbols. 173 * 174 * Register @fp to ftrace for enabling the probe on the symbols matched to @filter. 175 * If @notfilter is not NULL, the symbols matched the @notfilter are not probed. 176 * 177 * Return 0 if @fp is registered successfully, -errno if not. 178 */ 179 int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter) 180 { 181 struct ftrace_hash *hash; 182 unsigned char *str; 183 int ret, len; 184 185 if (!fp || !filter) 186 return -EINVAL; 187 188 fprobe_init(fp); 189 190 len = strlen(filter); 191 str = kstrdup(filter, GFP_KERNEL); 192 ret = ftrace_set_filter(&fp->ops, str, len, 0); 193 kfree(str); 194 if (ret) 195 return ret; 196 197 if (notfilter) { 198 len = strlen(notfilter); 199 str = kstrdup(notfilter, GFP_KERNEL); 200 ret = ftrace_set_notrace(&fp->ops, str, len, 0); 201 kfree(str); 202 if (ret) 203 goto out; 204 } 205 206 /* TODO: 207 * correctly calculate the total number of filtered symbols 208 * from both filter and notfilter. 209 */ 210 hash = rcu_access_pointer(fp->ops.local_hash.filter_hash); 211 if (WARN_ON_ONCE(!hash)) 212 goto out; 213 214 ret = fprobe_init_rethook(fp, (int)hash->count); 215 if (!ret) 216 ret = register_ftrace_function(&fp->ops); 217 218 out: 219 if (ret) 220 fprobe_fail_cleanup(fp); 221 return ret; 222 } 223 EXPORT_SYMBOL_GPL(register_fprobe); 224 225 /** 226 * register_fprobe_ips() - Register fprobe to ftrace by address. 227 * @fp: A fprobe data structure to be registered. 228 * @addrs: An array of target ftrace location addresses. 229 * @num: The number of entries of @addrs. 230 * 231 * Register @fp to ftrace for enabling the probe on the address given by @addrs. 232 * The @addrs must be the addresses of ftrace location address, which may be 233 * the symbol address + arch-dependent offset. 234 * If you unsure what this mean, please use other registration functions. 235 * 236 * Return 0 if @fp is registered successfully, -errno if not. 237 */ 238 int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num) 239 { 240 int ret; 241 242 if (!fp || !addrs || num <= 0) 243 return -EINVAL; 244 245 fprobe_init(fp); 246 247 ret = ftrace_set_filter_ips(&fp->ops, addrs, num, 0, 0); 248 if (ret) 249 return ret; 250 251 ret = fprobe_init_rethook(fp, num); 252 if (!ret) 253 ret = register_ftrace_function(&fp->ops); 254 255 if (ret) 256 fprobe_fail_cleanup(fp); 257 return ret; 258 } 259 EXPORT_SYMBOL_GPL(register_fprobe_ips); 260 261 /** 262 * register_fprobe_syms() - Register fprobe to ftrace by symbols. 263 * @fp: A fprobe data structure to be registered. 264 * @syms: An array of target symbols. 265 * @num: The number of entries of @syms. 266 * 267 * Register @fp to the symbols given by @syms array. This will be useful if 268 * you are sure the symbols exist in the kernel. 269 * 270 * Return 0 if @fp is registered successfully, -errno if not. 271 */ 272 int register_fprobe_syms(struct fprobe *fp, const char **syms, int num) 273 { 274 unsigned long *addrs; 275 int ret; 276 277 if (!fp || !syms || num <= 0) 278 return -EINVAL; 279 280 addrs = get_ftrace_locations(syms, num); 281 if (IS_ERR(addrs)) 282 return PTR_ERR(addrs); 283 284 ret = register_fprobe_ips(fp, addrs, num); 285 286 kfree(addrs); 287 288 return ret; 289 } 290 EXPORT_SYMBOL_GPL(register_fprobe_syms); 291 292 /** 293 * unregister_fprobe() - Unregister fprobe from ftrace 294 * @fp: A fprobe data structure to be unregistered. 295 * 296 * Unregister fprobe (and remove ftrace hooks from the function entries). 297 * 298 * Return 0 if @fp is unregistered successfully, -errno if not. 299 */ 300 int unregister_fprobe(struct fprobe *fp) 301 { 302 int ret; 303 304 if (!fp || fp->ops.func != fprobe_handler) 305 return -EINVAL; 306 307 /* 308 * rethook_free() starts disabling the rethook, but the rethook handlers 309 * may be running on other processors at this point. To make sure that all 310 * current running handlers are finished, call unregister_ftrace_function() 311 * after this. 312 */ 313 if (fp->rethook) 314 rethook_free(fp->rethook); 315 316 ret = unregister_ftrace_function(&fp->ops); 317 if (ret < 0) 318 return ret; 319 320 ftrace_free_filter(&fp->ops); 321 322 return ret; 323 } 324 EXPORT_SYMBOL_GPL(unregister_fprobe); 325