1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * fprobe - Simple ftrace probe wrapper for function entry. 4 */ 5 #define pr_fmt(fmt) "fprobe: " fmt 6 7 #include <linux/err.h> 8 #include <linux/fprobe.h> 9 #include <linux/kallsyms.h> 10 #include <linux/kprobes.h> 11 #include <linux/rethook.h> 12 #include <linux/slab.h> 13 #include <linux/sort.h> 14 15 #include "trace.h" 16 17 struct fprobe_rethook_node { 18 struct rethook_node node; 19 unsigned long entry_ip; 20 unsigned long entry_parent_ip; 21 char data[]; 22 }; 23 24 static inline void __fprobe_handler(unsigned long ip, unsigned long parent_ip, 25 struct ftrace_ops *ops, struct ftrace_regs *fregs) 26 { 27 struct fprobe_rethook_node *fpr; 28 struct rethook_node *rh = NULL; 29 struct fprobe *fp; 30 void *entry_data = NULL; 31 int ret = 0; 32 33 fp = container_of(ops, struct fprobe, ops); 34 35 if (fp->exit_handler) { 36 rh = rethook_try_get(fp->rethook); 37 if (!rh) { 38 fp->nmissed++; 39 return; 40 } 41 fpr = container_of(rh, struct fprobe_rethook_node, node); 42 fpr->entry_ip = ip; 43 fpr->entry_parent_ip = parent_ip; 44 if (fp->entry_data_size) 45 entry_data = fpr->data; 46 } 47 48 if (fp->entry_handler) 49 ret = fp->entry_handler(fp, ip, parent_ip, fregs, entry_data); 50 51 /* If entry_handler returns !0, nmissed is not counted. */ 52 if (rh) { 53 if (ret) 54 rethook_recycle(rh); 55 else 56 rethook_hook(rh, ftrace_get_regs(fregs), true); 57 } 58 } 59 60 static void fprobe_handler(unsigned long ip, unsigned long parent_ip, 61 struct ftrace_ops *ops, struct ftrace_regs *fregs) 62 { 63 struct fprobe *fp; 64 int bit; 65 66 fp = container_of(ops, struct fprobe, ops); 67 if (fprobe_disabled(fp)) 68 return; 69 70 /* recursion detection has to go before any traceable function and 71 * all functions before this point should be marked as notrace 72 */ 73 bit = ftrace_test_recursion_trylock(ip, parent_ip); 74 if (bit < 0) { 75 fp->nmissed++; 76 return; 77 } 78 __fprobe_handler(ip, parent_ip, ops, fregs); 79 ftrace_test_recursion_unlock(bit); 80 81 } 82 NOKPROBE_SYMBOL(fprobe_handler); 83 84 static void fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip, 85 struct ftrace_ops *ops, struct ftrace_regs *fregs) 86 { 87 struct fprobe *fp; 88 int bit; 89 90 fp = container_of(ops, struct fprobe, ops); 91 if (fprobe_disabled(fp)) 92 return; 93 94 /* recursion detection has to go before any traceable function and 95 * all functions called before this point should be marked as notrace 96 */ 97 bit = ftrace_test_recursion_trylock(ip, parent_ip); 98 if (bit < 0) { 99 fp->nmissed++; 100 return; 101 } 102 103 /* 104 * This user handler is shared with other kprobes and is not expected to be 105 * called recursively. So if any other kprobe handler is running, this will 106 * exit as kprobe does. See the section 'Share the callbacks with kprobes' 107 * in Documentation/trace/fprobe.rst for more information. 108 */ 109 if (unlikely(kprobe_running())) { 110 fp->nmissed++; 111 goto recursion_unlock; 112 } 113 114 kprobe_busy_begin(); 115 __fprobe_handler(ip, parent_ip, ops, fregs); 116 kprobe_busy_end(); 117 118 recursion_unlock: 119 ftrace_test_recursion_unlock(bit); 120 } 121 122 static void fprobe_exit_handler(struct rethook_node *rh, void *data, 123 unsigned long ret_ip, struct pt_regs *regs) 124 { 125 struct fprobe *fp = (struct fprobe *)data; 126 struct fprobe_rethook_node *fpr; 127 int bit; 128 129 if (!fp || fprobe_disabled(fp)) 130 return; 131 132 fpr = container_of(rh, struct fprobe_rethook_node, node); 133 134 /* 135 * we need to assure no calls to traceable functions in-between the 136 * end of fprobe_handler and the beginning of fprobe_exit_handler. 137 */ 138 bit = ftrace_test_recursion_trylock(fpr->entry_ip, fpr->entry_parent_ip); 139 if (bit < 0) { 140 fp->nmissed++; 141 return; 142 } 143 144 fp->exit_handler(fp, fpr->entry_ip, ret_ip, regs, 145 fp->entry_data_size ? (void *)fpr->data : NULL); 146 ftrace_test_recursion_unlock(bit); 147 } 148 NOKPROBE_SYMBOL(fprobe_exit_handler); 149 150 static int symbols_cmp(const void *a, const void *b) 151 { 152 const char **str_a = (const char **) a; 153 const char **str_b = (const char **) b; 154 155 return strcmp(*str_a, *str_b); 156 } 157 158 /* Convert ftrace location address from symbols */ 159 static unsigned long *get_ftrace_locations(const char **syms, int num) 160 { 161 unsigned long *addrs; 162 163 /* Convert symbols to symbol address */ 164 addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL); 165 if (!addrs) 166 return ERR_PTR(-ENOMEM); 167 168 /* ftrace_lookup_symbols expects sorted symbols */ 169 sort(syms, num, sizeof(*syms), symbols_cmp, NULL); 170 171 if (!ftrace_lookup_symbols(syms, num, addrs)) 172 return addrs; 173 174 kfree(addrs); 175 return ERR_PTR(-ENOENT); 176 } 177 178 static void fprobe_init(struct fprobe *fp) 179 { 180 fp->nmissed = 0; 181 if (fprobe_shared_with_kprobes(fp)) 182 fp->ops.func = fprobe_kprobe_handler; 183 else 184 fp->ops.func = fprobe_handler; 185 186 fp->ops.flags |= FTRACE_OPS_FL_SAVE_REGS; 187 } 188 189 static int fprobe_init_rethook(struct fprobe *fp, int num) 190 { 191 int size; 192 193 if (!fp->exit_handler) { 194 fp->rethook = NULL; 195 return 0; 196 } 197 198 /* Initialize rethook if needed */ 199 if (fp->nr_maxactive) 200 num = fp->nr_maxactive; 201 else 202 num *= num_possible_cpus() * 2; 203 if (num <= 0) 204 return -EINVAL; 205 206 size = sizeof(struct fprobe_rethook_node) + fp->entry_data_size; 207 208 /* Initialize rethook */ 209 fp->rethook = rethook_alloc((void *)fp, fprobe_exit_handler, size, num); 210 if (IS_ERR(fp->rethook)) 211 return PTR_ERR(fp->rethook); 212 213 return 0; 214 } 215 216 static void fprobe_fail_cleanup(struct fprobe *fp) 217 { 218 if (!IS_ERR_OR_NULL(fp->rethook)) { 219 /* Don't need to cleanup rethook->handler because this is not used. */ 220 rethook_free(fp->rethook); 221 fp->rethook = NULL; 222 } 223 ftrace_free_filter(&fp->ops); 224 } 225 226 /** 227 * register_fprobe() - Register fprobe to ftrace by pattern. 228 * @fp: A fprobe data structure to be registered. 229 * @filter: A wildcard pattern of probed symbols. 230 * @notfilter: A wildcard pattern of NOT probed symbols. 231 * 232 * Register @fp to ftrace for enabling the probe on the symbols matched to @filter. 233 * If @notfilter is not NULL, the symbols matched the @notfilter are not probed. 234 * 235 * Return 0 if @fp is registered successfully, -errno if not. 236 */ 237 int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter) 238 { 239 struct ftrace_hash *hash; 240 unsigned char *str; 241 int ret, len; 242 243 if (!fp || !filter) 244 return -EINVAL; 245 246 fprobe_init(fp); 247 248 len = strlen(filter); 249 str = kstrdup(filter, GFP_KERNEL); 250 ret = ftrace_set_filter(&fp->ops, str, len, 0); 251 kfree(str); 252 if (ret) 253 return ret; 254 255 if (notfilter) { 256 len = strlen(notfilter); 257 str = kstrdup(notfilter, GFP_KERNEL); 258 ret = ftrace_set_notrace(&fp->ops, str, len, 0); 259 kfree(str); 260 if (ret) 261 goto out; 262 } 263 264 /* TODO: 265 * correctly calculate the total number of filtered symbols 266 * from both filter and notfilter. 267 */ 268 hash = rcu_access_pointer(fp->ops.local_hash.filter_hash); 269 if (WARN_ON_ONCE(!hash)) 270 goto out; 271 272 ret = fprobe_init_rethook(fp, (int)hash->count); 273 if (!ret) 274 ret = register_ftrace_function(&fp->ops); 275 276 out: 277 if (ret) 278 fprobe_fail_cleanup(fp); 279 return ret; 280 } 281 EXPORT_SYMBOL_GPL(register_fprobe); 282 283 /** 284 * register_fprobe_ips() - Register fprobe to ftrace by address. 285 * @fp: A fprobe data structure to be registered. 286 * @addrs: An array of target ftrace location addresses. 287 * @num: The number of entries of @addrs. 288 * 289 * Register @fp to ftrace for enabling the probe on the address given by @addrs. 290 * The @addrs must be the addresses of ftrace location address, which may be 291 * the symbol address + arch-dependent offset. 292 * If you unsure what this mean, please use other registration functions. 293 * 294 * Return 0 if @fp is registered successfully, -errno if not. 295 */ 296 int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num) 297 { 298 int ret; 299 300 if (!fp || !addrs || num <= 0) 301 return -EINVAL; 302 303 fprobe_init(fp); 304 305 ret = ftrace_set_filter_ips(&fp->ops, addrs, num, 0, 0); 306 if (ret) 307 return ret; 308 309 ret = fprobe_init_rethook(fp, num); 310 if (!ret) 311 ret = register_ftrace_function(&fp->ops); 312 313 if (ret) 314 fprobe_fail_cleanup(fp); 315 return ret; 316 } 317 EXPORT_SYMBOL_GPL(register_fprobe_ips); 318 319 /** 320 * register_fprobe_syms() - Register fprobe to ftrace by symbols. 321 * @fp: A fprobe data structure to be registered. 322 * @syms: An array of target symbols. 323 * @num: The number of entries of @syms. 324 * 325 * Register @fp to the symbols given by @syms array. This will be useful if 326 * you are sure the symbols exist in the kernel. 327 * 328 * Return 0 if @fp is registered successfully, -errno if not. 329 */ 330 int register_fprobe_syms(struct fprobe *fp, const char **syms, int num) 331 { 332 unsigned long *addrs; 333 int ret; 334 335 if (!fp || !syms || num <= 0) 336 return -EINVAL; 337 338 addrs = get_ftrace_locations(syms, num); 339 if (IS_ERR(addrs)) 340 return PTR_ERR(addrs); 341 342 ret = register_fprobe_ips(fp, addrs, num); 343 344 kfree(addrs); 345 346 return ret; 347 } 348 EXPORT_SYMBOL_GPL(register_fprobe_syms); 349 350 bool fprobe_is_registered(struct fprobe *fp) 351 { 352 if (!fp || (fp->ops.saved_func != fprobe_handler && 353 fp->ops.saved_func != fprobe_kprobe_handler)) 354 return false; 355 return true; 356 } 357 358 /** 359 * unregister_fprobe() - Unregister fprobe from ftrace 360 * @fp: A fprobe data structure to be unregistered. 361 * 362 * Unregister fprobe (and remove ftrace hooks from the function entries). 363 * 364 * Return 0 if @fp is unregistered successfully, -errno if not. 365 */ 366 int unregister_fprobe(struct fprobe *fp) 367 { 368 int ret; 369 370 if (!fprobe_is_registered(fp)) 371 return -EINVAL; 372 373 if (!IS_ERR_OR_NULL(fp->rethook)) 374 rethook_stop(fp->rethook); 375 376 ret = unregister_ftrace_function(&fp->ops); 377 if (ret < 0) 378 return ret; 379 380 if (!IS_ERR_OR_NULL(fp->rethook)) 381 rethook_free(fp->rethook); 382 383 ftrace_free_filter(&fp->ops); 384 385 return ret; 386 } 387 EXPORT_SYMBOL_GPL(unregister_fprobe); 388