1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * fprobe - Simple ftrace probe wrapper for function entry. 4 */ 5 #define pr_fmt(fmt) "fprobe: " fmt 6 7 #include <linux/err.h> 8 #include <linux/fprobe.h> 9 #include <linux/kallsyms.h> 10 #include <linux/kprobes.h> 11 #include <linux/rethook.h> 12 #include <linux/slab.h> 13 #include <linux/sort.h> 14 15 #include "trace.h" 16 17 struct fprobe_rethook_node { 18 struct rethook_node node; 19 unsigned long entry_ip; 20 unsigned long entry_parent_ip; 21 char data[]; 22 }; 23 24 static inline void __fprobe_handler(unsigned long ip, unsigned long parent_ip, 25 struct ftrace_ops *ops, struct ftrace_regs *fregs) 26 { 27 struct fprobe_rethook_node *fpr; 28 struct rethook_node *rh = NULL; 29 struct fprobe *fp; 30 void *entry_data = NULL; 31 int ret = 0; 32 33 fp = container_of(ops, struct fprobe, ops); 34 35 if (fp->exit_handler) { 36 rh = rethook_try_get(fp->rethook); 37 if (!rh) { 38 fp->nmissed++; 39 return; 40 } 41 fpr = container_of(rh, struct fprobe_rethook_node, node); 42 fpr->entry_ip = ip; 43 fpr->entry_parent_ip = parent_ip; 44 if (fp->entry_data_size) 45 entry_data = fpr->data; 46 } 47 48 if (fp->entry_handler) 49 ret = fp->entry_handler(fp, ip, parent_ip, ftrace_get_regs(fregs), entry_data); 50 51 /* If entry_handler returns !0, nmissed is not counted. */ 52 if (rh) { 53 if (ret) 54 rethook_recycle(rh); 55 else 56 rethook_hook(rh, ftrace_get_regs(fregs), true); 57 } 58 } 59 60 static void fprobe_handler(unsigned long ip, unsigned long parent_ip, 61 struct ftrace_ops *ops, struct ftrace_regs *fregs) 62 { 63 struct fprobe *fp; 64 int bit; 65 66 fp = container_of(ops, struct fprobe, ops); 67 if (fprobe_disabled(fp)) 68 return; 69 70 /* recursion detection has to go before any traceable function and 71 * all functions before this point should be marked as notrace 72 */ 73 bit = ftrace_test_recursion_trylock(ip, parent_ip); 74 if (bit < 0) { 75 fp->nmissed++; 76 return; 77 } 78 __fprobe_handler(ip, parent_ip, ops, fregs); 79 ftrace_test_recursion_unlock(bit); 80 81 } 82 NOKPROBE_SYMBOL(fprobe_handler); 83 84 static void fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip, 85 struct ftrace_ops *ops, struct ftrace_regs *fregs) 86 { 87 struct fprobe *fp; 88 int bit; 89 90 fp = container_of(ops, struct fprobe, ops); 91 if (fprobe_disabled(fp)) 92 return; 93 94 /* recursion detection has to go before any traceable function and 95 * all functions called before this point should be marked as notrace 96 */ 97 bit = ftrace_test_recursion_trylock(ip, parent_ip); 98 if (bit < 0) { 99 fp->nmissed++; 100 return; 101 } 102 103 /* 104 * This user handler is shared with other kprobes and is not expected to be 105 * called recursively. So if any other kprobe handler is running, this will 106 * exit as kprobe does. See the section 'Share the callbacks with kprobes' 107 * in Documentation/trace/fprobe.rst for more information. 108 */ 109 if (unlikely(kprobe_running())) { 110 fp->nmissed++; 111 goto recursion_unlock; 112 } 113 114 kprobe_busy_begin(); 115 __fprobe_handler(ip, parent_ip, ops, fregs); 116 kprobe_busy_end(); 117 118 recursion_unlock: 119 ftrace_test_recursion_unlock(bit); 120 } 121 122 static void fprobe_exit_handler(struct rethook_node *rh, void *data, 123 unsigned long ret_ip, struct pt_regs *regs) 124 { 125 struct fprobe *fp = (struct fprobe *)data; 126 struct fprobe_rethook_node *fpr; 127 int bit; 128 129 if (!fp || fprobe_disabled(fp)) 130 return; 131 132 fpr = container_of(rh, struct fprobe_rethook_node, node); 133 134 /* 135 * we need to assure no calls to traceable functions in-between the 136 * end of fprobe_handler and the beginning of fprobe_exit_handler. 137 */ 138 bit = ftrace_test_recursion_trylock(fpr->entry_ip, fpr->entry_parent_ip); 139 if (bit < 0) { 140 fp->nmissed++; 141 return; 142 } 143 144 fp->exit_handler(fp, fpr->entry_ip, ret_ip, regs, 145 fp->entry_data_size ? (void *)fpr->data : NULL); 146 ftrace_test_recursion_unlock(bit); 147 } 148 NOKPROBE_SYMBOL(fprobe_exit_handler); 149 150 static int symbols_cmp(const void *a, const void *b) 151 { 152 const char **str_a = (const char **) a; 153 const char **str_b = (const char **) b; 154 155 return strcmp(*str_a, *str_b); 156 } 157 158 /* Convert ftrace location address from symbols */ 159 static unsigned long *get_ftrace_locations(const char **syms, int num) 160 { 161 unsigned long *addrs; 162 163 /* Convert symbols to symbol address */ 164 addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL); 165 if (!addrs) 166 return ERR_PTR(-ENOMEM); 167 168 /* ftrace_lookup_symbols expects sorted symbols */ 169 sort(syms, num, sizeof(*syms), symbols_cmp, NULL); 170 171 if (!ftrace_lookup_symbols(syms, num, addrs)) 172 return addrs; 173 174 kfree(addrs); 175 return ERR_PTR(-ENOENT); 176 } 177 178 static void fprobe_init(struct fprobe *fp) 179 { 180 fp->nmissed = 0; 181 if (fprobe_shared_with_kprobes(fp)) 182 fp->ops.func = fprobe_kprobe_handler; 183 else 184 fp->ops.func = fprobe_handler; 185 fp->ops.flags |= FTRACE_OPS_FL_SAVE_REGS; 186 } 187 188 static int fprobe_init_rethook(struct fprobe *fp, int num) 189 { 190 int size; 191 192 if (!fp->exit_handler) { 193 fp->rethook = NULL; 194 return 0; 195 } 196 197 /* Initialize rethook if needed */ 198 if (fp->nr_maxactive) 199 num = fp->nr_maxactive; 200 else 201 num *= num_possible_cpus() * 2; 202 if (num <= 0) 203 return -EINVAL; 204 205 size = sizeof(struct fprobe_rethook_node) + fp->entry_data_size; 206 207 /* Initialize rethook */ 208 fp->rethook = rethook_alloc((void *)fp, fprobe_exit_handler, size, num); 209 if (IS_ERR(fp->rethook)) 210 return PTR_ERR(fp->rethook); 211 212 return 0; 213 } 214 215 static void fprobe_fail_cleanup(struct fprobe *fp) 216 { 217 if (!IS_ERR_OR_NULL(fp->rethook)) { 218 /* Don't need to cleanup rethook->handler because this is not used. */ 219 rethook_free(fp->rethook); 220 fp->rethook = NULL; 221 } 222 ftrace_free_filter(&fp->ops); 223 } 224 225 /** 226 * register_fprobe() - Register fprobe to ftrace by pattern. 227 * @fp: A fprobe data structure to be registered. 228 * @filter: A wildcard pattern of probed symbols. 229 * @notfilter: A wildcard pattern of NOT probed symbols. 230 * 231 * Register @fp to ftrace for enabling the probe on the symbols matched to @filter. 232 * If @notfilter is not NULL, the symbols matched the @notfilter are not probed. 233 * 234 * Return 0 if @fp is registered successfully, -errno if not. 235 */ 236 int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter) 237 { 238 struct ftrace_hash *hash; 239 unsigned char *str; 240 int ret, len; 241 242 if (!fp || !filter) 243 return -EINVAL; 244 245 fprobe_init(fp); 246 247 len = strlen(filter); 248 str = kstrdup(filter, GFP_KERNEL); 249 ret = ftrace_set_filter(&fp->ops, str, len, 0); 250 kfree(str); 251 if (ret) 252 return ret; 253 254 if (notfilter) { 255 len = strlen(notfilter); 256 str = kstrdup(notfilter, GFP_KERNEL); 257 ret = ftrace_set_notrace(&fp->ops, str, len, 0); 258 kfree(str); 259 if (ret) 260 goto out; 261 } 262 263 /* TODO: 264 * correctly calculate the total number of filtered symbols 265 * from both filter and notfilter. 266 */ 267 hash = rcu_access_pointer(fp->ops.local_hash.filter_hash); 268 if (WARN_ON_ONCE(!hash)) 269 goto out; 270 271 ret = fprobe_init_rethook(fp, (int)hash->count); 272 if (!ret) 273 ret = register_ftrace_function(&fp->ops); 274 275 out: 276 if (ret) 277 fprobe_fail_cleanup(fp); 278 return ret; 279 } 280 EXPORT_SYMBOL_GPL(register_fprobe); 281 282 /** 283 * register_fprobe_ips() - Register fprobe to ftrace by address. 284 * @fp: A fprobe data structure to be registered. 285 * @addrs: An array of target ftrace location addresses. 286 * @num: The number of entries of @addrs. 287 * 288 * Register @fp to ftrace for enabling the probe on the address given by @addrs. 289 * The @addrs must be the addresses of ftrace location address, which may be 290 * the symbol address + arch-dependent offset. 291 * If you unsure what this mean, please use other registration functions. 292 * 293 * Return 0 if @fp is registered successfully, -errno if not. 294 */ 295 int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num) 296 { 297 int ret; 298 299 if (!fp || !addrs || num <= 0) 300 return -EINVAL; 301 302 fprobe_init(fp); 303 304 ret = ftrace_set_filter_ips(&fp->ops, addrs, num, 0, 0); 305 if (ret) 306 return ret; 307 308 ret = fprobe_init_rethook(fp, num); 309 if (!ret) 310 ret = register_ftrace_function(&fp->ops); 311 312 if (ret) 313 fprobe_fail_cleanup(fp); 314 return ret; 315 } 316 EXPORT_SYMBOL_GPL(register_fprobe_ips); 317 318 /** 319 * register_fprobe_syms() - Register fprobe to ftrace by symbols. 320 * @fp: A fprobe data structure to be registered. 321 * @syms: An array of target symbols. 322 * @num: The number of entries of @syms. 323 * 324 * Register @fp to the symbols given by @syms array. This will be useful if 325 * you are sure the symbols exist in the kernel. 326 * 327 * Return 0 if @fp is registered successfully, -errno if not. 328 */ 329 int register_fprobe_syms(struct fprobe *fp, const char **syms, int num) 330 { 331 unsigned long *addrs; 332 int ret; 333 334 if (!fp || !syms || num <= 0) 335 return -EINVAL; 336 337 addrs = get_ftrace_locations(syms, num); 338 if (IS_ERR(addrs)) 339 return PTR_ERR(addrs); 340 341 ret = register_fprobe_ips(fp, addrs, num); 342 343 kfree(addrs); 344 345 return ret; 346 } 347 EXPORT_SYMBOL_GPL(register_fprobe_syms); 348 349 bool fprobe_is_registered(struct fprobe *fp) 350 { 351 if (!fp || (fp->ops.saved_func != fprobe_handler && 352 fp->ops.saved_func != fprobe_kprobe_handler)) 353 return false; 354 return true; 355 } 356 357 /** 358 * unregister_fprobe() - Unregister fprobe from ftrace 359 * @fp: A fprobe data structure to be unregistered. 360 * 361 * Unregister fprobe (and remove ftrace hooks from the function entries). 362 * 363 * Return 0 if @fp is unregistered successfully, -errno if not. 364 */ 365 int unregister_fprobe(struct fprobe *fp) 366 { 367 int ret; 368 369 if (!fprobe_is_registered(fp)) 370 return -EINVAL; 371 372 if (!IS_ERR_OR_NULL(fp->rethook)) 373 rethook_stop(fp->rethook); 374 375 ret = unregister_ftrace_function(&fp->ops); 376 if (ret < 0) 377 return ret; 378 379 if (!IS_ERR_OR_NULL(fp->rethook)) 380 rethook_free(fp->rethook); 381 382 ftrace_free_filter(&fp->ops); 383 384 return ret; 385 } 386 EXPORT_SYMBOL_GPL(unregister_fprobe); 387