1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2011-2015 PLUMgrid, http://plumgrid.com 3 * Copyright (c) 2016 Facebook 4 */ 5 #include <linux/kernel.h> 6 #include <linux/types.h> 7 #include <linux/slab.h> 8 #include <linux/bpf.h> 9 #include <linux/bpf_verifier.h> 10 #include <linux/bpf_perf_event.h> 11 #include <linux/btf.h> 12 #include <linux/filter.h> 13 #include <linux/uaccess.h> 14 #include <linux/ctype.h> 15 #include <linux/kprobes.h> 16 #include <linux/spinlock.h> 17 #include <linux/syscalls.h> 18 #include <linux/error-injection.h> 19 #include <linux/btf_ids.h> 20 #include <linux/bpf_lsm.h> 21 #include <linux/fprobe.h> 22 #include <linux/bsearch.h> 23 #include <linux/sort.h> 24 #include <linux/key.h> 25 #include <linux/namei.h> 26 #include <linux/file.h> 27 28 #include <net/bpf_sk_storage.h> 29 30 #include <uapi/linux/bpf.h> 31 #include <uapi/linux/btf.h> 32 33 #include <asm/tlb.h> 34 35 #include "trace_probe.h" 36 #include "trace.h" 37 38 #define CREATE_TRACE_POINTS 39 #include "bpf_trace.h" 40 41 #define bpf_event_rcu_dereference(p) \ 42 rcu_dereference_protected(p, lockdep_is_held(&bpf_event_mutex)) 43 44 #define MAX_UPROBE_MULTI_CNT (1U << 20) 45 #define MAX_KPROBE_MULTI_CNT (1U << 20) 46 #define MAX_TRACING_MULTI_CNT (1U << 20) 47 48 #ifdef CONFIG_MODULES 49 struct bpf_trace_module { 50 struct module *module; 51 struct list_head list; 52 }; 53 54 static LIST_HEAD(bpf_trace_modules); 55 static DEFINE_MUTEX(bpf_module_mutex); 56 57 static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name) 58 { 59 struct bpf_raw_event_map *btp, *ret = NULL; 60 struct bpf_trace_module *btm; 61 unsigned int i; 62 63 mutex_lock(&bpf_module_mutex); 64 list_for_each_entry(btm, &bpf_trace_modules, list) { 65 for (i = 0; i < btm->module->num_bpf_raw_events; ++i) { 66 btp = &btm->module->bpf_raw_events[i]; 67 if (!strcmp(btp->tp->name, name)) { 68 if (try_module_get(btm->module)) 69 ret = btp; 70 goto out; 71 } 72 } 73 } 74 out: 75 mutex_unlock(&bpf_module_mutex); 76 return ret; 77 } 78 #else 79 static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name) 80 { 81 return NULL; 82 } 83 #endif /* CONFIG_MODULES */ 84 85 u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 86 u64 bpf_get_stack(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 87 88 static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size, 89 u64 flags, const struct btf **btf, 90 s32 *btf_id); 91 static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx); 92 static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx); 93 94 static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx); 95 static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx); 96 97 /** 98 * trace_call_bpf - invoke BPF program 99 * @call: tracepoint event 100 * @ctx: opaque context pointer 101 * 102 * kprobe handlers execute BPF programs via this helper. 103 * Can be used from static tracepoints in the future. 104 * 105 * Return: BPF programs always return an integer which is interpreted by 106 * kprobe handler as: 107 * 0 - return from kprobe (event is filtered out) 108 * 1 - store kprobe event into ring buffer 109 * Other values are reserved and currently alias to 1 110 */ 111 unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) 112 { 113 unsigned int ret; 114 115 cant_sleep(); 116 117 if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) { 118 /* 119 * since some bpf program is already running on this cpu, 120 * don't call into another bpf program (same or different) 121 * and don't send kprobe event into ring-buffer, 122 * so return zero here 123 */ 124 rcu_read_lock(); 125 bpf_prog_inc_misses_counters(rcu_dereference(call->prog_array)); 126 rcu_read_unlock(); 127 ret = 0; 128 goto out; 129 } 130 131 /* 132 * Instead of moving rcu_read_lock/rcu_dereference/rcu_read_unlock 133 * to all call sites, we did a bpf_prog_array_valid() there to check 134 * whether call->prog_array is empty or not, which is 135 * a heuristic to speed up execution. 136 * 137 * If bpf_prog_array_valid() fetched prog_array was 138 * non-NULL, we go into trace_call_bpf() and do the actual 139 * proper rcu_dereference() under RCU lock. 140 * If it turns out that prog_array is NULL then, we bail out. 141 * For the opposite, if the bpf_prog_array_valid() fetched pointer 142 * was NULL, you'll skip the prog_array with the risk of missing 143 * out of events when it was updated in between this and the 144 * rcu_dereference() which is accepted risk. 145 */ 146 rcu_read_lock(); 147 ret = bpf_prog_run_array(rcu_dereference(call->prog_array), 148 ctx, bpf_prog_run); 149 rcu_read_unlock(); 150 151 out: 152 __this_cpu_dec(bpf_prog_active); 153 154 return ret; 155 } 156 157 /** 158 * trace_call_bpf_faultable - invoke BPF program in faultable context 159 * @call: tracepoint event 160 * @ctx: opaque context pointer 161 * 162 * Variant of trace_call_bpf() for faultable tracepoints (syscall 163 * tracepoints). Supports sleepable BPF programs by using rcu_tasks_trace 164 * for lifetime protection and bpf_prog_run_array_sleepable() for per-program 165 * RCU flavor selection, following the uprobe pattern. 166 * 167 * Per-program recursion protection is provided by 168 * bpf_prog_run_array_sleepable(). Global bpf_prog_active is not 169 * needed because syscall tracepoints cannot self-recurse. 170 * 171 * Must be called from a faultable/preemptible context. 172 */ 173 unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx) 174 { 175 struct bpf_prog_array *prog_array; 176 177 might_fault(); 178 guard(rcu_tasks_trace)(); 179 180 prog_array = rcu_dereference_check(call->prog_array, 181 rcu_read_lock_trace_held()); 182 return bpf_prog_run_array_sleepable(prog_array, ctx, bpf_prog_run); 183 } 184 185 #ifdef CONFIG_BPF_KPROBE_OVERRIDE 186 BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc) 187 { 188 regs_set_return_value(regs, rc); 189 override_function_with_return(regs); 190 return 0; 191 } 192 193 static const struct bpf_func_proto bpf_override_return_proto = { 194 .func = bpf_override_return, 195 .gpl_only = true, 196 .ret_type = RET_INTEGER, 197 .arg1_type = ARG_PTR_TO_CTX, 198 .arg2_type = ARG_ANYTHING, 199 }; 200 #endif 201 202 static __always_inline int 203 bpf_probe_read_user_common(void *dst, u32 size, const void __user *unsafe_ptr) 204 { 205 int ret; 206 207 ret = copy_from_user_nofault(dst, unsafe_ptr, size); 208 if (unlikely(ret < 0)) 209 memset(dst, 0, size); 210 return ret; 211 } 212 213 BPF_CALL_3(bpf_probe_read_user, void *, dst, u32, size, 214 const void __user *, unsafe_ptr) 215 { 216 return bpf_probe_read_user_common(dst, size, unsafe_ptr); 217 } 218 219 const struct bpf_func_proto bpf_probe_read_user_proto = { 220 .func = bpf_probe_read_user, 221 .gpl_only = true, 222 .ret_type = RET_INTEGER, 223 .arg1_type = ARG_PTR_TO_UNINIT_MEM, 224 .arg2_type = ARG_MEM_SIZE_OR_ZERO, 225 .arg3_type = ARG_ANYTHING, 226 }; 227 228 static __always_inline int 229 bpf_probe_read_user_str_common(void *dst, u32 size, 230 const void __user *unsafe_ptr) 231 { 232 int ret; 233 234 /* 235 * NB: We rely on strncpy_from_user() not copying junk past the NUL 236 * terminator into `dst`. 237 * 238 * strncpy_from_user() does long-sized strides in the fast path. If the 239 * strncpy does not mask out the bytes after the NUL in `unsafe_ptr`, 240 * then there could be junk after the NUL in `dst`. If user takes `dst` 241 * and keys a hash map with it, then semantically identical strings can 242 * occupy multiple entries in the map. 243 */ 244 ret = strncpy_from_user_nofault(dst, unsafe_ptr, size); 245 if (unlikely(ret < 0)) 246 memset(dst, 0, size); 247 return ret; 248 } 249 250 BPF_CALL_3(bpf_probe_read_user_str, void *, dst, u32, size, 251 const void __user *, unsafe_ptr) 252 { 253 return bpf_probe_read_user_str_common(dst, size, unsafe_ptr); 254 } 255 256 const struct bpf_func_proto bpf_probe_read_user_str_proto = { 257 .func = bpf_probe_read_user_str, 258 .gpl_only = true, 259 .ret_type = RET_INTEGER, 260 .arg1_type = ARG_PTR_TO_UNINIT_MEM, 261 .arg2_type = ARG_MEM_SIZE_OR_ZERO, 262 .arg3_type = ARG_ANYTHING, 263 }; 264 265 BPF_CALL_3(bpf_probe_read_kernel, void *, dst, u32, size, 266 const void *, unsafe_ptr) 267 { 268 return bpf_probe_read_kernel_common(dst, size, unsafe_ptr); 269 } 270 271 const struct bpf_func_proto bpf_probe_read_kernel_proto = { 272 .func = bpf_probe_read_kernel, 273 .gpl_only = true, 274 .ret_type = RET_INTEGER, 275 .arg1_type = ARG_PTR_TO_UNINIT_MEM, 276 .arg2_type = ARG_MEM_SIZE_OR_ZERO, 277 .arg3_type = ARG_ANYTHING, 278 }; 279 280 static __always_inline int 281 bpf_probe_read_kernel_str_common(void *dst, u32 size, const void *unsafe_ptr) 282 { 283 int ret; 284 285 /* 286 * The strncpy_from_kernel_nofault() call will likely not fill the 287 * entire buffer, but that's okay in this circumstance as we're probing 288 * arbitrary memory anyway similar to bpf_probe_read_*() and might 289 * as well probe the stack. Thus, memory is explicitly cleared 290 * only in error case, so that improper users ignoring return 291 * code altogether don't copy garbage; otherwise length of string 292 * is returned that can be used for bpf_perf_event_output() et al. 293 */ 294 ret = strncpy_from_kernel_nofault(dst, unsafe_ptr, size); 295 if (unlikely(ret < 0)) 296 memset(dst, 0, size); 297 return ret; 298 } 299 300 BPF_CALL_3(bpf_probe_read_kernel_str, void *, dst, u32, size, 301 const void *, unsafe_ptr) 302 { 303 return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr); 304 } 305 306 const struct bpf_func_proto bpf_probe_read_kernel_str_proto = { 307 .func = bpf_probe_read_kernel_str, 308 .gpl_only = true, 309 .ret_type = RET_INTEGER, 310 .arg1_type = ARG_PTR_TO_UNINIT_MEM, 311 .arg2_type = ARG_MEM_SIZE_OR_ZERO, 312 .arg3_type = ARG_ANYTHING, 313 }; 314 315 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE 316 BPF_CALL_3(bpf_probe_read_compat, void *, dst, u32, size, 317 const void *, unsafe_ptr) 318 { 319 if ((unsigned long)unsafe_ptr < TASK_SIZE) { 320 return bpf_probe_read_user_common(dst, size, 321 (__force void __user *)unsafe_ptr); 322 } 323 return bpf_probe_read_kernel_common(dst, size, unsafe_ptr); 324 } 325 326 static const struct bpf_func_proto bpf_probe_read_compat_proto = { 327 .func = bpf_probe_read_compat, 328 .gpl_only = true, 329 .ret_type = RET_INTEGER, 330 .arg1_type = ARG_PTR_TO_UNINIT_MEM, 331 .arg2_type = ARG_MEM_SIZE_OR_ZERO, 332 .arg3_type = ARG_ANYTHING, 333 }; 334 335 BPF_CALL_3(bpf_probe_read_compat_str, void *, dst, u32, size, 336 const void *, unsafe_ptr) 337 { 338 if ((unsigned long)unsafe_ptr < TASK_SIZE) { 339 return bpf_probe_read_user_str_common(dst, size, 340 (__force void __user *)unsafe_ptr); 341 } 342 return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr); 343 } 344 345 static const struct bpf_func_proto bpf_probe_read_compat_str_proto = { 346 .func = bpf_probe_read_compat_str, 347 .gpl_only = true, 348 .ret_type = RET_INTEGER, 349 .arg1_type = ARG_PTR_TO_UNINIT_MEM, 350 .arg2_type = ARG_MEM_SIZE_OR_ZERO, 351 .arg3_type = ARG_ANYTHING, 352 }; 353 #endif /* CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE */ 354 355 BPF_CALL_3(bpf_probe_write_user, void __user *, unsafe_ptr, const void *, src, 356 u32, size) 357 { 358 /* 359 * Ensure we're in user context which is safe for the helper to 360 * run. This helper has no business in a kthread. 361 * 362 * access_ok() should prevent writing to non-user memory, but in 363 * some situations (nommu, temporary switch, etc) access_ok() does 364 * not provide enough validation, hence the check on KERNEL_DS. 365 * 366 * nmi_uaccess_okay() ensures the probe is not run in an interim 367 * state, when the task or mm are switched. This is specifically 368 * required to prevent the use of temporary mm. 369 */ 370 371 if (unlikely(in_interrupt() || 372 current->flags & (PF_KTHREAD | PF_EXITING))) 373 return -EPERM; 374 if (unlikely(!nmi_uaccess_okay())) 375 return -EPERM; 376 377 return copy_to_user_nofault(unsafe_ptr, src, size); 378 } 379 380 static const struct bpf_func_proto bpf_probe_write_user_proto = { 381 .func = bpf_probe_write_user, 382 .gpl_only = true, 383 .ret_type = RET_INTEGER, 384 .arg1_type = ARG_ANYTHING, 385 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, 386 .arg3_type = ARG_MEM_SIZE, 387 }; 388 389 #define MAX_TRACE_PRINTK_VARARGS 3 390 #define BPF_TRACE_PRINTK_SIZE 1024 391 392 BPF_CALL_5(bpf_trace_printk, char *, fmt, u32, fmt_size, u64, arg1, 393 u64, arg2, u64, arg3) 394 { 395 u64 args[MAX_TRACE_PRINTK_VARARGS] = { arg1, arg2, arg3 }; 396 struct bpf_bprintf_data data = { 397 .get_bin_args = true, 398 .get_buf = true, 399 }; 400 int ret; 401 402 ret = bpf_bprintf_prepare(fmt, fmt_size, args, 403 MAX_TRACE_PRINTK_VARARGS, &data); 404 if (ret < 0) 405 return ret; 406 407 ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt, data.bin_args); 408 409 trace_bpf_trace_printk(data.buf); 410 411 bpf_bprintf_cleanup(&data); 412 413 return ret; 414 } 415 416 static const struct bpf_func_proto bpf_trace_printk_proto = { 417 .func = bpf_trace_printk, 418 .gpl_only = true, 419 .ret_type = RET_INTEGER, 420 .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, 421 .arg2_type = ARG_MEM_SIZE, 422 }; 423 424 static void __set_printk_clr_event(struct work_struct *work) 425 { 426 /* 427 * This program might be calling bpf_trace_printk, 428 * so enable the associated bpf_trace/bpf_trace_printk event. 429 * Repeat this each time as it is possible a user has 430 * disabled bpf_trace_printk events. By loading a program 431 * calling bpf_trace_printk() however the user has expressed 432 * the intent to see such events. 433 */ 434 if (trace_set_clr_event("bpf_trace", "bpf_trace_printk", 1)) 435 pr_warn_ratelimited("could not enable bpf_trace_printk events"); 436 } 437 static DECLARE_WORK(set_printk_work, __set_printk_clr_event); 438 439 const struct bpf_func_proto *bpf_get_trace_printk_proto(void) 440 { 441 schedule_work(&set_printk_work); 442 return &bpf_trace_printk_proto; 443 } 444 445 BPF_CALL_4(bpf_trace_vprintk, char *, fmt, u32, fmt_size, const void *, args, 446 u32, data_len) 447 { 448 struct bpf_bprintf_data data = { 449 .get_bin_args = true, 450 .get_buf = true, 451 }; 452 int ret, num_args; 453 454 if (data_len & 7 || data_len > MAX_BPRINTF_VARARGS * 8 || 455 (data_len && !args)) 456 return -EINVAL; 457 num_args = data_len / 8; 458 459 ret = bpf_bprintf_prepare(fmt, fmt_size, args, num_args, &data); 460 if (ret < 0) 461 return ret; 462 463 ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt, data.bin_args); 464 465 trace_bpf_trace_printk(data.buf); 466 467 bpf_bprintf_cleanup(&data); 468 469 return ret; 470 } 471 472 static const struct bpf_func_proto bpf_trace_vprintk_proto = { 473 .func = bpf_trace_vprintk, 474 .gpl_only = true, 475 .ret_type = RET_INTEGER, 476 .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, 477 .arg2_type = ARG_MEM_SIZE, 478 .arg3_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, 479 .arg4_type = ARG_MEM_SIZE_OR_ZERO, 480 }; 481 482 const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void) 483 { 484 schedule_work(&set_printk_work); 485 return &bpf_trace_vprintk_proto; 486 } 487 488 BPF_CALL_5(bpf_seq_printf, struct seq_file *, m, char *, fmt, u32, fmt_size, 489 const void *, args, u32, data_len) 490 { 491 struct bpf_bprintf_data data = { 492 .get_bin_args = true, 493 }; 494 int err, num_args; 495 496 if (data_len & 7 || data_len > MAX_BPRINTF_VARARGS * 8 || 497 (data_len && !args)) 498 return -EINVAL; 499 num_args = data_len / 8; 500 501 err = bpf_bprintf_prepare(fmt, fmt_size, args, num_args, &data); 502 if (err < 0) 503 return err; 504 505 seq_bprintf(m, fmt, data.bin_args); 506 507 bpf_bprintf_cleanup(&data); 508 509 return seq_has_overflowed(m) ? -EOVERFLOW : 0; 510 } 511 512 BTF_ID_LIST_SINGLE(btf_seq_file_ids, struct, seq_file) 513 514 static const struct bpf_func_proto bpf_seq_printf_proto = { 515 .func = bpf_seq_printf, 516 .gpl_only = true, 517 .ret_type = RET_INTEGER, 518 .arg1_type = ARG_PTR_TO_BTF_ID, 519 .arg1_btf_id = &btf_seq_file_ids[0], 520 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, 521 .arg3_type = ARG_MEM_SIZE, 522 .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, 523 .arg5_type = ARG_MEM_SIZE_OR_ZERO, 524 }; 525 526 BPF_CALL_3(bpf_seq_write, struct seq_file *, m, const void *, data, u32, len) 527 { 528 return seq_write(m, data, len) ? -EOVERFLOW : 0; 529 } 530 531 static const struct bpf_func_proto bpf_seq_write_proto = { 532 .func = bpf_seq_write, 533 .gpl_only = true, 534 .ret_type = RET_INTEGER, 535 .arg1_type = ARG_PTR_TO_BTF_ID, 536 .arg1_btf_id = &btf_seq_file_ids[0], 537 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, 538 .arg3_type = ARG_MEM_SIZE_OR_ZERO, 539 }; 540 541 BPF_CALL_4(bpf_seq_printf_btf, struct seq_file *, m, struct btf_ptr *, ptr, 542 u32, btf_ptr_size, u64, flags) 543 { 544 const struct btf *btf; 545 s32 btf_id; 546 int ret; 547 548 ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id); 549 if (ret) 550 return ret; 551 552 return btf_type_seq_show_flags(btf, btf_id, ptr->ptr, m, flags); 553 } 554 555 static const struct bpf_func_proto bpf_seq_printf_btf_proto = { 556 .func = bpf_seq_printf_btf, 557 .gpl_only = true, 558 .ret_type = RET_INTEGER, 559 .arg1_type = ARG_PTR_TO_BTF_ID, 560 .arg1_btf_id = &btf_seq_file_ids[0], 561 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, 562 .arg3_type = ARG_MEM_SIZE_OR_ZERO, 563 .arg4_type = ARG_ANYTHING, 564 }; 565 566 static __always_inline int 567 get_map_perf_counter(struct bpf_map *map, u64 flags, 568 u64 *value, u64 *enabled, u64 *running) 569 { 570 struct bpf_array *array = container_of(map, struct bpf_array, map); 571 unsigned int cpu = smp_processor_id(); 572 u64 index = flags & BPF_F_INDEX_MASK; 573 struct bpf_event_entry *ee; 574 575 if (unlikely(flags & ~(BPF_F_INDEX_MASK))) 576 return -EINVAL; 577 if (index == BPF_F_CURRENT_CPU) 578 index = cpu; 579 if (unlikely(index >= array->map.max_entries)) 580 return -E2BIG; 581 582 ee = READ_ONCE(array->ptrs[index]); 583 if (!ee) 584 return -ENOENT; 585 586 return perf_event_read_local(ee->event, value, enabled, running); 587 } 588 589 BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags) 590 { 591 u64 value = 0; 592 int err; 593 594 err = get_map_perf_counter(map, flags, &value, NULL, NULL); 595 /* 596 * this api is ugly since we miss [-22..-2] range of valid 597 * counter values, but that's uapi 598 */ 599 if (err) 600 return err; 601 return value; 602 } 603 604 const struct bpf_func_proto bpf_perf_event_read_proto = { 605 .func = bpf_perf_event_read, 606 .gpl_only = true, 607 .ret_type = RET_INTEGER, 608 .arg1_type = ARG_CONST_MAP_PTR, 609 .arg2_type = ARG_ANYTHING, 610 }; 611 612 BPF_CALL_4(bpf_perf_event_read_value, struct bpf_map *, map, u64, flags, 613 struct bpf_perf_event_value *, buf, u32, size) 614 { 615 int err = -EINVAL; 616 617 if (unlikely(size != sizeof(struct bpf_perf_event_value))) 618 goto clear; 619 err = get_map_perf_counter(map, flags, &buf->counter, &buf->enabled, 620 &buf->running); 621 if (unlikely(err)) 622 goto clear; 623 return 0; 624 clear: 625 memset(buf, 0, size); 626 return err; 627 } 628 629 static const struct bpf_func_proto bpf_perf_event_read_value_proto = { 630 .func = bpf_perf_event_read_value, 631 .gpl_only = true, 632 .ret_type = RET_INTEGER, 633 .arg1_type = ARG_CONST_MAP_PTR, 634 .arg2_type = ARG_ANYTHING, 635 .arg3_type = ARG_PTR_TO_UNINIT_MEM, 636 .arg4_type = ARG_MEM_SIZE, 637 }; 638 639 const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void) 640 { 641 return &bpf_perf_event_read_value_proto; 642 } 643 644 static __always_inline u64 645 __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, 646 u64 flags, struct perf_raw_record *raw, 647 struct perf_sample_data *sd) 648 { 649 struct bpf_array *array = container_of(map, struct bpf_array, map); 650 unsigned int cpu = smp_processor_id(); 651 u64 index = flags & BPF_F_INDEX_MASK; 652 struct bpf_event_entry *ee; 653 struct perf_event *event; 654 655 if (index == BPF_F_CURRENT_CPU) 656 index = cpu; 657 if (unlikely(index >= array->map.max_entries)) 658 return -E2BIG; 659 660 ee = READ_ONCE(array->ptrs[index]); 661 if (!ee) 662 return -ENOENT; 663 664 event = ee->event; 665 if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE || 666 event->attr.config != PERF_COUNT_SW_BPF_OUTPUT)) 667 return -EINVAL; 668 669 if (unlikely(event->oncpu != cpu)) 670 return -EOPNOTSUPP; 671 672 perf_sample_save_raw_data(sd, event, raw); 673 674 return perf_event_output(event, sd, regs); 675 } 676 677 /* 678 * Support executing tracepoints in normal, irq, and nmi context that each call 679 * bpf_perf_event_output 680 */ 681 struct bpf_trace_sample_data { 682 struct perf_sample_data sds[3]; 683 }; 684 685 static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_trace_sds); 686 static DEFINE_PER_CPU(int, bpf_trace_nest_level); 687 BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map, 688 u64, flags, void *, data, u64, size) 689 { 690 struct bpf_trace_sample_data *sds; 691 struct perf_raw_record raw = { 692 .frag = { 693 .size = size, 694 .data = data, 695 }, 696 }; 697 struct perf_sample_data *sd; 698 int nest_level, err; 699 700 preempt_disable(); 701 sds = this_cpu_ptr(&bpf_trace_sds); 702 nest_level = this_cpu_inc_return(bpf_trace_nest_level); 703 704 if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(sds->sds))) { 705 err = -EBUSY; 706 goto out; 707 } 708 709 sd = &sds->sds[nest_level - 1]; 710 711 if (unlikely(flags & ~(BPF_F_INDEX_MASK))) { 712 err = -EINVAL; 713 goto out; 714 } 715 716 perf_sample_data_init(sd, 0, 0); 717 718 err = __bpf_perf_event_output(regs, map, flags, &raw, sd); 719 out: 720 this_cpu_dec(bpf_trace_nest_level); 721 preempt_enable(); 722 return err; 723 } 724 725 static const struct bpf_func_proto bpf_perf_event_output_proto = { 726 .func = bpf_perf_event_output, 727 .gpl_only = true, 728 .ret_type = RET_INTEGER, 729 .arg1_type = ARG_PTR_TO_CTX, 730 .arg2_type = ARG_CONST_MAP_PTR, 731 .arg3_type = ARG_ANYTHING, 732 .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, 733 .arg5_type = ARG_MEM_SIZE_OR_ZERO, 734 }; 735 736 static DEFINE_PER_CPU(int, bpf_event_output_nest_level); 737 struct bpf_nested_pt_regs { 738 struct pt_regs regs[3]; 739 }; 740 static DEFINE_PER_CPU(struct bpf_nested_pt_regs, bpf_pt_regs); 741 static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_misc_sds); 742 743 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 744 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy) 745 { 746 struct perf_raw_frag frag = { 747 .copy = ctx_copy, 748 .size = ctx_size, 749 .data = ctx, 750 }; 751 struct perf_raw_record raw = { 752 .frag = { 753 { 754 .next = ctx_size ? &frag : NULL, 755 }, 756 .size = meta_size, 757 .data = meta, 758 }, 759 }; 760 struct perf_sample_data *sd; 761 struct pt_regs *regs; 762 int nest_level; 763 u64 ret; 764 765 preempt_disable(); 766 nest_level = this_cpu_inc_return(bpf_event_output_nest_level); 767 768 if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(bpf_misc_sds.sds))) { 769 ret = -EBUSY; 770 goto out; 771 } 772 sd = this_cpu_ptr(&bpf_misc_sds.sds[nest_level - 1]); 773 regs = this_cpu_ptr(&bpf_pt_regs.regs[nest_level - 1]); 774 775 perf_fetch_caller_regs(regs); 776 perf_sample_data_init(sd, 0, 0); 777 778 ret = __bpf_perf_event_output(regs, map, flags, &raw, sd); 779 out: 780 this_cpu_dec(bpf_event_output_nest_level); 781 preempt_enable(); 782 return ret; 783 } 784 785 BPF_CALL_0(bpf_get_current_task) 786 { 787 return (long) current; 788 } 789 790 const struct bpf_func_proto bpf_get_current_task_proto = { 791 .func = bpf_get_current_task, 792 .gpl_only = true, 793 .ret_type = RET_INTEGER, 794 }; 795 796 BPF_CALL_0(bpf_get_current_task_btf) 797 { 798 return (unsigned long) current; 799 } 800 801 const struct bpf_func_proto bpf_get_current_task_btf_proto = { 802 .func = bpf_get_current_task_btf, 803 .gpl_only = true, 804 .ret_type = RET_PTR_TO_BTF_ID_TRUSTED, 805 .ret_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], 806 }; 807 808 BPF_CALL_1(bpf_task_pt_regs, struct task_struct *, task) 809 { 810 return (unsigned long) task_pt_regs(task); 811 } 812 813 BTF_ID_LIST_SINGLE(bpf_task_pt_regs_ids, struct, pt_regs) 814 815 const struct bpf_func_proto bpf_task_pt_regs_proto = { 816 .func = bpf_task_pt_regs, 817 .gpl_only = true, 818 .arg1_type = ARG_PTR_TO_BTF_ID, 819 .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], 820 .ret_type = RET_PTR_TO_BTF_ID, 821 .ret_btf_id = &bpf_task_pt_regs_ids[0], 822 }; 823 824 struct send_signal_irq_work { 825 struct irq_work irq_work; 826 /* The claim: NULL marks the slot free for the next caller. */ 827 struct task_struct *task; 828 u32 sig; 829 enum pid_type type; 830 bool has_siginfo; 831 struct kernel_siginfo info; 832 }; 833 834 static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work); 835 836 static void do_bpf_send_signal(struct irq_work *entry) 837 { 838 struct send_signal_irq_work *work; 839 struct kernel_siginfo *siginfo; 840 841 work = container_of(entry, struct send_signal_irq_work, irq_work); 842 siginfo = work->has_siginfo ? &work->info : SEND_SIG_PRIV; 843 844 group_send_sig_info(work->sig, siginfo, work->task, work->type); 845 put_task_struct(work->task); 846 /* Release once the fields are consumed. */ 847 smp_store_release(&work->task, NULL); 848 } 849 850 static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struct *task, u64 value) 851 { 852 struct send_signal_irq_work *work = NULL; 853 struct kernel_siginfo info; 854 struct kernel_siginfo *siginfo; 855 856 if (!task) { 857 task = current; 858 siginfo = SEND_SIG_PRIV; 859 } else { 860 clear_siginfo(&info); 861 info.si_signo = sig; 862 info.si_errno = 0; 863 info.si_code = SI_KERNEL; 864 info.si_pid = 0; 865 info.si_uid = 0; 866 info.si_value.sival_ptr = (void __user __force *)(unsigned long)value; 867 siginfo = &info; 868 } 869 870 /* 871 * Similar to bpf_probe_write_user, task needs to be 872 * in a sound condition. 873 */ 874 if (unlikely(task->flags & (PF_KTHREAD | PF_EXITING))) 875 return -EPERM; 876 /* Task should not be pid=1 to avoid kernel panic. */ 877 if (unlikely(is_global_init(task))) 878 return -EPERM; 879 880 if (preempt_count() != 0 || irqs_disabled()) { 881 /* Do an early check on signal validity. Otherwise, 882 * the error is lost in deferred irq_work. 883 */ 884 if (unlikely(!valid_signal(sig))) 885 return -EINVAL; 886 887 work = this_cpu_ptr(&send_signal_work); 888 if (cmpxchg(&work->task, NULL, task)) 889 return -EBUSY; 890 891 /* Add the current task, which is the target of sending signal, 892 * to the irq_work. The current task may change when queued 893 * irq works get executed. 894 */ 895 get_task_struct(task); 896 work->has_siginfo = siginfo == &info; 897 if (work->has_siginfo) 898 copy_siginfo(&work->info, &info); 899 work->sig = sig; 900 work->type = type; 901 irq_work_queue(&work->irq_work); 902 return 0; 903 } 904 905 return group_send_sig_info(sig, siginfo, task, type); 906 } 907 908 BPF_CALL_1(bpf_send_signal, u32, sig) 909 { 910 return bpf_send_signal_common(sig, PIDTYPE_TGID, NULL, 0); 911 } 912 913 const struct bpf_func_proto bpf_send_signal_proto = { 914 .func = bpf_send_signal, 915 .gpl_only = false, 916 .ret_type = RET_INTEGER, 917 .arg1_type = ARG_ANYTHING, 918 }; 919 920 BPF_CALL_1(bpf_send_signal_thread, u32, sig) 921 { 922 return bpf_send_signal_common(sig, PIDTYPE_PID, NULL, 0); 923 } 924 925 const struct bpf_func_proto bpf_send_signal_thread_proto = { 926 .func = bpf_send_signal_thread, 927 .gpl_only = false, 928 .ret_type = RET_INTEGER, 929 .arg1_type = ARG_ANYTHING, 930 }; 931 932 BPF_CALL_3(bpf_d_path, const struct path *, path, char *, buf, u32, sz) 933 { 934 struct path copy; 935 long len; 936 char *p; 937 938 if (!sz) 939 return 0; 940 941 /* 942 * The path pointer is verified as trusted and safe to use, 943 * but let's double check it's valid anyway to workaround 944 * potentially broken verifier. 945 */ 946 len = copy_from_kernel_nofault(©, path, sizeof(*path)); 947 if (len < 0) 948 return len; 949 950 p = d_path(©, buf, sz); 951 if (IS_ERR(p)) { 952 len = PTR_ERR(p); 953 } else { 954 len = buf + sz - p; 955 memmove(buf, p, len); 956 } 957 958 return len; 959 } 960 961 BTF_SET_START(btf_allowlist_d_path) 962 #ifdef CONFIG_SECURITY 963 BTF_ID(func, security_file_permission) 964 BTF_ID(func, security_inode_getattr) 965 BTF_ID(func, security_file_open) 966 #endif 967 #ifdef CONFIG_SECURITY_PATH 968 BTF_ID(func, security_path_truncate) 969 #endif 970 BTF_ID(func, vfs_truncate) 971 BTF_ID(func, vfs_fallocate) 972 BTF_ID(func, dentry_open) 973 BTF_ID(func, vfs_getattr) 974 BTF_ID(func, filp_close) 975 BTF_SET_END(btf_allowlist_d_path) 976 977 BTF_SET_START(btf_allowlist_d_path_fentry) 978 BTF_ID(func, filp_close_sync) 979 BTF_SET_END(btf_allowlist_d_path_fentry) 980 981 static bool bpf_d_path_allowed(const struct bpf_prog *prog) 982 { 983 if (prog->type == BPF_PROG_TYPE_TRACING && 984 prog->expected_attach_type == BPF_TRACE_ITER) 985 return true; 986 987 if (prog->type == BPF_PROG_TYPE_LSM) 988 return bpf_lsm_is_sleepable_hook(prog->aux->attach_btf_id); 989 990 if (btf_id_set_contains(&btf_allowlist_d_path, 991 prog->aux->attach_btf_id)) 992 return true; 993 994 if (btf_id_set_contains(&btf_allowlist_d_path_fentry, 995 prog->aux->attach_btf_id)) 996 return prog->expected_attach_type == BPF_TRACE_FENTRY; 997 998 return false; 999 } 1000 1001 BTF_ID_LIST_SINGLE(bpf_d_path_btf_ids, struct, path) 1002 1003 static const struct bpf_func_proto bpf_d_path_proto = { 1004 .func = bpf_d_path, 1005 .gpl_only = false, 1006 .ret_type = RET_INTEGER, 1007 .arg1_type = ARG_PTR_TO_BTF_ID, 1008 .arg1_btf_id = &bpf_d_path_btf_ids[0], 1009 .arg2_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, 1010 .arg3_type = ARG_MEM_SIZE_OR_ZERO, 1011 .allowed = bpf_d_path_allowed, 1012 }; 1013 1014 #define BTF_F_ALL (BTF_F_COMPACT | BTF_F_NONAME | \ 1015 BTF_F_PTR_RAW | BTF_F_ZERO) 1016 1017 static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size, 1018 u64 flags, const struct btf **btf, 1019 s32 *btf_id) 1020 { 1021 const struct btf_type *t; 1022 1023 if (unlikely(flags & ~(BTF_F_ALL))) 1024 return -EINVAL; 1025 1026 if (btf_ptr_size != sizeof(struct btf_ptr)) 1027 return -EINVAL; 1028 1029 *btf = bpf_get_btf_vmlinux(); 1030 1031 if (IS_ERR_OR_NULL(*btf)) 1032 return IS_ERR(*btf) ? PTR_ERR(*btf) : -EINVAL; 1033 1034 if (ptr->type_id > 0) 1035 *btf_id = ptr->type_id; 1036 else 1037 return -EINVAL; 1038 1039 if (*btf_id > 0) 1040 t = btf_type_by_id(*btf, *btf_id); 1041 if (*btf_id <= 0 || !t) 1042 return -ENOENT; 1043 1044 return 0; 1045 } 1046 1047 BPF_CALL_5(bpf_snprintf_btf, char *, str, u32, str_size, struct btf_ptr *, ptr, 1048 u32, btf_ptr_size, u64, flags) 1049 { 1050 const struct btf *btf; 1051 s32 btf_id; 1052 int ret; 1053 1054 ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id); 1055 if (ret) 1056 return ret; 1057 1058 return btf_type_snprintf_show(btf, btf_id, ptr->ptr, str, str_size, 1059 flags); 1060 } 1061 1062 const struct bpf_func_proto bpf_snprintf_btf_proto = { 1063 .func = bpf_snprintf_btf, 1064 .gpl_only = false, 1065 .ret_type = RET_INTEGER, 1066 .arg1_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, 1067 .arg2_type = ARG_MEM_SIZE, 1068 .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, 1069 .arg4_type = ARG_MEM_SIZE, 1070 .arg5_type = ARG_ANYTHING, 1071 }; 1072 1073 BPF_CALL_1(bpf_get_func_ip_tracing, void *, ctx) 1074 { 1075 /* This helper call is inlined by verifier. */ 1076 return ((u64 *)ctx)[-2]; 1077 } 1078 1079 static const struct bpf_func_proto bpf_get_func_ip_proto_tracing = { 1080 .func = bpf_get_func_ip_tracing, 1081 .gpl_only = true, 1082 .ret_type = RET_INTEGER, 1083 .arg1_type = ARG_PTR_TO_CTX, 1084 }; 1085 1086 static inline unsigned long get_entry_ip(unsigned long fentry_ip) 1087 { 1088 #ifdef CONFIG_X86_KERNEL_IBT 1089 if (is_endbr((void *)(fentry_ip - ENDBR_INSN_SIZE))) 1090 fentry_ip -= ENDBR_INSN_SIZE; 1091 #endif 1092 return fentry_ip; 1093 } 1094 1095 BPF_CALL_1(bpf_get_func_ip_kprobe, struct pt_regs *, regs) 1096 { 1097 struct bpf_trace_run_ctx *run_ctx __maybe_unused; 1098 struct kprobe *kp; 1099 1100 #ifdef CONFIG_UPROBES 1101 run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx); 1102 if (run_ctx->is_uprobe) 1103 return ((struct uprobe_dispatch_data *)current->utask->vaddr)->bp_addr; 1104 #endif 1105 1106 kp = kprobe_running(); 1107 1108 if (!kp || !(kp->flags & KPROBE_FLAG_ON_FUNC_ENTRY)) 1109 return 0; 1110 1111 return get_entry_ip((uintptr_t)kp->addr); 1112 } 1113 1114 static const struct bpf_func_proto bpf_get_func_ip_proto_kprobe = { 1115 .func = bpf_get_func_ip_kprobe, 1116 .gpl_only = true, 1117 .ret_type = RET_INTEGER, 1118 .arg1_type = ARG_PTR_TO_CTX, 1119 }; 1120 1121 BPF_CALL_1(bpf_get_func_ip_kprobe_multi, struct pt_regs *, regs) 1122 { 1123 return bpf_kprobe_multi_entry_ip(current->bpf_ctx); 1124 } 1125 1126 static const struct bpf_func_proto bpf_get_func_ip_proto_kprobe_multi = { 1127 .func = bpf_get_func_ip_kprobe_multi, 1128 .gpl_only = false, 1129 .ret_type = RET_INTEGER, 1130 .arg1_type = ARG_PTR_TO_CTX, 1131 }; 1132 1133 BPF_CALL_1(bpf_get_attach_cookie_kprobe_multi, struct pt_regs *, regs) 1134 { 1135 return bpf_kprobe_multi_cookie(current->bpf_ctx); 1136 } 1137 1138 static const struct bpf_func_proto bpf_get_attach_cookie_proto_kmulti = { 1139 .func = bpf_get_attach_cookie_kprobe_multi, 1140 .gpl_only = false, 1141 .ret_type = RET_INTEGER, 1142 .arg1_type = ARG_PTR_TO_CTX, 1143 }; 1144 1145 BPF_CALL_1(bpf_get_func_ip_uprobe_multi, struct pt_regs *, regs) 1146 { 1147 return bpf_uprobe_multi_entry_ip(current->bpf_ctx); 1148 } 1149 1150 static const struct bpf_func_proto bpf_get_func_ip_proto_uprobe_multi = { 1151 .func = bpf_get_func_ip_uprobe_multi, 1152 .gpl_only = false, 1153 .ret_type = RET_INTEGER, 1154 .arg1_type = ARG_PTR_TO_CTX, 1155 }; 1156 1157 BPF_CALL_1(bpf_get_attach_cookie_uprobe_multi, struct pt_regs *, regs) 1158 { 1159 return bpf_uprobe_multi_cookie(current->bpf_ctx); 1160 } 1161 1162 static const struct bpf_func_proto bpf_get_attach_cookie_proto_umulti = { 1163 .func = bpf_get_attach_cookie_uprobe_multi, 1164 .gpl_only = false, 1165 .ret_type = RET_INTEGER, 1166 .arg1_type = ARG_PTR_TO_CTX, 1167 }; 1168 1169 BPF_CALL_1(bpf_get_attach_cookie_trace, void *, ctx) 1170 { 1171 struct bpf_trace_run_ctx *run_ctx; 1172 1173 run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx); 1174 return run_ctx->bpf_cookie; 1175 } 1176 1177 static const struct bpf_func_proto bpf_get_attach_cookie_proto_trace = { 1178 .func = bpf_get_attach_cookie_trace, 1179 .gpl_only = false, 1180 .ret_type = RET_INTEGER, 1181 .arg1_type = ARG_PTR_TO_CTX, 1182 }; 1183 1184 BPF_CALL_1(bpf_get_attach_cookie_pe, struct bpf_perf_event_data_kern *, ctx) 1185 { 1186 return ctx->event->bpf_cookie; 1187 } 1188 1189 static const struct bpf_func_proto bpf_get_attach_cookie_proto_pe = { 1190 .func = bpf_get_attach_cookie_pe, 1191 .gpl_only = false, 1192 .ret_type = RET_INTEGER, 1193 .arg1_type = ARG_PTR_TO_CTX, 1194 }; 1195 1196 BPF_CALL_1(bpf_get_attach_cookie_tracing, void *, ctx) 1197 { 1198 struct bpf_trace_run_ctx *run_ctx; 1199 1200 run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx); 1201 return run_ctx->bpf_cookie; 1202 } 1203 1204 static const struct bpf_func_proto bpf_get_attach_cookie_proto_tracing = { 1205 .func = bpf_get_attach_cookie_tracing, 1206 .gpl_only = false, 1207 .ret_type = RET_INTEGER, 1208 .arg1_type = ARG_PTR_TO_CTX, 1209 }; 1210 1211 BPF_CALL_3(bpf_get_branch_snapshot, void *, buf, u32, size, u64, flags) 1212 { 1213 static const u32 br_entry_size = sizeof(struct perf_branch_entry); 1214 u32 entry_cnt = size / br_entry_size; 1215 1216 entry_cnt = static_call(perf_snapshot_branch_stack)(buf, entry_cnt); 1217 1218 if (unlikely(flags)) 1219 return -EINVAL; 1220 1221 if (!entry_cnt) 1222 return -ENOENT; 1223 1224 return entry_cnt * br_entry_size; 1225 } 1226 1227 const struct bpf_func_proto bpf_get_branch_snapshot_proto = { 1228 .func = bpf_get_branch_snapshot, 1229 .gpl_only = true, 1230 .ret_type = RET_INTEGER, 1231 .arg1_type = ARG_PTR_TO_UNINIT_MEM, 1232 .arg2_type = ARG_MEM_SIZE_OR_ZERO, 1233 }; 1234 1235 BPF_CALL_3(get_func_arg, void *, ctx, u32, n, u64 *, value) 1236 { 1237 /* This helper call is inlined by verifier. */ 1238 u64 nr_args = ((u64 *)ctx)[-1] & 0xFF; 1239 1240 if ((u64) n >= nr_args) 1241 return -EINVAL; 1242 *value = ((u64 *)ctx)[n]; 1243 return 0; 1244 } 1245 1246 static const struct bpf_func_proto bpf_get_func_arg_proto = { 1247 .func = get_func_arg, 1248 .ret_type = RET_INTEGER, 1249 .arg1_type = ARG_PTR_TO_CTX, 1250 .arg2_type = ARG_ANYTHING, 1251 .arg3_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, 1252 .arg3_size = sizeof(u64), 1253 }; 1254 1255 BPF_CALL_2(get_func_ret, void *, ctx, u64 *, value) 1256 { 1257 /* This helper call is inlined by verifier. */ 1258 u64 nr_args = ((u64 *)ctx)[-1] & 0xFF; 1259 1260 *value = ((u64 *)ctx)[nr_args]; 1261 return 0; 1262 } 1263 1264 static const struct bpf_func_proto bpf_get_func_ret_proto = { 1265 .func = get_func_ret, 1266 .ret_type = RET_INTEGER, 1267 .arg1_type = ARG_PTR_TO_CTX, 1268 .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, 1269 .arg2_size = sizeof(u64), 1270 }; 1271 1272 BPF_CALL_1(get_func_arg_cnt, void *, ctx) 1273 { 1274 /* This helper call is inlined by verifier. */ 1275 return ((u64 *)ctx)[-1] & 0xFF; 1276 } 1277 1278 static const struct bpf_func_proto bpf_get_func_arg_cnt_proto = { 1279 .func = get_func_arg_cnt, 1280 .ret_type = RET_INTEGER, 1281 .arg1_type = ARG_PTR_TO_CTX, 1282 }; 1283 1284 static const struct bpf_func_proto * 1285 bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1286 { 1287 const struct bpf_func_proto *func_proto; 1288 1289 switch (func_id) { 1290 case BPF_FUNC_get_smp_processor_id: 1291 return &bpf_get_smp_processor_id_proto; 1292 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE 1293 case BPF_FUNC_probe_read: 1294 return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ? 1295 NULL : &bpf_probe_read_compat_proto; 1296 case BPF_FUNC_probe_read_str: 1297 return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ? 1298 NULL : &bpf_probe_read_compat_str_proto; 1299 #endif 1300 case BPF_FUNC_get_func_ip: 1301 return &bpf_get_func_ip_proto_tracing; 1302 default: 1303 break; 1304 } 1305 1306 func_proto = bpf_base_func_proto(func_id, prog); 1307 if (func_proto) 1308 return func_proto; 1309 1310 if (!bpf_token_capable(prog->aux->token, CAP_SYS_ADMIN)) 1311 return NULL; 1312 1313 switch (func_id) { 1314 case BPF_FUNC_probe_write_user: 1315 return security_locked_down(LOCKDOWN_BPF_WRITE_USER) < 0 ? 1316 NULL : &bpf_probe_write_user_proto; 1317 default: 1318 return NULL; 1319 } 1320 } 1321 1322 static bool is_kprobe_multi(const struct bpf_prog *prog) 1323 { 1324 return prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI || 1325 prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION; 1326 } 1327 1328 static inline bool is_kprobe_session(const struct bpf_prog *prog) 1329 { 1330 return prog->type == BPF_PROG_TYPE_KPROBE && 1331 prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION; 1332 } 1333 1334 static inline bool is_uprobe_multi(const struct bpf_prog *prog) 1335 { 1336 return prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI || 1337 prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION; 1338 } 1339 1340 static inline bool is_uprobe_session(const struct bpf_prog *prog) 1341 { 1342 return prog->type == BPF_PROG_TYPE_KPROBE && 1343 prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION; 1344 } 1345 1346 static inline bool is_trace_fsession(const struct bpf_prog *prog) 1347 { 1348 return prog->type == BPF_PROG_TYPE_TRACING && 1349 (prog->expected_attach_type == BPF_TRACE_FSESSION || 1350 prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI); 1351 } 1352 1353 static const struct bpf_func_proto * 1354 kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1355 { 1356 switch (func_id) { 1357 case BPF_FUNC_perf_event_output: 1358 return &bpf_perf_event_output_proto; 1359 case BPF_FUNC_get_stackid: 1360 return &bpf_get_stackid_proto; 1361 case BPF_FUNC_get_stack: 1362 return prog->sleepable ? &bpf_get_stack_sleepable_proto : &bpf_get_stack_proto; 1363 #ifdef CONFIG_BPF_KPROBE_OVERRIDE 1364 case BPF_FUNC_override_return: 1365 return &bpf_override_return_proto; 1366 #endif 1367 case BPF_FUNC_get_func_ip: 1368 if (is_kprobe_multi(prog)) 1369 return &bpf_get_func_ip_proto_kprobe_multi; 1370 if (is_uprobe_multi(prog)) 1371 return &bpf_get_func_ip_proto_uprobe_multi; 1372 return &bpf_get_func_ip_proto_kprobe; 1373 case BPF_FUNC_get_attach_cookie: 1374 if (is_kprobe_multi(prog)) 1375 return &bpf_get_attach_cookie_proto_kmulti; 1376 if (is_uprobe_multi(prog)) 1377 return &bpf_get_attach_cookie_proto_umulti; 1378 return &bpf_get_attach_cookie_proto_trace; 1379 default: 1380 return bpf_tracing_func_proto(func_id, prog); 1381 } 1382 } 1383 1384 /* bpf+kprobe programs can access fields of 'struct pt_regs' */ 1385 static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type type, 1386 const struct bpf_prog *prog, 1387 struct bpf_insn_access_aux *info) 1388 { 1389 if (off < 0 || off >= sizeof(struct pt_regs)) 1390 return false; 1391 if (off % size != 0) 1392 return false; 1393 /* 1394 * Assertion for 32 bit to make sure last 8 byte access 1395 * (BPF_DW) to the last 4 byte member is disallowed. 1396 */ 1397 if (off + size > sizeof(struct pt_regs)) 1398 return false; 1399 1400 if (type == BPF_WRITE) { 1401 /* 1402 * pt_regs also holds what the kernel restores on the way to 1403 * user space: cs and flags on x86, pstate on arm64, status on 1404 * riscv. A prog can write the registers of the task only. 1405 */ 1406 if (!arch_uprobe_regs_writable(off, size)) 1407 return false; 1408 prog->aux->kprobe_write_ctx = true; 1409 } 1410 1411 return true; 1412 } 1413 1414 const struct bpf_verifier_ops kprobe_verifier_ops = { 1415 .get_func_proto = kprobe_prog_func_proto, 1416 .is_valid_access = kprobe_prog_is_valid_access, 1417 }; 1418 1419 const struct bpf_prog_ops kprobe_prog_ops = { 1420 }; 1421 1422 BPF_CALL_5(bpf_perf_event_output_tp, void *, tp_buff, struct bpf_map *, map, 1423 u64, flags, void *, data, u64, size) 1424 { 1425 struct pt_regs *regs = *(struct pt_regs **)tp_buff; 1426 1427 /* 1428 * r1 points to perf tracepoint buffer where first 8 bytes are hidden 1429 * from bpf program and contain a pointer to 'struct pt_regs'. Fetch it 1430 * from there and call the same bpf_perf_event_output() helper inline. 1431 */ 1432 return ____bpf_perf_event_output(regs, map, flags, data, size); 1433 } 1434 1435 static const struct bpf_func_proto bpf_perf_event_output_proto_tp = { 1436 .func = bpf_perf_event_output_tp, 1437 .gpl_only = true, 1438 .ret_type = RET_INTEGER, 1439 .arg1_type = ARG_PTR_TO_CTX, 1440 .arg2_type = ARG_CONST_MAP_PTR, 1441 .arg3_type = ARG_ANYTHING, 1442 .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, 1443 .arg5_type = ARG_MEM_SIZE_OR_ZERO, 1444 }; 1445 1446 BPF_CALL_3(bpf_get_stackid_tp, void *, tp_buff, struct bpf_map *, map, 1447 u64, flags) 1448 { 1449 struct pt_regs *regs = *(struct pt_regs **)tp_buff; 1450 1451 /* 1452 * Same comment as in bpf_perf_event_output_tp(), only that this time 1453 * the other helper's function body cannot be inlined due to being 1454 * external, thus we need to call raw helper function. 1455 */ 1456 return bpf_get_stackid((unsigned long) regs, (unsigned long) map, 1457 flags, 0, 0); 1458 } 1459 1460 static const struct bpf_func_proto bpf_get_stackid_proto_tp = { 1461 .func = bpf_get_stackid_tp, 1462 .gpl_only = true, 1463 .ret_type = RET_INTEGER, 1464 .arg1_type = ARG_PTR_TO_CTX, 1465 .arg2_type = ARG_CONST_MAP_PTR, 1466 .arg3_type = ARG_ANYTHING, 1467 }; 1468 1469 BPF_CALL_4(bpf_get_stack_tp, void *, tp_buff, void *, buf, u32, size, 1470 u64, flags) 1471 { 1472 struct pt_regs *regs = *(struct pt_regs **)tp_buff; 1473 1474 return bpf_get_stack((unsigned long) regs, (unsigned long) buf, 1475 (unsigned long) size, flags, 0); 1476 } 1477 1478 static const struct bpf_func_proto bpf_get_stack_proto_tp = { 1479 .func = bpf_get_stack_tp, 1480 .gpl_only = true, 1481 .ret_type = RET_INTEGER, 1482 .arg1_type = ARG_PTR_TO_CTX, 1483 .arg2_type = ARG_PTR_TO_UNINIT_MEM, 1484 .arg3_type = ARG_MEM_SIZE_OR_ZERO, 1485 .arg4_type = ARG_ANYTHING, 1486 }; 1487 1488 static const struct bpf_func_proto * 1489 tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1490 { 1491 switch (func_id) { 1492 case BPF_FUNC_perf_event_output: 1493 return &bpf_perf_event_output_proto_tp; 1494 case BPF_FUNC_get_stackid: 1495 return &bpf_get_stackid_proto_tp; 1496 case BPF_FUNC_get_stack: 1497 return &bpf_get_stack_proto_tp; 1498 case BPF_FUNC_get_attach_cookie: 1499 return &bpf_get_attach_cookie_proto_trace; 1500 default: 1501 return bpf_tracing_func_proto(func_id, prog); 1502 } 1503 } 1504 1505 static bool tp_prog_is_valid_access(int off, int size, enum bpf_access_type type, 1506 const struct bpf_prog *prog, 1507 struct bpf_insn_access_aux *info) 1508 { 1509 if (off < sizeof(void *) || off >= PERF_MAX_TRACE_SIZE) 1510 return false; 1511 if (type != BPF_READ) 1512 return false; 1513 if (off % size != 0) 1514 return false; 1515 1516 BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(__u64)); 1517 return true; 1518 } 1519 1520 const struct bpf_verifier_ops tracepoint_verifier_ops = { 1521 .get_func_proto = tp_prog_func_proto, 1522 .is_valid_access = tp_prog_is_valid_access, 1523 }; 1524 1525 const struct bpf_prog_ops tracepoint_prog_ops = { 1526 }; 1527 1528 BPF_CALL_3(bpf_perf_prog_read_value, struct bpf_perf_event_data_kern *, ctx, 1529 struct bpf_perf_event_value *, buf, u32, size) 1530 { 1531 int err = -EINVAL; 1532 1533 if (unlikely(size != sizeof(struct bpf_perf_event_value))) 1534 goto clear; 1535 err = perf_event_read_local(ctx->event, &buf->counter, &buf->enabled, 1536 &buf->running); 1537 if (unlikely(err)) 1538 goto clear; 1539 return 0; 1540 clear: 1541 memset(buf, 0, size); 1542 return err; 1543 } 1544 1545 static const struct bpf_func_proto bpf_perf_prog_read_value_proto = { 1546 .func = bpf_perf_prog_read_value, 1547 .gpl_only = true, 1548 .ret_type = RET_INTEGER, 1549 .arg1_type = ARG_PTR_TO_CTX, 1550 .arg2_type = ARG_PTR_TO_UNINIT_MEM, 1551 .arg3_type = ARG_MEM_SIZE, 1552 }; 1553 1554 BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx, 1555 void *, buf, u32, size, u64, flags) 1556 { 1557 static const u32 br_entry_size = sizeof(struct perf_branch_entry); 1558 struct perf_branch_stack *br_stack = ctx->data->br_stack; 1559 u32 to_copy; 1560 1561 if (unlikely(flags & ~BPF_F_GET_BRANCH_RECORDS_SIZE)) 1562 return -EINVAL; 1563 1564 if (unlikely(!(ctx->data->sample_flags & PERF_SAMPLE_BRANCH_STACK))) 1565 return -ENOENT; 1566 1567 if (unlikely(!br_stack)) 1568 return -ENOENT; 1569 1570 if (flags & BPF_F_GET_BRANCH_RECORDS_SIZE) 1571 return br_stack->nr * br_entry_size; 1572 1573 if (!buf || (size % br_entry_size != 0)) 1574 return -EINVAL; 1575 1576 to_copy = min_t(u32, br_stack->nr * br_entry_size, size); 1577 memcpy(buf, br_stack->entries, to_copy); 1578 1579 return to_copy; 1580 } 1581 1582 static const struct bpf_func_proto bpf_read_branch_records_proto = { 1583 .func = bpf_read_branch_records, 1584 .gpl_only = true, 1585 .ret_type = RET_INTEGER, 1586 .arg1_type = ARG_PTR_TO_CTX, 1587 .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_UNINIT | MEM_WRITE, 1588 .arg3_type = ARG_MEM_SIZE_OR_ZERO, 1589 .arg4_type = ARG_ANYTHING, 1590 }; 1591 1592 static const struct bpf_func_proto * 1593 pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1594 { 1595 switch (func_id) { 1596 case BPF_FUNC_perf_event_output: 1597 return &bpf_perf_event_output_proto_tp; 1598 case BPF_FUNC_get_stackid: 1599 return &bpf_get_stackid_proto_pe; 1600 case BPF_FUNC_get_stack: 1601 return &bpf_get_stack_proto_pe; 1602 case BPF_FUNC_perf_prog_read_value: 1603 return &bpf_perf_prog_read_value_proto; 1604 case BPF_FUNC_read_branch_records: 1605 return &bpf_read_branch_records_proto; 1606 case BPF_FUNC_get_attach_cookie: 1607 return &bpf_get_attach_cookie_proto_pe; 1608 default: 1609 return bpf_tracing_func_proto(func_id, prog); 1610 } 1611 } 1612 1613 /* 1614 * bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp 1615 * to avoid potential recursive reuse issue when/if tracepoints are added 1616 * inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack. 1617 * 1618 * Since raw tracepoints run despite bpf_prog_active, support concurrent usage 1619 * in normal, irq, and nmi context. 1620 */ 1621 struct bpf_raw_tp_regs { 1622 struct pt_regs regs[3]; 1623 }; 1624 static DEFINE_PER_CPU(struct bpf_raw_tp_regs, bpf_raw_tp_regs); 1625 static DEFINE_PER_CPU(int, bpf_raw_tp_nest_level); 1626 static struct pt_regs *get_bpf_raw_tp_regs(void) 1627 { 1628 struct bpf_raw_tp_regs *tp_regs = this_cpu_ptr(&bpf_raw_tp_regs); 1629 int nest_level = this_cpu_inc_return(bpf_raw_tp_nest_level); 1630 1631 if (nest_level > ARRAY_SIZE(tp_regs->regs)) { 1632 this_cpu_dec(bpf_raw_tp_nest_level); 1633 return ERR_PTR(-EBUSY); 1634 } 1635 1636 return &tp_regs->regs[nest_level - 1]; 1637 } 1638 1639 static void put_bpf_raw_tp_regs(void) 1640 { 1641 this_cpu_dec(bpf_raw_tp_nest_level); 1642 } 1643 1644 BPF_CALL_5(bpf_perf_event_output_raw_tp, struct bpf_raw_tracepoint_args *, args, 1645 struct bpf_map *, map, u64, flags, void *, data, u64, size) 1646 { 1647 struct pt_regs *regs = get_bpf_raw_tp_regs(); 1648 int ret; 1649 1650 if (IS_ERR(regs)) 1651 return PTR_ERR(regs); 1652 1653 perf_fetch_caller_regs(regs); 1654 ret = ____bpf_perf_event_output(regs, map, flags, data, size); 1655 1656 put_bpf_raw_tp_regs(); 1657 return ret; 1658 } 1659 1660 static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = { 1661 .func = bpf_perf_event_output_raw_tp, 1662 .gpl_only = true, 1663 .ret_type = RET_INTEGER, 1664 .arg1_type = ARG_PTR_TO_CTX, 1665 .arg2_type = ARG_CONST_MAP_PTR, 1666 .arg3_type = ARG_ANYTHING, 1667 .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, 1668 .arg5_type = ARG_MEM_SIZE_OR_ZERO, 1669 }; 1670 1671 extern const struct bpf_func_proto bpf_skb_output_proto; 1672 extern const struct bpf_func_proto bpf_xdp_output_proto; 1673 extern const struct bpf_func_proto bpf_xdp_get_buff_len_trace_proto; 1674 1675 BPF_CALL_3(bpf_get_stackid_raw_tp, struct bpf_raw_tracepoint_args *, args, 1676 struct bpf_map *, map, u64, flags) 1677 { 1678 struct pt_regs *regs = get_bpf_raw_tp_regs(); 1679 int ret; 1680 1681 if (IS_ERR(regs)) 1682 return PTR_ERR(regs); 1683 1684 perf_fetch_caller_regs(regs); 1685 /* similar to bpf_perf_event_output_tp, but pt_regs fetched differently */ 1686 ret = bpf_get_stackid((unsigned long) regs, (unsigned long) map, 1687 flags, 0, 0); 1688 put_bpf_raw_tp_regs(); 1689 return ret; 1690 } 1691 1692 static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = { 1693 .func = bpf_get_stackid_raw_tp, 1694 .gpl_only = true, 1695 .ret_type = RET_INTEGER, 1696 .arg1_type = ARG_PTR_TO_CTX, 1697 .arg2_type = ARG_CONST_MAP_PTR, 1698 .arg3_type = ARG_ANYTHING, 1699 }; 1700 1701 BPF_CALL_4(bpf_get_stack_raw_tp, struct bpf_raw_tracepoint_args *, args, 1702 void *, buf, u32, size, u64, flags) 1703 { 1704 struct pt_regs *regs = get_bpf_raw_tp_regs(); 1705 int ret; 1706 1707 if (IS_ERR(regs)) 1708 return PTR_ERR(regs); 1709 1710 perf_fetch_caller_regs(regs); 1711 ret = bpf_get_stack((unsigned long) regs, (unsigned long) buf, 1712 (unsigned long) size, flags, 0); 1713 put_bpf_raw_tp_regs(); 1714 return ret; 1715 } 1716 1717 static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { 1718 .func = bpf_get_stack_raw_tp, 1719 .gpl_only = true, 1720 .ret_type = RET_INTEGER, 1721 .arg1_type = ARG_PTR_TO_CTX, 1722 .arg2_type = ARG_PTR_TO_UNINIT_MEM, 1723 .arg3_type = ARG_MEM_SIZE_OR_ZERO, 1724 .arg4_type = ARG_ANYTHING, 1725 }; 1726 1727 static const struct bpf_func_proto * 1728 raw_tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1729 { 1730 switch (func_id) { 1731 case BPF_FUNC_perf_event_output: 1732 return &bpf_perf_event_output_proto_raw_tp; 1733 case BPF_FUNC_get_stackid: 1734 return &bpf_get_stackid_proto_raw_tp; 1735 case BPF_FUNC_get_stack: 1736 return &bpf_get_stack_proto_raw_tp; 1737 case BPF_FUNC_get_attach_cookie: 1738 return &bpf_get_attach_cookie_proto_tracing; 1739 default: 1740 return bpf_tracing_func_proto(func_id, prog); 1741 } 1742 } 1743 1744 const struct bpf_func_proto * 1745 tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1746 { 1747 const struct bpf_func_proto *fn; 1748 1749 switch (func_id) { 1750 #ifdef CONFIG_NET 1751 case BPF_FUNC_skb_output: 1752 return &bpf_skb_output_proto; 1753 case BPF_FUNC_xdp_output: 1754 return &bpf_xdp_output_proto; 1755 case BPF_FUNC_skc_to_tcp6_sock: 1756 return &bpf_skc_to_tcp6_sock_proto; 1757 case BPF_FUNC_skc_to_tcp_sock: 1758 return &bpf_skc_to_tcp_sock_proto; 1759 case BPF_FUNC_skc_to_tcp_timewait_sock: 1760 return &bpf_skc_to_tcp_timewait_sock_proto; 1761 case BPF_FUNC_skc_to_tcp_request_sock: 1762 return &bpf_skc_to_tcp_request_sock_proto; 1763 case BPF_FUNC_skc_to_udp6_sock: 1764 return &bpf_skc_to_udp6_sock_proto; 1765 case BPF_FUNC_skc_to_unix_sock: 1766 return &bpf_skc_to_unix_sock_proto; 1767 case BPF_FUNC_skc_to_mptcp_sock: 1768 return &bpf_skc_to_mptcp_sock_proto; 1769 case BPF_FUNC_sk_storage_get: 1770 return &bpf_sk_storage_get_tracing_proto; 1771 case BPF_FUNC_sk_storage_delete: 1772 return &bpf_sk_storage_delete_tracing_proto; 1773 case BPF_FUNC_sock_from_file: 1774 return &bpf_sock_from_file_proto; 1775 case BPF_FUNC_get_socket_cookie: 1776 return &bpf_get_socket_ptr_cookie_proto; 1777 case BPF_FUNC_xdp_get_buff_len: 1778 return &bpf_xdp_get_buff_len_trace_proto; 1779 #endif 1780 case BPF_FUNC_seq_printf: 1781 return prog->expected_attach_type == BPF_TRACE_ITER ? 1782 &bpf_seq_printf_proto : 1783 NULL; 1784 case BPF_FUNC_seq_write: 1785 return prog->expected_attach_type == BPF_TRACE_ITER ? 1786 &bpf_seq_write_proto : 1787 NULL; 1788 case BPF_FUNC_seq_printf_btf: 1789 return prog->expected_attach_type == BPF_TRACE_ITER ? 1790 &bpf_seq_printf_btf_proto : 1791 NULL; 1792 case BPF_FUNC_d_path: 1793 return &bpf_d_path_proto; 1794 case BPF_FUNC_get_func_arg: 1795 if (bpf_prog_has_trampoline(prog) || 1796 prog->expected_attach_type == BPF_TRACE_RAW_TP) 1797 return &bpf_get_func_arg_proto; 1798 return NULL; 1799 case BPF_FUNC_get_func_ret: 1800 return bpf_prog_has_trampoline(prog) ? &bpf_get_func_ret_proto : NULL; 1801 case BPF_FUNC_get_func_arg_cnt: 1802 if (bpf_prog_has_trampoline(prog) || 1803 prog->expected_attach_type == BPF_TRACE_RAW_TP) 1804 return &bpf_get_func_arg_cnt_proto; 1805 return NULL; 1806 case BPF_FUNC_get_attach_cookie: 1807 if (prog->type == BPF_PROG_TYPE_TRACING && 1808 prog->expected_attach_type == BPF_TRACE_RAW_TP) 1809 return &bpf_get_attach_cookie_proto_tracing; 1810 return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto_tracing : NULL; 1811 default: 1812 fn = raw_tp_prog_func_proto(func_id, prog); 1813 if (!fn && prog->expected_attach_type == BPF_TRACE_ITER) 1814 fn = bpf_iter_get_func_proto(func_id, prog); 1815 return fn; 1816 } 1817 } 1818 1819 static bool raw_tp_prog_is_valid_access(int off, int size, 1820 enum bpf_access_type type, 1821 const struct bpf_prog *prog, 1822 struct bpf_insn_access_aux *info) 1823 { 1824 return bpf_tracing_ctx_access(off, size, type); 1825 } 1826 1827 static bool tracing_prog_is_valid_access(int off, int size, 1828 enum bpf_access_type type, 1829 const struct bpf_prog *prog, 1830 struct bpf_insn_access_aux *info) 1831 { 1832 return bpf_tracing_btf_ctx_access(off, size, type, prog, info); 1833 } 1834 1835 int __weak bpf_prog_test_run_tracing(struct bpf_prog *prog, 1836 const union bpf_attr *kattr, 1837 union bpf_attr __user *uattr) 1838 { 1839 return -ENOTSUPP; 1840 } 1841 1842 const struct bpf_verifier_ops raw_tracepoint_verifier_ops = { 1843 .get_func_proto = raw_tp_prog_func_proto, 1844 .is_valid_access = raw_tp_prog_is_valid_access, 1845 }; 1846 1847 const struct bpf_prog_ops raw_tracepoint_prog_ops = { 1848 #ifdef CONFIG_NET 1849 .test_run = bpf_prog_test_run_raw_tp, 1850 #endif 1851 }; 1852 1853 const struct bpf_verifier_ops tracing_verifier_ops = { 1854 .get_func_proto = tracing_prog_func_proto, 1855 .is_valid_access = tracing_prog_is_valid_access, 1856 }; 1857 1858 const struct bpf_prog_ops tracing_prog_ops = { 1859 .test_run = bpf_prog_test_run_tracing, 1860 }; 1861 1862 static bool raw_tp_writable_prog_is_valid_access(int off, int size, 1863 enum bpf_access_type type, 1864 const struct bpf_prog *prog, 1865 struct bpf_insn_access_aux *info) 1866 { 1867 if (off == 0) { 1868 if (size != sizeof(u64) || type != BPF_READ) 1869 return false; 1870 info->reg_type = PTR_TO_TP_BUFFER; 1871 } 1872 return raw_tp_prog_is_valid_access(off, size, type, prog, info); 1873 } 1874 1875 const struct bpf_verifier_ops raw_tracepoint_writable_verifier_ops = { 1876 .get_func_proto = raw_tp_prog_func_proto, 1877 .is_valid_access = raw_tp_writable_prog_is_valid_access, 1878 }; 1879 1880 const struct bpf_prog_ops raw_tracepoint_writable_prog_ops = { 1881 }; 1882 1883 static bool pe_prog_is_valid_access(int off, int size, enum bpf_access_type type, 1884 const struct bpf_prog *prog, 1885 struct bpf_insn_access_aux *info) 1886 { 1887 const int size_u64 = sizeof(u64); 1888 1889 if (off < 0 || off >= sizeof(struct bpf_perf_event_data)) 1890 return false; 1891 if (type != BPF_READ) 1892 return false; 1893 if (off % size != 0) { 1894 if (sizeof(unsigned long) != 4) 1895 return false; 1896 if (size != 8) 1897 return false; 1898 if (off % size != 4) 1899 return false; 1900 } 1901 1902 switch (off) { 1903 case bpf_ctx_range(struct bpf_perf_event_data, sample_period): 1904 bpf_ctx_record_field_size(info, size_u64); 1905 if (!bpf_ctx_narrow_access_ok(off, size, size_u64)) 1906 return false; 1907 break; 1908 case bpf_ctx_range(struct bpf_perf_event_data, addr): 1909 bpf_ctx_record_field_size(info, size_u64); 1910 if (!bpf_ctx_narrow_access_ok(off, size, size_u64)) 1911 return false; 1912 break; 1913 default: 1914 if (size != sizeof(long)) 1915 return false; 1916 } 1917 1918 return true; 1919 } 1920 1921 static u32 pe_prog_convert_ctx_access(enum bpf_access_type type, 1922 const struct bpf_insn *si, 1923 struct bpf_insn *insn_buf, 1924 struct bpf_prog *prog, u32 *target_size) 1925 { 1926 struct bpf_insn *insn = insn_buf; 1927 1928 switch (si->off) { 1929 case offsetof(struct bpf_perf_event_data, sample_period): 1930 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, 1931 data), si->dst_reg, si->src_reg, 1932 offsetof(struct bpf_perf_event_data_kern, data)); 1933 *insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg, 1934 bpf_target_off(struct perf_sample_data, period, 8, 1935 target_size)); 1936 break; 1937 case offsetof(struct bpf_perf_event_data, addr): 1938 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, 1939 data), si->dst_reg, si->src_reg, 1940 offsetof(struct bpf_perf_event_data_kern, data)); 1941 *insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg, 1942 bpf_target_off(struct perf_sample_data, addr, 8, 1943 target_size)); 1944 break; 1945 default: 1946 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, 1947 regs), si->dst_reg, si->src_reg, 1948 offsetof(struct bpf_perf_event_data_kern, regs)); 1949 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(long), si->dst_reg, si->dst_reg, 1950 si->off); 1951 break; 1952 } 1953 1954 return insn - insn_buf; 1955 } 1956 1957 const struct bpf_verifier_ops perf_event_verifier_ops = { 1958 .get_func_proto = pe_prog_func_proto, 1959 .is_valid_access = pe_prog_is_valid_access, 1960 .convert_ctx_access = pe_prog_convert_ctx_access, 1961 }; 1962 1963 const struct bpf_prog_ops perf_event_prog_ops = { 1964 }; 1965 1966 static DEFINE_MUTEX(bpf_event_mutex); 1967 1968 #define BPF_TRACE_MAX_PROGS 64 1969 1970 int perf_event_attach_bpf_prog(struct perf_event *event, 1971 struct bpf_prog *prog, 1972 u64 bpf_cookie) 1973 { 1974 struct bpf_prog_array *old_array; 1975 struct bpf_prog_array *new_array; 1976 int ret = -EEXIST; 1977 1978 /* 1979 * Kprobe override only works if they are on the function entry, 1980 * and only if they are on the opt-in list. 1981 */ 1982 if (prog->kprobe_override && 1983 (!trace_kprobe_on_func_entry(event->tp_event) || 1984 !trace_kprobe_error_injectable(event->tp_event))) 1985 return -EINVAL; 1986 1987 mutex_lock(&bpf_event_mutex); 1988 1989 if (event->prog) 1990 goto unlock; 1991 1992 old_array = bpf_event_rcu_dereference(event->tp_event->prog_array); 1993 if (old_array && 1994 bpf_prog_array_length(old_array) >= BPF_TRACE_MAX_PROGS) { 1995 ret = -E2BIG; 1996 goto unlock; 1997 } 1998 1999 ret = bpf_prog_array_copy(old_array, NULL, prog, bpf_cookie, &new_array); 2000 if (ret < 0) 2001 goto unlock; 2002 2003 /* set the new array to event->tp_event and set event->prog */ 2004 event->prog = prog; 2005 event->bpf_cookie = bpf_cookie; 2006 rcu_assign_pointer(event->tp_event->prog_array, new_array); 2007 bpf_prog_array_free_sleepable(old_array); 2008 2009 unlock: 2010 mutex_unlock(&bpf_event_mutex); 2011 return ret; 2012 } 2013 2014 void perf_event_detach_bpf_prog(struct perf_event *event) 2015 { 2016 struct bpf_prog_array *old_array; 2017 struct bpf_prog_array *new_array; 2018 struct bpf_prog *prog = NULL; 2019 int ret; 2020 2021 mutex_lock(&bpf_event_mutex); 2022 2023 if (!event->prog) 2024 goto unlock; 2025 2026 old_array = bpf_event_rcu_dereference(event->tp_event->prog_array); 2027 if (!old_array) 2028 goto put; 2029 2030 ret = bpf_prog_array_copy(old_array, event->prog, NULL, 0, &new_array); 2031 if (ret < 0) { 2032 bpf_prog_array_delete_safe(old_array, event->prog); 2033 } else { 2034 rcu_assign_pointer(event->tp_event->prog_array, new_array); 2035 bpf_prog_array_free_sleepable(old_array); 2036 } 2037 2038 put: 2039 prog = event->prog; 2040 event->prog = NULL; 2041 2042 unlock: 2043 mutex_unlock(&bpf_event_mutex); 2044 2045 if (prog) { 2046 /* 2047 * It could be that the bpf_prog is not sleepable (and will be freed 2048 * via normal RCU), but is called from a point that supports sleepable 2049 * programs and uses tasks-trace-RCU. 2050 */ 2051 synchronize_rcu_tasks_trace(); 2052 2053 bpf_prog_put(prog); 2054 } 2055 } 2056 2057 int perf_event_query_prog_array(struct perf_event *event, void __user *info) 2058 { 2059 struct perf_event_query_bpf __user *uquery = info; 2060 struct perf_event_query_bpf query = {}; 2061 struct bpf_prog_array *progs; 2062 u32 *ids, prog_cnt, ids_len; 2063 int ret; 2064 2065 if (!perfmon_capable()) 2066 return -EPERM; 2067 if (event->attr.type != PERF_TYPE_TRACEPOINT) 2068 return -EINVAL; 2069 if (copy_from_user(&query, uquery, sizeof(query))) 2070 return -EFAULT; 2071 2072 ids_len = query.ids_len; 2073 if (ids_len > BPF_TRACE_MAX_PROGS) 2074 return -E2BIG; 2075 ids = kzalloc_objs(u32, ids_len, GFP_USER | __GFP_NOWARN); 2076 if (!ids) 2077 return -ENOMEM; 2078 /* 2079 * The above kcalloc returns ZERO_SIZE_PTR when ids_len = 0, which 2080 * is required when user only wants to check for uquery->prog_cnt. 2081 * There is no need to check for it since the case is handled 2082 * gracefully in bpf_prog_array_copy_info. 2083 */ 2084 2085 mutex_lock(&bpf_event_mutex); 2086 progs = bpf_event_rcu_dereference(event->tp_event->prog_array); 2087 ret = bpf_prog_array_copy_info(progs, ids, ids_len, &prog_cnt); 2088 mutex_unlock(&bpf_event_mutex); 2089 2090 if (copy_to_user(&uquery->prog_cnt, &prog_cnt, sizeof(prog_cnt)) || 2091 copy_to_user(uquery->ids, ids, ids_len * sizeof(u32))) 2092 ret = -EFAULT; 2093 2094 kfree(ids); 2095 return ret; 2096 } 2097 2098 extern struct bpf_raw_event_map __start__bpf_raw_tp[]; 2099 extern struct bpf_raw_event_map __stop__bpf_raw_tp[]; 2100 2101 struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name) 2102 { 2103 struct bpf_raw_event_map *btp = __start__bpf_raw_tp; 2104 2105 for (; btp < __stop__bpf_raw_tp; btp++) { 2106 if (!strcmp(btp->tp->name, name)) 2107 return btp; 2108 } 2109 2110 return bpf_get_raw_tracepoint_module(name); 2111 } 2112 2113 void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp) 2114 { 2115 struct module *mod; 2116 2117 guard(rcu)(); 2118 mod = __module_address((unsigned long)btp); 2119 module_put(mod); 2120 } 2121 2122 static __always_inline 2123 void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args) 2124 { 2125 struct srcu_ctr __percpu *scp = NULL; 2126 struct bpf_prog *prog = link->link.prog; 2127 bool sleepable = prog->sleepable; 2128 struct bpf_run_ctx *old_run_ctx; 2129 struct bpf_trace_run_ctx run_ctx; 2130 2131 if (sleepable) { 2132 scp = rcu_read_lock_tasks_trace(); 2133 migrate_disable(); 2134 } else { 2135 rcu_read_lock_dont_migrate(); 2136 } 2137 2138 if (unlikely(!bpf_prog_get_recursion_context(prog))) { 2139 bpf_prog_inc_misses_counter(prog); 2140 goto out; 2141 } 2142 2143 run_ctx.bpf_cookie = link->cookie; 2144 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); 2145 2146 (void)bpf_prog_run(prog, args); 2147 2148 bpf_reset_run_ctx(old_run_ctx); 2149 out: 2150 bpf_prog_put_recursion_context(prog); 2151 2152 if (sleepable) { 2153 migrate_enable(); 2154 rcu_read_unlock_tasks_trace(scp); 2155 } else { 2156 rcu_read_unlock_migrate(); 2157 } 2158 } 2159 2160 #define UNPACK(...) __VA_ARGS__ 2161 #define REPEAT_1(FN, DL, X, ...) FN(X) 2162 #define REPEAT_2(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_1(FN, DL, __VA_ARGS__) 2163 #define REPEAT_3(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_2(FN, DL, __VA_ARGS__) 2164 #define REPEAT_4(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_3(FN, DL, __VA_ARGS__) 2165 #define REPEAT_5(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_4(FN, DL, __VA_ARGS__) 2166 #define REPEAT_6(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_5(FN, DL, __VA_ARGS__) 2167 #define REPEAT_7(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_6(FN, DL, __VA_ARGS__) 2168 #define REPEAT_8(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_7(FN, DL, __VA_ARGS__) 2169 #define REPEAT_9(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_8(FN, DL, __VA_ARGS__) 2170 #define REPEAT_10(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_9(FN, DL, __VA_ARGS__) 2171 #define REPEAT_11(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_10(FN, DL, __VA_ARGS__) 2172 #define REPEAT_12(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_11(FN, DL, __VA_ARGS__) 2173 #define REPEAT(X, FN, DL, ...) REPEAT_##X(FN, DL, __VA_ARGS__) 2174 2175 #define SARG(X) u64 arg##X 2176 #define COPY(X) args[X] = arg##X 2177 2178 #define __DL_COM (,) 2179 #define __DL_SEM (;) 2180 2181 #define __SEQ_0_11 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 2182 2183 #define BPF_TRACE_DEFN_x(x) \ 2184 void bpf_trace_run##x(struct bpf_raw_tp_link *link, \ 2185 REPEAT(x, SARG, __DL_COM, __SEQ_0_11)) \ 2186 { \ 2187 u64 args[x]; \ 2188 REPEAT(x, COPY, __DL_SEM, __SEQ_0_11); \ 2189 __bpf_trace_run(link, args); \ 2190 } \ 2191 EXPORT_SYMBOL_GPL(bpf_trace_run##x) 2192 BPF_TRACE_DEFN_x(1); 2193 BPF_TRACE_DEFN_x(2); 2194 BPF_TRACE_DEFN_x(3); 2195 BPF_TRACE_DEFN_x(4); 2196 BPF_TRACE_DEFN_x(5); 2197 BPF_TRACE_DEFN_x(6); 2198 BPF_TRACE_DEFN_x(7); 2199 BPF_TRACE_DEFN_x(8); 2200 BPF_TRACE_DEFN_x(9); 2201 BPF_TRACE_DEFN_x(10); 2202 BPF_TRACE_DEFN_x(11); 2203 BPF_TRACE_DEFN_x(12); 2204 2205 int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link) 2206 { 2207 struct tracepoint *tp = btp->tp; 2208 struct bpf_prog *prog = link->link.prog; 2209 2210 /* 2211 * check that program doesn't access arguments beyond what's 2212 * available in this tracepoint 2213 */ 2214 if (prog->aux->max_ctx_offset > btp->num_args * sizeof(u64)) 2215 return -EINVAL; 2216 2217 if (prog->aux->max_tp_access > btp->writable_size) 2218 return -EINVAL; 2219 2220 return tracepoint_probe_register_may_exist(tp, (void *)btp->bpf_func, link); 2221 } 2222 2223 int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link) 2224 { 2225 return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, link); 2226 } 2227 2228 /* On success the caller releases *buf with kfree_const() */ 2229 int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, 2230 u32 *fd_type, const char **buf, 2231 u64 *probe_offset, u64 *probe_addr, 2232 unsigned long *missed) 2233 { 2234 bool is_tracepoint, is_syscall_tp; 2235 struct bpf_prog *prog; 2236 int flags, err = 0; 2237 2238 /* 2239 * The list of dynamic events is walked, and a probe which has siblings 2240 * can be deleted, with its strings, while its event is in use. 2241 */ 2242 guard(mutex)(&event_mutex); 2243 2244 prog = event->prog; 2245 if (!prog) 2246 return -ENOENT; 2247 2248 /* not supporting BPF_PROG_TYPE_PERF_EVENT yet */ 2249 if (prog->type == BPF_PROG_TYPE_PERF_EVENT) 2250 return -EOPNOTSUPP; 2251 2252 *prog_id = prog->aux->id; 2253 flags = event->tp_event->flags; 2254 is_tracepoint = flags & TRACE_EVENT_FL_TRACEPOINT; 2255 is_syscall_tp = is_syscall_trace_event(event->tp_event); 2256 2257 if (is_tracepoint || is_syscall_tp) { 2258 *buf = is_tracepoint ? event->tp_event->tp->name 2259 : event->tp_event->name; 2260 /* We allow NULL pointer for tracepoint */ 2261 if (fd_type) 2262 *fd_type = BPF_FD_TYPE_TRACEPOINT; 2263 if (probe_offset) 2264 *probe_offset = 0x0; 2265 if (probe_addr) 2266 *probe_addr = 0x0; 2267 } else { 2268 /* kprobe/uprobe */ 2269 err = -EOPNOTSUPP; 2270 #ifdef CONFIG_KPROBE_EVENTS 2271 if (flags & TRACE_EVENT_FL_KPROBE) 2272 err = bpf_get_kprobe_info(event, fd_type, buf, 2273 probe_offset, probe_addr, missed, 2274 event->attr.type == PERF_TYPE_TRACEPOINT); 2275 #endif 2276 #ifdef CONFIG_UPROBE_EVENTS 2277 if (flags & TRACE_EVENT_FL_UPROBE) 2278 err = bpf_get_uprobe_info(event, fd_type, buf, 2279 probe_offset, probe_addr, 2280 event->attr.type == PERF_TYPE_TRACEPOINT); 2281 #endif 2282 } 2283 2284 if (!err && *buf) { 2285 *buf = kstrdup_const(*buf, GFP_KERNEL); 2286 if (!*buf) 2287 err = -ENOMEM; 2288 } 2289 2290 return err; 2291 } 2292 2293 static int __init send_signal_irq_work_init(void) 2294 { 2295 int cpu; 2296 struct send_signal_irq_work *work; 2297 2298 for_each_possible_cpu(cpu) { 2299 work = per_cpu_ptr(&send_signal_work, cpu); 2300 init_irq_work(&work->irq_work, do_bpf_send_signal); 2301 } 2302 return 0; 2303 } 2304 2305 subsys_initcall(send_signal_irq_work_init); 2306 2307 #ifdef CONFIG_MODULES 2308 static int bpf_event_notify(struct notifier_block *nb, unsigned long op, 2309 void *module) 2310 { 2311 struct bpf_trace_module *btm, *tmp; 2312 struct module *mod = module; 2313 int ret = 0; 2314 2315 if (mod->num_bpf_raw_events == 0 || 2316 (op != MODULE_STATE_COMING && op != MODULE_STATE_GOING)) 2317 goto out; 2318 2319 mutex_lock(&bpf_module_mutex); 2320 2321 switch (op) { 2322 case MODULE_STATE_COMING: 2323 btm = kzalloc_obj(*btm); 2324 if (btm) { 2325 btm->module = module; 2326 list_add(&btm->list, &bpf_trace_modules); 2327 } else { 2328 ret = -ENOMEM; 2329 } 2330 break; 2331 case MODULE_STATE_GOING: 2332 list_for_each_entry_safe(btm, tmp, &bpf_trace_modules, list) { 2333 if (btm->module == module) { 2334 list_del(&btm->list); 2335 kfree(btm); 2336 break; 2337 } 2338 } 2339 break; 2340 } 2341 2342 mutex_unlock(&bpf_module_mutex); 2343 2344 out: 2345 return notifier_from_errno(ret); 2346 } 2347 2348 static struct notifier_block bpf_module_nb = { 2349 .notifier_call = bpf_event_notify, 2350 }; 2351 2352 static int __init bpf_event_init(void) 2353 { 2354 register_module_notifier(&bpf_module_nb); 2355 return 0; 2356 } 2357 2358 fs_initcall(bpf_event_init); 2359 #endif /* CONFIG_MODULES */ 2360 2361 struct bpf_session_run_ctx { 2362 struct bpf_run_ctx run_ctx; 2363 bool is_return; 2364 void *data; 2365 }; 2366 2367 #ifdef CONFIG_FPROBE 2368 struct bpf_kprobe_multi_link { 2369 struct bpf_link link; 2370 struct fprobe fp; 2371 unsigned long *addrs; 2372 u64 *cookies; 2373 u32 cnt; 2374 u32 mods_cnt; 2375 struct module **mods; 2376 }; 2377 2378 struct bpf_kprobe_multi_run_ctx { 2379 struct bpf_session_run_ctx session_ctx; 2380 struct bpf_kprobe_multi_link *link; 2381 unsigned long entry_ip; 2382 }; 2383 2384 struct user_syms { 2385 const char **syms; 2386 char *buf; 2387 }; 2388 2389 #ifndef CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS 2390 static DEFINE_PER_CPU(struct pt_regs, bpf_kprobe_multi_pt_regs); 2391 #define bpf_kprobe_multi_pt_regs_ptr() this_cpu_ptr(&bpf_kprobe_multi_pt_regs) 2392 #else 2393 #define bpf_kprobe_multi_pt_regs_ptr() (NULL) 2394 #endif 2395 2396 static unsigned long ftrace_get_entry_ip(unsigned long fentry_ip) 2397 { 2398 unsigned long ip = ftrace_get_symaddr(fentry_ip); 2399 2400 return ip ? : fentry_ip; 2401 } 2402 2403 static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 cnt) 2404 { 2405 unsigned long __user usymbol; 2406 const char **syms = NULL; 2407 char *buf = NULL, *p; 2408 int err = -ENOMEM; 2409 unsigned int i; 2410 2411 if (!access_ok(usyms, cnt * sizeof(*usyms))) 2412 return -EFAULT; 2413 2414 syms = kvmalloc_objs(*syms, cnt); 2415 if (!syms) 2416 return -ENOMEM; 2417 2418 buf = kvmalloc_array(cnt, KSYM_NAME_LEN, GFP_KERNEL); 2419 if (!buf) 2420 goto error; 2421 2422 for (p = buf, i = 0; i < cnt; i++) { 2423 if (get_user(usymbol, usyms + i)) { 2424 err = -EFAULT; 2425 goto error; 2426 } 2427 err = strncpy_from_user(p, (const char __user *) usymbol, KSYM_NAME_LEN); 2428 if (err == KSYM_NAME_LEN) 2429 err = -E2BIG; 2430 if (err < 0) 2431 goto error; 2432 syms[i] = p; 2433 p += err + 1; 2434 } 2435 2436 us->syms = syms; 2437 us->buf = buf; 2438 return 0; 2439 2440 error: 2441 kvfree(syms); 2442 kvfree(buf); 2443 return err; 2444 } 2445 2446 static void kprobe_multi_put_modules(struct module **mods, u32 cnt) 2447 { 2448 u32 i; 2449 2450 for (i = 0; i < cnt; i++) 2451 module_put(mods[i]); 2452 } 2453 2454 static void free_user_syms(struct user_syms *us) 2455 { 2456 kvfree(us->syms); 2457 kvfree(us->buf); 2458 } 2459 2460 static void bpf_kprobe_multi_link_release(struct bpf_link *link) 2461 { 2462 struct bpf_kprobe_multi_link *kmulti_link; 2463 2464 kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); 2465 /* Don't wait for RCU GP here. */ 2466 unregister_fprobe_async(&kmulti_link->fp); 2467 kprobe_multi_put_modules(kmulti_link->mods, kmulti_link->mods_cnt); 2468 } 2469 2470 static void bpf_kprobe_multi_link_dealloc(struct bpf_link *link) 2471 { 2472 struct bpf_kprobe_multi_link *kmulti_link; 2473 2474 kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); 2475 kvfree(kmulti_link->addrs); 2476 kvfree(kmulti_link->cookies); 2477 kfree(kmulti_link->mods); 2478 kfree(kmulti_link); 2479 } 2480 2481 static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link, 2482 struct bpf_link_info *info) 2483 { 2484 u64 __user *ucookies = u64_to_user_ptr(info->kprobe_multi.cookies); 2485 u64 __user *uaddrs = u64_to_user_ptr(info->kprobe_multi.addrs); 2486 struct bpf_kprobe_multi_link *kmulti_link; 2487 u32 ucount = info->kprobe_multi.count; 2488 int err = 0, i; 2489 2490 if (!uaddrs ^ !ucount) 2491 return -EINVAL; 2492 if (ucookies && !ucount) 2493 return -EINVAL; 2494 2495 kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); 2496 info->kprobe_multi.count = kmulti_link->cnt; 2497 info->kprobe_multi.flags = kmulti_link->link.flags; 2498 info->kprobe_multi.missed = kmulti_link->fp.nmissed; 2499 2500 if (!uaddrs) 2501 return 0; 2502 if (ucount < kmulti_link->cnt) 2503 err = -ENOSPC; 2504 else 2505 ucount = kmulti_link->cnt; 2506 2507 if (ucookies) { 2508 if (kmulti_link->cookies) { 2509 if (copy_to_user(ucookies, kmulti_link->cookies, ucount * sizeof(u64))) 2510 return -EFAULT; 2511 } else { 2512 for (i = 0; i < ucount; i++) { 2513 if (put_user(0, ucookies + i)) 2514 return -EFAULT; 2515 } 2516 } 2517 } 2518 2519 if (kallsyms_show_value(current_cred())) { 2520 if (copy_to_user(uaddrs, kmulti_link->addrs, ucount * sizeof(u64))) 2521 return -EFAULT; 2522 } else { 2523 for (i = 0; i < ucount; i++) { 2524 if (put_user(0, uaddrs + i)) 2525 return -EFAULT; 2526 } 2527 } 2528 return err; 2529 } 2530 2531 #ifdef CONFIG_PROC_FS 2532 static void bpf_kprobe_multi_show_fdinfo(const struct bpf_link *link, 2533 struct seq_file *seq) 2534 { 2535 struct bpf_kprobe_multi_link *kmulti_link; 2536 bool has_cookies; 2537 2538 kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); 2539 has_cookies = !!kmulti_link->cookies; 2540 2541 seq_printf(seq, 2542 "kprobe_cnt:\t%u\n" 2543 "missed:\t%lu\n", 2544 kmulti_link->cnt, 2545 kmulti_link->fp.nmissed); 2546 2547 seq_printf(seq, "%s\t %s\n", "cookie", "func"); 2548 for (int i = 0; i < kmulti_link->cnt; i++) { 2549 seq_printf(seq, 2550 "%llu\t %pS\n", 2551 has_cookies ? kmulti_link->cookies[i] : 0, 2552 (void *)kmulti_link->addrs[i]); 2553 } 2554 } 2555 #endif 2556 2557 static const struct bpf_link_ops bpf_kprobe_multi_link_lops = { 2558 .release = bpf_kprobe_multi_link_release, 2559 .dealloc_deferred = bpf_kprobe_multi_link_dealloc, 2560 .fill_link_info = bpf_kprobe_multi_link_fill_link_info, 2561 #ifdef CONFIG_PROC_FS 2562 .show_fdinfo = bpf_kprobe_multi_show_fdinfo, 2563 #endif 2564 }; 2565 2566 static void bpf_kprobe_multi_cookie_swap(void *a, void *b, int size, const void *priv) 2567 { 2568 const struct bpf_kprobe_multi_link *link = priv; 2569 unsigned long *addr_a = a, *addr_b = b; 2570 u64 *cookie_a, *cookie_b; 2571 2572 cookie_a = link->cookies + (addr_a - link->addrs); 2573 cookie_b = link->cookies + (addr_b - link->addrs); 2574 2575 /* swap addr_a/addr_b and cookie_a/cookie_b values */ 2576 swap(*addr_a, *addr_b); 2577 swap(*cookie_a, *cookie_b); 2578 } 2579 2580 static int bpf_kprobe_multi_addrs_cmp(const void *a, const void *b) 2581 { 2582 const unsigned long *addr_a = a, *addr_b = b; 2583 2584 if (*addr_a == *addr_b) 2585 return 0; 2586 return *addr_a < *addr_b ? -1 : 1; 2587 } 2588 2589 static int bpf_kprobe_multi_cookie_cmp(const void *a, const void *b, const void *priv) 2590 { 2591 return bpf_kprobe_multi_addrs_cmp(a, b); 2592 } 2593 2594 static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx) 2595 { 2596 struct bpf_kprobe_multi_run_ctx *run_ctx; 2597 struct bpf_kprobe_multi_link *link; 2598 u64 *cookie, entry_ip; 2599 unsigned long *addr; 2600 2601 if (WARN_ON_ONCE(!ctx)) 2602 return 0; 2603 run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx, 2604 session_ctx.run_ctx); 2605 link = run_ctx->link; 2606 if (!link->cookies) 2607 return 0; 2608 entry_ip = run_ctx->entry_ip; 2609 addr = bsearch(&entry_ip, link->addrs, link->cnt, sizeof(entry_ip), 2610 bpf_kprobe_multi_addrs_cmp); 2611 if (!addr) 2612 return 0; 2613 cookie = link->cookies + (addr - link->addrs); 2614 return *cookie; 2615 } 2616 2617 static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx) 2618 { 2619 struct bpf_kprobe_multi_run_ctx *run_ctx; 2620 2621 run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx, 2622 session_ctx.run_ctx); 2623 return run_ctx->entry_ip; 2624 } 2625 2626 static __always_inline int 2627 kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, 2628 unsigned long entry_ip, struct ftrace_regs *fregs, 2629 bool is_return, void *data) 2630 { 2631 struct bpf_kprobe_multi_run_ctx run_ctx = { 2632 .session_ctx = { 2633 .is_return = is_return, 2634 .data = data, 2635 }, 2636 .link = link, 2637 .entry_ip = entry_ip, 2638 }; 2639 struct bpf_run_ctx *old_run_ctx; 2640 struct pt_regs *regs; 2641 int err; 2642 2643 /* 2644 * graph tracer framework ensures we won't migrate, so there is no need 2645 * to use migrate_disable for bpf_prog_run again. The check here just for 2646 * __this_cpu_inc_return. 2647 */ 2648 cant_sleep(); 2649 2650 if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) { 2651 bpf_prog_inc_misses_counter(link->link.prog); 2652 err = 1; 2653 goto out; 2654 } 2655 2656 rcu_read_lock(); 2657 regs = ftrace_partial_regs(fregs, bpf_kprobe_multi_pt_regs_ptr()); 2658 old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx); 2659 err = bpf_prog_run(link->link.prog, regs); 2660 bpf_reset_run_ctx(old_run_ctx); 2661 ftrace_partial_regs_update(fregs, bpf_kprobe_multi_pt_regs_ptr()); 2662 rcu_read_unlock(); 2663 2664 out: 2665 __this_cpu_dec(bpf_prog_active); 2666 return err; 2667 } 2668 2669 static int 2670 kprobe_multi_link_handler(struct fprobe *fp, unsigned long fentry_ip, 2671 unsigned long ret_ip, struct ftrace_regs *fregs, 2672 void *data) 2673 { 2674 struct bpf_kprobe_multi_link *link; 2675 int err; 2676 2677 link = container_of(fp, struct bpf_kprobe_multi_link, fp); 2678 err = kprobe_multi_link_prog_run(link, ftrace_get_entry_ip(fentry_ip), 2679 fregs, false, data); 2680 return is_kprobe_session(link->link.prog) ? err : 0; 2681 } 2682 2683 static void 2684 kprobe_multi_link_exit_handler(struct fprobe *fp, unsigned long fentry_ip, 2685 unsigned long ret_ip, struct ftrace_regs *fregs, 2686 void *data) 2687 { 2688 struct bpf_kprobe_multi_link *link; 2689 2690 link = container_of(fp, struct bpf_kprobe_multi_link, fp); 2691 kprobe_multi_link_prog_run(link, ftrace_get_entry_ip(fentry_ip), 2692 fregs, true, data); 2693 } 2694 2695 static int symbols_cmp_r(const void *a, const void *b, const void *priv) 2696 { 2697 const char **str_a = (const char **) a; 2698 const char **str_b = (const char **) b; 2699 2700 return strcmp(*str_a, *str_b); 2701 } 2702 2703 struct multi_symbols_sort { 2704 const char **funcs; 2705 u64 *cookies; 2706 }; 2707 2708 static void symbols_swap_r(void *a, void *b, int size, const void *priv) 2709 { 2710 const struct multi_symbols_sort *data = priv; 2711 const char **name_a = a, **name_b = b; 2712 2713 swap(*name_a, *name_b); 2714 2715 /* If defined, swap also related cookies. */ 2716 if (data->cookies) { 2717 u64 *cookie_a, *cookie_b; 2718 2719 cookie_a = data->cookies + (name_a - data->funcs); 2720 cookie_b = data->cookies + (name_b - data->funcs); 2721 swap(*cookie_a, *cookie_b); 2722 } 2723 } 2724 2725 struct modules_array { 2726 struct module **mods; 2727 int mods_cnt; 2728 int mods_cap; 2729 }; 2730 2731 static int add_module(struct modules_array *arr, struct module *mod) 2732 { 2733 struct module **mods; 2734 2735 if (arr->mods_cnt == arr->mods_cap) { 2736 arr->mods_cap = max(16, arr->mods_cap * 3 / 2); 2737 mods = krealloc_array(arr->mods, arr->mods_cap, sizeof(*mods), GFP_KERNEL); 2738 if (!mods) 2739 return -ENOMEM; 2740 arr->mods = mods; 2741 } 2742 2743 arr->mods[arr->mods_cnt] = mod; 2744 arr->mods_cnt++; 2745 return 0; 2746 } 2747 2748 static bool has_module(struct modules_array *arr, struct module *mod) 2749 { 2750 int i; 2751 2752 for (i = arr->mods_cnt - 1; i >= 0; i--) { 2753 if (arr->mods[i] == mod) 2754 return true; 2755 } 2756 return false; 2757 } 2758 2759 static int get_modules_for_addrs(struct module ***mods, unsigned long *addrs, u32 addrs_cnt) 2760 { 2761 struct modules_array arr = {}; 2762 u32 i, err = 0; 2763 2764 for (i = 0; i < addrs_cnt; i++) { 2765 bool skip_add = false; 2766 struct module *mod; 2767 2768 scoped_guard(rcu) { 2769 mod = __module_address(addrs[i]); 2770 /* Either no module or it's already stored */ 2771 if (!mod || has_module(&arr, mod)) { 2772 skip_add = true; 2773 break; /* scoped_guard */ 2774 } 2775 if (!try_module_get(mod)) 2776 err = -EINVAL; 2777 } 2778 if (skip_add) 2779 continue; 2780 if (err) 2781 break; 2782 err = add_module(&arr, mod); 2783 if (err) { 2784 module_put(mod); 2785 break; 2786 } 2787 } 2788 2789 /* We return either err < 0 in case of error, ... */ 2790 if (err) { 2791 kprobe_multi_put_modules(arr.mods, arr.mods_cnt); 2792 kfree(arr.mods); 2793 return err; 2794 } 2795 2796 /* or number of modules found if everything is ok. */ 2797 *mods = arr.mods; 2798 return arr.mods_cnt; 2799 } 2800 2801 static int addrs_check_error_injection_list(unsigned long *addrs, u32 cnt) 2802 { 2803 u32 i; 2804 2805 for (i = 0; i < cnt; i++) { 2806 if (!within_error_injection_list(addrs[i])) 2807 return -EINVAL; 2808 } 2809 return 0; 2810 } 2811 2812 int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 2813 { 2814 struct bpf_kprobe_multi_link *link = NULL; 2815 struct bpf_link_primer link_primer; 2816 void __user *ucookies; 2817 unsigned long *addrs; 2818 u32 flags, cnt, size; 2819 void __user *uaddrs; 2820 u64 *cookies = NULL; 2821 void __user *usyms; 2822 int err; 2823 2824 /* no support for 32bit archs yet */ 2825 if (sizeof(u64) != sizeof(void *)) 2826 return -EOPNOTSUPP; 2827 2828 if (attr->link_create.flags) 2829 return -EINVAL; 2830 2831 if (!is_kprobe_multi(prog)) 2832 return -EINVAL; 2833 2834 /* kprobe_multi is not allowed to be sleepable. */ 2835 if (prog->sleepable) 2836 return -EINVAL; 2837 2838 /* Writing to context is not allowed for kprobes. */ 2839 if (prog->aux->kprobe_write_ctx) 2840 return -EINVAL; 2841 2842 flags = attr->link_create.kprobe_multi.flags; 2843 if (flags & ~BPF_F_KPROBE_MULTI_RETURN) 2844 return -EINVAL; 2845 2846 uaddrs = u64_to_user_ptr(attr->link_create.kprobe_multi.addrs); 2847 usyms = u64_to_user_ptr(attr->link_create.kprobe_multi.syms); 2848 if (!!uaddrs == !!usyms) 2849 return -EINVAL; 2850 2851 cnt = attr->link_create.kprobe_multi.cnt; 2852 if (!cnt) 2853 return -EINVAL; 2854 if (cnt > MAX_KPROBE_MULTI_CNT) 2855 return -E2BIG; 2856 2857 size = cnt * sizeof(*addrs); 2858 addrs = kvmalloc_objs(*addrs, cnt); 2859 if (!addrs) 2860 return -ENOMEM; 2861 2862 ucookies = u64_to_user_ptr(attr->link_create.kprobe_multi.cookies); 2863 if (ucookies) { 2864 cookies = kvmalloc_array(cnt, sizeof(*addrs), GFP_KERNEL); 2865 if (!cookies) { 2866 err = -ENOMEM; 2867 goto error; 2868 } 2869 if (copy_from_user(cookies, ucookies, size)) { 2870 err = -EFAULT; 2871 goto error; 2872 } 2873 } 2874 2875 if (uaddrs) { 2876 if (copy_from_user(addrs, uaddrs, size)) { 2877 err = -EFAULT; 2878 goto error; 2879 } 2880 } else { 2881 struct multi_symbols_sort data = { 2882 .cookies = cookies, 2883 }; 2884 struct user_syms us; 2885 2886 err = copy_user_syms(&us, usyms, cnt); 2887 if (err) 2888 goto error; 2889 2890 if (cookies) 2891 data.funcs = us.syms; 2892 2893 sort_r(us.syms, cnt, sizeof(*us.syms), symbols_cmp_r, 2894 symbols_swap_r, &data); 2895 2896 err = ftrace_lookup_symbols(us.syms, cnt, addrs); 2897 free_user_syms(&us); 2898 if (err) 2899 goto error; 2900 } 2901 2902 if (prog->kprobe_override && addrs_check_error_injection_list(addrs, cnt)) { 2903 err = -EINVAL; 2904 goto error; 2905 } 2906 2907 link = kzalloc_obj(*link); 2908 if (!link) { 2909 err = -ENOMEM; 2910 goto error; 2911 } 2912 2913 bpf_link_init(&link->link, BPF_LINK_TYPE_KPROBE_MULTI, 2914 &bpf_kprobe_multi_link_lops, prog, attr->link_create.attach_type); 2915 2916 err = bpf_link_prime(&link->link, &link_primer); 2917 if (err) 2918 goto error; 2919 2920 if (!(flags & BPF_F_KPROBE_MULTI_RETURN)) 2921 link->fp.entry_handler = kprobe_multi_link_handler; 2922 if ((flags & BPF_F_KPROBE_MULTI_RETURN) || is_kprobe_session(prog)) 2923 link->fp.exit_handler = kprobe_multi_link_exit_handler; 2924 if (is_kprobe_session(prog)) 2925 link->fp.entry_data_size = sizeof(u64); 2926 2927 link->addrs = addrs; 2928 link->cookies = cookies; 2929 link->cnt = cnt; 2930 link->link.flags = flags; 2931 2932 if (cookies) { 2933 /* 2934 * Sorting addresses will trigger sorting cookies as well 2935 * (check bpf_kprobe_multi_cookie_swap). This way we can 2936 * find cookie based on the address in bpf_get_attach_cookie 2937 * helper. 2938 */ 2939 sort_r(addrs, cnt, sizeof(*addrs), 2940 bpf_kprobe_multi_cookie_cmp, 2941 bpf_kprobe_multi_cookie_swap, 2942 link); 2943 } 2944 2945 err = get_modules_for_addrs(&link->mods, addrs, cnt); 2946 if (err < 0) { 2947 bpf_link_cleanup(&link_primer); 2948 return err; 2949 } 2950 link->mods_cnt = err; 2951 2952 err = register_fprobe_ips(&link->fp, addrs, cnt); 2953 if (err) { 2954 kprobe_multi_put_modules(link->mods, link->mods_cnt); 2955 bpf_link_cleanup(&link_primer); 2956 return err; 2957 } 2958 2959 return bpf_link_settle(&link_primer); 2960 2961 error: 2962 kfree(link); 2963 kvfree(addrs); 2964 kvfree(cookies); 2965 return err; 2966 } 2967 #else /* !CONFIG_FPROBE */ 2968 int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 2969 { 2970 return -EOPNOTSUPP; 2971 } 2972 static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx) 2973 { 2974 return 0; 2975 } 2976 static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx) 2977 { 2978 return 0; 2979 } 2980 #endif 2981 2982 #ifdef CONFIG_UPROBES 2983 struct bpf_uprobe_multi_link; 2984 2985 struct bpf_uprobe { 2986 struct bpf_uprobe_multi_link *link; 2987 loff_t offset; 2988 unsigned long ref_ctr_offset; 2989 u64 cookie; 2990 struct uprobe *uprobe; 2991 struct uprobe_consumer consumer; 2992 bool session; 2993 }; 2994 2995 struct bpf_uprobe_multi_link { 2996 struct path path; 2997 struct bpf_link link; 2998 u32 cnt; 2999 struct bpf_uprobe *uprobes; 3000 struct task_struct *task; 3001 }; 3002 3003 struct bpf_uprobe_multi_run_ctx { 3004 struct bpf_session_run_ctx session_ctx; 3005 unsigned long entry_ip; 3006 struct bpf_uprobe *uprobe; 3007 }; 3008 3009 static void bpf_uprobe_unregister(struct bpf_uprobe *uprobes, u32 cnt) 3010 { 3011 u32 i; 3012 3013 for (i = 0; i < cnt; i++) 3014 uprobe_unregister_nosync(uprobes[i].uprobe, &uprobes[i].consumer); 3015 3016 if (cnt) 3017 uprobe_unregister_sync(); 3018 } 3019 3020 static void bpf_uprobe_multi_link_release(struct bpf_link *link) 3021 { 3022 struct bpf_uprobe_multi_link *umulti_link; 3023 3024 umulti_link = container_of(link, struct bpf_uprobe_multi_link, link); 3025 bpf_uprobe_unregister(umulti_link->uprobes, umulti_link->cnt); 3026 if (umulti_link->task) 3027 put_task_struct(umulti_link->task); 3028 path_put(&umulti_link->path); 3029 } 3030 3031 static void bpf_uprobe_multi_link_dealloc(struct bpf_link *link) 3032 { 3033 struct bpf_uprobe_multi_link *umulti_link; 3034 3035 umulti_link = container_of(link, struct bpf_uprobe_multi_link, link); 3036 kvfree(umulti_link->uprobes); 3037 kfree(umulti_link); 3038 } 3039 3040 static int bpf_uprobe_multi_link_fill_link_info(const struct bpf_link *link, 3041 struct bpf_link_info *info) 3042 { 3043 u64 __user *uref_ctr_offsets = u64_to_user_ptr(info->uprobe_multi.ref_ctr_offsets); 3044 u64 __user *ucookies = u64_to_user_ptr(info->uprobe_multi.cookies); 3045 u64 __user *uoffsets = u64_to_user_ptr(info->uprobe_multi.offsets); 3046 u64 __user *upath = u64_to_user_ptr(info->uprobe_multi.path); 3047 u32 upath_size = info->uprobe_multi.path_size; 3048 struct bpf_uprobe_multi_link *umulti_link; 3049 u32 ucount = info->uprobe_multi.count; 3050 int err = 0, i; 3051 char *p, *buf; 3052 long left = 0; 3053 3054 if (!upath ^ !upath_size) 3055 return -EINVAL; 3056 3057 if ((uoffsets || uref_ctr_offsets || ucookies) && !ucount) 3058 return -EINVAL; 3059 3060 umulti_link = container_of(link, struct bpf_uprobe_multi_link, link); 3061 info->uprobe_multi.count = umulti_link->cnt; 3062 info->uprobe_multi.flags = umulti_link->link.flags; 3063 info->uprobe_multi.pid = umulti_link->task ? 3064 task_pid_nr_ns(umulti_link->task, task_active_pid_ns(current)) : 0; 3065 3066 upath_size = upath_size ? min_t(u32, upath_size, PATH_MAX) : PATH_MAX; 3067 buf = kmalloc(upath_size, GFP_KERNEL); 3068 if (!buf) 3069 return -ENOMEM; 3070 p = d_path(&umulti_link->path, buf, upath_size); 3071 if (IS_ERR(p)) { 3072 kfree(buf); 3073 return PTR_ERR(p); 3074 } 3075 upath_size = buf + upath_size - p; 3076 3077 if (upath) 3078 left = copy_to_user(upath, p, upath_size); 3079 kfree(buf); 3080 if (left) 3081 return -EFAULT; 3082 info->uprobe_multi.path_size = upath_size; 3083 3084 if (!uoffsets && !ucookies && !uref_ctr_offsets) 3085 return 0; 3086 3087 if (ucount < umulti_link->cnt) 3088 err = -ENOSPC; 3089 else 3090 ucount = umulti_link->cnt; 3091 3092 for (i = 0; i < ucount; i++) { 3093 if (uoffsets && 3094 put_user(umulti_link->uprobes[i].offset, uoffsets + i)) 3095 return -EFAULT; 3096 if (uref_ctr_offsets && 3097 put_user(umulti_link->uprobes[i].ref_ctr_offset, uref_ctr_offsets + i)) 3098 return -EFAULT; 3099 if (ucookies && 3100 put_user(umulti_link->uprobes[i].cookie, ucookies + i)) 3101 return -EFAULT; 3102 } 3103 3104 return err; 3105 } 3106 3107 #ifdef CONFIG_PROC_FS 3108 static void bpf_uprobe_multi_show_fdinfo(const struct bpf_link *link, 3109 struct seq_file *seq) 3110 { 3111 struct bpf_uprobe_multi_link *umulti_link; 3112 char *p, *buf; 3113 pid_t pid; 3114 3115 umulti_link = container_of(link, struct bpf_uprobe_multi_link, link); 3116 3117 buf = kmalloc(PATH_MAX, GFP_KERNEL); 3118 if (!buf) 3119 return; 3120 3121 p = d_path(&umulti_link->path, buf, PATH_MAX); 3122 if (IS_ERR(p)) { 3123 kfree(buf); 3124 return; 3125 } 3126 3127 pid = umulti_link->task ? 3128 task_pid_nr_ns(umulti_link->task, task_active_pid_ns(current)) : 0; 3129 seq_printf(seq, 3130 "uprobe_cnt:\t%u\n" 3131 "pid:\t%u\n" 3132 "path:\t%s\n", 3133 umulti_link->cnt, pid, p); 3134 3135 seq_printf(seq, "%s\t %s\t %s\n", "cookie", "offset", "ref_ctr_offset"); 3136 for (int i = 0; i < umulti_link->cnt; i++) { 3137 seq_printf(seq, 3138 "%llu\t %#llx\t %#lx\n", 3139 umulti_link->uprobes[i].cookie, 3140 umulti_link->uprobes[i].offset, 3141 umulti_link->uprobes[i].ref_ctr_offset); 3142 } 3143 3144 kfree(buf); 3145 } 3146 #endif 3147 3148 static const struct bpf_link_ops bpf_uprobe_multi_link_lops = { 3149 .release = bpf_uprobe_multi_link_release, 3150 .dealloc_deferred = bpf_uprobe_multi_link_dealloc, 3151 .fill_link_info = bpf_uprobe_multi_link_fill_link_info, 3152 #ifdef CONFIG_PROC_FS 3153 .show_fdinfo = bpf_uprobe_multi_show_fdinfo, 3154 #endif 3155 }; 3156 3157 static int uprobe_prog_run(struct bpf_uprobe *uprobe, 3158 unsigned long entry_ip, 3159 struct pt_regs *regs, 3160 bool is_return, void *data) 3161 { 3162 struct bpf_uprobe_multi_link *link = uprobe->link; 3163 struct bpf_uprobe_multi_run_ctx run_ctx = { 3164 .session_ctx = { 3165 .is_return = is_return, 3166 .data = data, 3167 }, 3168 .entry_ip = entry_ip, 3169 .uprobe = uprobe, 3170 }; 3171 struct bpf_prog *prog = link->link.prog; 3172 bool sleepable = prog->sleepable; 3173 struct bpf_run_ctx *old_run_ctx; 3174 int err; 3175 3176 if (link->task && !same_thread_group(current, link->task)) 3177 return 0; 3178 3179 if (sleepable) 3180 rcu_read_lock_trace(); 3181 else 3182 rcu_read_lock(); 3183 3184 migrate_disable(); 3185 3186 old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx); 3187 err = bpf_prog_run(link->link.prog, regs); 3188 bpf_reset_run_ctx(old_run_ctx); 3189 3190 migrate_enable(); 3191 3192 if (sleepable) 3193 rcu_read_unlock_trace(); 3194 else 3195 rcu_read_unlock(); 3196 return err; 3197 } 3198 3199 static bool 3200 uprobe_multi_link_filter(struct uprobe_consumer *con, struct mm_struct *mm) 3201 { 3202 struct bpf_uprobe *uprobe; 3203 3204 uprobe = container_of(con, struct bpf_uprobe, consumer); 3205 return uprobe->link->task->mm == mm; 3206 } 3207 3208 static int 3209 uprobe_multi_link_handler(struct uprobe_consumer *con, struct pt_regs *regs, 3210 __u64 *data) 3211 { 3212 struct bpf_uprobe *uprobe; 3213 int ret; 3214 3215 uprobe = container_of(con, struct bpf_uprobe, consumer); 3216 ret = uprobe_prog_run(uprobe, instruction_pointer(regs), regs, false, data); 3217 if (uprobe->session) 3218 return ret ? UPROBE_HANDLER_IGNORE : 0; 3219 return 0; 3220 } 3221 3222 static int 3223 uprobe_multi_link_ret_handler(struct uprobe_consumer *con, unsigned long func, struct pt_regs *regs, 3224 __u64 *data) 3225 { 3226 struct bpf_uprobe *uprobe; 3227 3228 uprobe = container_of(con, struct bpf_uprobe, consumer); 3229 uprobe_prog_run(uprobe, func, regs, true, data); 3230 return 0; 3231 } 3232 3233 static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx) 3234 { 3235 struct bpf_uprobe_multi_run_ctx *run_ctx; 3236 3237 run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, 3238 session_ctx.run_ctx); 3239 return run_ctx->entry_ip; 3240 } 3241 3242 static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx) 3243 { 3244 struct bpf_uprobe_multi_run_ctx *run_ctx; 3245 3246 run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, 3247 session_ctx.run_ctx); 3248 return run_ctx->uprobe->cookie; 3249 } 3250 3251 static int bpf_uprobe_multi_get_path(const union bpf_attr *attr, struct path *path) 3252 { 3253 void __user *upath = u64_to_user_ptr(attr->link_create.uprobe_multi.path); 3254 u32 path_fd = attr->link_create.uprobe_multi.path_fd; 3255 u32 flags = attr->link_create.uprobe_multi.flags; 3256 3257 if (flags & BPF_F_UPROBE_MULTI_PATH_FD) { 3258 /* 3259 * When BPF_F_UPROBE_MULTI_PATH_FD is set, the executable is 3260 * identified by path_fd, upath must be NULL. 3261 */ 3262 if (upath) 3263 return -EINVAL; 3264 3265 CLASS(fd, f)(path_fd); 3266 if (fd_empty(f)) 3267 return -EBADF; 3268 *path = fd_file(f)->f_path; 3269 path_get(path); 3270 return 0; 3271 } 3272 3273 /* 3274 * When BPF_F_UPROBE_MULTI_PATH_FD is not set, the path is resolved 3275 * relative to the cwd (AT_FDCWD) or absolute using the upath string. 3276 */ 3277 if (!upath || path_fd) 3278 return -EINVAL; 3279 3280 return user_path_at(AT_FDCWD, upath, LOOKUP_FOLLOW, path); 3281 } 3282 3283 int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 3284 { 3285 struct bpf_uprobe_multi_link *link = NULL; 3286 unsigned long __user *uref_ctr_offsets; 3287 struct bpf_link_primer link_primer; 3288 struct bpf_uprobe *uprobes = NULL; 3289 struct task_struct *task = NULL; 3290 unsigned long __user *uoffsets; 3291 u64 __user *ucookies; 3292 unsigned long size; 3293 u32 flags, cnt, i; 3294 struct path path; 3295 pid_t pid; 3296 int err; 3297 3298 /* no support for 32bit archs yet */ 3299 if (sizeof(u64) != sizeof(void *)) 3300 return -EOPNOTSUPP; 3301 3302 if (attr->link_create.flags) 3303 return -EINVAL; 3304 3305 if (!is_uprobe_multi(prog)) 3306 return -EINVAL; 3307 3308 flags = attr->link_create.uprobe_multi.flags; 3309 if (flags & ~(BPF_F_UPROBE_MULTI_RETURN | BPF_F_UPROBE_MULTI_PATH_FD)) 3310 return -EINVAL; 3311 3312 /* 3313 * offsets and cnt are mandatory, 3314 * ref_ctr_offsets and cookies are optional 3315 */ 3316 uoffsets = u64_to_user_ptr(attr->link_create.uprobe_multi.offsets); 3317 cnt = attr->link_create.uprobe_multi.cnt; 3318 pid = attr->link_create.uprobe_multi.pid; 3319 3320 if (!uoffsets || !cnt || pid < 0) 3321 return -EINVAL; 3322 if (cnt > MAX_UPROBE_MULTI_CNT) 3323 return -E2BIG; 3324 3325 uref_ctr_offsets = u64_to_user_ptr(attr->link_create.uprobe_multi.ref_ctr_offsets); 3326 ucookies = u64_to_user_ptr(attr->link_create.uprobe_multi.cookies); 3327 3328 /* 3329 * All uoffsets/uref_ctr_offsets/ucookies arrays have the same value 3330 * size. Check their address ranges up front, so that a bad pointer 3331 * fails before anything is looked up or allocated. 3332 */ 3333 size = sizeof(*uoffsets) * cnt; 3334 if (!access_ok(uoffsets, size) || !access_ok(uref_ctr_offsets, size) || 3335 !access_ok(ucookies, size)) 3336 return -EFAULT; 3337 3338 err = bpf_uprobe_multi_get_path(attr, &path); 3339 if (err) 3340 return err; 3341 3342 if (!d_is_reg(path.dentry)) { 3343 err = -EBADF; 3344 goto error_path_put; 3345 } 3346 3347 if (pid) { 3348 rcu_read_lock(); 3349 task = get_pid_task(find_vpid(pid), PIDTYPE_TGID); 3350 rcu_read_unlock(); 3351 if (!task) { 3352 err = -ESRCH; 3353 goto error_path_put; 3354 } 3355 } 3356 3357 err = -ENOMEM; 3358 3359 link = kzalloc_obj(*link); 3360 uprobes = kvzalloc_objs(*uprobes, cnt); 3361 3362 if (!uprobes || !link) 3363 goto error_free; 3364 3365 for (i = 0; i < cnt; i++) { 3366 if (get_user(uprobes[i].offset, uoffsets + i)) { 3367 err = -EFAULT; 3368 goto error_free; 3369 } 3370 if (uprobes[i].offset < 0) { 3371 err = -EINVAL; 3372 goto error_free; 3373 } 3374 if (uref_ctr_offsets && get_user(uprobes[i].ref_ctr_offset, uref_ctr_offsets + i)) { 3375 err = -EFAULT; 3376 goto error_free; 3377 } 3378 if (ucookies && get_user(uprobes[i].cookie, ucookies + i)) { 3379 err = -EFAULT; 3380 goto error_free; 3381 } 3382 3383 uprobes[i].link = link; 3384 3385 if (!(flags & BPF_F_UPROBE_MULTI_RETURN)) 3386 uprobes[i].consumer.handler = uprobe_multi_link_handler; 3387 if (flags & BPF_F_UPROBE_MULTI_RETURN || is_uprobe_session(prog)) 3388 uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler; 3389 if (is_uprobe_session(prog)) 3390 uprobes[i].session = true; 3391 if (pid) 3392 uprobes[i].consumer.filter = uprobe_multi_link_filter; 3393 } 3394 3395 link->cnt = cnt; 3396 link->uprobes = uprobes; 3397 link->path = path; 3398 link->task = task; 3399 link->link.flags = flags; 3400 3401 bpf_link_init(&link->link, BPF_LINK_TYPE_UPROBE_MULTI, 3402 &bpf_uprobe_multi_link_lops, prog, attr->link_create.attach_type); 3403 3404 for (i = 0; i < cnt; i++) { 3405 uprobes[i].uprobe = uprobe_register(d_real_inode(link->path.dentry), 3406 uprobes[i].offset, 3407 uprobes[i].ref_ctr_offset, 3408 &uprobes[i].consumer); 3409 if (IS_ERR(uprobes[i].uprobe)) { 3410 err = PTR_ERR(uprobes[i].uprobe); 3411 link->cnt = i; 3412 goto error_unregister; 3413 } 3414 } 3415 3416 err = bpf_link_prime(&link->link, &link_primer); 3417 if (err) 3418 goto error_unregister; 3419 3420 return bpf_link_settle(&link_primer); 3421 3422 error_unregister: 3423 bpf_uprobe_unregister(uprobes, link->cnt); 3424 3425 error_free: 3426 kvfree(uprobes); 3427 kfree(link); 3428 if (task) 3429 put_task_struct(task); 3430 error_path_put: 3431 path_put(&path); 3432 return err; 3433 } 3434 #else /* !CONFIG_UPROBES */ 3435 int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 3436 { 3437 return -EOPNOTSUPP; 3438 } 3439 static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx) 3440 { 3441 return 0; 3442 } 3443 static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx) 3444 { 3445 return 0; 3446 } 3447 #endif /* CONFIG_UPROBES */ 3448 3449 __bpf_kfunc_start_defs(); 3450 3451 __bpf_kfunc bool bpf_session_is_return(void *ctx) 3452 { 3453 struct bpf_session_run_ctx *session_ctx; 3454 3455 session_ctx = container_of(current->bpf_ctx, struct bpf_session_run_ctx, run_ctx); 3456 return session_ctx->is_return; 3457 } 3458 3459 __bpf_kfunc __u64 *bpf_session_cookie(void *ctx) 3460 { 3461 struct bpf_session_run_ctx *session_ctx; 3462 3463 session_ctx = container_of(current->bpf_ctx, struct bpf_session_run_ctx, run_ctx); 3464 return session_ctx->data; 3465 } 3466 3467 __bpf_kfunc_end_defs(); 3468 3469 BTF_KFUNCS_START(session_kfunc_set_ids) 3470 BTF_ID_FLAGS(func, bpf_session_is_return) 3471 BTF_ID_FLAGS(func, bpf_session_cookie) 3472 BTF_KFUNCS_END(session_kfunc_set_ids) 3473 3474 static int bpf_session_filter(const struct bpf_prog *prog, u32 kfunc_id) 3475 { 3476 if (!btf_id_set8_contains(&session_kfunc_set_ids, kfunc_id)) 3477 return 0; 3478 3479 if (!is_kprobe_session(prog) && !is_uprobe_session(prog) && !is_trace_fsession(prog)) 3480 return -EACCES; 3481 3482 return 0; 3483 } 3484 3485 static const struct btf_kfunc_id_set bpf_session_kfunc_set = { 3486 .owner = THIS_MODULE, 3487 .set = &session_kfunc_set_ids, 3488 .filter = bpf_session_filter, 3489 }; 3490 3491 static int __init bpf_trace_kfuncs_init(void) 3492 { 3493 int err = 0; 3494 3495 err = err ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_KPROBE, &bpf_session_kfunc_set); 3496 err = err ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_session_kfunc_set); 3497 3498 return err; 3499 } 3500 3501 late_initcall(bpf_trace_kfuncs_init); 3502 3503 typedef int (*copy_fn_t)(void *dst, const void *src, u32 size, struct task_struct *tsk); 3504 3505 /* 3506 * The __always_inline is to make sure the compiler doesn't 3507 * generate indirect calls into callbacks, which is expensive, 3508 * on some kernel configurations. This allows compiler to put 3509 * direct calls into all the specific callback implementations 3510 * (copy_user_data_sleepable, copy_user_data_nofault, and so on) 3511 */ 3512 static __always_inline int __bpf_dynptr_copy_str(const struct bpf_dynptr *dptr, u64 doff, u64 size, 3513 const void *unsafe_src, 3514 copy_fn_t str_copy_fn, 3515 struct task_struct *tsk) 3516 { 3517 const struct bpf_dynptr_kern *dst; 3518 u64 chunk_sz, off; 3519 void *dst_slice; 3520 int cnt, err; 3521 char buf[256]; 3522 3523 dst_slice = bpf_dynptr_slice_rdwr(dptr, doff, NULL, size); 3524 if (likely(dst_slice)) 3525 return str_copy_fn(dst_slice, unsafe_src, size, tsk); 3526 3527 dst = (struct bpf_dynptr_kern *)dptr; 3528 if (bpf_dynptr_check_off_len(dst, doff, size)) 3529 return -E2BIG; 3530 3531 for (off = 0; off < size; off += chunk_sz - 1) { 3532 chunk_sz = min_t(u64, sizeof(buf), size - off); 3533 /* Expect str_copy_fn to return count of copied bytes, including 3534 * zero terminator. Next iteration increment off by chunk_sz - 1 to 3535 * overwrite NUL. 3536 */ 3537 cnt = str_copy_fn(buf, unsafe_src + off, chunk_sz, tsk); 3538 if (cnt < 0) 3539 return cnt; 3540 err = __bpf_dynptr_write(dst, doff + off, buf, cnt, 0); 3541 if (err) 3542 return err; 3543 if (cnt < chunk_sz || chunk_sz == 1) /* we are done */ 3544 return off + cnt; 3545 } 3546 return off; 3547 } 3548 3549 static __always_inline int __bpf_dynptr_copy(const struct bpf_dynptr *dptr, u64 doff, 3550 u64 size, const void *unsafe_src, 3551 copy_fn_t copy_fn, struct task_struct *tsk) 3552 { 3553 const struct bpf_dynptr_kern *dst; 3554 void *dst_slice; 3555 char buf[256]; 3556 u64 off, chunk_sz; 3557 int err; 3558 3559 dst_slice = bpf_dynptr_slice_rdwr(dptr, doff, NULL, size); 3560 if (likely(dst_slice)) 3561 return copy_fn(dst_slice, unsafe_src, size, tsk); 3562 3563 dst = (struct bpf_dynptr_kern *)dptr; 3564 if (bpf_dynptr_check_off_len(dst, doff, size)) 3565 return -E2BIG; 3566 3567 for (off = 0; off < size; off += chunk_sz) { 3568 chunk_sz = min_t(u64, sizeof(buf), size - off); 3569 err = copy_fn(buf, unsafe_src + off, chunk_sz, tsk); 3570 if (err) 3571 return err; 3572 err = __bpf_dynptr_write(dst, doff + off, buf, chunk_sz, 0); 3573 if (err) 3574 return err; 3575 } 3576 return 0; 3577 } 3578 3579 static __always_inline int copy_user_data_nofault(void *dst, const void *unsafe_src, 3580 u32 size, struct task_struct *tsk) 3581 { 3582 return copy_from_user_nofault(dst, (const void __user *)unsafe_src, size); 3583 } 3584 3585 static __always_inline int copy_user_data_sleepable(void *dst, const void *unsafe_src, 3586 u32 size, struct task_struct *tsk) 3587 { 3588 int ret; 3589 3590 if (!tsk) { /* Read from the current task */ 3591 ret = copy_from_user(dst, (const void __user *)unsafe_src, size); 3592 if (ret) 3593 return -EFAULT; 3594 return 0; 3595 } 3596 3597 ret = access_process_vm(tsk, (unsigned long)unsafe_src, dst, size, 0); 3598 if (ret != size) 3599 return -EFAULT; 3600 return 0; 3601 } 3602 3603 static __always_inline int copy_kernel_data_nofault(void *dst, const void *unsafe_src, 3604 u32 size, struct task_struct *tsk) 3605 { 3606 return copy_from_kernel_nofault(dst, unsafe_src, size); 3607 } 3608 3609 static __always_inline int copy_user_str_nofault(void *dst, const void *unsafe_src, 3610 u32 size, struct task_struct *tsk) 3611 { 3612 return strncpy_from_user_nofault(dst, (const void __user *)unsafe_src, size); 3613 } 3614 3615 static __always_inline int copy_user_str_sleepable(void *dst, const void *unsafe_src, 3616 u32 size, struct task_struct *tsk) 3617 { 3618 int ret; 3619 3620 if (unlikely(size == 0)) 3621 return 0; 3622 3623 if (tsk) { 3624 ret = copy_remote_vm_str(tsk, (unsigned long)unsafe_src, dst, size, 0); 3625 } else { 3626 ret = strncpy_from_user(dst, (const void __user *)unsafe_src, size - 1); 3627 /* strncpy_from_user does not guarantee NUL termination */ 3628 if (ret >= 0) 3629 ((char *)dst)[ret] = '\0'; 3630 } 3631 3632 if (ret < 0) 3633 return ret; 3634 return ret + 1; 3635 } 3636 3637 static __always_inline int copy_kernel_str_nofault(void *dst, const void *unsafe_src, 3638 u32 size, struct task_struct *tsk) 3639 { 3640 return strncpy_from_kernel_nofault(dst, unsafe_src, size); 3641 } 3642 3643 __bpf_kfunc_start_defs(); 3644 3645 __bpf_kfunc int bpf_send_signal_task(struct task_struct *task, int sig, enum pid_type type, 3646 u64 value) 3647 { 3648 if (type != PIDTYPE_PID && type != PIDTYPE_TGID) 3649 return -EINVAL; 3650 3651 return bpf_send_signal_common(sig, type, task, value); 3652 } 3653 3654 __bpf_kfunc int bpf_probe_read_user_dynptr(const struct bpf_dynptr *dptr, u64 off, 3655 u64 size, const void __user *unsafe_ptr__ign) 3656 { 3657 return __bpf_dynptr_copy(dptr, off, size, (const void __force *)unsafe_ptr__ign, 3658 copy_user_data_nofault, NULL); 3659 } 3660 3661 __bpf_kfunc int bpf_probe_read_kernel_dynptr(const struct bpf_dynptr *dptr, u64 off, 3662 u64 size, const void *unsafe_ptr__ign) 3663 { 3664 return __bpf_dynptr_copy(dptr, off, size, unsafe_ptr__ign, 3665 copy_kernel_data_nofault, NULL); 3666 } 3667 3668 __bpf_kfunc int bpf_probe_read_user_str_dynptr(const struct bpf_dynptr *dptr, u64 off, 3669 u64 size, const void __user *unsafe_ptr__ign) 3670 { 3671 return __bpf_dynptr_copy_str(dptr, off, size, (const void __force *)unsafe_ptr__ign, 3672 copy_user_str_nofault, NULL); 3673 } 3674 3675 __bpf_kfunc int bpf_probe_read_kernel_str_dynptr(const struct bpf_dynptr *dptr, u64 off, 3676 u64 size, const void *unsafe_ptr__ign) 3677 { 3678 return __bpf_dynptr_copy_str(dptr, off, size, unsafe_ptr__ign, 3679 copy_kernel_str_nofault, NULL); 3680 } 3681 3682 __bpf_kfunc int bpf_copy_from_user_dynptr(const struct bpf_dynptr *dptr, u64 off, 3683 u64 size, const void __user *unsafe_ptr__ign) 3684 { 3685 return __bpf_dynptr_copy(dptr, off, size, (const void __force *)unsafe_ptr__ign, 3686 copy_user_data_sleepable, NULL); 3687 } 3688 3689 __bpf_kfunc int bpf_copy_from_user_str_dynptr(const struct bpf_dynptr *dptr, u64 off, 3690 u64 size, const void __user *unsafe_ptr__ign) 3691 { 3692 return __bpf_dynptr_copy_str(dptr, off, size, (const void __force *)unsafe_ptr__ign, 3693 copy_user_str_sleepable, NULL); 3694 } 3695 3696 __bpf_kfunc int bpf_copy_from_user_task_dynptr(const struct bpf_dynptr *dptr, u64 off, 3697 u64 size, const void __user *unsafe_ptr__ign, 3698 struct task_struct *tsk) 3699 { 3700 return __bpf_dynptr_copy(dptr, off, size, (const void __force *)unsafe_ptr__ign, 3701 copy_user_data_sleepable, tsk); 3702 } 3703 3704 __bpf_kfunc int bpf_copy_from_user_task_str_dynptr(const struct bpf_dynptr *dptr, u64 off, 3705 u64 size, const void __user *unsafe_ptr__ign, 3706 struct task_struct *tsk) 3707 { 3708 return __bpf_dynptr_copy_str(dptr, off, size, (const void __force *)unsafe_ptr__ign, 3709 copy_user_str_sleepable, tsk); 3710 } 3711 3712 __bpf_kfunc_end_defs(); 3713 3714 #if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) && \ 3715 defined(CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS) 3716 3717 static void bpf_tracing_multi_link_release(struct bpf_link *link) 3718 { 3719 struct bpf_tracing_multi_link *tr_link = 3720 container_of(link, struct bpf_tracing_multi_link, link); 3721 3722 bpf_trampoline_multi_detach(link->prog, tr_link); 3723 } 3724 3725 static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) 3726 { 3727 struct bpf_tracing_multi_link *tr_link = 3728 container_of(link, struct bpf_tracing_multi_link, link); 3729 3730 kvfree(tr_link->fexits); 3731 kvfree(tr_link->cookies); 3732 kvfree(tr_link); 3733 } 3734 3735 static int bpf_tracing_multi_link_fill_link_info(const struct bpf_link *link, 3736 struct bpf_link_info *info) 3737 { 3738 u64 __user *ucookies = u64_to_user_ptr(info->tracing_multi.cookies); 3739 u64 __user *uaddrs = u64_to_user_ptr(info->tracing_multi.addrs); 3740 u32 __user *uids = u64_to_user_ptr(info->tracing_multi.ids); 3741 struct bpf_tracing_multi_link *tr_link; 3742 u32 ucount = info->tracing_multi.count; 3743 bool has_cookies, show_addrs; 3744 int err = 0; 3745 3746 if ((uids || ucookies || uaddrs) && !ucount) 3747 return -EINVAL; 3748 3749 tr_link = container_of(link, struct bpf_tracing_multi_link, link); 3750 3751 info->tracing_multi.attach_type = tr_link->link.attach_type; 3752 info->tracing_multi.count = tr_link->nodes_cnt; 3753 info->tracing_multi.btf_obj_id = btf_obj_id(tr_link->link.prog->aux->attach_btf); 3754 3755 if (!uids && !ucookies && !uaddrs) 3756 return 0; 3757 3758 if (ucount < tr_link->nodes_cnt) 3759 err = -ENOSPC; 3760 else 3761 ucount = tr_link->nodes_cnt; 3762 3763 has_cookies = !!tr_link->cookies; 3764 show_addrs = kallsyms_show_value(current_cred()); 3765 3766 for (int i = 0; i < ucount; i++) { 3767 struct bpf_tracing_multi_node *mnode = &tr_link->nodes[i]; 3768 u64 addr, cookie; 3769 u32 id; 3770 3771 bpf_trampoline_unpack_key(mnode->trampoline->key, NULL, &id); 3772 3773 addr = show_addrs ? mnode->trampoline->ip : 0; 3774 cookie = has_cookies ? tr_link->cookies[i] : 0; 3775 3776 if (uids && put_user(id, uids + i)) 3777 return -EFAULT; 3778 if (uaddrs && put_user(addr, uaddrs + i)) 3779 return -EFAULT; 3780 if (ucookies && put_user(cookie, ucookies + i)) 3781 return -EFAULT; 3782 3783 cond_resched(); 3784 } 3785 3786 return err; 3787 } 3788 3789 #ifdef CONFIG_PROC_FS 3790 static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link, 3791 struct seq_file *seq) 3792 { 3793 struct bpf_tracing_multi_link *tr_link = 3794 container_of(link, struct bpf_tracing_multi_link, link); 3795 bool has_cookies = !!tr_link->cookies; 3796 3797 seq_printf(seq, "attach_type:\t%u\n", tr_link->link.attach_type); 3798 seq_printf(seq, "cnt:\t%u\n", tr_link->nodes_cnt); 3799 3800 seq_printf(seq, "%s\t %s\t %s\t %s\n", "obj-id", "btf-id", "cookie", "func"); 3801 for (int i = 0; i < tr_link->nodes_cnt; i++) { 3802 struct bpf_tracing_multi_node *mnode = &tr_link->nodes[i]; 3803 u32 btf_id, obj_id; 3804 3805 bpf_trampoline_unpack_key(mnode->trampoline->key, &obj_id, &btf_id); 3806 seq_printf(seq, "%u\t %u\t %llu\t %pS\n", 3807 obj_id, btf_id, 3808 has_cookies ? tr_link->cookies[i] : 0, 3809 (void *) mnode->trampoline->ip); 3810 3811 cond_resched(); 3812 } 3813 } 3814 #endif 3815 3816 static const struct bpf_link_ops bpf_tracing_multi_link_lops = { 3817 .release = bpf_tracing_multi_link_release, 3818 .dealloc_deferred = bpf_tracing_multi_link_dealloc, 3819 .fill_link_info = bpf_tracing_multi_link_fill_link_info, 3820 #ifdef CONFIG_PROC_FS 3821 .show_fdinfo = bpf_tracing_multi_show_fdinfo, 3822 #endif 3823 }; 3824 3825 static int ids_cmp_r(const void *pa, const void *pb, const void *priv __maybe_unused) 3826 { 3827 u32 a = *(u32 *) pa; 3828 u32 b = *(u32 *) pb; 3829 3830 return (a > b) - (a < b); 3831 } 3832 3833 static void ids_swap_r(void *a, void *b, int size __maybe_unused, 3834 const void *priv __maybe_unused) 3835 { 3836 u64 *cookie_a, *cookie_b, *cookies; 3837 u32 *id_a = a, *id_b = b, *ids; 3838 void **data = (void **) priv; 3839 3840 ids = data[0]; 3841 cookies = data[1]; 3842 3843 if (cookies) { 3844 cookie_a = cookies + (id_a - ids); 3845 cookie_b = cookies + (id_b - ids); 3846 swap(*cookie_a, *cookie_b); 3847 } 3848 swap(*id_a, *id_b); 3849 } 3850 3851 static int check_dup_ids(u32 *ids, u64 *cookies, u32 cnt) 3852 { 3853 void *data[2] = { ids, cookies }; 3854 int err = 0; 3855 3856 /* 3857 * Sort ids array (together with cookies array if defined) 3858 * and check it for duplicates. The ids and cookies arrays 3859 * are left sorted. 3860 */ 3861 sort_r_nonatomic(ids, cnt, sizeof(ids[0]), ids_cmp_r, ids_swap_r, data); 3862 3863 for (int i = 1; i < cnt; i++) { 3864 if (ids[i] == ids[i - 1]) { 3865 err = -EINVAL; 3866 break; 3867 } 3868 } 3869 return err; 3870 } 3871 3872 int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) 3873 { 3874 struct bpf_tracing_multi_link *link = NULL; 3875 struct bpf_tramp_node *fexits = NULL; 3876 struct bpf_link_primer link_primer; 3877 u32 cnt, *ids = NULL; 3878 u64 __user *ucookies; 3879 u64 *cookies = NULL; 3880 u32 __user *uids; 3881 int err; 3882 3883 uids = u64_to_user_ptr(attr->link_create.tracing_multi.ids); 3884 cnt = attr->link_create.tracing_multi.cnt; 3885 3886 if (!cnt || !uids) 3887 return -EINVAL; 3888 if (cnt > MAX_TRACING_MULTI_CNT) 3889 return -E2BIG; 3890 if (attr->link_create.flags || attr->link_create.target_fd) 3891 return -EINVAL; 3892 3893 ids = kvmalloc_objs(*ids, cnt); 3894 if (!ids) 3895 return -ENOMEM; 3896 3897 if (copy_from_user(ids, uids, cnt * sizeof(*ids))) { 3898 err = -EFAULT; 3899 goto error; 3900 } 3901 3902 ucookies = u64_to_user_ptr(attr->link_create.tracing_multi.cookies); 3903 if (ucookies) { 3904 cookies = kvmalloc_objs(*cookies, cnt); 3905 if (!cookies) { 3906 err = -ENOMEM; 3907 goto error; 3908 } 3909 if (copy_from_user(cookies, ucookies, cnt * sizeof(*cookies))) { 3910 err = -EFAULT; 3911 goto error; 3912 } 3913 } 3914 3915 err = check_dup_ids(ids, cookies, cnt); 3916 if (err) 3917 goto error; 3918 3919 if (prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI) { 3920 fexits = kvmalloc_objs(*fexits, cnt); 3921 if (!fexits) { 3922 err = -ENOMEM; 3923 goto error; 3924 } 3925 } 3926 3927 link = kvzalloc_flex(*link, nodes, cnt); 3928 if (!link) { 3929 err = -ENOMEM; 3930 goto error; 3931 } 3932 3933 bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING_MULTI, 3934 &bpf_tracing_multi_link_lops, prog, prog->expected_attach_type); 3935 3936 err = bpf_link_prime(&link->link, &link_primer); 3937 if (err) 3938 goto error; 3939 3940 link->nodes_cnt = cnt; 3941 link->cookies = cookies; 3942 link->fexits = fexits; 3943 3944 err = bpf_trampoline_multi_attach(prog, ids, link); 3945 kvfree(ids); 3946 if (err) { 3947 bpf_link_cleanup(&link_primer); 3948 return err; 3949 } 3950 return bpf_link_settle(&link_primer); 3951 3952 error: 3953 kvfree(fexits); 3954 kvfree(cookies); 3955 kvfree(ids); 3956 kvfree(link); 3957 return err; 3958 } 3959 3960 #else 3961 3962 int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) 3963 { 3964 return -EOPNOTSUPP; 3965 } 3966 3967 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS && CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */ 3968