1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 */ 4 #include <linux/bpf.h> 5 #include <linux/bpf_trace.h> 6 #include <linux/bpf_lirc.h> 7 #include <linux/btf.h> 8 #include <linux/syscalls.h> 9 #include <linux/slab.h> 10 #include <linux/sched/signal.h> 11 #include <linux/vmalloc.h> 12 #include <linux/mmzone.h> 13 #include <linux/anon_inodes.h> 14 #include <linux/fdtable.h> 15 #include <linux/file.h> 16 #include <linux/fs.h> 17 #include <linux/license.h> 18 #include <linux/filter.h> 19 #include <linux/version.h> 20 #include <linux/kernel.h> 21 #include <linux/idr.h> 22 #include <linux/cred.h> 23 #include <linux/timekeeping.h> 24 #include <linux/ctype.h> 25 #include <linux/nospec.h> 26 #include <linux/audit.h> 27 #include <uapi/linux/btf.h> 28 #include <linux/bpf_lsm.h> 29 30 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 31 (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 32 (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 33 #define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY) 34 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 35 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \ 36 IS_FD_HASH(map)) 37 38 #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) 39 40 DEFINE_PER_CPU(int, bpf_prog_active); 41 static DEFINE_IDR(prog_idr); 42 static DEFINE_SPINLOCK(prog_idr_lock); 43 static DEFINE_IDR(map_idr); 44 static DEFINE_SPINLOCK(map_idr_lock); 45 static DEFINE_IDR(link_idr); 46 static DEFINE_SPINLOCK(link_idr_lock); 47 48 int sysctl_unprivileged_bpf_disabled __read_mostly; 49 50 static const struct bpf_map_ops * const bpf_map_types[] = { 51 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) 52 #define BPF_MAP_TYPE(_id, _ops) \ 53 [_id] = &_ops, 54 #define BPF_LINK_TYPE(_id, _name) 55 #include <linux/bpf_types.h> 56 #undef BPF_PROG_TYPE 57 #undef BPF_MAP_TYPE 58 #undef BPF_LINK_TYPE 59 }; 60 61 /* 62 * If we're handed a bigger struct than we know of, ensure all the unknown bits 63 * are 0 - i.e. new user-space does not rely on any kernel feature extensions 64 * we don't know about yet. 65 * 66 * There is a ToCToU between this function call and the following 67 * copy_from_user() call. However, this is not a concern since this function is 68 * meant to be a future-proofing of bits. 69 */ 70 int bpf_check_uarg_tail_zero(void __user *uaddr, 71 size_t expected_size, 72 size_t actual_size) 73 { 74 unsigned char __user *addr; 75 unsigned char __user *end; 76 unsigned char val; 77 int err; 78 79 if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 80 return -E2BIG; 81 82 if (unlikely(!access_ok(uaddr, actual_size))) 83 return -EFAULT; 84 85 if (actual_size <= expected_size) 86 return 0; 87 88 addr = uaddr + expected_size; 89 end = uaddr + actual_size; 90 91 for (; addr < end; addr++) { 92 err = get_user(val, addr); 93 if (err) 94 return err; 95 if (val) 96 return -E2BIG; 97 } 98 99 return 0; 100 } 101 102 const struct bpf_map_ops bpf_map_offload_ops = { 103 .map_alloc = bpf_map_offload_map_alloc, 104 .map_free = bpf_map_offload_map_free, 105 .map_check_btf = map_check_no_btf, 106 }; 107 108 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 109 { 110 const struct bpf_map_ops *ops; 111 u32 type = attr->map_type; 112 struct bpf_map *map; 113 int err; 114 115 if (type >= ARRAY_SIZE(bpf_map_types)) 116 return ERR_PTR(-EINVAL); 117 type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types)); 118 ops = bpf_map_types[type]; 119 if (!ops) 120 return ERR_PTR(-EINVAL); 121 122 if (ops->map_alloc_check) { 123 err = ops->map_alloc_check(attr); 124 if (err) 125 return ERR_PTR(err); 126 } 127 if (attr->map_ifindex) 128 ops = &bpf_map_offload_ops; 129 map = ops->map_alloc(attr); 130 if (IS_ERR(map)) 131 return map; 132 map->ops = ops; 133 map->map_type = type; 134 return map; 135 } 136 137 static u32 bpf_map_value_size(struct bpf_map *map) 138 { 139 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 140 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 141 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 142 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 143 return round_up(map->value_size, 8) * num_possible_cpus(); 144 else if (IS_FD_MAP(map)) 145 return sizeof(u32); 146 else 147 return map->value_size; 148 } 149 150 static void maybe_wait_bpf_programs(struct bpf_map *map) 151 { 152 /* Wait for any running BPF programs to complete so that 153 * userspace, when we return to it, knows that all programs 154 * that could be running use the new map value. 155 */ 156 if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS || 157 map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 158 synchronize_rcu(); 159 } 160 161 static int bpf_map_update_value(struct bpf_map *map, struct fd f, void *key, 162 void *value, __u64 flags) 163 { 164 int err; 165 166 /* Need to create a kthread, thus must support schedule */ 167 if (bpf_map_is_dev_bound(map)) { 168 return bpf_map_offload_update_elem(map, key, value, flags); 169 } else if (map->map_type == BPF_MAP_TYPE_CPUMAP || 170 map->map_type == BPF_MAP_TYPE_SOCKHASH || 171 map->map_type == BPF_MAP_TYPE_SOCKMAP || 172 map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 173 return map->ops->map_update_elem(map, key, value, flags); 174 } else if (IS_FD_PROG_ARRAY(map)) { 175 return bpf_fd_array_map_update_elem(map, f.file, key, value, 176 flags); 177 } 178 179 bpf_disable_instrumentation(); 180 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 181 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 182 err = bpf_percpu_hash_update(map, key, value, flags); 183 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 184 err = bpf_percpu_array_update(map, key, value, flags); 185 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 186 err = bpf_percpu_cgroup_storage_update(map, key, value, 187 flags); 188 } else if (IS_FD_ARRAY(map)) { 189 rcu_read_lock(); 190 err = bpf_fd_array_map_update_elem(map, f.file, key, value, 191 flags); 192 rcu_read_unlock(); 193 } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 194 rcu_read_lock(); 195 err = bpf_fd_htab_map_update_elem(map, f.file, key, value, 196 flags); 197 rcu_read_unlock(); 198 } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 199 /* rcu_read_lock() is not needed */ 200 err = bpf_fd_reuseport_array_update_elem(map, key, value, 201 flags); 202 } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 203 map->map_type == BPF_MAP_TYPE_STACK) { 204 err = map->ops->map_push_elem(map, value, flags); 205 } else { 206 rcu_read_lock(); 207 err = map->ops->map_update_elem(map, key, value, flags); 208 rcu_read_unlock(); 209 } 210 bpf_enable_instrumentation(); 211 maybe_wait_bpf_programs(map); 212 213 return err; 214 } 215 216 static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value, 217 __u64 flags) 218 { 219 void *ptr; 220 int err; 221 222 if (bpf_map_is_dev_bound(map)) 223 return bpf_map_offload_lookup_elem(map, key, value); 224 225 bpf_disable_instrumentation(); 226 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 227 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 228 err = bpf_percpu_hash_copy(map, key, value); 229 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 230 err = bpf_percpu_array_copy(map, key, value); 231 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 232 err = bpf_percpu_cgroup_storage_copy(map, key, value); 233 } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 234 err = bpf_stackmap_copy(map, key, value); 235 } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) { 236 err = bpf_fd_array_map_lookup_elem(map, key, value); 237 } else if (IS_FD_HASH(map)) { 238 err = bpf_fd_htab_map_lookup_elem(map, key, value); 239 } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 240 err = bpf_fd_reuseport_array_lookup_elem(map, key, value); 241 } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 242 map->map_type == BPF_MAP_TYPE_STACK) { 243 err = map->ops->map_peek_elem(map, value); 244 } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 245 /* struct_ops map requires directly updating "value" */ 246 err = bpf_struct_ops_map_sys_lookup_elem(map, key, value); 247 } else { 248 rcu_read_lock(); 249 if (map->ops->map_lookup_elem_sys_only) 250 ptr = map->ops->map_lookup_elem_sys_only(map, key); 251 else 252 ptr = map->ops->map_lookup_elem(map, key); 253 if (IS_ERR(ptr)) { 254 err = PTR_ERR(ptr); 255 } else if (!ptr) { 256 err = -ENOENT; 257 } else { 258 err = 0; 259 if (flags & BPF_F_LOCK) 260 /* lock 'ptr' and copy everything but lock */ 261 copy_map_value_locked(map, value, ptr, true); 262 else 263 copy_map_value(map, value, ptr); 264 /* mask lock, since value wasn't zero inited */ 265 check_and_init_map_lock(map, value); 266 } 267 rcu_read_unlock(); 268 } 269 270 bpf_enable_instrumentation(); 271 maybe_wait_bpf_programs(map); 272 273 return err; 274 } 275 276 static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable) 277 { 278 /* We really just want to fail instead of triggering OOM killer 279 * under memory pressure, therefore we set __GFP_NORETRY to kmalloc, 280 * which is used for lower order allocation requests. 281 * 282 * It has been observed that higher order allocation requests done by 283 * vmalloc with __GFP_NORETRY being set might fail due to not trying 284 * to reclaim memory from the page cache, thus we set 285 * __GFP_RETRY_MAYFAIL to avoid such situations. 286 */ 287 288 const gfp_t flags = __GFP_NOWARN | __GFP_ZERO; 289 void *area; 290 291 if (size >= SIZE_MAX) 292 return NULL; 293 294 /* kmalloc()'ed memory can't be mmap()'ed */ 295 if (!mmapable && size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 296 area = kmalloc_node(size, GFP_USER | __GFP_NORETRY | flags, 297 numa_node); 298 if (area != NULL) 299 return area; 300 } 301 if (mmapable) { 302 BUG_ON(!PAGE_ALIGNED(size)); 303 return vmalloc_user_node_flags(size, numa_node, GFP_KERNEL | 304 __GFP_RETRY_MAYFAIL | flags); 305 } 306 return __vmalloc_node_flags_caller(size, numa_node, 307 GFP_KERNEL | __GFP_RETRY_MAYFAIL | 308 flags, __builtin_return_address(0)); 309 } 310 311 void *bpf_map_area_alloc(u64 size, int numa_node) 312 { 313 return __bpf_map_area_alloc(size, numa_node, false); 314 } 315 316 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node) 317 { 318 return __bpf_map_area_alloc(size, numa_node, true); 319 } 320 321 void bpf_map_area_free(void *area) 322 { 323 kvfree(area); 324 } 325 326 static u32 bpf_map_flags_retain_permanent(u32 flags) 327 { 328 /* Some map creation flags are not tied to the map object but 329 * rather to the map fd instead, so they have no meaning upon 330 * map object inspection since multiple file descriptors with 331 * different (access) properties can exist here. Thus, given 332 * this has zero meaning for the map itself, lets clear these 333 * from here. 334 */ 335 return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY); 336 } 337 338 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) 339 { 340 map->map_type = attr->map_type; 341 map->key_size = attr->key_size; 342 map->value_size = attr->value_size; 343 map->max_entries = attr->max_entries; 344 map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags); 345 map->numa_node = bpf_map_attr_numa_node(attr); 346 } 347 348 static int bpf_charge_memlock(struct user_struct *user, u32 pages) 349 { 350 unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 351 352 if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) { 353 atomic_long_sub(pages, &user->locked_vm); 354 return -EPERM; 355 } 356 return 0; 357 } 358 359 static void bpf_uncharge_memlock(struct user_struct *user, u32 pages) 360 { 361 if (user) 362 atomic_long_sub(pages, &user->locked_vm); 363 } 364 365 int bpf_map_charge_init(struct bpf_map_memory *mem, u64 size) 366 { 367 u32 pages = round_up(size, PAGE_SIZE) >> PAGE_SHIFT; 368 struct user_struct *user; 369 int ret; 370 371 if (size >= U32_MAX - PAGE_SIZE) 372 return -E2BIG; 373 374 user = get_current_user(); 375 ret = bpf_charge_memlock(user, pages); 376 if (ret) { 377 free_uid(user); 378 return ret; 379 } 380 381 mem->pages = pages; 382 mem->user = user; 383 384 return 0; 385 } 386 387 void bpf_map_charge_finish(struct bpf_map_memory *mem) 388 { 389 bpf_uncharge_memlock(mem->user, mem->pages); 390 free_uid(mem->user); 391 } 392 393 void bpf_map_charge_move(struct bpf_map_memory *dst, 394 struct bpf_map_memory *src) 395 { 396 *dst = *src; 397 398 /* Make sure src will not be used for the redundant uncharging. */ 399 memset(src, 0, sizeof(struct bpf_map_memory)); 400 } 401 402 int bpf_map_charge_memlock(struct bpf_map *map, u32 pages) 403 { 404 int ret; 405 406 ret = bpf_charge_memlock(map->memory.user, pages); 407 if (ret) 408 return ret; 409 map->memory.pages += pages; 410 return ret; 411 } 412 413 void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages) 414 { 415 bpf_uncharge_memlock(map->memory.user, pages); 416 map->memory.pages -= pages; 417 } 418 419 static int bpf_map_alloc_id(struct bpf_map *map) 420 { 421 int id; 422 423 idr_preload(GFP_KERNEL); 424 spin_lock_bh(&map_idr_lock); 425 id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 426 if (id > 0) 427 map->id = id; 428 spin_unlock_bh(&map_idr_lock); 429 idr_preload_end(); 430 431 if (WARN_ON_ONCE(!id)) 432 return -ENOSPC; 433 434 return id > 0 ? 0 : id; 435 } 436 437 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) 438 { 439 unsigned long flags; 440 441 /* Offloaded maps are removed from the IDR store when their device 442 * disappears - even if someone holds an fd to them they are unusable, 443 * the memory is gone, all ops will fail; they are simply waiting for 444 * refcnt to drop to be freed. 445 */ 446 if (!map->id) 447 return; 448 449 if (do_idr_lock) 450 spin_lock_irqsave(&map_idr_lock, flags); 451 else 452 __acquire(&map_idr_lock); 453 454 idr_remove(&map_idr, map->id); 455 map->id = 0; 456 457 if (do_idr_lock) 458 spin_unlock_irqrestore(&map_idr_lock, flags); 459 else 460 __release(&map_idr_lock); 461 } 462 463 /* called from workqueue */ 464 static void bpf_map_free_deferred(struct work_struct *work) 465 { 466 struct bpf_map *map = container_of(work, struct bpf_map, work); 467 struct bpf_map_memory mem; 468 469 bpf_map_charge_move(&mem, &map->memory); 470 security_bpf_map_free(map); 471 /* implementation dependent freeing */ 472 map->ops->map_free(map); 473 bpf_map_charge_finish(&mem); 474 } 475 476 static void bpf_map_put_uref(struct bpf_map *map) 477 { 478 if (atomic64_dec_and_test(&map->usercnt)) { 479 if (map->ops->map_release_uref) 480 map->ops->map_release_uref(map); 481 } 482 } 483 484 /* decrement map refcnt and schedule it for freeing via workqueue 485 * (unrelying map implementation ops->map_free() might sleep) 486 */ 487 static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) 488 { 489 if (atomic64_dec_and_test(&map->refcnt)) { 490 /* bpf_map_free_id() must be called first */ 491 bpf_map_free_id(map, do_idr_lock); 492 btf_put(map->btf); 493 INIT_WORK(&map->work, bpf_map_free_deferred); 494 schedule_work(&map->work); 495 } 496 } 497 498 void bpf_map_put(struct bpf_map *map) 499 { 500 __bpf_map_put(map, true); 501 } 502 EXPORT_SYMBOL_GPL(bpf_map_put); 503 504 void bpf_map_put_with_uref(struct bpf_map *map) 505 { 506 bpf_map_put_uref(map); 507 bpf_map_put(map); 508 } 509 510 static int bpf_map_release(struct inode *inode, struct file *filp) 511 { 512 struct bpf_map *map = filp->private_data; 513 514 if (map->ops->map_release) 515 map->ops->map_release(map, filp); 516 517 bpf_map_put_with_uref(map); 518 return 0; 519 } 520 521 static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f) 522 { 523 fmode_t mode = f.file->f_mode; 524 525 /* Our file permissions may have been overridden by global 526 * map permissions facing syscall side. 527 */ 528 if (READ_ONCE(map->frozen)) 529 mode &= ~FMODE_CAN_WRITE; 530 return mode; 531 } 532 533 #ifdef CONFIG_PROC_FS 534 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 535 { 536 const struct bpf_map *map = filp->private_data; 537 const struct bpf_array *array; 538 u32 type = 0, jited = 0; 539 540 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 541 array = container_of(map, struct bpf_array, map); 542 type = array->aux->type; 543 jited = array->aux->jited; 544 } 545 546 seq_printf(m, 547 "map_type:\t%u\n" 548 "key_size:\t%u\n" 549 "value_size:\t%u\n" 550 "max_entries:\t%u\n" 551 "map_flags:\t%#x\n" 552 "memlock:\t%llu\n" 553 "map_id:\t%u\n" 554 "frozen:\t%u\n", 555 map->map_type, 556 map->key_size, 557 map->value_size, 558 map->max_entries, 559 map->map_flags, 560 map->memory.pages * 1ULL << PAGE_SHIFT, 561 map->id, 562 READ_ONCE(map->frozen)); 563 if (type) { 564 seq_printf(m, "owner_prog_type:\t%u\n", type); 565 seq_printf(m, "owner_jited:\t%u\n", jited); 566 } 567 } 568 #endif 569 570 static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, 571 loff_t *ppos) 572 { 573 /* We need this handler such that alloc_file() enables 574 * f_mode with FMODE_CAN_READ. 575 */ 576 return -EINVAL; 577 } 578 579 static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, 580 size_t siz, loff_t *ppos) 581 { 582 /* We need this handler such that alloc_file() enables 583 * f_mode with FMODE_CAN_WRITE. 584 */ 585 return -EINVAL; 586 } 587 588 /* called for any extra memory-mapped regions (except initial) */ 589 static void bpf_map_mmap_open(struct vm_area_struct *vma) 590 { 591 struct bpf_map *map = vma->vm_file->private_data; 592 593 if (vma->vm_flags & VM_MAYWRITE) { 594 mutex_lock(&map->freeze_mutex); 595 map->writecnt++; 596 mutex_unlock(&map->freeze_mutex); 597 } 598 } 599 600 /* called for all unmapped memory region (including initial) */ 601 static void bpf_map_mmap_close(struct vm_area_struct *vma) 602 { 603 struct bpf_map *map = vma->vm_file->private_data; 604 605 if (vma->vm_flags & VM_MAYWRITE) { 606 mutex_lock(&map->freeze_mutex); 607 map->writecnt--; 608 mutex_unlock(&map->freeze_mutex); 609 } 610 } 611 612 static const struct vm_operations_struct bpf_map_default_vmops = { 613 .open = bpf_map_mmap_open, 614 .close = bpf_map_mmap_close, 615 }; 616 617 static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma) 618 { 619 struct bpf_map *map = filp->private_data; 620 int err; 621 622 if (!map->ops->map_mmap || map_value_has_spin_lock(map)) 623 return -ENOTSUPP; 624 625 if (!(vma->vm_flags & VM_SHARED)) 626 return -EINVAL; 627 628 mutex_lock(&map->freeze_mutex); 629 630 if ((vma->vm_flags & VM_WRITE) && map->frozen) { 631 err = -EPERM; 632 goto out; 633 } 634 635 /* set default open/close callbacks */ 636 vma->vm_ops = &bpf_map_default_vmops; 637 vma->vm_private_data = map; 638 vma->vm_flags &= ~VM_MAYEXEC; 639 if (!(vma->vm_flags & VM_WRITE)) 640 /* disallow re-mapping with PROT_WRITE */ 641 vma->vm_flags &= ~VM_MAYWRITE; 642 643 err = map->ops->map_mmap(map, vma); 644 if (err) 645 goto out; 646 647 if (vma->vm_flags & VM_MAYWRITE) 648 map->writecnt++; 649 out: 650 mutex_unlock(&map->freeze_mutex); 651 return err; 652 } 653 654 const struct file_operations bpf_map_fops = { 655 #ifdef CONFIG_PROC_FS 656 .show_fdinfo = bpf_map_show_fdinfo, 657 #endif 658 .release = bpf_map_release, 659 .read = bpf_dummy_read, 660 .write = bpf_dummy_write, 661 .mmap = bpf_map_mmap, 662 }; 663 664 int bpf_map_new_fd(struct bpf_map *map, int flags) 665 { 666 int ret; 667 668 ret = security_bpf_map(map, OPEN_FMODE(flags)); 669 if (ret < 0) 670 return ret; 671 672 return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 673 flags | O_CLOEXEC); 674 } 675 676 int bpf_get_file_flag(int flags) 677 { 678 if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) 679 return -EINVAL; 680 if (flags & BPF_F_RDONLY) 681 return O_RDONLY; 682 if (flags & BPF_F_WRONLY) 683 return O_WRONLY; 684 return O_RDWR; 685 } 686 687 /* helper macro to check that unused fields 'union bpf_attr' are zero */ 688 #define CHECK_ATTR(CMD) \ 689 memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 690 sizeof(attr->CMD##_LAST_FIELD), 0, \ 691 sizeof(*attr) - \ 692 offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 693 sizeof(attr->CMD##_LAST_FIELD)) != NULL 694 695 /* dst and src must have at least "size" number of bytes. 696 * Return strlen on success and < 0 on error. 697 */ 698 int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size) 699 { 700 const char *end = src + size; 701 const char *orig_src = src; 702 703 memset(dst, 0, size); 704 /* Copy all isalnum(), '_' and '.' chars. */ 705 while (src < end && *src) { 706 if (!isalnum(*src) && 707 *src != '_' && *src != '.') 708 return -EINVAL; 709 *dst++ = *src++; 710 } 711 712 /* No '\0' found in "size" number of bytes */ 713 if (src == end) 714 return -EINVAL; 715 716 return src - orig_src; 717 } 718 719 int map_check_no_btf(const struct bpf_map *map, 720 const struct btf *btf, 721 const struct btf_type *key_type, 722 const struct btf_type *value_type) 723 { 724 return -ENOTSUPP; 725 } 726 727 static int map_check_btf(struct bpf_map *map, const struct btf *btf, 728 u32 btf_key_id, u32 btf_value_id) 729 { 730 const struct btf_type *key_type, *value_type; 731 u32 key_size, value_size; 732 int ret = 0; 733 734 /* Some maps allow key to be unspecified. */ 735 if (btf_key_id) { 736 key_type = btf_type_id_size(btf, &btf_key_id, &key_size); 737 if (!key_type || key_size != map->key_size) 738 return -EINVAL; 739 } else { 740 key_type = btf_type_by_id(btf, 0); 741 if (!map->ops->map_check_btf) 742 return -EINVAL; 743 } 744 745 value_type = btf_type_id_size(btf, &btf_value_id, &value_size); 746 if (!value_type || value_size != map->value_size) 747 return -EINVAL; 748 749 map->spin_lock_off = btf_find_spin_lock(btf, value_type); 750 751 if (map_value_has_spin_lock(map)) { 752 if (map->map_flags & BPF_F_RDONLY_PROG) 753 return -EACCES; 754 if (map->map_type != BPF_MAP_TYPE_HASH && 755 map->map_type != BPF_MAP_TYPE_ARRAY && 756 map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 757 map->map_type != BPF_MAP_TYPE_SK_STORAGE) 758 return -ENOTSUPP; 759 if (map->spin_lock_off + sizeof(struct bpf_spin_lock) > 760 map->value_size) { 761 WARN_ONCE(1, 762 "verifier bug spin_lock_off %d value_size %d\n", 763 map->spin_lock_off, map->value_size); 764 return -EFAULT; 765 } 766 } 767 768 if (map->ops->map_check_btf) 769 ret = map->ops->map_check_btf(map, btf, key_type, value_type); 770 771 return ret; 772 } 773 774 #define BPF_MAP_CREATE_LAST_FIELD btf_vmlinux_value_type_id 775 /* called via syscall */ 776 static int map_create(union bpf_attr *attr) 777 { 778 int numa_node = bpf_map_attr_numa_node(attr); 779 struct bpf_map_memory mem; 780 struct bpf_map *map; 781 int f_flags; 782 int err; 783 784 err = CHECK_ATTR(BPF_MAP_CREATE); 785 if (err) 786 return -EINVAL; 787 788 if (attr->btf_vmlinux_value_type_id) { 789 if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS || 790 attr->btf_key_type_id || attr->btf_value_type_id) 791 return -EINVAL; 792 } else if (attr->btf_key_type_id && !attr->btf_value_type_id) { 793 return -EINVAL; 794 } 795 796 f_flags = bpf_get_file_flag(attr->map_flags); 797 if (f_flags < 0) 798 return f_flags; 799 800 if (numa_node != NUMA_NO_NODE && 801 ((unsigned int)numa_node >= nr_node_ids || 802 !node_online(numa_node))) 803 return -EINVAL; 804 805 /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 806 map = find_and_alloc_map(attr); 807 if (IS_ERR(map)) 808 return PTR_ERR(map); 809 810 err = bpf_obj_name_cpy(map->name, attr->map_name, 811 sizeof(attr->map_name)); 812 if (err < 0) 813 goto free_map; 814 815 atomic64_set(&map->refcnt, 1); 816 atomic64_set(&map->usercnt, 1); 817 mutex_init(&map->freeze_mutex); 818 819 map->spin_lock_off = -EINVAL; 820 if (attr->btf_key_type_id || attr->btf_value_type_id || 821 /* Even the map's value is a kernel's struct, 822 * the bpf_prog.o must have BTF to begin with 823 * to figure out the corresponding kernel's 824 * counter part. Thus, attr->btf_fd has 825 * to be valid also. 826 */ 827 attr->btf_vmlinux_value_type_id) { 828 struct btf *btf; 829 830 btf = btf_get_by_fd(attr->btf_fd); 831 if (IS_ERR(btf)) { 832 err = PTR_ERR(btf); 833 goto free_map; 834 } 835 map->btf = btf; 836 837 if (attr->btf_value_type_id) { 838 err = map_check_btf(map, btf, attr->btf_key_type_id, 839 attr->btf_value_type_id); 840 if (err) 841 goto free_map; 842 } 843 844 map->btf_key_type_id = attr->btf_key_type_id; 845 map->btf_value_type_id = attr->btf_value_type_id; 846 map->btf_vmlinux_value_type_id = 847 attr->btf_vmlinux_value_type_id; 848 } 849 850 err = security_bpf_map_alloc(map); 851 if (err) 852 goto free_map; 853 854 err = bpf_map_alloc_id(map); 855 if (err) 856 goto free_map_sec; 857 858 err = bpf_map_new_fd(map, f_flags); 859 if (err < 0) { 860 /* failed to allocate fd. 861 * bpf_map_put_with_uref() is needed because the above 862 * bpf_map_alloc_id() has published the map 863 * to the userspace and the userspace may 864 * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 865 */ 866 bpf_map_put_with_uref(map); 867 return err; 868 } 869 870 return err; 871 872 free_map_sec: 873 security_bpf_map_free(map); 874 free_map: 875 btf_put(map->btf); 876 bpf_map_charge_move(&mem, &map->memory); 877 map->ops->map_free(map); 878 bpf_map_charge_finish(&mem); 879 return err; 880 } 881 882 /* if error is returned, fd is released. 883 * On success caller should complete fd access with matching fdput() 884 */ 885 struct bpf_map *__bpf_map_get(struct fd f) 886 { 887 if (!f.file) 888 return ERR_PTR(-EBADF); 889 if (f.file->f_op != &bpf_map_fops) { 890 fdput(f); 891 return ERR_PTR(-EINVAL); 892 } 893 894 return f.file->private_data; 895 } 896 897 void bpf_map_inc(struct bpf_map *map) 898 { 899 atomic64_inc(&map->refcnt); 900 } 901 EXPORT_SYMBOL_GPL(bpf_map_inc); 902 903 void bpf_map_inc_with_uref(struct bpf_map *map) 904 { 905 atomic64_inc(&map->refcnt); 906 atomic64_inc(&map->usercnt); 907 } 908 EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref); 909 910 struct bpf_map *bpf_map_get(u32 ufd) 911 { 912 struct fd f = fdget(ufd); 913 struct bpf_map *map; 914 915 map = __bpf_map_get(f); 916 if (IS_ERR(map)) 917 return map; 918 919 bpf_map_inc(map); 920 fdput(f); 921 922 return map; 923 } 924 925 struct bpf_map *bpf_map_get_with_uref(u32 ufd) 926 { 927 struct fd f = fdget(ufd); 928 struct bpf_map *map; 929 930 map = __bpf_map_get(f); 931 if (IS_ERR(map)) 932 return map; 933 934 bpf_map_inc_with_uref(map); 935 fdput(f); 936 937 return map; 938 } 939 940 /* map_idr_lock should have been held */ 941 static struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref) 942 { 943 int refold; 944 945 refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0); 946 if (!refold) 947 return ERR_PTR(-ENOENT); 948 if (uref) 949 atomic64_inc(&map->usercnt); 950 951 return map; 952 } 953 954 struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map) 955 { 956 spin_lock_bh(&map_idr_lock); 957 map = __bpf_map_inc_not_zero(map, false); 958 spin_unlock_bh(&map_idr_lock); 959 960 return map; 961 } 962 EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero); 963 964 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 965 { 966 return -ENOTSUPP; 967 } 968 969 static void *__bpf_copy_key(void __user *ukey, u64 key_size) 970 { 971 if (key_size) 972 return memdup_user(ukey, key_size); 973 974 if (ukey) 975 return ERR_PTR(-EINVAL); 976 977 return NULL; 978 } 979 980 /* last field in 'union bpf_attr' used by this command */ 981 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags 982 983 static int map_lookup_elem(union bpf_attr *attr) 984 { 985 void __user *ukey = u64_to_user_ptr(attr->key); 986 void __user *uvalue = u64_to_user_ptr(attr->value); 987 int ufd = attr->map_fd; 988 struct bpf_map *map; 989 void *key, *value; 990 u32 value_size; 991 struct fd f; 992 int err; 993 994 if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 995 return -EINVAL; 996 997 if (attr->flags & ~BPF_F_LOCK) 998 return -EINVAL; 999 1000 f = fdget(ufd); 1001 map = __bpf_map_get(f); 1002 if (IS_ERR(map)) 1003 return PTR_ERR(map); 1004 if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 1005 err = -EPERM; 1006 goto err_put; 1007 } 1008 1009 if ((attr->flags & BPF_F_LOCK) && 1010 !map_value_has_spin_lock(map)) { 1011 err = -EINVAL; 1012 goto err_put; 1013 } 1014 1015 key = __bpf_copy_key(ukey, map->key_size); 1016 if (IS_ERR(key)) { 1017 err = PTR_ERR(key); 1018 goto err_put; 1019 } 1020 1021 value_size = bpf_map_value_size(map); 1022 1023 err = -ENOMEM; 1024 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1025 if (!value) 1026 goto free_key; 1027 1028 err = bpf_map_copy_value(map, key, value, attr->flags); 1029 if (err) 1030 goto free_value; 1031 1032 err = -EFAULT; 1033 if (copy_to_user(uvalue, value, value_size) != 0) 1034 goto free_value; 1035 1036 err = 0; 1037 1038 free_value: 1039 kfree(value); 1040 free_key: 1041 kfree(key); 1042 err_put: 1043 fdput(f); 1044 return err; 1045 } 1046 1047 1048 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 1049 1050 static int map_update_elem(union bpf_attr *attr) 1051 { 1052 void __user *ukey = u64_to_user_ptr(attr->key); 1053 void __user *uvalue = u64_to_user_ptr(attr->value); 1054 int ufd = attr->map_fd; 1055 struct bpf_map *map; 1056 void *key, *value; 1057 u32 value_size; 1058 struct fd f; 1059 int err; 1060 1061 if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 1062 return -EINVAL; 1063 1064 f = fdget(ufd); 1065 map = __bpf_map_get(f); 1066 if (IS_ERR(map)) 1067 return PTR_ERR(map); 1068 if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 1069 err = -EPERM; 1070 goto err_put; 1071 } 1072 1073 if ((attr->flags & BPF_F_LOCK) && 1074 !map_value_has_spin_lock(map)) { 1075 err = -EINVAL; 1076 goto err_put; 1077 } 1078 1079 key = __bpf_copy_key(ukey, map->key_size); 1080 if (IS_ERR(key)) { 1081 err = PTR_ERR(key); 1082 goto err_put; 1083 } 1084 1085 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 1086 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 1087 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 1088 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 1089 value_size = round_up(map->value_size, 8) * num_possible_cpus(); 1090 else 1091 value_size = map->value_size; 1092 1093 err = -ENOMEM; 1094 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1095 if (!value) 1096 goto free_key; 1097 1098 err = -EFAULT; 1099 if (copy_from_user(value, uvalue, value_size) != 0) 1100 goto free_value; 1101 1102 err = bpf_map_update_value(map, f, key, value, attr->flags); 1103 1104 free_value: 1105 kfree(value); 1106 free_key: 1107 kfree(key); 1108 err_put: 1109 fdput(f); 1110 return err; 1111 } 1112 1113 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 1114 1115 static int map_delete_elem(union bpf_attr *attr) 1116 { 1117 void __user *ukey = u64_to_user_ptr(attr->key); 1118 int ufd = attr->map_fd; 1119 struct bpf_map *map; 1120 struct fd f; 1121 void *key; 1122 int err; 1123 1124 if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 1125 return -EINVAL; 1126 1127 f = fdget(ufd); 1128 map = __bpf_map_get(f); 1129 if (IS_ERR(map)) 1130 return PTR_ERR(map); 1131 if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 1132 err = -EPERM; 1133 goto err_put; 1134 } 1135 1136 key = __bpf_copy_key(ukey, map->key_size); 1137 if (IS_ERR(key)) { 1138 err = PTR_ERR(key); 1139 goto err_put; 1140 } 1141 1142 if (bpf_map_is_dev_bound(map)) { 1143 err = bpf_map_offload_delete_elem(map, key); 1144 goto out; 1145 } else if (IS_FD_PROG_ARRAY(map) || 1146 map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 1147 /* These maps require sleepable context */ 1148 err = map->ops->map_delete_elem(map, key); 1149 goto out; 1150 } 1151 1152 bpf_disable_instrumentation(); 1153 rcu_read_lock(); 1154 err = map->ops->map_delete_elem(map, key); 1155 rcu_read_unlock(); 1156 bpf_enable_instrumentation(); 1157 maybe_wait_bpf_programs(map); 1158 out: 1159 kfree(key); 1160 err_put: 1161 fdput(f); 1162 return err; 1163 } 1164 1165 /* last field in 'union bpf_attr' used by this command */ 1166 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 1167 1168 static int map_get_next_key(union bpf_attr *attr) 1169 { 1170 void __user *ukey = u64_to_user_ptr(attr->key); 1171 void __user *unext_key = u64_to_user_ptr(attr->next_key); 1172 int ufd = attr->map_fd; 1173 struct bpf_map *map; 1174 void *key, *next_key; 1175 struct fd f; 1176 int err; 1177 1178 if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 1179 return -EINVAL; 1180 1181 f = fdget(ufd); 1182 map = __bpf_map_get(f); 1183 if (IS_ERR(map)) 1184 return PTR_ERR(map); 1185 if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 1186 err = -EPERM; 1187 goto err_put; 1188 } 1189 1190 if (ukey) { 1191 key = __bpf_copy_key(ukey, map->key_size); 1192 if (IS_ERR(key)) { 1193 err = PTR_ERR(key); 1194 goto err_put; 1195 } 1196 } else { 1197 key = NULL; 1198 } 1199 1200 err = -ENOMEM; 1201 next_key = kmalloc(map->key_size, GFP_USER); 1202 if (!next_key) 1203 goto free_key; 1204 1205 if (bpf_map_is_dev_bound(map)) { 1206 err = bpf_map_offload_get_next_key(map, key, next_key); 1207 goto out; 1208 } 1209 1210 rcu_read_lock(); 1211 err = map->ops->map_get_next_key(map, key, next_key); 1212 rcu_read_unlock(); 1213 out: 1214 if (err) 1215 goto free_next_key; 1216 1217 err = -EFAULT; 1218 if (copy_to_user(unext_key, next_key, map->key_size) != 0) 1219 goto free_next_key; 1220 1221 err = 0; 1222 1223 free_next_key: 1224 kfree(next_key); 1225 free_key: 1226 kfree(key); 1227 err_put: 1228 fdput(f); 1229 return err; 1230 } 1231 1232 int generic_map_delete_batch(struct bpf_map *map, 1233 const union bpf_attr *attr, 1234 union bpf_attr __user *uattr) 1235 { 1236 void __user *keys = u64_to_user_ptr(attr->batch.keys); 1237 u32 cp, max_count; 1238 int err = 0; 1239 void *key; 1240 1241 if (attr->batch.elem_flags & ~BPF_F_LOCK) 1242 return -EINVAL; 1243 1244 if ((attr->batch.elem_flags & BPF_F_LOCK) && 1245 !map_value_has_spin_lock(map)) { 1246 return -EINVAL; 1247 } 1248 1249 max_count = attr->batch.count; 1250 if (!max_count) 1251 return 0; 1252 1253 key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 1254 if (!key) 1255 return -ENOMEM; 1256 1257 for (cp = 0; cp < max_count; cp++) { 1258 err = -EFAULT; 1259 if (copy_from_user(key, keys + cp * map->key_size, 1260 map->key_size)) 1261 break; 1262 1263 if (bpf_map_is_dev_bound(map)) { 1264 err = bpf_map_offload_delete_elem(map, key); 1265 break; 1266 } 1267 1268 bpf_disable_instrumentation(); 1269 rcu_read_lock(); 1270 err = map->ops->map_delete_elem(map, key); 1271 rcu_read_unlock(); 1272 bpf_enable_instrumentation(); 1273 maybe_wait_bpf_programs(map); 1274 if (err) 1275 break; 1276 } 1277 if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) 1278 err = -EFAULT; 1279 1280 kfree(key); 1281 return err; 1282 } 1283 1284 int generic_map_update_batch(struct bpf_map *map, 1285 const union bpf_attr *attr, 1286 union bpf_attr __user *uattr) 1287 { 1288 void __user *values = u64_to_user_ptr(attr->batch.values); 1289 void __user *keys = u64_to_user_ptr(attr->batch.keys); 1290 u32 value_size, cp, max_count; 1291 int ufd = attr->map_fd; 1292 void *key, *value; 1293 struct fd f; 1294 int err = 0; 1295 1296 f = fdget(ufd); 1297 if (attr->batch.elem_flags & ~BPF_F_LOCK) 1298 return -EINVAL; 1299 1300 if ((attr->batch.elem_flags & BPF_F_LOCK) && 1301 !map_value_has_spin_lock(map)) { 1302 return -EINVAL; 1303 } 1304 1305 value_size = bpf_map_value_size(map); 1306 1307 max_count = attr->batch.count; 1308 if (!max_count) 1309 return 0; 1310 1311 key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 1312 if (!key) 1313 return -ENOMEM; 1314 1315 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1316 if (!value) { 1317 kfree(key); 1318 return -ENOMEM; 1319 } 1320 1321 for (cp = 0; cp < max_count; cp++) { 1322 err = -EFAULT; 1323 if (copy_from_user(key, keys + cp * map->key_size, 1324 map->key_size) || 1325 copy_from_user(value, values + cp * value_size, value_size)) 1326 break; 1327 1328 err = bpf_map_update_value(map, f, key, value, 1329 attr->batch.elem_flags); 1330 1331 if (err) 1332 break; 1333 } 1334 1335 if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) 1336 err = -EFAULT; 1337 1338 kfree(value); 1339 kfree(key); 1340 return err; 1341 } 1342 1343 #define MAP_LOOKUP_RETRIES 3 1344 1345 int generic_map_lookup_batch(struct bpf_map *map, 1346 const union bpf_attr *attr, 1347 union bpf_attr __user *uattr) 1348 { 1349 void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch); 1350 void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch); 1351 void __user *values = u64_to_user_ptr(attr->batch.values); 1352 void __user *keys = u64_to_user_ptr(attr->batch.keys); 1353 void *buf, *buf_prevkey, *prev_key, *key, *value; 1354 int err, retry = MAP_LOOKUP_RETRIES; 1355 u32 value_size, cp, max_count; 1356 1357 if (attr->batch.elem_flags & ~BPF_F_LOCK) 1358 return -EINVAL; 1359 1360 if ((attr->batch.elem_flags & BPF_F_LOCK) && 1361 !map_value_has_spin_lock(map)) 1362 return -EINVAL; 1363 1364 value_size = bpf_map_value_size(map); 1365 1366 max_count = attr->batch.count; 1367 if (!max_count) 1368 return 0; 1369 1370 if (put_user(0, &uattr->batch.count)) 1371 return -EFAULT; 1372 1373 buf_prevkey = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 1374 if (!buf_prevkey) 1375 return -ENOMEM; 1376 1377 buf = kmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN); 1378 if (!buf) { 1379 kvfree(buf_prevkey); 1380 return -ENOMEM; 1381 } 1382 1383 err = -EFAULT; 1384 prev_key = NULL; 1385 if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size)) 1386 goto free_buf; 1387 key = buf; 1388 value = key + map->key_size; 1389 if (ubatch) 1390 prev_key = buf_prevkey; 1391 1392 for (cp = 0; cp < max_count;) { 1393 rcu_read_lock(); 1394 err = map->ops->map_get_next_key(map, prev_key, key); 1395 rcu_read_unlock(); 1396 if (err) 1397 break; 1398 err = bpf_map_copy_value(map, key, value, 1399 attr->batch.elem_flags); 1400 1401 if (err == -ENOENT) { 1402 if (retry) { 1403 retry--; 1404 continue; 1405 } 1406 err = -EINTR; 1407 break; 1408 } 1409 1410 if (err) 1411 goto free_buf; 1412 1413 if (copy_to_user(keys + cp * map->key_size, key, 1414 map->key_size)) { 1415 err = -EFAULT; 1416 goto free_buf; 1417 } 1418 if (copy_to_user(values + cp * value_size, value, value_size)) { 1419 err = -EFAULT; 1420 goto free_buf; 1421 } 1422 1423 if (!prev_key) 1424 prev_key = buf_prevkey; 1425 1426 swap(prev_key, key); 1427 retry = MAP_LOOKUP_RETRIES; 1428 cp++; 1429 } 1430 1431 if (err == -EFAULT) 1432 goto free_buf; 1433 1434 if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) || 1435 (cp && copy_to_user(uobatch, prev_key, map->key_size)))) 1436 err = -EFAULT; 1437 1438 free_buf: 1439 kfree(buf_prevkey); 1440 kfree(buf); 1441 return err; 1442 } 1443 1444 #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value 1445 1446 static int map_lookup_and_delete_elem(union bpf_attr *attr) 1447 { 1448 void __user *ukey = u64_to_user_ptr(attr->key); 1449 void __user *uvalue = u64_to_user_ptr(attr->value); 1450 int ufd = attr->map_fd; 1451 struct bpf_map *map; 1452 void *key, *value; 1453 u32 value_size; 1454 struct fd f; 1455 int err; 1456 1457 if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) 1458 return -EINVAL; 1459 1460 f = fdget(ufd); 1461 map = __bpf_map_get(f); 1462 if (IS_ERR(map)) 1463 return PTR_ERR(map); 1464 if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 1465 err = -EPERM; 1466 goto err_put; 1467 } 1468 1469 key = __bpf_copy_key(ukey, map->key_size); 1470 if (IS_ERR(key)) { 1471 err = PTR_ERR(key); 1472 goto err_put; 1473 } 1474 1475 value_size = map->value_size; 1476 1477 err = -ENOMEM; 1478 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1479 if (!value) 1480 goto free_key; 1481 1482 if (map->map_type == BPF_MAP_TYPE_QUEUE || 1483 map->map_type == BPF_MAP_TYPE_STACK) { 1484 err = map->ops->map_pop_elem(map, value); 1485 } else { 1486 err = -ENOTSUPP; 1487 } 1488 1489 if (err) 1490 goto free_value; 1491 1492 if (copy_to_user(uvalue, value, value_size) != 0) 1493 goto free_value; 1494 1495 err = 0; 1496 1497 free_value: 1498 kfree(value); 1499 free_key: 1500 kfree(key); 1501 err_put: 1502 fdput(f); 1503 return err; 1504 } 1505 1506 #define BPF_MAP_FREEZE_LAST_FIELD map_fd 1507 1508 static int map_freeze(const union bpf_attr *attr) 1509 { 1510 int err = 0, ufd = attr->map_fd; 1511 struct bpf_map *map; 1512 struct fd f; 1513 1514 if (CHECK_ATTR(BPF_MAP_FREEZE)) 1515 return -EINVAL; 1516 1517 f = fdget(ufd); 1518 map = __bpf_map_get(f); 1519 if (IS_ERR(map)) 1520 return PTR_ERR(map); 1521 1522 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 1523 fdput(f); 1524 return -ENOTSUPP; 1525 } 1526 1527 mutex_lock(&map->freeze_mutex); 1528 1529 if (map->writecnt) { 1530 err = -EBUSY; 1531 goto err_put; 1532 } 1533 if (READ_ONCE(map->frozen)) { 1534 err = -EBUSY; 1535 goto err_put; 1536 } 1537 if (!capable(CAP_SYS_ADMIN)) { 1538 err = -EPERM; 1539 goto err_put; 1540 } 1541 1542 WRITE_ONCE(map->frozen, true); 1543 err_put: 1544 mutex_unlock(&map->freeze_mutex); 1545 fdput(f); 1546 return err; 1547 } 1548 1549 static const struct bpf_prog_ops * const bpf_prog_types[] = { 1550 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 1551 [_id] = & _name ## _prog_ops, 1552 #define BPF_MAP_TYPE(_id, _ops) 1553 #define BPF_LINK_TYPE(_id, _name) 1554 #include <linux/bpf_types.h> 1555 #undef BPF_PROG_TYPE 1556 #undef BPF_MAP_TYPE 1557 #undef BPF_LINK_TYPE 1558 }; 1559 1560 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 1561 { 1562 const struct bpf_prog_ops *ops; 1563 1564 if (type >= ARRAY_SIZE(bpf_prog_types)) 1565 return -EINVAL; 1566 type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types)); 1567 ops = bpf_prog_types[type]; 1568 if (!ops) 1569 return -EINVAL; 1570 1571 if (!bpf_prog_is_dev_bound(prog->aux)) 1572 prog->aux->ops = ops; 1573 else 1574 prog->aux->ops = &bpf_offload_prog_ops; 1575 prog->type = type; 1576 return 0; 1577 } 1578 1579 enum bpf_audit { 1580 BPF_AUDIT_LOAD, 1581 BPF_AUDIT_UNLOAD, 1582 BPF_AUDIT_MAX, 1583 }; 1584 1585 static const char * const bpf_audit_str[BPF_AUDIT_MAX] = { 1586 [BPF_AUDIT_LOAD] = "LOAD", 1587 [BPF_AUDIT_UNLOAD] = "UNLOAD", 1588 }; 1589 1590 static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op) 1591 { 1592 struct audit_context *ctx = NULL; 1593 struct audit_buffer *ab; 1594 1595 if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX)) 1596 return; 1597 if (audit_enabled == AUDIT_OFF) 1598 return; 1599 if (op == BPF_AUDIT_LOAD) 1600 ctx = audit_context(); 1601 ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF); 1602 if (unlikely(!ab)) 1603 return; 1604 audit_log_format(ab, "prog-id=%u op=%s", 1605 prog->aux->id, bpf_audit_str[op]); 1606 audit_log_end(ab); 1607 } 1608 1609 int __bpf_prog_charge(struct user_struct *user, u32 pages) 1610 { 1611 unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 1612 unsigned long user_bufs; 1613 1614 if (user) { 1615 user_bufs = atomic_long_add_return(pages, &user->locked_vm); 1616 if (user_bufs > memlock_limit) { 1617 atomic_long_sub(pages, &user->locked_vm); 1618 return -EPERM; 1619 } 1620 } 1621 1622 return 0; 1623 } 1624 1625 void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 1626 { 1627 if (user) 1628 atomic_long_sub(pages, &user->locked_vm); 1629 } 1630 1631 static int bpf_prog_charge_memlock(struct bpf_prog *prog) 1632 { 1633 struct user_struct *user = get_current_user(); 1634 int ret; 1635 1636 ret = __bpf_prog_charge(user, prog->pages); 1637 if (ret) { 1638 free_uid(user); 1639 return ret; 1640 } 1641 1642 prog->aux->user = user; 1643 return 0; 1644 } 1645 1646 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 1647 { 1648 struct user_struct *user = prog->aux->user; 1649 1650 __bpf_prog_uncharge(user, prog->pages); 1651 free_uid(user); 1652 } 1653 1654 static int bpf_prog_alloc_id(struct bpf_prog *prog) 1655 { 1656 int id; 1657 1658 idr_preload(GFP_KERNEL); 1659 spin_lock_bh(&prog_idr_lock); 1660 id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 1661 if (id > 0) 1662 prog->aux->id = id; 1663 spin_unlock_bh(&prog_idr_lock); 1664 idr_preload_end(); 1665 1666 /* id is in [1, INT_MAX) */ 1667 if (WARN_ON_ONCE(!id)) 1668 return -ENOSPC; 1669 1670 return id > 0 ? 0 : id; 1671 } 1672 1673 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) 1674 { 1675 /* cBPF to eBPF migrations are currently not in the idr store. 1676 * Offloaded programs are removed from the store when their device 1677 * disappears - even if someone grabs an fd to them they are unusable, 1678 * simply waiting for refcnt to drop to be freed. 1679 */ 1680 if (!prog->aux->id) 1681 return; 1682 1683 if (do_idr_lock) 1684 spin_lock_bh(&prog_idr_lock); 1685 else 1686 __acquire(&prog_idr_lock); 1687 1688 idr_remove(&prog_idr, prog->aux->id); 1689 prog->aux->id = 0; 1690 1691 if (do_idr_lock) 1692 spin_unlock_bh(&prog_idr_lock); 1693 else 1694 __release(&prog_idr_lock); 1695 } 1696 1697 static void __bpf_prog_put_rcu(struct rcu_head *rcu) 1698 { 1699 struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 1700 1701 kvfree(aux->func_info); 1702 kfree(aux->func_info_aux); 1703 bpf_prog_uncharge_memlock(aux->prog); 1704 security_bpf_prog_free(aux); 1705 bpf_prog_free(aux->prog); 1706 } 1707 1708 static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) 1709 { 1710 bpf_prog_kallsyms_del_all(prog); 1711 btf_put(prog->aux->btf); 1712 bpf_prog_free_linfo(prog); 1713 1714 if (deferred) 1715 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 1716 else 1717 __bpf_prog_put_rcu(&prog->aux->rcu); 1718 } 1719 1720 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 1721 { 1722 if (atomic64_dec_and_test(&prog->aux->refcnt)) { 1723 perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0); 1724 bpf_audit_prog(prog, BPF_AUDIT_UNLOAD); 1725 /* bpf_prog_free_id() must be called first */ 1726 bpf_prog_free_id(prog, do_idr_lock); 1727 __bpf_prog_put_noref(prog, true); 1728 } 1729 } 1730 1731 void bpf_prog_put(struct bpf_prog *prog) 1732 { 1733 __bpf_prog_put(prog, true); 1734 } 1735 EXPORT_SYMBOL_GPL(bpf_prog_put); 1736 1737 static int bpf_prog_release(struct inode *inode, struct file *filp) 1738 { 1739 struct bpf_prog *prog = filp->private_data; 1740 1741 bpf_prog_put(prog); 1742 return 0; 1743 } 1744 1745 static void bpf_prog_get_stats(const struct bpf_prog *prog, 1746 struct bpf_prog_stats *stats) 1747 { 1748 u64 nsecs = 0, cnt = 0; 1749 int cpu; 1750 1751 for_each_possible_cpu(cpu) { 1752 const struct bpf_prog_stats *st; 1753 unsigned int start; 1754 u64 tnsecs, tcnt; 1755 1756 st = per_cpu_ptr(prog->aux->stats, cpu); 1757 do { 1758 start = u64_stats_fetch_begin_irq(&st->syncp); 1759 tnsecs = st->nsecs; 1760 tcnt = st->cnt; 1761 } while (u64_stats_fetch_retry_irq(&st->syncp, start)); 1762 nsecs += tnsecs; 1763 cnt += tcnt; 1764 } 1765 stats->nsecs = nsecs; 1766 stats->cnt = cnt; 1767 } 1768 1769 #ifdef CONFIG_PROC_FS 1770 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 1771 { 1772 const struct bpf_prog *prog = filp->private_data; 1773 char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 1774 struct bpf_prog_stats stats; 1775 1776 bpf_prog_get_stats(prog, &stats); 1777 bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 1778 seq_printf(m, 1779 "prog_type:\t%u\n" 1780 "prog_jited:\t%u\n" 1781 "prog_tag:\t%s\n" 1782 "memlock:\t%llu\n" 1783 "prog_id:\t%u\n" 1784 "run_time_ns:\t%llu\n" 1785 "run_cnt:\t%llu\n", 1786 prog->type, 1787 prog->jited, 1788 prog_tag, 1789 prog->pages * 1ULL << PAGE_SHIFT, 1790 prog->aux->id, 1791 stats.nsecs, 1792 stats.cnt); 1793 } 1794 #endif 1795 1796 const struct file_operations bpf_prog_fops = { 1797 #ifdef CONFIG_PROC_FS 1798 .show_fdinfo = bpf_prog_show_fdinfo, 1799 #endif 1800 .release = bpf_prog_release, 1801 .read = bpf_dummy_read, 1802 .write = bpf_dummy_write, 1803 }; 1804 1805 int bpf_prog_new_fd(struct bpf_prog *prog) 1806 { 1807 int ret; 1808 1809 ret = security_bpf_prog(prog); 1810 if (ret < 0) 1811 return ret; 1812 1813 return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 1814 O_RDWR | O_CLOEXEC); 1815 } 1816 1817 static struct bpf_prog *____bpf_prog_get(struct fd f) 1818 { 1819 if (!f.file) 1820 return ERR_PTR(-EBADF); 1821 if (f.file->f_op != &bpf_prog_fops) { 1822 fdput(f); 1823 return ERR_PTR(-EINVAL); 1824 } 1825 1826 return f.file->private_data; 1827 } 1828 1829 void bpf_prog_add(struct bpf_prog *prog, int i) 1830 { 1831 atomic64_add(i, &prog->aux->refcnt); 1832 } 1833 EXPORT_SYMBOL_GPL(bpf_prog_add); 1834 1835 void bpf_prog_sub(struct bpf_prog *prog, int i) 1836 { 1837 /* Only to be used for undoing previous bpf_prog_add() in some 1838 * error path. We still know that another entity in our call 1839 * path holds a reference to the program, thus atomic_sub() can 1840 * be safely used in such cases! 1841 */ 1842 WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0); 1843 } 1844 EXPORT_SYMBOL_GPL(bpf_prog_sub); 1845 1846 void bpf_prog_inc(struct bpf_prog *prog) 1847 { 1848 atomic64_inc(&prog->aux->refcnt); 1849 } 1850 EXPORT_SYMBOL_GPL(bpf_prog_inc); 1851 1852 /* prog_idr_lock should have been held */ 1853 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 1854 { 1855 int refold; 1856 1857 refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0); 1858 1859 if (!refold) 1860 return ERR_PTR(-ENOENT); 1861 1862 return prog; 1863 } 1864 EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 1865 1866 bool bpf_prog_get_ok(struct bpf_prog *prog, 1867 enum bpf_prog_type *attach_type, bool attach_drv) 1868 { 1869 /* not an attachment, just a refcount inc, always allow */ 1870 if (!attach_type) 1871 return true; 1872 1873 if (prog->type != *attach_type) 1874 return false; 1875 if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv) 1876 return false; 1877 1878 return true; 1879 } 1880 1881 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, 1882 bool attach_drv) 1883 { 1884 struct fd f = fdget(ufd); 1885 struct bpf_prog *prog; 1886 1887 prog = ____bpf_prog_get(f); 1888 if (IS_ERR(prog)) 1889 return prog; 1890 if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) { 1891 prog = ERR_PTR(-EINVAL); 1892 goto out; 1893 } 1894 1895 bpf_prog_inc(prog); 1896 out: 1897 fdput(f); 1898 return prog; 1899 } 1900 1901 struct bpf_prog *bpf_prog_get(u32 ufd) 1902 { 1903 return __bpf_prog_get(ufd, NULL, false); 1904 } 1905 1906 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 1907 bool attach_drv) 1908 { 1909 return __bpf_prog_get(ufd, &type, attach_drv); 1910 } 1911 EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev); 1912 1913 /* Initially all BPF programs could be loaded w/o specifying 1914 * expected_attach_type. Later for some of them specifying expected_attach_type 1915 * at load time became required so that program could be validated properly. 1916 * Programs of types that are allowed to be loaded both w/ and w/o (for 1917 * backward compatibility) expected_attach_type, should have the default attach 1918 * type assigned to expected_attach_type for the latter case, so that it can be 1919 * validated later at attach time. 1920 * 1921 * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if 1922 * prog type requires it but has some attach types that have to be backward 1923 * compatible. 1924 */ 1925 static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) 1926 { 1927 switch (attr->prog_type) { 1928 case BPF_PROG_TYPE_CGROUP_SOCK: 1929 /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't 1930 * exist so checking for non-zero is the way to go here. 1931 */ 1932 if (!attr->expected_attach_type) 1933 attr->expected_attach_type = 1934 BPF_CGROUP_INET_SOCK_CREATE; 1935 break; 1936 } 1937 } 1938 1939 static int 1940 bpf_prog_load_check_attach(enum bpf_prog_type prog_type, 1941 enum bpf_attach_type expected_attach_type, 1942 u32 btf_id, u32 prog_fd) 1943 { 1944 if (btf_id) { 1945 if (btf_id > BTF_MAX_TYPE) 1946 return -EINVAL; 1947 1948 switch (prog_type) { 1949 case BPF_PROG_TYPE_TRACING: 1950 case BPF_PROG_TYPE_LSM: 1951 case BPF_PROG_TYPE_STRUCT_OPS: 1952 case BPF_PROG_TYPE_EXT: 1953 break; 1954 default: 1955 return -EINVAL; 1956 } 1957 } 1958 1959 if (prog_fd && prog_type != BPF_PROG_TYPE_TRACING && 1960 prog_type != BPF_PROG_TYPE_EXT) 1961 return -EINVAL; 1962 1963 switch (prog_type) { 1964 case BPF_PROG_TYPE_CGROUP_SOCK: 1965 switch (expected_attach_type) { 1966 case BPF_CGROUP_INET_SOCK_CREATE: 1967 case BPF_CGROUP_INET4_POST_BIND: 1968 case BPF_CGROUP_INET6_POST_BIND: 1969 return 0; 1970 default: 1971 return -EINVAL; 1972 } 1973 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 1974 switch (expected_attach_type) { 1975 case BPF_CGROUP_INET4_BIND: 1976 case BPF_CGROUP_INET6_BIND: 1977 case BPF_CGROUP_INET4_CONNECT: 1978 case BPF_CGROUP_INET6_CONNECT: 1979 case BPF_CGROUP_UDP4_SENDMSG: 1980 case BPF_CGROUP_UDP6_SENDMSG: 1981 case BPF_CGROUP_UDP4_RECVMSG: 1982 case BPF_CGROUP_UDP6_RECVMSG: 1983 return 0; 1984 default: 1985 return -EINVAL; 1986 } 1987 case BPF_PROG_TYPE_CGROUP_SKB: 1988 switch (expected_attach_type) { 1989 case BPF_CGROUP_INET_INGRESS: 1990 case BPF_CGROUP_INET_EGRESS: 1991 return 0; 1992 default: 1993 return -EINVAL; 1994 } 1995 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 1996 switch (expected_attach_type) { 1997 case BPF_CGROUP_SETSOCKOPT: 1998 case BPF_CGROUP_GETSOCKOPT: 1999 return 0; 2000 default: 2001 return -EINVAL; 2002 } 2003 case BPF_PROG_TYPE_EXT: 2004 if (expected_attach_type) 2005 return -EINVAL; 2006 /* fallthrough */ 2007 default: 2008 return 0; 2009 } 2010 } 2011 2012 /* last field in 'union bpf_attr' used by this command */ 2013 #define BPF_PROG_LOAD_LAST_FIELD attach_prog_fd 2014 2015 static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) 2016 { 2017 enum bpf_prog_type type = attr->prog_type; 2018 struct bpf_prog *prog; 2019 int err; 2020 char license[128]; 2021 bool is_gpl; 2022 2023 if (CHECK_ATTR(BPF_PROG_LOAD)) 2024 return -EINVAL; 2025 2026 if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | 2027 BPF_F_ANY_ALIGNMENT | 2028 BPF_F_TEST_STATE_FREQ | 2029 BPF_F_TEST_RND_HI32)) 2030 return -EINVAL; 2031 2032 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && 2033 (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && 2034 !capable(CAP_SYS_ADMIN)) 2035 return -EPERM; 2036 2037 /* copy eBPF program license from user space */ 2038 if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 2039 sizeof(license) - 1) < 0) 2040 return -EFAULT; 2041 license[sizeof(license) - 1] = 0; 2042 2043 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 2044 is_gpl = license_is_gpl_compatible(license); 2045 2046 if (attr->insn_cnt == 0 || 2047 attr->insn_cnt > (capable(CAP_SYS_ADMIN) ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) 2048 return -E2BIG; 2049 if (type != BPF_PROG_TYPE_SOCKET_FILTER && 2050 type != BPF_PROG_TYPE_CGROUP_SKB && 2051 !capable(CAP_SYS_ADMIN)) 2052 return -EPERM; 2053 2054 bpf_prog_load_fixup_attach_type(attr); 2055 if (bpf_prog_load_check_attach(type, attr->expected_attach_type, 2056 attr->attach_btf_id, 2057 attr->attach_prog_fd)) 2058 return -EINVAL; 2059 2060 /* plain bpf_prog allocation */ 2061 prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 2062 if (!prog) 2063 return -ENOMEM; 2064 2065 prog->expected_attach_type = attr->expected_attach_type; 2066 prog->aux->attach_btf_id = attr->attach_btf_id; 2067 if (attr->attach_prog_fd) { 2068 struct bpf_prog *tgt_prog; 2069 2070 tgt_prog = bpf_prog_get(attr->attach_prog_fd); 2071 if (IS_ERR(tgt_prog)) { 2072 err = PTR_ERR(tgt_prog); 2073 goto free_prog_nouncharge; 2074 } 2075 prog->aux->linked_prog = tgt_prog; 2076 } 2077 2078 prog->aux->offload_requested = !!attr->prog_ifindex; 2079 2080 err = security_bpf_prog_alloc(prog->aux); 2081 if (err) 2082 goto free_prog_nouncharge; 2083 2084 err = bpf_prog_charge_memlock(prog); 2085 if (err) 2086 goto free_prog_sec; 2087 2088 prog->len = attr->insn_cnt; 2089 2090 err = -EFAULT; 2091 if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 2092 bpf_prog_insn_size(prog)) != 0) 2093 goto free_prog; 2094 2095 prog->orig_prog = NULL; 2096 prog->jited = 0; 2097 2098 atomic64_set(&prog->aux->refcnt, 1); 2099 prog->gpl_compatible = is_gpl ? 1 : 0; 2100 2101 if (bpf_prog_is_dev_bound(prog->aux)) { 2102 err = bpf_prog_offload_init(prog, attr); 2103 if (err) 2104 goto free_prog; 2105 } 2106 2107 /* find program type: socket_filter vs tracing_filter */ 2108 err = find_prog_type(type, prog); 2109 if (err < 0) 2110 goto free_prog; 2111 2112 prog->aux->load_time = ktime_get_boottime_ns(); 2113 err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, 2114 sizeof(attr->prog_name)); 2115 if (err < 0) 2116 goto free_prog; 2117 2118 /* run eBPF verifier */ 2119 err = bpf_check(&prog, attr, uattr); 2120 if (err < 0) 2121 goto free_used_maps; 2122 2123 prog = bpf_prog_select_runtime(prog, &err); 2124 if (err < 0) 2125 goto free_used_maps; 2126 2127 err = bpf_prog_alloc_id(prog); 2128 if (err) 2129 goto free_used_maps; 2130 2131 /* Upon success of bpf_prog_alloc_id(), the BPF prog is 2132 * effectively publicly exposed. However, retrieving via 2133 * bpf_prog_get_fd_by_id() will take another reference, 2134 * therefore it cannot be gone underneath us. 2135 * 2136 * Only for the time /after/ successful bpf_prog_new_fd() 2137 * and before returning to userspace, we might just hold 2138 * one reference and any parallel close on that fd could 2139 * rip everything out. Hence, below notifications must 2140 * happen before bpf_prog_new_fd(). 2141 * 2142 * Also, any failure handling from this point onwards must 2143 * be using bpf_prog_put() given the program is exposed. 2144 */ 2145 bpf_prog_kallsyms_add(prog); 2146 perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); 2147 bpf_audit_prog(prog, BPF_AUDIT_LOAD); 2148 2149 err = bpf_prog_new_fd(prog); 2150 if (err < 0) 2151 bpf_prog_put(prog); 2152 return err; 2153 2154 free_used_maps: 2155 /* In case we have subprogs, we need to wait for a grace 2156 * period before we can tear down JIT memory since symbols 2157 * are already exposed under kallsyms. 2158 */ 2159 __bpf_prog_put_noref(prog, prog->aux->func_cnt); 2160 return err; 2161 free_prog: 2162 bpf_prog_uncharge_memlock(prog); 2163 free_prog_sec: 2164 security_bpf_prog_free(prog->aux); 2165 free_prog_nouncharge: 2166 bpf_prog_free(prog); 2167 return err; 2168 } 2169 2170 #define BPF_OBJ_LAST_FIELD file_flags 2171 2172 static int bpf_obj_pin(const union bpf_attr *attr) 2173 { 2174 if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0) 2175 return -EINVAL; 2176 2177 return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 2178 } 2179 2180 static int bpf_obj_get(const union bpf_attr *attr) 2181 { 2182 if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || 2183 attr->file_flags & ~BPF_OBJ_FLAG_MASK) 2184 return -EINVAL; 2185 2186 return bpf_obj_get_user(u64_to_user_ptr(attr->pathname), 2187 attr->file_flags); 2188 } 2189 2190 void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 2191 const struct bpf_link_ops *ops, struct bpf_prog *prog) 2192 { 2193 atomic64_set(&link->refcnt, 1); 2194 link->type = type; 2195 link->id = 0; 2196 link->ops = ops; 2197 link->prog = prog; 2198 } 2199 2200 static void bpf_link_free_id(int id) 2201 { 2202 if (!id) 2203 return; 2204 2205 spin_lock_bh(&link_idr_lock); 2206 idr_remove(&link_idr, id); 2207 spin_unlock_bh(&link_idr_lock); 2208 } 2209 2210 /* Clean up bpf_link and corresponding anon_inode file and FD. After 2211 * anon_inode is created, bpf_link can't be just kfree()'d due to deferred 2212 * anon_inode's release() call. This helper marksbpf_link as 2213 * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt 2214 * is not decremented, it's the responsibility of a calling code that failed 2215 * to complete bpf_link initialization. 2216 */ 2217 void bpf_link_cleanup(struct bpf_link_primer *primer) 2218 { 2219 primer->link->prog = NULL; 2220 bpf_link_free_id(primer->id); 2221 fput(primer->file); 2222 put_unused_fd(primer->fd); 2223 } 2224 2225 void bpf_link_inc(struct bpf_link *link) 2226 { 2227 atomic64_inc(&link->refcnt); 2228 } 2229 2230 /* bpf_link_free is guaranteed to be called from process context */ 2231 static void bpf_link_free(struct bpf_link *link) 2232 { 2233 bpf_link_free_id(link->id); 2234 if (link->prog) { 2235 /* detach BPF program, clean up used resources */ 2236 link->ops->release(link); 2237 bpf_prog_put(link->prog); 2238 } 2239 /* free bpf_link and its containing memory */ 2240 link->ops->dealloc(link); 2241 } 2242 2243 static void bpf_link_put_deferred(struct work_struct *work) 2244 { 2245 struct bpf_link *link = container_of(work, struct bpf_link, work); 2246 2247 bpf_link_free(link); 2248 } 2249 2250 /* bpf_link_put can be called from atomic context, but ensures that resources 2251 * are freed from process context 2252 */ 2253 void bpf_link_put(struct bpf_link *link) 2254 { 2255 if (!atomic64_dec_and_test(&link->refcnt)) 2256 return; 2257 2258 if (in_atomic()) { 2259 INIT_WORK(&link->work, bpf_link_put_deferred); 2260 schedule_work(&link->work); 2261 } else { 2262 bpf_link_free(link); 2263 } 2264 } 2265 2266 static int bpf_link_release(struct inode *inode, struct file *filp) 2267 { 2268 struct bpf_link *link = filp->private_data; 2269 2270 bpf_link_put(link); 2271 return 0; 2272 } 2273 2274 #ifdef CONFIG_PROC_FS 2275 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) 2276 #define BPF_MAP_TYPE(_id, _ops) 2277 #define BPF_LINK_TYPE(_id, _name) [_id] = #_name, 2278 static const char *bpf_link_type_strs[] = { 2279 [BPF_LINK_TYPE_UNSPEC] = "<invalid>", 2280 #include <linux/bpf_types.h> 2281 }; 2282 #undef BPF_PROG_TYPE 2283 #undef BPF_MAP_TYPE 2284 #undef BPF_LINK_TYPE 2285 2286 static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) 2287 { 2288 const struct bpf_link *link = filp->private_data; 2289 const struct bpf_prog *prog = link->prog; 2290 char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 2291 2292 bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 2293 seq_printf(m, 2294 "link_type:\t%s\n" 2295 "link_id:\t%u\n" 2296 "prog_tag:\t%s\n" 2297 "prog_id:\t%u\n", 2298 bpf_link_type_strs[link->type], 2299 link->id, 2300 prog_tag, 2301 prog->aux->id); 2302 if (link->ops->show_fdinfo) 2303 link->ops->show_fdinfo(link, m); 2304 } 2305 #endif 2306 2307 static const struct file_operations bpf_link_fops = { 2308 #ifdef CONFIG_PROC_FS 2309 .show_fdinfo = bpf_link_show_fdinfo, 2310 #endif 2311 .release = bpf_link_release, 2312 .read = bpf_dummy_read, 2313 .write = bpf_dummy_write, 2314 }; 2315 2316 static int bpf_link_alloc_id(struct bpf_link *link) 2317 { 2318 int id; 2319 2320 idr_preload(GFP_KERNEL); 2321 spin_lock_bh(&link_idr_lock); 2322 id = idr_alloc_cyclic(&link_idr, link, 1, INT_MAX, GFP_ATOMIC); 2323 spin_unlock_bh(&link_idr_lock); 2324 idr_preload_end(); 2325 2326 return id; 2327 } 2328 2329 /* Prepare bpf_link to be exposed to user-space by allocating anon_inode file, 2330 * reserving unused FD and allocating ID from link_idr. This is to be paired 2331 * with bpf_link_settle() to install FD and ID and expose bpf_link to 2332 * user-space, if bpf_link is successfully attached. If not, bpf_link and 2333 * pre-allocated resources are to be freed with bpf_cleanup() call. All the 2334 * transient state is passed around in struct bpf_link_primer. 2335 * This is preferred way to create and initialize bpf_link, especially when 2336 * there are complicated and expensive operations inbetween creating bpf_link 2337 * itself and attaching it to BPF hook. By using bpf_link_prime() and 2338 * bpf_link_settle() kernel code using bpf_link doesn't have to perform 2339 * expensive (and potentially failing) roll back operations in a rare case 2340 * that file, FD, or ID can't be allocated. 2341 */ 2342 int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer) 2343 { 2344 struct file *file; 2345 int fd, id; 2346 2347 fd = get_unused_fd_flags(O_CLOEXEC); 2348 if (fd < 0) 2349 return fd; 2350 2351 2352 id = bpf_link_alloc_id(link); 2353 if (id < 0) { 2354 put_unused_fd(fd); 2355 return id; 2356 } 2357 2358 file = anon_inode_getfile("bpf_link", &bpf_link_fops, link, O_CLOEXEC); 2359 if (IS_ERR(file)) { 2360 bpf_link_free_id(id); 2361 put_unused_fd(fd); 2362 return PTR_ERR(file); 2363 } 2364 2365 primer->link = link; 2366 primer->file = file; 2367 primer->fd = fd; 2368 primer->id = id; 2369 return 0; 2370 } 2371 2372 int bpf_link_settle(struct bpf_link_primer *primer) 2373 { 2374 /* make bpf_link fetchable by ID */ 2375 spin_lock_bh(&link_idr_lock); 2376 primer->link->id = primer->id; 2377 spin_unlock_bh(&link_idr_lock); 2378 /* make bpf_link fetchable by FD */ 2379 fd_install(primer->fd, primer->file); 2380 /* pass through installed FD */ 2381 return primer->fd; 2382 } 2383 2384 int bpf_link_new_fd(struct bpf_link *link) 2385 { 2386 return anon_inode_getfd("bpf-link", &bpf_link_fops, link, O_CLOEXEC); 2387 } 2388 2389 struct bpf_link *bpf_link_get_from_fd(u32 ufd) 2390 { 2391 struct fd f = fdget(ufd); 2392 struct bpf_link *link; 2393 2394 if (!f.file) 2395 return ERR_PTR(-EBADF); 2396 if (f.file->f_op != &bpf_link_fops) { 2397 fdput(f); 2398 return ERR_PTR(-EINVAL); 2399 } 2400 2401 link = f.file->private_data; 2402 bpf_link_inc(link); 2403 fdput(f); 2404 2405 return link; 2406 } 2407 2408 struct bpf_tracing_link { 2409 struct bpf_link link; 2410 enum bpf_attach_type attach_type; 2411 }; 2412 2413 static void bpf_tracing_link_release(struct bpf_link *link) 2414 { 2415 WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog)); 2416 } 2417 2418 static void bpf_tracing_link_dealloc(struct bpf_link *link) 2419 { 2420 struct bpf_tracing_link *tr_link = 2421 container_of(link, struct bpf_tracing_link, link); 2422 2423 kfree(tr_link); 2424 } 2425 2426 static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link, 2427 struct seq_file *seq) 2428 { 2429 struct bpf_tracing_link *tr_link = 2430 container_of(link, struct bpf_tracing_link, link); 2431 2432 seq_printf(seq, 2433 "attach_type:\t%d\n", 2434 tr_link->attach_type); 2435 } 2436 2437 static int bpf_tracing_link_fill_link_info(const struct bpf_link *link, 2438 struct bpf_link_info *info) 2439 { 2440 struct bpf_tracing_link *tr_link = 2441 container_of(link, struct bpf_tracing_link, link); 2442 2443 info->tracing.attach_type = tr_link->attach_type; 2444 2445 return 0; 2446 } 2447 2448 static const struct bpf_link_ops bpf_tracing_link_lops = { 2449 .release = bpf_tracing_link_release, 2450 .dealloc = bpf_tracing_link_dealloc, 2451 .show_fdinfo = bpf_tracing_link_show_fdinfo, 2452 .fill_link_info = bpf_tracing_link_fill_link_info, 2453 }; 2454 2455 static int bpf_tracing_prog_attach(struct bpf_prog *prog) 2456 { 2457 struct bpf_link_primer link_primer; 2458 struct bpf_tracing_link *link; 2459 int err; 2460 2461 switch (prog->type) { 2462 case BPF_PROG_TYPE_TRACING: 2463 if (prog->expected_attach_type != BPF_TRACE_FENTRY && 2464 prog->expected_attach_type != BPF_TRACE_FEXIT && 2465 prog->expected_attach_type != BPF_MODIFY_RETURN) { 2466 err = -EINVAL; 2467 goto out_put_prog; 2468 } 2469 break; 2470 case BPF_PROG_TYPE_EXT: 2471 if (prog->expected_attach_type != 0) { 2472 err = -EINVAL; 2473 goto out_put_prog; 2474 } 2475 break; 2476 case BPF_PROG_TYPE_LSM: 2477 if (prog->expected_attach_type != BPF_LSM_MAC) { 2478 err = -EINVAL; 2479 goto out_put_prog; 2480 } 2481 break; 2482 default: 2483 err = -EINVAL; 2484 goto out_put_prog; 2485 } 2486 2487 link = kzalloc(sizeof(*link), GFP_USER); 2488 if (!link) { 2489 err = -ENOMEM; 2490 goto out_put_prog; 2491 } 2492 bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING, 2493 &bpf_tracing_link_lops, prog); 2494 link->attach_type = prog->expected_attach_type; 2495 2496 err = bpf_link_prime(&link->link, &link_primer); 2497 if (err) { 2498 kfree(link); 2499 goto out_put_prog; 2500 } 2501 2502 err = bpf_trampoline_link_prog(prog); 2503 if (err) { 2504 bpf_link_cleanup(&link_primer); 2505 goto out_put_prog; 2506 } 2507 2508 return bpf_link_settle(&link_primer); 2509 out_put_prog: 2510 bpf_prog_put(prog); 2511 return err; 2512 } 2513 2514 struct bpf_raw_tp_link { 2515 struct bpf_link link; 2516 struct bpf_raw_event_map *btp; 2517 }; 2518 2519 static void bpf_raw_tp_link_release(struct bpf_link *link) 2520 { 2521 struct bpf_raw_tp_link *raw_tp = 2522 container_of(link, struct bpf_raw_tp_link, link); 2523 2524 bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog); 2525 bpf_put_raw_tracepoint(raw_tp->btp); 2526 } 2527 2528 static void bpf_raw_tp_link_dealloc(struct bpf_link *link) 2529 { 2530 struct bpf_raw_tp_link *raw_tp = 2531 container_of(link, struct bpf_raw_tp_link, link); 2532 2533 kfree(raw_tp); 2534 } 2535 2536 static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link, 2537 struct seq_file *seq) 2538 { 2539 struct bpf_raw_tp_link *raw_tp_link = 2540 container_of(link, struct bpf_raw_tp_link, link); 2541 2542 seq_printf(seq, 2543 "tp_name:\t%s\n", 2544 raw_tp_link->btp->tp->name); 2545 } 2546 2547 static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link, 2548 struct bpf_link_info *info) 2549 { 2550 struct bpf_raw_tp_link *raw_tp_link = 2551 container_of(link, struct bpf_raw_tp_link, link); 2552 char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name); 2553 const char *tp_name = raw_tp_link->btp->tp->name; 2554 u32 ulen = info->raw_tracepoint.tp_name_len; 2555 size_t tp_len = strlen(tp_name); 2556 2557 if (ulen && !ubuf) 2558 return -EINVAL; 2559 2560 info->raw_tracepoint.tp_name_len = tp_len + 1; 2561 2562 if (!ubuf) 2563 return 0; 2564 2565 if (ulen >= tp_len + 1) { 2566 if (copy_to_user(ubuf, tp_name, tp_len + 1)) 2567 return -EFAULT; 2568 } else { 2569 char zero = '\0'; 2570 2571 if (copy_to_user(ubuf, tp_name, ulen - 1)) 2572 return -EFAULT; 2573 if (put_user(zero, ubuf + ulen - 1)) 2574 return -EFAULT; 2575 return -ENOSPC; 2576 } 2577 2578 return 0; 2579 } 2580 2581 static const struct bpf_link_ops bpf_raw_tp_link_lops = { 2582 .release = bpf_raw_tp_link_release, 2583 .dealloc = bpf_raw_tp_link_dealloc, 2584 .show_fdinfo = bpf_raw_tp_link_show_fdinfo, 2585 .fill_link_info = bpf_raw_tp_link_fill_link_info, 2586 }; 2587 2588 #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd 2589 2590 static int bpf_raw_tracepoint_open(const union bpf_attr *attr) 2591 { 2592 struct bpf_link_primer link_primer; 2593 struct bpf_raw_tp_link *link; 2594 struct bpf_raw_event_map *btp; 2595 struct bpf_prog *prog; 2596 const char *tp_name; 2597 char buf[128]; 2598 int err; 2599 2600 if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN)) 2601 return -EINVAL; 2602 2603 prog = bpf_prog_get(attr->raw_tracepoint.prog_fd); 2604 if (IS_ERR(prog)) 2605 return PTR_ERR(prog); 2606 2607 switch (prog->type) { 2608 case BPF_PROG_TYPE_TRACING: 2609 case BPF_PROG_TYPE_EXT: 2610 case BPF_PROG_TYPE_LSM: 2611 if (attr->raw_tracepoint.name) { 2612 /* The attach point for this category of programs 2613 * should be specified via btf_id during program load. 2614 */ 2615 err = -EINVAL; 2616 goto out_put_prog; 2617 } 2618 if (prog->type == BPF_PROG_TYPE_TRACING && 2619 prog->expected_attach_type == BPF_TRACE_RAW_TP) { 2620 tp_name = prog->aux->attach_func_name; 2621 break; 2622 } 2623 return bpf_tracing_prog_attach(prog); 2624 case BPF_PROG_TYPE_RAW_TRACEPOINT: 2625 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 2626 if (strncpy_from_user(buf, 2627 u64_to_user_ptr(attr->raw_tracepoint.name), 2628 sizeof(buf) - 1) < 0) { 2629 err = -EFAULT; 2630 goto out_put_prog; 2631 } 2632 buf[sizeof(buf) - 1] = 0; 2633 tp_name = buf; 2634 break; 2635 default: 2636 err = -EINVAL; 2637 goto out_put_prog; 2638 } 2639 2640 btp = bpf_get_raw_tracepoint(tp_name); 2641 if (!btp) { 2642 err = -ENOENT; 2643 goto out_put_prog; 2644 } 2645 2646 link = kzalloc(sizeof(*link), GFP_USER); 2647 if (!link) { 2648 err = -ENOMEM; 2649 goto out_put_btp; 2650 } 2651 bpf_link_init(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT, 2652 &bpf_raw_tp_link_lops, prog); 2653 link->btp = btp; 2654 2655 err = bpf_link_prime(&link->link, &link_primer); 2656 if (err) { 2657 kfree(link); 2658 goto out_put_btp; 2659 } 2660 2661 err = bpf_probe_register(link->btp, prog); 2662 if (err) { 2663 bpf_link_cleanup(&link_primer); 2664 goto out_put_btp; 2665 } 2666 2667 return bpf_link_settle(&link_primer); 2668 2669 out_put_btp: 2670 bpf_put_raw_tracepoint(btp); 2671 out_put_prog: 2672 bpf_prog_put(prog); 2673 return err; 2674 } 2675 2676 static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, 2677 enum bpf_attach_type attach_type) 2678 { 2679 switch (prog->type) { 2680 case BPF_PROG_TYPE_CGROUP_SOCK: 2681 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 2682 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 2683 return attach_type == prog->expected_attach_type ? 0 : -EINVAL; 2684 case BPF_PROG_TYPE_CGROUP_SKB: 2685 return prog->enforce_expected_attach_type && 2686 prog->expected_attach_type != attach_type ? 2687 -EINVAL : 0; 2688 default: 2689 return 0; 2690 } 2691 } 2692 2693 static enum bpf_prog_type 2694 attach_type_to_prog_type(enum bpf_attach_type attach_type) 2695 { 2696 switch (attach_type) { 2697 case BPF_CGROUP_INET_INGRESS: 2698 case BPF_CGROUP_INET_EGRESS: 2699 return BPF_PROG_TYPE_CGROUP_SKB; 2700 break; 2701 case BPF_CGROUP_INET_SOCK_CREATE: 2702 case BPF_CGROUP_INET4_POST_BIND: 2703 case BPF_CGROUP_INET6_POST_BIND: 2704 return BPF_PROG_TYPE_CGROUP_SOCK; 2705 case BPF_CGROUP_INET4_BIND: 2706 case BPF_CGROUP_INET6_BIND: 2707 case BPF_CGROUP_INET4_CONNECT: 2708 case BPF_CGROUP_INET6_CONNECT: 2709 case BPF_CGROUP_UDP4_SENDMSG: 2710 case BPF_CGROUP_UDP6_SENDMSG: 2711 case BPF_CGROUP_UDP4_RECVMSG: 2712 case BPF_CGROUP_UDP6_RECVMSG: 2713 return BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 2714 case BPF_CGROUP_SOCK_OPS: 2715 return BPF_PROG_TYPE_SOCK_OPS; 2716 case BPF_CGROUP_DEVICE: 2717 return BPF_PROG_TYPE_CGROUP_DEVICE; 2718 case BPF_SK_MSG_VERDICT: 2719 return BPF_PROG_TYPE_SK_MSG; 2720 case BPF_SK_SKB_STREAM_PARSER: 2721 case BPF_SK_SKB_STREAM_VERDICT: 2722 return BPF_PROG_TYPE_SK_SKB; 2723 case BPF_LIRC_MODE2: 2724 return BPF_PROG_TYPE_LIRC_MODE2; 2725 case BPF_FLOW_DISSECTOR: 2726 return BPF_PROG_TYPE_FLOW_DISSECTOR; 2727 case BPF_CGROUP_SYSCTL: 2728 return BPF_PROG_TYPE_CGROUP_SYSCTL; 2729 case BPF_CGROUP_GETSOCKOPT: 2730 case BPF_CGROUP_SETSOCKOPT: 2731 return BPF_PROG_TYPE_CGROUP_SOCKOPT; 2732 case BPF_TRACE_ITER: 2733 return BPF_PROG_TYPE_TRACING; 2734 default: 2735 return BPF_PROG_TYPE_UNSPEC; 2736 } 2737 } 2738 2739 #define BPF_PROG_ATTACH_LAST_FIELD replace_bpf_fd 2740 2741 #define BPF_F_ATTACH_MASK \ 2742 (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI | BPF_F_REPLACE) 2743 2744 static int bpf_prog_attach(const union bpf_attr *attr) 2745 { 2746 enum bpf_prog_type ptype; 2747 struct bpf_prog *prog; 2748 int ret; 2749 2750 if (!capable(CAP_NET_ADMIN)) 2751 return -EPERM; 2752 2753 if (CHECK_ATTR(BPF_PROG_ATTACH)) 2754 return -EINVAL; 2755 2756 if (attr->attach_flags & ~BPF_F_ATTACH_MASK) 2757 return -EINVAL; 2758 2759 ptype = attach_type_to_prog_type(attr->attach_type); 2760 if (ptype == BPF_PROG_TYPE_UNSPEC) 2761 return -EINVAL; 2762 2763 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 2764 if (IS_ERR(prog)) 2765 return PTR_ERR(prog); 2766 2767 if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) { 2768 bpf_prog_put(prog); 2769 return -EINVAL; 2770 } 2771 2772 switch (ptype) { 2773 case BPF_PROG_TYPE_SK_SKB: 2774 case BPF_PROG_TYPE_SK_MSG: 2775 ret = sock_map_get_from_fd(attr, prog); 2776 break; 2777 case BPF_PROG_TYPE_LIRC_MODE2: 2778 ret = lirc_prog_attach(attr, prog); 2779 break; 2780 case BPF_PROG_TYPE_FLOW_DISSECTOR: 2781 ret = skb_flow_dissector_bpf_prog_attach(attr, prog); 2782 break; 2783 case BPF_PROG_TYPE_CGROUP_DEVICE: 2784 case BPF_PROG_TYPE_CGROUP_SKB: 2785 case BPF_PROG_TYPE_CGROUP_SOCK: 2786 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 2787 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 2788 case BPF_PROG_TYPE_CGROUP_SYSCTL: 2789 case BPF_PROG_TYPE_SOCK_OPS: 2790 ret = cgroup_bpf_prog_attach(attr, ptype, prog); 2791 break; 2792 default: 2793 ret = -EINVAL; 2794 } 2795 2796 if (ret) 2797 bpf_prog_put(prog); 2798 return ret; 2799 } 2800 2801 #define BPF_PROG_DETACH_LAST_FIELD attach_type 2802 2803 static int bpf_prog_detach(const union bpf_attr *attr) 2804 { 2805 enum bpf_prog_type ptype; 2806 2807 if (!capable(CAP_NET_ADMIN)) 2808 return -EPERM; 2809 2810 if (CHECK_ATTR(BPF_PROG_DETACH)) 2811 return -EINVAL; 2812 2813 ptype = attach_type_to_prog_type(attr->attach_type); 2814 2815 switch (ptype) { 2816 case BPF_PROG_TYPE_SK_MSG: 2817 case BPF_PROG_TYPE_SK_SKB: 2818 return sock_map_get_from_fd(attr, NULL); 2819 case BPF_PROG_TYPE_LIRC_MODE2: 2820 return lirc_prog_detach(attr); 2821 case BPF_PROG_TYPE_FLOW_DISSECTOR: 2822 return skb_flow_dissector_bpf_prog_detach(attr); 2823 case BPF_PROG_TYPE_CGROUP_DEVICE: 2824 case BPF_PROG_TYPE_CGROUP_SKB: 2825 case BPF_PROG_TYPE_CGROUP_SOCK: 2826 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 2827 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 2828 case BPF_PROG_TYPE_CGROUP_SYSCTL: 2829 case BPF_PROG_TYPE_SOCK_OPS: 2830 return cgroup_bpf_prog_detach(attr, ptype); 2831 default: 2832 return -EINVAL; 2833 } 2834 } 2835 2836 #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt 2837 2838 static int bpf_prog_query(const union bpf_attr *attr, 2839 union bpf_attr __user *uattr) 2840 { 2841 if (!capable(CAP_NET_ADMIN)) 2842 return -EPERM; 2843 if (CHECK_ATTR(BPF_PROG_QUERY)) 2844 return -EINVAL; 2845 if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE) 2846 return -EINVAL; 2847 2848 switch (attr->query.attach_type) { 2849 case BPF_CGROUP_INET_INGRESS: 2850 case BPF_CGROUP_INET_EGRESS: 2851 case BPF_CGROUP_INET_SOCK_CREATE: 2852 case BPF_CGROUP_INET4_BIND: 2853 case BPF_CGROUP_INET6_BIND: 2854 case BPF_CGROUP_INET4_POST_BIND: 2855 case BPF_CGROUP_INET6_POST_BIND: 2856 case BPF_CGROUP_INET4_CONNECT: 2857 case BPF_CGROUP_INET6_CONNECT: 2858 case BPF_CGROUP_UDP4_SENDMSG: 2859 case BPF_CGROUP_UDP6_SENDMSG: 2860 case BPF_CGROUP_UDP4_RECVMSG: 2861 case BPF_CGROUP_UDP6_RECVMSG: 2862 case BPF_CGROUP_SOCK_OPS: 2863 case BPF_CGROUP_DEVICE: 2864 case BPF_CGROUP_SYSCTL: 2865 case BPF_CGROUP_GETSOCKOPT: 2866 case BPF_CGROUP_SETSOCKOPT: 2867 return cgroup_bpf_prog_query(attr, uattr); 2868 case BPF_LIRC_MODE2: 2869 return lirc_prog_query(attr, uattr); 2870 case BPF_FLOW_DISSECTOR: 2871 return skb_flow_dissector_prog_query(attr, uattr); 2872 default: 2873 return -EINVAL; 2874 } 2875 } 2876 2877 #define BPF_PROG_TEST_RUN_LAST_FIELD test.ctx_out 2878 2879 static int bpf_prog_test_run(const union bpf_attr *attr, 2880 union bpf_attr __user *uattr) 2881 { 2882 struct bpf_prog *prog; 2883 int ret = -ENOTSUPP; 2884 2885 if (!capable(CAP_SYS_ADMIN)) 2886 return -EPERM; 2887 if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 2888 return -EINVAL; 2889 2890 if ((attr->test.ctx_size_in && !attr->test.ctx_in) || 2891 (!attr->test.ctx_size_in && attr->test.ctx_in)) 2892 return -EINVAL; 2893 2894 if ((attr->test.ctx_size_out && !attr->test.ctx_out) || 2895 (!attr->test.ctx_size_out && attr->test.ctx_out)) 2896 return -EINVAL; 2897 2898 prog = bpf_prog_get(attr->test.prog_fd); 2899 if (IS_ERR(prog)) 2900 return PTR_ERR(prog); 2901 2902 if (prog->aux->ops->test_run) 2903 ret = prog->aux->ops->test_run(prog, attr, uattr); 2904 2905 bpf_prog_put(prog); 2906 return ret; 2907 } 2908 2909 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 2910 2911 static int bpf_obj_get_next_id(const union bpf_attr *attr, 2912 union bpf_attr __user *uattr, 2913 struct idr *idr, 2914 spinlock_t *lock) 2915 { 2916 u32 next_id = attr->start_id; 2917 int err = 0; 2918 2919 if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 2920 return -EINVAL; 2921 2922 if (!capable(CAP_SYS_ADMIN)) 2923 return -EPERM; 2924 2925 next_id++; 2926 spin_lock_bh(lock); 2927 if (!idr_get_next(idr, &next_id)) 2928 err = -ENOENT; 2929 spin_unlock_bh(lock); 2930 2931 if (!err) 2932 err = put_user(next_id, &uattr->next_id); 2933 2934 return err; 2935 } 2936 2937 struct bpf_map *bpf_map_get_curr_or_next(u32 *id) 2938 { 2939 struct bpf_map *map; 2940 2941 spin_lock_bh(&map_idr_lock); 2942 again: 2943 map = idr_get_next(&map_idr, id); 2944 if (map) { 2945 map = __bpf_map_inc_not_zero(map, false); 2946 if (IS_ERR(map)) { 2947 (*id)++; 2948 goto again; 2949 } 2950 } 2951 spin_unlock_bh(&map_idr_lock); 2952 2953 return map; 2954 } 2955 2956 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 2957 2958 struct bpf_prog *bpf_prog_by_id(u32 id) 2959 { 2960 struct bpf_prog *prog; 2961 2962 if (!id) 2963 return ERR_PTR(-ENOENT); 2964 2965 spin_lock_bh(&prog_idr_lock); 2966 prog = idr_find(&prog_idr, id); 2967 if (prog) 2968 prog = bpf_prog_inc_not_zero(prog); 2969 else 2970 prog = ERR_PTR(-ENOENT); 2971 spin_unlock_bh(&prog_idr_lock); 2972 return prog; 2973 } 2974 2975 static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 2976 { 2977 struct bpf_prog *prog; 2978 u32 id = attr->prog_id; 2979 int fd; 2980 2981 if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 2982 return -EINVAL; 2983 2984 if (!capable(CAP_SYS_ADMIN)) 2985 return -EPERM; 2986 2987 prog = bpf_prog_by_id(id); 2988 if (IS_ERR(prog)) 2989 return PTR_ERR(prog); 2990 2991 fd = bpf_prog_new_fd(prog); 2992 if (fd < 0) 2993 bpf_prog_put(prog); 2994 2995 return fd; 2996 } 2997 2998 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags 2999 3000 static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 3001 { 3002 struct bpf_map *map; 3003 u32 id = attr->map_id; 3004 int f_flags; 3005 int fd; 3006 3007 if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || 3008 attr->open_flags & ~BPF_OBJ_FLAG_MASK) 3009 return -EINVAL; 3010 3011 if (!capable(CAP_SYS_ADMIN)) 3012 return -EPERM; 3013 3014 f_flags = bpf_get_file_flag(attr->open_flags); 3015 if (f_flags < 0) 3016 return f_flags; 3017 3018 spin_lock_bh(&map_idr_lock); 3019 map = idr_find(&map_idr, id); 3020 if (map) 3021 map = __bpf_map_inc_not_zero(map, true); 3022 else 3023 map = ERR_PTR(-ENOENT); 3024 spin_unlock_bh(&map_idr_lock); 3025 3026 if (IS_ERR(map)) 3027 return PTR_ERR(map); 3028 3029 fd = bpf_map_new_fd(map, f_flags); 3030 if (fd < 0) 3031 bpf_map_put_with_uref(map); 3032 3033 return fd; 3034 } 3035 3036 static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, 3037 unsigned long addr, u32 *off, 3038 u32 *type) 3039 { 3040 const struct bpf_map *map; 3041 int i; 3042 3043 for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { 3044 map = prog->aux->used_maps[i]; 3045 if (map == (void *)addr) { 3046 *type = BPF_PSEUDO_MAP_FD; 3047 return map; 3048 } 3049 if (!map->ops->map_direct_value_meta) 3050 continue; 3051 if (!map->ops->map_direct_value_meta(map, addr, off)) { 3052 *type = BPF_PSEUDO_MAP_VALUE; 3053 return map; 3054 } 3055 } 3056 3057 return NULL; 3058 } 3059 3060 static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog) 3061 { 3062 const struct bpf_map *map; 3063 struct bpf_insn *insns; 3064 u32 off, type; 3065 u64 imm; 3066 int i; 3067 3068 insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), 3069 GFP_USER); 3070 if (!insns) 3071 return insns; 3072 3073 for (i = 0; i < prog->len; i++) { 3074 if (insns[i].code == (BPF_JMP | BPF_TAIL_CALL)) { 3075 insns[i].code = BPF_JMP | BPF_CALL; 3076 insns[i].imm = BPF_FUNC_tail_call; 3077 /* fall-through */ 3078 } 3079 if (insns[i].code == (BPF_JMP | BPF_CALL) || 3080 insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) { 3081 if (insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) 3082 insns[i].code = BPF_JMP | BPF_CALL; 3083 if (!bpf_dump_raw_ok()) 3084 insns[i].imm = 0; 3085 continue; 3086 } 3087 3088 if (insns[i].code != (BPF_LD | BPF_IMM | BPF_DW)) 3089 continue; 3090 3091 imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; 3092 map = bpf_map_from_imm(prog, imm, &off, &type); 3093 if (map) { 3094 insns[i].src_reg = type; 3095 insns[i].imm = map->id; 3096 insns[i + 1].imm = off; 3097 continue; 3098 } 3099 } 3100 3101 return insns; 3102 } 3103 3104 static int set_info_rec_size(struct bpf_prog_info *info) 3105 { 3106 /* 3107 * Ensure info.*_rec_size is the same as kernel expected size 3108 * 3109 * or 3110 * 3111 * Only allow zero *_rec_size if both _rec_size and _cnt are 3112 * zero. In this case, the kernel will set the expected 3113 * _rec_size back to the info. 3114 */ 3115 3116 if ((info->nr_func_info || info->func_info_rec_size) && 3117 info->func_info_rec_size != sizeof(struct bpf_func_info)) 3118 return -EINVAL; 3119 3120 if ((info->nr_line_info || info->line_info_rec_size) && 3121 info->line_info_rec_size != sizeof(struct bpf_line_info)) 3122 return -EINVAL; 3123 3124 if ((info->nr_jited_line_info || info->jited_line_info_rec_size) && 3125 info->jited_line_info_rec_size != sizeof(__u64)) 3126 return -EINVAL; 3127 3128 info->func_info_rec_size = sizeof(struct bpf_func_info); 3129 info->line_info_rec_size = sizeof(struct bpf_line_info); 3130 info->jited_line_info_rec_size = sizeof(__u64); 3131 3132 return 0; 3133 } 3134 3135 static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, 3136 const union bpf_attr *attr, 3137 union bpf_attr __user *uattr) 3138 { 3139 struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 3140 struct bpf_prog_info info; 3141 u32 info_len = attr->info.info_len; 3142 struct bpf_prog_stats stats; 3143 char __user *uinsns; 3144 u32 ulen; 3145 int err; 3146 3147 err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 3148 if (err) 3149 return err; 3150 info_len = min_t(u32, sizeof(info), info_len); 3151 3152 memset(&info, 0, sizeof(info)); 3153 if (copy_from_user(&info, uinfo, info_len)) 3154 return -EFAULT; 3155 3156 info.type = prog->type; 3157 info.id = prog->aux->id; 3158 info.load_time = prog->aux->load_time; 3159 info.created_by_uid = from_kuid_munged(current_user_ns(), 3160 prog->aux->user->uid); 3161 info.gpl_compatible = prog->gpl_compatible; 3162 3163 memcpy(info.tag, prog->tag, sizeof(prog->tag)); 3164 memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); 3165 3166 ulen = info.nr_map_ids; 3167 info.nr_map_ids = prog->aux->used_map_cnt; 3168 ulen = min_t(u32, info.nr_map_ids, ulen); 3169 if (ulen) { 3170 u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); 3171 u32 i; 3172 3173 for (i = 0; i < ulen; i++) 3174 if (put_user(prog->aux->used_maps[i]->id, 3175 &user_map_ids[i])) 3176 return -EFAULT; 3177 } 3178 3179 err = set_info_rec_size(&info); 3180 if (err) 3181 return err; 3182 3183 bpf_prog_get_stats(prog, &stats); 3184 info.run_time_ns = stats.nsecs; 3185 info.run_cnt = stats.cnt; 3186 3187 if (!capable(CAP_SYS_ADMIN)) { 3188 info.jited_prog_len = 0; 3189 info.xlated_prog_len = 0; 3190 info.nr_jited_ksyms = 0; 3191 info.nr_jited_func_lens = 0; 3192 info.nr_func_info = 0; 3193 info.nr_line_info = 0; 3194 info.nr_jited_line_info = 0; 3195 goto done; 3196 } 3197 3198 ulen = info.xlated_prog_len; 3199 info.xlated_prog_len = bpf_prog_insn_size(prog); 3200 if (info.xlated_prog_len && ulen) { 3201 struct bpf_insn *insns_sanitized; 3202 bool fault; 3203 3204 if (prog->blinded && !bpf_dump_raw_ok()) { 3205 info.xlated_prog_insns = 0; 3206 goto done; 3207 } 3208 insns_sanitized = bpf_insn_prepare_dump(prog); 3209 if (!insns_sanitized) 3210 return -ENOMEM; 3211 uinsns = u64_to_user_ptr(info.xlated_prog_insns); 3212 ulen = min_t(u32, info.xlated_prog_len, ulen); 3213 fault = copy_to_user(uinsns, insns_sanitized, ulen); 3214 kfree(insns_sanitized); 3215 if (fault) 3216 return -EFAULT; 3217 } 3218 3219 if (bpf_prog_is_dev_bound(prog->aux)) { 3220 err = bpf_prog_offload_info_fill(&info, prog); 3221 if (err) 3222 return err; 3223 goto done; 3224 } 3225 3226 /* NOTE: the following code is supposed to be skipped for offload. 3227 * bpf_prog_offload_info_fill() is the place to fill similar fields 3228 * for offload. 3229 */ 3230 ulen = info.jited_prog_len; 3231 if (prog->aux->func_cnt) { 3232 u32 i; 3233 3234 info.jited_prog_len = 0; 3235 for (i = 0; i < prog->aux->func_cnt; i++) 3236 info.jited_prog_len += prog->aux->func[i]->jited_len; 3237 } else { 3238 info.jited_prog_len = prog->jited_len; 3239 } 3240 3241 if (info.jited_prog_len && ulen) { 3242 if (bpf_dump_raw_ok()) { 3243 uinsns = u64_to_user_ptr(info.jited_prog_insns); 3244 ulen = min_t(u32, info.jited_prog_len, ulen); 3245 3246 /* for multi-function programs, copy the JITed 3247 * instructions for all the functions 3248 */ 3249 if (prog->aux->func_cnt) { 3250 u32 len, free, i; 3251 u8 *img; 3252 3253 free = ulen; 3254 for (i = 0; i < prog->aux->func_cnt; i++) { 3255 len = prog->aux->func[i]->jited_len; 3256 len = min_t(u32, len, free); 3257 img = (u8 *) prog->aux->func[i]->bpf_func; 3258 if (copy_to_user(uinsns, img, len)) 3259 return -EFAULT; 3260 uinsns += len; 3261 free -= len; 3262 if (!free) 3263 break; 3264 } 3265 } else { 3266 if (copy_to_user(uinsns, prog->bpf_func, ulen)) 3267 return -EFAULT; 3268 } 3269 } else { 3270 info.jited_prog_insns = 0; 3271 } 3272 } 3273 3274 ulen = info.nr_jited_ksyms; 3275 info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; 3276 if (ulen) { 3277 if (bpf_dump_raw_ok()) { 3278 unsigned long ksym_addr; 3279 u64 __user *user_ksyms; 3280 u32 i; 3281 3282 /* copy the address of the kernel symbol 3283 * corresponding to each function 3284 */ 3285 ulen = min_t(u32, info.nr_jited_ksyms, ulen); 3286 user_ksyms = u64_to_user_ptr(info.jited_ksyms); 3287 if (prog->aux->func_cnt) { 3288 for (i = 0; i < ulen; i++) { 3289 ksym_addr = (unsigned long) 3290 prog->aux->func[i]->bpf_func; 3291 if (put_user((u64) ksym_addr, 3292 &user_ksyms[i])) 3293 return -EFAULT; 3294 } 3295 } else { 3296 ksym_addr = (unsigned long) prog->bpf_func; 3297 if (put_user((u64) ksym_addr, &user_ksyms[0])) 3298 return -EFAULT; 3299 } 3300 } else { 3301 info.jited_ksyms = 0; 3302 } 3303 } 3304 3305 ulen = info.nr_jited_func_lens; 3306 info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; 3307 if (ulen) { 3308 if (bpf_dump_raw_ok()) { 3309 u32 __user *user_lens; 3310 u32 func_len, i; 3311 3312 /* copy the JITed image lengths for each function */ 3313 ulen = min_t(u32, info.nr_jited_func_lens, ulen); 3314 user_lens = u64_to_user_ptr(info.jited_func_lens); 3315 if (prog->aux->func_cnt) { 3316 for (i = 0; i < ulen; i++) { 3317 func_len = 3318 prog->aux->func[i]->jited_len; 3319 if (put_user(func_len, &user_lens[i])) 3320 return -EFAULT; 3321 } 3322 } else { 3323 func_len = prog->jited_len; 3324 if (put_user(func_len, &user_lens[0])) 3325 return -EFAULT; 3326 } 3327 } else { 3328 info.jited_func_lens = 0; 3329 } 3330 } 3331 3332 if (prog->aux->btf) 3333 info.btf_id = btf_id(prog->aux->btf); 3334 3335 ulen = info.nr_func_info; 3336 info.nr_func_info = prog->aux->func_info_cnt; 3337 if (info.nr_func_info && ulen) { 3338 char __user *user_finfo; 3339 3340 user_finfo = u64_to_user_ptr(info.func_info); 3341 ulen = min_t(u32, info.nr_func_info, ulen); 3342 if (copy_to_user(user_finfo, prog->aux->func_info, 3343 info.func_info_rec_size * ulen)) 3344 return -EFAULT; 3345 } 3346 3347 ulen = info.nr_line_info; 3348 info.nr_line_info = prog->aux->nr_linfo; 3349 if (info.nr_line_info && ulen) { 3350 __u8 __user *user_linfo; 3351 3352 user_linfo = u64_to_user_ptr(info.line_info); 3353 ulen = min_t(u32, info.nr_line_info, ulen); 3354 if (copy_to_user(user_linfo, prog->aux->linfo, 3355 info.line_info_rec_size * ulen)) 3356 return -EFAULT; 3357 } 3358 3359 ulen = info.nr_jited_line_info; 3360 if (prog->aux->jited_linfo) 3361 info.nr_jited_line_info = prog->aux->nr_linfo; 3362 else 3363 info.nr_jited_line_info = 0; 3364 if (info.nr_jited_line_info && ulen) { 3365 if (bpf_dump_raw_ok()) { 3366 __u64 __user *user_linfo; 3367 u32 i; 3368 3369 user_linfo = u64_to_user_ptr(info.jited_line_info); 3370 ulen = min_t(u32, info.nr_jited_line_info, ulen); 3371 for (i = 0; i < ulen; i++) { 3372 if (put_user((__u64)(long)prog->aux->jited_linfo[i], 3373 &user_linfo[i])) 3374 return -EFAULT; 3375 } 3376 } else { 3377 info.jited_line_info = 0; 3378 } 3379 } 3380 3381 ulen = info.nr_prog_tags; 3382 info.nr_prog_tags = prog->aux->func_cnt ? : 1; 3383 if (ulen) { 3384 __u8 __user (*user_prog_tags)[BPF_TAG_SIZE]; 3385 u32 i; 3386 3387 user_prog_tags = u64_to_user_ptr(info.prog_tags); 3388 ulen = min_t(u32, info.nr_prog_tags, ulen); 3389 if (prog->aux->func_cnt) { 3390 for (i = 0; i < ulen; i++) { 3391 if (copy_to_user(user_prog_tags[i], 3392 prog->aux->func[i]->tag, 3393 BPF_TAG_SIZE)) 3394 return -EFAULT; 3395 } 3396 } else { 3397 if (copy_to_user(user_prog_tags[0], 3398 prog->tag, BPF_TAG_SIZE)) 3399 return -EFAULT; 3400 } 3401 } 3402 3403 done: 3404 if (copy_to_user(uinfo, &info, info_len) || 3405 put_user(info_len, &uattr->info.info_len)) 3406 return -EFAULT; 3407 3408 return 0; 3409 } 3410 3411 static int bpf_map_get_info_by_fd(struct bpf_map *map, 3412 const union bpf_attr *attr, 3413 union bpf_attr __user *uattr) 3414 { 3415 struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 3416 struct bpf_map_info info; 3417 u32 info_len = attr->info.info_len; 3418 int err; 3419 3420 err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 3421 if (err) 3422 return err; 3423 info_len = min_t(u32, sizeof(info), info_len); 3424 3425 memset(&info, 0, sizeof(info)); 3426 info.type = map->map_type; 3427 info.id = map->id; 3428 info.key_size = map->key_size; 3429 info.value_size = map->value_size; 3430 info.max_entries = map->max_entries; 3431 info.map_flags = map->map_flags; 3432 memcpy(info.name, map->name, sizeof(map->name)); 3433 3434 if (map->btf) { 3435 info.btf_id = btf_id(map->btf); 3436 info.btf_key_type_id = map->btf_key_type_id; 3437 info.btf_value_type_id = map->btf_value_type_id; 3438 } 3439 info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; 3440 3441 if (bpf_map_is_dev_bound(map)) { 3442 err = bpf_map_offload_info_fill(&info, map); 3443 if (err) 3444 return err; 3445 } 3446 3447 if (copy_to_user(uinfo, &info, info_len) || 3448 put_user(info_len, &uattr->info.info_len)) 3449 return -EFAULT; 3450 3451 return 0; 3452 } 3453 3454 static int bpf_btf_get_info_by_fd(struct btf *btf, 3455 const union bpf_attr *attr, 3456 union bpf_attr __user *uattr) 3457 { 3458 struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info); 3459 u32 info_len = attr->info.info_len; 3460 int err; 3461 3462 err = bpf_check_uarg_tail_zero(uinfo, sizeof(*uinfo), info_len); 3463 if (err) 3464 return err; 3465 3466 return btf_get_info_by_fd(btf, attr, uattr); 3467 } 3468 3469 static int bpf_link_get_info_by_fd(struct bpf_link *link, 3470 const union bpf_attr *attr, 3471 union bpf_attr __user *uattr) 3472 { 3473 struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info); 3474 struct bpf_link_info info; 3475 u32 info_len = attr->info.info_len; 3476 int err; 3477 3478 err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 3479 if (err) 3480 return err; 3481 info_len = min_t(u32, sizeof(info), info_len); 3482 3483 memset(&info, 0, sizeof(info)); 3484 if (copy_from_user(&info, uinfo, info_len)) 3485 return -EFAULT; 3486 3487 info.type = link->type; 3488 info.id = link->id; 3489 info.prog_id = link->prog->aux->id; 3490 3491 if (link->ops->fill_link_info) { 3492 err = link->ops->fill_link_info(link, &info); 3493 if (err) 3494 return err; 3495 } 3496 3497 if (copy_to_user(uinfo, &info, info_len) || 3498 put_user(info_len, &uattr->info.info_len)) 3499 return -EFAULT; 3500 3501 return 0; 3502 } 3503 3504 3505 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 3506 3507 static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 3508 union bpf_attr __user *uattr) 3509 { 3510 int ufd = attr->info.bpf_fd; 3511 struct fd f; 3512 int err; 3513 3514 if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 3515 return -EINVAL; 3516 3517 f = fdget(ufd); 3518 if (!f.file) 3519 return -EBADFD; 3520 3521 if (f.file->f_op == &bpf_prog_fops) 3522 err = bpf_prog_get_info_by_fd(f.file->private_data, attr, 3523 uattr); 3524 else if (f.file->f_op == &bpf_map_fops) 3525 err = bpf_map_get_info_by_fd(f.file->private_data, attr, 3526 uattr); 3527 else if (f.file->f_op == &btf_fops) 3528 err = bpf_btf_get_info_by_fd(f.file->private_data, attr, uattr); 3529 else if (f.file->f_op == &bpf_link_fops) 3530 err = bpf_link_get_info_by_fd(f.file->private_data, 3531 attr, uattr); 3532 else 3533 err = -EINVAL; 3534 3535 fdput(f); 3536 return err; 3537 } 3538 3539 #define BPF_BTF_LOAD_LAST_FIELD btf_log_level 3540 3541 static int bpf_btf_load(const union bpf_attr *attr) 3542 { 3543 if (CHECK_ATTR(BPF_BTF_LOAD)) 3544 return -EINVAL; 3545 3546 if (!capable(CAP_SYS_ADMIN)) 3547 return -EPERM; 3548 3549 return btf_new_fd(attr); 3550 } 3551 3552 #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id 3553 3554 static int bpf_btf_get_fd_by_id(const union bpf_attr *attr) 3555 { 3556 if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID)) 3557 return -EINVAL; 3558 3559 if (!capable(CAP_SYS_ADMIN)) 3560 return -EPERM; 3561 3562 return btf_get_fd_by_id(attr->btf_id); 3563 } 3564 3565 static int bpf_task_fd_query_copy(const union bpf_attr *attr, 3566 union bpf_attr __user *uattr, 3567 u32 prog_id, u32 fd_type, 3568 const char *buf, u64 probe_offset, 3569 u64 probe_addr) 3570 { 3571 char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf); 3572 u32 len = buf ? strlen(buf) : 0, input_len; 3573 int err = 0; 3574 3575 if (put_user(len, &uattr->task_fd_query.buf_len)) 3576 return -EFAULT; 3577 input_len = attr->task_fd_query.buf_len; 3578 if (input_len && ubuf) { 3579 if (!len) { 3580 /* nothing to copy, just make ubuf NULL terminated */ 3581 char zero = '\0'; 3582 3583 if (put_user(zero, ubuf)) 3584 return -EFAULT; 3585 } else if (input_len >= len + 1) { 3586 /* ubuf can hold the string with NULL terminator */ 3587 if (copy_to_user(ubuf, buf, len + 1)) 3588 return -EFAULT; 3589 } else { 3590 /* ubuf cannot hold the string with NULL terminator, 3591 * do a partial copy with NULL terminator. 3592 */ 3593 char zero = '\0'; 3594 3595 err = -ENOSPC; 3596 if (copy_to_user(ubuf, buf, input_len - 1)) 3597 return -EFAULT; 3598 if (put_user(zero, ubuf + input_len - 1)) 3599 return -EFAULT; 3600 } 3601 } 3602 3603 if (put_user(prog_id, &uattr->task_fd_query.prog_id) || 3604 put_user(fd_type, &uattr->task_fd_query.fd_type) || 3605 put_user(probe_offset, &uattr->task_fd_query.probe_offset) || 3606 put_user(probe_addr, &uattr->task_fd_query.probe_addr)) 3607 return -EFAULT; 3608 3609 return err; 3610 } 3611 3612 #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr 3613 3614 static int bpf_task_fd_query(const union bpf_attr *attr, 3615 union bpf_attr __user *uattr) 3616 { 3617 pid_t pid = attr->task_fd_query.pid; 3618 u32 fd = attr->task_fd_query.fd; 3619 const struct perf_event *event; 3620 struct files_struct *files; 3621 struct task_struct *task; 3622 struct file *file; 3623 int err; 3624 3625 if (CHECK_ATTR(BPF_TASK_FD_QUERY)) 3626 return -EINVAL; 3627 3628 if (!capable(CAP_SYS_ADMIN)) 3629 return -EPERM; 3630 3631 if (attr->task_fd_query.flags != 0) 3632 return -EINVAL; 3633 3634 task = get_pid_task(find_vpid(pid), PIDTYPE_PID); 3635 if (!task) 3636 return -ENOENT; 3637 3638 files = get_files_struct(task); 3639 put_task_struct(task); 3640 if (!files) 3641 return -ENOENT; 3642 3643 err = 0; 3644 spin_lock(&files->file_lock); 3645 file = fcheck_files(files, fd); 3646 if (!file) 3647 err = -EBADF; 3648 else 3649 get_file(file); 3650 spin_unlock(&files->file_lock); 3651 put_files_struct(files); 3652 3653 if (err) 3654 goto out; 3655 3656 if (file->f_op == &bpf_link_fops) { 3657 struct bpf_link *link = file->private_data; 3658 3659 if (link->ops == &bpf_raw_tp_link_lops) { 3660 struct bpf_raw_tp_link *raw_tp = 3661 container_of(link, struct bpf_raw_tp_link, link); 3662 struct bpf_raw_event_map *btp = raw_tp->btp; 3663 3664 err = bpf_task_fd_query_copy(attr, uattr, 3665 raw_tp->link.prog->aux->id, 3666 BPF_FD_TYPE_RAW_TRACEPOINT, 3667 btp->tp->name, 0, 0); 3668 goto put_file; 3669 } 3670 goto out_not_supp; 3671 } 3672 3673 event = perf_get_event(file); 3674 if (!IS_ERR(event)) { 3675 u64 probe_offset, probe_addr; 3676 u32 prog_id, fd_type; 3677 const char *buf; 3678 3679 err = bpf_get_perf_event_info(event, &prog_id, &fd_type, 3680 &buf, &probe_offset, 3681 &probe_addr); 3682 if (!err) 3683 err = bpf_task_fd_query_copy(attr, uattr, prog_id, 3684 fd_type, buf, 3685 probe_offset, 3686 probe_addr); 3687 goto put_file; 3688 } 3689 3690 out_not_supp: 3691 err = -ENOTSUPP; 3692 put_file: 3693 fput(file); 3694 out: 3695 return err; 3696 } 3697 3698 #define BPF_MAP_BATCH_LAST_FIELD batch.flags 3699 3700 #define BPF_DO_BATCH(fn) \ 3701 do { \ 3702 if (!fn) { \ 3703 err = -ENOTSUPP; \ 3704 goto err_put; \ 3705 } \ 3706 err = fn(map, attr, uattr); \ 3707 } while (0) 3708 3709 static int bpf_map_do_batch(const union bpf_attr *attr, 3710 union bpf_attr __user *uattr, 3711 int cmd) 3712 { 3713 struct bpf_map *map; 3714 int err, ufd; 3715 struct fd f; 3716 3717 if (CHECK_ATTR(BPF_MAP_BATCH)) 3718 return -EINVAL; 3719 3720 ufd = attr->batch.map_fd; 3721 f = fdget(ufd); 3722 map = __bpf_map_get(f); 3723 if (IS_ERR(map)) 3724 return PTR_ERR(map); 3725 3726 if ((cmd == BPF_MAP_LOOKUP_BATCH || 3727 cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) && 3728 !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 3729 err = -EPERM; 3730 goto err_put; 3731 } 3732 3733 if (cmd != BPF_MAP_LOOKUP_BATCH && 3734 !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 3735 err = -EPERM; 3736 goto err_put; 3737 } 3738 3739 if (cmd == BPF_MAP_LOOKUP_BATCH) 3740 BPF_DO_BATCH(map->ops->map_lookup_batch); 3741 else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) 3742 BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch); 3743 else if (cmd == BPF_MAP_UPDATE_BATCH) 3744 BPF_DO_BATCH(map->ops->map_update_batch); 3745 else 3746 BPF_DO_BATCH(map->ops->map_delete_batch); 3747 3748 err_put: 3749 fdput(f); 3750 return err; 3751 } 3752 3753 static int tracing_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 3754 { 3755 if (attr->link_create.attach_type == BPF_TRACE_ITER && 3756 prog->expected_attach_type == BPF_TRACE_ITER) 3757 return bpf_iter_link_attach(attr, prog); 3758 3759 return -EINVAL; 3760 } 3761 3762 #define BPF_LINK_CREATE_LAST_FIELD link_create.flags 3763 static int link_create(union bpf_attr *attr) 3764 { 3765 enum bpf_prog_type ptype; 3766 struct bpf_prog *prog; 3767 int ret; 3768 3769 if (!capable(CAP_NET_ADMIN)) 3770 return -EPERM; 3771 3772 if (CHECK_ATTR(BPF_LINK_CREATE)) 3773 return -EINVAL; 3774 3775 ptype = attach_type_to_prog_type(attr->link_create.attach_type); 3776 if (ptype == BPF_PROG_TYPE_UNSPEC) 3777 return -EINVAL; 3778 3779 prog = bpf_prog_get_type(attr->link_create.prog_fd, ptype); 3780 if (IS_ERR(prog)) 3781 return PTR_ERR(prog); 3782 3783 ret = bpf_prog_attach_check_attach_type(prog, 3784 attr->link_create.attach_type); 3785 if (ret) 3786 goto err_out; 3787 3788 switch (ptype) { 3789 case BPF_PROG_TYPE_CGROUP_SKB: 3790 case BPF_PROG_TYPE_CGROUP_SOCK: 3791 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 3792 case BPF_PROG_TYPE_SOCK_OPS: 3793 case BPF_PROG_TYPE_CGROUP_DEVICE: 3794 case BPF_PROG_TYPE_CGROUP_SYSCTL: 3795 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 3796 ret = cgroup_bpf_link_attach(attr, prog); 3797 break; 3798 case BPF_PROG_TYPE_TRACING: 3799 ret = tracing_bpf_link_attach(attr, prog); 3800 break; 3801 default: 3802 ret = -EINVAL; 3803 } 3804 3805 err_out: 3806 if (ret < 0) 3807 bpf_prog_put(prog); 3808 return ret; 3809 } 3810 3811 #define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd 3812 3813 static int link_update(union bpf_attr *attr) 3814 { 3815 struct bpf_prog *old_prog = NULL, *new_prog; 3816 struct bpf_link *link; 3817 u32 flags; 3818 int ret; 3819 3820 if (!capable(CAP_NET_ADMIN)) 3821 return -EPERM; 3822 3823 if (CHECK_ATTR(BPF_LINK_UPDATE)) 3824 return -EINVAL; 3825 3826 flags = attr->link_update.flags; 3827 if (flags & ~BPF_F_REPLACE) 3828 return -EINVAL; 3829 3830 link = bpf_link_get_from_fd(attr->link_update.link_fd); 3831 if (IS_ERR(link)) 3832 return PTR_ERR(link); 3833 3834 new_prog = bpf_prog_get(attr->link_update.new_prog_fd); 3835 if (IS_ERR(new_prog)) { 3836 ret = PTR_ERR(new_prog); 3837 goto out_put_link; 3838 } 3839 3840 if (flags & BPF_F_REPLACE) { 3841 old_prog = bpf_prog_get(attr->link_update.old_prog_fd); 3842 if (IS_ERR(old_prog)) { 3843 ret = PTR_ERR(old_prog); 3844 old_prog = NULL; 3845 goto out_put_progs; 3846 } 3847 } else if (attr->link_update.old_prog_fd) { 3848 ret = -EINVAL; 3849 goto out_put_progs; 3850 } 3851 3852 if (link->ops->update_prog) 3853 ret = link->ops->update_prog(link, new_prog, old_prog); 3854 else 3855 ret = EINVAL; 3856 3857 out_put_progs: 3858 if (old_prog) 3859 bpf_prog_put(old_prog); 3860 if (ret) 3861 bpf_prog_put(new_prog); 3862 out_put_link: 3863 bpf_link_put(link); 3864 return ret; 3865 } 3866 3867 static int bpf_link_inc_not_zero(struct bpf_link *link) 3868 { 3869 return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? 0 : -ENOENT; 3870 } 3871 3872 #define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id 3873 3874 static int bpf_link_get_fd_by_id(const union bpf_attr *attr) 3875 { 3876 struct bpf_link *link; 3877 u32 id = attr->link_id; 3878 int fd, err; 3879 3880 if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID)) 3881 return -EINVAL; 3882 3883 if (!capable(CAP_SYS_ADMIN)) 3884 return -EPERM; 3885 3886 spin_lock_bh(&link_idr_lock); 3887 link = idr_find(&link_idr, id); 3888 /* before link is "settled", ID is 0, pretend it doesn't exist yet */ 3889 if (link) { 3890 if (link->id) 3891 err = bpf_link_inc_not_zero(link); 3892 else 3893 err = -EAGAIN; 3894 } else { 3895 err = -ENOENT; 3896 } 3897 spin_unlock_bh(&link_idr_lock); 3898 3899 if (err) 3900 return err; 3901 3902 fd = bpf_link_new_fd(link); 3903 if (fd < 0) 3904 bpf_link_put(link); 3905 3906 return fd; 3907 } 3908 3909 DEFINE_MUTEX(bpf_stats_enabled_mutex); 3910 3911 static int bpf_stats_release(struct inode *inode, struct file *file) 3912 { 3913 mutex_lock(&bpf_stats_enabled_mutex); 3914 static_key_slow_dec(&bpf_stats_enabled_key.key); 3915 mutex_unlock(&bpf_stats_enabled_mutex); 3916 return 0; 3917 } 3918 3919 static const struct file_operations bpf_stats_fops = { 3920 .release = bpf_stats_release, 3921 }; 3922 3923 static int bpf_enable_runtime_stats(void) 3924 { 3925 int fd; 3926 3927 mutex_lock(&bpf_stats_enabled_mutex); 3928 3929 /* Set a very high limit to avoid overflow */ 3930 if (static_key_count(&bpf_stats_enabled_key.key) > INT_MAX / 2) { 3931 mutex_unlock(&bpf_stats_enabled_mutex); 3932 return -EBUSY; 3933 } 3934 3935 fd = anon_inode_getfd("bpf-stats", &bpf_stats_fops, NULL, O_CLOEXEC); 3936 if (fd >= 0) 3937 static_key_slow_inc(&bpf_stats_enabled_key.key); 3938 3939 mutex_unlock(&bpf_stats_enabled_mutex); 3940 return fd; 3941 } 3942 3943 #define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type 3944 3945 static int bpf_enable_stats(union bpf_attr *attr) 3946 { 3947 3948 if (CHECK_ATTR(BPF_ENABLE_STATS)) 3949 return -EINVAL; 3950 3951 if (!capable(CAP_SYS_ADMIN)) 3952 return -EPERM; 3953 3954 switch (attr->enable_stats.type) { 3955 case BPF_STATS_RUN_TIME: 3956 return bpf_enable_runtime_stats(); 3957 default: 3958 break; 3959 } 3960 return -EINVAL; 3961 } 3962 3963 #define BPF_ITER_CREATE_LAST_FIELD iter_create.flags 3964 3965 static int bpf_iter_create(union bpf_attr *attr) 3966 { 3967 struct bpf_link *link; 3968 int err; 3969 3970 if (CHECK_ATTR(BPF_ITER_CREATE)) 3971 return -EINVAL; 3972 3973 if (attr->iter_create.flags) 3974 return -EINVAL; 3975 3976 link = bpf_link_get_from_fd(attr->iter_create.link_fd); 3977 if (IS_ERR(link)) 3978 return PTR_ERR(link); 3979 3980 err = bpf_iter_new_fd(link); 3981 bpf_link_put(link); 3982 3983 return err; 3984 } 3985 3986 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 3987 { 3988 union bpf_attr attr; 3989 int err; 3990 3991 if (sysctl_unprivileged_bpf_disabled && !capable(CAP_SYS_ADMIN)) 3992 return -EPERM; 3993 3994 err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size); 3995 if (err) 3996 return err; 3997 size = min_t(u32, size, sizeof(attr)); 3998 3999 /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 4000 memset(&attr, 0, sizeof(attr)); 4001 if (copy_from_user(&attr, uattr, size) != 0) 4002 return -EFAULT; 4003 4004 err = security_bpf(cmd, &attr, size); 4005 if (err < 0) 4006 return err; 4007 4008 switch (cmd) { 4009 case BPF_MAP_CREATE: 4010 err = map_create(&attr); 4011 break; 4012 case BPF_MAP_LOOKUP_ELEM: 4013 err = map_lookup_elem(&attr); 4014 break; 4015 case BPF_MAP_UPDATE_ELEM: 4016 err = map_update_elem(&attr); 4017 break; 4018 case BPF_MAP_DELETE_ELEM: 4019 err = map_delete_elem(&attr); 4020 break; 4021 case BPF_MAP_GET_NEXT_KEY: 4022 err = map_get_next_key(&attr); 4023 break; 4024 case BPF_MAP_FREEZE: 4025 err = map_freeze(&attr); 4026 break; 4027 case BPF_PROG_LOAD: 4028 err = bpf_prog_load(&attr, uattr); 4029 break; 4030 case BPF_OBJ_PIN: 4031 err = bpf_obj_pin(&attr); 4032 break; 4033 case BPF_OBJ_GET: 4034 err = bpf_obj_get(&attr); 4035 break; 4036 case BPF_PROG_ATTACH: 4037 err = bpf_prog_attach(&attr); 4038 break; 4039 case BPF_PROG_DETACH: 4040 err = bpf_prog_detach(&attr); 4041 break; 4042 case BPF_PROG_QUERY: 4043 err = bpf_prog_query(&attr, uattr); 4044 break; 4045 case BPF_PROG_TEST_RUN: 4046 err = bpf_prog_test_run(&attr, uattr); 4047 break; 4048 case BPF_PROG_GET_NEXT_ID: 4049 err = bpf_obj_get_next_id(&attr, uattr, 4050 &prog_idr, &prog_idr_lock); 4051 break; 4052 case BPF_MAP_GET_NEXT_ID: 4053 err = bpf_obj_get_next_id(&attr, uattr, 4054 &map_idr, &map_idr_lock); 4055 break; 4056 case BPF_BTF_GET_NEXT_ID: 4057 err = bpf_obj_get_next_id(&attr, uattr, 4058 &btf_idr, &btf_idr_lock); 4059 break; 4060 case BPF_PROG_GET_FD_BY_ID: 4061 err = bpf_prog_get_fd_by_id(&attr); 4062 break; 4063 case BPF_MAP_GET_FD_BY_ID: 4064 err = bpf_map_get_fd_by_id(&attr); 4065 break; 4066 case BPF_OBJ_GET_INFO_BY_FD: 4067 err = bpf_obj_get_info_by_fd(&attr, uattr); 4068 break; 4069 case BPF_RAW_TRACEPOINT_OPEN: 4070 err = bpf_raw_tracepoint_open(&attr); 4071 break; 4072 case BPF_BTF_LOAD: 4073 err = bpf_btf_load(&attr); 4074 break; 4075 case BPF_BTF_GET_FD_BY_ID: 4076 err = bpf_btf_get_fd_by_id(&attr); 4077 break; 4078 case BPF_TASK_FD_QUERY: 4079 err = bpf_task_fd_query(&attr, uattr); 4080 break; 4081 case BPF_MAP_LOOKUP_AND_DELETE_ELEM: 4082 err = map_lookup_and_delete_elem(&attr); 4083 break; 4084 case BPF_MAP_LOOKUP_BATCH: 4085 err = bpf_map_do_batch(&attr, uattr, BPF_MAP_LOOKUP_BATCH); 4086 break; 4087 case BPF_MAP_LOOKUP_AND_DELETE_BATCH: 4088 err = bpf_map_do_batch(&attr, uattr, 4089 BPF_MAP_LOOKUP_AND_DELETE_BATCH); 4090 break; 4091 case BPF_MAP_UPDATE_BATCH: 4092 err = bpf_map_do_batch(&attr, uattr, BPF_MAP_UPDATE_BATCH); 4093 break; 4094 case BPF_MAP_DELETE_BATCH: 4095 err = bpf_map_do_batch(&attr, uattr, BPF_MAP_DELETE_BATCH); 4096 break; 4097 case BPF_LINK_CREATE: 4098 err = link_create(&attr); 4099 break; 4100 case BPF_LINK_UPDATE: 4101 err = link_update(&attr); 4102 break; 4103 case BPF_LINK_GET_FD_BY_ID: 4104 err = bpf_link_get_fd_by_id(&attr); 4105 break; 4106 case BPF_LINK_GET_NEXT_ID: 4107 err = bpf_obj_get_next_id(&attr, uattr, 4108 &link_idr, &link_idr_lock); 4109 break; 4110 case BPF_ENABLE_STATS: 4111 err = bpf_enable_stats(&attr); 4112 break; 4113 case BPF_ITER_CREATE: 4114 err = bpf_iter_create(&attr); 4115 break; 4116 default: 4117 err = -EINVAL; 4118 break; 4119 } 4120 4121 return err; 4122 } 4123