1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 2 * 3 * This program is free software; you can redistribute it and/or 4 * modify it under the terms of version 2 of the GNU General Public 5 * License as published by the Free Software Foundation. 6 * 7 * This program is distributed in the hope that it will be useful, but 8 * WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 10 * General Public License for more details. 11 */ 12 #include <linux/bpf.h> 13 #include <linux/bpf_trace.h> 14 #include <linux/syscalls.h> 15 #include <linux/slab.h> 16 #include <linux/sched/signal.h> 17 #include <linux/vmalloc.h> 18 #include <linux/mmzone.h> 19 #include <linux/anon_inodes.h> 20 #include <linux/file.h> 21 #include <linux/license.h> 22 #include <linux/filter.h> 23 #include <linux/version.h> 24 #include <linux/kernel.h> 25 #include <linux/idr.h> 26 27 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ 28 (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 29 (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 30 (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 31 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 32 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map)) 33 34 DEFINE_PER_CPU(int, bpf_prog_active); 35 static DEFINE_IDR(prog_idr); 36 static DEFINE_SPINLOCK(prog_idr_lock); 37 static DEFINE_IDR(map_idr); 38 static DEFINE_SPINLOCK(map_idr_lock); 39 40 int sysctl_unprivileged_bpf_disabled __read_mostly; 41 42 static const struct bpf_map_ops * const bpf_map_types[] = { 43 #define BPF_PROG_TYPE(_id, _ops) 44 #define BPF_MAP_TYPE(_id, _ops) \ 45 [_id] = &_ops, 46 #include <linux/bpf_types.h> 47 #undef BPF_PROG_TYPE 48 #undef BPF_MAP_TYPE 49 }; 50 51 /* 52 * If we're handed a bigger struct than we know of, ensure all the unknown bits 53 * are 0 - i.e. new user-space does not rely on any kernel feature extensions 54 * we don't know about yet. 55 * 56 * There is a ToCToU between this function call and the following 57 * copy_from_user() call. However, this is not a concern since this function is 58 * meant to be a future-proofing of bits. 59 */ 60 static int check_uarg_tail_zero(void __user *uaddr, 61 size_t expected_size, 62 size_t actual_size) 63 { 64 unsigned char __user *addr; 65 unsigned char __user *end; 66 unsigned char val; 67 int err; 68 69 if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 70 return -E2BIG; 71 72 if (unlikely(!access_ok(VERIFY_READ, uaddr, actual_size))) 73 return -EFAULT; 74 75 if (actual_size <= expected_size) 76 return 0; 77 78 addr = uaddr + expected_size; 79 end = uaddr + actual_size; 80 81 for (; addr < end; addr++) { 82 err = get_user(val, addr); 83 if (err) 84 return err; 85 if (val) 86 return -E2BIG; 87 } 88 89 return 0; 90 } 91 92 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 93 { 94 struct bpf_map *map; 95 96 if (attr->map_type >= ARRAY_SIZE(bpf_map_types) || 97 !bpf_map_types[attr->map_type]) 98 return ERR_PTR(-EINVAL); 99 100 map = bpf_map_types[attr->map_type]->map_alloc(attr); 101 if (IS_ERR(map)) 102 return map; 103 map->ops = bpf_map_types[attr->map_type]; 104 map->map_type = attr->map_type; 105 return map; 106 } 107 108 void *bpf_map_area_alloc(size_t size, int numa_node) 109 { 110 /* We definitely need __GFP_NORETRY, so OOM killer doesn't 111 * trigger under memory pressure as we really just want to 112 * fail instead. 113 */ 114 const gfp_t flags = __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO; 115 void *area; 116 117 if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 118 area = kmalloc_node(size, GFP_USER | flags, numa_node); 119 if (area != NULL) 120 return area; 121 } 122 123 return __vmalloc_node_flags_caller(size, numa_node, GFP_KERNEL | flags, 124 __builtin_return_address(0)); 125 } 126 127 void bpf_map_area_free(void *area) 128 { 129 kvfree(area); 130 } 131 132 int bpf_map_precharge_memlock(u32 pages) 133 { 134 struct user_struct *user = get_current_user(); 135 unsigned long memlock_limit, cur; 136 137 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 138 cur = atomic_long_read(&user->locked_vm); 139 free_uid(user); 140 if (cur + pages > memlock_limit) 141 return -EPERM; 142 return 0; 143 } 144 145 static int bpf_map_charge_memlock(struct bpf_map *map) 146 { 147 struct user_struct *user = get_current_user(); 148 unsigned long memlock_limit; 149 150 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 151 152 atomic_long_add(map->pages, &user->locked_vm); 153 154 if (atomic_long_read(&user->locked_vm) > memlock_limit) { 155 atomic_long_sub(map->pages, &user->locked_vm); 156 free_uid(user); 157 return -EPERM; 158 } 159 map->user = user; 160 return 0; 161 } 162 163 static void bpf_map_uncharge_memlock(struct bpf_map *map) 164 { 165 struct user_struct *user = map->user; 166 167 atomic_long_sub(map->pages, &user->locked_vm); 168 free_uid(user); 169 } 170 171 static int bpf_map_alloc_id(struct bpf_map *map) 172 { 173 int id; 174 175 spin_lock_bh(&map_idr_lock); 176 id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 177 if (id > 0) 178 map->id = id; 179 spin_unlock_bh(&map_idr_lock); 180 181 if (WARN_ON_ONCE(!id)) 182 return -ENOSPC; 183 184 return id > 0 ? 0 : id; 185 } 186 187 static void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) 188 { 189 if (do_idr_lock) 190 spin_lock_bh(&map_idr_lock); 191 else 192 __acquire(&map_idr_lock); 193 194 idr_remove(&map_idr, map->id); 195 196 if (do_idr_lock) 197 spin_unlock_bh(&map_idr_lock); 198 else 199 __release(&map_idr_lock); 200 } 201 202 /* called from workqueue */ 203 static void bpf_map_free_deferred(struct work_struct *work) 204 { 205 struct bpf_map *map = container_of(work, struct bpf_map, work); 206 207 bpf_map_uncharge_memlock(map); 208 /* implementation dependent freeing */ 209 map->ops->map_free(map); 210 } 211 212 static void bpf_map_put_uref(struct bpf_map *map) 213 { 214 if (atomic_dec_and_test(&map->usercnt)) { 215 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) 216 bpf_fd_array_map_clear(map); 217 } 218 } 219 220 /* decrement map refcnt and schedule it for freeing via workqueue 221 * (unrelying map implementation ops->map_free() might sleep) 222 */ 223 static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) 224 { 225 if (atomic_dec_and_test(&map->refcnt)) { 226 /* bpf_map_free_id() must be called first */ 227 bpf_map_free_id(map, do_idr_lock); 228 INIT_WORK(&map->work, bpf_map_free_deferred); 229 schedule_work(&map->work); 230 } 231 } 232 233 void bpf_map_put(struct bpf_map *map) 234 { 235 __bpf_map_put(map, true); 236 } 237 238 void bpf_map_put_with_uref(struct bpf_map *map) 239 { 240 bpf_map_put_uref(map); 241 bpf_map_put(map); 242 } 243 244 static int bpf_map_release(struct inode *inode, struct file *filp) 245 { 246 struct bpf_map *map = filp->private_data; 247 248 if (map->ops->map_release) 249 map->ops->map_release(map, filp); 250 251 bpf_map_put_with_uref(map); 252 return 0; 253 } 254 255 #ifdef CONFIG_PROC_FS 256 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 257 { 258 const struct bpf_map *map = filp->private_data; 259 const struct bpf_array *array; 260 u32 owner_prog_type = 0; 261 u32 owner_jited = 0; 262 263 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 264 array = container_of(map, struct bpf_array, map); 265 owner_prog_type = array->owner_prog_type; 266 owner_jited = array->owner_jited; 267 } 268 269 seq_printf(m, 270 "map_type:\t%u\n" 271 "key_size:\t%u\n" 272 "value_size:\t%u\n" 273 "max_entries:\t%u\n" 274 "map_flags:\t%#x\n" 275 "memlock:\t%llu\n", 276 map->map_type, 277 map->key_size, 278 map->value_size, 279 map->max_entries, 280 map->map_flags, 281 map->pages * 1ULL << PAGE_SHIFT); 282 283 if (owner_prog_type) { 284 seq_printf(m, "owner_prog_type:\t%u\n", 285 owner_prog_type); 286 seq_printf(m, "owner_jited:\t%u\n", 287 owner_jited); 288 } 289 } 290 #endif 291 292 static const struct file_operations bpf_map_fops = { 293 #ifdef CONFIG_PROC_FS 294 .show_fdinfo = bpf_map_show_fdinfo, 295 #endif 296 .release = bpf_map_release, 297 }; 298 299 int bpf_map_new_fd(struct bpf_map *map) 300 { 301 return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 302 O_RDWR | O_CLOEXEC); 303 } 304 305 /* helper macro to check that unused fields 'union bpf_attr' are zero */ 306 #define CHECK_ATTR(CMD) \ 307 memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 308 sizeof(attr->CMD##_LAST_FIELD), 0, \ 309 sizeof(*attr) - \ 310 offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 311 sizeof(attr->CMD##_LAST_FIELD)) != NULL 312 313 #define BPF_MAP_CREATE_LAST_FIELD numa_node 314 /* called via syscall */ 315 static int map_create(union bpf_attr *attr) 316 { 317 int numa_node = bpf_map_attr_numa_node(attr); 318 struct bpf_map *map; 319 int err; 320 321 err = CHECK_ATTR(BPF_MAP_CREATE); 322 if (err) 323 return -EINVAL; 324 325 if (numa_node != NUMA_NO_NODE && 326 ((unsigned int)numa_node >= nr_node_ids || 327 !node_online(numa_node))) 328 return -EINVAL; 329 330 /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 331 map = find_and_alloc_map(attr); 332 if (IS_ERR(map)) 333 return PTR_ERR(map); 334 335 atomic_set(&map->refcnt, 1); 336 atomic_set(&map->usercnt, 1); 337 338 err = bpf_map_charge_memlock(map); 339 if (err) 340 goto free_map_nouncharge; 341 342 err = bpf_map_alloc_id(map); 343 if (err) 344 goto free_map; 345 346 err = bpf_map_new_fd(map); 347 if (err < 0) { 348 /* failed to allocate fd. 349 * bpf_map_put() is needed because the above 350 * bpf_map_alloc_id() has published the map 351 * to the userspace and the userspace may 352 * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 353 */ 354 bpf_map_put(map); 355 return err; 356 } 357 358 trace_bpf_map_create(map, err); 359 return err; 360 361 free_map: 362 bpf_map_uncharge_memlock(map); 363 free_map_nouncharge: 364 map->ops->map_free(map); 365 return err; 366 } 367 368 /* if error is returned, fd is released. 369 * On success caller should complete fd access with matching fdput() 370 */ 371 struct bpf_map *__bpf_map_get(struct fd f) 372 { 373 if (!f.file) 374 return ERR_PTR(-EBADF); 375 if (f.file->f_op != &bpf_map_fops) { 376 fdput(f); 377 return ERR_PTR(-EINVAL); 378 } 379 380 return f.file->private_data; 381 } 382 383 /* prog's and map's refcnt limit */ 384 #define BPF_MAX_REFCNT 32768 385 386 struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref) 387 { 388 if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) { 389 atomic_dec(&map->refcnt); 390 return ERR_PTR(-EBUSY); 391 } 392 if (uref) 393 atomic_inc(&map->usercnt); 394 return map; 395 } 396 397 struct bpf_map *bpf_map_get_with_uref(u32 ufd) 398 { 399 struct fd f = fdget(ufd); 400 struct bpf_map *map; 401 402 map = __bpf_map_get(f); 403 if (IS_ERR(map)) 404 return map; 405 406 map = bpf_map_inc(map, true); 407 fdput(f); 408 409 return map; 410 } 411 412 /* map_idr_lock should have been held */ 413 static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map, 414 bool uref) 415 { 416 int refold; 417 418 refold = __atomic_add_unless(&map->refcnt, 1, 0); 419 420 if (refold >= BPF_MAX_REFCNT) { 421 __bpf_map_put(map, false); 422 return ERR_PTR(-EBUSY); 423 } 424 425 if (!refold) 426 return ERR_PTR(-ENOENT); 427 428 if (uref) 429 atomic_inc(&map->usercnt); 430 431 return map; 432 } 433 434 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 435 { 436 return -ENOTSUPP; 437 } 438 439 /* last field in 'union bpf_attr' used by this command */ 440 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value 441 442 static int map_lookup_elem(union bpf_attr *attr) 443 { 444 void __user *ukey = u64_to_user_ptr(attr->key); 445 void __user *uvalue = u64_to_user_ptr(attr->value); 446 int ufd = attr->map_fd; 447 struct bpf_map *map; 448 void *key, *value, *ptr; 449 u32 value_size; 450 struct fd f; 451 int err; 452 453 if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 454 return -EINVAL; 455 456 f = fdget(ufd); 457 map = __bpf_map_get(f); 458 if (IS_ERR(map)) 459 return PTR_ERR(map); 460 461 key = memdup_user(ukey, map->key_size); 462 if (IS_ERR(key)) { 463 err = PTR_ERR(key); 464 goto err_put; 465 } 466 467 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 468 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 469 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 470 value_size = round_up(map->value_size, 8) * num_possible_cpus(); 471 else if (IS_FD_MAP(map)) 472 value_size = sizeof(u32); 473 else 474 value_size = map->value_size; 475 476 err = -ENOMEM; 477 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 478 if (!value) 479 goto free_key; 480 481 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 482 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 483 err = bpf_percpu_hash_copy(map, key, value); 484 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 485 err = bpf_percpu_array_copy(map, key, value); 486 } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 487 err = bpf_stackmap_copy(map, key, value); 488 } else if (IS_FD_ARRAY(map)) { 489 err = bpf_fd_array_map_lookup_elem(map, key, value); 490 } else if (IS_FD_HASH(map)) { 491 err = bpf_fd_htab_map_lookup_elem(map, key, value); 492 } else { 493 rcu_read_lock(); 494 ptr = map->ops->map_lookup_elem(map, key); 495 if (ptr) 496 memcpy(value, ptr, value_size); 497 rcu_read_unlock(); 498 err = ptr ? 0 : -ENOENT; 499 } 500 501 if (err) 502 goto free_value; 503 504 err = -EFAULT; 505 if (copy_to_user(uvalue, value, value_size) != 0) 506 goto free_value; 507 508 trace_bpf_map_lookup_elem(map, ufd, key, value); 509 err = 0; 510 511 free_value: 512 kfree(value); 513 free_key: 514 kfree(key); 515 err_put: 516 fdput(f); 517 return err; 518 } 519 520 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 521 522 static int map_update_elem(union bpf_attr *attr) 523 { 524 void __user *ukey = u64_to_user_ptr(attr->key); 525 void __user *uvalue = u64_to_user_ptr(attr->value); 526 int ufd = attr->map_fd; 527 struct bpf_map *map; 528 void *key, *value; 529 u32 value_size; 530 struct fd f; 531 int err; 532 533 if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 534 return -EINVAL; 535 536 f = fdget(ufd); 537 map = __bpf_map_get(f); 538 if (IS_ERR(map)) 539 return PTR_ERR(map); 540 541 key = memdup_user(ukey, map->key_size); 542 if (IS_ERR(key)) { 543 err = PTR_ERR(key); 544 goto err_put; 545 } 546 547 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 548 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 549 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 550 value_size = round_up(map->value_size, 8) * num_possible_cpus(); 551 else 552 value_size = map->value_size; 553 554 err = -ENOMEM; 555 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 556 if (!value) 557 goto free_key; 558 559 err = -EFAULT; 560 if (copy_from_user(value, uvalue, value_size) != 0) 561 goto free_value; 562 563 /* must increment bpf_prog_active to avoid kprobe+bpf triggering from 564 * inside bpf map update or delete otherwise deadlocks are possible 565 */ 566 preempt_disable(); 567 __this_cpu_inc(bpf_prog_active); 568 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 569 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 570 err = bpf_percpu_hash_update(map, key, value, attr->flags); 571 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 572 err = bpf_percpu_array_update(map, key, value, attr->flags); 573 } else if (map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || 574 map->map_type == BPF_MAP_TYPE_PROG_ARRAY || 575 map->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || 576 map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) { 577 rcu_read_lock(); 578 err = bpf_fd_array_map_update_elem(map, f.file, key, value, 579 attr->flags); 580 rcu_read_unlock(); 581 } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 582 rcu_read_lock(); 583 err = bpf_fd_htab_map_update_elem(map, f.file, key, value, 584 attr->flags); 585 rcu_read_unlock(); 586 } else { 587 rcu_read_lock(); 588 err = map->ops->map_update_elem(map, key, value, attr->flags); 589 rcu_read_unlock(); 590 } 591 __this_cpu_dec(bpf_prog_active); 592 preempt_enable(); 593 594 if (!err) 595 trace_bpf_map_update_elem(map, ufd, key, value); 596 free_value: 597 kfree(value); 598 free_key: 599 kfree(key); 600 err_put: 601 fdput(f); 602 return err; 603 } 604 605 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 606 607 static int map_delete_elem(union bpf_attr *attr) 608 { 609 void __user *ukey = u64_to_user_ptr(attr->key); 610 int ufd = attr->map_fd; 611 struct bpf_map *map; 612 struct fd f; 613 void *key; 614 int err; 615 616 if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 617 return -EINVAL; 618 619 f = fdget(ufd); 620 map = __bpf_map_get(f); 621 if (IS_ERR(map)) 622 return PTR_ERR(map); 623 624 key = memdup_user(ukey, map->key_size); 625 if (IS_ERR(key)) { 626 err = PTR_ERR(key); 627 goto err_put; 628 } 629 630 preempt_disable(); 631 __this_cpu_inc(bpf_prog_active); 632 rcu_read_lock(); 633 err = map->ops->map_delete_elem(map, key); 634 rcu_read_unlock(); 635 __this_cpu_dec(bpf_prog_active); 636 preempt_enable(); 637 638 if (!err) 639 trace_bpf_map_delete_elem(map, ufd, key); 640 kfree(key); 641 err_put: 642 fdput(f); 643 return err; 644 } 645 646 /* last field in 'union bpf_attr' used by this command */ 647 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 648 649 static int map_get_next_key(union bpf_attr *attr) 650 { 651 void __user *ukey = u64_to_user_ptr(attr->key); 652 void __user *unext_key = u64_to_user_ptr(attr->next_key); 653 int ufd = attr->map_fd; 654 struct bpf_map *map; 655 void *key, *next_key; 656 struct fd f; 657 int err; 658 659 if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 660 return -EINVAL; 661 662 f = fdget(ufd); 663 map = __bpf_map_get(f); 664 if (IS_ERR(map)) 665 return PTR_ERR(map); 666 667 if (ukey) { 668 key = memdup_user(ukey, map->key_size); 669 if (IS_ERR(key)) { 670 err = PTR_ERR(key); 671 goto err_put; 672 } 673 } else { 674 key = NULL; 675 } 676 677 err = -ENOMEM; 678 next_key = kmalloc(map->key_size, GFP_USER); 679 if (!next_key) 680 goto free_key; 681 682 rcu_read_lock(); 683 err = map->ops->map_get_next_key(map, key, next_key); 684 rcu_read_unlock(); 685 if (err) 686 goto free_next_key; 687 688 err = -EFAULT; 689 if (copy_to_user(unext_key, next_key, map->key_size) != 0) 690 goto free_next_key; 691 692 trace_bpf_map_next_key(map, ufd, key, next_key); 693 err = 0; 694 695 free_next_key: 696 kfree(next_key); 697 free_key: 698 kfree(key); 699 err_put: 700 fdput(f); 701 return err; 702 } 703 704 static const struct bpf_verifier_ops * const bpf_prog_types[] = { 705 #define BPF_PROG_TYPE(_id, _ops) \ 706 [_id] = &_ops, 707 #define BPF_MAP_TYPE(_id, _ops) 708 #include <linux/bpf_types.h> 709 #undef BPF_PROG_TYPE 710 #undef BPF_MAP_TYPE 711 }; 712 713 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 714 { 715 if (type >= ARRAY_SIZE(bpf_prog_types) || !bpf_prog_types[type]) 716 return -EINVAL; 717 718 prog->aux->ops = bpf_prog_types[type]; 719 prog->type = type; 720 return 0; 721 } 722 723 /* drop refcnt on maps used by eBPF program and free auxilary data */ 724 static void free_used_maps(struct bpf_prog_aux *aux) 725 { 726 int i; 727 728 for (i = 0; i < aux->used_map_cnt; i++) 729 bpf_map_put(aux->used_maps[i]); 730 731 kfree(aux->used_maps); 732 } 733 734 int __bpf_prog_charge(struct user_struct *user, u32 pages) 735 { 736 unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 737 unsigned long user_bufs; 738 739 if (user) { 740 user_bufs = atomic_long_add_return(pages, &user->locked_vm); 741 if (user_bufs > memlock_limit) { 742 atomic_long_sub(pages, &user->locked_vm); 743 return -EPERM; 744 } 745 } 746 747 return 0; 748 } 749 750 void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 751 { 752 if (user) 753 atomic_long_sub(pages, &user->locked_vm); 754 } 755 756 static int bpf_prog_charge_memlock(struct bpf_prog *prog) 757 { 758 struct user_struct *user = get_current_user(); 759 int ret; 760 761 ret = __bpf_prog_charge(user, prog->pages); 762 if (ret) { 763 free_uid(user); 764 return ret; 765 } 766 767 prog->aux->user = user; 768 return 0; 769 } 770 771 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 772 { 773 struct user_struct *user = prog->aux->user; 774 775 __bpf_prog_uncharge(user, prog->pages); 776 free_uid(user); 777 } 778 779 static int bpf_prog_alloc_id(struct bpf_prog *prog) 780 { 781 int id; 782 783 spin_lock_bh(&prog_idr_lock); 784 id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 785 if (id > 0) 786 prog->aux->id = id; 787 spin_unlock_bh(&prog_idr_lock); 788 789 /* id is in [1, INT_MAX) */ 790 if (WARN_ON_ONCE(!id)) 791 return -ENOSPC; 792 793 return id > 0 ? 0 : id; 794 } 795 796 static void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) 797 { 798 /* cBPF to eBPF migrations are currently not in the idr store. */ 799 if (!prog->aux->id) 800 return; 801 802 if (do_idr_lock) 803 spin_lock_bh(&prog_idr_lock); 804 else 805 __acquire(&prog_idr_lock); 806 807 idr_remove(&prog_idr, prog->aux->id); 808 809 if (do_idr_lock) 810 spin_unlock_bh(&prog_idr_lock); 811 else 812 __release(&prog_idr_lock); 813 } 814 815 static void __bpf_prog_put_rcu(struct rcu_head *rcu) 816 { 817 struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 818 819 free_used_maps(aux); 820 bpf_prog_uncharge_memlock(aux->prog); 821 bpf_prog_free(aux->prog); 822 } 823 824 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 825 { 826 if (atomic_dec_and_test(&prog->aux->refcnt)) { 827 trace_bpf_prog_put_rcu(prog); 828 /* bpf_prog_free_id() must be called first */ 829 bpf_prog_free_id(prog, do_idr_lock); 830 bpf_prog_kallsyms_del(prog); 831 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 832 } 833 } 834 835 void bpf_prog_put(struct bpf_prog *prog) 836 { 837 __bpf_prog_put(prog, true); 838 } 839 EXPORT_SYMBOL_GPL(bpf_prog_put); 840 841 static int bpf_prog_release(struct inode *inode, struct file *filp) 842 { 843 struct bpf_prog *prog = filp->private_data; 844 845 bpf_prog_put(prog); 846 return 0; 847 } 848 849 #ifdef CONFIG_PROC_FS 850 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 851 { 852 const struct bpf_prog *prog = filp->private_data; 853 char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 854 855 bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 856 seq_printf(m, 857 "prog_type:\t%u\n" 858 "prog_jited:\t%u\n" 859 "prog_tag:\t%s\n" 860 "memlock:\t%llu\n", 861 prog->type, 862 prog->jited, 863 prog_tag, 864 prog->pages * 1ULL << PAGE_SHIFT); 865 } 866 #endif 867 868 static const struct file_operations bpf_prog_fops = { 869 #ifdef CONFIG_PROC_FS 870 .show_fdinfo = bpf_prog_show_fdinfo, 871 #endif 872 .release = bpf_prog_release, 873 }; 874 875 int bpf_prog_new_fd(struct bpf_prog *prog) 876 { 877 return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 878 O_RDWR | O_CLOEXEC); 879 } 880 881 static struct bpf_prog *____bpf_prog_get(struct fd f) 882 { 883 if (!f.file) 884 return ERR_PTR(-EBADF); 885 if (f.file->f_op != &bpf_prog_fops) { 886 fdput(f); 887 return ERR_PTR(-EINVAL); 888 } 889 890 return f.file->private_data; 891 } 892 893 struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i) 894 { 895 if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) { 896 atomic_sub(i, &prog->aux->refcnt); 897 return ERR_PTR(-EBUSY); 898 } 899 return prog; 900 } 901 EXPORT_SYMBOL_GPL(bpf_prog_add); 902 903 void bpf_prog_sub(struct bpf_prog *prog, int i) 904 { 905 /* Only to be used for undoing previous bpf_prog_add() in some 906 * error path. We still know that another entity in our call 907 * path holds a reference to the program, thus atomic_sub() can 908 * be safely used in such cases! 909 */ 910 WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0); 911 } 912 EXPORT_SYMBOL_GPL(bpf_prog_sub); 913 914 struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog) 915 { 916 return bpf_prog_add(prog, 1); 917 } 918 EXPORT_SYMBOL_GPL(bpf_prog_inc); 919 920 /* prog_idr_lock should have been held */ 921 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 922 { 923 int refold; 924 925 refold = __atomic_add_unless(&prog->aux->refcnt, 1, 0); 926 927 if (refold >= BPF_MAX_REFCNT) { 928 __bpf_prog_put(prog, false); 929 return ERR_PTR(-EBUSY); 930 } 931 932 if (!refold) 933 return ERR_PTR(-ENOENT); 934 935 return prog; 936 } 937 EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 938 939 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *type) 940 { 941 struct fd f = fdget(ufd); 942 struct bpf_prog *prog; 943 944 prog = ____bpf_prog_get(f); 945 if (IS_ERR(prog)) 946 return prog; 947 if (type && prog->type != *type) { 948 prog = ERR_PTR(-EINVAL); 949 goto out; 950 } 951 952 prog = bpf_prog_inc(prog); 953 out: 954 fdput(f); 955 return prog; 956 } 957 958 struct bpf_prog *bpf_prog_get(u32 ufd) 959 { 960 return __bpf_prog_get(ufd, NULL); 961 } 962 963 struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type) 964 { 965 struct bpf_prog *prog = __bpf_prog_get(ufd, &type); 966 967 if (!IS_ERR(prog)) 968 trace_bpf_prog_get_type(prog); 969 return prog; 970 } 971 EXPORT_SYMBOL_GPL(bpf_prog_get_type); 972 973 /* last field in 'union bpf_attr' used by this command */ 974 #define BPF_PROG_LOAD_LAST_FIELD prog_flags 975 976 static int bpf_prog_load(union bpf_attr *attr) 977 { 978 enum bpf_prog_type type = attr->prog_type; 979 struct bpf_prog *prog; 980 int err; 981 char license[128]; 982 bool is_gpl; 983 984 if (CHECK_ATTR(BPF_PROG_LOAD)) 985 return -EINVAL; 986 987 if (attr->prog_flags & ~BPF_F_STRICT_ALIGNMENT) 988 return -EINVAL; 989 990 /* copy eBPF program license from user space */ 991 if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 992 sizeof(license) - 1) < 0) 993 return -EFAULT; 994 license[sizeof(license) - 1] = 0; 995 996 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 997 is_gpl = license_is_gpl_compatible(license); 998 999 if (attr->insn_cnt == 0 || attr->insn_cnt > BPF_MAXINSNS) 1000 return -E2BIG; 1001 1002 if (type == BPF_PROG_TYPE_KPROBE && 1003 attr->kern_version != LINUX_VERSION_CODE) 1004 return -EINVAL; 1005 1006 if (type != BPF_PROG_TYPE_SOCKET_FILTER && 1007 type != BPF_PROG_TYPE_CGROUP_SKB && 1008 !capable(CAP_SYS_ADMIN)) 1009 return -EPERM; 1010 1011 /* plain bpf_prog allocation */ 1012 prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 1013 if (!prog) 1014 return -ENOMEM; 1015 1016 err = bpf_prog_charge_memlock(prog); 1017 if (err) 1018 goto free_prog_nouncharge; 1019 1020 prog->len = attr->insn_cnt; 1021 1022 err = -EFAULT; 1023 if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 1024 bpf_prog_insn_size(prog)) != 0) 1025 goto free_prog; 1026 1027 prog->orig_prog = NULL; 1028 prog->jited = 0; 1029 1030 atomic_set(&prog->aux->refcnt, 1); 1031 prog->gpl_compatible = is_gpl ? 1 : 0; 1032 1033 /* find program type: socket_filter vs tracing_filter */ 1034 err = find_prog_type(type, prog); 1035 if (err < 0) 1036 goto free_prog; 1037 1038 /* run eBPF verifier */ 1039 err = bpf_check(&prog, attr); 1040 if (err < 0) 1041 goto free_used_maps; 1042 1043 /* eBPF program is ready to be JITed */ 1044 prog = bpf_prog_select_runtime(prog, &err); 1045 if (err < 0) 1046 goto free_used_maps; 1047 1048 err = bpf_prog_alloc_id(prog); 1049 if (err) 1050 goto free_used_maps; 1051 1052 err = bpf_prog_new_fd(prog); 1053 if (err < 0) { 1054 /* failed to allocate fd. 1055 * bpf_prog_put() is needed because the above 1056 * bpf_prog_alloc_id() has published the prog 1057 * to the userspace and the userspace may 1058 * have refcnt-ed it through BPF_PROG_GET_FD_BY_ID. 1059 */ 1060 bpf_prog_put(prog); 1061 return err; 1062 } 1063 1064 bpf_prog_kallsyms_add(prog); 1065 trace_bpf_prog_load(prog, err); 1066 return err; 1067 1068 free_used_maps: 1069 free_used_maps(prog->aux); 1070 free_prog: 1071 bpf_prog_uncharge_memlock(prog); 1072 free_prog_nouncharge: 1073 bpf_prog_free(prog); 1074 return err; 1075 } 1076 1077 #define BPF_OBJ_LAST_FIELD bpf_fd 1078 1079 static int bpf_obj_pin(const union bpf_attr *attr) 1080 { 1081 if (CHECK_ATTR(BPF_OBJ)) 1082 return -EINVAL; 1083 1084 return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 1085 } 1086 1087 static int bpf_obj_get(const union bpf_attr *attr) 1088 { 1089 if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0) 1090 return -EINVAL; 1091 1092 return bpf_obj_get_user(u64_to_user_ptr(attr->pathname)); 1093 } 1094 1095 #ifdef CONFIG_CGROUP_BPF 1096 1097 #define BPF_PROG_ATTACH_LAST_FIELD attach_flags 1098 1099 static int sockmap_get_from_fd(const union bpf_attr *attr) 1100 { 1101 int ufd = attr->target_fd; 1102 struct bpf_prog *prog; 1103 struct bpf_map *map; 1104 struct fd f; 1105 int err; 1106 1107 f = fdget(ufd); 1108 map = __bpf_map_get(f); 1109 if (IS_ERR(map)) 1110 return PTR_ERR(map); 1111 1112 prog = bpf_prog_get_type(attr->attach_bpf_fd, BPF_PROG_TYPE_SK_SKB); 1113 if (IS_ERR(prog)) { 1114 fdput(f); 1115 return PTR_ERR(prog); 1116 } 1117 1118 err = sock_map_attach_prog(map, prog, attr->attach_type); 1119 if (err) { 1120 fdput(f); 1121 bpf_prog_put(prog); 1122 return err; 1123 } 1124 1125 fdput(f); 1126 return 0; 1127 } 1128 1129 static int bpf_prog_attach(const union bpf_attr *attr) 1130 { 1131 enum bpf_prog_type ptype; 1132 struct bpf_prog *prog; 1133 struct cgroup *cgrp; 1134 int ret; 1135 1136 if (!capable(CAP_NET_ADMIN)) 1137 return -EPERM; 1138 1139 if (CHECK_ATTR(BPF_PROG_ATTACH)) 1140 return -EINVAL; 1141 1142 if (attr->attach_flags & ~BPF_F_ALLOW_OVERRIDE) 1143 return -EINVAL; 1144 1145 switch (attr->attach_type) { 1146 case BPF_CGROUP_INET_INGRESS: 1147 case BPF_CGROUP_INET_EGRESS: 1148 ptype = BPF_PROG_TYPE_CGROUP_SKB; 1149 break; 1150 case BPF_CGROUP_INET_SOCK_CREATE: 1151 ptype = BPF_PROG_TYPE_CGROUP_SOCK; 1152 break; 1153 case BPF_CGROUP_SOCK_OPS: 1154 ptype = BPF_PROG_TYPE_SOCK_OPS; 1155 break; 1156 case BPF_SK_SKB_STREAM_PARSER: 1157 case BPF_SK_SKB_STREAM_VERDICT: 1158 return sockmap_get_from_fd(attr); 1159 default: 1160 return -EINVAL; 1161 } 1162 1163 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 1164 if (IS_ERR(prog)) 1165 return PTR_ERR(prog); 1166 1167 cgrp = cgroup_get_from_fd(attr->target_fd); 1168 if (IS_ERR(cgrp)) { 1169 bpf_prog_put(prog); 1170 return PTR_ERR(cgrp); 1171 } 1172 1173 ret = cgroup_bpf_update(cgrp, prog, attr->attach_type, 1174 attr->attach_flags & BPF_F_ALLOW_OVERRIDE); 1175 if (ret) 1176 bpf_prog_put(prog); 1177 cgroup_put(cgrp); 1178 1179 return ret; 1180 } 1181 1182 #define BPF_PROG_DETACH_LAST_FIELD attach_type 1183 1184 static int bpf_prog_detach(const union bpf_attr *attr) 1185 { 1186 struct cgroup *cgrp; 1187 int ret; 1188 1189 if (!capable(CAP_NET_ADMIN)) 1190 return -EPERM; 1191 1192 if (CHECK_ATTR(BPF_PROG_DETACH)) 1193 return -EINVAL; 1194 1195 switch (attr->attach_type) { 1196 case BPF_CGROUP_INET_INGRESS: 1197 case BPF_CGROUP_INET_EGRESS: 1198 case BPF_CGROUP_INET_SOCK_CREATE: 1199 case BPF_CGROUP_SOCK_OPS: 1200 cgrp = cgroup_get_from_fd(attr->target_fd); 1201 if (IS_ERR(cgrp)) 1202 return PTR_ERR(cgrp); 1203 1204 ret = cgroup_bpf_update(cgrp, NULL, attr->attach_type, false); 1205 cgroup_put(cgrp); 1206 break; 1207 1208 default: 1209 return -EINVAL; 1210 } 1211 1212 return ret; 1213 } 1214 1215 #endif /* CONFIG_CGROUP_BPF */ 1216 1217 #define BPF_PROG_TEST_RUN_LAST_FIELD test.duration 1218 1219 static int bpf_prog_test_run(const union bpf_attr *attr, 1220 union bpf_attr __user *uattr) 1221 { 1222 struct bpf_prog *prog; 1223 int ret = -ENOTSUPP; 1224 1225 if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 1226 return -EINVAL; 1227 1228 prog = bpf_prog_get(attr->test.prog_fd); 1229 if (IS_ERR(prog)) 1230 return PTR_ERR(prog); 1231 1232 if (prog->aux->ops->test_run) 1233 ret = prog->aux->ops->test_run(prog, attr, uattr); 1234 1235 bpf_prog_put(prog); 1236 return ret; 1237 } 1238 1239 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 1240 1241 static int bpf_obj_get_next_id(const union bpf_attr *attr, 1242 union bpf_attr __user *uattr, 1243 struct idr *idr, 1244 spinlock_t *lock) 1245 { 1246 u32 next_id = attr->start_id; 1247 int err = 0; 1248 1249 if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 1250 return -EINVAL; 1251 1252 if (!capable(CAP_SYS_ADMIN)) 1253 return -EPERM; 1254 1255 next_id++; 1256 spin_lock_bh(lock); 1257 if (!idr_get_next(idr, &next_id)) 1258 err = -ENOENT; 1259 spin_unlock_bh(lock); 1260 1261 if (!err) 1262 err = put_user(next_id, &uattr->next_id); 1263 1264 return err; 1265 } 1266 1267 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 1268 1269 static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 1270 { 1271 struct bpf_prog *prog; 1272 u32 id = attr->prog_id; 1273 int fd; 1274 1275 if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 1276 return -EINVAL; 1277 1278 if (!capable(CAP_SYS_ADMIN)) 1279 return -EPERM; 1280 1281 spin_lock_bh(&prog_idr_lock); 1282 prog = idr_find(&prog_idr, id); 1283 if (prog) 1284 prog = bpf_prog_inc_not_zero(prog); 1285 else 1286 prog = ERR_PTR(-ENOENT); 1287 spin_unlock_bh(&prog_idr_lock); 1288 1289 if (IS_ERR(prog)) 1290 return PTR_ERR(prog); 1291 1292 fd = bpf_prog_new_fd(prog); 1293 if (fd < 0) 1294 bpf_prog_put(prog); 1295 1296 return fd; 1297 } 1298 1299 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD map_id 1300 1301 static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 1302 { 1303 struct bpf_map *map; 1304 u32 id = attr->map_id; 1305 int fd; 1306 1307 if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID)) 1308 return -EINVAL; 1309 1310 if (!capable(CAP_SYS_ADMIN)) 1311 return -EPERM; 1312 1313 spin_lock_bh(&map_idr_lock); 1314 map = idr_find(&map_idr, id); 1315 if (map) 1316 map = bpf_map_inc_not_zero(map, true); 1317 else 1318 map = ERR_PTR(-ENOENT); 1319 spin_unlock_bh(&map_idr_lock); 1320 1321 if (IS_ERR(map)) 1322 return PTR_ERR(map); 1323 1324 fd = bpf_map_new_fd(map); 1325 if (fd < 0) 1326 bpf_map_put(map); 1327 1328 return fd; 1329 } 1330 1331 static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, 1332 const union bpf_attr *attr, 1333 union bpf_attr __user *uattr) 1334 { 1335 struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 1336 struct bpf_prog_info info = {}; 1337 u32 info_len = attr->info.info_len; 1338 char __user *uinsns; 1339 u32 ulen; 1340 int err; 1341 1342 err = check_uarg_tail_zero(uinfo, sizeof(info), info_len); 1343 if (err) 1344 return err; 1345 info_len = min_t(u32, sizeof(info), info_len); 1346 1347 if (copy_from_user(&info, uinfo, info_len)) 1348 return -EFAULT; 1349 1350 info.type = prog->type; 1351 info.id = prog->aux->id; 1352 1353 memcpy(info.tag, prog->tag, sizeof(prog->tag)); 1354 1355 if (!capable(CAP_SYS_ADMIN)) { 1356 info.jited_prog_len = 0; 1357 info.xlated_prog_len = 0; 1358 goto done; 1359 } 1360 1361 ulen = info.jited_prog_len; 1362 info.jited_prog_len = prog->jited_len; 1363 if (info.jited_prog_len && ulen) { 1364 uinsns = u64_to_user_ptr(info.jited_prog_insns); 1365 ulen = min_t(u32, info.jited_prog_len, ulen); 1366 if (copy_to_user(uinsns, prog->bpf_func, ulen)) 1367 return -EFAULT; 1368 } 1369 1370 ulen = info.xlated_prog_len; 1371 info.xlated_prog_len = bpf_prog_insn_size(prog); 1372 if (info.xlated_prog_len && ulen) { 1373 uinsns = u64_to_user_ptr(info.xlated_prog_insns); 1374 ulen = min_t(u32, info.xlated_prog_len, ulen); 1375 if (copy_to_user(uinsns, prog->insnsi, ulen)) 1376 return -EFAULT; 1377 } 1378 1379 done: 1380 if (copy_to_user(uinfo, &info, info_len) || 1381 put_user(info_len, &uattr->info.info_len)) 1382 return -EFAULT; 1383 1384 return 0; 1385 } 1386 1387 static int bpf_map_get_info_by_fd(struct bpf_map *map, 1388 const union bpf_attr *attr, 1389 union bpf_attr __user *uattr) 1390 { 1391 struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 1392 struct bpf_map_info info = {}; 1393 u32 info_len = attr->info.info_len; 1394 int err; 1395 1396 err = check_uarg_tail_zero(uinfo, sizeof(info), info_len); 1397 if (err) 1398 return err; 1399 info_len = min_t(u32, sizeof(info), info_len); 1400 1401 info.type = map->map_type; 1402 info.id = map->id; 1403 info.key_size = map->key_size; 1404 info.value_size = map->value_size; 1405 info.max_entries = map->max_entries; 1406 info.map_flags = map->map_flags; 1407 1408 if (copy_to_user(uinfo, &info, info_len) || 1409 put_user(info_len, &uattr->info.info_len)) 1410 return -EFAULT; 1411 1412 return 0; 1413 } 1414 1415 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 1416 1417 static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 1418 union bpf_attr __user *uattr) 1419 { 1420 int ufd = attr->info.bpf_fd; 1421 struct fd f; 1422 int err; 1423 1424 if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 1425 return -EINVAL; 1426 1427 f = fdget(ufd); 1428 if (!f.file) 1429 return -EBADFD; 1430 1431 if (f.file->f_op == &bpf_prog_fops) 1432 err = bpf_prog_get_info_by_fd(f.file->private_data, attr, 1433 uattr); 1434 else if (f.file->f_op == &bpf_map_fops) 1435 err = bpf_map_get_info_by_fd(f.file->private_data, attr, 1436 uattr); 1437 else 1438 err = -EINVAL; 1439 1440 fdput(f); 1441 return err; 1442 } 1443 1444 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 1445 { 1446 union bpf_attr attr = {}; 1447 int err; 1448 1449 if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled) 1450 return -EPERM; 1451 1452 err = check_uarg_tail_zero(uattr, sizeof(attr), size); 1453 if (err) 1454 return err; 1455 size = min_t(u32, size, sizeof(attr)); 1456 1457 /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 1458 if (copy_from_user(&attr, uattr, size) != 0) 1459 return -EFAULT; 1460 1461 switch (cmd) { 1462 case BPF_MAP_CREATE: 1463 err = map_create(&attr); 1464 break; 1465 case BPF_MAP_LOOKUP_ELEM: 1466 err = map_lookup_elem(&attr); 1467 break; 1468 case BPF_MAP_UPDATE_ELEM: 1469 err = map_update_elem(&attr); 1470 break; 1471 case BPF_MAP_DELETE_ELEM: 1472 err = map_delete_elem(&attr); 1473 break; 1474 case BPF_MAP_GET_NEXT_KEY: 1475 err = map_get_next_key(&attr); 1476 break; 1477 case BPF_PROG_LOAD: 1478 err = bpf_prog_load(&attr); 1479 break; 1480 case BPF_OBJ_PIN: 1481 err = bpf_obj_pin(&attr); 1482 break; 1483 case BPF_OBJ_GET: 1484 err = bpf_obj_get(&attr); 1485 break; 1486 #ifdef CONFIG_CGROUP_BPF 1487 case BPF_PROG_ATTACH: 1488 err = bpf_prog_attach(&attr); 1489 break; 1490 case BPF_PROG_DETACH: 1491 err = bpf_prog_detach(&attr); 1492 break; 1493 #endif 1494 case BPF_PROG_TEST_RUN: 1495 err = bpf_prog_test_run(&attr, uattr); 1496 break; 1497 case BPF_PROG_GET_NEXT_ID: 1498 err = bpf_obj_get_next_id(&attr, uattr, 1499 &prog_idr, &prog_idr_lock); 1500 break; 1501 case BPF_MAP_GET_NEXT_ID: 1502 err = bpf_obj_get_next_id(&attr, uattr, 1503 &map_idr, &map_idr_lock); 1504 break; 1505 case BPF_PROG_GET_FD_BY_ID: 1506 err = bpf_prog_get_fd_by_id(&attr); 1507 break; 1508 case BPF_MAP_GET_FD_BY_ID: 1509 err = bpf_map_get_fd_by_id(&attr); 1510 break; 1511 case BPF_OBJ_GET_INFO_BY_FD: 1512 err = bpf_obj_get_info_by_fd(&attr, uattr); 1513 break; 1514 default: 1515 err = -EINVAL; 1516 break; 1517 } 1518 1519 return err; 1520 } 1521