1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 2 * 3 * This program is free software; you can redistribute it and/or 4 * modify it under the terms of version 2 of the GNU General Public 5 * License as published by the Free Software Foundation. 6 * 7 * This program is distributed in the hope that it will be useful, but 8 * WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 10 * General Public License for more details. 11 */ 12 #include <linux/bpf.h> 13 #include <linux/syscalls.h> 14 #include <linux/slab.h> 15 #include <linux/anon_inodes.h> 16 #include <linux/file.h> 17 #include <linux/license.h> 18 #include <linux/filter.h> 19 #include <linux/version.h> 20 21 DEFINE_PER_CPU(int, bpf_prog_active); 22 23 int sysctl_unprivileged_bpf_disabled __read_mostly; 24 25 static LIST_HEAD(bpf_map_types); 26 27 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 28 { 29 struct bpf_map_type_list *tl; 30 struct bpf_map *map; 31 32 list_for_each_entry(tl, &bpf_map_types, list_node) { 33 if (tl->type == attr->map_type) { 34 map = tl->ops->map_alloc(attr); 35 if (IS_ERR(map)) 36 return map; 37 map->ops = tl->ops; 38 map->map_type = attr->map_type; 39 return map; 40 } 41 } 42 return ERR_PTR(-EINVAL); 43 } 44 45 /* boot time registration of different map implementations */ 46 void bpf_register_map_type(struct bpf_map_type_list *tl) 47 { 48 list_add(&tl->list_node, &bpf_map_types); 49 } 50 51 int bpf_map_precharge_memlock(u32 pages) 52 { 53 struct user_struct *user = get_current_user(); 54 unsigned long memlock_limit, cur; 55 56 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 57 cur = atomic_long_read(&user->locked_vm); 58 free_uid(user); 59 if (cur + pages > memlock_limit) 60 return -EPERM; 61 return 0; 62 } 63 64 static int bpf_map_charge_memlock(struct bpf_map *map) 65 { 66 struct user_struct *user = get_current_user(); 67 unsigned long memlock_limit; 68 69 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 70 71 atomic_long_add(map->pages, &user->locked_vm); 72 73 if (atomic_long_read(&user->locked_vm) > memlock_limit) { 74 atomic_long_sub(map->pages, &user->locked_vm); 75 free_uid(user); 76 return -EPERM; 77 } 78 map->user = user; 79 return 0; 80 } 81 82 static void bpf_map_uncharge_memlock(struct bpf_map *map) 83 { 84 struct user_struct *user = map->user; 85 86 atomic_long_sub(map->pages, &user->locked_vm); 87 free_uid(user); 88 } 89 90 /* called from workqueue */ 91 static void bpf_map_free_deferred(struct work_struct *work) 92 { 93 struct bpf_map *map = container_of(work, struct bpf_map, work); 94 95 bpf_map_uncharge_memlock(map); 96 /* implementation dependent freeing */ 97 map->ops->map_free(map); 98 } 99 100 static void bpf_map_put_uref(struct bpf_map *map) 101 { 102 if (atomic_dec_and_test(&map->usercnt)) { 103 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) 104 bpf_fd_array_map_clear(map); 105 } 106 } 107 108 /* decrement map refcnt and schedule it for freeing via workqueue 109 * (unrelying map implementation ops->map_free() might sleep) 110 */ 111 void bpf_map_put(struct bpf_map *map) 112 { 113 if (atomic_dec_and_test(&map->refcnt)) { 114 INIT_WORK(&map->work, bpf_map_free_deferred); 115 schedule_work(&map->work); 116 } 117 } 118 119 void bpf_map_put_with_uref(struct bpf_map *map) 120 { 121 bpf_map_put_uref(map); 122 bpf_map_put(map); 123 } 124 125 static int bpf_map_release(struct inode *inode, struct file *filp) 126 { 127 struct bpf_map *map = filp->private_data; 128 129 if (map->ops->map_release) 130 map->ops->map_release(map, filp); 131 132 bpf_map_put_with_uref(map); 133 return 0; 134 } 135 136 #ifdef CONFIG_PROC_FS 137 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 138 { 139 const struct bpf_map *map = filp->private_data; 140 141 seq_printf(m, 142 "map_type:\t%u\n" 143 "key_size:\t%u\n" 144 "value_size:\t%u\n" 145 "max_entries:\t%u\n" 146 "map_flags:\t%#x\n", 147 map->map_type, 148 map->key_size, 149 map->value_size, 150 map->max_entries, 151 map->map_flags); 152 } 153 #endif 154 155 static const struct file_operations bpf_map_fops = { 156 #ifdef CONFIG_PROC_FS 157 .show_fdinfo = bpf_map_show_fdinfo, 158 #endif 159 .release = bpf_map_release, 160 }; 161 162 int bpf_map_new_fd(struct bpf_map *map) 163 { 164 return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 165 O_RDWR | O_CLOEXEC); 166 } 167 168 /* helper macro to check that unused fields 'union bpf_attr' are zero */ 169 #define CHECK_ATTR(CMD) \ 170 memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 171 sizeof(attr->CMD##_LAST_FIELD), 0, \ 172 sizeof(*attr) - \ 173 offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 174 sizeof(attr->CMD##_LAST_FIELD)) != NULL 175 176 #define BPF_MAP_CREATE_LAST_FIELD map_flags 177 /* called via syscall */ 178 static int map_create(union bpf_attr *attr) 179 { 180 struct bpf_map *map; 181 int err; 182 183 err = CHECK_ATTR(BPF_MAP_CREATE); 184 if (err) 185 return -EINVAL; 186 187 /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 188 map = find_and_alloc_map(attr); 189 if (IS_ERR(map)) 190 return PTR_ERR(map); 191 192 atomic_set(&map->refcnt, 1); 193 atomic_set(&map->usercnt, 1); 194 195 err = bpf_map_charge_memlock(map); 196 if (err) 197 goto free_map; 198 199 err = bpf_map_new_fd(map); 200 if (err < 0) 201 /* failed to allocate fd */ 202 goto free_map; 203 204 return err; 205 206 free_map: 207 map->ops->map_free(map); 208 return err; 209 } 210 211 /* if error is returned, fd is released. 212 * On success caller should complete fd access with matching fdput() 213 */ 214 struct bpf_map *__bpf_map_get(struct fd f) 215 { 216 if (!f.file) 217 return ERR_PTR(-EBADF); 218 if (f.file->f_op != &bpf_map_fops) { 219 fdput(f); 220 return ERR_PTR(-EINVAL); 221 } 222 223 return f.file->private_data; 224 } 225 226 /* prog's and map's refcnt limit */ 227 #define BPF_MAX_REFCNT 32768 228 229 struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref) 230 { 231 if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) { 232 atomic_dec(&map->refcnt); 233 return ERR_PTR(-EBUSY); 234 } 235 if (uref) 236 atomic_inc(&map->usercnt); 237 return map; 238 } 239 240 struct bpf_map *bpf_map_get_with_uref(u32 ufd) 241 { 242 struct fd f = fdget(ufd); 243 struct bpf_map *map; 244 245 map = __bpf_map_get(f); 246 if (IS_ERR(map)) 247 return map; 248 249 map = bpf_map_inc(map, true); 250 fdput(f); 251 252 return map; 253 } 254 255 /* helper to convert user pointers passed inside __aligned_u64 fields */ 256 static void __user *u64_to_ptr(__u64 val) 257 { 258 return (void __user *) (unsigned long) val; 259 } 260 261 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 262 { 263 return -ENOTSUPP; 264 } 265 266 /* last field in 'union bpf_attr' used by this command */ 267 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value 268 269 static int map_lookup_elem(union bpf_attr *attr) 270 { 271 void __user *ukey = u64_to_ptr(attr->key); 272 void __user *uvalue = u64_to_ptr(attr->value); 273 int ufd = attr->map_fd; 274 struct bpf_map *map; 275 void *key, *value, *ptr; 276 u32 value_size; 277 struct fd f; 278 int err; 279 280 if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 281 return -EINVAL; 282 283 f = fdget(ufd); 284 map = __bpf_map_get(f); 285 if (IS_ERR(map)) 286 return PTR_ERR(map); 287 288 err = -ENOMEM; 289 key = kmalloc(map->key_size, GFP_USER); 290 if (!key) 291 goto err_put; 292 293 err = -EFAULT; 294 if (copy_from_user(key, ukey, map->key_size) != 0) 295 goto free_key; 296 297 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 298 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 299 value_size = round_up(map->value_size, 8) * num_possible_cpus(); 300 else 301 value_size = map->value_size; 302 303 err = -ENOMEM; 304 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 305 if (!value) 306 goto free_key; 307 308 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH) { 309 err = bpf_percpu_hash_copy(map, key, value); 310 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 311 err = bpf_percpu_array_copy(map, key, value); 312 } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 313 err = bpf_stackmap_copy(map, key, value); 314 } else { 315 rcu_read_lock(); 316 ptr = map->ops->map_lookup_elem(map, key); 317 if (ptr) 318 memcpy(value, ptr, value_size); 319 rcu_read_unlock(); 320 err = ptr ? 0 : -ENOENT; 321 } 322 323 if (err) 324 goto free_value; 325 326 err = -EFAULT; 327 if (copy_to_user(uvalue, value, value_size) != 0) 328 goto free_value; 329 330 err = 0; 331 332 free_value: 333 kfree(value); 334 free_key: 335 kfree(key); 336 err_put: 337 fdput(f); 338 return err; 339 } 340 341 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 342 343 static int map_update_elem(union bpf_attr *attr) 344 { 345 void __user *ukey = u64_to_ptr(attr->key); 346 void __user *uvalue = u64_to_ptr(attr->value); 347 int ufd = attr->map_fd; 348 struct bpf_map *map; 349 void *key, *value; 350 u32 value_size; 351 struct fd f; 352 int err; 353 354 if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 355 return -EINVAL; 356 357 f = fdget(ufd); 358 map = __bpf_map_get(f); 359 if (IS_ERR(map)) 360 return PTR_ERR(map); 361 362 err = -ENOMEM; 363 key = kmalloc(map->key_size, GFP_USER); 364 if (!key) 365 goto err_put; 366 367 err = -EFAULT; 368 if (copy_from_user(key, ukey, map->key_size) != 0) 369 goto free_key; 370 371 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 372 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 373 value_size = round_up(map->value_size, 8) * num_possible_cpus(); 374 else 375 value_size = map->value_size; 376 377 err = -ENOMEM; 378 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 379 if (!value) 380 goto free_key; 381 382 err = -EFAULT; 383 if (copy_from_user(value, uvalue, value_size) != 0) 384 goto free_value; 385 386 /* must increment bpf_prog_active to avoid kprobe+bpf triggering from 387 * inside bpf map update or delete otherwise deadlocks are possible 388 */ 389 preempt_disable(); 390 __this_cpu_inc(bpf_prog_active); 391 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH) { 392 err = bpf_percpu_hash_update(map, key, value, attr->flags); 393 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 394 err = bpf_percpu_array_update(map, key, value, attr->flags); 395 } else if (map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || 396 map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 397 rcu_read_lock(); 398 err = bpf_fd_array_map_update_elem(map, f.file, key, value, 399 attr->flags); 400 rcu_read_unlock(); 401 } else { 402 rcu_read_lock(); 403 err = map->ops->map_update_elem(map, key, value, attr->flags); 404 rcu_read_unlock(); 405 } 406 __this_cpu_dec(bpf_prog_active); 407 preempt_enable(); 408 409 free_value: 410 kfree(value); 411 free_key: 412 kfree(key); 413 err_put: 414 fdput(f); 415 return err; 416 } 417 418 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 419 420 static int map_delete_elem(union bpf_attr *attr) 421 { 422 void __user *ukey = u64_to_ptr(attr->key); 423 int ufd = attr->map_fd; 424 struct bpf_map *map; 425 struct fd f; 426 void *key; 427 int err; 428 429 if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 430 return -EINVAL; 431 432 f = fdget(ufd); 433 map = __bpf_map_get(f); 434 if (IS_ERR(map)) 435 return PTR_ERR(map); 436 437 err = -ENOMEM; 438 key = kmalloc(map->key_size, GFP_USER); 439 if (!key) 440 goto err_put; 441 442 err = -EFAULT; 443 if (copy_from_user(key, ukey, map->key_size) != 0) 444 goto free_key; 445 446 preempt_disable(); 447 __this_cpu_inc(bpf_prog_active); 448 rcu_read_lock(); 449 err = map->ops->map_delete_elem(map, key); 450 rcu_read_unlock(); 451 __this_cpu_dec(bpf_prog_active); 452 preempt_enable(); 453 454 free_key: 455 kfree(key); 456 err_put: 457 fdput(f); 458 return err; 459 } 460 461 /* last field in 'union bpf_attr' used by this command */ 462 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 463 464 static int map_get_next_key(union bpf_attr *attr) 465 { 466 void __user *ukey = u64_to_ptr(attr->key); 467 void __user *unext_key = u64_to_ptr(attr->next_key); 468 int ufd = attr->map_fd; 469 struct bpf_map *map; 470 void *key, *next_key; 471 struct fd f; 472 int err; 473 474 if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 475 return -EINVAL; 476 477 f = fdget(ufd); 478 map = __bpf_map_get(f); 479 if (IS_ERR(map)) 480 return PTR_ERR(map); 481 482 err = -ENOMEM; 483 key = kmalloc(map->key_size, GFP_USER); 484 if (!key) 485 goto err_put; 486 487 err = -EFAULT; 488 if (copy_from_user(key, ukey, map->key_size) != 0) 489 goto free_key; 490 491 err = -ENOMEM; 492 next_key = kmalloc(map->key_size, GFP_USER); 493 if (!next_key) 494 goto free_key; 495 496 rcu_read_lock(); 497 err = map->ops->map_get_next_key(map, key, next_key); 498 rcu_read_unlock(); 499 if (err) 500 goto free_next_key; 501 502 err = -EFAULT; 503 if (copy_to_user(unext_key, next_key, map->key_size) != 0) 504 goto free_next_key; 505 506 err = 0; 507 508 free_next_key: 509 kfree(next_key); 510 free_key: 511 kfree(key); 512 err_put: 513 fdput(f); 514 return err; 515 } 516 517 static LIST_HEAD(bpf_prog_types); 518 519 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 520 { 521 struct bpf_prog_type_list *tl; 522 523 list_for_each_entry(tl, &bpf_prog_types, list_node) { 524 if (tl->type == type) { 525 prog->aux->ops = tl->ops; 526 prog->type = type; 527 return 0; 528 } 529 } 530 531 return -EINVAL; 532 } 533 534 void bpf_register_prog_type(struct bpf_prog_type_list *tl) 535 { 536 list_add(&tl->list_node, &bpf_prog_types); 537 } 538 539 /* fixup insn->imm field of bpf_call instructions: 540 * if (insn->imm == BPF_FUNC_map_lookup_elem) 541 * insn->imm = bpf_map_lookup_elem - __bpf_call_base; 542 * else if (insn->imm == BPF_FUNC_map_update_elem) 543 * insn->imm = bpf_map_update_elem - __bpf_call_base; 544 * else ... 545 * 546 * this function is called after eBPF program passed verification 547 */ 548 static void fixup_bpf_calls(struct bpf_prog *prog) 549 { 550 const struct bpf_func_proto *fn; 551 int i; 552 553 for (i = 0; i < prog->len; i++) { 554 struct bpf_insn *insn = &prog->insnsi[i]; 555 556 if (insn->code == (BPF_JMP | BPF_CALL)) { 557 /* we reach here when program has bpf_call instructions 558 * and it passed bpf_check(), means that 559 * ops->get_func_proto must have been supplied, check it 560 */ 561 BUG_ON(!prog->aux->ops->get_func_proto); 562 563 if (insn->imm == BPF_FUNC_get_route_realm) 564 prog->dst_needed = 1; 565 if (insn->imm == BPF_FUNC_get_prandom_u32) 566 bpf_user_rnd_init_once(); 567 if (insn->imm == BPF_FUNC_tail_call) { 568 /* mark bpf_tail_call as different opcode 569 * to avoid conditional branch in 570 * interpeter for every normal call 571 * and to prevent accidental JITing by 572 * JIT compiler that doesn't support 573 * bpf_tail_call yet 574 */ 575 insn->imm = 0; 576 insn->code |= BPF_X; 577 continue; 578 } 579 580 fn = prog->aux->ops->get_func_proto(insn->imm); 581 /* all functions that have prototype and verifier allowed 582 * programs to call them, must be real in-kernel functions 583 */ 584 BUG_ON(!fn->func); 585 insn->imm = fn->func - __bpf_call_base; 586 } 587 } 588 } 589 590 /* drop refcnt on maps used by eBPF program and free auxilary data */ 591 static void free_used_maps(struct bpf_prog_aux *aux) 592 { 593 int i; 594 595 for (i = 0; i < aux->used_map_cnt; i++) 596 bpf_map_put(aux->used_maps[i]); 597 598 kfree(aux->used_maps); 599 } 600 601 static int bpf_prog_charge_memlock(struct bpf_prog *prog) 602 { 603 struct user_struct *user = get_current_user(); 604 unsigned long memlock_limit; 605 606 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 607 608 atomic_long_add(prog->pages, &user->locked_vm); 609 if (atomic_long_read(&user->locked_vm) > memlock_limit) { 610 atomic_long_sub(prog->pages, &user->locked_vm); 611 free_uid(user); 612 return -EPERM; 613 } 614 prog->aux->user = user; 615 return 0; 616 } 617 618 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 619 { 620 struct user_struct *user = prog->aux->user; 621 622 atomic_long_sub(prog->pages, &user->locked_vm); 623 free_uid(user); 624 } 625 626 static void __prog_put_common(struct rcu_head *rcu) 627 { 628 struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 629 630 free_used_maps(aux); 631 bpf_prog_uncharge_memlock(aux->prog); 632 bpf_prog_free(aux->prog); 633 } 634 635 /* version of bpf_prog_put() that is called after a grace period */ 636 void bpf_prog_put_rcu(struct bpf_prog *prog) 637 { 638 if (atomic_dec_and_test(&prog->aux->refcnt)) 639 call_rcu(&prog->aux->rcu, __prog_put_common); 640 } 641 642 void bpf_prog_put(struct bpf_prog *prog) 643 { 644 if (atomic_dec_and_test(&prog->aux->refcnt)) 645 __prog_put_common(&prog->aux->rcu); 646 } 647 EXPORT_SYMBOL_GPL(bpf_prog_put); 648 649 static int bpf_prog_release(struct inode *inode, struct file *filp) 650 { 651 struct bpf_prog *prog = filp->private_data; 652 653 bpf_prog_put_rcu(prog); 654 return 0; 655 } 656 657 static const struct file_operations bpf_prog_fops = { 658 .release = bpf_prog_release, 659 }; 660 661 int bpf_prog_new_fd(struct bpf_prog *prog) 662 { 663 return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 664 O_RDWR | O_CLOEXEC); 665 } 666 667 static struct bpf_prog *__bpf_prog_get(struct fd f) 668 { 669 if (!f.file) 670 return ERR_PTR(-EBADF); 671 if (f.file->f_op != &bpf_prog_fops) { 672 fdput(f); 673 return ERR_PTR(-EINVAL); 674 } 675 676 return f.file->private_data; 677 } 678 679 struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog) 680 { 681 if (atomic_inc_return(&prog->aux->refcnt) > BPF_MAX_REFCNT) { 682 atomic_dec(&prog->aux->refcnt); 683 return ERR_PTR(-EBUSY); 684 } 685 return prog; 686 } 687 688 /* called by sockets/tracing/seccomp before attaching program to an event 689 * pairs with bpf_prog_put() 690 */ 691 struct bpf_prog *bpf_prog_get(u32 ufd) 692 { 693 struct fd f = fdget(ufd); 694 struct bpf_prog *prog; 695 696 prog = __bpf_prog_get(f); 697 if (IS_ERR(prog)) 698 return prog; 699 700 prog = bpf_prog_inc(prog); 701 fdput(f); 702 703 return prog; 704 } 705 EXPORT_SYMBOL_GPL(bpf_prog_get); 706 707 /* last field in 'union bpf_attr' used by this command */ 708 #define BPF_PROG_LOAD_LAST_FIELD kern_version 709 710 static int bpf_prog_load(union bpf_attr *attr) 711 { 712 enum bpf_prog_type type = attr->prog_type; 713 struct bpf_prog *prog; 714 int err; 715 char license[128]; 716 bool is_gpl; 717 718 if (CHECK_ATTR(BPF_PROG_LOAD)) 719 return -EINVAL; 720 721 /* copy eBPF program license from user space */ 722 if (strncpy_from_user(license, u64_to_ptr(attr->license), 723 sizeof(license) - 1) < 0) 724 return -EFAULT; 725 license[sizeof(license) - 1] = 0; 726 727 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 728 is_gpl = license_is_gpl_compatible(license); 729 730 if (attr->insn_cnt >= BPF_MAXINSNS) 731 return -EINVAL; 732 733 if (type == BPF_PROG_TYPE_KPROBE && 734 attr->kern_version != LINUX_VERSION_CODE) 735 return -EINVAL; 736 737 if (type != BPF_PROG_TYPE_SOCKET_FILTER && !capable(CAP_SYS_ADMIN)) 738 return -EPERM; 739 740 /* plain bpf_prog allocation */ 741 prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 742 if (!prog) 743 return -ENOMEM; 744 745 err = bpf_prog_charge_memlock(prog); 746 if (err) 747 goto free_prog_nouncharge; 748 749 prog->len = attr->insn_cnt; 750 751 err = -EFAULT; 752 if (copy_from_user(prog->insns, u64_to_ptr(attr->insns), 753 prog->len * sizeof(struct bpf_insn)) != 0) 754 goto free_prog; 755 756 prog->orig_prog = NULL; 757 prog->jited = 0; 758 759 atomic_set(&prog->aux->refcnt, 1); 760 prog->gpl_compatible = is_gpl ? 1 : 0; 761 762 /* find program type: socket_filter vs tracing_filter */ 763 err = find_prog_type(type, prog); 764 if (err < 0) 765 goto free_prog; 766 767 /* run eBPF verifier */ 768 err = bpf_check(&prog, attr); 769 if (err < 0) 770 goto free_used_maps; 771 772 /* fixup BPF_CALL->imm field */ 773 fixup_bpf_calls(prog); 774 775 /* eBPF program is ready to be JITed */ 776 prog = bpf_prog_select_runtime(prog, &err); 777 if (err < 0) 778 goto free_used_maps; 779 780 err = bpf_prog_new_fd(prog); 781 if (err < 0) 782 /* failed to allocate fd */ 783 goto free_used_maps; 784 785 return err; 786 787 free_used_maps: 788 free_used_maps(prog->aux); 789 free_prog: 790 bpf_prog_uncharge_memlock(prog); 791 free_prog_nouncharge: 792 bpf_prog_free(prog); 793 return err; 794 } 795 796 #define BPF_OBJ_LAST_FIELD bpf_fd 797 798 static int bpf_obj_pin(const union bpf_attr *attr) 799 { 800 if (CHECK_ATTR(BPF_OBJ)) 801 return -EINVAL; 802 803 return bpf_obj_pin_user(attr->bpf_fd, u64_to_ptr(attr->pathname)); 804 } 805 806 static int bpf_obj_get(const union bpf_attr *attr) 807 { 808 if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0) 809 return -EINVAL; 810 811 return bpf_obj_get_user(u64_to_ptr(attr->pathname)); 812 } 813 814 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 815 { 816 union bpf_attr attr = {}; 817 int err; 818 819 if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled) 820 return -EPERM; 821 822 if (!access_ok(VERIFY_READ, uattr, 1)) 823 return -EFAULT; 824 825 if (size > PAGE_SIZE) /* silly large */ 826 return -E2BIG; 827 828 /* If we're handed a bigger struct than we know of, 829 * ensure all the unknown bits are 0 - i.e. new 830 * user-space does not rely on any kernel feature 831 * extensions we dont know about yet. 832 */ 833 if (size > sizeof(attr)) { 834 unsigned char __user *addr; 835 unsigned char __user *end; 836 unsigned char val; 837 838 addr = (void __user *)uattr + sizeof(attr); 839 end = (void __user *)uattr + size; 840 841 for (; addr < end; addr++) { 842 err = get_user(val, addr); 843 if (err) 844 return err; 845 if (val) 846 return -E2BIG; 847 } 848 size = sizeof(attr); 849 } 850 851 /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 852 if (copy_from_user(&attr, uattr, size) != 0) 853 return -EFAULT; 854 855 switch (cmd) { 856 case BPF_MAP_CREATE: 857 err = map_create(&attr); 858 break; 859 case BPF_MAP_LOOKUP_ELEM: 860 err = map_lookup_elem(&attr); 861 break; 862 case BPF_MAP_UPDATE_ELEM: 863 err = map_update_elem(&attr); 864 break; 865 case BPF_MAP_DELETE_ELEM: 866 err = map_delete_elem(&attr); 867 break; 868 case BPF_MAP_GET_NEXT_KEY: 869 err = map_get_next_key(&attr); 870 break; 871 case BPF_PROG_LOAD: 872 err = bpf_prog_load(&attr); 873 break; 874 case BPF_OBJ_PIN: 875 err = bpf_obj_pin(&attr); 876 break; 877 case BPF_OBJ_GET: 878 err = bpf_obj_get(&attr); 879 break; 880 default: 881 err = -EINVAL; 882 break; 883 } 884 885 return err; 886 } 887