199c55f7dSAlexei Starovoitov /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 299c55f7dSAlexei Starovoitov * 399c55f7dSAlexei Starovoitov * This program is free software; you can redistribute it and/or 499c55f7dSAlexei Starovoitov * modify it under the terms of version 2 of the GNU General Public 599c55f7dSAlexei Starovoitov * License as published by the Free Software Foundation. 699c55f7dSAlexei Starovoitov * 799c55f7dSAlexei Starovoitov * This program is distributed in the hope that it will be useful, but 899c55f7dSAlexei Starovoitov * WITHOUT ANY WARRANTY; without even the implied warranty of 999c55f7dSAlexei Starovoitov * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 1099c55f7dSAlexei Starovoitov * General Public License for more details. 1199c55f7dSAlexei Starovoitov */ 1299c55f7dSAlexei Starovoitov #include <linux/bpf.h> 1399c55f7dSAlexei Starovoitov #include <linux/syscalls.h> 1499c55f7dSAlexei Starovoitov #include <linux/slab.h> 1599c55f7dSAlexei Starovoitov #include <linux/anon_inodes.h> 16db20fd2bSAlexei Starovoitov #include <linux/file.h> 1709756af4SAlexei Starovoitov #include <linux/license.h> 1809756af4SAlexei Starovoitov #include <linux/filter.h> 192541517cSAlexei Starovoitov #include <linux/version.h> 20535e7b4bSMickaël Salaün #include <linux/kernel.h> 2199c55f7dSAlexei Starovoitov 22b121d1e7SAlexei Starovoitov DEFINE_PER_CPU(int, bpf_prog_active); 23b121d1e7SAlexei Starovoitov 241be7f75dSAlexei Starovoitov int sysctl_unprivileged_bpf_disabled __read_mostly; 251be7f75dSAlexei Starovoitov 2699c55f7dSAlexei Starovoitov static LIST_HEAD(bpf_map_types); 2799c55f7dSAlexei Starovoitov 2899c55f7dSAlexei Starovoitov static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 2999c55f7dSAlexei Starovoitov { 3099c55f7dSAlexei Starovoitov struct bpf_map_type_list *tl; 3199c55f7dSAlexei Starovoitov struct bpf_map *map; 3299c55f7dSAlexei Starovoitov 3399c55f7dSAlexei Starovoitov list_for_each_entry(tl, &bpf_map_types, list_node) { 3499c55f7dSAlexei Starovoitov if (tl->type == attr->map_type) { 3599c55f7dSAlexei Starovoitov map = tl->ops->map_alloc(attr); 3699c55f7dSAlexei Starovoitov if (IS_ERR(map)) 3799c55f7dSAlexei Starovoitov return map; 3899c55f7dSAlexei Starovoitov map->ops = tl->ops; 3999c55f7dSAlexei Starovoitov map->map_type = attr->map_type; 4099c55f7dSAlexei Starovoitov return map; 4199c55f7dSAlexei Starovoitov } 4299c55f7dSAlexei Starovoitov } 4399c55f7dSAlexei Starovoitov return ERR_PTR(-EINVAL); 4499c55f7dSAlexei Starovoitov } 4599c55f7dSAlexei Starovoitov 4699c55f7dSAlexei Starovoitov /* boot time registration of different map implementations */ 4799c55f7dSAlexei Starovoitov void bpf_register_map_type(struct bpf_map_type_list *tl) 4899c55f7dSAlexei Starovoitov { 4999c55f7dSAlexei Starovoitov list_add(&tl->list_node, &bpf_map_types); 5099c55f7dSAlexei Starovoitov } 5199c55f7dSAlexei Starovoitov 526c905981SAlexei Starovoitov int bpf_map_precharge_memlock(u32 pages) 536c905981SAlexei Starovoitov { 546c905981SAlexei Starovoitov struct user_struct *user = get_current_user(); 556c905981SAlexei Starovoitov unsigned long memlock_limit, cur; 566c905981SAlexei Starovoitov 576c905981SAlexei Starovoitov memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 586c905981SAlexei Starovoitov cur = atomic_long_read(&user->locked_vm); 596c905981SAlexei Starovoitov free_uid(user); 606c905981SAlexei Starovoitov if (cur + pages > memlock_limit) 616c905981SAlexei Starovoitov return -EPERM; 626c905981SAlexei Starovoitov return 0; 636c905981SAlexei Starovoitov } 646c905981SAlexei Starovoitov 65aaac3ba9SAlexei Starovoitov static int bpf_map_charge_memlock(struct bpf_map *map) 66aaac3ba9SAlexei Starovoitov { 67aaac3ba9SAlexei Starovoitov struct user_struct *user = get_current_user(); 68aaac3ba9SAlexei Starovoitov unsigned long memlock_limit; 69aaac3ba9SAlexei Starovoitov 70aaac3ba9SAlexei Starovoitov memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 71aaac3ba9SAlexei Starovoitov 72aaac3ba9SAlexei Starovoitov atomic_long_add(map->pages, &user->locked_vm); 73aaac3ba9SAlexei Starovoitov 74aaac3ba9SAlexei Starovoitov if (atomic_long_read(&user->locked_vm) > memlock_limit) { 75aaac3ba9SAlexei Starovoitov atomic_long_sub(map->pages, &user->locked_vm); 76aaac3ba9SAlexei Starovoitov free_uid(user); 77aaac3ba9SAlexei Starovoitov return -EPERM; 78aaac3ba9SAlexei Starovoitov } 79aaac3ba9SAlexei Starovoitov map->user = user; 80aaac3ba9SAlexei Starovoitov return 0; 81aaac3ba9SAlexei Starovoitov } 82aaac3ba9SAlexei Starovoitov 83aaac3ba9SAlexei Starovoitov static void bpf_map_uncharge_memlock(struct bpf_map *map) 84aaac3ba9SAlexei Starovoitov { 85aaac3ba9SAlexei Starovoitov struct user_struct *user = map->user; 86aaac3ba9SAlexei Starovoitov 87aaac3ba9SAlexei Starovoitov atomic_long_sub(map->pages, &user->locked_vm); 88aaac3ba9SAlexei Starovoitov free_uid(user); 89aaac3ba9SAlexei Starovoitov } 90aaac3ba9SAlexei Starovoitov 9199c55f7dSAlexei Starovoitov /* called from workqueue */ 9299c55f7dSAlexei Starovoitov static void bpf_map_free_deferred(struct work_struct *work) 9399c55f7dSAlexei Starovoitov { 9499c55f7dSAlexei Starovoitov struct bpf_map *map = container_of(work, struct bpf_map, work); 9599c55f7dSAlexei Starovoitov 96aaac3ba9SAlexei Starovoitov bpf_map_uncharge_memlock(map); 9799c55f7dSAlexei Starovoitov /* implementation dependent freeing */ 9899c55f7dSAlexei Starovoitov map->ops->map_free(map); 9999c55f7dSAlexei Starovoitov } 10099c55f7dSAlexei Starovoitov 101c9da161cSDaniel Borkmann static void bpf_map_put_uref(struct bpf_map *map) 102c9da161cSDaniel Borkmann { 103c9da161cSDaniel Borkmann if (atomic_dec_and_test(&map->usercnt)) { 104c9da161cSDaniel Borkmann if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) 105c9da161cSDaniel Borkmann bpf_fd_array_map_clear(map); 106c9da161cSDaniel Borkmann } 107c9da161cSDaniel Borkmann } 108c9da161cSDaniel Borkmann 10999c55f7dSAlexei Starovoitov /* decrement map refcnt and schedule it for freeing via workqueue 11099c55f7dSAlexei Starovoitov * (unrelying map implementation ops->map_free() might sleep) 11199c55f7dSAlexei Starovoitov */ 11299c55f7dSAlexei Starovoitov void bpf_map_put(struct bpf_map *map) 11399c55f7dSAlexei Starovoitov { 11499c55f7dSAlexei Starovoitov if (atomic_dec_and_test(&map->refcnt)) { 11599c55f7dSAlexei Starovoitov INIT_WORK(&map->work, bpf_map_free_deferred); 11699c55f7dSAlexei Starovoitov schedule_work(&map->work); 11799c55f7dSAlexei Starovoitov } 11899c55f7dSAlexei Starovoitov } 11999c55f7dSAlexei Starovoitov 120c9da161cSDaniel Borkmann void bpf_map_put_with_uref(struct bpf_map *map) 121c9da161cSDaniel Borkmann { 122c9da161cSDaniel Borkmann bpf_map_put_uref(map); 123c9da161cSDaniel Borkmann bpf_map_put(map); 124c9da161cSDaniel Borkmann } 125c9da161cSDaniel Borkmann 12699c55f7dSAlexei Starovoitov static int bpf_map_release(struct inode *inode, struct file *filp) 12799c55f7dSAlexei Starovoitov { 12861d1b6a4SDaniel Borkmann struct bpf_map *map = filp->private_data; 12961d1b6a4SDaniel Borkmann 13061d1b6a4SDaniel Borkmann if (map->ops->map_release) 13161d1b6a4SDaniel Borkmann map->ops->map_release(map, filp); 13261d1b6a4SDaniel Borkmann 13361d1b6a4SDaniel Borkmann bpf_map_put_with_uref(map); 13499c55f7dSAlexei Starovoitov return 0; 13599c55f7dSAlexei Starovoitov } 13699c55f7dSAlexei Starovoitov 137f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 138f99bf205SDaniel Borkmann static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 139f99bf205SDaniel Borkmann { 140f99bf205SDaniel Borkmann const struct bpf_map *map = filp->private_data; 141f99bf205SDaniel Borkmann 142f99bf205SDaniel Borkmann seq_printf(m, 143f99bf205SDaniel Borkmann "map_type:\t%u\n" 144f99bf205SDaniel Borkmann "key_size:\t%u\n" 145f99bf205SDaniel Borkmann "value_size:\t%u\n" 146322cea2fSDaniel Borkmann "max_entries:\t%u\n" 147322cea2fSDaniel Borkmann "map_flags:\t%#x\n", 148f99bf205SDaniel Borkmann map->map_type, 149f99bf205SDaniel Borkmann map->key_size, 150f99bf205SDaniel Borkmann map->value_size, 151322cea2fSDaniel Borkmann map->max_entries, 152322cea2fSDaniel Borkmann map->map_flags); 153f99bf205SDaniel Borkmann } 154f99bf205SDaniel Borkmann #endif 155f99bf205SDaniel Borkmann 15699c55f7dSAlexei Starovoitov static const struct file_operations bpf_map_fops = { 157f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 158f99bf205SDaniel Borkmann .show_fdinfo = bpf_map_show_fdinfo, 159f99bf205SDaniel Borkmann #endif 16099c55f7dSAlexei Starovoitov .release = bpf_map_release, 16199c55f7dSAlexei Starovoitov }; 16299c55f7dSAlexei Starovoitov 163b2197755SDaniel Borkmann int bpf_map_new_fd(struct bpf_map *map) 164aa79781bSDaniel Borkmann { 165aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 166aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC); 167aa79781bSDaniel Borkmann } 168aa79781bSDaniel Borkmann 16999c55f7dSAlexei Starovoitov /* helper macro to check that unused fields 'union bpf_attr' are zero */ 17099c55f7dSAlexei Starovoitov #define CHECK_ATTR(CMD) \ 17199c55f7dSAlexei Starovoitov memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 17299c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD), 0, \ 17399c55f7dSAlexei Starovoitov sizeof(*attr) - \ 17499c55f7dSAlexei Starovoitov offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 17599c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD)) != NULL 17699c55f7dSAlexei Starovoitov 1776c905981SAlexei Starovoitov #define BPF_MAP_CREATE_LAST_FIELD map_flags 17899c55f7dSAlexei Starovoitov /* called via syscall */ 17999c55f7dSAlexei Starovoitov static int map_create(union bpf_attr *attr) 18099c55f7dSAlexei Starovoitov { 18199c55f7dSAlexei Starovoitov struct bpf_map *map; 18299c55f7dSAlexei Starovoitov int err; 18399c55f7dSAlexei Starovoitov 18499c55f7dSAlexei Starovoitov err = CHECK_ATTR(BPF_MAP_CREATE); 18599c55f7dSAlexei Starovoitov if (err) 18699c55f7dSAlexei Starovoitov return -EINVAL; 18799c55f7dSAlexei Starovoitov 18899c55f7dSAlexei Starovoitov /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 18999c55f7dSAlexei Starovoitov map = find_and_alloc_map(attr); 19099c55f7dSAlexei Starovoitov if (IS_ERR(map)) 19199c55f7dSAlexei Starovoitov return PTR_ERR(map); 19299c55f7dSAlexei Starovoitov 19399c55f7dSAlexei Starovoitov atomic_set(&map->refcnt, 1); 194c9da161cSDaniel Borkmann atomic_set(&map->usercnt, 1); 19599c55f7dSAlexei Starovoitov 196aaac3ba9SAlexei Starovoitov err = bpf_map_charge_memlock(map); 197aaac3ba9SAlexei Starovoitov if (err) 19820b2b24fSDaniel Borkmann goto free_map_nouncharge; 199aaac3ba9SAlexei Starovoitov 200aa79781bSDaniel Borkmann err = bpf_map_new_fd(map); 20199c55f7dSAlexei Starovoitov if (err < 0) 20299c55f7dSAlexei Starovoitov /* failed to allocate fd */ 20399c55f7dSAlexei Starovoitov goto free_map; 20499c55f7dSAlexei Starovoitov 20599c55f7dSAlexei Starovoitov return err; 20699c55f7dSAlexei Starovoitov 20799c55f7dSAlexei Starovoitov free_map: 20820b2b24fSDaniel Borkmann bpf_map_uncharge_memlock(map); 20920b2b24fSDaniel Borkmann free_map_nouncharge: 21099c55f7dSAlexei Starovoitov map->ops->map_free(map); 21199c55f7dSAlexei Starovoitov return err; 21299c55f7dSAlexei Starovoitov } 21399c55f7dSAlexei Starovoitov 214db20fd2bSAlexei Starovoitov /* if error is returned, fd is released. 215db20fd2bSAlexei Starovoitov * On success caller should complete fd access with matching fdput() 216db20fd2bSAlexei Starovoitov */ 217c2101297SDaniel Borkmann struct bpf_map *__bpf_map_get(struct fd f) 218db20fd2bSAlexei Starovoitov { 219db20fd2bSAlexei Starovoitov if (!f.file) 220db20fd2bSAlexei Starovoitov return ERR_PTR(-EBADF); 221db20fd2bSAlexei Starovoitov if (f.file->f_op != &bpf_map_fops) { 222db20fd2bSAlexei Starovoitov fdput(f); 223db20fd2bSAlexei Starovoitov return ERR_PTR(-EINVAL); 224db20fd2bSAlexei Starovoitov } 225db20fd2bSAlexei Starovoitov 226c2101297SDaniel Borkmann return f.file->private_data; 227c2101297SDaniel Borkmann } 228c2101297SDaniel Borkmann 22992117d84SAlexei Starovoitov /* prog's and map's refcnt limit */ 23092117d84SAlexei Starovoitov #define BPF_MAX_REFCNT 32768 23192117d84SAlexei Starovoitov 23292117d84SAlexei Starovoitov struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref) 233c9da161cSDaniel Borkmann { 23492117d84SAlexei Starovoitov if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) { 23592117d84SAlexei Starovoitov atomic_dec(&map->refcnt); 23692117d84SAlexei Starovoitov return ERR_PTR(-EBUSY); 23792117d84SAlexei Starovoitov } 238c9da161cSDaniel Borkmann if (uref) 239c9da161cSDaniel Borkmann atomic_inc(&map->usercnt); 24092117d84SAlexei Starovoitov return map; 241c9da161cSDaniel Borkmann } 242c9da161cSDaniel Borkmann 243c9da161cSDaniel Borkmann struct bpf_map *bpf_map_get_with_uref(u32 ufd) 244c2101297SDaniel Borkmann { 245c2101297SDaniel Borkmann struct fd f = fdget(ufd); 246c2101297SDaniel Borkmann struct bpf_map *map; 247c2101297SDaniel Borkmann 248c2101297SDaniel Borkmann map = __bpf_map_get(f); 249c2101297SDaniel Borkmann if (IS_ERR(map)) 250c2101297SDaniel Borkmann return map; 251c2101297SDaniel Borkmann 25292117d84SAlexei Starovoitov map = bpf_map_inc(map, true); 253c2101297SDaniel Borkmann fdput(f); 254db20fd2bSAlexei Starovoitov 255db20fd2bSAlexei Starovoitov return map; 256db20fd2bSAlexei Starovoitov } 257db20fd2bSAlexei Starovoitov 258b8cdc051SAlexei Starovoitov int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 259b8cdc051SAlexei Starovoitov { 260b8cdc051SAlexei Starovoitov return -ENOTSUPP; 261b8cdc051SAlexei Starovoitov } 262b8cdc051SAlexei Starovoitov 263db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 264db20fd2bSAlexei Starovoitov #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value 265db20fd2bSAlexei Starovoitov 266db20fd2bSAlexei Starovoitov static int map_lookup_elem(union bpf_attr *attr) 267db20fd2bSAlexei Starovoitov { 268535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 269535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 270db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 271db20fd2bSAlexei Starovoitov struct bpf_map *map; 2728ebe667cSAlexei Starovoitov void *key, *value, *ptr; 27315a07b33SAlexei Starovoitov u32 value_size; 274592867bfSDaniel Borkmann struct fd f; 275db20fd2bSAlexei Starovoitov int err; 276db20fd2bSAlexei Starovoitov 277db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 278db20fd2bSAlexei Starovoitov return -EINVAL; 279db20fd2bSAlexei Starovoitov 280592867bfSDaniel Borkmann f = fdget(ufd); 281c2101297SDaniel Borkmann map = __bpf_map_get(f); 282db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 283db20fd2bSAlexei Starovoitov return PTR_ERR(map); 284db20fd2bSAlexei Starovoitov 285db20fd2bSAlexei Starovoitov err = -ENOMEM; 286db20fd2bSAlexei Starovoitov key = kmalloc(map->key_size, GFP_USER); 287db20fd2bSAlexei Starovoitov if (!key) 288db20fd2bSAlexei Starovoitov goto err_put; 289db20fd2bSAlexei Starovoitov 290db20fd2bSAlexei Starovoitov err = -EFAULT; 291db20fd2bSAlexei Starovoitov if (copy_from_user(key, ukey, map->key_size) != 0) 292db20fd2bSAlexei Starovoitov goto free_key; 293db20fd2bSAlexei Starovoitov 29415a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 2958f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 29615a07b33SAlexei Starovoitov map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 29715a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 29815a07b33SAlexei Starovoitov else 29915a07b33SAlexei Starovoitov value_size = map->value_size; 30015a07b33SAlexei Starovoitov 3018ebe667cSAlexei Starovoitov err = -ENOMEM; 30215a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 303db20fd2bSAlexei Starovoitov if (!value) 3048ebe667cSAlexei Starovoitov goto free_key; 3058ebe667cSAlexei Starovoitov 3068f844938SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 3078f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 30815a07b33SAlexei Starovoitov err = bpf_percpu_hash_copy(map, key, value); 30915a07b33SAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 31015a07b33SAlexei Starovoitov err = bpf_percpu_array_copy(map, key, value); 311557c0c6eSAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 312557c0c6eSAlexei Starovoitov err = bpf_stackmap_copy(map, key, value); 31315a07b33SAlexei Starovoitov } else { 3148ebe667cSAlexei Starovoitov rcu_read_lock(); 3158ebe667cSAlexei Starovoitov ptr = map->ops->map_lookup_elem(map, key); 3168ebe667cSAlexei Starovoitov if (ptr) 31715a07b33SAlexei Starovoitov memcpy(value, ptr, value_size); 3188ebe667cSAlexei Starovoitov rcu_read_unlock(); 31915a07b33SAlexei Starovoitov err = ptr ? 0 : -ENOENT; 32015a07b33SAlexei Starovoitov } 3218ebe667cSAlexei Starovoitov 32215a07b33SAlexei Starovoitov if (err) 3238ebe667cSAlexei Starovoitov goto free_value; 324db20fd2bSAlexei Starovoitov 325db20fd2bSAlexei Starovoitov err = -EFAULT; 32615a07b33SAlexei Starovoitov if (copy_to_user(uvalue, value, value_size) != 0) 3278ebe667cSAlexei Starovoitov goto free_value; 328db20fd2bSAlexei Starovoitov 329db20fd2bSAlexei Starovoitov err = 0; 330db20fd2bSAlexei Starovoitov 3318ebe667cSAlexei Starovoitov free_value: 3328ebe667cSAlexei Starovoitov kfree(value); 333db20fd2bSAlexei Starovoitov free_key: 334db20fd2bSAlexei Starovoitov kfree(key); 335db20fd2bSAlexei Starovoitov err_put: 336db20fd2bSAlexei Starovoitov fdput(f); 337db20fd2bSAlexei Starovoitov return err; 338db20fd2bSAlexei Starovoitov } 339db20fd2bSAlexei Starovoitov 3403274f520SAlexei Starovoitov #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 341db20fd2bSAlexei Starovoitov 342db20fd2bSAlexei Starovoitov static int map_update_elem(union bpf_attr *attr) 343db20fd2bSAlexei Starovoitov { 344535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 345535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 346db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 347db20fd2bSAlexei Starovoitov struct bpf_map *map; 348db20fd2bSAlexei Starovoitov void *key, *value; 34915a07b33SAlexei Starovoitov u32 value_size; 350592867bfSDaniel Borkmann struct fd f; 351db20fd2bSAlexei Starovoitov int err; 352db20fd2bSAlexei Starovoitov 353db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 354db20fd2bSAlexei Starovoitov return -EINVAL; 355db20fd2bSAlexei Starovoitov 356592867bfSDaniel Borkmann f = fdget(ufd); 357c2101297SDaniel Borkmann map = __bpf_map_get(f); 358db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 359db20fd2bSAlexei Starovoitov return PTR_ERR(map); 360db20fd2bSAlexei Starovoitov 361db20fd2bSAlexei Starovoitov err = -ENOMEM; 362db20fd2bSAlexei Starovoitov key = kmalloc(map->key_size, GFP_USER); 363db20fd2bSAlexei Starovoitov if (!key) 364db20fd2bSAlexei Starovoitov goto err_put; 365db20fd2bSAlexei Starovoitov 366db20fd2bSAlexei Starovoitov err = -EFAULT; 367db20fd2bSAlexei Starovoitov if (copy_from_user(key, ukey, map->key_size) != 0) 368db20fd2bSAlexei Starovoitov goto free_key; 369db20fd2bSAlexei Starovoitov 37015a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 3718f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 37215a07b33SAlexei Starovoitov map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 37315a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 37415a07b33SAlexei Starovoitov else 37515a07b33SAlexei Starovoitov value_size = map->value_size; 37615a07b33SAlexei Starovoitov 377db20fd2bSAlexei Starovoitov err = -ENOMEM; 37815a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 379db20fd2bSAlexei Starovoitov if (!value) 380db20fd2bSAlexei Starovoitov goto free_key; 381db20fd2bSAlexei Starovoitov 382db20fd2bSAlexei Starovoitov err = -EFAULT; 38315a07b33SAlexei Starovoitov if (copy_from_user(value, uvalue, value_size) != 0) 384db20fd2bSAlexei Starovoitov goto free_value; 385db20fd2bSAlexei Starovoitov 386b121d1e7SAlexei Starovoitov /* must increment bpf_prog_active to avoid kprobe+bpf triggering from 387b121d1e7SAlexei Starovoitov * inside bpf map update or delete otherwise deadlocks are possible 388b121d1e7SAlexei Starovoitov */ 389b121d1e7SAlexei Starovoitov preempt_disable(); 390b121d1e7SAlexei Starovoitov __this_cpu_inc(bpf_prog_active); 3918f844938SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 3928f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 39315a07b33SAlexei Starovoitov err = bpf_percpu_hash_update(map, key, value, attr->flags); 39415a07b33SAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 39515a07b33SAlexei Starovoitov err = bpf_percpu_array_update(map, key, value, attr->flags); 396d056a788SDaniel Borkmann } else if (map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || 3974ed8ec52SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_PROG_ARRAY || 3984ed8ec52SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_CGROUP_ARRAY) { 399d056a788SDaniel Borkmann rcu_read_lock(); 400d056a788SDaniel Borkmann err = bpf_fd_array_map_update_elem(map, f.file, key, value, 401d056a788SDaniel Borkmann attr->flags); 402d056a788SDaniel Borkmann rcu_read_unlock(); 40315a07b33SAlexei Starovoitov } else { 404db20fd2bSAlexei Starovoitov rcu_read_lock(); 4053274f520SAlexei Starovoitov err = map->ops->map_update_elem(map, key, value, attr->flags); 406db20fd2bSAlexei Starovoitov rcu_read_unlock(); 40715a07b33SAlexei Starovoitov } 408b121d1e7SAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 409b121d1e7SAlexei Starovoitov preempt_enable(); 410db20fd2bSAlexei Starovoitov 411db20fd2bSAlexei Starovoitov free_value: 412db20fd2bSAlexei Starovoitov kfree(value); 413db20fd2bSAlexei Starovoitov free_key: 414db20fd2bSAlexei Starovoitov kfree(key); 415db20fd2bSAlexei Starovoitov err_put: 416db20fd2bSAlexei Starovoitov fdput(f); 417db20fd2bSAlexei Starovoitov return err; 418db20fd2bSAlexei Starovoitov } 419db20fd2bSAlexei Starovoitov 420db20fd2bSAlexei Starovoitov #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 421db20fd2bSAlexei Starovoitov 422db20fd2bSAlexei Starovoitov static int map_delete_elem(union bpf_attr *attr) 423db20fd2bSAlexei Starovoitov { 424535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 425db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 426db20fd2bSAlexei Starovoitov struct bpf_map *map; 427592867bfSDaniel Borkmann struct fd f; 428db20fd2bSAlexei Starovoitov void *key; 429db20fd2bSAlexei Starovoitov int err; 430db20fd2bSAlexei Starovoitov 431db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 432db20fd2bSAlexei Starovoitov return -EINVAL; 433db20fd2bSAlexei Starovoitov 434592867bfSDaniel Borkmann f = fdget(ufd); 435c2101297SDaniel Borkmann map = __bpf_map_get(f); 436db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 437db20fd2bSAlexei Starovoitov return PTR_ERR(map); 438db20fd2bSAlexei Starovoitov 439db20fd2bSAlexei Starovoitov err = -ENOMEM; 440db20fd2bSAlexei Starovoitov key = kmalloc(map->key_size, GFP_USER); 441db20fd2bSAlexei Starovoitov if (!key) 442db20fd2bSAlexei Starovoitov goto err_put; 443db20fd2bSAlexei Starovoitov 444db20fd2bSAlexei Starovoitov err = -EFAULT; 445db20fd2bSAlexei Starovoitov if (copy_from_user(key, ukey, map->key_size) != 0) 446db20fd2bSAlexei Starovoitov goto free_key; 447db20fd2bSAlexei Starovoitov 448b121d1e7SAlexei Starovoitov preempt_disable(); 449b121d1e7SAlexei Starovoitov __this_cpu_inc(bpf_prog_active); 450db20fd2bSAlexei Starovoitov rcu_read_lock(); 451db20fd2bSAlexei Starovoitov err = map->ops->map_delete_elem(map, key); 452db20fd2bSAlexei Starovoitov rcu_read_unlock(); 453b121d1e7SAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 454b121d1e7SAlexei Starovoitov preempt_enable(); 455db20fd2bSAlexei Starovoitov 456db20fd2bSAlexei Starovoitov free_key: 457db20fd2bSAlexei Starovoitov kfree(key); 458db20fd2bSAlexei Starovoitov err_put: 459db20fd2bSAlexei Starovoitov fdput(f); 460db20fd2bSAlexei Starovoitov return err; 461db20fd2bSAlexei Starovoitov } 462db20fd2bSAlexei Starovoitov 463db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 464db20fd2bSAlexei Starovoitov #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 465db20fd2bSAlexei Starovoitov 466db20fd2bSAlexei Starovoitov static int map_get_next_key(union bpf_attr *attr) 467db20fd2bSAlexei Starovoitov { 468535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 469535e7b4bSMickaël Salaün void __user *unext_key = u64_to_user_ptr(attr->next_key); 470db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 471db20fd2bSAlexei Starovoitov struct bpf_map *map; 472db20fd2bSAlexei Starovoitov void *key, *next_key; 473592867bfSDaniel Borkmann struct fd f; 474db20fd2bSAlexei Starovoitov int err; 475db20fd2bSAlexei Starovoitov 476db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 477db20fd2bSAlexei Starovoitov return -EINVAL; 478db20fd2bSAlexei Starovoitov 479592867bfSDaniel Borkmann f = fdget(ufd); 480c2101297SDaniel Borkmann map = __bpf_map_get(f); 481db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 482db20fd2bSAlexei Starovoitov return PTR_ERR(map); 483db20fd2bSAlexei Starovoitov 484db20fd2bSAlexei Starovoitov err = -ENOMEM; 485db20fd2bSAlexei Starovoitov key = kmalloc(map->key_size, GFP_USER); 486db20fd2bSAlexei Starovoitov if (!key) 487db20fd2bSAlexei Starovoitov goto err_put; 488db20fd2bSAlexei Starovoitov 489db20fd2bSAlexei Starovoitov err = -EFAULT; 490db20fd2bSAlexei Starovoitov if (copy_from_user(key, ukey, map->key_size) != 0) 491db20fd2bSAlexei Starovoitov goto free_key; 492db20fd2bSAlexei Starovoitov 493db20fd2bSAlexei Starovoitov err = -ENOMEM; 494db20fd2bSAlexei Starovoitov next_key = kmalloc(map->key_size, GFP_USER); 495db20fd2bSAlexei Starovoitov if (!next_key) 496db20fd2bSAlexei Starovoitov goto free_key; 497db20fd2bSAlexei Starovoitov 498db20fd2bSAlexei Starovoitov rcu_read_lock(); 499db20fd2bSAlexei Starovoitov err = map->ops->map_get_next_key(map, key, next_key); 500db20fd2bSAlexei Starovoitov rcu_read_unlock(); 501db20fd2bSAlexei Starovoitov if (err) 502db20fd2bSAlexei Starovoitov goto free_next_key; 503db20fd2bSAlexei Starovoitov 504db20fd2bSAlexei Starovoitov err = -EFAULT; 505db20fd2bSAlexei Starovoitov if (copy_to_user(unext_key, next_key, map->key_size) != 0) 506db20fd2bSAlexei Starovoitov goto free_next_key; 507db20fd2bSAlexei Starovoitov 508db20fd2bSAlexei Starovoitov err = 0; 509db20fd2bSAlexei Starovoitov 510db20fd2bSAlexei Starovoitov free_next_key: 511db20fd2bSAlexei Starovoitov kfree(next_key); 512db20fd2bSAlexei Starovoitov free_key: 513db20fd2bSAlexei Starovoitov kfree(key); 514db20fd2bSAlexei Starovoitov err_put: 515db20fd2bSAlexei Starovoitov fdput(f); 516db20fd2bSAlexei Starovoitov return err; 517db20fd2bSAlexei Starovoitov } 518db20fd2bSAlexei Starovoitov 51909756af4SAlexei Starovoitov static LIST_HEAD(bpf_prog_types); 52009756af4SAlexei Starovoitov 52109756af4SAlexei Starovoitov static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 52209756af4SAlexei Starovoitov { 52309756af4SAlexei Starovoitov struct bpf_prog_type_list *tl; 52409756af4SAlexei Starovoitov 52509756af4SAlexei Starovoitov list_for_each_entry(tl, &bpf_prog_types, list_node) { 52609756af4SAlexei Starovoitov if (tl->type == type) { 52709756af4SAlexei Starovoitov prog->aux->ops = tl->ops; 52824701eceSDaniel Borkmann prog->type = type; 52909756af4SAlexei Starovoitov return 0; 53009756af4SAlexei Starovoitov } 53109756af4SAlexei Starovoitov } 53224701eceSDaniel Borkmann 53309756af4SAlexei Starovoitov return -EINVAL; 53409756af4SAlexei Starovoitov } 53509756af4SAlexei Starovoitov 53609756af4SAlexei Starovoitov void bpf_register_prog_type(struct bpf_prog_type_list *tl) 53709756af4SAlexei Starovoitov { 53809756af4SAlexei Starovoitov list_add(&tl->list_node, &bpf_prog_types); 53909756af4SAlexei Starovoitov } 54009756af4SAlexei Starovoitov 5410a542a86SAlexei Starovoitov /* fixup insn->imm field of bpf_call instructions: 5420a542a86SAlexei Starovoitov * if (insn->imm == BPF_FUNC_map_lookup_elem) 5430a542a86SAlexei Starovoitov * insn->imm = bpf_map_lookup_elem - __bpf_call_base; 5440a542a86SAlexei Starovoitov * else if (insn->imm == BPF_FUNC_map_update_elem) 5450a542a86SAlexei Starovoitov * insn->imm = bpf_map_update_elem - __bpf_call_base; 5460a542a86SAlexei Starovoitov * else ... 5470a542a86SAlexei Starovoitov * 5480a542a86SAlexei Starovoitov * this function is called after eBPF program passed verification 5490a542a86SAlexei Starovoitov */ 5500a542a86SAlexei Starovoitov static void fixup_bpf_calls(struct bpf_prog *prog) 5510a542a86SAlexei Starovoitov { 5520a542a86SAlexei Starovoitov const struct bpf_func_proto *fn; 5530a542a86SAlexei Starovoitov int i; 5540a542a86SAlexei Starovoitov 5550a542a86SAlexei Starovoitov for (i = 0; i < prog->len; i++) { 5560a542a86SAlexei Starovoitov struct bpf_insn *insn = &prog->insnsi[i]; 5570a542a86SAlexei Starovoitov 5580a542a86SAlexei Starovoitov if (insn->code == (BPF_JMP | BPF_CALL)) { 5590a542a86SAlexei Starovoitov /* we reach here when program has bpf_call instructions 5600a542a86SAlexei Starovoitov * and it passed bpf_check(), means that 5610a542a86SAlexei Starovoitov * ops->get_func_proto must have been supplied, check it 5620a542a86SAlexei Starovoitov */ 5630a542a86SAlexei Starovoitov BUG_ON(!prog->aux->ops->get_func_proto); 5640a542a86SAlexei Starovoitov 565c46646d0SDaniel Borkmann if (insn->imm == BPF_FUNC_get_route_realm) 566c46646d0SDaniel Borkmann prog->dst_needed = 1; 5673ad00405SDaniel Borkmann if (insn->imm == BPF_FUNC_get_prandom_u32) 5683ad00405SDaniel Borkmann bpf_user_rnd_init_once(); 56904fd61abSAlexei Starovoitov if (insn->imm == BPF_FUNC_tail_call) { 57004fd61abSAlexei Starovoitov /* mark bpf_tail_call as different opcode 57104fd61abSAlexei Starovoitov * to avoid conditional branch in 57204fd61abSAlexei Starovoitov * interpeter for every normal call 57304fd61abSAlexei Starovoitov * and to prevent accidental JITing by 57404fd61abSAlexei Starovoitov * JIT compiler that doesn't support 57504fd61abSAlexei Starovoitov * bpf_tail_call yet 57604fd61abSAlexei Starovoitov */ 57704fd61abSAlexei Starovoitov insn->imm = 0; 57804fd61abSAlexei Starovoitov insn->code |= BPF_X; 57904fd61abSAlexei Starovoitov continue; 58004fd61abSAlexei Starovoitov } 58104fd61abSAlexei Starovoitov 5820a542a86SAlexei Starovoitov fn = prog->aux->ops->get_func_proto(insn->imm); 5830a542a86SAlexei Starovoitov /* all functions that have prototype and verifier allowed 5840a542a86SAlexei Starovoitov * programs to call them, must be real in-kernel functions 5850a542a86SAlexei Starovoitov */ 5860a542a86SAlexei Starovoitov BUG_ON(!fn->func); 5870a542a86SAlexei Starovoitov insn->imm = fn->func - __bpf_call_base; 5880a542a86SAlexei Starovoitov } 5890a542a86SAlexei Starovoitov } 5900a542a86SAlexei Starovoitov } 5910a542a86SAlexei Starovoitov 59209756af4SAlexei Starovoitov /* drop refcnt on maps used by eBPF program and free auxilary data */ 59309756af4SAlexei Starovoitov static void free_used_maps(struct bpf_prog_aux *aux) 59409756af4SAlexei Starovoitov { 59509756af4SAlexei Starovoitov int i; 59609756af4SAlexei Starovoitov 59709756af4SAlexei Starovoitov for (i = 0; i < aux->used_map_cnt; i++) 59809756af4SAlexei Starovoitov bpf_map_put(aux->used_maps[i]); 59909756af4SAlexei Starovoitov 60009756af4SAlexei Starovoitov kfree(aux->used_maps); 60109756af4SAlexei Starovoitov } 60209756af4SAlexei Starovoitov 603aaac3ba9SAlexei Starovoitov static int bpf_prog_charge_memlock(struct bpf_prog *prog) 604aaac3ba9SAlexei Starovoitov { 605aaac3ba9SAlexei Starovoitov struct user_struct *user = get_current_user(); 606aaac3ba9SAlexei Starovoitov unsigned long memlock_limit; 607aaac3ba9SAlexei Starovoitov 608aaac3ba9SAlexei Starovoitov memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 609aaac3ba9SAlexei Starovoitov 610aaac3ba9SAlexei Starovoitov atomic_long_add(prog->pages, &user->locked_vm); 611aaac3ba9SAlexei Starovoitov if (atomic_long_read(&user->locked_vm) > memlock_limit) { 612aaac3ba9SAlexei Starovoitov atomic_long_sub(prog->pages, &user->locked_vm); 613aaac3ba9SAlexei Starovoitov free_uid(user); 614aaac3ba9SAlexei Starovoitov return -EPERM; 615aaac3ba9SAlexei Starovoitov } 616aaac3ba9SAlexei Starovoitov prog->aux->user = user; 617aaac3ba9SAlexei Starovoitov return 0; 618aaac3ba9SAlexei Starovoitov } 619aaac3ba9SAlexei Starovoitov 620aaac3ba9SAlexei Starovoitov static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 621aaac3ba9SAlexei Starovoitov { 622aaac3ba9SAlexei Starovoitov struct user_struct *user = prog->aux->user; 623aaac3ba9SAlexei Starovoitov 624aaac3ba9SAlexei Starovoitov atomic_long_sub(prog->pages, &user->locked_vm); 625aaac3ba9SAlexei Starovoitov free_uid(user); 626aaac3ba9SAlexei Starovoitov } 627aaac3ba9SAlexei Starovoitov 6281aacde3dSDaniel Borkmann static void __bpf_prog_put_rcu(struct rcu_head *rcu) 629abf2e7d6SAlexei Starovoitov { 630abf2e7d6SAlexei Starovoitov struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 631abf2e7d6SAlexei Starovoitov 632abf2e7d6SAlexei Starovoitov free_used_maps(aux); 633aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(aux->prog); 634abf2e7d6SAlexei Starovoitov bpf_prog_free(aux->prog); 635abf2e7d6SAlexei Starovoitov } 636abf2e7d6SAlexei Starovoitov 63709756af4SAlexei Starovoitov void bpf_prog_put(struct bpf_prog *prog) 63809756af4SAlexei Starovoitov { 639e9d8afa9SDaniel Borkmann if (atomic_dec_and_test(&prog->aux->refcnt)) 6401aacde3dSDaniel Borkmann call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 64109756af4SAlexei Starovoitov } 642e2e9b654SDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_put); 64309756af4SAlexei Starovoitov 64409756af4SAlexei Starovoitov static int bpf_prog_release(struct inode *inode, struct file *filp) 64509756af4SAlexei Starovoitov { 64609756af4SAlexei Starovoitov struct bpf_prog *prog = filp->private_data; 64709756af4SAlexei Starovoitov 6481aacde3dSDaniel Borkmann bpf_prog_put(prog); 64909756af4SAlexei Starovoitov return 0; 65009756af4SAlexei Starovoitov } 65109756af4SAlexei Starovoitov 65209756af4SAlexei Starovoitov static const struct file_operations bpf_prog_fops = { 65309756af4SAlexei Starovoitov .release = bpf_prog_release, 65409756af4SAlexei Starovoitov }; 65509756af4SAlexei Starovoitov 656b2197755SDaniel Borkmann int bpf_prog_new_fd(struct bpf_prog *prog) 657aa79781bSDaniel Borkmann { 658aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 659aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC); 660aa79781bSDaniel Borkmann } 661aa79781bSDaniel Borkmann 662113214beSDaniel Borkmann static struct bpf_prog *____bpf_prog_get(struct fd f) 66309756af4SAlexei Starovoitov { 66409756af4SAlexei Starovoitov if (!f.file) 66509756af4SAlexei Starovoitov return ERR_PTR(-EBADF); 66609756af4SAlexei Starovoitov if (f.file->f_op != &bpf_prog_fops) { 66709756af4SAlexei Starovoitov fdput(f); 66809756af4SAlexei Starovoitov return ERR_PTR(-EINVAL); 66909756af4SAlexei Starovoitov } 67009756af4SAlexei Starovoitov 671c2101297SDaniel Borkmann return f.file->private_data; 67209756af4SAlexei Starovoitov } 67309756af4SAlexei Starovoitov 67459d3656dSBrenden Blanco struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i) 67592117d84SAlexei Starovoitov { 67659d3656dSBrenden Blanco if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) { 67759d3656dSBrenden Blanco atomic_sub(i, &prog->aux->refcnt); 67892117d84SAlexei Starovoitov return ERR_PTR(-EBUSY); 67992117d84SAlexei Starovoitov } 68092117d84SAlexei Starovoitov return prog; 68192117d84SAlexei Starovoitov } 68259d3656dSBrenden Blanco EXPORT_SYMBOL_GPL(bpf_prog_add); 68359d3656dSBrenden Blanco 684c540594fSDaniel Borkmann void bpf_prog_sub(struct bpf_prog *prog, int i) 685c540594fSDaniel Borkmann { 686c540594fSDaniel Borkmann /* Only to be used for undoing previous bpf_prog_add() in some 687c540594fSDaniel Borkmann * error path. We still know that another entity in our call 688c540594fSDaniel Borkmann * path holds a reference to the program, thus atomic_sub() can 689c540594fSDaniel Borkmann * be safely used in such cases! 690c540594fSDaniel Borkmann */ 691c540594fSDaniel Borkmann WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0); 692c540594fSDaniel Borkmann } 693c540594fSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_sub); 694c540594fSDaniel Borkmann 69559d3656dSBrenden Blanco struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog) 69659d3656dSBrenden Blanco { 69759d3656dSBrenden Blanco return bpf_prog_add(prog, 1); 69859d3656dSBrenden Blanco } 699*97bc402dSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_inc); 70092117d84SAlexei Starovoitov 701113214beSDaniel Borkmann static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *type) 70209756af4SAlexei Starovoitov { 70309756af4SAlexei Starovoitov struct fd f = fdget(ufd); 70409756af4SAlexei Starovoitov struct bpf_prog *prog; 70509756af4SAlexei Starovoitov 706113214beSDaniel Borkmann prog = ____bpf_prog_get(f); 70709756af4SAlexei Starovoitov if (IS_ERR(prog)) 70809756af4SAlexei Starovoitov return prog; 709113214beSDaniel Borkmann if (type && prog->type != *type) { 710113214beSDaniel Borkmann prog = ERR_PTR(-EINVAL); 711113214beSDaniel Borkmann goto out; 712113214beSDaniel Borkmann } 71309756af4SAlexei Starovoitov 71492117d84SAlexei Starovoitov prog = bpf_prog_inc(prog); 715113214beSDaniel Borkmann out: 71609756af4SAlexei Starovoitov fdput(f); 71709756af4SAlexei Starovoitov return prog; 71809756af4SAlexei Starovoitov } 719113214beSDaniel Borkmann 720113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get(u32 ufd) 721113214beSDaniel Borkmann { 722113214beSDaniel Borkmann return __bpf_prog_get(ufd, NULL); 723113214beSDaniel Borkmann } 724113214beSDaniel Borkmann 725113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type) 726113214beSDaniel Borkmann { 727113214beSDaniel Borkmann return __bpf_prog_get(ufd, &type); 728113214beSDaniel Borkmann } 729113214beSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_get_type); 73009756af4SAlexei Starovoitov 73109756af4SAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 7322541517cSAlexei Starovoitov #define BPF_PROG_LOAD_LAST_FIELD kern_version 73309756af4SAlexei Starovoitov 73409756af4SAlexei Starovoitov static int bpf_prog_load(union bpf_attr *attr) 73509756af4SAlexei Starovoitov { 73609756af4SAlexei Starovoitov enum bpf_prog_type type = attr->prog_type; 73709756af4SAlexei Starovoitov struct bpf_prog *prog; 73809756af4SAlexei Starovoitov int err; 73909756af4SAlexei Starovoitov char license[128]; 74009756af4SAlexei Starovoitov bool is_gpl; 74109756af4SAlexei Starovoitov 74209756af4SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_LOAD)) 74309756af4SAlexei Starovoitov return -EINVAL; 74409756af4SAlexei Starovoitov 74509756af4SAlexei Starovoitov /* copy eBPF program license from user space */ 746535e7b4bSMickaël Salaün if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 74709756af4SAlexei Starovoitov sizeof(license) - 1) < 0) 74809756af4SAlexei Starovoitov return -EFAULT; 74909756af4SAlexei Starovoitov license[sizeof(license) - 1] = 0; 75009756af4SAlexei Starovoitov 75109756af4SAlexei Starovoitov /* eBPF programs must be GPL compatible to use GPL-ed functions */ 75209756af4SAlexei Starovoitov is_gpl = license_is_gpl_compatible(license); 75309756af4SAlexei Starovoitov 75409756af4SAlexei Starovoitov if (attr->insn_cnt >= BPF_MAXINSNS) 75509756af4SAlexei Starovoitov return -EINVAL; 75609756af4SAlexei Starovoitov 7572541517cSAlexei Starovoitov if (type == BPF_PROG_TYPE_KPROBE && 7582541517cSAlexei Starovoitov attr->kern_version != LINUX_VERSION_CODE) 7592541517cSAlexei Starovoitov return -EINVAL; 7602541517cSAlexei Starovoitov 7611be7f75dSAlexei Starovoitov if (type != BPF_PROG_TYPE_SOCKET_FILTER && !capable(CAP_SYS_ADMIN)) 7621be7f75dSAlexei Starovoitov return -EPERM; 7631be7f75dSAlexei Starovoitov 76409756af4SAlexei Starovoitov /* plain bpf_prog allocation */ 76509756af4SAlexei Starovoitov prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 76609756af4SAlexei Starovoitov if (!prog) 76709756af4SAlexei Starovoitov return -ENOMEM; 76809756af4SAlexei Starovoitov 769aaac3ba9SAlexei Starovoitov err = bpf_prog_charge_memlock(prog); 770aaac3ba9SAlexei Starovoitov if (err) 771aaac3ba9SAlexei Starovoitov goto free_prog_nouncharge; 772aaac3ba9SAlexei Starovoitov 77309756af4SAlexei Starovoitov prog->len = attr->insn_cnt; 77409756af4SAlexei Starovoitov 77509756af4SAlexei Starovoitov err = -EFAULT; 776535e7b4bSMickaël Salaün if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 77709756af4SAlexei Starovoitov prog->len * sizeof(struct bpf_insn)) != 0) 77809756af4SAlexei Starovoitov goto free_prog; 77909756af4SAlexei Starovoitov 78009756af4SAlexei Starovoitov prog->orig_prog = NULL; 781a91263d5SDaniel Borkmann prog->jited = 0; 78209756af4SAlexei Starovoitov 78309756af4SAlexei Starovoitov atomic_set(&prog->aux->refcnt, 1); 784a91263d5SDaniel Borkmann prog->gpl_compatible = is_gpl ? 1 : 0; 78509756af4SAlexei Starovoitov 78609756af4SAlexei Starovoitov /* find program type: socket_filter vs tracing_filter */ 78709756af4SAlexei Starovoitov err = find_prog_type(type, prog); 78809756af4SAlexei Starovoitov if (err < 0) 78909756af4SAlexei Starovoitov goto free_prog; 79009756af4SAlexei Starovoitov 79109756af4SAlexei Starovoitov /* run eBPF verifier */ 7929bac3d6dSAlexei Starovoitov err = bpf_check(&prog, attr); 79309756af4SAlexei Starovoitov if (err < 0) 79409756af4SAlexei Starovoitov goto free_used_maps; 79509756af4SAlexei Starovoitov 7960a542a86SAlexei Starovoitov /* fixup BPF_CALL->imm field */ 7970a542a86SAlexei Starovoitov fixup_bpf_calls(prog); 7980a542a86SAlexei Starovoitov 79909756af4SAlexei Starovoitov /* eBPF program is ready to be JITed */ 800d1c55ab5SDaniel Borkmann prog = bpf_prog_select_runtime(prog, &err); 80104fd61abSAlexei Starovoitov if (err < 0) 80204fd61abSAlexei Starovoitov goto free_used_maps; 80309756af4SAlexei Starovoitov 804aa79781bSDaniel Borkmann err = bpf_prog_new_fd(prog); 80509756af4SAlexei Starovoitov if (err < 0) 80609756af4SAlexei Starovoitov /* failed to allocate fd */ 80709756af4SAlexei Starovoitov goto free_used_maps; 80809756af4SAlexei Starovoitov 80909756af4SAlexei Starovoitov return err; 81009756af4SAlexei Starovoitov 81109756af4SAlexei Starovoitov free_used_maps: 81209756af4SAlexei Starovoitov free_used_maps(prog->aux); 81309756af4SAlexei Starovoitov free_prog: 814aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(prog); 815aaac3ba9SAlexei Starovoitov free_prog_nouncharge: 81609756af4SAlexei Starovoitov bpf_prog_free(prog); 81709756af4SAlexei Starovoitov return err; 81809756af4SAlexei Starovoitov } 81909756af4SAlexei Starovoitov 820b2197755SDaniel Borkmann #define BPF_OBJ_LAST_FIELD bpf_fd 821b2197755SDaniel Borkmann 822b2197755SDaniel Borkmann static int bpf_obj_pin(const union bpf_attr *attr) 823b2197755SDaniel Borkmann { 824b2197755SDaniel Borkmann if (CHECK_ATTR(BPF_OBJ)) 825b2197755SDaniel Borkmann return -EINVAL; 826b2197755SDaniel Borkmann 827535e7b4bSMickaël Salaün return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 828b2197755SDaniel Borkmann } 829b2197755SDaniel Borkmann 830b2197755SDaniel Borkmann static int bpf_obj_get(const union bpf_attr *attr) 831b2197755SDaniel Borkmann { 832b2197755SDaniel Borkmann if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0) 833b2197755SDaniel Borkmann return -EINVAL; 834b2197755SDaniel Borkmann 835535e7b4bSMickaël Salaün return bpf_obj_get_user(u64_to_user_ptr(attr->pathname)); 836b2197755SDaniel Borkmann } 837b2197755SDaniel Borkmann 83899c55f7dSAlexei Starovoitov SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 83999c55f7dSAlexei Starovoitov { 84099c55f7dSAlexei Starovoitov union bpf_attr attr = {}; 84199c55f7dSAlexei Starovoitov int err; 84299c55f7dSAlexei Starovoitov 8431be7f75dSAlexei Starovoitov if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled) 84499c55f7dSAlexei Starovoitov return -EPERM; 84599c55f7dSAlexei Starovoitov 84699c55f7dSAlexei Starovoitov if (!access_ok(VERIFY_READ, uattr, 1)) 84799c55f7dSAlexei Starovoitov return -EFAULT; 84899c55f7dSAlexei Starovoitov 84999c55f7dSAlexei Starovoitov if (size > PAGE_SIZE) /* silly large */ 85099c55f7dSAlexei Starovoitov return -E2BIG; 85199c55f7dSAlexei Starovoitov 85299c55f7dSAlexei Starovoitov /* If we're handed a bigger struct than we know of, 85399c55f7dSAlexei Starovoitov * ensure all the unknown bits are 0 - i.e. new 85499c55f7dSAlexei Starovoitov * user-space does not rely on any kernel feature 85599c55f7dSAlexei Starovoitov * extensions we dont know about yet. 85699c55f7dSAlexei Starovoitov */ 85799c55f7dSAlexei Starovoitov if (size > sizeof(attr)) { 85899c55f7dSAlexei Starovoitov unsigned char __user *addr; 85999c55f7dSAlexei Starovoitov unsigned char __user *end; 86099c55f7dSAlexei Starovoitov unsigned char val; 86199c55f7dSAlexei Starovoitov 86299c55f7dSAlexei Starovoitov addr = (void __user *)uattr + sizeof(attr); 86399c55f7dSAlexei Starovoitov end = (void __user *)uattr + size; 86499c55f7dSAlexei Starovoitov 86599c55f7dSAlexei Starovoitov for (; addr < end; addr++) { 86699c55f7dSAlexei Starovoitov err = get_user(val, addr); 86799c55f7dSAlexei Starovoitov if (err) 86899c55f7dSAlexei Starovoitov return err; 86999c55f7dSAlexei Starovoitov if (val) 87099c55f7dSAlexei Starovoitov return -E2BIG; 87199c55f7dSAlexei Starovoitov } 87299c55f7dSAlexei Starovoitov size = sizeof(attr); 87399c55f7dSAlexei Starovoitov } 87499c55f7dSAlexei Starovoitov 87599c55f7dSAlexei Starovoitov /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 87699c55f7dSAlexei Starovoitov if (copy_from_user(&attr, uattr, size) != 0) 87799c55f7dSAlexei Starovoitov return -EFAULT; 87899c55f7dSAlexei Starovoitov 87999c55f7dSAlexei Starovoitov switch (cmd) { 88099c55f7dSAlexei Starovoitov case BPF_MAP_CREATE: 88199c55f7dSAlexei Starovoitov err = map_create(&attr); 88299c55f7dSAlexei Starovoitov break; 883db20fd2bSAlexei Starovoitov case BPF_MAP_LOOKUP_ELEM: 884db20fd2bSAlexei Starovoitov err = map_lookup_elem(&attr); 885db20fd2bSAlexei Starovoitov break; 886db20fd2bSAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 887db20fd2bSAlexei Starovoitov err = map_update_elem(&attr); 888db20fd2bSAlexei Starovoitov break; 889db20fd2bSAlexei Starovoitov case BPF_MAP_DELETE_ELEM: 890db20fd2bSAlexei Starovoitov err = map_delete_elem(&attr); 891db20fd2bSAlexei Starovoitov break; 892db20fd2bSAlexei Starovoitov case BPF_MAP_GET_NEXT_KEY: 893db20fd2bSAlexei Starovoitov err = map_get_next_key(&attr); 894db20fd2bSAlexei Starovoitov break; 89509756af4SAlexei Starovoitov case BPF_PROG_LOAD: 89609756af4SAlexei Starovoitov err = bpf_prog_load(&attr); 89709756af4SAlexei Starovoitov break; 898b2197755SDaniel Borkmann case BPF_OBJ_PIN: 899b2197755SDaniel Borkmann err = bpf_obj_pin(&attr); 900b2197755SDaniel Borkmann break; 901b2197755SDaniel Borkmann case BPF_OBJ_GET: 902b2197755SDaniel Borkmann err = bpf_obj_get(&attr); 903b2197755SDaniel Borkmann break; 90499c55f7dSAlexei Starovoitov default: 90599c55f7dSAlexei Starovoitov err = -EINVAL; 90699c55f7dSAlexei Starovoitov break; 90799c55f7dSAlexei Starovoitov } 90899c55f7dSAlexei Starovoitov 90999c55f7dSAlexei Starovoitov return err; 91099c55f7dSAlexei Starovoitov } 911