199c55f7dSAlexei Starovoitov /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 299c55f7dSAlexei Starovoitov * 399c55f7dSAlexei Starovoitov * This program is free software; you can redistribute it and/or 499c55f7dSAlexei Starovoitov * modify it under the terms of version 2 of the GNU General Public 599c55f7dSAlexei Starovoitov * License as published by the Free Software Foundation. 699c55f7dSAlexei Starovoitov * 799c55f7dSAlexei Starovoitov * This program is distributed in the hope that it will be useful, but 899c55f7dSAlexei Starovoitov * WITHOUT ANY WARRANTY; without even the implied warranty of 999c55f7dSAlexei Starovoitov * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 1099c55f7dSAlexei Starovoitov * General Public License for more details. 1199c55f7dSAlexei Starovoitov */ 1299c55f7dSAlexei Starovoitov #include <linux/bpf.h> 13a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 1499c55f7dSAlexei Starovoitov #include <linux/syscalls.h> 1599c55f7dSAlexei Starovoitov #include <linux/slab.h> 163f07c014SIngo Molnar #include <linux/sched/signal.h> 17d407bd25SDaniel Borkmann #include <linux/vmalloc.h> 18d407bd25SDaniel Borkmann #include <linux/mmzone.h> 1999c55f7dSAlexei Starovoitov #include <linux/anon_inodes.h> 20db20fd2bSAlexei Starovoitov #include <linux/file.h> 2109756af4SAlexei Starovoitov #include <linux/license.h> 2209756af4SAlexei Starovoitov #include <linux/filter.h> 232541517cSAlexei Starovoitov #include <linux/version.h> 24535e7b4bSMickaël Salaün #include <linux/kernel.h> 25dc4bb0e2SMartin KaFai Lau #include <linux/idr.h> 2699c55f7dSAlexei Starovoitov 2714dc6f04SMartin KaFai Lau #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ 2814dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 2914dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 3014dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 3114dc6f04SMartin KaFai Lau #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 3214dc6f04SMartin KaFai Lau #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map)) 3314dc6f04SMartin KaFai Lau 34b121d1e7SAlexei Starovoitov DEFINE_PER_CPU(int, bpf_prog_active); 35dc4bb0e2SMartin KaFai Lau static DEFINE_IDR(prog_idr); 36dc4bb0e2SMartin KaFai Lau static DEFINE_SPINLOCK(prog_idr_lock); 37f3f1c054SMartin KaFai Lau static DEFINE_IDR(map_idr); 38f3f1c054SMartin KaFai Lau static DEFINE_SPINLOCK(map_idr_lock); 39b121d1e7SAlexei Starovoitov 401be7f75dSAlexei Starovoitov int sysctl_unprivileged_bpf_disabled __read_mostly; 411be7f75dSAlexei Starovoitov 4240077e0cSJohannes Berg static const struct bpf_map_ops * const bpf_map_types[] = { 4340077e0cSJohannes Berg #define BPF_PROG_TYPE(_id, _ops) 4440077e0cSJohannes Berg #define BPF_MAP_TYPE(_id, _ops) \ 4540077e0cSJohannes Berg [_id] = &_ops, 4640077e0cSJohannes Berg #include <linux/bpf_types.h> 4740077e0cSJohannes Berg #undef BPF_PROG_TYPE 4840077e0cSJohannes Berg #undef BPF_MAP_TYPE 4940077e0cSJohannes Berg }; 5099c55f7dSAlexei Starovoitov 5199c55f7dSAlexei Starovoitov static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 5299c55f7dSAlexei Starovoitov { 5399c55f7dSAlexei Starovoitov struct bpf_map *map; 5499c55f7dSAlexei Starovoitov 5540077e0cSJohannes Berg if (attr->map_type >= ARRAY_SIZE(bpf_map_types) || 5640077e0cSJohannes Berg !bpf_map_types[attr->map_type]) 5740077e0cSJohannes Berg return ERR_PTR(-EINVAL); 5840077e0cSJohannes Berg 5940077e0cSJohannes Berg map = bpf_map_types[attr->map_type]->map_alloc(attr); 6099c55f7dSAlexei Starovoitov if (IS_ERR(map)) 6199c55f7dSAlexei Starovoitov return map; 6240077e0cSJohannes Berg map->ops = bpf_map_types[attr->map_type]; 6399c55f7dSAlexei Starovoitov map->map_type = attr->map_type; 6499c55f7dSAlexei Starovoitov return map; 6599c55f7dSAlexei Starovoitov } 6699c55f7dSAlexei Starovoitov 67d407bd25SDaniel Borkmann void *bpf_map_area_alloc(size_t size) 68d407bd25SDaniel Borkmann { 69d407bd25SDaniel Borkmann /* We definitely need __GFP_NORETRY, so OOM killer doesn't 70d407bd25SDaniel Borkmann * trigger under memory pressure as we really just want to 71d407bd25SDaniel Borkmann * fail instead. 72d407bd25SDaniel Borkmann */ 73d407bd25SDaniel Borkmann const gfp_t flags = __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO; 74d407bd25SDaniel Borkmann void *area; 75d407bd25SDaniel Borkmann 76d407bd25SDaniel Borkmann if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 77d407bd25SDaniel Borkmann area = kmalloc(size, GFP_USER | flags); 78d407bd25SDaniel Borkmann if (area != NULL) 79d407bd25SDaniel Borkmann return area; 80d407bd25SDaniel Borkmann } 81d407bd25SDaniel Borkmann 8219809c2dSMichal Hocko return __vmalloc(size, GFP_KERNEL | flags, PAGE_KERNEL); 83d407bd25SDaniel Borkmann } 84d407bd25SDaniel Borkmann 85d407bd25SDaniel Borkmann void bpf_map_area_free(void *area) 86d407bd25SDaniel Borkmann { 87d407bd25SDaniel Borkmann kvfree(area); 88d407bd25SDaniel Borkmann } 89d407bd25SDaniel Borkmann 906c905981SAlexei Starovoitov int bpf_map_precharge_memlock(u32 pages) 916c905981SAlexei Starovoitov { 926c905981SAlexei Starovoitov struct user_struct *user = get_current_user(); 936c905981SAlexei Starovoitov unsigned long memlock_limit, cur; 946c905981SAlexei Starovoitov 956c905981SAlexei Starovoitov memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 966c905981SAlexei Starovoitov cur = atomic_long_read(&user->locked_vm); 976c905981SAlexei Starovoitov free_uid(user); 986c905981SAlexei Starovoitov if (cur + pages > memlock_limit) 996c905981SAlexei Starovoitov return -EPERM; 1006c905981SAlexei Starovoitov return 0; 1016c905981SAlexei Starovoitov } 1026c905981SAlexei Starovoitov 103aaac3ba9SAlexei Starovoitov static int bpf_map_charge_memlock(struct bpf_map *map) 104aaac3ba9SAlexei Starovoitov { 105aaac3ba9SAlexei Starovoitov struct user_struct *user = get_current_user(); 106aaac3ba9SAlexei Starovoitov unsigned long memlock_limit; 107aaac3ba9SAlexei Starovoitov 108aaac3ba9SAlexei Starovoitov memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 109aaac3ba9SAlexei Starovoitov 110aaac3ba9SAlexei Starovoitov atomic_long_add(map->pages, &user->locked_vm); 111aaac3ba9SAlexei Starovoitov 112aaac3ba9SAlexei Starovoitov if (atomic_long_read(&user->locked_vm) > memlock_limit) { 113aaac3ba9SAlexei Starovoitov atomic_long_sub(map->pages, &user->locked_vm); 114aaac3ba9SAlexei Starovoitov free_uid(user); 115aaac3ba9SAlexei Starovoitov return -EPERM; 116aaac3ba9SAlexei Starovoitov } 117aaac3ba9SAlexei Starovoitov map->user = user; 118aaac3ba9SAlexei Starovoitov return 0; 119aaac3ba9SAlexei Starovoitov } 120aaac3ba9SAlexei Starovoitov 121aaac3ba9SAlexei Starovoitov static void bpf_map_uncharge_memlock(struct bpf_map *map) 122aaac3ba9SAlexei Starovoitov { 123aaac3ba9SAlexei Starovoitov struct user_struct *user = map->user; 124aaac3ba9SAlexei Starovoitov 125aaac3ba9SAlexei Starovoitov atomic_long_sub(map->pages, &user->locked_vm); 126aaac3ba9SAlexei Starovoitov free_uid(user); 127aaac3ba9SAlexei Starovoitov } 128aaac3ba9SAlexei Starovoitov 129f3f1c054SMartin KaFai Lau static int bpf_map_alloc_id(struct bpf_map *map) 130f3f1c054SMartin KaFai Lau { 131f3f1c054SMartin KaFai Lau int id; 132f3f1c054SMartin KaFai Lau 133f3f1c054SMartin KaFai Lau spin_lock_bh(&map_idr_lock); 134f3f1c054SMartin KaFai Lau id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 135f3f1c054SMartin KaFai Lau if (id > 0) 136f3f1c054SMartin KaFai Lau map->id = id; 137f3f1c054SMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 138f3f1c054SMartin KaFai Lau 139f3f1c054SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 140f3f1c054SMartin KaFai Lau return -ENOSPC; 141f3f1c054SMartin KaFai Lau 142f3f1c054SMartin KaFai Lau return id > 0 ? 0 : id; 143f3f1c054SMartin KaFai Lau } 144f3f1c054SMartin KaFai Lau 145bd5f5f4eSMartin KaFai Lau static void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) 146f3f1c054SMartin KaFai Lau { 147bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 148f3f1c054SMartin KaFai Lau spin_lock_bh(&map_idr_lock); 149bd5f5f4eSMartin KaFai Lau else 150bd5f5f4eSMartin KaFai Lau __acquire(&map_idr_lock); 151bd5f5f4eSMartin KaFai Lau 152f3f1c054SMartin KaFai Lau idr_remove(&map_idr, map->id); 153bd5f5f4eSMartin KaFai Lau 154bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 155f3f1c054SMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 156bd5f5f4eSMartin KaFai Lau else 157bd5f5f4eSMartin KaFai Lau __release(&map_idr_lock); 158f3f1c054SMartin KaFai Lau } 159f3f1c054SMartin KaFai Lau 16099c55f7dSAlexei Starovoitov /* called from workqueue */ 16199c55f7dSAlexei Starovoitov static void bpf_map_free_deferred(struct work_struct *work) 16299c55f7dSAlexei Starovoitov { 16399c55f7dSAlexei Starovoitov struct bpf_map *map = container_of(work, struct bpf_map, work); 16499c55f7dSAlexei Starovoitov 165aaac3ba9SAlexei Starovoitov bpf_map_uncharge_memlock(map); 16699c55f7dSAlexei Starovoitov /* implementation dependent freeing */ 16799c55f7dSAlexei Starovoitov map->ops->map_free(map); 16899c55f7dSAlexei Starovoitov } 16999c55f7dSAlexei Starovoitov 170c9da161cSDaniel Borkmann static void bpf_map_put_uref(struct bpf_map *map) 171c9da161cSDaniel Borkmann { 172c9da161cSDaniel Borkmann if (atomic_dec_and_test(&map->usercnt)) { 173c9da161cSDaniel Borkmann if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) 174c9da161cSDaniel Borkmann bpf_fd_array_map_clear(map); 175c9da161cSDaniel Borkmann } 176c9da161cSDaniel Borkmann } 177c9da161cSDaniel Borkmann 17899c55f7dSAlexei Starovoitov /* decrement map refcnt and schedule it for freeing via workqueue 17999c55f7dSAlexei Starovoitov * (unrelying map implementation ops->map_free() might sleep) 18099c55f7dSAlexei Starovoitov */ 181bd5f5f4eSMartin KaFai Lau static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) 18299c55f7dSAlexei Starovoitov { 18399c55f7dSAlexei Starovoitov if (atomic_dec_and_test(&map->refcnt)) { 18434ad5580SMartin KaFai Lau /* bpf_map_free_id() must be called first */ 185bd5f5f4eSMartin KaFai Lau bpf_map_free_id(map, do_idr_lock); 18699c55f7dSAlexei Starovoitov INIT_WORK(&map->work, bpf_map_free_deferred); 18799c55f7dSAlexei Starovoitov schedule_work(&map->work); 18899c55f7dSAlexei Starovoitov } 18999c55f7dSAlexei Starovoitov } 19099c55f7dSAlexei Starovoitov 191bd5f5f4eSMartin KaFai Lau void bpf_map_put(struct bpf_map *map) 192bd5f5f4eSMartin KaFai Lau { 193bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, true); 194bd5f5f4eSMartin KaFai Lau } 195bd5f5f4eSMartin KaFai Lau 196c9da161cSDaniel Borkmann void bpf_map_put_with_uref(struct bpf_map *map) 197c9da161cSDaniel Borkmann { 198c9da161cSDaniel Borkmann bpf_map_put_uref(map); 199c9da161cSDaniel Borkmann bpf_map_put(map); 200c9da161cSDaniel Borkmann } 201c9da161cSDaniel Borkmann 20299c55f7dSAlexei Starovoitov static int bpf_map_release(struct inode *inode, struct file *filp) 20399c55f7dSAlexei Starovoitov { 20461d1b6a4SDaniel Borkmann struct bpf_map *map = filp->private_data; 20561d1b6a4SDaniel Borkmann 20661d1b6a4SDaniel Borkmann if (map->ops->map_release) 20761d1b6a4SDaniel Borkmann map->ops->map_release(map, filp); 20861d1b6a4SDaniel Borkmann 20961d1b6a4SDaniel Borkmann bpf_map_put_with_uref(map); 21099c55f7dSAlexei Starovoitov return 0; 21199c55f7dSAlexei Starovoitov } 21299c55f7dSAlexei Starovoitov 213f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 214f99bf205SDaniel Borkmann static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 215f99bf205SDaniel Borkmann { 216f99bf205SDaniel Borkmann const struct bpf_map *map = filp->private_data; 21721116b70SDaniel Borkmann const struct bpf_array *array; 21821116b70SDaniel Borkmann u32 owner_prog_type = 0; 219*9780c0abSDaniel Borkmann u32 owner_jited = 0; 22021116b70SDaniel Borkmann 22121116b70SDaniel Borkmann if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 22221116b70SDaniel Borkmann array = container_of(map, struct bpf_array, map); 22321116b70SDaniel Borkmann owner_prog_type = array->owner_prog_type; 224*9780c0abSDaniel Borkmann owner_jited = array->owner_jited; 22521116b70SDaniel Borkmann } 226f99bf205SDaniel Borkmann 227f99bf205SDaniel Borkmann seq_printf(m, 228f99bf205SDaniel Borkmann "map_type:\t%u\n" 229f99bf205SDaniel Borkmann "key_size:\t%u\n" 230f99bf205SDaniel Borkmann "value_size:\t%u\n" 231322cea2fSDaniel Borkmann "max_entries:\t%u\n" 23221116b70SDaniel Borkmann "map_flags:\t%#x\n" 23321116b70SDaniel Borkmann "memlock:\t%llu\n", 234f99bf205SDaniel Borkmann map->map_type, 235f99bf205SDaniel Borkmann map->key_size, 236f99bf205SDaniel Borkmann map->value_size, 237322cea2fSDaniel Borkmann map->max_entries, 23821116b70SDaniel Borkmann map->map_flags, 23921116b70SDaniel Borkmann map->pages * 1ULL << PAGE_SHIFT); 24021116b70SDaniel Borkmann 241*9780c0abSDaniel Borkmann if (owner_prog_type) { 24221116b70SDaniel Borkmann seq_printf(m, "owner_prog_type:\t%u\n", 24321116b70SDaniel Borkmann owner_prog_type); 244*9780c0abSDaniel Borkmann seq_printf(m, "owner_jited:\t%u\n", 245*9780c0abSDaniel Borkmann owner_jited); 246*9780c0abSDaniel Borkmann } 247f99bf205SDaniel Borkmann } 248f99bf205SDaniel Borkmann #endif 249f99bf205SDaniel Borkmann 25099c55f7dSAlexei Starovoitov static const struct file_operations bpf_map_fops = { 251f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 252f99bf205SDaniel Borkmann .show_fdinfo = bpf_map_show_fdinfo, 253f99bf205SDaniel Borkmann #endif 25499c55f7dSAlexei Starovoitov .release = bpf_map_release, 25599c55f7dSAlexei Starovoitov }; 25699c55f7dSAlexei Starovoitov 257b2197755SDaniel Borkmann int bpf_map_new_fd(struct bpf_map *map) 258aa79781bSDaniel Borkmann { 259aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 260aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC); 261aa79781bSDaniel Borkmann } 262aa79781bSDaniel Borkmann 26399c55f7dSAlexei Starovoitov /* helper macro to check that unused fields 'union bpf_attr' are zero */ 26499c55f7dSAlexei Starovoitov #define CHECK_ATTR(CMD) \ 26599c55f7dSAlexei Starovoitov memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 26699c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD), 0, \ 26799c55f7dSAlexei Starovoitov sizeof(*attr) - \ 26899c55f7dSAlexei Starovoitov offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 26999c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD)) != NULL 27099c55f7dSAlexei Starovoitov 27156f668dfSMartin KaFai Lau #define BPF_MAP_CREATE_LAST_FIELD inner_map_fd 27299c55f7dSAlexei Starovoitov /* called via syscall */ 27399c55f7dSAlexei Starovoitov static int map_create(union bpf_attr *attr) 27499c55f7dSAlexei Starovoitov { 27599c55f7dSAlexei Starovoitov struct bpf_map *map; 27699c55f7dSAlexei Starovoitov int err; 27799c55f7dSAlexei Starovoitov 27899c55f7dSAlexei Starovoitov err = CHECK_ATTR(BPF_MAP_CREATE); 27999c55f7dSAlexei Starovoitov if (err) 28099c55f7dSAlexei Starovoitov return -EINVAL; 28199c55f7dSAlexei Starovoitov 28299c55f7dSAlexei Starovoitov /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 28399c55f7dSAlexei Starovoitov map = find_and_alloc_map(attr); 28499c55f7dSAlexei Starovoitov if (IS_ERR(map)) 28599c55f7dSAlexei Starovoitov return PTR_ERR(map); 28699c55f7dSAlexei Starovoitov 28799c55f7dSAlexei Starovoitov atomic_set(&map->refcnt, 1); 288c9da161cSDaniel Borkmann atomic_set(&map->usercnt, 1); 28999c55f7dSAlexei Starovoitov 290aaac3ba9SAlexei Starovoitov err = bpf_map_charge_memlock(map); 291aaac3ba9SAlexei Starovoitov if (err) 29220b2b24fSDaniel Borkmann goto free_map_nouncharge; 293aaac3ba9SAlexei Starovoitov 294f3f1c054SMartin KaFai Lau err = bpf_map_alloc_id(map); 295f3f1c054SMartin KaFai Lau if (err) 296f3f1c054SMartin KaFai Lau goto free_map; 297f3f1c054SMartin KaFai Lau 298aa79781bSDaniel Borkmann err = bpf_map_new_fd(map); 299bd5f5f4eSMartin KaFai Lau if (err < 0) { 300bd5f5f4eSMartin KaFai Lau /* failed to allocate fd. 301bd5f5f4eSMartin KaFai Lau * bpf_map_put() is needed because the above 302bd5f5f4eSMartin KaFai Lau * bpf_map_alloc_id() has published the map 303bd5f5f4eSMartin KaFai Lau * to the userspace and the userspace may 304bd5f5f4eSMartin KaFai Lau * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 305bd5f5f4eSMartin KaFai Lau */ 306bd5f5f4eSMartin KaFai Lau bpf_map_put(map); 307bd5f5f4eSMartin KaFai Lau return err; 308bd5f5f4eSMartin KaFai Lau } 30999c55f7dSAlexei Starovoitov 310a67edbf4SDaniel Borkmann trace_bpf_map_create(map, err); 31199c55f7dSAlexei Starovoitov return err; 31299c55f7dSAlexei Starovoitov 31399c55f7dSAlexei Starovoitov free_map: 31420b2b24fSDaniel Borkmann bpf_map_uncharge_memlock(map); 31520b2b24fSDaniel Borkmann free_map_nouncharge: 31699c55f7dSAlexei Starovoitov map->ops->map_free(map); 31799c55f7dSAlexei Starovoitov return err; 31899c55f7dSAlexei Starovoitov } 31999c55f7dSAlexei Starovoitov 320db20fd2bSAlexei Starovoitov /* if error is returned, fd is released. 321db20fd2bSAlexei Starovoitov * On success caller should complete fd access with matching fdput() 322db20fd2bSAlexei Starovoitov */ 323c2101297SDaniel Borkmann struct bpf_map *__bpf_map_get(struct fd f) 324db20fd2bSAlexei Starovoitov { 325db20fd2bSAlexei Starovoitov if (!f.file) 326db20fd2bSAlexei Starovoitov return ERR_PTR(-EBADF); 327db20fd2bSAlexei Starovoitov if (f.file->f_op != &bpf_map_fops) { 328db20fd2bSAlexei Starovoitov fdput(f); 329db20fd2bSAlexei Starovoitov return ERR_PTR(-EINVAL); 330db20fd2bSAlexei Starovoitov } 331db20fd2bSAlexei Starovoitov 332c2101297SDaniel Borkmann return f.file->private_data; 333c2101297SDaniel Borkmann } 334c2101297SDaniel Borkmann 33592117d84SAlexei Starovoitov /* prog's and map's refcnt limit */ 33692117d84SAlexei Starovoitov #define BPF_MAX_REFCNT 32768 33792117d84SAlexei Starovoitov 33892117d84SAlexei Starovoitov struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref) 339c9da161cSDaniel Borkmann { 34092117d84SAlexei Starovoitov if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) { 34192117d84SAlexei Starovoitov atomic_dec(&map->refcnt); 34292117d84SAlexei Starovoitov return ERR_PTR(-EBUSY); 34392117d84SAlexei Starovoitov } 344c9da161cSDaniel Borkmann if (uref) 345c9da161cSDaniel Borkmann atomic_inc(&map->usercnt); 34692117d84SAlexei Starovoitov return map; 347c9da161cSDaniel Borkmann } 348c9da161cSDaniel Borkmann 349c9da161cSDaniel Borkmann struct bpf_map *bpf_map_get_with_uref(u32 ufd) 350c2101297SDaniel Borkmann { 351c2101297SDaniel Borkmann struct fd f = fdget(ufd); 352c2101297SDaniel Borkmann struct bpf_map *map; 353c2101297SDaniel Borkmann 354c2101297SDaniel Borkmann map = __bpf_map_get(f); 355c2101297SDaniel Borkmann if (IS_ERR(map)) 356c2101297SDaniel Borkmann return map; 357c2101297SDaniel Borkmann 35892117d84SAlexei Starovoitov map = bpf_map_inc(map, true); 359c2101297SDaniel Borkmann fdput(f); 360db20fd2bSAlexei Starovoitov 361db20fd2bSAlexei Starovoitov return map; 362db20fd2bSAlexei Starovoitov } 363db20fd2bSAlexei Starovoitov 364bd5f5f4eSMartin KaFai Lau /* map_idr_lock should have been held */ 365bd5f5f4eSMartin KaFai Lau static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map, 366bd5f5f4eSMartin KaFai Lau bool uref) 367bd5f5f4eSMartin KaFai Lau { 368bd5f5f4eSMartin KaFai Lau int refold; 369bd5f5f4eSMartin KaFai Lau 370bd5f5f4eSMartin KaFai Lau refold = __atomic_add_unless(&map->refcnt, 1, 0); 371bd5f5f4eSMartin KaFai Lau 372bd5f5f4eSMartin KaFai Lau if (refold >= BPF_MAX_REFCNT) { 373bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, false); 374bd5f5f4eSMartin KaFai Lau return ERR_PTR(-EBUSY); 375bd5f5f4eSMartin KaFai Lau } 376bd5f5f4eSMartin KaFai Lau 377bd5f5f4eSMartin KaFai Lau if (!refold) 378bd5f5f4eSMartin KaFai Lau return ERR_PTR(-ENOENT); 379bd5f5f4eSMartin KaFai Lau 380bd5f5f4eSMartin KaFai Lau if (uref) 381bd5f5f4eSMartin KaFai Lau atomic_inc(&map->usercnt); 382bd5f5f4eSMartin KaFai Lau 383bd5f5f4eSMartin KaFai Lau return map; 384bd5f5f4eSMartin KaFai Lau } 385bd5f5f4eSMartin KaFai Lau 386b8cdc051SAlexei Starovoitov int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 387b8cdc051SAlexei Starovoitov { 388b8cdc051SAlexei Starovoitov return -ENOTSUPP; 389b8cdc051SAlexei Starovoitov } 390b8cdc051SAlexei Starovoitov 391db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 392db20fd2bSAlexei Starovoitov #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value 393db20fd2bSAlexei Starovoitov 394db20fd2bSAlexei Starovoitov static int map_lookup_elem(union bpf_attr *attr) 395db20fd2bSAlexei Starovoitov { 396535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 397535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 398db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 399db20fd2bSAlexei Starovoitov struct bpf_map *map; 4008ebe667cSAlexei Starovoitov void *key, *value, *ptr; 40115a07b33SAlexei Starovoitov u32 value_size; 402592867bfSDaniel Borkmann struct fd f; 403db20fd2bSAlexei Starovoitov int err; 404db20fd2bSAlexei Starovoitov 405db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 406db20fd2bSAlexei Starovoitov return -EINVAL; 407db20fd2bSAlexei Starovoitov 408592867bfSDaniel Borkmann f = fdget(ufd); 409c2101297SDaniel Borkmann map = __bpf_map_get(f); 410db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 411db20fd2bSAlexei Starovoitov return PTR_ERR(map); 412db20fd2bSAlexei Starovoitov 413db20fd2bSAlexei Starovoitov err = -ENOMEM; 414db20fd2bSAlexei Starovoitov key = kmalloc(map->key_size, GFP_USER); 415db20fd2bSAlexei Starovoitov if (!key) 416db20fd2bSAlexei Starovoitov goto err_put; 417db20fd2bSAlexei Starovoitov 418db20fd2bSAlexei Starovoitov err = -EFAULT; 419db20fd2bSAlexei Starovoitov if (copy_from_user(key, ukey, map->key_size) != 0) 420db20fd2bSAlexei Starovoitov goto free_key; 421db20fd2bSAlexei Starovoitov 42215a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 4238f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 42415a07b33SAlexei Starovoitov map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 42515a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 42614dc6f04SMartin KaFai Lau else if (IS_FD_MAP(map)) 42714dc6f04SMartin KaFai Lau value_size = sizeof(u32); 42815a07b33SAlexei Starovoitov else 42915a07b33SAlexei Starovoitov value_size = map->value_size; 43015a07b33SAlexei Starovoitov 4318ebe667cSAlexei Starovoitov err = -ENOMEM; 43215a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 433db20fd2bSAlexei Starovoitov if (!value) 4348ebe667cSAlexei Starovoitov goto free_key; 4358ebe667cSAlexei Starovoitov 4368f844938SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 4378f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 43815a07b33SAlexei Starovoitov err = bpf_percpu_hash_copy(map, key, value); 43915a07b33SAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 44015a07b33SAlexei Starovoitov err = bpf_percpu_array_copy(map, key, value); 441557c0c6eSAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 442557c0c6eSAlexei Starovoitov err = bpf_stackmap_copy(map, key, value); 44314dc6f04SMartin KaFai Lau } else if (IS_FD_ARRAY(map)) { 44414dc6f04SMartin KaFai Lau err = bpf_fd_array_map_lookup_elem(map, key, value); 44514dc6f04SMartin KaFai Lau } else if (IS_FD_HASH(map)) { 44614dc6f04SMartin KaFai Lau err = bpf_fd_htab_map_lookup_elem(map, key, value); 44715a07b33SAlexei Starovoitov } else { 4488ebe667cSAlexei Starovoitov rcu_read_lock(); 4498ebe667cSAlexei Starovoitov ptr = map->ops->map_lookup_elem(map, key); 4508ebe667cSAlexei Starovoitov if (ptr) 45115a07b33SAlexei Starovoitov memcpy(value, ptr, value_size); 4528ebe667cSAlexei Starovoitov rcu_read_unlock(); 45315a07b33SAlexei Starovoitov err = ptr ? 0 : -ENOENT; 45415a07b33SAlexei Starovoitov } 4558ebe667cSAlexei Starovoitov 45615a07b33SAlexei Starovoitov if (err) 4578ebe667cSAlexei Starovoitov goto free_value; 458db20fd2bSAlexei Starovoitov 459db20fd2bSAlexei Starovoitov err = -EFAULT; 46015a07b33SAlexei Starovoitov if (copy_to_user(uvalue, value, value_size) != 0) 4618ebe667cSAlexei Starovoitov goto free_value; 462db20fd2bSAlexei Starovoitov 463a67edbf4SDaniel Borkmann trace_bpf_map_lookup_elem(map, ufd, key, value); 464db20fd2bSAlexei Starovoitov err = 0; 465db20fd2bSAlexei Starovoitov 4668ebe667cSAlexei Starovoitov free_value: 4678ebe667cSAlexei Starovoitov kfree(value); 468db20fd2bSAlexei Starovoitov free_key: 469db20fd2bSAlexei Starovoitov kfree(key); 470db20fd2bSAlexei Starovoitov err_put: 471db20fd2bSAlexei Starovoitov fdput(f); 472db20fd2bSAlexei Starovoitov return err; 473db20fd2bSAlexei Starovoitov } 474db20fd2bSAlexei Starovoitov 4753274f520SAlexei Starovoitov #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 476db20fd2bSAlexei Starovoitov 477db20fd2bSAlexei Starovoitov static int map_update_elem(union bpf_attr *attr) 478db20fd2bSAlexei Starovoitov { 479535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 480535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 481db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 482db20fd2bSAlexei Starovoitov struct bpf_map *map; 483db20fd2bSAlexei Starovoitov void *key, *value; 48415a07b33SAlexei Starovoitov u32 value_size; 485592867bfSDaniel Borkmann struct fd f; 486db20fd2bSAlexei Starovoitov int err; 487db20fd2bSAlexei Starovoitov 488db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 489db20fd2bSAlexei Starovoitov return -EINVAL; 490db20fd2bSAlexei Starovoitov 491592867bfSDaniel Borkmann f = fdget(ufd); 492c2101297SDaniel Borkmann map = __bpf_map_get(f); 493db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 494db20fd2bSAlexei Starovoitov return PTR_ERR(map); 495db20fd2bSAlexei Starovoitov 496db20fd2bSAlexei Starovoitov err = -ENOMEM; 497db20fd2bSAlexei Starovoitov key = kmalloc(map->key_size, GFP_USER); 498db20fd2bSAlexei Starovoitov if (!key) 499db20fd2bSAlexei Starovoitov goto err_put; 500db20fd2bSAlexei Starovoitov 501db20fd2bSAlexei Starovoitov err = -EFAULT; 502db20fd2bSAlexei Starovoitov if (copy_from_user(key, ukey, map->key_size) != 0) 503db20fd2bSAlexei Starovoitov goto free_key; 504db20fd2bSAlexei Starovoitov 50515a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 5068f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 50715a07b33SAlexei Starovoitov map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 50815a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 50915a07b33SAlexei Starovoitov else 51015a07b33SAlexei Starovoitov value_size = map->value_size; 51115a07b33SAlexei Starovoitov 512db20fd2bSAlexei Starovoitov err = -ENOMEM; 51315a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 514db20fd2bSAlexei Starovoitov if (!value) 515db20fd2bSAlexei Starovoitov goto free_key; 516db20fd2bSAlexei Starovoitov 517db20fd2bSAlexei Starovoitov err = -EFAULT; 51815a07b33SAlexei Starovoitov if (copy_from_user(value, uvalue, value_size) != 0) 519db20fd2bSAlexei Starovoitov goto free_value; 520db20fd2bSAlexei Starovoitov 521b121d1e7SAlexei Starovoitov /* must increment bpf_prog_active to avoid kprobe+bpf triggering from 522b121d1e7SAlexei Starovoitov * inside bpf map update or delete otherwise deadlocks are possible 523b121d1e7SAlexei Starovoitov */ 524b121d1e7SAlexei Starovoitov preempt_disable(); 525b121d1e7SAlexei Starovoitov __this_cpu_inc(bpf_prog_active); 5268f844938SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 5278f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 52815a07b33SAlexei Starovoitov err = bpf_percpu_hash_update(map, key, value, attr->flags); 52915a07b33SAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 53015a07b33SAlexei Starovoitov err = bpf_percpu_array_update(map, key, value, attr->flags); 531d056a788SDaniel Borkmann } else if (map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || 5324ed8ec52SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_PROG_ARRAY || 53356f668dfSMartin KaFai Lau map->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || 53456f668dfSMartin KaFai Lau map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) { 535d056a788SDaniel Borkmann rcu_read_lock(); 536d056a788SDaniel Borkmann err = bpf_fd_array_map_update_elem(map, f.file, key, value, 537d056a788SDaniel Borkmann attr->flags); 538d056a788SDaniel Borkmann rcu_read_unlock(); 539bcc6b1b7SMartin KaFai Lau } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 540bcc6b1b7SMartin KaFai Lau rcu_read_lock(); 541bcc6b1b7SMartin KaFai Lau err = bpf_fd_htab_map_update_elem(map, f.file, key, value, 542bcc6b1b7SMartin KaFai Lau attr->flags); 543bcc6b1b7SMartin KaFai Lau rcu_read_unlock(); 54415a07b33SAlexei Starovoitov } else { 545db20fd2bSAlexei Starovoitov rcu_read_lock(); 5463274f520SAlexei Starovoitov err = map->ops->map_update_elem(map, key, value, attr->flags); 547db20fd2bSAlexei Starovoitov rcu_read_unlock(); 54815a07b33SAlexei Starovoitov } 549b121d1e7SAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 550b121d1e7SAlexei Starovoitov preempt_enable(); 551db20fd2bSAlexei Starovoitov 552a67edbf4SDaniel Borkmann if (!err) 553a67edbf4SDaniel Borkmann trace_bpf_map_update_elem(map, ufd, key, value); 554db20fd2bSAlexei Starovoitov free_value: 555db20fd2bSAlexei Starovoitov kfree(value); 556db20fd2bSAlexei Starovoitov free_key: 557db20fd2bSAlexei Starovoitov kfree(key); 558db20fd2bSAlexei Starovoitov err_put: 559db20fd2bSAlexei Starovoitov fdput(f); 560db20fd2bSAlexei Starovoitov return err; 561db20fd2bSAlexei Starovoitov } 562db20fd2bSAlexei Starovoitov 563db20fd2bSAlexei Starovoitov #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 564db20fd2bSAlexei Starovoitov 565db20fd2bSAlexei Starovoitov static int map_delete_elem(union bpf_attr *attr) 566db20fd2bSAlexei Starovoitov { 567535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 568db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 569db20fd2bSAlexei Starovoitov struct bpf_map *map; 570592867bfSDaniel Borkmann struct fd f; 571db20fd2bSAlexei Starovoitov void *key; 572db20fd2bSAlexei Starovoitov int err; 573db20fd2bSAlexei Starovoitov 574db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 575db20fd2bSAlexei Starovoitov return -EINVAL; 576db20fd2bSAlexei Starovoitov 577592867bfSDaniel Borkmann f = fdget(ufd); 578c2101297SDaniel Borkmann map = __bpf_map_get(f); 579db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 580db20fd2bSAlexei Starovoitov return PTR_ERR(map); 581db20fd2bSAlexei Starovoitov 582db20fd2bSAlexei Starovoitov err = -ENOMEM; 583db20fd2bSAlexei Starovoitov key = kmalloc(map->key_size, GFP_USER); 584db20fd2bSAlexei Starovoitov if (!key) 585db20fd2bSAlexei Starovoitov goto err_put; 586db20fd2bSAlexei Starovoitov 587db20fd2bSAlexei Starovoitov err = -EFAULT; 588db20fd2bSAlexei Starovoitov if (copy_from_user(key, ukey, map->key_size) != 0) 589db20fd2bSAlexei Starovoitov goto free_key; 590db20fd2bSAlexei Starovoitov 591b121d1e7SAlexei Starovoitov preempt_disable(); 592b121d1e7SAlexei Starovoitov __this_cpu_inc(bpf_prog_active); 593db20fd2bSAlexei Starovoitov rcu_read_lock(); 594db20fd2bSAlexei Starovoitov err = map->ops->map_delete_elem(map, key); 595db20fd2bSAlexei Starovoitov rcu_read_unlock(); 596b121d1e7SAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 597b121d1e7SAlexei Starovoitov preempt_enable(); 598db20fd2bSAlexei Starovoitov 599a67edbf4SDaniel Borkmann if (!err) 600a67edbf4SDaniel Borkmann trace_bpf_map_delete_elem(map, ufd, key); 601db20fd2bSAlexei Starovoitov free_key: 602db20fd2bSAlexei Starovoitov kfree(key); 603db20fd2bSAlexei Starovoitov err_put: 604db20fd2bSAlexei Starovoitov fdput(f); 605db20fd2bSAlexei Starovoitov return err; 606db20fd2bSAlexei Starovoitov } 607db20fd2bSAlexei Starovoitov 608db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 609db20fd2bSAlexei Starovoitov #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 610db20fd2bSAlexei Starovoitov 611db20fd2bSAlexei Starovoitov static int map_get_next_key(union bpf_attr *attr) 612db20fd2bSAlexei Starovoitov { 613535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 614535e7b4bSMickaël Salaün void __user *unext_key = u64_to_user_ptr(attr->next_key); 615db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 616db20fd2bSAlexei Starovoitov struct bpf_map *map; 617db20fd2bSAlexei Starovoitov void *key, *next_key; 618592867bfSDaniel Borkmann struct fd f; 619db20fd2bSAlexei Starovoitov int err; 620db20fd2bSAlexei Starovoitov 621db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 622db20fd2bSAlexei Starovoitov return -EINVAL; 623db20fd2bSAlexei Starovoitov 624592867bfSDaniel Borkmann f = fdget(ufd); 625c2101297SDaniel Borkmann map = __bpf_map_get(f); 626db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 627db20fd2bSAlexei Starovoitov return PTR_ERR(map); 628db20fd2bSAlexei Starovoitov 6298fe45924STeng Qin if (ukey) { 630db20fd2bSAlexei Starovoitov err = -ENOMEM; 631db20fd2bSAlexei Starovoitov key = kmalloc(map->key_size, GFP_USER); 632db20fd2bSAlexei Starovoitov if (!key) 633db20fd2bSAlexei Starovoitov goto err_put; 634db20fd2bSAlexei Starovoitov 635db20fd2bSAlexei Starovoitov err = -EFAULT; 636db20fd2bSAlexei Starovoitov if (copy_from_user(key, ukey, map->key_size) != 0) 637db20fd2bSAlexei Starovoitov goto free_key; 6388fe45924STeng Qin } else { 6398fe45924STeng Qin key = NULL; 6408fe45924STeng Qin } 641db20fd2bSAlexei Starovoitov 642db20fd2bSAlexei Starovoitov err = -ENOMEM; 643db20fd2bSAlexei Starovoitov next_key = kmalloc(map->key_size, GFP_USER); 644db20fd2bSAlexei Starovoitov if (!next_key) 645db20fd2bSAlexei Starovoitov goto free_key; 646db20fd2bSAlexei Starovoitov 647db20fd2bSAlexei Starovoitov rcu_read_lock(); 648db20fd2bSAlexei Starovoitov err = map->ops->map_get_next_key(map, key, next_key); 649db20fd2bSAlexei Starovoitov rcu_read_unlock(); 650db20fd2bSAlexei Starovoitov if (err) 651db20fd2bSAlexei Starovoitov goto free_next_key; 652db20fd2bSAlexei Starovoitov 653db20fd2bSAlexei Starovoitov err = -EFAULT; 654db20fd2bSAlexei Starovoitov if (copy_to_user(unext_key, next_key, map->key_size) != 0) 655db20fd2bSAlexei Starovoitov goto free_next_key; 656db20fd2bSAlexei Starovoitov 657a67edbf4SDaniel Borkmann trace_bpf_map_next_key(map, ufd, key, next_key); 658db20fd2bSAlexei Starovoitov err = 0; 659db20fd2bSAlexei Starovoitov 660db20fd2bSAlexei Starovoitov free_next_key: 661db20fd2bSAlexei Starovoitov kfree(next_key); 662db20fd2bSAlexei Starovoitov free_key: 663db20fd2bSAlexei Starovoitov kfree(key); 664db20fd2bSAlexei Starovoitov err_put: 665db20fd2bSAlexei Starovoitov fdput(f); 666db20fd2bSAlexei Starovoitov return err; 667db20fd2bSAlexei Starovoitov } 668db20fd2bSAlexei Starovoitov 669be9370a7SJohannes Berg static const struct bpf_verifier_ops * const bpf_prog_types[] = { 670be9370a7SJohannes Berg #define BPF_PROG_TYPE(_id, _ops) \ 671be9370a7SJohannes Berg [_id] = &_ops, 67240077e0cSJohannes Berg #define BPF_MAP_TYPE(_id, _ops) 673be9370a7SJohannes Berg #include <linux/bpf_types.h> 674be9370a7SJohannes Berg #undef BPF_PROG_TYPE 67540077e0cSJohannes Berg #undef BPF_MAP_TYPE 676be9370a7SJohannes Berg }; 67709756af4SAlexei Starovoitov 67809756af4SAlexei Starovoitov static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 67909756af4SAlexei Starovoitov { 680be9370a7SJohannes Berg if (type >= ARRAY_SIZE(bpf_prog_types) || !bpf_prog_types[type]) 681be9370a7SJohannes Berg return -EINVAL; 68209756af4SAlexei Starovoitov 683be9370a7SJohannes Berg prog->aux->ops = bpf_prog_types[type]; 68424701eceSDaniel Borkmann prog->type = type; 68509756af4SAlexei Starovoitov return 0; 68609756af4SAlexei Starovoitov } 68709756af4SAlexei Starovoitov 68809756af4SAlexei Starovoitov /* drop refcnt on maps used by eBPF program and free auxilary data */ 68909756af4SAlexei Starovoitov static void free_used_maps(struct bpf_prog_aux *aux) 69009756af4SAlexei Starovoitov { 69109756af4SAlexei Starovoitov int i; 69209756af4SAlexei Starovoitov 69309756af4SAlexei Starovoitov for (i = 0; i < aux->used_map_cnt; i++) 69409756af4SAlexei Starovoitov bpf_map_put(aux->used_maps[i]); 69509756af4SAlexei Starovoitov 69609756af4SAlexei Starovoitov kfree(aux->used_maps); 69709756af4SAlexei Starovoitov } 69809756af4SAlexei Starovoitov 6995ccb071eSDaniel Borkmann int __bpf_prog_charge(struct user_struct *user, u32 pages) 7005ccb071eSDaniel Borkmann { 7015ccb071eSDaniel Borkmann unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 7025ccb071eSDaniel Borkmann unsigned long user_bufs; 7035ccb071eSDaniel Borkmann 7045ccb071eSDaniel Borkmann if (user) { 7055ccb071eSDaniel Borkmann user_bufs = atomic_long_add_return(pages, &user->locked_vm); 7065ccb071eSDaniel Borkmann if (user_bufs > memlock_limit) { 7075ccb071eSDaniel Borkmann atomic_long_sub(pages, &user->locked_vm); 7085ccb071eSDaniel Borkmann return -EPERM; 7095ccb071eSDaniel Borkmann } 7105ccb071eSDaniel Borkmann } 7115ccb071eSDaniel Borkmann 7125ccb071eSDaniel Borkmann return 0; 7135ccb071eSDaniel Borkmann } 7145ccb071eSDaniel Borkmann 7155ccb071eSDaniel Borkmann void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 7165ccb071eSDaniel Borkmann { 7175ccb071eSDaniel Borkmann if (user) 7185ccb071eSDaniel Borkmann atomic_long_sub(pages, &user->locked_vm); 7195ccb071eSDaniel Borkmann } 7205ccb071eSDaniel Borkmann 721aaac3ba9SAlexei Starovoitov static int bpf_prog_charge_memlock(struct bpf_prog *prog) 722aaac3ba9SAlexei Starovoitov { 723aaac3ba9SAlexei Starovoitov struct user_struct *user = get_current_user(); 7245ccb071eSDaniel Borkmann int ret; 725aaac3ba9SAlexei Starovoitov 7265ccb071eSDaniel Borkmann ret = __bpf_prog_charge(user, prog->pages); 7275ccb071eSDaniel Borkmann if (ret) { 728aaac3ba9SAlexei Starovoitov free_uid(user); 7295ccb071eSDaniel Borkmann return ret; 730aaac3ba9SAlexei Starovoitov } 7315ccb071eSDaniel Borkmann 732aaac3ba9SAlexei Starovoitov prog->aux->user = user; 733aaac3ba9SAlexei Starovoitov return 0; 734aaac3ba9SAlexei Starovoitov } 735aaac3ba9SAlexei Starovoitov 736aaac3ba9SAlexei Starovoitov static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 737aaac3ba9SAlexei Starovoitov { 738aaac3ba9SAlexei Starovoitov struct user_struct *user = prog->aux->user; 739aaac3ba9SAlexei Starovoitov 7405ccb071eSDaniel Borkmann __bpf_prog_uncharge(user, prog->pages); 741aaac3ba9SAlexei Starovoitov free_uid(user); 742aaac3ba9SAlexei Starovoitov } 743aaac3ba9SAlexei Starovoitov 744dc4bb0e2SMartin KaFai Lau static int bpf_prog_alloc_id(struct bpf_prog *prog) 745dc4bb0e2SMartin KaFai Lau { 746dc4bb0e2SMartin KaFai Lau int id; 747dc4bb0e2SMartin KaFai Lau 748dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 749dc4bb0e2SMartin KaFai Lau id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 750dc4bb0e2SMartin KaFai Lau if (id > 0) 751dc4bb0e2SMartin KaFai Lau prog->aux->id = id; 752dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 753dc4bb0e2SMartin KaFai Lau 754dc4bb0e2SMartin KaFai Lau /* id is in [1, INT_MAX) */ 755dc4bb0e2SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 756dc4bb0e2SMartin KaFai Lau return -ENOSPC; 757dc4bb0e2SMartin KaFai Lau 758dc4bb0e2SMartin KaFai Lau return id > 0 ? 0 : id; 759dc4bb0e2SMartin KaFai Lau } 760dc4bb0e2SMartin KaFai Lau 761b16d9aa4SMartin KaFai Lau static void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) 762dc4bb0e2SMartin KaFai Lau { 763dc4bb0e2SMartin KaFai Lau /* cBPF to eBPF migrations are currently not in the idr store. */ 764dc4bb0e2SMartin KaFai Lau if (!prog->aux->id) 765dc4bb0e2SMartin KaFai Lau return; 766dc4bb0e2SMartin KaFai Lau 767b16d9aa4SMartin KaFai Lau if (do_idr_lock) 768dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 769b16d9aa4SMartin KaFai Lau else 770b16d9aa4SMartin KaFai Lau __acquire(&prog_idr_lock); 771b16d9aa4SMartin KaFai Lau 772dc4bb0e2SMartin KaFai Lau idr_remove(&prog_idr, prog->aux->id); 773b16d9aa4SMartin KaFai Lau 774b16d9aa4SMartin KaFai Lau if (do_idr_lock) 775dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 776b16d9aa4SMartin KaFai Lau else 777b16d9aa4SMartin KaFai Lau __release(&prog_idr_lock); 778dc4bb0e2SMartin KaFai Lau } 779dc4bb0e2SMartin KaFai Lau 7801aacde3dSDaniel Borkmann static void __bpf_prog_put_rcu(struct rcu_head *rcu) 781abf2e7d6SAlexei Starovoitov { 782abf2e7d6SAlexei Starovoitov struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 783abf2e7d6SAlexei Starovoitov 784abf2e7d6SAlexei Starovoitov free_used_maps(aux); 785aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(aux->prog); 786abf2e7d6SAlexei Starovoitov bpf_prog_free(aux->prog); 787abf2e7d6SAlexei Starovoitov } 788abf2e7d6SAlexei Starovoitov 789b16d9aa4SMartin KaFai Lau static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 79009756af4SAlexei Starovoitov { 791a67edbf4SDaniel Borkmann if (atomic_dec_and_test(&prog->aux->refcnt)) { 792a67edbf4SDaniel Borkmann trace_bpf_prog_put_rcu(prog); 79334ad5580SMartin KaFai Lau /* bpf_prog_free_id() must be called first */ 794b16d9aa4SMartin KaFai Lau bpf_prog_free_id(prog, do_idr_lock); 79574451e66SDaniel Borkmann bpf_prog_kallsyms_del(prog); 7961aacde3dSDaniel Borkmann call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 79709756af4SAlexei Starovoitov } 798a67edbf4SDaniel Borkmann } 799b16d9aa4SMartin KaFai Lau 800b16d9aa4SMartin KaFai Lau void bpf_prog_put(struct bpf_prog *prog) 801b16d9aa4SMartin KaFai Lau { 802b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, true); 803b16d9aa4SMartin KaFai Lau } 804e2e9b654SDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_put); 80509756af4SAlexei Starovoitov 80609756af4SAlexei Starovoitov static int bpf_prog_release(struct inode *inode, struct file *filp) 80709756af4SAlexei Starovoitov { 80809756af4SAlexei Starovoitov struct bpf_prog *prog = filp->private_data; 80909756af4SAlexei Starovoitov 8101aacde3dSDaniel Borkmann bpf_prog_put(prog); 81109756af4SAlexei Starovoitov return 0; 81209756af4SAlexei Starovoitov } 81309756af4SAlexei Starovoitov 8147bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 8157bd509e3SDaniel Borkmann static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 8167bd509e3SDaniel Borkmann { 8177bd509e3SDaniel Borkmann const struct bpf_prog *prog = filp->private_data; 818f1f7714eSDaniel Borkmann char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 8197bd509e3SDaniel Borkmann 820f1f7714eSDaniel Borkmann bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 8217bd509e3SDaniel Borkmann seq_printf(m, 8227bd509e3SDaniel Borkmann "prog_type:\t%u\n" 8237bd509e3SDaniel Borkmann "prog_jited:\t%u\n" 824f1f7714eSDaniel Borkmann "prog_tag:\t%s\n" 8257bd509e3SDaniel Borkmann "memlock:\t%llu\n", 8267bd509e3SDaniel Borkmann prog->type, 8277bd509e3SDaniel Borkmann prog->jited, 828f1f7714eSDaniel Borkmann prog_tag, 8297bd509e3SDaniel Borkmann prog->pages * 1ULL << PAGE_SHIFT); 8307bd509e3SDaniel Borkmann } 8317bd509e3SDaniel Borkmann #endif 8327bd509e3SDaniel Borkmann 83309756af4SAlexei Starovoitov static const struct file_operations bpf_prog_fops = { 8347bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 8357bd509e3SDaniel Borkmann .show_fdinfo = bpf_prog_show_fdinfo, 8367bd509e3SDaniel Borkmann #endif 83709756af4SAlexei Starovoitov .release = bpf_prog_release, 83809756af4SAlexei Starovoitov }; 83909756af4SAlexei Starovoitov 840b2197755SDaniel Borkmann int bpf_prog_new_fd(struct bpf_prog *prog) 841aa79781bSDaniel Borkmann { 842aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 843aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC); 844aa79781bSDaniel Borkmann } 845aa79781bSDaniel Borkmann 846113214beSDaniel Borkmann static struct bpf_prog *____bpf_prog_get(struct fd f) 84709756af4SAlexei Starovoitov { 84809756af4SAlexei Starovoitov if (!f.file) 84909756af4SAlexei Starovoitov return ERR_PTR(-EBADF); 85009756af4SAlexei Starovoitov if (f.file->f_op != &bpf_prog_fops) { 85109756af4SAlexei Starovoitov fdput(f); 85209756af4SAlexei Starovoitov return ERR_PTR(-EINVAL); 85309756af4SAlexei Starovoitov } 85409756af4SAlexei Starovoitov 855c2101297SDaniel Borkmann return f.file->private_data; 85609756af4SAlexei Starovoitov } 85709756af4SAlexei Starovoitov 85859d3656dSBrenden Blanco struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i) 85992117d84SAlexei Starovoitov { 86059d3656dSBrenden Blanco if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) { 86159d3656dSBrenden Blanco atomic_sub(i, &prog->aux->refcnt); 86292117d84SAlexei Starovoitov return ERR_PTR(-EBUSY); 86392117d84SAlexei Starovoitov } 86492117d84SAlexei Starovoitov return prog; 86592117d84SAlexei Starovoitov } 86659d3656dSBrenden Blanco EXPORT_SYMBOL_GPL(bpf_prog_add); 86759d3656dSBrenden Blanco 868c540594fSDaniel Borkmann void bpf_prog_sub(struct bpf_prog *prog, int i) 869c540594fSDaniel Borkmann { 870c540594fSDaniel Borkmann /* Only to be used for undoing previous bpf_prog_add() in some 871c540594fSDaniel Borkmann * error path. We still know that another entity in our call 872c540594fSDaniel Borkmann * path holds a reference to the program, thus atomic_sub() can 873c540594fSDaniel Borkmann * be safely used in such cases! 874c540594fSDaniel Borkmann */ 875c540594fSDaniel Borkmann WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0); 876c540594fSDaniel Borkmann } 877c540594fSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_sub); 878c540594fSDaniel Borkmann 87959d3656dSBrenden Blanco struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog) 88059d3656dSBrenden Blanco { 88159d3656dSBrenden Blanco return bpf_prog_add(prog, 1); 88259d3656dSBrenden Blanco } 88397bc402dSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_inc); 88492117d84SAlexei Starovoitov 885b16d9aa4SMartin KaFai Lau /* prog_idr_lock should have been held */ 886b16d9aa4SMartin KaFai Lau static struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 887b16d9aa4SMartin KaFai Lau { 888b16d9aa4SMartin KaFai Lau int refold; 889b16d9aa4SMartin KaFai Lau 890b16d9aa4SMartin KaFai Lau refold = __atomic_add_unless(&prog->aux->refcnt, 1, 0); 891b16d9aa4SMartin KaFai Lau 892b16d9aa4SMartin KaFai Lau if (refold >= BPF_MAX_REFCNT) { 893b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, false); 894b16d9aa4SMartin KaFai Lau return ERR_PTR(-EBUSY); 895b16d9aa4SMartin KaFai Lau } 896b16d9aa4SMartin KaFai Lau 897b16d9aa4SMartin KaFai Lau if (!refold) 898b16d9aa4SMartin KaFai Lau return ERR_PTR(-ENOENT); 899b16d9aa4SMartin KaFai Lau 900b16d9aa4SMartin KaFai Lau return prog; 901b16d9aa4SMartin KaFai Lau } 902b16d9aa4SMartin KaFai Lau 903113214beSDaniel Borkmann static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *type) 90409756af4SAlexei Starovoitov { 90509756af4SAlexei Starovoitov struct fd f = fdget(ufd); 90609756af4SAlexei Starovoitov struct bpf_prog *prog; 90709756af4SAlexei Starovoitov 908113214beSDaniel Borkmann prog = ____bpf_prog_get(f); 90909756af4SAlexei Starovoitov if (IS_ERR(prog)) 91009756af4SAlexei Starovoitov return prog; 911113214beSDaniel Borkmann if (type && prog->type != *type) { 912113214beSDaniel Borkmann prog = ERR_PTR(-EINVAL); 913113214beSDaniel Borkmann goto out; 914113214beSDaniel Borkmann } 91509756af4SAlexei Starovoitov 91692117d84SAlexei Starovoitov prog = bpf_prog_inc(prog); 917113214beSDaniel Borkmann out: 91809756af4SAlexei Starovoitov fdput(f); 91909756af4SAlexei Starovoitov return prog; 92009756af4SAlexei Starovoitov } 921113214beSDaniel Borkmann 922113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get(u32 ufd) 923113214beSDaniel Borkmann { 924113214beSDaniel Borkmann return __bpf_prog_get(ufd, NULL); 925113214beSDaniel Borkmann } 926113214beSDaniel Borkmann 927113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type) 928113214beSDaniel Borkmann { 929a67edbf4SDaniel Borkmann struct bpf_prog *prog = __bpf_prog_get(ufd, &type); 930a67edbf4SDaniel Borkmann 931a67edbf4SDaniel Borkmann if (!IS_ERR(prog)) 932a67edbf4SDaniel Borkmann trace_bpf_prog_get_type(prog); 933a67edbf4SDaniel Borkmann return prog; 934113214beSDaniel Borkmann } 935113214beSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_get_type); 93609756af4SAlexei Starovoitov 93709756af4SAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 938e07b98d9SDavid S. Miller #define BPF_PROG_LOAD_LAST_FIELD prog_flags 93909756af4SAlexei Starovoitov 94009756af4SAlexei Starovoitov static int bpf_prog_load(union bpf_attr *attr) 94109756af4SAlexei Starovoitov { 94209756af4SAlexei Starovoitov enum bpf_prog_type type = attr->prog_type; 94309756af4SAlexei Starovoitov struct bpf_prog *prog; 94409756af4SAlexei Starovoitov int err; 94509756af4SAlexei Starovoitov char license[128]; 94609756af4SAlexei Starovoitov bool is_gpl; 94709756af4SAlexei Starovoitov 94809756af4SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_LOAD)) 94909756af4SAlexei Starovoitov return -EINVAL; 95009756af4SAlexei Starovoitov 951e07b98d9SDavid S. Miller if (attr->prog_flags & ~BPF_F_STRICT_ALIGNMENT) 952e07b98d9SDavid S. Miller return -EINVAL; 953e07b98d9SDavid S. Miller 95409756af4SAlexei Starovoitov /* copy eBPF program license from user space */ 955535e7b4bSMickaël Salaün if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 95609756af4SAlexei Starovoitov sizeof(license) - 1) < 0) 95709756af4SAlexei Starovoitov return -EFAULT; 95809756af4SAlexei Starovoitov license[sizeof(license) - 1] = 0; 95909756af4SAlexei Starovoitov 96009756af4SAlexei Starovoitov /* eBPF programs must be GPL compatible to use GPL-ed functions */ 96109756af4SAlexei Starovoitov is_gpl = license_is_gpl_compatible(license); 96209756af4SAlexei Starovoitov 963ef0915caSDaniel Borkmann if (attr->insn_cnt == 0 || attr->insn_cnt > BPF_MAXINSNS) 964ef0915caSDaniel Borkmann return -E2BIG; 96509756af4SAlexei Starovoitov 9662541517cSAlexei Starovoitov if (type == BPF_PROG_TYPE_KPROBE && 9672541517cSAlexei Starovoitov attr->kern_version != LINUX_VERSION_CODE) 9682541517cSAlexei Starovoitov return -EINVAL; 9692541517cSAlexei Starovoitov 97080b7d819SChenbo Feng if (type != BPF_PROG_TYPE_SOCKET_FILTER && 97180b7d819SChenbo Feng type != BPF_PROG_TYPE_CGROUP_SKB && 97280b7d819SChenbo Feng !capable(CAP_SYS_ADMIN)) 9731be7f75dSAlexei Starovoitov return -EPERM; 9741be7f75dSAlexei Starovoitov 97509756af4SAlexei Starovoitov /* plain bpf_prog allocation */ 97609756af4SAlexei Starovoitov prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 97709756af4SAlexei Starovoitov if (!prog) 97809756af4SAlexei Starovoitov return -ENOMEM; 97909756af4SAlexei Starovoitov 980aaac3ba9SAlexei Starovoitov err = bpf_prog_charge_memlock(prog); 981aaac3ba9SAlexei Starovoitov if (err) 982aaac3ba9SAlexei Starovoitov goto free_prog_nouncharge; 983aaac3ba9SAlexei Starovoitov 98409756af4SAlexei Starovoitov prog->len = attr->insn_cnt; 98509756af4SAlexei Starovoitov 98609756af4SAlexei Starovoitov err = -EFAULT; 987535e7b4bSMickaël Salaün if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 988aafe6ae9SDaniel Borkmann bpf_prog_insn_size(prog)) != 0) 98909756af4SAlexei Starovoitov goto free_prog; 99009756af4SAlexei Starovoitov 99109756af4SAlexei Starovoitov prog->orig_prog = NULL; 992a91263d5SDaniel Borkmann prog->jited = 0; 99309756af4SAlexei Starovoitov 99409756af4SAlexei Starovoitov atomic_set(&prog->aux->refcnt, 1); 995a91263d5SDaniel Borkmann prog->gpl_compatible = is_gpl ? 1 : 0; 99609756af4SAlexei Starovoitov 99709756af4SAlexei Starovoitov /* find program type: socket_filter vs tracing_filter */ 99809756af4SAlexei Starovoitov err = find_prog_type(type, prog); 99909756af4SAlexei Starovoitov if (err < 0) 100009756af4SAlexei Starovoitov goto free_prog; 100109756af4SAlexei Starovoitov 100209756af4SAlexei Starovoitov /* run eBPF verifier */ 10039bac3d6dSAlexei Starovoitov err = bpf_check(&prog, attr); 100409756af4SAlexei Starovoitov if (err < 0) 100509756af4SAlexei Starovoitov goto free_used_maps; 100609756af4SAlexei Starovoitov 100709756af4SAlexei Starovoitov /* eBPF program is ready to be JITed */ 1008d1c55ab5SDaniel Borkmann prog = bpf_prog_select_runtime(prog, &err); 100904fd61abSAlexei Starovoitov if (err < 0) 101004fd61abSAlexei Starovoitov goto free_used_maps; 101109756af4SAlexei Starovoitov 1012dc4bb0e2SMartin KaFai Lau err = bpf_prog_alloc_id(prog); 1013dc4bb0e2SMartin KaFai Lau if (err) 1014dc4bb0e2SMartin KaFai Lau goto free_used_maps; 1015dc4bb0e2SMartin KaFai Lau 1016aa79781bSDaniel Borkmann err = bpf_prog_new_fd(prog); 1017b16d9aa4SMartin KaFai Lau if (err < 0) { 1018b16d9aa4SMartin KaFai Lau /* failed to allocate fd. 1019b16d9aa4SMartin KaFai Lau * bpf_prog_put() is needed because the above 1020b16d9aa4SMartin KaFai Lau * bpf_prog_alloc_id() has published the prog 1021b16d9aa4SMartin KaFai Lau * to the userspace and the userspace may 1022b16d9aa4SMartin KaFai Lau * have refcnt-ed it through BPF_PROG_GET_FD_BY_ID. 1023b16d9aa4SMartin KaFai Lau */ 1024b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 1025b16d9aa4SMartin KaFai Lau return err; 1026b16d9aa4SMartin KaFai Lau } 102709756af4SAlexei Starovoitov 102874451e66SDaniel Borkmann bpf_prog_kallsyms_add(prog); 1029a67edbf4SDaniel Borkmann trace_bpf_prog_load(prog, err); 103009756af4SAlexei Starovoitov return err; 103109756af4SAlexei Starovoitov 103209756af4SAlexei Starovoitov free_used_maps: 103309756af4SAlexei Starovoitov free_used_maps(prog->aux); 103409756af4SAlexei Starovoitov free_prog: 1035aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(prog); 1036aaac3ba9SAlexei Starovoitov free_prog_nouncharge: 103709756af4SAlexei Starovoitov bpf_prog_free(prog); 103809756af4SAlexei Starovoitov return err; 103909756af4SAlexei Starovoitov } 104009756af4SAlexei Starovoitov 1041b2197755SDaniel Borkmann #define BPF_OBJ_LAST_FIELD bpf_fd 1042b2197755SDaniel Borkmann 1043b2197755SDaniel Borkmann static int bpf_obj_pin(const union bpf_attr *attr) 1044b2197755SDaniel Borkmann { 1045b2197755SDaniel Borkmann if (CHECK_ATTR(BPF_OBJ)) 1046b2197755SDaniel Borkmann return -EINVAL; 1047b2197755SDaniel Borkmann 1048535e7b4bSMickaël Salaün return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 1049b2197755SDaniel Borkmann } 1050b2197755SDaniel Borkmann 1051b2197755SDaniel Borkmann static int bpf_obj_get(const union bpf_attr *attr) 1052b2197755SDaniel Borkmann { 1053b2197755SDaniel Borkmann if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0) 1054b2197755SDaniel Borkmann return -EINVAL; 1055b2197755SDaniel Borkmann 1056535e7b4bSMickaël Salaün return bpf_obj_get_user(u64_to_user_ptr(attr->pathname)); 1057b2197755SDaniel Borkmann } 1058b2197755SDaniel Borkmann 1059f4324551SDaniel Mack #ifdef CONFIG_CGROUP_BPF 1060f4324551SDaniel Mack 10617f677633SAlexei Starovoitov #define BPF_PROG_ATTACH_LAST_FIELD attach_flags 1062f4324551SDaniel Mack 1063f4324551SDaniel Mack static int bpf_prog_attach(const union bpf_attr *attr) 1064f4324551SDaniel Mack { 10657f677633SAlexei Starovoitov enum bpf_prog_type ptype; 1066f4324551SDaniel Mack struct bpf_prog *prog; 1067f4324551SDaniel Mack struct cgroup *cgrp; 10687f677633SAlexei Starovoitov int ret; 1069f4324551SDaniel Mack 1070f4324551SDaniel Mack if (!capable(CAP_NET_ADMIN)) 1071f4324551SDaniel Mack return -EPERM; 1072f4324551SDaniel Mack 1073f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_ATTACH)) 1074f4324551SDaniel Mack return -EINVAL; 1075f4324551SDaniel Mack 10767f677633SAlexei Starovoitov if (attr->attach_flags & ~BPF_F_ALLOW_OVERRIDE) 10777f677633SAlexei Starovoitov return -EINVAL; 10787f677633SAlexei Starovoitov 1079f4324551SDaniel Mack switch (attr->attach_type) { 1080f4324551SDaniel Mack case BPF_CGROUP_INET_INGRESS: 1081f4324551SDaniel Mack case BPF_CGROUP_INET_EGRESS: 1082b2cd1257SDavid Ahern ptype = BPF_PROG_TYPE_CGROUP_SKB; 1083b2cd1257SDavid Ahern break; 108461023658SDavid Ahern case BPF_CGROUP_INET_SOCK_CREATE: 108561023658SDavid Ahern ptype = BPF_PROG_TYPE_CGROUP_SOCK; 108661023658SDavid Ahern break; 108740304b2aSLawrence Brakmo case BPF_CGROUP_SOCK_OPS: 108840304b2aSLawrence Brakmo ptype = BPF_PROG_TYPE_SOCK_OPS; 108940304b2aSLawrence Brakmo break; 1090b2cd1257SDavid Ahern default: 1091b2cd1257SDavid Ahern return -EINVAL; 1092b2cd1257SDavid Ahern } 1093b2cd1257SDavid Ahern 1094b2cd1257SDavid Ahern prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 1095f4324551SDaniel Mack if (IS_ERR(prog)) 1096f4324551SDaniel Mack return PTR_ERR(prog); 1097f4324551SDaniel Mack 1098f4324551SDaniel Mack cgrp = cgroup_get_from_fd(attr->target_fd); 1099f4324551SDaniel Mack if (IS_ERR(cgrp)) { 1100f4324551SDaniel Mack bpf_prog_put(prog); 1101f4324551SDaniel Mack return PTR_ERR(cgrp); 1102f4324551SDaniel Mack } 1103f4324551SDaniel Mack 11047f677633SAlexei Starovoitov ret = cgroup_bpf_update(cgrp, prog, attr->attach_type, 11057f677633SAlexei Starovoitov attr->attach_flags & BPF_F_ALLOW_OVERRIDE); 11067f677633SAlexei Starovoitov if (ret) 11077f677633SAlexei Starovoitov bpf_prog_put(prog); 1108f4324551SDaniel Mack cgroup_put(cgrp); 1109f4324551SDaniel Mack 11107f677633SAlexei Starovoitov return ret; 1111f4324551SDaniel Mack } 1112f4324551SDaniel Mack 1113f4324551SDaniel Mack #define BPF_PROG_DETACH_LAST_FIELD attach_type 1114f4324551SDaniel Mack 1115f4324551SDaniel Mack static int bpf_prog_detach(const union bpf_attr *attr) 1116f4324551SDaniel Mack { 1117f4324551SDaniel Mack struct cgroup *cgrp; 11187f677633SAlexei Starovoitov int ret; 1119f4324551SDaniel Mack 1120f4324551SDaniel Mack if (!capable(CAP_NET_ADMIN)) 1121f4324551SDaniel Mack return -EPERM; 1122f4324551SDaniel Mack 1123f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_DETACH)) 1124f4324551SDaniel Mack return -EINVAL; 1125f4324551SDaniel Mack 1126f4324551SDaniel Mack switch (attr->attach_type) { 1127f4324551SDaniel Mack case BPF_CGROUP_INET_INGRESS: 1128f4324551SDaniel Mack case BPF_CGROUP_INET_EGRESS: 112961023658SDavid Ahern case BPF_CGROUP_INET_SOCK_CREATE: 113040304b2aSLawrence Brakmo case BPF_CGROUP_SOCK_OPS: 1131f4324551SDaniel Mack cgrp = cgroup_get_from_fd(attr->target_fd); 1132f4324551SDaniel Mack if (IS_ERR(cgrp)) 1133f4324551SDaniel Mack return PTR_ERR(cgrp); 1134f4324551SDaniel Mack 11357f677633SAlexei Starovoitov ret = cgroup_bpf_update(cgrp, NULL, attr->attach_type, false); 1136f4324551SDaniel Mack cgroup_put(cgrp); 1137f4324551SDaniel Mack break; 1138f4324551SDaniel Mack 1139f4324551SDaniel Mack default: 1140f4324551SDaniel Mack return -EINVAL; 1141f4324551SDaniel Mack } 1142f4324551SDaniel Mack 11437f677633SAlexei Starovoitov return ret; 1144f4324551SDaniel Mack } 114540304b2aSLawrence Brakmo 1146f4324551SDaniel Mack #endif /* CONFIG_CGROUP_BPF */ 1147f4324551SDaniel Mack 11481cf1cae9SAlexei Starovoitov #define BPF_PROG_TEST_RUN_LAST_FIELD test.duration 11491cf1cae9SAlexei Starovoitov 11501cf1cae9SAlexei Starovoitov static int bpf_prog_test_run(const union bpf_attr *attr, 11511cf1cae9SAlexei Starovoitov union bpf_attr __user *uattr) 11521cf1cae9SAlexei Starovoitov { 11531cf1cae9SAlexei Starovoitov struct bpf_prog *prog; 11541cf1cae9SAlexei Starovoitov int ret = -ENOTSUPP; 11551cf1cae9SAlexei Starovoitov 11561cf1cae9SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 11571cf1cae9SAlexei Starovoitov return -EINVAL; 11581cf1cae9SAlexei Starovoitov 11591cf1cae9SAlexei Starovoitov prog = bpf_prog_get(attr->test.prog_fd); 11601cf1cae9SAlexei Starovoitov if (IS_ERR(prog)) 11611cf1cae9SAlexei Starovoitov return PTR_ERR(prog); 11621cf1cae9SAlexei Starovoitov 11631cf1cae9SAlexei Starovoitov if (prog->aux->ops->test_run) 11641cf1cae9SAlexei Starovoitov ret = prog->aux->ops->test_run(prog, attr, uattr); 11651cf1cae9SAlexei Starovoitov 11661cf1cae9SAlexei Starovoitov bpf_prog_put(prog); 11671cf1cae9SAlexei Starovoitov return ret; 11681cf1cae9SAlexei Starovoitov } 11691cf1cae9SAlexei Starovoitov 117034ad5580SMartin KaFai Lau #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 117134ad5580SMartin KaFai Lau 117234ad5580SMartin KaFai Lau static int bpf_obj_get_next_id(const union bpf_attr *attr, 117334ad5580SMartin KaFai Lau union bpf_attr __user *uattr, 117434ad5580SMartin KaFai Lau struct idr *idr, 117534ad5580SMartin KaFai Lau spinlock_t *lock) 117634ad5580SMartin KaFai Lau { 117734ad5580SMartin KaFai Lau u32 next_id = attr->start_id; 117834ad5580SMartin KaFai Lau int err = 0; 117934ad5580SMartin KaFai Lau 118034ad5580SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 118134ad5580SMartin KaFai Lau return -EINVAL; 118234ad5580SMartin KaFai Lau 118334ad5580SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 118434ad5580SMartin KaFai Lau return -EPERM; 118534ad5580SMartin KaFai Lau 118634ad5580SMartin KaFai Lau next_id++; 118734ad5580SMartin KaFai Lau spin_lock_bh(lock); 118834ad5580SMartin KaFai Lau if (!idr_get_next(idr, &next_id)) 118934ad5580SMartin KaFai Lau err = -ENOENT; 119034ad5580SMartin KaFai Lau spin_unlock_bh(lock); 119134ad5580SMartin KaFai Lau 119234ad5580SMartin KaFai Lau if (!err) 119334ad5580SMartin KaFai Lau err = put_user(next_id, &uattr->next_id); 119434ad5580SMartin KaFai Lau 119534ad5580SMartin KaFai Lau return err; 119634ad5580SMartin KaFai Lau } 119734ad5580SMartin KaFai Lau 1198b16d9aa4SMartin KaFai Lau #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 1199b16d9aa4SMartin KaFai Lau 1200b16d9aa4SMartin KaFai Lau static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 1201b16d9aa4SMartin KaFai Lau { 1202b16d9aa4SMartin KaFai Lau struct bpf_prog *prog; 1203b16d9aa4SMartin KaFai Lau u32 id = attr->prog_id; 1204b16d9aa4SMartin KaFai Lau int fd; 1205b16d9aa4SMartin KaFai Lau 1206b16d9aa4SMartin KaFai Lau if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 1207b16d9aa4SMartin KaFai Lau return -EINVAL; 1208b16d9aa4SMartin KaFai Lau 1209b16d9aa4SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 1210b16d9aa4SMartin KaFai Lau return -EPERM; 1211b16d9aa4SMartin KaFai Lau 1212b16d9aa4SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1213b16d9aa4SMartin KaFai Lau prog = idr_find(&prog_idr, id); 1214b16d9aa4SMartin KaFai Lau if (prog) 1215b16d9aa4SMartin KaFai Lau prog = bpf_prog_inc_not_zero(prog); 1216b16d9aa4SMartin KaFai Lau else 1217b16d9aa4SMartin KaFai Lau prog = ERR_PTR(-ENOENT); 1218b16d9aa4SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1219b16d9aa4SMartin KaFai Lau 1220b16d9aa4SMartin KaFai Lau if (IS_ERR(prog)) 1221b16d9aa4SMartin KaFai Lau return PTR_ERR(prog); 1222b16d9aa4SMartin KaFai Lau 1223b16d9aa4SMartin KaFai Lau fd = bpf_prog_new_fd(prog); 1224b16d9aa4SMartin KaFai Lau if (fd < 0) 1225b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 1226b16d9aa4SMartin KaFai Lau 1227b16d9aa4SMartin KaFai Lau return fd; 1228b16d9aa4SMartin KaFai Lau } 1229b16d9aa4SMartin KaFai Lau 1230bd5f5f4eSMartin KaFai Lau #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD map_id 1231bd5f5f4eSMartin KaFai Lau 1232bd5f5f4eSMartin KaFai Lau static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 1233bd5f5f4eSMartin KaFai Lau { 1234bd5f5f4eSMartin KaFai Lau struct bpf_map *map; 1235bd5f5f4eSMartin KaFai Lau u32 id = attr->map_id; 1236bd5f5f4eSMartin KaFai Lau int fd; 1237bd5f5f4eSMartin KaFai Lau 1238bd5f5f4eSMartin KaFai Lau if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID)) 1239bd5f5f4eSMartin KaFai Lau return -EINVAL; 1240bd5f5f4eSMartin KaFai Lau 1241bd5f5f4eSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 1242bd5f5f4eSMartin KaFai Lau return -EPERM; 1243bd5f5f4eSMartin KaFai Lau 1244bd5f5f4eSMartin KaFai Lau spin_lock_bh(&map_idr_lock); 1245bd5f5f4eSMartin KaFai Lau map = idr_find(&map_idr, id); 1246bd5f5f4eSMartin KaFai Lau if (map) 1247bd5f5f4eSMartin KaFai Lau map = bpf_map_inc_not_zero(map, true); 1248bd5f5f4eSMartin KaFai Lau else 1249bd5f5f4eSMartin KaFai Lau map = ERR_PTR(-ENOENT); 1250bd5f5f4eSMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 1251bd5f5f4eSMartin KaFai Lau 1252bd5f5f4eSMartin KaFai Lau if (IS_ERR(map)) 1253bd5f5f4eSMartin KaFai Lau return PTR_ERR(map); 1254bd5f5f4eSMartin KaFai Lau 1255bd5f5f4eSMartin KaFai Lau fd = bpf_map_new_fd(map); 1256bd5f5f4eSMartin KaFai Lau if (fd < 0) 1257bd5f5f4eSMartin KaFai Lau bpf_map_put(map); 1258bd5f5f4eSMartin KaFai Lau 1259bd5f5f4eSMartin KaFai Lau return fd; 1260bd5f5f4eSMartin KaFai Lau } 1261bd5f5f4eSMartin KaFai Lau 12621e270976SMartin KaFai Lau static int check_uarg_tail_zero(void __user *uaddr, 12631e270976SMartin KaFai Lau size_t expected_size, 12641e270976SMartin KaFai Lau size_t actual_size) 12651e270976SMartin KaFai Lau { 12661e270976SMartin KaFai Lau unsigned char __user *addr; 12671e270976SMartin KaFai Lau unsigned char __user *end; 12681e270976SMartin KaFai Lau unsigned char val; 12691e270976SMartin KaFai Lau int err; 12701e270976SMartin KaFai Lau 12711e270976SMartin KaFai Lau if (actual_size <= expected_size) 12721e270976SMartin KaFai Lau return 0; 12731e270976SMartin KaFai Lau 12741e270976SMartin KaFai Lau addr = uaddr + expected_size; 12751e270976SMartin KaFai Lau end = uaddr + actual_size; 12761e270976SMartin KaFai Lau 12771e270976SMartin KaFai Lau for (; addr < end; addr++) { 12781e270976SMartin KaFai Lau err = get_user(val, addr); 12791e270976SMartin KaFai Lau if (err) 12801e270976SMartin KaFai Lau return err; 12811e270976SMartin KaFai Lau if (val) 12821e270976SMartin KaFai Lau return -E2BIG; 12831e270976SMartin KaFai Lau } 12841e270976SMartin KaFai Lau 12851e270976SMartin KaFai Lau return 0; 12861e270976SMartin KaFai Lau } 12871e270976SMartin KaFai Lau 12881e270976SMartin KaFai Lau static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, 12891e270976SMartin KaFai Lau const union bpf_attr *attr, 12901e270976SMartin KaFai Lau union bpf_attr __user *uattr) 12911e270976SMartin KaFai Lau { 12921e270976SMartin KaFai Lau struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 12931e270976SMartin KaFai Lau struct bpf_prog_info info = {}; 12941e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 12951e270976SMartin KaFai Lau char __user *uinsns; 12961e270976SMartin KaFai Lau u32 ulen; 12971e270976SMartin KaFai Lau int err; 12981e270976SMartin KaFai Lau 12991e270976SMartin KaFai Lau err = check_uarg_tail_zero(uinfo, sizeof(info), info_len); 13001e270976SMartin KaFai Lau if (err) 13011e270976SMartin KaFai Lau return err; 13021e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 13031e270976SMartin KaFai Lau 13041e270976SMartin KaFai Lau if (copy_from_user(&info, uinfo, info_len)) 13051e270976SMartin KaFai Lau return err; 13061e270976SMartin KaFai Lau 13071e270976SMartin KaFai Lau info.type = prog->type; 13081e270976SMartin KaFai Lau info.id = prog->aux->id; 13091e270976SMartin KaFai Lau 13101e270976SMartin KaFai Lau memcpy(info.tag, prog->tag, sizeof(prog->tag)); 13111e270976SMartin KaFai Lau 13121e270976SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) { 13131e270976SMartin KaFai Lau info.jited_prog_len = 0; 13141e270976SMartin KaFai Lau info.xlated_prog_len = 0; 13151e270976SMartin KaFai Lau goto done; 13161e270976SMartin KaFai Lau } 13171e270976SMartin KaFai Lau 13181e270976SMartin KaFai Lau ulen = info.jited_prog_len; 13191e270976SMartin KaFai Lau info.jited_prog_len = prog->jited_len; 13201e270976SMartin KaFai Lau if (info.jited_prog_len && ulen) { 13211e270976SMartin KaFai Lau uinsns = u64_to_user_ptr(info.jited_prog_insns); 13221e270976SMartin KaFai Lau ulen = min_t(u32, info.jited_prog_len, ulen); 13231e270976SMartin KaFai Lau if (copy_to_user(uinsns, prog->bpf_func, ulen)) 13241e270976SMartin KaFai Lau return -EFAULT; 13251e270976SMartin KaFai Lau } 13261e270976SMartin KaFai Lau 13271e270976SMartin KaFai Lau ulen = info.xlated_prog_len; 13281e270976SMartin KaFai Lau info.xlated_prog_len = bpf_prog_size(prog->len); 13291e270976SMartin KaFai Lau if (info.xlated_prog_len && ulen) { 13301e270976SMartin KaFai Lau uinsns = u64_to_user_ptr(info.xlated_prog_insns); 13311e270976SMartin KaFai Lau ulen = min_t(u32, info.xlated_prog_len, ulen); 13321e270976SMartin KaFai Lau if (copy_to_user(uinsns, prog->insnsi, ulen)) 13331e270976SMartin KaFai Lau return -EFAULT; 13341e270976SMartin KaFai Lau } 13351e270976SMartin KaFai Lau 13361e270976SMartin KaFai Lau done: 13371e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 13381e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 13391e270976SMartin KaFai Lau return -EFAULT; 13401e270976SMartin KaFai Lau 13411e270976SMartin KaFai Lau return 0; 13421e270976SMartin KaFai Lau } 13431e270976SMartin KaFai Lau 13441e270976SMartin KaFai Lau static int bpf_map_get_info_by_fd(struct bpf_map *map, 13451e270976SMartin KaFai Lau const union bpf_attr *attr, 13461e270976SMartin KaFai Lau union bpf_attr __user *uattr) 13471e270976SMartin KaFai Lau { 13481e270976SMartin KaFai Lau struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 13491e270976SMartin KaFai Lau struct bpf_map_info info = {}; 13501e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 13511e270976SMartin KaFai Lau int err; 13521e270976SMartin KaFai Lau 13531e270976SMartin KaFai Lau err = check_uarg_tail_zero(uinfo, sizeof(info), info_len); 13541e270976SMartin KaFai Lau if (err) 13551e270976SMartin KaFai Lau return err; 13561e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 13571e270976SMartin KaFai Lau 13581e270976SMartin KaFai Lau info.type = map->map_type; 13591e270976SMartin KaFai Lau info.id = map->id; 13601e270976SMartin KaFai Lau info.key_size = map->key_size; 13611e270976SMartin KaFai Lau info.value_size = map->value_size; 13621e270976SMartin KaFai Lau info.max_entries = map->max_entries; 13631e270976SMartin KaFai Lau info.map_flags = map->map_flags; 13641e270976SMartin KaFai Lau 13651e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 13661e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 13671e270976SMartin KaFai Lau return -EFAULT; 13681e270976SMartin KaFai Lau 13691e270976SMartin KaFai Lau return 0; 13701e270976SMartin KaFai Lau } 13711e270976SMartin KaFai Lau 13721e270976SMartin KaFai Lau #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 13731e270976SMartin KaFai Lau 13741e270976SMartin KaFai Lau static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 13751e270976SMartin KaFai Lau union bpf_attr __user *uattr) 13761e270976SMartin KaFai Lau { 13771e270976SMartin KaFai Lau int ufd = attr->info.bpf_fd; 13781e270976SMartin KaFai Lau struct fd f; 13791e270976SMartin KaFai Lau int err; 13801e270976SMartin KaFai Lau 13811e270976SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 13821e270976SMartin KaFai Lau return -EINVAL; 13831e270976SMartin KaFai Lau 13841e270976SMartin KaFai Lau f = fdget(ufd); 13851e270976SMartin KaFai Lau if (!f.file) 13861e270976SMartin KaFai Lau return -EBADFD; 13871e270976SMartin KaFai Lau 13881e270976SMartin KaFai Lau if (f.file->f_op == &bpf_prog_fops) 13891e270976SMartin KaFai Lau err = bpf_prog_get_info_by_fd(f.file->private_data, attr, 13901e270976SMartin KaFai Lau uattr); 13911e270976SMartin KaFai Lau else if (f.file->f_op == &bpf_map_fops) 13921e270976SMartin KaFai Lau err = bpf_map_get_info_by_fd(f.file->private_data, attr, 13931e270976SMartin KaFai Lau uattr); 13941e270976SMartin KaFai Lau else 13951e270976SMartin KaFai Lau err = -EINVAL; 13961e270976SMartin KaFai Lau 13971e270976SMartin KaFai Lau fdput(f); 13981e270976SMartin KaFai Lau return err; 13991e270976SMartin KaFai Lau } 14001e270976SMartin KaFai Lau 140199c55f7dSAlexei Starovoitov SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 140299c55f7dSAlexei Starovoitov { 140399c55f7dSAlexei Starovoitov union bpf_attr attr = {}; 140499c55f7dSAlexei Starovoitov int err; 140599c55f7dSAlexei Starovoitov 14061be7f75dSAlexei Starovoitov if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled) 140799c55f7dSAlexei Starovoitov return -EPERM; 140899c55f7dSAlexei Starovoitov 140999c55f7dSAlexei Starovoitov if (!access_ok(VERIFY_READ, uattr, 1)) 141099c55f7dSAlexei Starovoitov return -EFAULT; 141199c55f7dSAlexei Starovoitov 141299c55f7dSAlexei Starovoitov if (size > PAGE_SIZE) /* silly large */ 141399c55f7dSAlexei Starovoitov return -E2BIG; 141499c55f7dSAlexei Starovoitov 141599c55f7dSAlexei Starovoitov /* If we're handed a bigger struct than we know of, 141699c55f7dSAlexei Starovoitov * ensure all the unknown bits are 0 - i.e. new 141799c55f7dSAlexei Starovoitov * user-space does not rely on any kernel feature 141899c55f7dSAlexei Starovoitov * extensions we dont know about yet. 141999c55f7dSAlexei Starovoitov */ 14201e270976SMartin KaFai Lau err = check_uarg_tail_zero(uattr, sizeof(attr), size); 142199c55f7dSAlexei Starovoitov if (err) 142299c55f7dSAlexei Starovoitov return err; 14231e270976SMartin KaFai Lau size = min_t(u32, size, sizeof(attr)); 142499c55f7dSAlexei Starovoitov 142599c55f7dSAlexei Starovoitov /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 142699c55f7dSAlexei Starovoitov if (copy_from_user(&attr, uattr, size) != 0) 142799c55f7dSAlexei Starovoitov return -EFAULT; 142899c55f7dSAlexei Starovoitov 142999c55f7dSAlexei Starovoitov switch (cmd) { 143099c55f7dSAlexei Starovoitov case BPF_MAP_CREATE: 143199c55f7dSAlexei Starovoitov err = map_create(&attr); 143299c55f7dSAlexei Starovoitov break; 1433db20fd2bSAlexei Starovoitov case BPF_MAP_LOOKUP_ELEM: 1434db20fd2bSAlexei Starovoitov err = map_lookup_elem(&attr); 1435db20fd2bSAlexei Starovoitov break; 1436db20fd2bSAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 1437db20fd2bSAlexei Starovoitov err = map_update_elem(&attr); 1438db20fd2bSAlexei Starovoitov break; 1439db20fd2bSAlexei Starovoitov case BPF_MAP_DELETE_ELEM: 1440db20fd2bSAlexei Starovoitov err = map_delete_elem(&attr); 1441db20fd2bSAlexei Starovoitov break; 1442db20fd2bSAlexei Starovoitov case BPF_MAP_GET_NEXT_KEY: 1443db20fd2bSAlexei Starovoitov err = map_get_next_key(&attr); 1444db20fd2bSAlexei Starovoitov break; 144509756af4SAlexei Starovoitov case BPF_PROG_LOAD: 144609756af4SAlexei Starovoitov err = bpf_prog_load(&attr); 144709756af4SAlexei Starovoitov break; 1448b2197755SDaniel Borkmann case BPF_OBJ_PIN: 1449b2197755SDaniel Borkmann err = bpf_obj_pin(&attr); 1450b2197755SDaniel Borkmann break; 1451b2197755SDaniel Borkmann case BPF_OBJ_GET: 1452b2197755SDaniel Borkmann err = bpf_obj_get(&attr); 1453b2197755SDaniel Borkmann break; 1454f4324551SDaniel Mack #ifdef CONFIG_CGROUP_BPF 1455f4324551SDaniel Mack case BPF_PROG_ATTACH: 1456f4324551SDaniel Mack err = bpf_prog_attach(&attr); 1457f4324551SDaniel Mack break; 1458f4324551SDaniel Mack case BPF_PROG_DETACH: 1459f4324551SDaniel Mack err = bpf_prog_detach(&attr); 1460f4324551SDaniel Mack break; 1461f4324551SDaniel Mack #endif 14621cf1cae9SAlexei Starovoitov case BPF_PROG_TEST_RUN: 14631cf1cae9SAlexei Starovoitov err = bpf_prog_test_run(&attr, uattr); 14641cf1cae9SAlexei Starovoitov break; 146534ad5580SMartin KaFai Lau case BPF_PROG_GET_NEXT_ID: 146634ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 146734ad5580SMartin KaFai Lau &prog_idr, &prog_idr_lock); 146834ad5580SMartin KaFai Lau break; 146934ad5580SMartin KaFai Lau case BPF_MAP_GET_NEXT_ID: 147034ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 147134ad5580SMartin KaFai Lau &map_idr, &map_idr_lock); 147234ad5580SMartin KaFai Lau break; 1473b16d9aa4SMartin KaFai Lau case BPF_PROG_GET_FD_BY_ID: 1474b16d9aa4SMartin KaFai Lau err = bpf_prog_get_fd_by_id(&attr); 1475b16d9aa4SMartin KaFai Lau break; 1476bd5f5f4eSMartin KaFai Lau case BPF_MAP_GET_FD_BY_ID: 1477bd5f5f4eSMartin KaFai Lau err = bpf_map_get_fd_by_id(&attr); 1478bd5f5f4eSMartin KaFai Lau break; 14791e270976SMartin KaFai Lau case BPF_OBJ_GET_INFO_BY_FD: 14801e270976SMartin KaFai Lau err = bpf_obj_get_info_by_fd(&attr, uattr); 14811e270976SMartin KaFai Lau break; 148299c55f7dSAlexei Starovoitov default: 148399c55f7dSAlexei Starovoitov err = -EINVAL; 148499c55f7dSAlexei Starovoitov break; 148599c55f7dSAlexei Starovoitov } 148699c55f7dSAlexei Starovoitov 148799c55f7dSAlexei Starovoitov return err; 148899c55f7dSAlexei Starovoitov } 1489