15b497af4SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 299c55f7dSAlexei Starovoitov /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 399c55f7dSAlexei Starovoitov */ 499c55f7dSAlexei Starovoitov #include <linux/bpf.h> 5a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 6f4364dcfSSean Young #include <linux/bpf_lirc.h> 74a1e7c0cSToke Høiland-Jørgensen #include <linux/bpf_verifier.h> 8f56a653cSMartin KaFai Lau #include <linux/btf.h> 999c55f7dSAlexei Starovoitov #include <linux/syscalls.h> 1099c55f7dSAlexei Starovoitov #include <linux/slab.h> 113f07c014SIngo Molnar #include <linux/sched/signal.h> 12d407bd25SDaniel Borkmann #include <linux/vmalloc.h> 13d407bd25SDaniel Borkmann #include <linux/mmzone.h> 1499c55f7dSAlexei Starovoitov #include <linux/anon_inodes.h> 1541bdc4b4SYonghong Song #include <linux/fdtable.h> 16db20fd2bSAlexei Starovoitov #include <linux/file.h> 1741bdc4b4SYonghong Song #include <linux/fs.h> 1809756af4SAlexei Starovoitov #include <linux/license.h> 1909756af4SAlexei Starovoitov #include <linux/filter.h> 20535e7b4bSMickaël Salaün #include <linux/kernel.h> 21dc4bb0e2SMartin KaFai Lau #include <linux/idr.h> 22cb4d2b3fSMartin KaFai Lau #include <linux/cred.h> 23cb4d2b3fSMartin KaFai Lau #include <linux/timekeeping.h> 24cb4d2b3fSMartin KaFai Lau #include <linux/ctype.h> 259ef09e35SMark Rutland #include <linux/nospec.h> 26bae141f5SDaniel Borkmann #include <linux/audit.h> 27ccfe29ebSAlexei Starovoitov #include <uapi/linux/btf.h> 28ca5999fdSMike Rapoport #include <linux/pgtable.h> 299e4e01dfSKP Singh #include <linux/bpf_lsm.h> 30457f4436SAndrii Nakryiko #include <linux/poll.h> 31a3fd7ceeSJakub Sitnicki #include <linux/bpf-netns.h> 321e6c62a8SAlexei Starovoitov #include <linux/rcupdate_trace.h> 3348edc1f7SRoman Gushchin #include <linux/memcontrol.h> 3499c55f7dSAlexei Starovoitov 35da765a2fSDaniel Borkmann #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 3614dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 3714dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 38da765a2fSDaniel Borkmann #define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY) 3914dc6f04SMartin KaFai Lau #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 40da765a2fSDaniel Borkmann #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \ 41da765a2fSDaniel Borkmann IS_FD_HASH(map)) 4214dc6f04SMartin KaFai Lau 436e71b04aSChenbo Feng #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) 446e71b04aSChenbo Feng 45b121d1e7SAlexei Starovoitov DEFINE_PER_CPU(int, bpf_prog_active); 46dc4bb0e2SMartin KaFai Lau static DEFINE_IDR(prog_idr); 47dc4bb0e2SMartin KaFai Lau static DEFINE_SPINLOCK(prog_idr_lock); 48f3f1c054SMartin KaFai Lau static DEFINE_IDR(map_idr); 49f3f1c054SMartin KaFai Lau static DEFINE_SPINLOCK(map_idr_lock); 50a3b80e10SAndrii Nakryiko static DEFINE_IDR(link_idr); 51a3b80e10SAndrii Nakryiko static DEFINE_SPINLOCK(link_idr_lock); 52b121d1e7SAlexei Starovoitov 531be7f75dSAlexei Starovoitov int sysctl_unprivileged_bpf_disabled __read_mostly; 541be7f75dSAlexei Starovoitov 5540077e0cSJohannes Berg static const struct bpf_map_ops * const bpf_map_types[] = { 5691cc1a99SAlexei Starovoitov #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) 5740077e0cSJohannes Berg #define BPF_MAP_TYPE(_id, _ops) \ 5840077e0cSJohannes Berg [_id] = &_ops, 59f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) 6040077e0cSJohannes Berg #include <linux/bpf_types.h> 6140077e0cSJohannes Berg #undef BPF_PROG_TYPE 6240077e0cSJohannes Berg #undef BPF_MAP_TYPE 63f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 6440077e0cSJohannes Berg }; 6599c55f7dSAlexei Starovoitov 66752ba56fSMickaël Salaün /* 67752ba56fSMickaël Salaün * If we're handed a bigger struct than we know of, ensure all the unknown bits 68752ba56fSMickaël Salaün * are 0 - i.e. new user-space does not rely on any kernel feature extensions 69752ba56fSMickaël Salaün * we don't know about yet. 70752ba56fSMickaël Salaün * 71752ba56fSMickaël Salaün * There is a ToCToU between this function call and the following 72752ba56fSMickaël Salaün * copy_from_user() call. However, this is not a concern since this function is 73752ba56fSMickaël Salaün * meant to be a future-proofing of bits. 74752ba56fSMickaël Salaün */ 75dcab51f1SMartin KaFai Lau int bpf_check_uarg_tail_zero(void __user *uaddr, 7658291a74SMickaël Salaün size_t expected_size, 7758291a74SMickaël Salaün size_t actual_size) 7858291a74SMickaël Salaün { 79b7e4b65fSAl Viro unsigned char __user *addr = uaddr + expected_size; 80b7e4b65fSAl Viro int res; 8158291a74SMickaël Salaün 82752ba56fSMickaël Salaün if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 83752ba56fSMickaël Salaün return -E2BIG; 84752ba56fSMickaël Salaün 8558291a74SMickaël Salaün if (actual_size <= expected_size) 8658291a74SMickaël Salaün return 0; 8758291a74SMickaël Salaün 88b7e4b65fSAl Viro res = check_zeroed_user(addr, actual_size - expected_size); 89b7e4b65fSAl Viro if (res < 0) 90b7e4b65fSAl Viro return res; 91b7e4b65fSAl Viro return res ? 0 : -E2BIG; 9258291a74SMickaël Salaün } 9358291a74SMickaël Salaün 94a3884572SJakub Kicinski const struct bpf_map_ops bpf_map_offload_ops = { 95f4d05259SMartin KaFai Lau .map_meta_equal = bpf_map_meta_equal, 96a3884572SJakub Kicinski .map_alloc = bpf_map_offload_map_alloc, 97a3884572SJakub Kicinski .map_free = bpf_map_offload_map_free, 98e8d2bec0SDaniel Borkmann .map_check_btf = map_check_no_btf, 99a3884572SJakub Kicinski }; 100a3884572SJakub Kicinski 10199c55f7dSAlexei Starovoitov static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 10299c55f7dSAlexei Starovoitov { 1031110f3a9SJakub Kicinski const struct bpf_map_ops *ops; 1049ef09e35SMark Rutland u32 type = attr->map_type; 10599c55f7dSAlexei Starovoitov struct bpf_map *map; 1061110f3a9SJakub Kicinski int err; 10799c55f7dSAlexei Starovoitov 1089ef09e35SMark Rutland if (type >= ARRAY_SIZE(bpf_map_types)) 1091110f3a9SJakub Kicinski return ERR_PTR(-EINVAL); 1109ef09e35SMark Rutland type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types)); 1119ef09e35SMark Rutland ops = bpf_map_types[type]; 1121110f3a9SJakub Kicinski if (!ops) 11340077e0cSJohannes Berg return ERR_PTR(-EINVAL); 11440077e0cSJohannes Berg 1151110f3a9SJakub Kicinski if (ops->map_alloc_check) { 1161110f3a9SJakub Kicinski err = ops->map_alloc_check(attr); 1171110f3a9SJakub Kicinski if (err) 1181110f3a9SJakub Kicinski return ERR_PTR(err); 1191110f3a9SJakub Kicinski } 120a3884572SJakub Kicinski if (attr->map_ifindex) 121a3884572SJakub Kicinski ops = &bpf_map_offload_ops; 1221110f3a9SJakub Kicinski map = ops->map_alloc(attr); 12399c55f7dSAlexei Starovoitov if (IS_ERR(map)) 12499c55f7dSAlexei Starovoitov return map; 1251110f3a9SJakub Kicinski map->ops = ops; 1269ef09e35SMark Rutland map->map_type = type; 12799c55f7dSAlexei Starovoitov return map; 12899c55f7dSAlexei Starovoitov } 12999c55f7dSAlexei Starovoitov 13080ee81e0SRoman Gushchin static u32 bpf_map_value_size(const struct bpf_map *map) 13115c14a3dSBrian Vazquez { 13215c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 13315c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 13415c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 13515c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 13615c14a3dSBrian Vazquez return round_up(map->value_size, 8) * num_possible_cpus(); 13715c14a3dSBrian Vazquez else if (IS_FD_MAP(map)) 13815c14a3dSBrian Vazquez return sizeof(u32); 13915c14a3dSBrian Vazquez else 14015c14a3dSBrian Vazquez return map->value_size; 14115c14a3dSBrian Vazquez } 14215c14a3dSBrian Vazquez 14315c14a3dSBrian Vazquez static void maybe_wait_bpf_programs(struct bpf_map *map) 14415c14a3dSBrian Vazquez { 14515c14a3dSBrian Vazquez /* Wait for any running BPF programs to complete so that 14615c14a3dSBrian Vazquez * userspace, when we return to it, knows that all programs 14715c14a3dSBrian Vazquez * that could be running use the new map value. 14815c14a3dSBrian Vazquez */ 14915c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS || 15015c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 15115c14a3dSBrian Vazquez synchronize_rcu(); 15215c14a3dSBrian Vazquez } 15315c14a3dSBrian Vazquez 15415c14a3dSBrian Vazquez static int bpf_map_update_value(struct bpf_map *map, struct fd f, void *key, 15515c14a3dSBrian Vazquez void *value, __u64 flags) 15615c14a3dSBrian Vazquez { 15715c14a3dSBrian Vazquez int err; 15815c14a3dSBrian Vazquez 15915c14a3dSBrian Vazquez /* Need to create a kthread, thus must support schedule */ 16015c14a3dSBrian Vazquez if (bpf_map_is_dev_bound(map)) { 16115c14a3dSBrian Vazquez return bpf_map_offload_update_elem(map, key, value, flags); 16215c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_CPUMAP || 16315c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 16415c14a3dSBrian Vazquez return map->ops->map_update_elem(map, key, value, flags); 16513b79d3fSLorenz Bauer } else if (map->map_type == BPF_MAP_TYPE_SOCKHASH || 16613b79d3fSLorenz Bauer map->map_type == BPF_MAP_TYPE_SOCKMAP) { 16713b79d3fSLorenz Bauer return sock_map_update_elem_sys(map, key, value, flags); 16815c14a3dSBrian Vazquez } else if (IS_FD_PROG_ARRAY(map)) { 16915c14a3dSBrian Vazquez return bpf_fd_array_map_update_elem(map, f.file, key, value, 17015c14a3dSBrian Vazquez flags); 17115c14a3dSBrian Vazquez } 17215c14a3dSBrian Vazquez 173b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 17415c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 17515c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 17615c14a3dSBrian Vazquez err = bpf_percpu_hash_update(map, key, value, flags); 17715c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 17815c14a3dSBrian Vazquez err = bpf_percpu_array_update(map, key, value, flags); 17915c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 18015c14a3dSBrian Vazquez err = bpf_percpu_cgroup_storage_update(map, key, value, 18115c14a3dSBrian Vazquez flags); 18215c14a3dSBrian Vazquez } else if (IS_FD_ARRAY(map)) { 18315c14a3dSBrian Vazquez rcu_read_lock(); 18415c14a3dSBrian Vazquez err = bpf_fd_array_map_update_elem(map, f.file, key, value, 18515c14a3dSBrian Vazquez flags); 18615c14a3dSBrian Vazquez rcu_read_unlock(); 18715c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 18815c14a3dSBrian Vazquez rcu_read_lock(); 18915c14a3dSBrian Vazquez err = bpf_fd_htab_map_update_elem(map, f.file, key, value, 19015c14a3dSBrian Vazquez flags); 19115c14a3dSBrian Vazquez rcu_read_unlock(); 19215c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 19315c14a3dSBrian Vazquez /* rcu_read_lock() is not needed */ 19415c14a3dSBrian Vazquez err = bpf_fd_reuseport_array_update_elem(map, key, value, 19515c14a3dSBrian Vazquez flags); 19615c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 19715c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_STACK) { 19815c14a3dSBrian Vazquez err = map->ops->map_push_elem(map, value, flags); 19915c14a3dSBrian Vazquez } else { 20015c14a3dSBrian Vazquez rcu_read_lock(); 20115c14a3dSBrian Vazquez err = map->ops->map_update_elem(map, key, value, flags); 20215c14a3dSBrian Vazquez rcu_read_unlock(); 20315c14a3dSBrian Vazquez } 204b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 20515c14a3dSBrian Vazquez maybe_wait_bpf_programs(map); 20615c14a3dSBrian Vazquez 20715c14a3dSBrian Vazquez return err; 20815c14a3dSBrian Vazquez } 20915c14a3dSBrian Vazquez 21015c14a3dSBrian Vazquez static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value, 21115c14a3dSBrian Vazquez __u64 flags) 21215c14a3dSBrian Vazquez { 21315c14a3dSBrian Vazquez void *ptr; 21415c14a3dSBrian Vazquez int err; 21515c14a3dSBrian Vazquez 216cb4d03abSBrian Vazquez if (bpf_map_is_dev_bound(map)) 217cb4d03abSBrian Vazquez return bpf_map_offload_lookup_elem(map, key, value); 21815c14a3dSBrian Vazquez 219b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 22015c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 22115c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 22215c14a3dSBrian Vazquez err = bpf_percpu_hash_copy(map, key, value); 22315c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 22415c14a3dSBrian Vazquez err = bpf_percpu_array_copy(map, key, value); 22515c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 22615c14a3dSBrian Vazquez err = bpf_percpu_cgroup_storage_copy(map, key, value); 22715c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 22815c14a3dSBrian Vazquez err = bpf_stackmap_copy(map, key, value); 22915c14a3dSBrian Vazquez } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) { 23015c14a3dSBrian Vazquez err = bpf_fd_array_map_lookup_elem(map, key, value); 23115c14a3dSBrian Vazquez } else if (IS_FD_HASH(map)) { 23215c14a3dSBrian Vazquez err = bpf_fd_htab_map_lookup_elem(map, key, value); 23315c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 23415c14a3dSBrian Vazquez err = bpf_fd_reuseport_array_lookup_elem(map, key, value); 23515c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 23615c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_STACK) { 23715c14a3dSBrian Vazquez err = map->ops->map_peek_elem(map, value); 23815c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 23915c14a3dSBrian Vazquez /* struct_ops map requires directly updating "value" */ 24015c14a3dSBrian Vazquez err = bpf_struct_ops_map_sys_lookup_elem(map, key, value); 24115c14a3dSBrian Vazquez } else { 24215c14a3dSBrian Vazquez rcu_read_lock(); 24315c14a3dSBrian Vazquez if (map->ops->map_lookup_elem_sys_only) 24415c14a3dSBrian Vazquez ptr = map->ops->map_lookup_elem_sys_only(map, key); 24515c14a3dSBrian Vazquez else 24615c14a3dSBrian Vazquez ptr = map->ops->map_lookup_elem(map, key); 24715c14a3dSBrian Vazquez if (IS_ERR(ptr)) { 24815c14a3dSBrian Vazquez err = PTR_ERR(ptr); 24915c14a3dSBrian Vazquez } else if (!ptr) { 25015c14a3dSBrian Vazquez err = -ENOENT; 25115c14a3dSBrian Vazquez } else { 25215c14a3dSBrian Vazquez err = 0; 25315c14a3dSBrian Vazquez if (flags & BPF_F_LOCK) 25415c14a3dSBrian Vazquez /* lock 'ptr' and copy everything but lock */ 25515c14a3dSBrian Vazquez copy_map_value_locked(map, value, ptr, true); 25615c14a3dSBrian Vazquez else 25715c14a3dSBrian Vazquez copy_map_value(map, value, ptr); 25815c14a3dSBrian Vazquez /* mask lock, since value wasn't zero inited */ 25915c14a3dSBrian Vazquez check_and_init_map_lock(map, value); 26015c14a3dSBrian Vazquez } 26115c14a3dSBrian Vazquez rcu_read_unlock(); 26215c14a3dSBrian Vazquez } 26315c14a3dSBrian Vazquez 264b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 26515c14a3dSBrian Vazquez maybe_wait_bpf_programs(map); 26615c14a3dSBrian Vazquez 26715c14a3dSBrian Vazquez return err; 26815c14a3dSBrian Vazquez } 26915c14a3dSBrian Vazquez 270d5299b67SRoman Gushchin /* Please, do not use this function outside from the map creation path 271d5299b67SRoman Gushchin * (e.g. in map update path) without taking care of setting the active 272d5299b67SRoman Gushchin * memory cgroup (see at bpf_map_kmalloc_node() for example). 273d5299b67SRoman Gushchin */ 274196e8ca7SDaniel Borkmann static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable) 275d407bd25SDaniel Borkmann { 276f01a7dbeSMartynas Pumputis /* We really just want to fail instead of triggering OOM killer 277f01a7dbeSMartynas Pumputis * under memory pressure, therefore we set __GFP_NORETRY to kmalloc, 278f01a7dbeSMartynas Pumputis * which is used for lower order allocation requests. 279f01a7dbeSMartynas Pumputis * 280f01a7dbeSMartynas Pumputis * It has been observed that higher order allocation requests done by 281f01a7dbeSMartynas Pumputis * vmalloc with __GFP_NORETRY being set might fail due to not trying 282f01a7dbeSMartynas Pumputis * to reclaim memory from the page cache, thus we set 283f01a7dbeSMartynas Pumputis * __GFP_RETRY_MAYFAIL to avoid such situations. 284d407bd25SDaniel Borkmann */ 285f01a7dbeSMartynas Pumputis 286d5299b67SRoman Gushchin const gfp_t gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_ACCOUNT; 287041de93fSChristoph Hellwig unsigned int flags = 0; 288041de93fSChristoph Hellwig unsigned long align = 1; 289d407bd25SDaniel Borkmann void *area; 290d407bd25SDaniel Borkmann 291196e8ca7SDaniel Borkmann if (size >= SIZE_MAX) 292196e8ca7SDaniel Borkmann return NULL; 293196e8ca7SDaniel Borkmann 294fc970227SAndrii Nakryiko /* kmalloc()'ed memory can't be mmap()'ed */ 295041de93fSChristoph Hellwig if (mmapable) { 296041de93fSChristoph Hellwig BUG_ON(!PAGE_ALIGNED(size)); 297041de93fSChristoph Hellwig align = SHMLBA; 298041de93fSChristoph Hellwig flags = VM_USERMAP; 299041de93fSChristoph Hellwig } else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 300041de93fSChristoph Hellwig area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY, 301f01a7dbeSMartynas Pumputis numa_node); 302d407bd25SDaniel Borkmann if (area != NULL) 303d407bd25SDaniel Borkmann return area; 304d407bd25SDaniel Borkmann } 305041de93fSChristoph Hellwig 306041de93fSChristoph Hellwig return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, 307041de93fSChristoph Hellwig gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL, 308041de93fSChristoph Hellwig flags, numa_node, __builtin_return_address(0)); 309d407bd25SDaniel Borkmann } 310d407bd25SDaniel Borkmann 311196e8ca7SDaniel Borkmann void *bpf_map_area_alloc(u64 size, int numa_node) 312fc970227SAndrii Nakryiko { 313fc970227SAndrii Nakryiko return __bpf_map_area_alloc(size, numa_node, false); 314fc970227SAndrii Nakryiko } 315fc970227SAndrii Nakryiko 316196e8ca7SDaniel Borkmann void *bpf_map_area_mmapable_alloc(u64 size, int numa_node) 317fc970227SAndrii Nakryiko { 318fc970227SAndrii Nakryiko return __bpf_map_area_alloc(size, numa_node, true); 319fc970227SAndrii Nakryiko } 320fc970227SAndrii Nakryiko 321d407bd25SDaniel Borkmann void bpf_map_area_free(void *area) 322d407bd25SDaniel Borkmann { 323d407bd25SDaniel Borkmann kvfree(area); 324d407bd25SDaniel Borkmann } 325d407bd25SDaniel Borkmann 326be70bcd5SDaniel Borkmann static u32 bpf_map_flags_retain_permanent(u32 flags) 327be70bcd5SDaniel Borkmann { 328be70bcd5SDaniel Borkmann /* Some map creation flags are not tied to the map object but 329be70bcd5SDaniel Borkmann * rather to the map fd instead, so they have no meaning upon 330be70bcd5SDaniel Borkmann * map object inspection since multiple file descriptors with 331be70bcd5SDaniel Borkmann * different (access) properties can exist here. Thus, given 332be70bcd5SDaniel Borkmann * this has zero meaning for the map itself, lets clear these 333be70bcd5SDaniel Borkmann * from here. 334be70bcd5SDaniel Borkmann */ 335be70bcd5SDaniel Borkmann return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY); 336be70bcd5SDaniel Borkmann } 337be70bcd5SDaniel Borkmann 338bd475643SJakub Kicinski void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) 339bd475643SJakub Kicinski { 340bd475643SJakub Kicinski map->map_type = attr->map_type; 341bd475643SJakub Kicinski map->key_size = attr->key_size; 342bd475643SJakub Kicinski map->value_size = attr->value_size; 343bd475643SJakub Kicinski map->max_entries = attr->max_entries; 344be70bcd5SDaniel Borkmann map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags); 345bd475643SJakub Kicinski map->numa_node = bpf_map_attr_numa_node(attr); 346bd475643SJakub Kicinski } 347bd475643SJakub Kicinski 348f3f1c054SMartin KaFai Lau static int bpf_map_alloc_id(struct bpf_map *map) 349f3f1c054SMartin KaFai Lau { 350f3f1c054SMartin KaFai Lau int id; 351f3f1c054SMartin KaFai Lau 352b76354cdSShaohua Li idr_preload(GFP_KERNEL); 353f3f1c054SMartin KaFai Lau spin_lock_bh(&map_idr_lock); 354f3f1c054SMartin KaFai Lau id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 355f3f1c054SMartin KaFai Lau if (id > 0) 356f3f1c054SMartin KaFai Lau map->id = id; 357f3f1c054SMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 358b76354cdSShaohua Li idr_preload_end(); 359f3f1c054SMartin KaFai Lau 360f3f1c054SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 361f3f1c054SMartin KaFai Lau return -ENOSPC; 362f3f1c054SMartin KaFai Lau 363f3f1c054SMartin KaFai Lau return id > 0 ? 0 : id; 364f3f1c054SMartin KaFai Lau } 365f3f1c054SMartin KaFai Lau 366a3884572SJakub Kicinski void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) 367f3f1c054SMartin KaFai Lau { 368930651a7SEric Dumazet unsigned long flags; 369930651a7SEric Dumazet 370a3884572SJakub Kicinski /* Offloaded maps are removed from the IDR store when their device 371a3884572SJakub Kicinski * disappears - even if someone holds an fd to them they are unusable, 372a3884572SJakub Kicinski * the memory is gone, all ops will fail; they are simply waiting for 373a3884572SJakub Kicinski * refcnt to drop to be freed. 374a3884572SJakub Kicinski */ 375a3884572SJakub Kicinski if (!map->id) 376a3884572SJakub Kicinski return; 377a3884572SJakub Kicinski 378bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 379930651a7SEric Dumazet spin_lock_irqsave(&map_idr_lock, flags); 380bd5f5f4eSMartin KaFai Lau else 381bd5f5f4eSMartin KaFai Lau __acquire(&map_idr_lock); 382bd5f5f4eSMartin KaFai Lau 383f3f1c054SMartin KaFai Lau idr_remove(&map_idr, map->id); 384a3884572SJakub Kicinski map->id = 0; 385bd5f5f4eSMartin KaFai Lau 386bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 387930651a7SEric Dumazet spin_unlock_irqrestore(&map_idr_lock, flags); 388bd5f5f4eSMartin KaFai Lau else 389bd5f5f4eSMartin KaFai Lau __release(&map_idr_lock); 390f3f1c054SMartin KaFai Lau } 391f3f1c054SMartin KaFai Lau 39248edc1f7SRoman Gushchin #ifdef CONFIG_MEMCG_KMEM 39348edc1f7SRoman Gushchin static void bpf_map_save_memcg(struct bpf_map *map) 39448edc1f7SRoman Gushchin { 39548edc1f7SRoman Gushchin map->memcg = get_mem_cgroup_from_mm(current->mm); 39648edc1f7SRoman Gushchin } 39748edc1f7SRoman Gushchin 39848edc1f7SRoman Gushchin static void bpf_map_release_memcg(struct bpf_map *map) 39948edc1f7SRoman Gushchin { 40048edc1f7SRoman Gushchin mem_cgroup_put(map->memcg); 40148edc1f7SRoman Gushchin } 40248edc1f7SRoman Gushchin 40348edc1f7SRoman Gushchin void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, 40448edc1f7SRoman Gushchin int node) 40548edc1f7SRoman Gushchin { 40648edc1f7SRoman Gushchin struct mem_cgroup *old_memcg; 40748edc1f7SRoman Gushchin void *ptr; 40848edc1f7SRoman Gushchin 40948edc1f7SRoman Gushchin old_memcg = set_active_memcg(map->memcg); 41048edc1f7SRoman Gushchin ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node); 41148edc1f7SRoman Gushchin set_active_memcg(old_memcg); 41248edc1f7SRoman Gushchin 41348edc1f7SRoman Gushchin return ptr; 41448edc1f7SRoman Gushchin } 41548edc1f7SRoman Gushchin 41648edc1f7SRoman Gushchin void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags) 41748edc1f7SRoman Gushchin { 41848edc1f7SRoman Gushchin struct mem_cgroup *old_memcg; 41948edc1f7SRoman Gushchin void *ptr; 42048edc1f7SRoman Gushchin 42148edc1f7SRoman Gushchin old_memcg = set_active_memcg(map->memcg); 42248edc1f7SRoman Gushchin ptr = kzalloc(size, flags | __GFP_ACCOUNT); 42348edc1f7SRoman Gushchin set_active_memcg(old_memcg); 42448edc1f7SRoman Gushchin 42548edc1f7SRoman Gushchin return ptr; 42648edc1f7SRoman Gushchin } 42748edc1f7SRoman Gushchin 42848edc1f7SRoman Gushchin void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, 42948edc1f7SRoman Gushchin size_t align, gfp_t flags) 43048edc1f7SRoman Gushchin { 43148edc1f7SRoman Gushchin struct mem_cgroup *old_memcg; 43248edc1f7SRoman Gushchin void __percpu *ptr; 43348edc1f7SRoman Gushchin 43448edc1f7SRoman Gushchin old_memcg = set_active_memcg(map->memcg); 43548edc1f7SRoman Gushchin ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT); 43648edc1f7SRoman Gushchin set_active_memcg(old_memcg); 43748edc1f7SRoman Gushchin 43848edc1f7SRoman Gushchin return ptr; 43948edc1f7SRoman Gushchin } 44048edc1f7SRoman Gushchin 44148edc1f7SRoman Gushchin #else 44248edc1f7SRoman Gushchin static void bpf_map_save_memcg(struct bpf_map *map) 44348edc1f7SRoman Gushchin { 44448edc1f7SRoman Gushchin } 44548edc1f7SRoman Gushchin 44648edc1f7SRoman Gushchin static void bpf_map_release_memcg(struct bpf_map *map) 44748edc1f7SRoman Gushchin { 44848edc1f7SRoman Gushchin } 44948edc1f7SRoman Gushchin #endif 45048edc1f7SRoman Gushchin 45199c55f7dSAlexei Starovoitov /* called from workqueue */ 45299c55f7dSAlexei Starovoitov static void bpf_map_free_deferred(struct work_struct *work) 45399c55f7dSAlexei Starovoitov { 45499c55f7dSAlexei Starovoitov struct bpf_map *map = container_of(work, struct bpf_map, work); 45599c55f7dSAlexei Starovoitov 456afdb09c7SChenbo Feng security_bpf_map_free(map); 45748edc1f7SRoman Gushchin bpf_map_release_memcg(map); 45899c55f7dSAlexei Starovoitov /* implementation dependent freeing */ 45999c55f7dSAlexei Starovoitov map->ops->map_free(map); 46099c55f7dSAlexei Starovoitov } 46199c55f7dSAlexei Starovoitov 462c9da161cSDaniel Borkmann static void bpf_map_put_uref(struct bpf_map *map) 463c9da161cSDaniel Borkmann { 4641e0bd5a0SAndrii Nakryiko if (atomic64_dec_and_test(&map->usercnt)) { 465ba6b8de4SJohn Fastabend if (map->ops->map_release_uref) 466ba6b8de4SJohn Fastabend map->ops->map_release_uref(map); 467c9da161cSDaniel Borkmann } 468c9da161cSDaniel Borkmann } 469c9da161cSDaniel Borkmann 47099c55f7dSAlexei Starovoitov /* decrement map refcnt and schedule it for freeing via workqueue 47199c55f7dSAlexei Starovoitov * (unrelying map implementation ops->map_free() might sleep) 47299c55f7dSAlexei Starovoitov */ 473bd5f5f4eSMartin KaFai Lau static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) 47499c55f7dSAlexei Starovoitov { 4751e0bd5a0SAndrii Nakryiko if (atomic64_dec_and_test(&map->refcnt)) { 47634ad5580SMartin KaFai Lau /* bpf_map_free_id() must be called first */ 477bd5f5f4eSMartin KaFai Lau bpf_map_free_id(map, do_idr_lock); 47878958fcaSMartin KaFai Lau btf_put(map->btf); 47999c55f7dSAlexei Starovoitov INIT_WORK(&map->work, bpf_map_free_deferred); 48099c55f7dSAlexei Starovoitov schedule_work(&map->work); 48199c55f7dSAlexei Starovoitov } 48299c55f7dSAlexei Starovoitov } 48399c55f7dSAlexei Starovoitov 484bd5f5f4eSMartin KaFai Lau void bpf_map_put(struct bpf_map *map) 485bd5f5f4eSMartin KaFai Lau { 486bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, true); 487bd5f5f4eSMartin KaFai Lau } 488630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_put); 489bd5f5f4eSMartin KaFai Lau 490c9da161cSDaniel Borkmann void bpf_map_put_with_uref(struct bpf_map *map) 491c9da161cSDaniel Borkmann { 492c9da161cSDaniel Borkmann bpf_map_put_uref(map); 493c9da161cSDaniel Borkmann bpf_map_put(map); 494c9da161cSDaniel Borkmann } 495c9da161cSDaniel Borkmann 49699c55f7dSAlexei Starovoitov static int bpf_map_release(struct inode *inode, struct file *filp) 49799c55f7dSAlexei Starovoitov { 49861d1b6a4SDaniel Borkmann struct bpf_map *map = filp->private_data; 49961d1b6a4SDaniel Borkmann 50061d1b6a4SDaniel Borkmann if (map->ops->map_release) 50161d1b6a4SDaniel Borkmann map->ops->map_release(map, filp); 50261d1b6a4SDaniel Borkmann 50361d1b6a4SDaniel Borkmann bpf_map_put_with_uref(map); 50499c55f7dSAlexei Starovoitov return 0; 50599c55f7dSAlexei Starovoitov } 50699c55f7dSAlexei Starovoitov 50787df15deSDaniel Borkmann static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f) 50887df15deSDaniel Borkmann { 50987df15deSDaniel Borkmann fmode_t mode = f.file->f_mode; 51087df15deSDaniel Borkmann 51187df15deSDaniel Borkmann /* Our file permissions may have been overridden by global 51287df15deSDaniel Borkmann * map permissions facing syscall side. 51387df15deSDaniel Borkmann */ 51487df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) 51587df15deSDaniel Borkmann mode &= ~FMODE_CAN_WRITE; 51687df15deSDaniel Borkmann return mode; 51787df15deSDaniel Borkmann } 51887df15deSDaniel Borkmann 519f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 52080ee81e0SRoman Gushchin /* Provides an approximation of the map's memory footprint. 52180ee81e0SRoman Gushchin * Used only to provide a backward compatibility and display 52280ee81e0SRoman Gushchin * a reasonable "memlock" info. 52380ee81e0SRoman Gushchin */ 52480ee81e0SRoman Gushchin static unsigned long bpf_map_memory_footprint(const struct bpf_map *map) 52580ee81e0SRoman Gushchin { 52680ee81e0SRoman Gushchin unsigned long size; 52780ee81e0SRoman Gushchin 52880ee81e0SRoman Gushchin size = round_up(map->key_size + bpf_map_value_size(map), 8); 52980ee81e0SRoman Gushchin 53080ee81e0SRoman Gushchin return round_up(map->max_entries * size, PAGE_SIZE); 53180ee81e0SRoman Gushchin } 53280ee81e0SRoman Gushchin 533f99bf205SDaniel Borkmann static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 534f99bf205SDaniel Borkmann { 535f99bf205SDaniel Borkmann const struct bpf_map *map = filp->private_data; 53621116b70SDaniel Borkmann const struct bpf_array *array; 5372beee5f5SDaniel Borkmann u32 type = 0, jited = 0; 53821116b70SDaniel Borkmann 53921116b70SDaniel Borkmann if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 54021116b70SDaniel Borkmann array = container_of(map, struct bpf_array, map); 5412beee5f5SDaniel Borkmann type = array->aux->type; 5422beee5f5SDaniel Borkmann jited = array->aux->jited; 54321116b70SDaniel Borkmann } 544f99bf205SDaniel Borkmann 545f99bf205SDaniel Borkmann seq_printf(m, 546f99bf205SDaniel Borkmann "map_type:\t%u\n" 547f99bf205SDaniel Borkmann "key_size:\t%u\n" 548f99bf205SDaniel Borkmann "value_size:\t%u\n" 549322cea2fSDaniel Borkmann "max_entries:\t%u\n" 55021116b70SDaniel Borkmann "map_flags:\t%#x\n" 55180ee81e0SRoman Gushchin "memlock:\t%lu\n" 55287df15deSDaniel Borkmann "map_id:\t%u\n" 55387df15deSDaniel Borkmann "frozen:\t%u\n", 554f99bf205SDaniel Borkmann map->map_type, 555f99bf205SDaniel Borkmann map->key_size, 556f99bf205SDaniel Borkmann map->value_size, 557322cea2fSDaniel Borkmann map->max_entries, 55821116b70SDaniel Borkmann map->map_flags, 55980ee81e0SRoman Gushchin bpf_map_memory_footprint(map), 56087df15deSDaniel Borkmann map->id, 56187df15deSDaniel Borkmann READ_ONCE(map->frozen)); 5622beee5f5SDaniel Borkmann if (type) { 5632beee5f5SDaniel Borkmann seq_printf(m, "owner_prog_type:\t%u\n", type); 5642beee5f5SDaniel Borkmann seq_printf(m, "owner_jited:\t%u\n", jited); 5659780c0abSDaniel Borkmann } 566f99bf205SDaniel Borkmann } 567f99bf205SDaniel Borkmann #endif 568f99bf205SDaniel Borkmann 5696e71b04aSChenbo Feng static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, 5706e71b04aSChenbo Feng loff_t *ppos) 5716e71b04aSChenbo Feng { 5726e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 5736e71b04aSChenbo Feng * f_mode with FMODE_CAN_READ. 5746e71b04aSChenbo Feng */ 5756e71b04aSChenbo Feng return -EINVAL; 5766e71b04aSChenbo Feng } 5776e71b04aSChenbo Feng 5786e71b04aSChenbo Feng static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, 5796e71b04aSChenbo Feng size_t siz, loff_t *ppos) 5806e71b04aSChenbo Feng { 5816e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 5826e71b04aSChenbo Feng * f_mode with FMODE_CAN_WRITE. 5836e71b04aSChenbo Feng */ 5846e71b04aSChenbo Feng return -EINVAL; 5856e71b04aSChenbo Feng } 5866e71b04aSChenbo Feng 587fc970227SAndrii Nakryiko /* called for any extra memory-mapped regions (except initial) */ 588fc970227SAndrii Nakryiko static void bpf_map_mmap_open(struct vm_area_struct *vma) 589fc970227SAndrii Nakryiko { 590fc970227SAndrii Nakryiko struct bpf_map *map = vma->vm_file->private_data; 591fc970227SAndrii Nakryiko 5921f6cb19bSAndrii Nakryiko if (vma->vm_flags & VM_MAYWRITE) { 593fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 594fc970227SAndrii Nakryiko map->writecnt++; 595fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 596fc970227SAndrii Nakryiko } 597fc970227SAndrii Nakryiko } 598fc970227SAndrii Nakryiko 599fc970227SAndrii Nakryiko /* called for all unmapped memory region (including initial) */ 600fc970227SAndrii Nakryiko static void bpf_map_mmap_close(struct vm_area_struct *vma) 601fc970227SAndrii Nakryiko { 602fc970227SAndrii Nakryiko struct bpf_map *map = vma->vm_file->private_data; 603fc970227SAndrii Nakryiko 6041f6cb19bSAndrii Nakryiko if (vma->vm_flags & VM_MAYWRITE) { 605fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 606fc970227SAndrii Nakryiko map->writecnt--; 607fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 608fc970227SAndrii Nakryiko } 609fc970227SAndrii Nakryiko } 610fc970227SAndrii Nakryiko 611fc970227SAndrii Nakryiko static const struct vm_operations_struct bpf_map_default_vmops = { 612fc970227SAndrii Nakryiko .open = bpf_map_mmap_open, 613fc970227SAndrii Nakryiko .close = bpf_map_mmap_close, 614fc970227SAndrii Nakryiko }; 615fc970227SAndrii Nakryiko 616fc970227SAndrii Nakryiko static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma) 617fc970227SAndrii Nakryiko { 618fc970227SAndrii Nakryiko struct bpf_map *map = filp->private_data; 619fc970227SAndrii Nakryiko int err; 620fc970227SAndrii Nakryiko 621fc970227SAndrii Nakryiko if (!map->ops->map_mmap || map_value_has_spin_lock(map)) 622fc970227SAndrii Nakryiko return -ENOTSUPP; 623fc970227SAndrii Nakryiko 624fc970227SAndrii Nakryiko if (!(vma->vm_flags & VM_SHARED)) 625fc970227SAndrii Nakryiko return -EINVAL; 626fc970227SAndrii Nakryiko 627fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 628fc970227SAndrii Nakryiko 629dfeb376dSAndrii Nakryiko if (vma->vm_flags & VM_WRITE) { 630dfeb376dSAndrii Nakryiko if (map->frozen) { 631fc970227SAndrii Nakryiko err = -EPERM; 632fc970227SAndrii Nakryiko goto out; 633fc970227SAndrii Nakryiko } 634dfeb376dSAndrii Nakryiko /* map is meant to be read-only, so do not allow mapping as 635dfeb376dSAndrii Nakryiko * writable, because it's possible to leak a writable page 636dfeb376dSAndrii Nakryiko * reference and allows user-space to still modify it after 637dfeb376dSAndrii Nakryiko * freezing, while verifier will assume contents do not change 638dfeb376dSAndrii Nakryiko */ 639dfeb376dSAndrii Nakryiko if (map->map_flags & BPF_F_RDONLY_PROG) { 640dfeb376dSAndrii Nakryiko err = -EACCES; 641dfeb376dSAndrii Nakryiko goto out; 642dfeb376dSAndrii Nakryiko } 643dfeb376dSAndrii Nakryiko } 644fc970227SAndrii Nakryiko 645fc970227SAndrii Nakryiko /* set default open/close callbacks */ 646fc970227SAndrii Nakryiko vma->vm_ops = &bpf_map_default_vmops; 647fc970227SAndrii Nakryiko vma->vm_private_data = map; 6481f6cb19bSAndrii Nakryiko vma->vm_flags &= ~VM_MAYEXEC; 6491f6cb19bSAndrii Nakryiko if (!(vma->vm_flags & VM_WRITE)) 6501f6cb19bSAndrii Nakryiko /* disallow re-mapping with PROT_WRITE */ 6511f6cb19bSAndrii Nakryiko vma->vm_flags &= ~VM_MAYWRITE; 652fc970227SAndrii Nakryiko 653fc970227SAndrii Nakryiko err = map->ops->map_mmap(map, vma); 654fc970227SAndrii Nakryiko if (err) 655fc970227SAndrii Nakryiko goto out; 656fc970227SAndrii Nakryiko 6571f6cb19bSAndrii Nakryiko if (vma->vm_flags & VM_MAYWRITE) 658fc970227SAndrii Nakryiko map->writecnt++; 659fc970227SAndrii Nakryiko out: 660fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 661fc970227SAndrii Nakryiko return err; 662fc970227SAndrii Nakryiko } 663fc970227SAndrii Nakryiko 664457f4436SAndrii Nakryiko static __poll_t bpf_map_poll(struct file *filp, struct poll_table_struct *pts) 665457f4436SAndrii Nakryiko { 666457f4436SAndrii Nakryiko struct bpf_map *map = filp->private_data; 667457f4436SAndrii Nakryiko 668457f4436SAndrii Nakryiko if (map->ops->map_poll) 669457f4436SAndrii Nakryiko return map->ops->map_poll(map, filp, pts); 670457f4436SAndrii Nakryiko 671457f4436SAndrii Nakryiko return EPOLLERR; 672457f4436SAndrii Nakryiko } 673457f4436SAndrii Nakryiko 674f66e448cSChenbo Feng const struct file_operations bpf_map_fops = { 675f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 676f99bf205SDaniel Borkmann .show_fdinfo = bpf_map_show_fdinfo, 677f99bf205SDaniel Borkmann #endif 67899c55f7dSAlexei Starovoitov .release = bpf_map_release, 6796e71b04aSChenbo Feng .read = bpf_dummy_read, 6806e71b04aSChenbo Feng .write = bpf_dummy_write, 681fc970227SAndrii Nakryiko .mmap = bpf_map_mmap, 682457f4436SAndrii Nakryiko .poll = bpf_map_poll, 68399c55f7dSAlexei Starovoitov }; 68499c55f7dSAlexei Starovoitov 6856e71b04aSChenbo Feng int bpf_map_new_fd(struct bpf_map *map, int flags) 686aa79781bSDaniel Borkmann { 687afdb09c7SChenbo Feng int ret; 688afdb09c7SChenbo Feng 689afdb09c7SChenbo Feng ret = security_bpf_map(map, OPEN_FMODE(flags)); 690afdb09c7SChenbo Feng if (ret < 0) 691afdb09c7SChenbo Feng return ret; 692afdb09c7SChenbo Feng 693aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 6946e71b04aSChenbo Feng flags | O_CLOEXEC); 6956e71b04aSChenbo Feng } 6966e71b04aSChenbo Feng 6976e71b04aSChenbo Feng int bpf_get_file_flag(int flags) 6986e71b04aSChenbo Feng { 6996e71b04aSChenbo Feng if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) 7006e71b04aSChenbo Feng return -EINVAL; 7016e71b04aSChenbo Feng if (flags & BPF_F_RDONLY) 7026e71b04aSChenbo Feng return O_RDONLY; 7036e71b04aSChenbo Feng if (flags & BPF_F_WRONLY) 7046e71b04aSChenbo Feng return O_WRONLY; 7056e71b04aSChenbo Feng return O_RDWR; 706aa79781bSDaniel Borkmann } 707aa79781bSDaniel Borkmann 70899c55f7dSAlexei Starovoitov /* helper macro to check that unused fields 'union bpf_attr' are zero */ 70999c55f7dSAlexei Starovoitov #define CHECK_ATTR(CMD) \ 71099c55f7dSAlexei Starovoitov memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 71199c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD), 0, \ 71299c55f7dSAlexei Starovoitov sizeof(*attr) - \ 71399c55f7dSAlexei Starovoitov offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 71499c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD)) != NULL 71599c55f7dSAlexei Starovoitov 7168e7ae251SMartin KaFai Lau /* dst and src must have at least "size" number of bytes. 7178e7ae251SMartin KaFai Lau * Return strlen on success and < 0 on error. 718cb4d2b3fSMartin KaFai Lau */ 7198e7ae251SMartin KaFai Lau int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size) 720cb4d2b3fSMartin KaFai Lau { 7218e7ae251SMartin KaFai Lau const char *end = src + size; 7228e7ae251SMartin KaFai Lau const char *orig_src = src; 723cb4d2b3fSMartin KaFai Lau 7248e7ae251SMartin KaFai Lau memset(dst, 0, size); 7253e0ddc4fSDaniel Borkmann /* Copy all isalnum(), '_' and '.' chars. */ 726cb4d2b3fSMartin KaFai Lau while (src < end && *src) { 7273e0ddc4fSDaniel Borkmann if (!isalnum(*src) && 7283e0ddc4fSDaniel Borkmann *src != '_' && *src != '.') 729cb4d2b3fSMartin KaFai Lau return -EINVAL; 730cb4d2b3fSMartin KaFai Lau *dst++ = *src++; 731cb4d2b3fSMartin KaFai Lau } 732cb4d2b3fSMartin KaFai Lau 7338e7ae251SMartin KaFai Lau /* No '\0' found in "size" number of bytes */ 734cb4d2b3fSMartin KaFai Lau if (src == end) 735cb4d2b3fSMartin KaFai Lau return -EINVAL; 736cb4d2b3fSMartin KaFai Lau 7378e7ae251SMartin KaFai Lau return src - orig_src; 738cb4d2b3fSMartin KaFai Lau } 739cb4d2b3fSMartin KaFai Lau 740e8d2bec0SDaniel Borkmann int map_check_no_btf(const struct bpf_map *map, 7411b2b234bSRoman Gushchin const struct btf *btf, 742e8d2bec0SDaniel Borkmann const struct btf_type *key_type, 743e8d2bec0SDaniel Borkmann const struct btf_type *value_type) 744e8d2bec0SDaniel Borkmann { 745e8d2bec0SDaniel Borkmann return -ENOTSUPP; 746e8d2bec0SDaniel Borkmann } 747e8d2bec0SDaniel Borkmann 748d83525caSAlexei Starovoitov static int map_check_btf(struct bpf_map *map, const struct btf *btf, 749e8d2bec0SDaniel Borkmann u32 btf_key_id, u32 btf_value_id) 750e8d2bec0SDaniel Borkmann { 751e8d2bec0SDaniel Borkmann const struct btf_type *key_type, *value_type; 752e8d2bec0SDaniel Borkmann u32 key_size, value_size; 753e8d2bec0SDaniel Borkmann int ret = 0; 754e8d2bec0SDaniel Borkmann 7552824ecb7SDaniel Borkmann /* Some maps allow key to be unspecified. */ 7562824ecb7SDaniel Borkmann if (btf_key_id) { 757e8d2bec0SDaniel Borkmann key_type = btf_type_id_size(btf, &btf_key_id, &key_size); 758e8d2bec0SDaniel Borkmann if (!key_type || key_size != map->key_size) 759e8d2bec0SDaniel Borkmann return -EINVAL; 7602824ecb7SDaniel Borkmann } else { 7612824ecb7SDaniel Borkmann key_type = btf_type_by_id(btf, 0); 7622824ecb7SDaniel Borkmann if (!map->ops->map_check_btf) 7632824ecb7SDaniel Borkmann return -EINVAL; 7642824ecb7SDaniel Borkmann } 765e8d2bec0SDaniel Borkmann 766e8d2bec0SDaniel Borkmann value_type = btf_type_id_size(btf, &btf_value_id, &value_size); 767e8d2bec0SDaniel Borkmann if (!value_type || value_size != map->value_size) 768e8d2bec0SDaniel Borkmann return -EINVAL; 769e8d2bec0SDaniel Borkmann 770d83525caSAlexei Starovoitov map->spin_lock_off = btf_find_spin_lock(btf, value_type); 771d83525caSAlexei Starovoitov 772d83525caSAlexei Starovoitov if (map_value_has_spin_lock(map)) { 773591fe988SDaniel Borkmann if (map->map_flags & BPF_F_RDONLY_PROG) 774591fe988SDaniel Borkmann return -EACCES; 775d83525caSAlexei Starovoitov if (map->map_type != BPF_MAP_TYPE_HASH && 776e16d2f1aSAlexei Starovoitov map->map_type != BPF_MAP_TYPE_ARRAY && 7776ac99e8fSMartin KaFai Lau map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 7788ea63684SKP Singh map->map_type != BPF_MAP_TYPE_SK_STORAGE && 7794cf1bc1fSKP Singh map->map_type != BPF_MAP_TYPE_INODE_STORAGE && 7804cf1bc1fSKP Singh map->map_type != BPF_MAP_TYPE_TASK_STORAGE) 781d83525caSAlexei Starovoitov return -ENOTSUPP; 782d83525caSAlexei Starovoitov if (map->spin_lock_off + sizeof(struct bpf_spin_lock) > 783d83525caSAlexei Starovoitov map->value_size) { 784d83525caSAlexei Starovoitov WARN_ONCE(1, 785d83525caSAlexei Starovoitov "verifier bug spin_lock_off %d value_size %d\n", 786d83525caSAlexei Starovoitov map->spin_lock_off, map->value_size); 787d83525caSAlexei Starovoitov return -EFAULT; 788d83525caSAlexei Starovoitov } 789d83525caSAlexei Starovoitov } 790d83525caSAlexei Starovoitov 791e8d2bec0SDaniel Borkmann if (map->ops->map_check_btf) 7921b2b234bSRoman Gushchin ret = map->ops->map_check_btf(map, btf, key_type, value_type); 793e8d2bec0SDaniel Borkmann 794e8d2bec0SDaniel Borkmann return ret; 795e8d2bec0SDaniel Borkmann } 796e8d2bec0SDaniel Borkmann 79785d33df3SMartin KaFai Lau #define BPF_MAP_CREATE_LAST_FIELD btf_vmlinux_value_type_id 79899c55f7dSAlexei Starovoitov /* called via syscall */ 79999c55f7dSAlexei Starovoitov static int map_create(union bpf_attr *attr) 80099c55f7dSAlexei Starovoitov { 80196eabe7aSMartin KaFai Lau int numa_node = bpf_map_attr_numa_node(attr); 80299c55f7dSAlexei Starovoitov struct bpf_map *map; 8036e71b04aSChenbo Feng int f_flags; 80499c55f7dSAlexei Starovoitov int err; 80599c55f7dSAlexei Starovoitov 80699c55f7dSAlexei Starovoitov err = CHECK_ATTR(BPF_MAP_CREATE); 80799c55f7dSAlexei Starovoitov if (err) 80899c55f7dSAlexei Starovoitov return -EINVAL; 80999c55f7dSAlexei Starovoitov 81085d33df3SMartin KaFai Lau if (attr->btf_vmlinux_value_type_id) { 81185d33df3SMartin KaFai Lau if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS || 81285d33df3SMartin KaFai Lau attr->btf_key_type_id || attr->btf_value_type_id) 81385d33df3SMartin KaFai Lau return -EINVAL; 81485d33df3SMartin KaFai Lau } else if (attr->btf_key_type_id && !attr->btf_value_type_id) { 81585d33df3SMartin KaFai Lau return -EINVAL; 81685d33df3SMartin KaFai Lau } 81785d33df3SMartin KaFai Lau 8186e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->map_flags); 8196e71b04aSChenbo Feng if (f_flags < 0) 8206e71b04aSChenbo Feng return f_flags; 8216e71b04aSChenbo Feng 82296eabe7aSMartin KaFai Lau if (numa_node != NUMA_NO_NODE && 82396e5ae4eSEric Dumazet ((unsigned int)numa_node >= nr_node_ids || 82496e5ae4eSEric Dumazet !node_online(numa_node))) 82596eabe7aSMartin KaFai Lau return -EINVAL; 82696eabe7aSMartin KaFai Lau 82799c55f7dSAlexei Starovoitov /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 82899c55f7dSAlexei Starovoitov map = find_and_alloc_map(attr); 82999c55f7dSAlexei Starovoitov if (IS_ERR(map)) 83099c55f7dSAlexei Starovoitov return PTR_ERR(map); 83199c55f7dSAlexei Starovoitov 8328e7ae251SMartin KaFai Lau err = bpf_obj_name_cpy(map->name, attr->map_name, 8338e7ae251SMartin KaFai Lau sizeof(attr->map_name)); 8348e7ae251SMartin KaFai Lau if (err < 0) 835b936ca64SRoman Gushchin goto free_map; 836ad5b177bSMartin KaFai Lau 8371e0bd5a0SAndrii Nakryiko atomic64_set(&map->refcnt, 1); 8381e0bd5a0SAndrii Nakryiko atomic64_set(&map->usercnt, 1); 839fc970227SAndrii Nakryiko mutex_init(&map->freeze_mutex); 84099c55f7dSAlexei Starovoitov 84185d33df3SMartin KaFai Lau map->spin_lock_off = -EINVAL; 84285d33df3SMartin KaFai Lau if (attr->btf_key_type_id || attr->btf_value_type_id || 84385d33df3SMartin KaFai Lau /* Even the map's value is a kernel's struct, 84485d33df3SMartin KaFai Lau * the bpf_prog.o must have BTF to begin with 84585d33df3SMartin KaFai Lau * to figure out the corresponding kernel's 84685d33df3SMartin KaFai Lau * counter part. Thus, attr->btf_fd has 84785d33df3SMartin KaFai Lau * to be valid also. 84885d33df3SMartin KaFai Lau */ 84985d33df3SMartin KaFai Lau attr->btf_vmlinux_value_type_id) { 850a26ca7c9SMartin KaFai Lau struct btf *btf; 851a26ca7c9SMartin KaFai Lau 852a26ca7c9SMartin KaFai Lau btf = btf_get_by_fd(attr->btf_fd); 853a26ca7c9SMartin KaFai Lau if (IS_ERR(btf)) { 854a26ca7c9SMartin KaFai Lau err = PTR_ERR(btf); 855b936ca64SRoman Gushchin goto free_map; 856a26ca7c9SMartin KaFai Lau } 857350a5c4dSAlexei Starovoitov if (btf_is_kernel(btf)) { 858350a5c4dSAlexei Starovoitov btf_put(btf); 859350a5c4dSAlexei Starovoitov err = -EACCES; 860350a5c4dSAlexei Starovoitov goto free_map; 861350a5c4dSAlexei Starovoitov } 86285d33df3SMartin KaFai Lau map->btf = btf; 863a26ca7c9SMartin KaFai Lau 86485d33df3SMartin KaFai Lau if (attr->btf_value_type_id) { 865e8d2bec0SDaniel Borkmann err = map_check_btf(map, btf, attr->btf_key_type_id, 8669b2cf328SMartin KaFai Lau attr->btf_value_type_id); 86785d33df3SMartin KaFai Lau if (err) 868b936ca64SRoman Gushchin goto free_map; 869a26ca7c9SMartin KaFai Lau } 870a26ca7c9SMartin KaFai Lau 8719b2cf328SMartin KaFai Lau map->btf_key_type_id = attr->btf_key_type_id; 8729b2cf328SMartin KaFai Lau map->btf_value_type_id = attr->btf_value_type_id; 87385d33df3SMartin KaFai Lau map->btf_vmlinux_value_type_id = 87485d33df3SMartin KaFai Lau attr->btf_vmlinux_value_type_id; 875a26ca7c9SMartin KaFai Lau } 876a26ca7c9SMartin KaFai Lau 877afdb09c7SChenbo Feng err = security_bpf_map_alloc(map); 878aaac3ba9SAlexei Starovoitov if (err) 879b936ca64SRoman Gushchin goto free_map; 880afdb09c7SChenbo Feng 881f3f1c054SMartin KaFai Lau err = bpf_map_alloc_id(map); 882f3f1c054SMartin KaFai Lau if (err) 883b936ca64SRoman Gushchin goto free_map_sec; 884f3f1c054SMartin KaFai Lau 88548edc1f7SRoman Gushchin bpf_map_save_memcg(map); 88648edc1f7SRoman Gushchin 8876e71b04aSChenbo Feng err = bpf_map_new_fd(map, f_flags); 888bd5f5f4eSMartin KaFai Lau if (err < 0) { 889bd5f5f4eSMartin KaFai Lau /* failed to allocate fd. 890352d20d6SPeng Sun * bpf_map_put_with_uref() is needed because the above 891bd5f5f4eSMartin KaFai Lau * bpf_map_alloc_id() has published the map 892bd5f5f4eSMartin KaFai Lau * to the userspace and the userspace may 893bd5f5f4eSMartin KaFai Lau * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 894bd5f5f4eSMartin KaFai Lau */ 895352d20d6SPeng Sun bpf_map_put_with_uref(map); 896bd5f5f4eSMartin KaFai Lau return err; 897bd5f5f4eSMartin KaFai Lau } 89899c55f7dSAlexei Starovoitov 89999c55f7dSAlexei Starovoitov return err; 90099c55f7dSAlexei Starovoitov 901afdb09c7SChenbo Feng free_map_sec: 902afdb09c7SChenbo Feng security_bpf_map_free(map); 903b936ca64SRoman Gushchin free_map: 904a26ca7c9SMartin KaFai Lau btf_put(map->btf); 90599c55f7dSAlexei Starovoitov map->ops->map_free(map); 90699c55f7dSAlexei Starovoitov return err; 90799c55f7dSAlexei Starovoitov } 90899c55f7dSAlexei Starovoitov 909db20fd2bSAlexei Starovoitov /* if error is returned, fd is released. 910db20fd2bSAlexei Starovoitov * On success caller should complete fd access with matching fdput() 911db20fd2bSAlexei Starovoitov */ 912c2101297SDaniel Borkmann struct bpf_map *__bpf_map_get(struct fd f) 913db20fd2bSAlexei Starovoitov { 914db20fd2bSAlexei Starovoitov if (!f.file) 915db20fd2bSAlexei Starovoitov return ERR_PTR(-EBADF); 916db20fd2bSAlexei Starovoitov if (f.file->f_op != &bpf_map_fops) { 917db20fd2bSAlexei Starovoitov fdput(f); 918db20fd2bSAlexei Starovoitov return ERR_PTR(-EINVAL); 919db20fd2bSAlexei Starovoitov } 920db20fd2bSAlexei Starovoitov 921c2101297SDaniel Borkmann return f.file->private_data; 922c2101297SDaniel Borkmann } 923c2101297SDaniel Borkmann 9241e0bd5a0SAndrii Nakryiko void bpf_map_inc(struct bpf_map *map) 925c9da161cSDaniel Borkmann { 9261e0bd5a0SAndrii Nakryiko atomic64_inc(&map->refcnt); 927c9da161cSDaniel Borkmann } 928630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_inc); 929c9da161cSDaniel Borkmann 9301e0bd5a0SAndrii Nakryiko void bpf_map_inc_with_uref(struct bpf_map *map) 9311e0bd5a0SAndrii Nakryiko { 9321e0bd5a0SAndrii Nakryiko atomic64_inc(&map->refcnt); 9331e0bd5a0SAndrii Nakryiko atomic64_inc(&map->usercnt); 9341e0bd5a0SAndrii Nakryiko } 9351e0bd5a0SAndrii Nakryiko EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref); 9361e0bd5a0SAndrii Nakryiko 9371ed4d924SMartin KaFai Lau struct bpf_map *bpf_map_get(u32 ufd) 9381ed4d924SMartin KaFai Lau { 9391ed4d924SMartin KaFai Lau struct fd f = fdget(ufd); 9401ed4d924SMartin KaFai Lau struct bpf_map *map; 9411ed4d924SMartin KaFai Lau 9421ed4d924SMartin KaFai Lau map = __bpf_map_get(f); 9431ed4d924SMartin KaFai Lau if (IS_ERR(map)) 9441ed4d924SMartin KaFai Lau return map; 9451ed4d924SMartin KaFai Lau 9461ed4d924SMartin KaFai Lau bpf_map_inc(map); 9471ed4d924SMartin KaFai Lau fdput(f); 9481ed4d924SMartin KaFai Lau 9491ed4d924SMartin KaFai Lau return map; 9501ed4d924SMartin KaFai Lau } 9511ed4d924SMartin KaFai Lau 952c9da161cSDaniel Borkmann struct bpf_map *bpf_map_get_with_uref(u32 ufd) 953c2101297SDaniel Borkmann { 954c2101297SDaniel Borkmann struct fd f = fdget(ufd); 955c2101297SDaniel Borkmann struct bpf_map *map; 956c2101297SDaniel Borkmann 957c2101297SDaniel Borkmann map = __bpf_map_get(f); 958c2101297SDaniel Borkmann if (IS_ERR(map)) 959c2101297SDaniel Borkmann return map; 960c2101297SDaniel Borkmann 9611e0bd5a0SAndrii Nakryiko bpf_map_inc_with_uref(map); 962c2101297SDaniel Borkmann fdput(f); 963db20fd2bSAlexei Starovoitov 964db20fd2bSAlexei Starovoitov return map; 965db20fd2bSAlexei Starovoitov } 966db20fd2bSAlexei Starovoitov 967bd5f5f4eSMartin KaFai Lau /* map_idr_lock should have been held */ 9681e0bd5a0SAndrii Nakryiko static struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref) 969bd5f5f4eSMartin KaFai Lau { 970bd5f5f4eSMartin KaFai Lau int refold; 971bd5f5f4eSMartin KaFai Lau 9721e0bd5a0SAndrii Nakryiko refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0); 973bd5f5f4eSMartin KaFai Lau if (!refold) 974bd5f5f4eSMartin KaFai Lau return ERR_PTR(-ENOENT); 975bd5f5f4eSMartin KaFai Lau if (uref) 9761e0bd5a0SAndrii Nakryiko atomic64_inc(&map->usercnt); 977bd5f5f4eSMartin KaFai Lau 978bd5f5f4eSMartin KaFai Lau return map; 979bd5f5f4eSMartin KaFai Lau } 980bd5f5f4eSMartin KaFai Lau 9811e0bd5a0SAndrii Nakryiko struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map) 982b0e4701cSStanislav Fomichev { 983b0e4701cSStanislav Fomichev spin_lock_bh(&map_idr_lock); 9841e0bd5a0SAndrii Nakryiko map = __bpf_map_inc_not_zero(map, false); 985b0e4701cSStanislav Fomichev spin_unlock_bh(&map_idr_lock); 986b0e4701cSStanislav Fomichev 987b0e4701cSStanislav Fomichev return map; 988b0e4701cSStanislav Fomichev } 989b0e4701cSStanislav Fomichev EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero); 990b0e4701cSStanislav Fomichev 991b8cdc051SAlexei Starovoitov int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 992b8cdc051SAlexei Starovoitov { 993b8cdc051SAlexei Starovoitov return -ENOTSUPP; 994b8cdc051SAlexei Starovoitov } 995b8cdc051SAlexei Starovoitov 996c9d29f46SMauricio Vasquez B static void *__bpf_copy_key(void __user *ukey, u64 key_size) 997c9d29f46SMauricio Vasquez B { 998c9d29f46SMauricio Vasquez B if (key_size) 999c9d29f46SMauricio Vasquez B return memdup_user(ukey, key_size); 1000c9d29f46SMauricio Vasquez B 1001c9d29f46SMauricio Vasquez B if (ukey) 1002c9d29f46SMauricio Vasquez B return ERR_PTR(-EINVAL); 1003c9d29f46SMauricio Vasquez B 1004c9d29f46SMauricio Vasquez B return NULL; 1005c9d29f46SMauricio Vasquez B } 1006c9d29f46SMauricio Vasquez B 1007db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 100896049f3aSAlexei Starovoitov #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags 1009db20fd2bSAlexei Starovoitov 1010db20fd2bSAlexei Starovoitov static int map_lookup_elem(union bpf_attr *attr) 1011db20fd2bSAlexei Starovoitov { 1012535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1013535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 1014db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1015db20fd2bSAlexei Starovoitov struct bpf_map *map; 101615c14a3dSBrian Vazquez void *key, *value; 101715a07b33SAlexei Starovoitov u32 value_size; 1018592867bfSDaniel Borkmann struct fd f; 1019db20fd2bSAlexei Starovoitov int err; 1020db20fd2bSAlexei Starovoitov 1021db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 1022db20fd2bSAlexei Starovoitov return -EINVAL; 1023db20fd2bSAlexei Starovoitov 102496049f3aSAlexei Starovoitov if (attr->flags & ~BPF_F_LOCK) 102596049f3aSAlexei Starovoitov return -EINVAL; 102696049f3aSAlexei Starovoitov 1027592867bfSDaniel Borkmann f = fdget(ufd); 1028c2101297SDaniel Borkmann map = __bpf_map_get(f); 1029db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1030db20fd2bSAlexei Starovoitov return PTR_ERR(map); 103187df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 10326e71b04aSChenbo Feng err = -EPERM; 10336e71b04aSChenbo Feng goto err_put; 10346e71b04aSChenbo Feng } 10356e71b04aSChenbo Feng 103696049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 103796049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 103896049f3aSAlexei Starovoitov err = -EINVAL; 103996049f3aSAlexei Starovoitov goto err_put; 104096049f3aSAlexei Starovoitov } 104196049f3aSAlexei Starovoitov 1042c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1043e4448ed8SAl Viro if (IS_ERR(key)) { 1044e4448ed8SAl Viro err = PTR_ERR(key); 1045db20fd2bSAlexei Starovoitov goto err_put; 1046e4448ed8SAl Viro } 1047db20fd2bSAlexei Starovoitov 104815c14a3dSBrian Vazquez value_size = bpf_map_value_size(map); 104915a07b33SAlexei Starovoitov 10508ebe667cSAlexei Starovoitov err = -ENOMEM; 105115a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1052db20fd2bSAlexei Starovoitov if (!value) 10538ebe667cSAlexei Starovoitov goto free_key; 10548ebe667cSAlexei Starovoitov 105515c14a3dSBrian Vazquez err = bpf_map_copy_value(map, key, value, attr->flags); 105615a07b33SAlexei Starovoitov if (err) 10578ebe667cSAlexei Starovoitov goto free_value; 1058db20fd2bSAlexei Starovoitov 1059db20fd2bSAlexei Starovoitov err = -EFAULT; 106015a07b33SAlexei Starovoitov if (copy_to_user(uvalue, value, value_size) != 0) 10618ebe667cSAlexei Starovoitov goto free_value; 1062db20fd2bSAlexei Starovoitov 1063db20fd2bSAlexei Starovoitov err = 0; 1064db20fd2bSAlexei Starovoitov 10658ebe667cSAlexei Starovoitov free_value: 10668ebe667cSAlexei Starovoitov kfree(value); 1067db20fd2bSAlexei Starovoitov free_key: 1068db20fd2bSAlexei Starovoitov kfree(key); 1069db20fd2bSAlexei Starovoitov err_put: 1070db20fd2bSAlexei Starovoitov fdput(f); 1071db20fd2bSAlexei Starovoitov return err; 1072db20fd2bSAlexei Starovoitov } 1073db20fd2bSAlexei Starovoitov 10741ae80cf3SDaniel Colascione 10753274f520SAlexei Starovoitov #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 1076db20fd2bSAlexei Starovoitov 1077db20fd2bSAlexei Starovoitov static int map_update_elem(union bpf_attr *attr) 1078db20fd2bSAlexei Starovoitov { 1079535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1080535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 1081db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1082db20fd2bSAlexei Starovoitov struct bpf_map *map; 1083db20fd2bSAlexei Starovoitov void *key, *value; 108415a07b33SAlexei Starovoitov u32 value_size; 1085592867bfSDaniel Borkmann struct fd f; 1086db20fd2bSAlexei Starovoitov int err; 1087db20fd2bSAlexei Starovoitov 1088db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 1089db20fd2bSAlexei Starovoitov return -EINVAL; 1090db20fd2bSAlexei Starovoitov 1091592867bfSDaniel Borkmann f = fdget(ufd); 1092c2101297SDaniel Borkmann map = __bpf_map_get(f); 1093db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1094db20fd2bSAlexei Starovoitov return PTR_ERR(map); 109587df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 10966e71b04aSChenbo Feng err = -EPERM; 10976e71b04aSChenbo Feng goto err_put; 10986e71b04aSChenbo Feng } 10996e71b04aSChenbo Feng 110096049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 110196049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 110296049f3aSAlexei Starovoitov err = -EINVAL; 110396049f3aSAlexei Starovoitov goto err_put; 110496049f3aSAlexei Starovoitov } 110596049f3aSAlexei Starovoitov 1106c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1107e4448ed8SAl Viro if (IS_ERR(key)) { 1108e4448ed8SAl Viro err = PTR_ERR(key); 1109db20fd2bSAlexei Starovoitov goto err_put; 1110e4448ed8SAl Viro } 1111db20fd2bSAlexei Starovoitov 111215a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 11138f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 1114b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 1115b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 111615a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 111715a07b33SAlexei Starovoitov else 111815a07b33SAlexei Starovoitov value_size = map->value_size; 111915a07b33SAlexei Starovoitov 1120db20fd2bSAlexei Starovoitov err = -ENOMEM; 112115a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1122db20fd2bSAlexei Starovoitov if (!value) 1123db20fd2bSAlexei Starovoitov goto free_key; 1124db20fd2bSAlexei Starovoitov 1125db20fd2bSAlexei Starovoitov err = -EFAULT; 112615a07b33SAlexei Starovoitov if (copy_from_user(value, uvalue, value_size) != 0) 1127db20fd2bSAlexei Starovoitov goto free_value; 1128db20fd2bSAlexei Starovoitov 112915c14a3dSBrian Vazquez err = bpf_map_update_value(map, f, key, value, attr->flags); 11306710e112SJesper Dangaard Brouer 1131db20fd2bSAlexei Starovoitov free_value: 1132db20fd2bSAlexei Starovoitov kfree(value); 1133db20fd2bSAlexei Starovoitov free_key: 1134db20fd2bSAlexei Starovoitov kfree(key); 1135db20fd2bSAlexei Starovoitov err_put: 1136db20fd2bSAlexei Starovoitov fdput(f); 1137db20fd2bSAlexei Starovoitov return err; 1138db20fd2bSAlexei Starovoitov } 1139db20fd2bSAlexei Starovoitov 1140db20fd2bSAlexei Starovoitov #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 1141db20fd2bSAlexei Starovoitov 1142db20fd2bSAlexei Starovoitov static int map_delete_elem(union bpf_attr *attr) 1143db20fd2bSAlexei Starovoitov { 1144535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1145db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1146db20fd2bSAlexei Starovoitov struct bpf_map *map; 1147592867bfSDaniel Borkmann struct fd f; 1148db20fd2bSAlexei Starovoitov void *key; 1149db20fd2bSAlexei Starovoitov int err; 1150db20fd2bSAlexei Starovoitov 1151db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 1152db20fd2bSAlexei Starovoitov return -EINVAL; 1153db20fd2bSAlexei Starovoitov 1154592867bfSDaniel Borkmann f = fdget(ufd); 1155c2101297SDaniel Borkmann map = __bpf_map_get(f); 1156db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1157db20fd2bSAlexei Starovoitov return PTR_ERR(map); 115887df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 11596e71b04aSChenbo Feng err = -EPERM; 11606e71b04aSChenbo Feng goto err_put; 11616e71b04aSChenbo Feng } 11626e71b04aSChenbo Feng 1163c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1164e4448ed8SAl Viro if (IS_ERR(key)) { 1165e4448ed8SAl Viro err = PTR_ERR(key); 1166db20fd2bSAlexei Starovoitov goto err_put; 1167e4448ed8SAl Viro } 1168db20fd2bSAlexei Starovoitov 1169a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1170a3884572SJakub Kicinski err = bpf_map_offload_delete_elem(map, key); 1171a3884572SJakub Kicinski goto out; 117285d33df3SMartin KaFai Lau } else if (IS_FD_PROG_ARRAY(map) || 117385d33df3SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 117485d33df3SMartin KaFai Lau /* These maps require sleepable context */ 1175da765a2fSDaniel Borkmann err = map->ops->map_delete_elem(map, key); 1176da765a2fSDaniel Borkmann goto out; 1177a3884572SJakub Kicinski } 1178a3884572SJakub Kicinski 1179b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 1180db20fd2bSAlexei Starovoitov rcu_read_lock(); 1181db20fd2bSAlexei Starovoitov err = map->ops->map_delete_elem(map, key); 1182db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1183b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 11841ae80cf3SDaniel Colascione maybe_wait_bpf_programs(map); 1185a3884572SJakub Kicinski out: 1186db20fd2bSAlexei Starovoitov kfree(key); 1187db20fd2bSAlexei Starovoitov err_put: 1188db20fd2bSAlexei Starovoitov fdput(f); 1189db20fd2bSAlexei Starovoitov return err; 1190db20fd2bSAlexei Starovoitov } 1191db20fd2bSAlexei Starovoitov 1192db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 1193db20fd2bSAlexei Starovoitov #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 1194db20fd2bSAlexei Starovoitov 1195db20fd2bSAlexei Starovoitov static int map_get_next_key(union bpf_attr *attr) 1196db20fd2bSAlexei Starovoitov { 1197535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1198535e7b4bSMickaël Salaün void __user *unext_key = u64_to_user_ptr(attr->next_key); 1199db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1200db20fd2bSAlexei Starovoitov struct bpf_map *map; 1201db20fd2bSAlexei Starovoitov void *key, *next_key; 1202592867bfSDaniel Borkmann struct fd f; 1203db20fd2bSAlexei Starovoitov int err; 1204db20fd2bSAlexei Starovoitov 1205db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 1206db20fd2bSAlexei Starovoitov return -EINVAL; 1207db20fd2bSAlexei Starovoitov 1208592867bfSDaniel Borkmann f = fdget(ufd); 1209c2101297SDaniel Borkmann map = __bpf_map_get(f); 1210db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1211db20fd2bSAlexei Starovoitov return PTR_ERR(map); 121287df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 12136e71b04aSChenbo Feng err = -EPERM; 12146e71b04aSChenbo Feng goto err_put; 12156e71b04aSChenbo Feng } 12166e71b04aSChenbo Feng 12178fe45924STeng Qin if (ukey) { 1218c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1219e4448ed8SAl Viro if (IS_ERR(key)) { 1220e4448ed8SAl Viro err = PTR_ERR(key); 1221db20fd2bSAlexei Starovoitov goto err_put; 1222e4448ed8SAl Viro } 12238fe45924STeng Qin } else { 12248fe45924STeng Qin key = NULL; 12258fe45924STeng Qin } 1226db20fd2bSAlexei Starovoitov 1227db20fd2bSAlexei Starovoitov err = -ENOMEM; 1228db20fd2bSAlexei Starovoitov next_key = kmalloc(map->key_size, GFP_USER); 1229db20fd2bSAlexei Starovoitov if (!next_key) 1230db20fd2bSAlexei Starovoitov goto free_key; 1231db20fd2bSAlexei Starovoitov 1232a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1233a3884572SJakub Kicinski err = bpf_map_offload_get_next_key(map, key, next_key); 1234a3884572SJakub Kicinski goto out; 1235a3884572SJakub Kicinski } 1236a3884572SJakub Kicinski 1237db20fd2bSAlexei Starovoitov rcu_read_lock(); 1238db20fd2bSAlexei Starovoitov err = map->ops->map_get_next_key(map, key, next_key); 1239db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1240a3884572SJakub Kicinski out: 1241db20fd2bSAlexei Starovoitov if (err) 1242db20fd2bSAlexei Starovoitov goto free_next_key; 1243db20fd2bSAlexei Starovoitov 1244db20fd2bSAlexei Starovoitov err = -EFAULT; 1245db20fd2bSAlexei Starovoitov if (copy_to_user(unext_key, next_key, map->key_size) != 0) 1246db20fd2bSAlexei Starovoitov goto free_next_key; 1247db20fd2bSAlexei Starovoitov 1248db20fd2bSAlexei Starovoitov err = 0; 1249db20fd2bSAlexei Starovoitov 1250db20fd2bSAlexei Starovoitov free_next_key: 1251db20fd2bSAlexei Starovoitov kfree(next_key); 1252db20fd2bSAlexei Starovoitov free_key: 1253db20fd2bSAlexei Starovoitov kfree(key); 1254db20fd2bSAlexei Starovoitov err_put: 1255db20fd2bSAlexei Starovoitov fdput(f); 1256db20fd2bSAlexei Starovoitov return err; 1257db20fd2bSAlexei Starovoitov } 1258db20fd2bSAlexei Starovoitov 1259aa2e93b8SBrian Vazquez int generic_map_delete_batch(struct bpf_map *map, 1260aa2e93b8SBrian Vazquez const union bpf_attr *attr, 1261aa2e93b8SBrian Vazquez union bpf_attr __user *uattr) 1262aa2e93b8SBrian Vazquez { 1263aa2e93b8SBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1264aa2e93b8SBrian Vazquez u32 cp, max_count; 1265aa2e93b8SBrian Vazquez int err = 0; 1266aa2e93b8SBrian Vazquez void *key; 1267aa2e93b8SBrian Vazquez 1268aa2e93b8SBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1269aa2e93b8SBrian Vazquez return -EINVAL; 1270aa2e93b8SBrian Vazquez 1271aa2e93b8SBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1272aa2e93b8SBrian Vazquez !map_value_has_spin_lock(map)) { 1273aa2e93b8SBrian Vazquez return -EINVAL; 1274aa2e93b8SBrian Vazquez } 1275aa2e93b8SBrian Vazquez 1276aa2e93b8SBrian Vazquez max_count = attr->batch.count; 1277aa2e93b8SBrian Vazquez if (!max_count) 1278aa2e93b8SBrian Vazquez return 0; 1279aa2e93b8SBrian Vazquez 12802e3a94aaSBrian Vazquez key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 12812e3a94aaSBrian Vazquez if (!key) 12822e3a94aaSBrian Vazquez return -ENOMEM; 12832e3a94aaSBrian Vazquez 1284aa2e93b8SBrian Vazquez for (cp = 0; cp < max_count; cp++) { 12852e3a94aaSBrian Vazquez err = -EFAULT; 12862e3a94aaSBrian Vazquez if (copy_from_user(key, keys + cp * map->key_size, 12872e3a94aaSBrian Vazquez map->key_size)) 1288aa2e93b8SBrian Vazquez break; 1289aa2e93b8SBrian Vazquez 1290aa2e93b8SBrian Vazquez if (bpf_map_is_dev_bound(map)) { 1291aa2e93b8SBrian Vazquez err = bpf_map_offload_delete_elem(map, key); 1292aa2e93b8SBrian Vazquez break; 1293aa2e93b8SBrian Vazquez } 1294aa2e93b8SBrian Vazquez 1295b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 1296aa2e93b8SBrian Vazquez rcu_read_lock(); 1297aa2e93b8SBrian Vazquez err = map->ops->map_delete_elem(map, key); 1298aa2e93b8SBrian Vazquez rcu_read_unlock(); 1299b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 1300aa2e93b8SBrian Vazquez maybe_wait_bpf_programs(map); 1301aa2e93b8SBrian Vazquez if (err) 1302aa2e93b8SBrian Vazquez break; 1303aa2e93b8SBrian Vazquez } 1304aa2e93b8SBrian Vazquez if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) 1305aa2e93b8SBrian Vazquez err = -EFAULT; 13062e3a94aaSBrian Vazquez 13072e3a94aaSBrian Vazquez kfree(key); 1308aa2e93b8SBrian Vazquez return err; 1309aa2e93b8SBrian Vazquez } 1310aa2e93b8SBrian Vazquez 1311aa2e93b8SBrian Vazquez int generic_map_update_batch(struct bpf_map *map, 1312aa2e93b8SBrian Vazquez const union bpf_attr *attr, 1313aa2e93b8SBrian Vazquez union bpf_attr __user *uattr) 1314aa2e93b8SBrian Vazquez { 1315aa2e93b8SBrian Vazquez void __user *values = u64_to_user_ptr(attr->batch.values); 1316aa2e93b8SBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1317aa2e93b8SBrian Vazquez u32 value_size, cp, max_count; 1318aa2e93b8SBrian Vazquez int ufd = attr->map_fd; 1319aa2e93b8SBrian Vazquez void *key, *value; 1320aa2e93b8SBrian Vazquez struct fd f; 1321aa2e93b8SBrian Vazquez int err = 0; 1322aa2e93b8SBrian Vazquez 1323aa2e93b8SBrian Vazquez f = fdget(ufd); 1324aa2e93b8SBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1325aa2e93b8SBrian Vazquez return -EINVAL; 1326aa2e93b8SBrian Vazquez 1327aa2e93b8SBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1328aa2e93b8SBrian Vazquez !map_value_has_spin_lock(map)) { 1329aa2e93b8SBrian Vazquez return -EINVAL; 1330aa2e93b8SBrian Vazquez } 1331aa2e93b8SBrian Vazquez 1332aa2e93b8SBrian Vazquez value_size = bpf_map_value_size(map); 1333aa2e93b8SBrian Vazquez 1334aa2e93b8SBrian Vazquez max_count = attr->batch.count; 1335aa2e93b8SBrian Vazquez if (!max_count) 1336aa2e93b8SBrian Vazquez return 0; 1337aa2e93b8SBrian Vazquez 13382e3a94aaSBrian Vazquez key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 13392e3a94aaSBrian Vazquez if (!key) 1340aa2e93b8SBrian Vazquez return -ENOMEM; 1341aa2e93b8SBrian Vazquez 13422e3a94aaSBrian Vazquez value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 13432e3a94aaSBrian Vazquez if (!value) { 13442e3a94aaSBrian Vazquez kfree(key); 13452e3a94aaSBrian Vazquez return -ENOMEM; 1346aa2e93b8SBrian Vazquez } 13472e3a94aaSBrian Vazquez 13482e3a94aaSBrian Vazquez for (cp = 0; cp < max_count; cp++) { 1349aa2e93b8SBrian Vazquez err = -EFAULT; 13502e3a94aaSBrian Vazquez if (copy_from_user(key, keys + cp * map->key_size, 13512e3a94aaSBrian Vazquez map->key_size) || 13522e3a94aaSBrian Vazquez copy_from_user(value, values + cp * value_size, value_size)) 1353aa2e93b8SBrian Vazquez break; 1354aa2e93b8SBrian Vazquez 1355aa2e93b8SBrian Vazquez err = bpf_map_update_value(map, f, key, value, 1356aa2e93b8SBrian Vazquez attr->batch.elem_flags); 1357aa2e93b8SBrian Vazquez 1358aa2e93b8SBrian Vazquez if (err) 1359aa2e93b8SBrian Vazquez break; 1360aa2e93b8SBrian Vazquez } 1361aa2e93b8SBrian Vazquez 1362aa2e93b8SBrian Vazquez if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) 1363aa2e93b8SBrian Vazquez err = -EFAULT; 1364aa2e93b8SBrian Vazquez 1365aa2e93b8SBrian Vazquez kfree(value); 1366aa2e93b8SBrian Vazquez kfree(key); 1367aa2e93b8SBrian Vazquez return err; 1368aa2e93b8SBrian Vazquez } 1369aa2e93b8SBrian Vazquez 1370cb4d03abSBrian Vazquez #define MAP_LOOKUP_RETRIES 3 1371cb4d03abSBrian Vazquez 1372cb4d03abSBrian Vazquez int generic_map_lookup_batch(struct bpf_map *map, 1373cb4d03abSBrian Vazquez const union bpf_attr *attr, 1374cb4d03abSBrian Vazquez union bpf_attr __user *uattr) 1375cb4d03abSBrian Vazquez { 1376cb4d03abSBrian Vazquez void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch); 1377cb4d03abSBrian Vazquez void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch); 1378cb4d03abSBrian Vazquez void __user *values = u64_to_user_ptr(attr->batch.values); 1379cb4d03abSBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1380cb4d03abSBrian Vazquez void *buf, *buf_prevkey, *prev_key, *key, *value; 1381cb4d03abSBrian Vazquez int err, retry = MAP_LOOKUP_RETRIES; 1382cb4d03abSBrian Vazquez u32 value_size, cp, max_count; 1383cb4d03abSBrian Vazquez 1384cb4d03abSBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1385cb4d03abSBrian Vazquez return -EINVAL; 1386cb4d03abSBrian Vazquez 1387cb4d03abSBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1388cb4d03abSBrian Vazquez !map_value_has_spin_lock(map)) 1389cb4d03abSBrian Vazquez return -EINVAL; 1390cb4d03abSBrian Vazquez 1391cb4d03abSBrian Vazquez value_size = bpf_map_value_size(map); 1392cb4d03abSBrian Vazquez 1393cb4d03abSBrian Vazquez max_count = attr->batch.count; 1394cb4d03abSBrian Vazquez if (!max_count) 1395cb4d03abSBrian Vazquez return 0; 1396cb4d03abSBrian Vazquez 1397cb4d03abSBrian Vazquez if (put_user(0, &uattr->batch.count)) 1398cb4d03abSBrian Vazquez return -EFAULT; 1399cb4d03abSBrian Vazquez 1400cb4d03abSBrian Vazquez buf_prevkey = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 1401cb4d03abSBrian Vazquez if (!buf_prevkey) 1402cb4d03abSBrian Vazquez return -ENOMEM; 1403cb4d03abSBrian Vazquez 1404cb4d03abSBrian Vazquez buf = kmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN); 1405cb4d03abSBrian Vazquez if (!buf) { 1406bb2359f4SDenis Efremov kfree(buf_prevkey); 1407cb4d03abSBrian Vazquez return -ENOMEM; 1408cb4d03abSBrian Vazquez } 1409cb4d03abSBrian Vazquez 1410cb4d03abSBrian Vazquez err = -EFAULT; 1411cb4d03abSBrian Vazquez prev_key = NULL; 1412cb4d03abSBrian Vazquez if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size)) 1413cb4d03abSBrian Vazquez goto free_buf; 1414cb4d03abSBrian Vazquez key = buf; 1415cb4d03abSBrian Vazquez value = key + map->key_size; 1416cb4d03abSBrian Vazquez if (ubatch) 1417cb4d03abSBrian Vazquez prev_key = buf_prevkey; 1418cb4d03abSBrian Vazquez 1419cb4d03abSBrian Vazquez for (cp = 0; cp < max_count;) { 1420cb4d03abSBrian Vazquez rcu_read_lock(); 1421cb4d03abSBrian Vazquez err = map->ops->map_get_next_key(map, prev_key, key); 1422cb4d03abSBrian Vazquez rcu_read_unlock(); 1423cb4d03abSBrian Vazquez if (err) 1424cb4d03abSBrian Vazquez break; 1425cb4d03abSBrian Vazquez err = bpf_map_copy_value(map, key, value, 1426cb4d03abSBrian Vazquez attr->batch.elem_flags); 1427cb4d03abSBrian Vazquez 1428cb4d03abSBrian Vazquez if (err == -ENOENT) { 1429cb4d03abSBrian Vazquez if (retry) { 1430cb4d03abSBrian Vazquez retry--; 1431cb4d03abSBrian Vazquez continue; 1432cb4d03abSBrian Vazquez } 1433cb4d03abSBrian Vazquez err = -EINTR; 1434cb4d03abSBrian Vazquez break; 1435cb4d03abSBrian Vazquez } 1436cb4d03abSBrian Vazquez 1437cb4d03abSBrian Vazquez if (err) 1438cb4d03abSBrian Vazquez goto free_buf; 1439cb4d03abSBrian Vazquez 1440cb4d03abSBrian Vazquez if (copy_to_user(keys + cp * map->key_size, key, 1441cb4d03abSBrian Vazquez map->key_size)) { 1442cb4d03abSBrian Vazquez err = -EFAULT; 1443cb4d03abSBrian Vazquez goto free_buf; 1444cb4d03abSBrian Vazquez } 1445cb4d03abSBrian Vazquez if (copy_to_user(values + cp * value_size, value, value_size)) { 1446cb4d03abSBrian Vazquez err = -EFAULT; 1447cb4d03abSBrian Vazquez goto free_buf; 1448cb4d03abSBrian Vazquez } 1449cb4d03abSBrian Vazquez 1450cb4d03abSBrian Vazquez if (!prev_key) 1451cb4d03abSBrian Vazquez prev_key = buf_prevkey; 1452cb4d03abSBrian Vazquez 1453cb4d03abSBrian Vazquez swap(prev_key, key); 1454cb4d03abSBrian Vazquez retry = MAP_LOOKUP_RETRIES; 1455cb4d03abSBrian Vazquez cp++; 1456cb4d03abSBrian Vazquez } 1457cb4d03abSBrian Vazquez 1458cb4d03abSBrian Vazquez if (err == -EFAULT) 1459cb4d03abSBrian Vazquez goto free_buf; 1460cb4d03abSBrian Vazquez 1461cb4d03abSBrian Vazquez if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) || 1462cb4d03abSBrian Vazquez (cp && copy_to_user(uobatch, prev_key, map->key_size)))) 1463cb4d03abSBrian Vazquez err = -EFAULT; 1464cb4d03abSBrian Vazquez 1465cb4d03abSBrian Vazquez free_buf: 1466cb4d03abSBrian Vazquez kfree(buf_prevkey); 1467cb4d03abSBrian Vazquez kfree(buf); 1468cb4d03abSBrian Vazquez return err; 1469cb4d03abSBrian Vazquez } 1470cb4d03abSBrian Vazquez 1471bd513cd0SMauricio Vasquez B #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value 1472bd513cd0SMauricio Vasquez B 1473bd513cd0SMauricio Vasquez B static int map_lookup_and_delete_elem(union bpf_attr *attr) 1474bd513cd0SMauricio Vasquez B { 1475bd513cd0SMauricio Vasquez B void __user *ukey = u64_to_user_ptr(attr->key); 1476bd513cd0SMauricio Vasquez B void __user *uvalue = u64_to_user_ptr(attr->value); 1477bd513cd0SMauricio Vasquez B int ufd = attr->map_fd; 1478bd513cd0SMauricio Vasquez B struct bpf_map *map; 1479540fefc0SAlexei Starovoitov void *key, *value; 1480bd513cd0SMauricio Vasquez B u32 value_size; 1481bd513cd0SMauricio Vasquez B struct fd f; 1482bd513cd0SMauricio Vasquez B int err; 1483bd513cd0SMauricio Vasquez B 1484bd513cd0SMauricio Vasquez B if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) 1485bd513cd0SMauricio Vasquez B return -EINVAL; 1486bd513cd0SMauricio Vasquez B 1487bd513cd0SMauricio Vasquez B f = fdget(ufd); 1488bd513cd0SMauricio Vasquez B map = __bpf_map_get(f); 1489bd513cd0SMauricio Vasquez B if (IS_ERR(map)) 1490bd513cd0SMauricio Vasquez B return PTR_ERR(map); 14911ea0f912SAnton Protopopov if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) || 14921ea0f912SAnton Protopopov !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 1493bd513cd0SMauricio Vasquez B err = -EPERM; 1494bd513cd0SMauricio Vasquez B goto err_put; 1495bd513cd0SMauricio Vasquez B } 1496bd513cd0SMauricio Vasquez B 1497bd513cd0SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1498bd513cd0SMauricio Vasquez B if (IS_ERR(key)) { 1499bd513cd0SMauricio Vasquez B err = PTR_ERR(key); 1500bd513cd0SMauricio Vasquez B goto err_put; 1501bd513cd0SMauricio Vasquez B } 1502bd513cd0SMauricio Vasquez B 1503bd513cd0SMauricio Vasquez B value_size = map->value_size; 1504bd513cd0SMauricio Vasquez B 1505bd513cd0SMauricio Vasquez B err = -ENOMEM; 1506bd513cd0SMauricio Vasquez B value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1507bd513cd0SMauricio Vasquez B if (!value) 1508bd513cd0SMauricio Vasquez B goto free_key; 1509bd513cd0SMauricio Vasquez B 1510bd513cd0SMauricio Vasquez B if (map->map_type == BPF_MAP_TYPE_QUEUE || 1511bd513cd0SMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 1512bd513cd0SMauricio Vasquez B err = map->ops->map_pop_elem(map, value); 1513bd513cd0SMauricio Vasquez B } else { 1514bd513cd0SMauricio Vasquez B err = -ENOTSUPP; 1515bd513cd0SMauricio Vasquez B } 1516bd513cd0SMauricio Vasquez B 1517bd513cd0SMauricio Vasquez B if (err) 1518bd513cd0SMauricio Vasquez B goto free_value; 1519bd513cd0SMauricio Vasquez B 15207f645462SWei Yongjun if (copy_to_user(uvalue, value, value_size) != 0) { 15217f645462SWei Yongjun err = -EFAULT; 1522bd513cd0SMauricio Vasquez B goto free_value; 15237f645462SWei Yongjun } 1524bd513cd0SMauricio Vasquez B 1525bd513cd0SMauricio Vasquez B err = 0; 1526bd513cd0SMauricio Vasquez B 1527bd513cd0SMauricio Vasquez B free_value: 1528bd513cd0SMauricio Vasquez B kfree(value); 1529bd513cd0SMauricio Vasquez B free_key: 1530bd513cd0SMauricio Vasquez B kfree(key); 1531bd513cd0SMauricio Vasquez B err_put: 1532bd513cd0SMauricio Vasquez B fdput(f); 1533bd513cd0SMauricio Vasquez B return err; 1534bd513cd0SMauricio Vasquez B } 1535bd513cd0SMauricio Vasquez B 153687df15deSDaniel Borkmann #define BPF_MAP_FREEZE_LAST_FIELD map_fd 153787df15deSDaniel Borkmann 153887df15deSDaniel Borkmann static int map_freeze(const union bpf_attr *attr) 153987df15deSDaniel Borkmann { 154087df15deSDaniel Borkmann int err = 0, ufd = attr->map_fd; 154187df15deSDaniel Borkmann struct bpf_map *map; 154287df15deSDaniel Borkmann struct fd f; 154387df15deSDaniel Borkmann 154487df15deSDaniel Borkmann if (CHECK_ATTR(BPF_MAP_FREEZE)) 154587df15deSDaniel Borkmann return -EINVAL; 154687df15deSDaniel Borkmann 154787df15deSDaniel Borkmann f = fdget(ufd); 154887df15deSDaniel Borkmann map = __bpf_map_get(f); 154987df15deSDaniel Borkmann if (IS_ERR(map)) 155087df15deSDaniel Borkmann return PTR_ERR(map); 1551fc970227SAndrii Nakryiko 1552849b4d94SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 1553849b4d94SMartin KaFai Lau fdput(f); 1554849b4d94SMartin KaFai Lau return -ENOTSUPP; 1555849b4d94SMartin KaFai Lau } 1556849b4d94SMartin KaFai Lau 1557fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 1558fc970227SAndrii Nakryiko 1559fc970227SAndrii Nakryiko if (map->writecnt) { 1560fc970227SAndrii Nakryiko err = -EBUSY; 1561fc970227SAndrii Nakryiko goto err_put; 1562fc970227SAndrii Nakryiko } 156387df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) { 156487df15deSDaniel Borkmann err = -EBUSY; 156587df15deSDaniel Borkmann goto err_put; 156687df15deSDaniel Borkmann } 15672c78ee89SAlexei Starovoitov if (!bpf_capable()) { 156887df15deSDaniel Borkmann err = -EPERM; 156987df15deSDaniel Borkmann goto err_put; 157087df15deSDaniel Borkmann } 157187df15deSDaniel Borkmann 157287df15deSDaniel Borkmann WRITE_ONCE(map->frozen, true); 157387df15deSDaniel Borkmann err_put: 1574fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 157587df15deSDaniel Borkmann fdput(f); 157687df15deSDaniel Borkmann return err; 157787df15deSDaniel Borkmann } 157887df15deSDaniel Borkmann 15797de16e3aSJakub Kicinski static const struct bpf_prog_ops * const bpf_prog_types[] = { 158091cc1a99SAlexei Starovoitov #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 15817de16e3aSJakub Kicinski [_id] = & _name ## _prog_ops, 15827de16e3aSJakub Kicinski #define BPF_MAP_TYPE(_id, _ops) 1583f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) 15847de16e3aSJakub Kicinski #include <linux/bpf_types.h> 15857de16e3aSJakub Kicinski #undef BPF_PROG_TYPE 15867de16e3aSJakub Kicinski #undef BPF_MAP_TYPE 1587f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 15887de16e3aSJakub Kicinski }; 15897de16e3aSJakub Kicinski 159009756af4SAlexei Starovoitov static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 159109756af4SAlexei Starovoitov { 1592d0f1a451SDaniel Borkmann const struct bpf_prog_ops *ops; 1593d0f1a451SDaniel Borkmann 1594d0f1a451SDaniel Borkmann if (type >= ARRAY_SIZE(bpf_prog_types)) 1595d0f1a451SDaniel Borkmann return -EINVAL; 1596d0f1a451SDaniel Borkmann type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types)); 1597d0f1a451SDaniel Borkmann ops = bpf_prog_types[type]; 1598d0f1a451SDaniel Borkmann if (!ops) 1599be9370a7SJohannes Berg return -EINVAL; 160009756af4SAlexei Starovoitov 1601ab3f0063SJakub Kicinski if (!bpf_prog_is_dev_bound(prog->aux)) 1602d0f1a451SDaniel Borkmann prog->aux->ops = ops; 1603ab3f0063SJakub Kicinski else 1604ab3f0063SJakub Kicinski prog->aux->ops = &bpf_offload_prog_ops; 160524701eceSDaniel Borkmann prog->type = type; 160609756af4SAlexei Starovoitov return 0; 160709756af4SAlexei Starovoitov } 160809756af4SAlexei Starovoitov 1609bae141f5SDaniel Borkmann enum bpf_audit { 1610bae141f5SDaniel Borkmann BPF_AUDIT_LOAD, 1611bae141f5SDaniel Borkmann BPF_AUDIT_UNLOAD, 1612bae141f5SDaniel Borkmann BPF_AUDIT_MAX, 1613bae141f5SDaniel Borkmann }; 1614bae141f5SDaniel Borkmann 1615bae141f5SDaniel Borkmann static const char * const bpf_audit_str[BPF_AUDIT_MAX] = { 1616bae141f5SDaniel Borkmann [BPF_AUDIT_LOAD] = "LOAD", 1617bae141f5SDaniel Borkmann [BPF_AUDIT_UNLOAD] = "UNLOAD", 1618bae141f5SDaniel Borkmann }; 1619bae141f5SDaniel Borkmann 1620bae141f5SDaniel Borkmann static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op) 1621bae141f5SDaniel Borkmann { 1622bae141f5SDaniel Borkmann struct audit_context *ctx = NULL; 1623bae141f5SDaniel Borkmann struct audit_buffer *ab; 1624bae141f5SDaniel Borkmann 1625bae141f5SDaniel Borkmann if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX)) 1626bae141f5SDaniel Borkmann return; 1627bae141f5SDaniel Borkmann if (audit_enabled == AUDIT_OFF) 1628bae141f5SDaniel Borkmann return; 1629bae141f5SDaniel Borkmann if (op == BPF_AUDIT_LOAD) 1630bae141f5SDaniel Borkmann ctx = audit_context(); 1631bae141f5SDaniel Borkmann ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF); 1632bae141f5SDaniel Borkmann if (unlikely(!ab)) 1633bae141f5SDaniel Borkmann return; 1634bae141f5SDaniel Borkmann audit_log_format(ab, "prog-id=%u op=%s", 1635bae141f5SDaniel Borkmann prog->aux->id, bpf_audit_str[op]); 1636bae141f5SDaniel Borkmann audit_log_end(ab); 1637bae141f5SDaniel Borkmann } 1638bae141f5SDaniel Borkmann 1639dc4bb0e2SMartin KaFai Lau static int bpf_prog_alloc_id(struct bpf_prog *prog) 1640dc4bb0e2SMartin KaFai Lau { 1641dc4bb0e2SMartin KaFai Lau int id; 1642dc4bb0e2SMartin KaFai Lau 1643b76354cdSShaohua Li idr_preload(GFP_KERNEL); 1644dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1645dc4bb0e2SMartin KaFai Lau id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 1646dc4bb0e2SMartin KaFai Lau if (id > 0) 1647dc4bb0e2SMartin KaFai Lau prog->aux->id = id; 1648dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1649b76354cdSShaohua Li idr_preload_end(); 1650dc4bb0e2SMartin KaFai Lau 1651dc4bb0e2SMartin KaFai Lau /* id is in [1, INT_MAX) */ 1652dc4bb0e2SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 1653dc4bb0e2SMartin KaFai Lau return -ENOSPC; 1654dc4bb0e2SMartin KaFai Lau 1655dc4bb0e2SMartin KaFai Lau return id > 0 ? 0 : id; 1656dc4bb0e2SMartin KaFai Lau } 1657dc4bb0e2SMartin KaFai Lau 1658ad8ad79fSJakub Kicinski void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) 1659dc4bb0e2SMartin KaFai Lau { 1660ad8ad79fSJakub Kicinski /* cBPF to eBPF migrations are currently not in the idr store. 1661ad8ad79fSJakub Kicinski * Offloaded programs are removed from the store when their device 1662ad8ad79fSJakub Kicinski * disappears - even if someone grabs an fd to them they are unusable, 1663ad8ad79fSJakub Kicinski * simply waiting for refcnt to drop to be freed. 1664ad8ad79fSJakub Kicinski */ 1665dc4bb0e2SMartin KaFai Lau if (!prog->aux->id) 1666dc4bb0e2SMartin KaFai Lau return; 1667dc4bb0e2SMartin KaFai Lau 1668b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1669dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1670b16d9aa4SMartin KaFai Lau else 1671b16d9aa4SMartin KaFai Lau __acquire(&prog_idr_lock); 1672b16d9aa4SMartin KaFai Lau 1673dc4bb0e2SMartin KaFai Lau idr_remove(&prog_idr, prog->aux->id); 1674ad8ad79fSJakub Kicinski prog->aux->id = 0; 1675b16d9aa4SMartin KaFai Lau 1676b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1677dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1678b16d9aa4SMartin KaFai Lau else 1679b16d9aa4SMartin KaFai Lau __release(&prog_idr_lock); 1680dc4bb0e2SMartin KaFai Lau } 1681dc4bb0e2SMartin KaFai Lau 16821aacde3dSDaniel Borkmann static void __bpf_prog_put_rcu(struct rcu_head *rcu) 1683abf2e7d6SAlexei Starovoitov { 1684abf2e7d6SAlexei Starovoitov struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 1685abf2e7d6SAlexei Starovoitov 16863b4d9eb2SDaniel Borkmann kvfree(aux->func_info); 16878c1b6e69SAlexei Starovoitov kfree(aux->func_info_aux); 16883ac1f01bSRoman Gushchin free_uid(aux->user); 1689afdb09c7SChenbo Feng security_bpf_prog_free(aux); 1690abf2e7d6SAlexei Starovoitov bpf_prog_free(aux->prog); 1691abf2e7d6SAlexei Starovoitov } 1692abf2e7d6SAlexei Starovoitov 1693cd7455f1SDaniel Borkmann static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) 1694cd7455f1SDaniel Borkmann { 1695cd7455f1SDaniel Borkmann bpf_prog_kallsyms_del_all(prog); 1696cd7455f1SDaniel Borkmann btf_put(prog->aux->btf); 1697e16301fbSMartin KaFai Lau kvfree(prog->aux->jited_linfo); 1698e16301fbSMartin KaFai Lau kvfree(prog->aux->linfo); 1699e6ac2450SMartin KaFai Lau kfree(prog->aux->kfunc_tab); 170022dc4a0fSAndrii Nakryiko if (prog->aux->attach_btf) 170122dc4a0fSAndrii Nakryiko btf_put(prog->aux->attach_btf); 1702cd7455f1SDaniel Borkmann 17031e6c62a8SAlexei Starovoitov if (deferred) { 17041e6c62a8SAlexei Starovoitov if (prog->aux->sleepable) 17051e6c62a8SAlexei Starovoitov call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); 1706cd7455f1SDaniel Borkmann else 17071e6c62a8SAlexei Starovoitov call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 17081e6c62a8SAlexei Starovoitov } else { 1709cd7455f1SDaniel Borkmann __bpf_prog_put_rcu(&prog->aux->rcu); 1710cd7455f1SDaniel Borkmann } 17111e6c62a8SAlexei Starovoitov } 1712cd7455f1SDaniel Borkmann 1713b16d9aa4SMartin KaFai Lau static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 171409756af4SAlexei Starovoitov { 171585192dbfSAndrii Nakryiko if (atomic64_dec_and_test(&prog->aux->refcnt)) { 17166ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0); 1717bae141f5SDaniel Borkmann bpf_audit_prog(prog, BPF_AUDIT_UNLOAD); 171834ad5580SMartin KaFai Lau /* bpf_prog_free_id() must be called first */ 1719b16d9aa4SMartin KaFai Lau bpf_prog_free_id(prog, do_idr_lock); 1720cd7455f1SDaniel Borkmann __bpf_prog_put_noref(prog, true); 172109756af4SAlexei Starovoitov } 1722a67edbf4SDaniel Borkmann } 1723b16d9aa4SMartin KaFai Lau 1724b16d9aa4SMartin KaFai Lau void bpf_prog_put(struct bpf_prog *prog) 1725b16d9aa4SMartin KaFai Lau { 1726b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, true); 1727b16d9aa4SMartin KaFai Lau } 1728e2e9b654SDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_put); 172909756af4SAlexei Starovoitov 173009756af4SAlexei Starovoitov static int bpf_prog_release(struct inode *inode, struct file *filp) 173109756af4SAlexei Starovoitov { 173209756af4SAlexei Starovoitov struct bpf_prog *prog = filp->private_data; 173309756af4SAlexei Starovoitov 17341aacde3dSDaniel Borkmann bpf_prog_put(prog); 173509756af4SAlexei Starovoitov return 0; 173609756af4SAlexei Starovoitov } 173709756af4SAlexei Starovoitov 1738492ecee8SAlexei Starovoitov static void bpf_prog_get_stats(const struct bpf_prog *prog, 1739492ecee8SAlexei Starovoitov struct bpf_prog_stats *stats) 1740492ecee8SAlexei Starovoitov { 17419ed9e9baSAlexei Starovoitov u64 nsecs = 0, cnt = 0, misses = 0; 1742492ecee8SAlexei Starovoitov int cpu; 1743492ecee8SAlexei Starovoitov 1744492ecee8SAlexei Starovoitov for_each_possible_cpu(cpu) { 1745492ecee8SAlexei Starovoitov const struct bpf_prog_stats *st; 1746492ecee8SAlexei Starovoitov unsigned int start; 17479ed9e9baSAlexei Starovoitov u64 tnsecs, tcnt, tmisses; 1748492ecee8SAlexei Starovoitov 1749700d4796SAlexei Starovoitov st = per_cpu_ptr(prog->stats, cpu); 1750492ecee8SAlexei Starovoitov do { 1751492ecee8SAlexei Starovoitov start = u64_stats_fetch_begin_irq(&st->syncp); 1752492ecee8SAlexei Starovoitov tnsecs = st->nsecs; 1753492ecee8SAlexei Starovoitov tcnt = st->cnt; 17549ed9e9baSAlexei Starovoitov tmisses = st->misses; 1755492ecee8SAlexei Starovoitov } while (u64_stats_fetch_retry_irq(&st->syncp, start)); 1756492ecee8SAlexei Starovoitov nsecs += tnsecs; 1757492ecee8SAlexei Starovoitov cnt += tcnt; 17589ed9e9baSAlexei Starovoitov misses += tmisses; 1759492ecee8SAlexei Starovoitov } 1760492ecee8SAlexei Starovoitov stats->nsecs = nsecs; 1761492ecee8SAlexei Starovoitov stats->cnt = cnt; 17629ed9e9baSAlexei Starovoitov stats->misses = misses; 1763492ecee8SAlexei Starovoitov } 1764492ecee8SAlexei Starovoitov 17657bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 17667bd509e3SDaniel Borkmann static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 17677bd509e3SDaniel Borkmann { 17687bd509e3SDaniel Borkmann const struct bpf_prog *prog = filp->private_data; 1769f1f7714eSDaniel Borkmann char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 1770492ecee8SAlexei Starovoitov struct bpf_prog_stats stats; 17717bd509e3SDaniel Borkmann 1772492ecee8SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 1773f1f7714eSDaniel Borkmann bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 17747bd509e3SDaniel Borkmann seq_printf(m, 17757bd509e3SDaniel Borkmann "prog_type:\t%u\n" 17767bd509e3SDaniel Borkmann "prog_jited:\t%u\n" 1777f1f7714eSDaniel Borkmann "prog_tag:\t%s\n" 17784316b409SDaniel Borkmann "memlock:\t%llu\n" 1779492ecee8SAlexei Starovoitov "prog_id:\t%u\n" 1780492ecee8SAlexei Starovoitov "run_time_ns:\t%llu\n" 17819ed9e9baSAlexei Starovoitov "run_cnt:\t%llu\n" 17829ed9e9baSAlexei Starovoitov "recursion_misses:\t%llu\n", 17837bd509e3SDaniel Borkmann prog->type, 17847bd509e3SDaniel Borkmann prog->jited, 1785f1f7714eSDaniel Borkmann prog_tag, 17864316b409SDaniel Borkmann prog->pages * 1ULL << PAGE_SHIFT, 1787492ecee8SAlexei Starovoitov prog->aux->id, 1788492ecee8SAlexei Starovoitov stats.nsecs, 17899ed9e9baSAlexei Starovoitov stats.cnt, 17909ed9e9baSAlexei Starovoitov stats.misses); 17917bd509e3SDaniel Borkmann } 17927bd509e3SDaniel Borkmann #endif 17937bd509e3SDaniel Borkmann 1794f66e448cSChenbo Feng const struct file_operations bpf_prog_fops = { 17957bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 17967bd509e3SDaniel Borkmann .show_fdinfo = bpf_prog_show_fdinfo, 17977bd509e3SDaniel Borkmann #endif 179809756af4SAlexei Starovoitov .release = bpf_prog_release, 17996e71b04aSChenbo Feng .read = bpf_dummy_read, 18006e71b04aSChenbo Feng .write = bpf_dummy_write, 180109756af4SAlexei Starovoitov }; 180209756af4SAlexei Starovoitov 1803b2197755SDaniel Borkmann int bpf_prog_new_fd(struct bpf_prog *prog) 1804aa79781bSDaniel Borkmann { 1805afdb09c7SChenbo Feng int ret; 1806afdb09c7SChenbo Feng 1807afdb09c7SChenbo Feng ret = security_bpf_prog(prog); 1808afdb09c7SChenbo Feng if (ret < 0) 1809afdb09c7SChenbo Feng return ret; 1810afdb09c7SChenbo Feng 1811aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 1812aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC); 1813aa79781bSDaniel Borkmann } 1814aa79781bSDaniel Borkmann 1815113214beSDaniel Borkmann static struct bpf_prog *____bpf_prog_get(struct fd f) 181609756af4SAlexei Starovoitov { 181709756af4SAlexei Starovoitov if (!f.file) 181809756af4SAlexei Starovoitov return ERR_PTR(-EBADF); 181909756af4SAlexei Starovoitov if (f.file->f_op != &bpf_prog_fops) { 182009756af4SAlexei Starovoitov fdput(f); 182109756af4SAlexei Starovoitov return ERR_PTR(-EINVAL); 182209756af4SAlexei Starovoitov } 182309756af4SAlexei Starovoitov 1824c2101297SDaniel Borkmann return f.file->private_data; 182509756af4SAlexei Starovoitov } 182609756af4SAlexei Starovoitov 182785192dbfSAndrii Nakryiko void bpf_prog_add(struct bpf_prog *prog, int i) 182892117d84SAlexei Starovoitov { 182985192dbfSAndrii Nakryiko atomic64_add(i, &prog->aux->refcnt); 183092117d84SAlexei Starovoitov } 183159d3656dSBrenden Blanco EXPORT_SYMBOL_GPL(bpf_prog_add); 183259d3656dSBrenden Blanco 1833c540594fSDaniel Borkmann void bpf_prog_sub(struct bpf_prog *prog, int i) 1834c540594fSDaniel Borkmann { 1835c540594fSDaniel Borkmann /* Only to be used for undoing previous bpf_prog_add() in some 1836c540594fSDaniel Borkmann * error path. We still know that another entity in our call 1837c540594fSDaniel Borkmann * path holds a reference to the program, thus atomic_sub() can 1838c540594fSDaniel Borkmann * be safely used in such cases! 1839c540594fSDaniel Borkmann */ 184085192dbfSAndrii Nakryiko WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0); 1841c540594fSDaniel Borkmann } 1842c540594fSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_sub); 1843c540594fSDaniel Borkmann 184485192dbfSAndrii Nakryiko void bpf_prog_inc(struct bpf_prog *prog) 184559d3656dSBrenden Blanco { 184685192dbfSAndrii Nakryiko atomic64_inc(&prog->aux->refcnt); 184759d3656dSBrenden Blanco } 184897bc402dSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_inc); 184992117d84SAlexei Starovoitov 1850b16d9aa4SMartin KaFai Lau /* prog_idr_lock should have been held */ 1851a6f6df69SJohn Fastabend struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 1852b16d9aa4SMartin KaFai Lau { 1853b16d9aa4SMartin KaFai Lau int refold; 1854b16d9aa4SMartin KaFai Lau 185585192dbfSAndrii Nakryiko refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0); 1856b16d9aa4SMartin KaFai Lau 1857b16d9aa4SMartin KaFai Lau if (!refold) 1858b16d9aa4SMartin KaFai Lau return ERR_PTR(-ENOENT); 1859b16d9aa4SMartin KaFai Lau 1860b16d9aa4SMartin KaFai Lau return prog; 1861b16d9aa4SMartin KaFai Lau } 1862a6f6df69SJohn Fastabend EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 1863b16d9aa4SMartin KaFai Lau 1864040ee692SAl Viro bool bpf_prog_get_ok(struct bpf_prog *prog, 1865288b3de5SJakub Kicinski enum bpf_prog_type *attach_type, bool attach_drv) 1866248f346fSJakub Kicinski { 1867288b3de5SJakub Kicinski /* not an attachment, just a refcount inc, always allow */ 1868288b3de5SJakub Kicinski if (!attach_type) 1869288b3de5SJakub Kicinski return true; 1870248f346fSJakub Kicinski 1871248f346fSJakub Kicinski if (prog->type != *attach_type) 1872248f346fSJakub Kicinski return false; 1873288b3de5SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv) 1874248f346fSJakub Kicinski return false; 1875248f346fSJakub Kicinski 1876248f346fSJakub Kicinski return true; 1877248f346fSJakub Kicinski } 1878248f346fSJakub Kicinski 1879248f346fSJakub Kicinski static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, 1880288b3de5SJakub Kicinski bool attach_drv) 188109756af4SAlexei Starovoitov { 188209756af4SAlexei Starovoitov struct fd f = fdget(ufd); 188309756af4SAlexei Starovoitov struct bpf_prog *prog; 188409756af4SAlexei Starovoitov 1885113214beSDaniel Borkmann prog = ____bpf_prog_get(f); 188609756af4SAlexei Starovoitov if (IS_ERR(prog)) 188709756af4SAlexei Starovoitov return prog; 1888288b3de5SJakub Kicinski if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) { 1889113214beSDaniel Borkmann prog = ERR_PTR(-EINVAL); 1890113214beSDaniel Borkmann goto out; 1891113214beSDaniel Borkmann } 189209756af4SAlexei Starovoitov 189385192dbfSAndrii Nakryiko bpf_prog_inc(prog); 1894113214beSDaniel Borkmann out: 189509756af4SAlexei Starovoitov fdput(f); 189609756af4SAlexei Starovoitov return prog; 189709756af4SAlexei Starovoitov } 1898113214beSDaniel Borkmann 1899113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get(u32 ufd) 1900113214beSDaniel Borkmann { 1901288b3de5SJakub Kicinski return __bpf_prog_get(ufd, NULL, false); 1902113214beSDaniel Borkmann } 1903113214beSDaniel Borkmann 1904248f346fSJakub Kicinski struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 1905288b3de5SJakub Kicinski bool attach_drv) 1906248f346fSJakub Kicinski { 19074d220ed0SAlexei Starovoitov return __bpf_prog_get(ufd, &type, attach_drv); 1908248f346fSJakub Kicinski } 19096c8dfe21SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev); 1910248f346fSJakub Kicinski 1911aac3fc32SAndrey Ignatov /* Initially all BPF programs could be loaded w/o specifying 1912aac3fc32SAndrey Ignatov * expected_attach_type. Later for some of them specifying expected_attach_type 1913aac3fc32SAndrey Ignatov * at load time became required so that program could be validated properly. 1914aac3fc32SAndrey Ignatov * Programs of types that are allowed to be loaded both w/ and w/o (for 1915aac3fc32SAndrey Ignatov * backward compatibility) expected_attach_type, should have the default attach 1916aac3fc32SAndrey Ignatov * type assigned to expected_attach_type for the latter case, so that it can be 1917aac3fc32SAndrey Ignatov * validated later at attach time. 1918aac3fc32SAndrey Ignatov * 1919aac3fc32SAndrey Ignatov * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if 1920aac3fc32SAndrey Ignatov * prog type requires it but has some attach types that have to be backward 1921aac3fc32SAndrey Ignatov * compatible. 1922aac3fc32SAndrey Ignatov */ 1923aac3fc32SAndrey Ignatov static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) 1924aac3fc32SAndrey Ignatov { 1925aac3fc32SAndrey Ignatov switch (attr->prog_type) { 1926aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1927aac3fc32SAndrey Ignatov /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't 1928aac3fc32SAndrey Ignatov * exist so checking for non-zero is the way to go here. 1929aac3fc32SAndrey Ignatov */ 1930aac3fc32SAndrey Ignatov if (!attr->expected_attach_type) 1931aac3fc32SAndrey Ignatov attr->expected_attach_type = 1932aac3fc32SAndrey Ignatov BPF_CGROUP_INET_SOCK_CREATE; 1933aac3fc32SAndrey Ignatov break; 1934aac3fc32SAndrey Ignatov } 1935aac3fc32SAndrey Ignatov } 1936aac3fc32SAndrey Ignatov 19375e43f899SAndrey Ignatov static int 1938ccfe29ebSAlexei Starovoitov bpf_prog_load_check_attach(enum bpf_prog_type prog_type, 1939ccfe29ebSAlexei Starovoitov enum bpf_attach_type expected_attach_type, 1940290248a5SAndrii Nakryiko struct btf *attach_btf, u32 btf_id, 1941290248a5SAndrii Nakryiko struct bpf_prog *dst_prog) 19425e43f899SAndrey Ignatov { 194327ae7997SMartin KaFai Lau if (btf_id) { 1944c108e3c1SAlexei Starovoitov if (btf_id > BTF_MAX_TYPE) 1945c108e3c1SAlexei Starovoitov return -EINVAL; 194627ae7997SMartin KaFai Lau 1947290248a5SAndrii Nakryiko if (!attach_btf && !dst_prog) 1948290248a5SAndrii Nakryiko return -EINVAL; 1949290248a5SAndrii Nakryiko 195027ae7997SMartin KaFai Lau switch (prog_type) { 195127ae7997SMartin KaFai Lau case BPF_PROG_TYPE_TRACING: 19529e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 195327ae7997SMartin KaFai Lau case BPF_PROG_TYPE_STRUCT_OPS: 1954be8704ffSAlexei Starovoitov case BPF_PROG_TYPE_EXT: 1955c108e3c1SAlexei Starovoitov break; 1956c108e3c1SAlexei Starovoitov default: 1957c108e3c1SAlexei Starovoitov return -EINVAL; 1958c108e3c1SAlexei Starovoitov } 195927ae7997SMartin KaFai Lau } 196027ae7997SMartin KaFai Lau 1961290248a5SAndrii Nakryiko if (attach_btf && (!btf_id || dst_prog)) 1962290248a5SAndrii Nakryiko return -EINVAL; 1963290248a5SAndrii Nakryiko 1964290248a5SAndrii Nakryiko if (dst_prog && prog_type != BPF_PROG_TYPE_TRACING && 1965be8704ffSAlexei Starovoitov prog_type != BPF_PROG_TYPE_EXT) 196627ae7997SMartin KaFai Lau return -EINVAL; 1967c108e3c1SAlexei Starovoitov 1968c108e3c1SAlexei Starovoitov switch (prog_type) { 1969aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1970aac3fc32SAndrey Ignatov switch (expected_attach_type) { 1971aac3fc32SAndrey Ignatov case BPF_CGROUP_INET_SOCK_CREATE: 1972f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 1973aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1974aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 1975aac3fc32SAndrey Ignatov return 0; 1976aac3fc32SAndrey Ignatov default: 1977aac3fc32SAndrey Ignatov return -EINVAL; 1978aac3fc32SAndrey Ignatov } 19794fbac77dSAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 19804fbac77dSAndrey Ignatov switch (expected_attach_type) { 19814fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 19824fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1983d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1984d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 19851b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 19861b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 19871b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 19881b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 19891cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 19901cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 1991983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG: 1992983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG: 19935e43f899SAndrey Ignatov return 0; 19944fbac77dSAndrey Ignatov default: 19954fbac77dSAndrey Ignatov return -EINVAL; 19964fbac77dSAndrey Ignatov } 19975cf1e914Sbrakmo case BPF_PROG_TYPE_CGROUP_SKB: 19985cf1e914Sbrakmo switch (expected_attach_type) { 19995cf1e914Sbrakmo case BPF_CGROUP_INET_INGRESS: 20005cf1e914Sbrakmo case BPF_CGROUP_INET_EGRESS: 20015cf1e914Sbrakmo return 0; 20025cf1e914Sbrakmo default: 20035cf1e914Sbrakmo return -EINVAL; 20045cf1e914Sbrakmo } 20050d01da6aSStanislav Fomichev case BPF_PROG_TYPE_CGROUP_SOCKOPT: 20060d01da6aSStanislav Fomichev switch (expected_attach_type) { 20070d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT: 20080d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT: 20090d01da6aSStanislav Fomichev return 0; 20100d01da6aSStanislav Fomichev default: 20110d01da6aSStanislav Fomichev return -EINVAL; 20120d01da6aSStanislav Fomichev } 2013e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP: 2014e9ddbb77SJakub Sitnicki if (expected_attach_type == BPF_SK_LOOKUP) 2015e9ddbb77SJakub Sitnicki return 0; 2016e9ddbb77SJakub Sitnicki return -EINVAL; 2017be8704ffSAlexei Starovoitov case BPF_PROG_TYPE_EXT: 2018be8704ffSAlexei Starovoitov if (expected_attach_type) 2019be8704ffSAlexei Starovoitov return -EINVAL; 2020df561f66SGustavo A. R. Silva fallthrough; 20214fbac77dSAndrey Ignatov default: 20224fbac77dSAndrey Ignatov return 0; 20234fbac77dSAndrey Ignatov } 20245e43f899SAndrey Ignatov } 20255e43f899SAndrey Ignatov 20262c78ee89SAlexei Starovoitov static bool is_net_admin_prog_type(enum bpf_prog_type prog_type) 20272c78ee89SAlexei Starovoitov { 20282c78ee89SAlexei Starovoitov switch (prog_type) { 20292c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SCHED_CLS: 20302c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SCHED_ACT: 20312c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_XDP: 20322c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_IN: 20332c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_OUT: 20342c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_XMIT: 20352c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_SEG6LOCAL: 20362c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_SKB: 20372c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_MSG: 20382c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LIRC_MODE2: 20392c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_FLOW_DISSECTOR: 20402c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_DEVICE: 20412c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCK: 20422c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 20432c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCKOPT: 20442c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SYSCTL: 20452c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SOCK_OPS: 20462c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_EXT: /* extends any prog */ 20472c78ee89SAlexei Starovoitov return true; 20482c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SKB: 20492c78ee89SAlexei Starovoitov /* always unpriv */ 20502c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_REUSEPORT: 20512c78ee89SAlexei Starovoitov /* equivalent to SOCKET_FILTER. need CAP_BPF only */ 20522c78ee89SAlexei Starovoitov default: 20532c78ee89SAlexei Starovoitov return false; 20542c78ee89SAlexei Starovoitov } 20552c78ee89SAlexei Starovoitov } 20562c78ee89SAlexei Starovoitov 20572c78ee89SAlexei Starovoitov static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) 20582c78ee89SAlexei Starovoitov { 20592c78ee89SAlexei Starovoitov switch (prog_type) { 20602c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_KPROBE: 20612c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_TRACEPOINT: 20622c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_PERF_EVENT: 20632c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_RAW_TRACEPOINT: 20642c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 20652c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_TRACING: 20662c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LSM: 20672c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_STRUCT_OPS: /* has access to struct sock */ 20682c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_EXT: /* extends any prog */ 20692c78ee89SAlexei Starovoitov return true; 20702c78ee89SAlexei Starovoitov default: 20712c78ee89SAlexei Starovoitov return false; 20722c78ee89SAlexei Starovoitov } 20732c78ee89SAlexei Starovoitov } 20742c78ee89SAlexei Starovoitov 207509756af4SAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 20765b92a28aSAlexei Starovoitov #define BPF_PROG_LOAD_LAST_FIELD attach_prog_fd 207709756af4SAlexei Starovoitov 2078838e9690SYonghong Song static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) 207909756af4SAlexei Starovoitov { 208009756af4SAlexei Starovoitov enum bpf_prog_type type = attr->prog_type; 2081290248a5SAndrii Nakryiko struct bpf_prog *prog, *dst_prog = NULL; 2082290248a5SAndrii Nakryiko struct btf *attach_btf = NULL; 208309756af4SAlexei Starovoitov int err; 208409756af4SAlexei Starovoitov char license[128]; 208509756af4SAlexei Starovoitov bool is_gpl; 208609756af4SAlexei Starovoitov 208709756af4SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_LOAD)) 208809756af4SAlexei Starovoitov return -EINVAL; 208909756af4SAlexei Starovoitov 2090c240eff6SJiong Wang if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | 2091c240eff6SJiong Wang BPF_F_ANY_ALIGNMENT | 209210d274e8SAlexei Starovoitov BPF_F_TEST_STATE_FREQ | 20931e6c62a8SAlexei Starovoitov BPF_F_SLEEPABLE | 2094c240eff6SJiong Wang BPF_F_TEST_RND_HI32)) 2095e07b98d9SDavid S. Miller return -EINVAL; 2096e07b98d9SDavid S. Miller 2097e9ee9efcSDavid Miller if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && 2098e9ee9efcSDavid Miller (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && 20992c78ee89SAlexei Starovoitov !bpf_capable()) 2100e9ee9efcSDavid Miller return -EPERM; 2101e9ee9efcSDavid Miller 210209756af4SAlexei Starovoitov /* copy eBPF program license from user space */ 2103535e7b4bSMickaël Salaün if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 210409756af4SAlexei Starovoitov sizeof(license) - 1) < 0) 210509756af4SAlexei Starovoitov return -EFAULT; 210609756af4SAlexei Starovoitov license[sizeof(license) - 1] = 0; 210709756af4SAlexei Starovoitov 210809756af4SAlexei Starovoitov /* eBPF programs must be GPL compatible to use GPL-ed functions */ 210909756af4SAlexei Starovoitov is_gpl = license_is_gpl_compatible(license); 211009756af4SAlexei Starovoitov 2111c04c0d2bSAlexei Starovoitov if (attr->insn_cnt == 0 || 21122c78ee89SAlexei Starovoitov attr->insn_cnt > (bpf_capable() ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) 2113ef0915caSDaniel Borkmann return -E2BIG; 211480b7d819SChenbo Feng if (type != BPF_PROG_TYPE_SOCKET_FILTER && 211580b7d819SChenbo Feng type != BPF_PROG_TYPE_CGROUP_SKB && 21162c78ee89SAlexei Starovoitov !bpf_capable()) 21172c78ee89SAlexei Starovoitov return -EPERM; 21182c78ee89SAlexei Starovoitov 2119b338cb92SMaciej Żenczykowski if (is_net_admin_prog_type(type) && !capable(CAP_NET_ADMIN) && !capable(CAP_SYS_ADMIN)) 21202c78ee89SAlexei Starovoitov return -EPERM; 21212c78ee89SAlexei Starovoitov if (is_perfmon_prog_type(type) && !perfmon_capable()) 21221be7f75dSAlexei Starovoitov return -EPERM; 21231be7f75dSAlexei Starovoitov 2124290248a5SAndrii Nakryiko /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog 2125290248a5SAndrii Nakryiko * or btf, we need to check which one it is 2126290248a5SAndrii Nakryiko */ 2127290248a5SAndrii Nakryiko if (attr->attach_prog_fd) { 2128290248a5SAndrii Nakryiko dst_prog = bpf_prog_get(attr->attach_prog_fd); 2129290248a5SAndrii Nakryiko if (IS_ERR(dst_prog)) { 2130290248a5SAndrii Nakryiko dst_prog = NULL; 2131290248a5SAndrii Nakryiko attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd); 2132290248a5SAndrii Nakryiko if (IS_ERR(attach_btf)) 2133290248a5SAndrii Nakryiko return -EINVAL; 2134290248a5SAndrii Nakryiko if (!btf_is_kernel(attach_btf)) { 21358bdd8e27SAndrii Nakryiko /* attaching through specifying bpf_prog's BTF 21368bdd8e27SAndrii Nakryiko * objects directly might be supported eventually 21378bdd8e27SAndrii Nakryiko */ 2138290248a5SAndrii Nakryiko btf_put(attach_btf); 21398bdd8e27SAndrii Nakryiko return -ENOTSUPP; 2140290248a5SAndrii Nakryiko } 2141290248a5SAndrii Nakryiko } 2142290248a5SAndrii Nakryiko } else if (attr->attach_btf_id) { 2143290248a5SAndrii Nakryiko /* fall back to vmlinux BTF, if BTF type ID is specified */ 2144290248a5SAndrii Nakryiko attach_btf = bpf_get_btf_vmlinux(); 2145290248a5SAndrii Nakryiko if (IS_ERR(attach_btf)) 2146290248a5SAndrii Nakryiko return PTR_ERR(attach_btf); 2147290248a5SAndrii Nakryiko if (!attach_btf) 2148290248a5SAndrii Nakryiko return -EINVAL; 2149290248a5SAndrii Nakryiko btf_get(attach_btf); 2150290248a5SAndrii Nakryiko } 2151290248a5SAndrii Nakryiko 2152aac3fc32SAndrey Ignatov bpf_prog_load_fixup_attach_type(attr); 2153ccfe29ebSAlexei Starovoitov if (bpf_prog_load_check_attach(type, attr->expected_attach_type, 2154290248a5SAndrii Nakryiko attach_btf, attr->attach_btf_id, 2155290248a5SAndrii Nakryiko dst_prog)) { 2156290248a5SAndrii Nakryiko if (dst_prog) 2157290248a5SAndrii Nakryiko bpf_prog_put(dst_prog); 2158290248a5SAndrii Nakryiko if (attach_btf) 2159290248a5SAndrii Nakryiko btf_put(attach_btf); 21605e43f899SAndrey Ignatov return -EINVAL; 2161290248a5SAndrii Nakryiko } 21625e43f899SAndrey Ignatov 216309756af4SAlexei Starovoitov /* plain bpf_prog allocation */ 216409756af4SAlexei Starovoitov prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 2165290248a5SAndrii Nakryiko if (!prog) { 2166290248a5SAndrii Nakryiko if (dst_prog) 2167290248a5SAndrii Nakryiko bpf_prog_put(dst_prog); 2168290248a5SAndrii Nakryiko if (attach_btf) 2169290248a5SAndrii Nakryiko btf_put(attach_btf); 217009756af4SAlexei Starovoitov return -ENOMEM; 2171290248a5SAndrii Nakryiko } 217209756af4SAlexei Starovoitov 21735e43f899SAndrey Ignatov prog->expected_attach_type = attr->expected_attach_type; 2174290248a5SAndrii Nakryiko prog->aux->attach_btf = attach_btf; 2175ccfe29ebSAlexei Starovoitov prog->aux->attach_btf_id = attr->attach_btf_id; 21763aac1eadSToke Høiland-Jørgensen prog->aux->dst_prog = dst_prog; 21779a18eedbSJakub Kicinski prog->aux->offload_requested = !!attr->prog_ifindex; 21781e6c62a8SAlexei Starovoitov prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE; 21799a18eedbSJakub Kicinski 2180afdb09c7SChenbo Feng err = security_bpf_prog_alloc(prog->aux); 2181aaac3ba9SAlexei Starovoitov if (err) 21823ac1f01bSRoman Gushchin goto free_prog; 2183aaac3ba9SAlexei Starovoitov 21843ac1f01bSRoman Gushchin prog->aux->user = get_current_user(); 218509756af4SAlexei Starovoitov prog->len = attr->insn_cnt; 218609756af4SAlexei Starovoitov 218709756af4SAlexei Starovoitov err = -EFAULT; 2188535e7b4bSMickaël Salaün if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 2189aafe6ae9SDaniel Borkmann bpf_prog_insn_size(prog)) != 0) 21903ac1f01bSRoman Gushchin goto free_prog_sec; 219109756af4SAlexei Starovoitov 219209756af4SAlexei Starovoitov prog->orig_prog = NULL; 2193a91263d5SDaniel Borkmann prog->jited = 0; 219409756af4SAlexei Starovoitov 219585192dbfSAndrii Nakryiko atomic64_set(&prog->aux->refcnt, 1); 2196a91263d5SDaniel Borkmann prog->gpl_compatible = is_gpl ? 1 : 0; 219709756af4SAlexei Starovoitov 21989a18eedbSJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 2199ab3f0063SJakub Kicinski err = bpf_prog_offload_init(prog, attr); 2200ab3f0063SJakub Kicinski if (err) 22013ac1f01bSRoman Gushchin goto free_prog_sec; 2202ab3f0063SJakub Kicinski } 2203ab3f0063SJakub Kicinski 220409756af4SAlexei Starovoitov /* find program type: socket_filter vs tracing_filter */ 220509756af4SAlexei Starovoitov err = find_prog_type(type, prog); 220609756af4SAlexei Starovoitov if (err < 0) 22073ac1f01bSRoman Gushchin goto free_prog_sec; 220809756af4SAlexei Starovoitov 22099285ec4cSJason A. Donenfeld prog->aux->load_time = ktime_get_boottime_ns(); 22108e7ae251SMartin KaFai Lau err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, 22118e7ae251SMartin KaFai Lau sizeof(attr->prog_name)); 22128e7ae251SMartin KaFai Lau if (err < 0) 22133ac1f01bSRoman Gushchin goto free_prog_sec; 2214cb4d2b3fSMartin KaFai Lau 221509756af4SAlexei Starovoitov /* run eBPF verifier */ 2216838e9690SYonghong Song err = bpf_check(&prog, attr, uattr); 221709756af4SAlexei Starovoitov if (err < 0) 221809756af4SAlexei Starovoitov goto free_used_maps; 221909756af4SAlexei Starovoitov 2220d1c55ab5SDaniel Borkmann prog = bpf_prog_select_runtime(prog, &err); 222104fd61abSAlexei Starovoitov if (err < 0) 222204fd61abSAlexei Starovoitov goto free_used_maps; 222309756af4SAlexei Starovoitov 2224dc4bb0e2SMartin KaFai Lau err = bpf_prog_alloc_id(prog); 2225dc4bb0e2SMartin KaFai Lau if (err) 2226dc4bb0e2SMartin KaFai Lau goto free_used_maps; 2227dc4bb0e2SMartin KaFai Lau 2228c751798aSDaniel Borkmann /* Upon success of bpf_prog_alloc_id(), the BPF prog is 2229c751798aSDaniel Borkmann * effectively publicly exposed. However, retrieving via 2230c751798aSDaniel Borkmann * bpf_prog_get_fd_by_id() will take another reference, 2231c751798aSDaniel Borkmann * therefore it cannot be gone underneath us. 2232c751798aSDaniel Borkmann * 2233c751798aSDaniel Borkmann * Only for the time /after/ successful bpf_prog_new_fd() 2234c751798aSDaniel Borkmann * and before returning to userspace, we might just hold 2235c751798aSDaniel Borkmann * one reference and any parallel close on that fd could 2236c751798aSDaniel Borkmann * rip everything out. Hence, below notifications must 2237c751798aSDaniel Borkmann * happen before bpf_prog_new_fd(). 2238c751798aSDaniel Borkmann * 2239c751798aSDaniel Borkmann * Also, any failure handling from this point onwards must 2240c751798aSDaniel Borkmann * be using bpf_prog_put() given the program is exposed. 2241b16d9aa4SMartin KaFai Lau */ 224274451e66SDaniel Borkmann bpf_prog_kallsyms_add(prog); 22436ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); 2244bae141f5SDaniel Borkmann bpf_audit_prog(prog, BPF_AUDIT_LOAD); 2245c751798aSDaniel Borkmann 2246c751798aSDaniel Borkmann err = bpf_prog_new_fd(prog); 2247c751798aSDaniel Borkmann if (err < 0) 2248c751798aSDaniel Borkmann bpf_prog_put(prog); 224909756af4SAlexei Starovoitov return err; 225009756af4SAlexei Starovoitov 225109756af4SAlexei Starovoitov free_used_maps: 2252cd7455f1SDaniel Borkmann /* In case we have subprogs, we need to wait for a grace 2253cd7455f1SDaniel Borkmann * period before we can tear down JIT memory since symbols 2254cd7455f1SDaniel Borkmann * are already exposed under kallsyms. 2255cd7455f1SDaniel Borkmann */ 2256cd7455f1SDaniel Borkmann __bpf_prog_put_noref(prog, prog->aux->func_cnt); 2257cd7455f1SDaniel Borkmann return err; 2258afdb09c7SChenbo Feng free_prog_sec: 22593ac1f01bSRoman Gushchin free_uid(prog->aux->user); 2260afdb09c7SChenbo Feng security_bpf_prog_free(prog->aux); 22613ac1f01bSRoman Gushchin free_prog: 226222dc4a0fSAndrii Nakryiko if (prog->aux->attach_btf) 226322dc4a0fSAndrii Nakryiko btf_put(prog->aux->attach_btf); 226409756af4SAlexei Starovoitov bpf_prog_free(prog); 226509756af4SAlexei Starovoitov return err; 226609756af4SAlexei Starovoitov } 226709756af4SAlexei Starovoitov 22686e71b04aSChenbo Feng #define BPF_OBJ_LAST_FIELD file_flags 2269b2197755SDaniel Borkmann 2270b2197755SDaniel Borkmann static int bpf_obj_pin(const union bpf_attr *attr) 2271b2197755SDaniel Borkmann { 22726e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0) 2273b2197755SDaniel Borkmann return -EINVAL; 2274b2197755SDaniel Borkmann 2275535e7b4bSMickaël Salaün return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 2276b2197755SDaniel Borkmann } 2277b2197755SDaniel Borkmann 2278b2197755SDaniel Borkmann static int bpf_obj_get(const union bpf_attr *attr) 2279b2197755SDaniel Borkmann { 22806e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || 22816e71b04aSChenbo Feng attr->file_flags & ~BPF_OBJ_FLAG_MASK) 2282b2197755SDaniel Borkmann return -EINVAL; 2283b2197755SDaniel Borkmann 22846e71b04aSChenbo Feng return bpf_obj_get_user(u64_to_user_ptr(attr->pathname), 22856e71b04aSChenbo Feng attr->file_flags); 2286b2197755SDaniel Borkmann } 2287b2197755SDaniel Borkmann 2288f2e10bffSAndrii Nakryiko void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 2289a3b80e10SAndrii Nakryiko const struct bpf_link_ops *ops, struct bpf_prog *prog) 229070ed506cSAndrii Nakryiko { 229170ed506cSAndrii Nakryiko atomic64_set(&link->refcnt, 1); 2292f2e10bffSAndrii Nakryiko link->type = type; 2293a3b80e10SAndrii Nakryiko link->id = 0; 229470ed506cSAndrii Nakryiko link->ops = ops; 229570ed506cSAndrii Nakryiko link->prog = prog; 229670ed506cSAndrii Nakryiko } 229770ed506cSAndrii Nakryiko 2298a3b80e10SAndrii Nakryiko static void bpf_link_free_id(int id) 2299a3b80e10SAndrii Nakryiko { 2300a3b80e10SAndrii Nakryiko if (!id) 2301a3b80e10SAndrii Nakryiko return; 2302a3b80e10SAndrii Nakryiko 2303a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 2304a3b80e10SAndrii Nakryiko idr_remove(&link_idr, id); 2305a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 2306a3b80e10SAndrii Nakryiko } 2307a3b80e10SAndrii Nakryiko 230898868668SAndrii Nakryiko /* Clean up bpf_link and corresponding anon_inode file and FD. After 230998868668SAndrii Nakryiko * anon_inode is created, bpf_link can't be just kfree()'d due to deferred 2310a3b80e10SAndrii Nakryiko * anon_inode's release() call. This helper marksbpf_link as 2311a3b80e10SAndrii Nakryiko * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt 2312a3b80e10SAndrii Nakryiko * is not decremented, it's the responsibility of a calling code that failed 2313a3b80e10SAndrii Nakryiko * to complete bpf_link initialization. 231498868668SAndrii Nakryiko */ 2315a3b80e10SAndrii Nakryiko void bpf_link_cleanup(struct bpf_link_primer *primer) 2316babf3164SAndrii Nakryiko { 2317a3b80e10SAndrii Nakryiko primer->link->prog = NULL; 2318a3b80e10SAndrii Nakryiko bpf_link_free_id(primer->id); 2319a3b80e10SAndrii Nakryiko fput(primer->file); 2320a3b80e10SAndrii Nakryiko put_unused_fd(primer->fd); 2321babf3164SAndrii Nakryiko } 2322babf3164SAndrii Nakryiko 232370ed506cSAndrii Nakryiko void bpf_link_inc(struct bpf_link *link) 232470ed506cSAndrii Nakryiko { 232570ed506cSAndrii Nakryiko atomic64_inc(&link->refcnt); 232670ed506cSAndrii Nakryiko } 232770ed506cSAndrii Nakryiko 232870ed506cSAndrii Nakryiko /* bpf_link_free is guaranteed to be called from process context */ 232970ed506cSAndrii Nakryiko static void bpf_link_free(struct bpf_link *link) 233070ed506cSAndrii Nakryiko { 2331a3b80e10SAndrii Nakryiko bpf_link_free_id(link->id); 2332babf3164SAndrii Nakryiko if (link->prog) { 2333babf3164SAndrii Nakryiko /* detach BPF program, clean up used resources */ 233470ed506cSAndrii Nakryiko link->ops->release(link); 2335babf3164SAndrii Nakryiko bpf_prog_put(link->prog); 2336babf3164SAndrii Nakryiko } 2337babf3164SAndrii Nakryiko /* free bpf_link and its containing memory */ 2338babf3164SAndrii Nakryiko link->ops->dealloc(link); 233970ed506cSAndrii Nakryiko } 234070ed506cSAndrii Nakryiko 234170ed506cSAndrii Nakryiko static void bpf_link_put_deferred(struct work_struct *work) 234270ed506cSAndrii Nakryiko { 234370ed506cSAndrii Nakryiko struct bpf_link *link = container_of(work, struct bpf_link, work); 234470ed506cSAndrii Nakryiko 234570ed506cSAndrii Nakryiko bpf_link_free(link); 234670ed506cSAndrii Nakryiko } 234770ed506cSAndrii Nakryiko 234870ed506cSAndrii Nakryiko /* bpf_link_put can be called from atomic context, but ensures that resources 234970ed506cSAndrii Nakryiko * are freed from process context 235070ed506cSAndrii Nakryiko */ 235170ed506cSAndrii Nakryiko void bpf_link_put(struct bpf_link *link) 235270ed506cSAndrii Nakryiko { 235370ed506cSAndrii Nakryiko if (!atomic64_dec_and_test(&link->refcnt)) 235470ed506cSAndrii Nakryiko return; 235570ed506cSAndrii Nakryiko 2356f00f2f7fSAlexei Starovoitov if (in_atomic()) { 235770ed506cSAndrii Nakryiko INIT_WORK(&link->work, bpf_link_put_deferred); 235870ed506cSAndrii Nakryiko schedule_work(&link->work); 2359f00f2f7fSAlexei Starovoitov } else { 2360f00f2f7fSAlexei Starovoitov bpf_link_free(link); 2361f00f2f7fSAlexei Starovoitov } 236270ed506cSAndrii Nakryiko } 236370ed506cSAndrii Nakryiko 236470ed506cSAndrii Nakryiko static int bpf_link_release(struct inode *inode, struct file *filp) 236570ed506cSAndrii Nakryiko { 236670ed506cSAndrii Nakryiko struct bpf_link *link = filp->private_data; 236770ed506cSAndrii Nakryiko 236870ed506cSAndrii Nakryiko bpf_link_put(link); 2369fec56f58SAlexei Starovoitov return 0; 2370fec56f58SAlexei Starovoitov } 2371fec56f58SAlexei Starovoitov 237270ed506cSAndrii Nakryiko #ifdef CONFIG_PROC_FS 2373f2e10bffSAndrii Nakryiko #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) 2374f2e10bffSAndrii Nakryiko #define BPF_MAP_TYPE(_id, _ops) 2375f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) [_id] = #_name, 2376f2e10bffSAndrii Nakryiko static const char *bpf_link_type_strs[] = { 2377f2e10bffSAndrii Nakryiko [BPF_LINK_TYPE_UNSPEC] = "<invalid>", 2378f2e10bffSAndrii Nakryiko #include <linux/bpf_types.h> 2379f2e10bffSAndrii Nakryiko }; 2380f2e10bffSAndrii Nakryiko #undef BPF_PROG_TYPE 2381f2e10bffSAndrii Nakryiko #undef BPF_MAP_TYPE 2382f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 238370ed506cSAndrii Nakryiko 238470ed506cSAndrii Nakryiko static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) 238570ed506cSAndrii Nakryiko { 238670ed506cSAndrii Nakryiko const struct bpf_link *link = filp->private_data; 238770ed506cSAndrii Nakryiko const struct bpf_prog *prog = link->prog; 238870ed506cSAndrii Nakryiko char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 238970ed506cSAndrii Nakryiko 239070ed506cSAndrii Nakryiko bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 239170ed506cSAndrii Nakryiko seq_printf(m, 239270ed506cSAndrii Nakryiko "link_type:\t%s\n" 2393a3b80e10SAndrii Nakryiko "link_id:\t%u\n" 239470ed506cSAndrii Nakryiko "prog_tag:\t%s\n" 239570ed506cSAndrii Nakryiko "prog_id:\t%u\n", 2396f2e10bffSAndrii Nakryiko bpf_link_type_strs[link->type], 2397a3b80e10SAndrii Nakryiko link->id, 239870ed506cSAndrii Nakryiko prog_tag, 239970ed506cSAndrii Nakryiko prog->aux->id); 2400f2e10bffSAndrii Nakryiko if (link->ops->show_fdinfo) 2401f2e10bffSAndrii Nakryiko link->ops->show_fdinfo(link, m); 240270ed506cSAndrii Nakryiko } 240370ed506cSAndrii Nakryiko #endif 240470ed506cSAndrii Nakryiko 24056f302bfbSZou Wei static const struct file_operations bpf_link_fops = { 240670ed506cSAndrii Nakryiko #ifdef CONFIG_PROC_FS 240770ed506cSAndrii Nakryiko .show_fdinfo = bpf_link_show_fdinfo, 240870ed506cSAndrii Nakryiko #endif 240970ed506cSAndrii Nakryiko .release = bpf_link_release, 2410fec56f58SAlexei Starovoitov .read = bpf_dummy_read, 2411fec56f58SAlexei Starovoitov .write = bpf_dummy_write, 2412fec56f58SAlexei Starovoitov }; 2413fec56f58SAlexei Starovoitov 2414a3b80e10SAndrii Nakryiko static int bpf_link_alloc_id(struct bpf_link *link) 241570ed506cSAndrii Nakryiko { 2416a3b80e10SAndrii Nakryiko int id; 2417a3b80e10SAndrii Nakryiko 2418a3b80e10SAndrii Nakryiko idr_preload(GFP_KERNEL); 2419a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 2420a3b80e10SAndrii Nakryiko id = idr_alloc_cyclic(&link_idr, link, 1, INT_MAX, GFP_ATOMIC); 2421a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 2422a3b80e10SAndrii Nakryiko idr_preload_end(); 2423a3b80e10SAndrii Nakryiko 2424a3b80e10SAndrii Nakryiko return id; 242570ed506cSAndrii Nakryiko } 242670ed506cSAndrii Nakryiko 2427a3b80e10SAndrii Nakryiko /* Prepare bpf_link to be exposed to user-space by allocating anon_inode file, 2428a3b80e10SAndrii Nakryiko * reserving unused FD and allocating ID from link_idr. This is to be paired 2429a3b80e10SAndrii Nakryiko * with bpf_link_settle() to install FD and ID and expose bpf_link to 2430a3b80e10SAndrii Nakryiko * user-space, if bpf_link is successfully attached. If not, bpf_link and 2431a3b80e10SAndrii Nakryiko * pre-allocated resources are to be freed with bpf_cleanup() call. All the 2432a3b80e10SAndrii Nakryiko * transient state is passed around in struct bpf_link_primer. 2433a3b80e10SAndrii Nakryiko * This is preferred way to create and initialize bpf_link, especially when 2434a3b80e10SAndrii Nakryiko * there are complicated and expensive operations inbetween creating bpf_link 2435a3b80e10SAndrii Nakryiko * itself and attaching it to BPF hook. By using bpf_link_prime() and 2436a3b80e10SAndrii Nakryiko * bpf_link_settle() kernel code using bpf_link doesn't have to perform 2437a3b80e10SAndrii Nakryiko * expensive (and potentially failing) roll back operations in a rare case 2438a3b80e10SAndrii Nakryiko * that file, FD, or ID can't be allocated. 2439babf3164SAndrii Nakryiko */ 2440a3b80e10SAndrii Nakryiko int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer) 2441babf3164SAndrii Nakryiko { 2442babf3164SAndrii Nakryiko struct file *file; 2443a3b80e10SAndrii Nakryiko int fd, id; 2444babf3164SAndrii Nakryiko 2445babf3164SAndrii Nakryiko fd = get_unused_fd_flags(O_CLOEXEC); 2446babf3164SAndrii Nakryiko if (fd < 0) 2447a3b80e10SAndrii Nakryiko return fd; 2448babf3164SAndrii Nakryiko 2449babf3164SAndrii Nakryiko 2450a3b80e10SAndrii Nakryiko id = bpf_link_alloc_id(link); 2451a3b80e10SAndrii Nakryiko if (id < 0) { 2452a3b80e10SAndrii Nakryiko put_unused_fd(fd); 2453a3b80e10SAndrii Nakryiko return id; 2454a3b80e10SAndrii Nakryiko } 2455babf3164SAndrii Nakryiko 2456babf3164SAndrii Nakryiko file = anon_inode_getfile("bpf_link", &bpf_link_fops, link, O_CLOEXEC); 2457babf3164SAndrii Nakryiko if (IS_ERR(file)) { 2458138c6767SAndrii Nakryiko bpf_link_free_id(id); 2459babf3164SAndrii Nakryiko put_unused_fd(fd); 2460138c6767SAndrii Nakryiko return PTR_ERR(file); 2461babf3164SAndrii Nakryiko } 2462babf3164SAndrii Nakryiko 2463a3b80e10SAndrii Nakryiko primer->link = link; 2464a3b80e10SAndrii Nakryiko primer->file = file; 2465a3b80e10SAndrii Nakryiko primer->fd = fd; 2466a3b80e10SAndrii Nakryiko primer->id = id; 2467a3b80e10SAndrii Nakryiko return 0; 2468a3b80e10SAndrii Nakryiko } 2469a3b80e10SAndrii Nakryiko 2470a3b80e10SAndrii Nakryiko int bpf_link_settle(struct bpf_link_primer *primer) 2471a3b80e10SAndrii Nakryiko { 2472a3b80e10SAndrii Nakryiko /* make bpf_link fetchable by ID */ 2473a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 2474a3b80e10SAndrii Nakryiko primer->link->id = primer->id; 2475a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 2476a3b80e10SAndrii Nakryiko /* make bpf_link fetchable by FD */ 2477a3b80e10SAndrii Nakryiko fd_install(primer->fd, primer->file); 2478a3b80e10SAndrii Nakryiko /* pass through installed FD */ 2479a3b80e10SAndrii Nakryiko return primer->fd; 2480a3b80e10SAndrii Nakryiko } 2481a3b80e10SAndrii Nakryiko 2482a3b80e10SAndrii Nakryiko int bpf_link_new_fd(struct bpf_link *link) 2483a3b80e10SAndrii Nakryiko { 2484a3b80e10SAndrii Nakryiko return anon_inode_getfd("bpf-link", &bpf_link_fops, link, O_CLOEXEC); 2485babf3164SAndrii Nakryiko } 2486babf3164SAndrii Nakryiko 248770ed506cSAndrii Nakryiko struct bpf_link *bpf_link_get_from_fd(u32 ufd) 248870ed506cSAndrii Nakryiko { 248970ed506cSAndrii Nakryiko struct fd f = fdget(ufd); 249070ed506cSAndrii Nakryiko struct bpf_link *link; 249170ed506cSAndrii Nakryiko 249270ed506cSAndrii Nakryiko if (!f.file) 249370ed506cSAndrii Nakryiko return ERR_PTR(-EBADF); 249470ed506cSAndrii Nakryiko if (f.file->f_op != &bpf_link_fops) { 249570ed506cSAndrii Nakryiko fdput(f); 249670ed506cSAndrii Nakryiko return ERR_PTR(-EINVAL); 249770ed506cSAndrii Nakryiko } 249870ed506cSAndrii Nakryiko 249970ed506cSAndrii Nakryiko link = f.file->private_data; 250070ed506cSAndrii Nakryiko bpf_link_inc(link); 250170ed506cSAndrii Nakryiko fdput(f); 250270ed506cSAndrii Nakryiko 250370ed506cSAndrii Nakryiko return link; 250470ed506cSAndrii Nakryiko } 250570ed506cSAndrii Nakryiko 250670ed506cSAndrii Nakryiko struct bpf_tracing_link { 250770ed506cSAndrii Nakryiko struct bpf_link link; 2508f2e10bffSAndrii Nakryiko enum bpf_attach_type attach_type; 25093aac1eadSToke Høiland-Jørgensen struct bpf_trampoline *trampoline; 25103aac1eadSToke Høiland-Jørgensen struct bpf_prog *tgt_prog; 251170ed506cSAndrii Nakryiko }; 251270ed506cSAndrii Nakryiko 251370ed506cSAndrii Nakryiko static void bpf_tracing_link_release(struct bpf_link *link) 251470ed506cSAndrii Nakryiko { 25153aac1eadSToke Høiland-Jørgensen struct bpf_tracing_link *tr_link = 25163aac1eadSToke Høiland-Jørgensen container_of(link, struct bpf_tracing_link, link); 25173aac1eadSToke Høiland-Jørgensen 25183aac1eadSToke Høiland-Jørgensen WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog, 25193aac1eadSToke Høiland-Jørgensen tr_link->trampoline)); 25203aac1eadSToke Høiland-Jørgensen 25213aac1eadSToke Høiland-Jørgensen bpf_trampoline_put(tr_link->trampoline); 25223aac1eadSToke Høiland-Jørgensen 25233aac1eadSToke Høiland-Jørgensen /* tgt_prog is NULL if target is a kernel function */ 25243aac1eadSToke Høiland-Jørgensen if (tr_link->tgt_prog) 25253aac1eadSToke Høiland-Jørgensen bpf_prog_put(tr_link->tgt_prog); 2526babf3164SAndrii Nakryiko } 2527babf3164SAndrii Nakryiko 2528babf3164SAndrii Nakryiko static void bpf_tracing_link_dealloc(struct bpf_link *link) 2529babf3164SAndrii Nakryiko { 253070ed506cSAndrii Nakryiko struct bpf_tracing_link *tr_link = 253170ed506cSAndrii Nakryiko container_of(link, struct bpf_tracing_link, link); 253270ed506cSAndrii Nakryiko 253370ed506cSAndrii Nakryiko kfree(tr_link); 253470ed506cSAndrii Nakryiko } 253570ed506cSAndrii Nakryiko 2536f2e10bffSAndrii Nakryiko static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link, 2537f2e10bffSAndrii Nakryiko struct seq_file *seq) 2538f2e10bffSAndrii Nakryiko { 2539f2e10bffSAndrii Nakryiko struct bpf_tracing_link *tr_link = 2540f2e10bffSAndrii Nakryiko container_of(link, struct bpf_tracing_link, link); 2541f2e10bffSAndrii Nakryiko 2542f2e10bffSAndrii Nakryiko seq_printf(seq, 2543f2e10bffSAndrii Nakryiko "attach_type:\t%d\n", 2544f2e10bffSAndrii Nakryiko tr_link->attach_type); 2545f2e10bffSAndrii Nakryiko } 2546f2e10bffSAndrii Nakryiko 2547f2e10bffSAndrii Nakryiko static int bpf_tracing_link_fill_link_info(const struct bpf_link *link, 2548f2e10bffSAndrii Nakryiko struct bpf_link_info *info) 2549f2e10bffSAndrii Nakryiko { 2550f2e10bffSAndrii Nakryiko struct bpf_tracing_link *tr_link = 2551f2e10bffSAndrii Nakryiko container_of(link, struct bpf_tracing_link, link); 2552f2e10bffSAndrii Nakryiko 2553f2e10bffSAndrii Nakryiko info->tracing.attach_type = tr_link->attach_type; 2554441e8c66SToke Høiland-Jørgensen bpf_trampoline_unpack_key(tr_link->trampoline->key, 2555441e8c66SToke Høiland-Jørgensen &info->tracing.target_obj_id, 2556441e8c66SToke Høiland-Jørgensen &info->tracing.target_btf_id); 2557f2e10bffSAndrii Nakryiko 2558f2e10bffSAndrii Nakryiko return 0; 2559f2e10bffSAndrii Nakryiko } 2560f2e10bffSAndrii Nakryiko 256170ed506cSAndrii Nakryiko static const struct bpf_link_ops bpf_tracing_link_lops = { 256270ed506cSAndrii Nakryiko .release = bpf_tracing_link_release, 2563babf3164SAndrii Nakryiko .dealloc = bpf_tracing_link_dealloc, 2564f2e10bffSAndrii Nakryiko .show_fdinfo = bpf_tracing_link_show_fdinfo, 2565f2e10bffSAndrii Nakryiko .fill_link_info = bpf_tracing_link_fill_link_info, 256670ed506cSAndrii Nakryiko }; 256770ed506cSAndrii Nakryiko 25684a1e7c0cSToke Høiland-Jørgensen static int bpf_tracing_prog_attach(struct bpf_prog *prog, 25694a1e7c0cSToke Høiland-Jørgensen int tgt_prog_fd, 25704a1e7c0cSToke Høiland-Jørgensen u32 btf_id) 2571fec56f58SAlexei Starovoitov { 2572a3b80e10SAndrii Nakryiko struct bpf_link_primer link_primer; 25733aac1eadSToke Høiland-Jørgensen struct bpf_prog *tgt_prog = NULL; 25744a1e7c0cSToke Høiland-Jørgensen struct bpf_trampoline *tr = NULL; 257570ed506cSAndrii Nakryiko struct bpf_tracing_link *link; 25764a1e7c0cSToke Høiland-Jørgensen u64 key = 0; 2577a3b80e10SAndrii Nakryiko int err; 2578fec56f58SAlexei Starovoitov 25799e4e01dfSKP Singh switch (prog->type) { 25809e4e01dfSKP Singh case BPF_PROG_TYPE_TRACING: 2581fec56f58SAlexei Starovoitov if (prog->expected_attach_type != BPF_TRACE_FENTRY && 2582be8704ffSAlexei Starovoitov prog->expected_attach_type != BPF_TRACE_FEXIT && 25839e4e01dfSKP Singh prog->expected_attach_type != BPF_MODIFY_RETURN) { 25849e4e01dfSKP Singh err = -EINVAL; 25859e4e01dfSKP Singh goto out_put_prog; 25869e4e01dfSKP Singh } 25879e4e01dfSKP Singh break; 25889e4e01dfSKP Singh case BPF_PROG_TYPE_EXT: 25899e4e01dfSKP Singh if (prog->expected_attach_type != 0) { 25909e4e01dfSKP Singh err = -EINVAL; 25919e4e01dfSKP Singh goto out_put_prog; 25929e4e01dfSKP Singh } 25939e4e01dfSKP Singh break; 25949e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 25959e4e01dfSKP Singh if (prog->expected_attach_type != BPF_LSM_MAC) { 25969e4e01dfSKP Singh err = -EINVAL; 25979e4e01dfSKP Singh goto out_put_prog; 25989e4e01dfSKP Singh } 25999e4e01dfSKP Singh break; 26009e4e01dfSKP Singh default: 2601fec56f58SAlexei Starovoitov err = -EINVAL; 2602fec56f58SAlexei Starovoitov goto out_put_prog; 2603fec56f58SAlexei Starovoitov } 2604fec56f58SAlexei Starovoitov 26054a1e7c0cSToke Høiland-Jørgensen if (!!tgt_prog_fd != !!btf_id) { 26064a1e7c0cSToke Høiland-Jørgensen err = -EINVAL; 26074a1e7c0cSToke Høiland-Jørgensen goto out_put_prog; 26084a1e7c0cSToke Høiland-Jørgensen } 26094a1e7c0cSToke Høiland-Jørgensen 26104a1e7c0cSToke Høiland-Jørgensen if (tgt_prog_fd) { 26114a1e7c0cSToke Høiland-Jørgensen /* For now we only allow new targets for BPF_PROG_TYPE_EXT */ 26124a1e7c0cSToke Høiland-Jørgensen if (prog->type != BPF_PROG_TYPE_EXT) { 26134a1e7c0cSToke Høiland-Jørgensen err = -EINVAL; 26144a1e7c0cSToke Høiland-Jørgensen goto out_put_prog; 26154a1e7c0cSToke Høiland-Jørgensen } 26164a1e7c0cSToke Høiland-Jørgensen 26174a1e7c0cSToke Høiland-Jørgensen tgt_prog = bpf_prog_get(tgt_prog_fd); 26184a1e7c0cSToke Høiland-Jørgensen if (IS_ERR(tgt_prog)) { 26194a1e7c0cSToke Høiland-Jørgensen err = PTR_ERR(tgt_prog); 26204a1e7c0cSToke Høiland-Jørgensen tgt_prog = NULL; 26214a1e7c0cSToke Høiland-Jørgensen goto out_put_prog; 26224a1e7c0cSToke Høiland-Jørgensen } 26234a1e7c0cSToke Høiland-Jørgensen 262422dc4a0fSAndrii Nakryiko key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id); 26254a1e7c0cSToke Høiland-Jørgensen } 26264a1e7c0cSToke Høiland-Jørgensen 262770ed506cSAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER); 262870ed506cSAndrii Nakryiko if (!link) { 262970ed506cSAndrii Nakryiko err = -ENOMEM; 2630fec56f58SAlexei Starovoitov goto out_put_prog; 2631fec56f58SAlexei Starovoitov } 2632f2e10bffSAndrii Nakryiko bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING, 2633f2e10bffSAndrii Nakryiko &bpf_tracing_link_lops, prog); 2634f2e10bffSAndrii Nakryiko link->attach_type = prog->expected_attach_type; 2635fec56f58SAlexei Starovoitov 26363aac1eadSToke Høiland-Jørgensen mutex_lock(&prog->aux->dst_mutex); 2637babf3164SAndrii Nakryiko 26384a1e7c0cSToke Høiland-Jørgensen /* There are a few possible cases here: 26394a1e7c0cSToke Høiland-Jørgensen * 26404a1e7c0cSToke Høiland-Jørgensen * - if prog->aux->dst_trampoline is set, the program was just loaded 26414a1e7c0cSToke Høiland-Jørgensen * and not yet attached to anything, so we can use the values stored 26424a1e7c0cSToke Høiland-Jørgensen * in prog->aux 26434a1e7c0cSToke Høiland-Jørgensen * 26444a1e7c0cSToke Høiland-Jørgensen * - if prog->aux->dst_trampoline is NULL, the program has already been 26454a1e7c0cSToke Høiland-Jørgensen * attached to a target and its initial target was cleared (below) 26464a1e7c0cSToke Høiland-Jørgensen * 26474a1e7c0cSToke Høiland-Jørgensen * - if tgt_prog != NULL, the caller specified tgt_prog_fd + 26484a1e7c0cSToke Høiland-Jørgensen * target_btf_id using the link_create API. 26494a1e7c0cSToke Høiland-Jørgensen * 26504a1e7c0cSToke Høiland-Jørgensen * - if tgt_prog == NULL when this function was called using the old 26514a1e7c0cSToke Høiland-Jørgensen * raw_tracepoint_open API, and we need a target from prog->aux 26524a1e7c0cSToke Høiland-Jørgensen * 2653*f3a95075SJiri Olsa * - if prog->aux->dst_trampoline and tgt_prog is NULL, the program 2654*f3a95075SJiri Olsa * was detached and is going for re-attachment. 26554a1e7c0cSToke Høiland-Jørgensen */ 26564a1e7c0cSToke Høiland-Jørgensen if (!prog->aux->dst_trampoline && !tgt_prog) { 2657*f3a95075SJiri Olsa /* 2658*f3a95075SJiri Olsa * Allow re-attach for TRACING and LSM programs. If it's 2659*f3a95075SJiri Olsa * currently linked, bpf_trampoline_link_prog will fail. 2660*f3a95075SJiri Olsa * EXT programs need to specify tgt_prog_fd, so they 2661*f3a95075SJiri Olsa * re-attach in separate code path. 2662*f3a95075SJiri Olsa */ 2663*f3a95075SJiri Olsa if (prog->type != BPF_PROG_TYPE_TRACING && 2664*f3a95075SJiri Olsa prog->type != BPF_PROG_TYPE_LSM) { 2665*f3a95075SJiri Olsa err = -EINVAL; 26663aac1eadSToke Høiland-Jørgensen goto out_unlock; 26673aac1eadSToke Høiland-Jørgensen } 2668*f3a95075SJiri Olsa btf_id = prog->aux->attach_btf_id; 2669*f3a95075SJiri Olsa key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id); 2670*f3a95075SJiri Olsa } 26714a1e7c0cSToke Høiland-Jørgensen 26724a1e7c0cSToke Høiland-Jørgensen if (!prog->aux->dst_trampoline || 26734a1e7c0cSToke Høiland-Jørgensen (key && key != prog->aux->dst_trampoline->key)) { 26744a1e7c0cSToke Høiland-Jørgensen /* If there is no saved target, or the specified target is 26754a1e7c0cSToke Høiland-Jørgensen * different from the destination specified at load time, we 26764a1e7c0cSToke Høiland-Jørgensen * need a new trampoline and a check for compatibility 26774a1e7c0cSToke Høiland-Jørgensen */ 26784a1e7c0cSToke Høiland-Jørgensen struct bpf_attach_target_info tgt_info = {}; 26794a1e7c0cSToke Høiland-Jørgensen 26804a1e7c0cSToke Høiland-Jørgensen err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id, 26814a1e7c0cSToke Høiland-Jørgensen &tgt_info); 26824a1e7c0cSToke Høiland-Jørgensen if (err) 26834a1e7c0cSToke Høiland-Jørgensen goto out_unlock; 26844a1e7c0cSToke Høiland-Jørgensen 26854a1e7c0cSToke Høiland-Jørgensen tr = bpf_trampoline_get(key, &tgt_info); 26864a1e7c0cSToke Høiland-Jørgensen if (!tr) { 26874a1e7c0cSToke Høiland-Jørgensen err = -ENOMEM; 26884a1e7c0cSToke Høiland-Jørgensen goto out_unlock; 26894a1e7c0cSToke Høiland-Jørgensen } 26904a1e7c0cSToke Høiland-Jørgensen } else { 26914a1e7c0cSToke Høiland-Jørgensen /* The caller didn't specify a target, or the target was the 26924a1e7c0cSToke Høiland-Jørgensen * same as the destination supplied during program load. This 26934a1e7c0cSToke Høiland-Jørgensen * means we can reuse the trampoline and reference from program 26944a1e7c0cSToke Høiland-Jørgensen * load time, and there is no need to allocate a new one. This 26954a1e7c0cSToke Høiland-Jørgensen * can only happen once for any program, as the saved values in 26964a1e7c0cSToke Høiland-Jørgensen * prog->aux are cleared below. 26974a1e7c0cSToke Høiland-Jørgensen */ 26983aac1eadSToke Høiland-Jørgensen tr = prog->aux->dst_trampoline; 26993aac1eadSToke Høiland-Jørgensen tgt_prog = prog->aux->dst_prog; 27004a1e7c0cSToke Høiland-Jørgensen } 27013aac1eadSToke Høiland-Jørgensen 27023aac1eadSToke Høiland-Jørgensen err = bpf_link_prime(&link->link, &link_primer); 27033aac1eadSToke Høiland-Jørgensen if (err) 27043aac1eadSToke Høiland-Jørgensen goto out_unlock; 27053aac1eadSToke Høiland-Jørgensen 27063aac1eadSToke Høiland-Jørgensen err = bpf_trampoline_link_prog(prog, tr); 2707babf3164SAndrii Nakryiko if (err) { 2708a3b80e10SAndrii Nakryiko bpf_link_cleanup(&link_primer); 27093aac1eadSToke Høiland-Jørgensen link = NULL; 27103aac1eadSToke Høiland-Jørgensen goto out_unlock; 2711babf3164SAndrii Nakryiko } 2712babf3164SAndrii Nakryiko 27133aac1eadSToke Høiland-Jørgensen link->tgt_prog = tgt_prog; 27143aac1eadSToke Høiland-Jørgensen link->trampoline = tr; 27153aac1eadSToke Høiland-Jørgensen 27164a1e7c0cSToke Høiland-Jørgensen /* Always clear the trampoline and target prog from prog->aux to make 27174a1e7c0cSToke Høiland-Jørgensen * sure the original attach destination is not kept alive after a 27184a1e7c0cSToke Høiland-Jørgensen * program is (re-)attached to another target. 27194a1e7c0cSToke Høiland-Jørgensen */ 27204a1e7c0cSToke Høiland-Jørgensen if (prog->aux->dst_prog && 27214a1e7c0cSToke Høiland-Jørgensen (tgt_prog_fd || tr != prog->aux->dst_trampoline)) 27224a1e7c0cSToke Høiland-Jørgensen /* got extra prog ref from syscall, or attaching to different prog */ 27234a1e7c0cSToke Høiland-Jørgensen bpf_prog_put(prog->aux->dst_prog); 27244a1e7c0cSToke Høiland-Jørgensen if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline) 27254a1e7c0cSToke Høiland-Jørgensen /* we allocated a new trampoline, so free the old one */ 27264a1e7c0cSToke Høiland-Jørgensen bpf_trampoline_put(prog->aux->dst_trampoline); 27274a1e7c0cSToke Høiland-Jørgensen 27283aac1eadSToke Høiland-Jørgensen prog->aux->dst_prog = NULL; 27293aac1eadSToke Høiland-Jørgensen prog->aux->dst_trampoline = NULL; 27303aac1eadSToke Høiland-Jørgensen mutex_unlock(&prog->aux->dst_mutex); 27313aac1eadSToke Høiland-Jørgensen 2732a3b80e10SAndrii Nakryiko return bpf_link_settle(&link_primer); 27333aac1eadSToke Høiland-Jørgensen out_unlock: 27344a1e7c0cSToke Høiland-Jørgensen if (tr && tr != prog->aux->dst_trampoline) 27354a1e7c0cSToke Høiland-Jørgensen bpf_trampoline_put(tr); 27363aac1eadSToke Høiland-Jørgensen mutex_unlock(&prog->aux->dst_mutex); 27373aac1eadSToke Høiland-Jørgensen kfree(link); 2738fec56f58SAlexei Starovoitov out_put_prog: 27394a1e7c0cSToke Høiland-Jørgensen if (tgt_prog_fd && tgt_prog) 27404a1e7c0cSToke Høiland-Jørgensen bpf_prog_put(tgt_prog); 2741fec56f58SAlexei Starovoitov return err; 2742fec56f58SAlexei Starovoitov } 2743fec56f58SAlexei Starovoitov 274470ed506cSAndrii Nakryiko struct bpf_raw_tp_link { 274570ed506cSAndrii Nakryiko struct bpf_link link; 2746c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 2747c4f6699dSAlexei Starovoitov }; 2748c4f6699dSAlexei Starovoitov 274970ed506cSAndrii Nakryiko static void bpf_raw_tp_link_release(struct bpf_link *link) 2750c4f6699dSAlexei Starovoitov { 275170ed506cSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 275270ed506cSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 2753c4f6699dSAlexei Starovoitov 275470ed506cSAndrii Nakryiko bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog); 2755a38d1107SMatt Mullins bpf_put_raw_tracepoint(raw_tp->btp); 2756babf3164SAndrii Nakryiko } 2757babf3164SAndrii Nakryiko 2758babf3164SAndrii Nakryiko static void bpf_raw_tp_link_dealloc(struct bpf_link *link) 2759babf3164SAndrii Nakryiko { 2760babf3164SAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 2761babf3164SAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 2762babf3164SAndrii Nakryiko 2763c4f6699dSAlexei Starovoitov kfree(raw_tp); 2764c4f6699dSAlexei Starovoitov } 2765c4f6699dSAlexei Starovoitov 2766f2e10bffSAndrii Nakryiko static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link, 2767f2e10bffSAndrii Nakryiko struct seq_file *seq) 2768f2e10bffSAndrii Nakryiko { 2769f2e10bffSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp_link = 2770f2e10bffSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 2771f2e10bffSAndrii Nakryiko 2772f2e10bffSAndrii Nakryiko seq_printf(seq, 2773f2e10bffSAndrii Nakryiko "tp_name:\t%s\n", 2774f2e10bffSAndrii Nakryiko raw_tp_link->btp->tp->name); 2775f2e10bffSAndrii Nakryiko } 2776f2e10bffSAndrii Nakryiko 2777f2e10bffSAndrii Nakryiko static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link, 2778f2e10bffSAndrii Nakryiko struct bpf_link_info *info) 2779f2e10bffSAndrii Nakryiko { 2780f2e10bffSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp_link = 2781f2e10bffSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 2782f2e10bffSAndrii Nakryiko char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name); 2783f2e10bffSAndrii Nakryiko const char *tp_name = raw_tp_link->btp->tp->name; 2784f2e10bffSAndrii Nakryiko u32 ulen = info->raw_tracepoint.tp_name_len; 2785f2e10bffSAndrii Nakryiko size_t tp_len = strlen(tp_name); 2786f2e10bffSAndrii Nakryiko 2787b474959dSYonghong Song if (!ulen ^ !ubuf) 2788f2e10bffSAndrii Nakryiko return -EINVAL; 2789f2e10bffSAndrii Nakryiko 2790f2e10bffSAndrii Nakryiko info->raw_tracepoint.tp_name_len = tp_len + 1; 2791f2e10bffSAndrii Nakryiko 2792f2e10bffSAndrii Nakryiko if (!ubuf) 2793f2e10bffSAndrii Nakryiko return 0; 2794f2e10bffSAndrii Nakryiko 2795f2e10bffSAndrii Nakryiko if (ulen >= tp_len + 1) { 2796f2e10bffSAndrii Nakryiko if (copy_to_user(ubuf, tp_name, tp_len + 1)) 2797f2e10bffSAndrii Nakryiko return -EFAULT; 2798f2e10bffSAndrii Nakryiko } else { 2799f2e10bffSAndrii Nakryiko char zero = '\0'; 2800f2e10bffSAndrii Nakryiko 2801f2e10bffSAndrii Nakryiko if (copy_to_user(ubuf, tp_name, ulen - 1)) 2802f2e10bffSAndrii Nakryiko return -EFAULT; 2803f2e10bffSAndrii Nakryiko if (put_user(zero, ubuf + ulen - 1)) 2804f2e10bffSAndrii Nakryiko return -EFAULT; 2805f2e10bffSAndrii Nakryiko return -ENOSPC; 2806f2e10bffSAndrii Nakryiko } 2807f2e10bffSAndrii Nakryiko 2808f2e10bffSAndrii Nakryiko return 0; 2809f2e10bffSAndrii Nakryiko } 2810f2e10bffSAndrii Nakryiko 2811a3b80e10SAndrii Nakryiko static const struct bpf_link_ops bpf_raw_tp_link_lops = { 281270ed506cSAndrii Nakryiko .release = bpf_raw_tp_link_release, 2813babf3164SAndrii Nakryiko .dealloc = bpf_raw_tp_link_dealloc, 2814f2e10bffSAndrii Nakryiko .show_fdinfo = bpf_raw_tp_link_show_fdinfo, 2815f2e10bffSAndrii Nakryiko .fill_link_info = bpf_raw_tp_link_fill_link_info, 2816c4f6699dSAlexei Starovoitov }; 2817c4f6699dSAlexei Starovoitov 2818c4f6699dSAlexei Starovoitov #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd 2819c4f6699dSAlexei Starovoitov 2820c4f6699dSAlexei Starovoitov static int bpf_raw_tracepoint_open(const union bpf_attr *attr) 2821c4f6699dSAlexei Starovoitov { 2822a3b80e10SAndrii Nakryiko struct bpf_link_primer link_primer; 2823babf3164SAndrii Nakryiko struct bpf_raw_tp_link *link; 2824c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 2825c4f6699dSAlexei Starovoitov struct bpf_prog *prog; 2826ac4414b5SAlexei Starovoitov const char *tp_name; 2827ac4414b5SAlexei Starovoitov char buf[128]; 2828a3b80e10SAndrii Nakryiko int err; 2829c4f6699dSAlexei Starovoitov 2830ac4414b5SAlexei Starovoitov if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN)) 2831ac4414b5SAlexei Starovoitov return -EINVAL; 2832ac4414b5SAlexei Starovoitov 2833ac4414b5SAlexei Starovoitov prog = bpf_prog_get(attr->raw_tracepoint.prog_fd); 2834ac4414b5SAlexei Starovoitov if (IS_ERR(prog)) 2835ac4414b5SAlexei Starovoitov return PTR_ERR(prog); 2836ac4414b5SAlexei Starovoitov 28379e4e01dfSKP Singh switch (prog->type) { 28389e4e01dfSKP Singh case BPF_PROG_TYPE_TRACING: 28399e4e01dfSKP Singh case BPF_PROG_TYPE_EXT: 28409e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 2841ac4414b5SAlexei Starovoitov if (attr->raw_tracepoint.name) { 2842fec56f58SAlexei Starovoitov /* The attach point for this category of programs 2843fec56f58SAlexei Starovoitov * should be specified via btf_id during program load. 2844ac4414b5SAlexei Starovoitov */ 2845ac4414b5SAlexei Starovoitov err = -EINVAL; 2846ac4414b5SAlexei Starovoitov goto out_put_prog; 2847ac4414b5SAlexei Starovoitov } 28489e4e01dfSKP Singh if (prog->type == BPF_PROG_TYPE_TRACING && 28499e4e01dfSKP Singh prog->expected_attach_type == BPF_TRACE_RAW_TP) { 285038207291SMartin KaFai Lau tp_name = prog->aux->attach_func_name; 28519e4e01dfSKP Singh break; 28529e4e01dfSKP Singh } 28535541075aSJiri Olsa err = bpf_tracing_prog_attach(prog, 0, 0); 28545541075aSJiri Olsa if (err >= 0) 28555541075aSJiri Olsa return err; 28565541075aSJiri Olsa goto out_put_prog; 28579e4e01dfSKP Singh case BPF_PROG_TYPE_RAW_TRACEPOINT: 28589e4e01dfSKP Singh case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 2859ac4414b5SAlexei Starovoitov if (strncpy_from_user(buf, 2860ac4414b5SAlexei Starovoitov u64_to_user_ptr(attr->raw_tracepoint.name), 2861ac4414b5SAlexei Starovoitov sizeof(buf) - 1) < 0) { 2862ac4414b5SAlexei Starovoitov err = -EFAULT; 2863ac4414b5SAlexei Starovoitov goto out_put_prog; 2864ac4414b5SAlexei Starovoitov } 2865ac4414b5SAlexei Starovoitov buf[sizeof(buf) - 1] = 0; 2866ac4414b5SAlexei Starovoitov tp_name = buf; 28679e4e01dfSKP Singh break; 28689e4e01dfSKP Singh default: 28699e4e01dfSKP Singh err = -EINVAL; 28709e4e01dfSKP Singh goto out_put_prog; 2871ac4414b5SAlexei Starovoitov } 2872c4f6699dSAlexei Starovoitov 2873a38d1107SMatt Mullins btp = bpf_get_raw_tracepoint(tp_name); 2874ac4414b5SAlexei Starovoitov if (!btp) { 2875ac4414b5SAlexei Starovoitov err = -ENOENT; 2876ac4414b5SAlexei Starovoitov goto out_put_prog; 2877ac4414b5SAlexei Starovoitov } 2878c4f6699dSAlexei Starovoitov 2879babf3164SAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER); 2880babf3164SAndrii Nakryiko if (!link) { 2881a38d1107SMatt Mullins err = -ENOMEM; 2882a38d1107SMatt Mullins goto out_put_btp; 2883a38d1107SMatt Mullins } 2884f2e10bffSAndrii Nakryiko bpf_link_init(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT, 2885f2e10bffSAndrii Nakryiko &bpf_raw_tp_link_lops, prog); 2886babf3164SAndrii Nakryiko link->btp = btp; 2887c4f6699dSAlexei Starovoitov 2888a3b80e10SAndrii Nakryiko err = bpf_link_prime(&link->link, &link_primer); 2889a3b80e10SAndrii Nakryiko if (err) { 2890babf3164SAndrii Nakryiko kfree(link); 2891babf3164SAndrii Nakryiko goto out_put_btp; 2892c4f6699dSAlexei Starovoitov } 2893babf3164SAndrii Nakryiko 2894babf3164SAndrii Nakryiko err = bpf_probe_register(link->btp, prog); 2895babf3164SAndrii Nakryiko if (err) { 2896a3b80e10SAndrii Nakryiko bpf_link_cleanup(&link_primer); 2897babf3164SAndrii Nakryiko goto out_put_btp; 2898babf3164SAndrii Nakryiko } 2899babf3164SAndrii Nakryiko 2900a3b80e10SAndrii Nakryiko return bpf_link_settle(&link_primer); 2901c4f6699dSAlexei Starovoitov 2902a38d1107SMatt Mullins out_put_btp: 2903a38d1107SMatt Mullins bpf_put_raw_tracepoint(btp); 2904ac4414b5SAlexei Starovoitov out_put_prog: 2905ac4414b5SAlexei Starovoitov bpf_prog_put(prog); 2906c4f6699dSAlexei Starovoitov return err; 2907c4f6699dSAlexei Starovoitov } 2908c4f6699dSAlexei Starovoitov 290933491588SAnders Roxell static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, 291033491588SAnders Roxell enum bpf_attach_type attach_type) 291133491588SAnders Roxell { 291233491588SAnders Roxell switch (prog->type) { 291333491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK: 291433491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 29150d01da6aSStanislav Fomichev case BPF_PROG_TYPE_CGROUP_SOCKOPT: 2916e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP: 291733491588SAnders Roxell return attach_type == prog->expected_attach_type ? 0 : -EINVAL; 29185cf1e914Sbrakmo case BPF_PROG_TYPE_CGROUP_SKB: 29192c78ee89SAlexei Starovoitov if (!capable(CAP_NET_ADMIN)) 29202c78ee89SAlexei Starovoitov /* cg-skb progs can be loaded by unpriv user. 29212c78ee89SAlexei Starovoitov * check permissions at attach time. 29222c78ee89SAlexei Starovoitov */ 29232c78ee89SAlexei Starovoitov return -EPERM; 29245cf1e914Sbrakmo return prog->enforce_expected_attach_type && 29255cf1e914Sbrakmo prog->expected_attach_type != attach_type ? 29265cf1e914Sbrakmo -EINVAL : 0; 292733491588SAnders Roxell default: 292833491588SAnders Roxell return 0; 292933491588SAnders Roxell } 293033491588SAnders Roxell } 293133491588SAnders Roxell 2932e28784e3SAndrii Nakryiko static enum bpf_prog_type 2933e28784e3SAndrii Nakryiko attach_type_to_prog_type(enum bpf_attach_type attach_type) 2934e28784e3SAndrii Nakryiko { 2935e28784e3SAndrii Nakryiko switch (attach_type) { 2936e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_INGRESS: 2937e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_EGRESS: 2938e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SKB; 2939e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_SOCK_CREATE: 2940f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 2941e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_POST_BIND: 2942e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_POST_BIND: 2943e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCK; 2944e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_BIND: 2945e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_BIND: 2946e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_CONNECT: 2947e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_CONNECT: 29481b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 29491b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 29501b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 29511b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 2952e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP4_SENDMSG: 2953e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP6_SENDMSG: 2954e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP4_RECVMSG: 2955e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP6_RECVMSG: 2956e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 2957e28784e3SAndrii Nakryiko case BPF_CGROUP_SOCK_OPS: 2958e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SOCK_OPS; 2959e28784e3SAndrii Nakryiko case BPF_CGROUP_DEVICE: 2960e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_DEVICE; 2961e28784e3SAndrii Nakryiko case BPF_SK_MSG_VERDICT: 2962e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SK_MSG; 2963e28784e3SAndrii Nakryiko case BPF_SK_SKB_STREAM_PARSER: 2964e28784e3SAndrii Nakryiko case BPF_SK_SKB_STREAM_VERDICT: 2965a7ba4558SCong Wang case BPF_SK_SKB_VERDICT: 2966e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SK_SKB; 2967e28784e3SAndrii Nakryiko case BPF_LIRC_MODE2: 2968e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_LIRC_MODE2; 2969e28784e3SAndrii Nakryiko case BPF_FLOW_DISSECTOR: 2970e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_FLOW_DISSECTOR; 2971e28784e3SAndrii Nakryiko case BPF_CGROUP_SYSCTL: 2972e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SYSCTL; 2973e28784e3SAndrii Nakryiko case BPF_CGROUP_GETSOCKOPT: 2974e28784e3SAndrii Nakryiko case BPF_CGROUP_SETSOCKOPT: 2975e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCKOPT; 2976de4e05caSYonghong Song case BPF_TRACE_ITER: 2977de4e05caSYonghong Song return BPF_PROG_TYPE_TRACING; 2978e9ddbb77SJakub Sitnicki case BPF_SK_LOOKUP: 2979e9ddbb77SJakub Sitnicki return BPF_PROG_TYPE_SK_LOOKUP; 2980aa8d3a71SAndrii Nakryiko case BPF_XDP: 2981aa8d3a71SAndrii Nakryiko return BPF_PROG_TYPE_XDP; 2982e28784e3SAndrii Nakryiko default: 2983e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_UNSPEC; 2984e28784e3SAndrii Nakryiko } 2985e28784e3SAndrii Nakryiko } 2986e28784e3SAndrii Nakryiko 29877dd68b32SAndrey Ignatov #define BPF_PROG_ATTACH_LAST_FIELD replace_bpf_fd 2988174a79ffSJohn Fastabend 2989324bda9eSAlexei Starovoitov #define BPF_F_ATTACH_MASK \ 29907dd68b32SAndrey Ignatov (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI | BPF_F_REPLACE) 2991324bda9eSAlexei Starovoitov 2992f4324551SDaniel Mack static int bpf_prog_attach(const union bpf_attr *attr) 2993f4324551SDaniel Mack { 29947f677633SAlexei Starovoitov enum bpf_prog_type ptype; 2995f4324551SDaniel Mack struct bpf_prog *prog; 29967f677633SAlexei Starovoitov int ret; 2997f4324551SDaniel Mack 2998f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_ATTACH)) 2999f4324551SDaniel Mack return -EINVAL; 3000f4324551SDaniel Mack 3001324bda9eSAlexei Starovoitov if (attr->attach_flags & ~BPF_F_ATTACH_MASK) 30027f677633SAlexei Starovoitov return -EINVAL; 30037f677633SAlexei Starovoitov 3004e28784e3SAndrii Nakryiko ptype = attach_type_to_prog_type(attr->attach_type); 3005e28784e3SAndrii Nakryiko if (ptype == BPF_PROG_TYPE_UNSPEC) 3006b2cd1257SDavid Ahern return -EINVAL; 3007b2cd1257SDavid Ahern 3008b2cd1257SDavid Ahern prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 3009f4324551SDaniel Mack if (IS_ERR(prog)) 3010f4324551SDaniel Mack return PTR_ERR(prog); 3011f4324551SDaniel Mack 30125e43f899SAndrey Ignatov if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) { 30135e43f899SAndrey Ignatov bpf_prog_put(prog); 30145e43f899SAndrey Ignatov return -EINVAL; 30155e43f899SAndrey Ignatov } 30165e43f899SAndrey Ignatov 3017fdb5c453SSean Young switch (ptype) { 3018fdb5c453SSean Young case BPF_PROG_TYPE_SK_SKB: 3019fdb5c453SSean Young case BPF_PROG_TYPE_SK_MSG: 3020604326b4SDaniel Borkmann ret = sock_map_get_from_fd(attr, prog); 3021fdb5c453SSean Young break; 3022fdb5c453SSean Young case BPF_PROG_TYPE_LIRC_MODE2: 3023fdb5c453SSean Young ret = lirc_prog_attach(attr, prog); 3024fdb5c453SSean Young break; 3025d58e468bSPetar Penkov case BPF_PROG_TYPE_FLOW_DISSECTOR: 3026a3fd7ceeSJakub Sitnicki ret = netns_bpf_prog_attach(attr, prog); 3027d58e468bSPetar Penkov break; 3028e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 3029e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 3030e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 3031e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 3032e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 3033e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 3034e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 3035fdb5c453SSean Young ret = cgroup_bpf_prog_attach(attr, ptype, prog); 3036e28784e3SAndrii Nakryiko break; 3037e28784e3SAndrii Nakryiko default: 3038e28784e3SAndrii Nakryiko ret = -EINVAL; 3039f4324551SDaniel Mack } 3040f4324551SDaniel Mack 30417f677633SAlexei Starovoitov if (ret) 30427f677633SAlexei Starovoitov bpf_prog_put(prog); 30437f677633SAlexei Starovoitov return ret; 3044f4324551SDaniel Mack } 3045f4324551SDaniel Mack 3046f4324551SDaniel Mack #define BPF_PROG_DETACH_LAST_FIELD attach_type 3047f4324551SDaniel Mack 3048f4324551SDaniel Mack static int bpf_prog_detach(const union bpf_attr *attr) 3049f4324551SDaniel Mack { 3050324bda9eSAlexei Starovoitov enum bpf_prog_type ptype; 3051f4324551SDaniel Mack 3052f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_DETACH)) 3053f4324551SDaniel Mack return -EINVAL; 3054f4324551SDaniel Mack 3055e28784e3SAndrii Nakryiko ptype = attach_type_to_prog_type(attr->attach_type); 3056e28784e3SAndrii Nakryiko 3057e28784e3SAndrii Nakryiko switch (ptype) { 3058e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SK_MSG: 3059e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SK_SKB: 3060bb0de313SLorenz Bauer return sock_map_prog_detach(attr, ptype); 3061e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_LIRC_MODE2: 3062f4364dcfSSean Young return lirc_prog_detach(attr); 3063e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_FLOW_DISSECTOR: 30644ac2add6SLorenz Bauer return netns_bpf_prog_detach(attr, ptype); 3065e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 3066e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 3067e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 3068e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 3069e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 3070e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 3071e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 3072e28784e3SAndrii Nakryiko return cgroup_bpf_prog_detach(attr, ptype); 3073f4324551SDaniel Mack default: 3074f4324551SDaniel Mack return -EINVAL; 3075f4324551SDaniel Mack } 3076f4324551SDaniel Mack } 307740304b2aSLawrence Brakmo 3078468e2f64SAlexei Starovoitov #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt 3079468e2f64SAlexei Starovoitov 3080468e2f64SAlexei Starovoitov static int bpf_prog_query(const union bpf_attr *attr, 3081468e2f64SAlexei Starovoitov union bpf_attr __user *uattr) 3082468e2f64SAlexei Starovoitov { 3083468e2f64SAlexei Starovoitov if (!capable(CAP_NET_ADMIN)) 3084468e2f64SAlexei Starovoitov return -EPERM; 3085468e2f64SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_QUERY)) 3086468e2f64SAlexei Starovoitov return -EINVAL; 3087468e2f64SAlexei Starovoitov if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE) 3088468e2f64SAlexei Starovoitov return -EINVAL; 3089468e2f64SAlexei Starovoitov 3090468e2f64SAlexei Starovoitov switch (attr->query.attach_type) { 3091468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_INGRESS: 3092468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_EGRESS: 3093468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_SOCK_CREATE: 3094f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 30954fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 30964fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 3097aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 3098aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 3099d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 3100d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 31011b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 31021b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 31031b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 31041b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 31051cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 31061cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 3107983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG: 3108983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG: 3109468e2f64SAlexei Starovoitov case BPF_CGROUP_SOCK_OPS: 3110ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 31117b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 31120d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT: 31130d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT: 3114e28784e3SAndrii Nakryiko return cgroup_bpf_prog_query(attr, uattr); 3115f4364dcfSSean Young case BPF_LIRC_MODE2: 3116f4364dcfSSean Young return lirc_prog_query(attr, uattr); 3117118c8e9aSStanislav Fomichev case BPF_FLOW_DISSECTOR: 3118e9ddbb77SJakub Sitnicki case BPF_SK_LOOKUP: 3119a3fd7ceeSJakub Sitnicki return netns_bpf_prog_query(attr, uattr); 3120468e2f64SAlexei Starovoitov default: 3121468e2f64SAlexei Starovoitov return -EINVAL; 3122468e2f64SAlexei Starovoitov } 3123468e2f64SAlexei Starovoitov } 3124f4324551SDaniel Mack 31251b4d60ecSSong Liu #define BPF_PROG_TEST_RUN_LAST_FIELD test.cpu 31261cf1cae9SAlexei Starovoitov 31271cf1cae9SAlexei Starovoitov static int bpf_prog_test_run(const union bpf_attr *attr, 31281cf1cae9SAlexei Starovoitov union bpf_attr __user *uattr) 31291cf1cae9SAlexei Starovoitov { 31301cf1cae9SAlexei Starovoitov struct bpf_prog *prog; 31311cf1cae9SAlexei Starovoitov int ret = -ENOTSUPP; 31321cf1cae9SAlexei Starovoitov 31331cf1cae9SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 31341cf1cae9SAlexei Starovoitov return -EINVAL; 31351cf1cae9SAlexei Starovoitov 3136b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_in && !attr->test.ctx_in) || 3137b0b9395dSStanislav Fomichev (!attr->test.ctx_size_in && attr->test.ctx_in)) 3138b0b9395dSStanislav Fomichev return -EINVAL; 3139b0b9395dSStanislav Fomichev 3140b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_out && !attr->test.ctx_out) || 3141b0b9395dSStanislav Fomichev (!attr->test.ctx_size_out && attr->test.ctx_out)) 3142b0b9395dSStanislav Fomichev return -EINVAL; 3143b0b9395dSStanislav Fomichev 31441cf1cae9SAlexei Starovoitov prog = bpf_prog_get(attr->test.prog_fd); 31451cf1cae9SAlexei Starovoitov if (IS_ERR(prog)) 31461cf1cae9SAlexei Starovoitov return PTR_ERR(prog); 31471cf1cae9SAlexei Starovoitov 31481cf1cae9SAlexei Starovoitov if (prog->aux->ops->test_run) 31491cf1cae9SAlexei Starovoitov ret = prog->aux->ops->test_run(prog, attr, uattr); 31501cf1cae9SAlexei Starovoitov 31511cf1cae9SAlexei Starovoitov bpf_prog_put(prog); 31521cf1cae9SAlexei Starovoitov return ret; 31531cf1cae9SAlexei Starovoitov } 31541cf1cae9SAlexei Starovoitov 315534ad5580SMartin KaFai Lau #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 315634ad5580SMartin KaFai Lau 315734ad5580SMartin KaFai Lau static int bpf_obj_get_next_id(const union bpf_attr *attr, 315834ad5580SMartin KaFai Lau union bpf_attr __user *uattr, 315934ad5580SMartin KaFai Lau struct idr *idr, 316034ad5580SMartin KaFai Lau spinlock_t *lock) 316134ad5580SMartin KaFai Lau { 316234ad5580SMartin KaFai Lau u32 next_id = attr->start_id; 316334ad5580SMartin KaFai Lau int err = 0; 316434ad5580SMartin KaFai Lau 316534ad5580SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 316634ad5580SMartin KaFai Lau return -EINVAL; 316734ad5580SMartin KaFai Lau 316834ad5580SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 316934ad5580SMartin KaFai Lau return -EPERM; 317034ad5580SMartin KaFai Lau 317134ad5580SMartin KaFai Lau next_id++; 317234ad5580SMartin KaFai Lau spin_lock_bh(lock); 317334ad5580SMartin KaFai Lau if (!idr_get_next(idr, &next_id)) 317434ad5580SMartin KaFai Lau err = -ENOENT; 317534ad5580SMartin KaFai Lau spin_unlock_bh(lock); 317634ad5580SMartin KaFai Lau 317734ad5580SMartin KaFai Lau if (!err) 317834ad5580SMartin KaFai Lau err = put_user(next_id, &uattr->next_id); 317934ad5580SMartin KaFai Lau 318034ad5580SMartin KaFai Lau return err; 318134ad5580SMartin KaFai Lau } 318234ad5580SMartin KaFai Lau 31836086d29dSYonghong Song struct bpf_map *bpf_map_get_curr_or_next(u32 *id) 31846086d29dSYonghong Song { 31856086d29dSYonghong Song struct bpf_map *map; 31866086d29dSYonghong Song 31876086d29dSYonghong Song spin_lock_bh(&map_idr_lock); 31886086d29dSYonghong Song again: 31896086d29dSYonghong Song map = idr_get_next(&map_idr, id); 31906086d29dSYonghong Song if (map) { 31916086d29dSYonghong Song map = __bpf_map_inc_not_zero(map, false); 31926086d29dSYonghong Song if (IS_ERR(map)) { 31936086d29dSYonghong Song (*id)++; 31946086d29dSYonghong Song goto again; 31956086d29dSYonghong Song } 31966086d29dSYonghong Song } 31976086d29dSYonghong Song spin_unlock_bh(&map_idr_lock); 31986086d29dSYonghong Song 31996086d29dSYonghong Song return map; 32006086d29dSYonghong Song } 32016086d29dSYonghong Song 3202a228a64fSAlexei Starovoitov struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id) 3203a228a64fSAlexei Starovoitov { 3204a228a64fSAlexei Starovoitov struct bpf_prog *prog; 3205a228a64fSAlexei Starovoitov 3206a228a64fSAlexei Starovoitov spin_lock_bh(&prog_idr_lock); 3207a228a64fSAlexei Starovoitov again: 3208a228a64fSAlexei Starovoitov prog = idr_get_next(&prog_idr, id); 3209a228a64fSAlexei Starovoitov if (prog) { 3210a228a64fSAlexei Starovoitov prog = bpf_prog_inc_not_zero(prog); 3211a228a64fSAlexei Starovoitov if (IS_ERR(prog)) { 3212a228a64fSAlexei Starovoitov (*id)++; 3213a228a64fSAlexei Starovoitov goto again; 3214a228a64fSAlexei Starovoitov } 3215a228a64fSAlexei Starovoitov } 3216a228a64fSAlexei Starovoitov spin_unlock_bh(&prog_idr_lock); 3217a228a64fSAlexei Starovoitov 3218a228a64fSAlexei Starovoitov return prog; 3219a228a64fSAlexei Starovoitov } 3220a228a64fSAlexei Starovoitov 3221b16d9aa4SMartin KaFai Lau #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 3222b16d9aa4SMartin KaFai Lau 32237e6897f9SBjörn Töpel struct bpf_prog *bpf_prog_by_id(u32 id) 32247e6897f9SBjörn Töpel { 32257e6897f9SBjörn Töpel struct bpf_prog *prog; 32267e6897f9SBjörn Töpel 32277e6897f9SBjörn Töpel if (!id) 32287e6897f9SBjörn Töpel return ERR_PTR(-ENOENT); 32297e6897f9SBjörn Töpel 32307e6897f9SBjörn Töpel spin_lock_bh(&prog_idr_lock); 32317e6897f9SBjörn Töpel prog = idr_find(&prog_idr, id); 32327e6897f9SBjörn Töpel if (prog) 32337e6897f9SBjörn Töpel prog = bpf_prog_inc_not_zero(prog); 32347e6897f9SBjörn Töpel else 32357e6897f9SBjörn Töpel prog = ERR_PTR(-ENOENT); 32367e6897f9SBjörn Töpel spin_unlock_bh(&prog_idr_lock); 32377e6897f9SBjörn Töpel return prog; 32387e6897f9SBjörn Töpel } 32397e6897f9SBjörn Töpel 3240b16d9aa4SMartin KaFai Lau static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 3241b16d9aa4SMartin KaFai Lau { 3242b16d9aa4SMartin KaFai Lau struct bpf_prog *prog; 3243b16d9aa4SMartin KaFai Lau u32 id = attr->prog_id; 3244b16d9aa4SMartin KaFai Lau int fd; 3245b16d9aa4SMartin KaFai Lau 3246b16d9aa4SMartin KaFai Lau if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 3247b16d9aa4SMartin KaFai Lau return -EINVAL; 3248b16d9aa4SMartin KaFai Lau 3249b16d9aa4SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 3250b16d9aa4SMartin KaFai Lau return -EPERM; 3251b16d9aa4SMartin KaFai Lau 32527e6897f9SBjörn Töpel prog = bpf_prog_by_id(id); 3253b16d9aa4SMartin KaFai Lau if (IS_ERR(prog)) 3254b16d9aa4SMartin KaFai Lau return PTR_ERR(prog); 3255b16d9aa4SMartin KaFai Lau 3256b16d9aa4SMartin KaFai Lau fd = bpf_prog_new_fd(prog); 3257b16d9aa4SMartin KaFai Lau if (fd < 0) 3258b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 3259b16d9aa4SMartin KaFai Lau 3260b16d9aa4SMartin KaFai Lau return fd; 3261b16d9aa4SMartin KaFai Lau } 3262b16d9aa4SMartin KaFai Lau 32636e71b04aSChenbo Feng #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags 3264bd5f5f4eSMartin KaFai Lau 3265bd5f5f4eSMartin KaFai Lau static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 3266bd5f5f4eSMartin KaFai Lau { 3267bd5f5f4eSMartin KaFai Lau struct bpf_map *map; 3268bd5f5f4eSMartin KaFai Lau u32 id = attr->map_id; 32696e71b04aSChenbo Feng int f_flags; 3270bd5f5f4eSMartin KaFai Lau int fd; 3271bd5f5f4eSMartin KaFai Lau 32726e71b04aSChenbo Feng if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || 32736e71b04aSChenbo Feng attr->open_flags & ~BPF_OBJ_FLAG_MASK) 3274bd5f5f4eSMartin KaFai Lau return -EINVAL; 3275bd5f5f4eSMartin KaFai Lau 3276bd5f5f4eSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 3277bd5f5f4eSMartin KaFai Lau return -EPERM; 3278bd5f5f4eSMartin KaFai Lau 32796e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->open_flags); 32806e71b04aSChenbo Feng if (f_flags < 0) 32816e71b04aSChenbo Feng return f_flags; 32826e71b04aSChenbo Feng 3283bd5f5f4eSMartin KaFai Lau spin_lock_bh(&map_idr_lock); 3284bd5f5f4eSMartin KaFai Lau map = idr_find(&map_idr, id); 3285bd5f5f4eSMartin KaFai Lau if (map) 3286b0e4701cSStanislav Fomichev map = __bpf_map_inc_not_zero(map, true); 3287bd5f5f4eSMartin KaFai Lau else 3288bd5f5f4eSMartin KaFai Lau map = ERR_PTR(-ENOENT); 3289bd5f5f4eSMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 3290bd5f5f4eSMartin KaFai Lau 3291bd5f5f4eSMartin KaFai Lau if (IS_ERR(map)) 3292bd5f5f4eSMartin KaFai Lau return PTR_ERR(map); 3293bd5f5f4eSMartin KaFai Lau 32946e71b04aSChenbo Feng fd = bpf_map_new_fd(map, f_flags); 3295bd5f5f4eSMartin KaFai Lau if (fd < 0) 3296781e6282SPeng Sun bpf_map_put_with_uref(map); 3297bd5f5f4eSMartin KaFai Lau 3298bd5f5f4eSMartin KaFai Lau return fd; 3299bd5f5f4eSMartin KaFai Lau } 3300bd5f5f4eSMartin KaFai Lau 33017105e828SDaniel Borkmann static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, 3302d8eca5bbSDaniel Borkmann unsigned long addr, u32 *off, 3303d8eca5bbSDaniel Borkmann u32 *type) 33047105e828SDaniel Borkmann { 3305d8eca5bbSDaniel Borkmann const struct bpf_map *map; 33067105e828SDaniel Borkmann int i; 33077105e828SDaniel Borkmann 3308984fe94fSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 3309d8eca5bbSDaniel Borkmann for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { 3310d8eca5bbSDaniel Borkmann map = prog->aux->used_maps[i]; 3311d8eca5bbSDaniel Borkmann if (map == (void *)addr) { 3312d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_FD; 3313984fe94fSYiFei Zhu goto out; 3314d8eca5bbSDaniel Borkmann } 3315d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta) 3316d8eca5bbSDaniel Borkmann continue; 3317d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta(map, addr, off)) { 3318d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_VALUE; 3319984fe94fSYiFei Zhu goto out; 3320d8eca5bbSDaniel Borkmann } 3321d8eca5bbSDaniel Borkmann } 3322984fe94fSYiFei Zhu map = NULL; 3323d8eca5bbSDaniel Borkmann 3324984fe94fSYiFei Zhu out: 3325984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 3326984fe94fSYiFei Zhu return map; 33277105e828SDaniel Borkmann } 33287105e828SDaniel Borkmann 332963960260SKees Cook static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, 333063960260SKees Cook const struct cred *f_cred) 33317105e828SDaniel Borkmann { 33327105e828SDaniel Borkmann const struct bpf_map *map; 33337105e828SDaniel Borkmann struct bpf_insn *insns; 3334d8eca5bbSDaniel Borkmann u32 off, type; 33357105e828SDaniel Borkmann u64 imm; 333629fcb05bSAndrii Nakryiko u8 code; 33377105e828SDaniel Borkmann int i; 33387105e828SDaniel Borkmann 33397105e828SDaniel Borkmann insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), 33407105e828SDaniel Borkmann GFP_USER); 33417105e828SDaniel Borkmann if (!insns) 33427105e828SDaniel Borkmann return insns; 33437105e828SDaniel Borkmann 33447105e828SDaniel Borkmann for (i = 0; i < prog->len; i++) { 334529fcb05bSAndrii Nakryiko code = insns[i].code; 334629fcb05bSAndrii Nakryiko 334729fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_TAIL_CALL)) { 33487105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 33497105e828SDaniel Borkmann insns[i].imm = BPF_FUNC_tail_call; 33507105e828SDaniel Borkmann /* fall-through */ 33517105e828SDaniel Borkmann } 335229fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_CALL) || 335329fcb05bSAndrii Nakryiko code == (BPF_JMP | BPF_CALL_ARGS)) { 335429fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_CALL_ARGS)) 33557105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 335663960260SKees Cook if (!bpf_dump_raw_ok(f_cred)) 33577105e828SDaniel Borkmann insns[i].imm = 0; 33587105e828SDaniel Borkmann continue; 33597105e828SDaniel Borkmann } 336029fcb05bSAndrii Nakryiko if (BPF_CLASS(code) == BPF_LDX && BPF_MODE(code) == BPF_PROBE_MEM) { 336129fcb05bSAndrii Nakryiko insns[i].code = BPF_LDX | BPF_SIZE(code) | BPF_MEM; 336229fcb05bSAndrii Nakryiko continue; 336329fcb05bSAndrii Nakryiko } 33647105e828SDaniel Borkmann 336529fcb05bSAndrii Nakryiko if (code != (BPF_LD | BPF_IMM | BPF_DW)) 33667105e828SDaniel Borkmann continue; 33677105e828SDaniel Borkmann 33687105e828SDaniel Borkmann imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; 3369d8eca5bbSDaniel Borkmann map = bpf_map_from_imm(prog, imm, &off, &type); 33707105e828SDaniel Borkmann if (map) { 3371d8eca5bbSDaniel Borkmann insns[i].src_reg = type; 33727105e828SDaniel Borkmann insns[i].imm = map->id; 3373d8eca5bbSDaniel Borkmann insns[i + 1].imm = off; 33747105e828SDaniel Borkmann continue; 33757105e828SDaniel Borkmann } 33767105e828SDaniel Borkmann } 33777105e828SDaniel Borkmann 33787105e828SDaniel Borkmann return insns; 33797105e828SDaniel Borkmann } 33807105e828SDaniel Borkmann 3381c454a46bSMartin KaFai Lau static int set_info_rec_size(struct bpf_prog_info *info) 3382c454a46bSMartin KaFai Lau { 3383c454a46bSMartin KaFai Lau /* 3384c454a46bSMartin KaFai Lau * Ensure info.*_rec_size is the same as kernel expected size 3385c454a46bSMartin KaFai Lau * 3386c454a46bSMartin KaFai Lau * or 3387c454a46bSMartin KaFai Lau * 3388c454a46bSMartin KaFai Lau * Only allow zero *_rec_size if both _rec_size and _cnt are 3389c454a46bSMartin KaFai Lau * zero. In this case, the kernel will set the expected 3390c454a46bSMartin KaFai Lau * _rec_size back to the info. 3391c454a46bSMartin KaFai Lau */ 3392c454a46bSMartin KaFai Lau 339311d8b82dSYonghong Song if ((info->nr_func_info || info->func_info_rec_size) && 3394c454a46bSMartin KaFai Lau info->func_info_rec_size != sizeof(struct bpf_func_info)) 3395c454a46bSMartin KaFai Lau return -EINVAL; 3396c454a46bSMartin KaFai Lau 339711d8b82dSYonghong Song if ((info->nr_line_info || info->line_info_rec_size) && 3398c454a46bSMartin KaFai Lau info->line_info_rec_size != sizeof(struct bpf_line_info)) 3399c454a46bSMartin KaFai Lau return -EINVAL; 3400c454a46bSMartin KaFai Lau 340111d8b82dSYonghong Song if ((info->nr_jited_line_info || info->jited_line_info_rec_size) && 3402c454a46bSMartin KaFai Lau info->jited_line_info_rec_size != sizeof(__u64)) 3403c454a46bSMartin KaFai Lau return -EINVAL; 3404c454a46bSMartin KaFai Lau 3405c454a46bSMartin KaFai Lau info->func_info_rec_size = sizeof(struct bpf_func_info); 3406c454a46bSMartin KaFai Lau info->line_info_rec_size = sizeof(struct bpf_line_info); 3407c454a46bSMartin KaFai Lau info->jited_line_info_rec_size = sizeof(__u64); 3408c454a46bSMartin KaFai Lau 3409c454a46bSMartin KaFai Lau return 0; 3410c454a46bSMartin KaFai Lau } 3411c454a46bSMartin KaFai Lau 341263960260SKees Cook static int bpf_prog_get_info_by_fd(struct file *file, 341363960260SKees Cook struct bpf_prog *prog, 34141e270976SMartin KaFai Lau const union bpf_attr *attr, 34151e270976SMartin KaFai Lau union bpf_attr __user *uattr) 34161e270976SMartin KaFai Lau { 34171e270976SMartin KaFai Lau struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 34185c6f2588SGreg Kroah-Hartman struct bpf_prog_info info; 34191e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 34205f8f8b93SAlexei Starovoitov struct bpf_prog_stats stats; 34211e270976SMartin KaFai Lau char __user *uinsns; 34221e270976SMartin KaFai Lau u32 ulen; 34231e270976SMartin KaFai Lau int err; 34241e270976SMartin KaFai Lau 3425dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 34261e270976SMartin KaFai Lau if (err) 34271e270976SMartin KaFai Lau return err; 34281e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 34291e270976SMartin KaFai Lau 34305c6f2588SGreg Kroah-Hartman memset(&info, 0, sizeof(info)); 34311e270976SMartin KaFai Lau if (copy_from_user(&info, uinfo, info_len)) 343289b09689SDaniel Borkmann return -EFAULT; 34331e270976SMartin KaFai Lau 34341e270976SMartin KaFai Lau info.type = prog->type; 34351e270976SMartin KaFai Lau info.id = prog->aux->id; 3436cb4d2b3fSMartin KaFai Lau info.load_time = prog->aux->load_time; 3437cb4d2b3fSMartin KaFai Lau info.created_by_uid = from_kuid_munged(current_user_ns(), 3438cb4d2b3fSMartin KaFai Lau prog->aux->user->uid); 3439b85fab0eSJiri Olsa info.gpl_compatible = prog->gpl_compatible; 34401e270976SMartin KaFai Lau 34411e270976SMartin KaFai Lau memcpy(info.tag, prog->tag, sizeof(prog->tag)); 3442cb4d2b3fSMartin KaFai Lau memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); 3443cb4d2b3fSMartin KaFai Lau 3444984fe94fSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 3445cb4d2b3fSMartin KaFai Lau ulen = info.nr_map_ids; 3446cb4d2b3fSMartin KaFai Lau info.nr_map_ids = prog->aux->used_map_cnt; 3447cb4d2b3fSMartin KaFai Lau ulen = min_t(u32, info.nr_map_ids, ulen); 3448cb4d2b3fSMartin KaFai Lau if (ulen) { 3449721e08daSMartin KaFai Lau u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); 3450cb4d2b3fSMartin KaFai Lau u32 i; 3451cb4d2b3fSMartin KaFai Lau 3452cb4d2b3fSMartin KaFai Lau for (i = 0; i < ulen; i++) 3453cb4d2b3fSMartin KaFai Lau if (put_user(prog->aux->used_maps[i]->id, 3454984fe94fSYiFei Zhu &user_map_ids[i])) { 3455984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 3456cb4d2b3fSMartin KaFai Lau return -EFAULT; 3457cb4d2b3fSMartin KaFai Lau } 3458984fe94fSYiFei Zhu } 3459984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 34601e270976SMartin KaFai Lau 3461c454a46bSMartin KaFai Lau err = set_info_rec_size(&info); 3462c454a46bSMartin KaFai Lau if (err) 3463c454a46bSMartin KaFai Lau return err; 34647337224fSMartin KaFai Lau 34655f8f8b93SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 34665f8f8b93SAlexei Starovoitov info.run_time_ns = stats.nsecs; 34675f8f8b93SAlexei Starovoitov info.run_cnt = stats.cnt; 34689ed9e9baSAlexei Starovoitov info.recursion_misses = stats.misses; 34695f8f8b93SAlexei Starovoitov 34702c78ee89SAlexei Starovoitov if (!bpf_capable()) { 34711e270976SMartin KaFai Lau info.jited_prog_len = 0; 34721e270976SMartin KaFai Lau info.xlated_prog_len = 0; 3473dbecd738SSandipan Das info.nr_jited_ksyms = 0; 347428c2fae7SDaniel Borkmann info.nr_jited_func_lens = 0; 347511d8b82dSYonghong Song info.nr_func_info = 0; 347611d8b82dSYonghong Song info.nr_line_info = 0; 347711d8b82dSYonghong Song info.nr_jited_line_info = 0; 34781e270976SMartin KaFai Lau goto done; 34791e270976SMartin KaFai Lau } 34801e270976SMartin KaFai Lau 34811e270976SMartin KaFai Lau ulen = info.xlated_prog_len; 34829975a54bSDaniel Borkmann info.xlated_prog_len = bpf_prog_insn_size(prog); 34831e270976SMartin KaFai Lau if (info.xlated_prog_len && ulen) { 34847105e828SDaniel Borkmann struct bpf_insn *insns_sanitized; 34857105e828SDaniel Borkmann bool fault; 34867105e828SDaniel Borkmann 348763960260SKees Cook if (prog->blinded && !bpf_dump_raw_ok(file->f_cred)) { 34887105e828SDaniel Borkmann info.xlated_prog_insns = 0; 34897105e828SDaniel Borkmann goto done; 34907105e828SDaniel Borkmann } 349163960260SKees Cook insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred); 34927105e828SDaniel Borkmann if (!insns_sanitized) 34937105e828SDaniel Borkmann return -ENOMEM; 34941e270976SMartin KaFai Lau uinsns = u64_to_user_ptr(info.xlated_prog_insns); 34951e270976SMartin KaFai Lau ulen = min_t(u32, info.xlated_prog_len, ulen); 34967105e828SDaniel Borkmann fault = copy_to_user(uinsns, insns_sanitized, ulen); 34977105e828SDaniel Borkmann kfree(insns_sanitized); 34987105e828SDaniel Borkmann if (fault) 34991e270976SMartin KaFai Lau return -EFAULT; 35001e270976SMartin KaFai Lau } 35011e270976SMartin KaFai Lau 3502675fc275SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 3503675fc275SJakub Kicinski err = bpf_prog_offload_info_fill(&info, prog); 3504675fc275SJakub Kicinski if (err) 3505675fc275SJakub Kicinski return err; 3506fcfb126dSJiong Wang goto done; 3507fcfb126dSJiong Wang } 3508fcfb126dSJiong Wang 3509fcfb126dSJiong Wang /* NOTE: the following code is supposed to be skipped for offload. 3510fcfb126dSJiong Wang * bpf_prog_offload_info_fill() is the place to fill similar fields 3511fcfb126dSJiong Wang * for offload. 3512fcfb126dSJiong Wang */ 3513fcfb126dSJiong Wang ulen = info.jited_prog_len; 35144d56a76eSSandipan Das if (prog->aux->func_cnt) { 35154d56a76eSSandipan Das u32 i; 35164d56a76eSSandipan Das 35174d56a76eSSandipan Das info.jited_prog_len = 0; 35184d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) 35194d56a76eSSandipan Das info.jited_prog_len += prog->aux->func[i]->jited_len; 35204d56a76eSSandipan Das } else { 3521fcfb126dSJiong Wang info.jited_prog_len = prog->jited_len; 35224d56a76eSSandipan Das } 35234d56a76eSSandipan Das 3524fcfb126dSJiong Wang if (info.jited_prog_len && ulen) { 352563960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 3526fcfb126dSJiong Wang uinsns = u64_to_user_ptr(info.jited_prog_insns); 3527fcfb126dSJiong Wang ulen = min_t(u32, info.jited_prog_len, ulen); 35284d56a76eSSandipan Das 35294d56a76eSSandipan Das /* for multi-function programs, copy the JITed 35304d56a76eSSandipan Das * instructions for all the functions 35314d56a76eSSandipan Das */ 35324d56a76eSSandipan Das if (prog->aux->func_cnt) { 35334d56a76eSSandipan Das u32 len, free, i; 35344d56a76eSSandipan Das u8 *img; 35354d56a76eSSandipan Das 35364d56a76eSSandipan Das free = ulen; 35374d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) { 35384d56a76eSSandipan Das len = prog->aux->func[i]->jited_len; 35394d56a76eSSandipan Das len = min_t(u32, len, free); 35404d56a76eSSandipan Das img = (u8 *) prog->aux->func[i]->bpf_func; 35414d56a76eSSandipan Das if (copy_to_user(uinsns, img, len)) 35424d56a76eSSandipan Das return -EFAULT; 35434d56a76eSSandipan Das uinsns += len; 35444d56a76eSSandipan Das free -= len; 35454d56a76eSSandipan Das if (!free) 35464d56a76eSSandipan Das break; 35474d56a76eSSandipan Das } 35484d56a76eSSandipan Das } else { 3549fcfb126dSJiong Wang if (copy_to_user(uinsns, prog->bpf_func, ulen)) 3550fcfb126dSJiong Wang return -EFAULT; 35514d56a76eSSandipan Das } 3552fcfb126dSJiong Wang } else { 3553fcfb126dSJiong Wang info.jited_prog_insns = 0; 3554fcfb126dSJiong Wang } 3555675fc275SJakub Kicinski } 3556675fc275SJakub Kicinski 3557dbecd738SSandipan Das ulen = info.nr_jited_ksyms; 3558ff1889fcSSong Liu info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; 35597a5725ddSSong Liu if (ulen) { 356063960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 3561ff1889fcSSong Liu unsigned long ksym_addr; 3562dbecd738SSandipan Das u64 __user *user_ksyms; 3563dbecd738SSandipan Das u32 i; 3564dbecd738SSandipan Das 3565dbecd738SSandipan Das /* copy the address of the kernel symbol 3566dbecd738SSandipan Das * corresponding to each function 3567dbecd738SSandipan Das */ 3568dbecd738SSandipan Das ulen = min_t(u32, info.nr_jited_ksyms, ulen); 3569dbecd738SSandipan Das user_ksyms = u64_to_user_ptr(info.jited_ksyms); 3570ff1889fcSSong Liu if (prog->aux->func_cnt) { 3571dbecd738SSandipan Das for (i = 0; i < ulen; i++) { 3572ff1889fcSSong Liu ksym_addr = (unsigned long) 3573ff1889fcSSong Liu prog->aux->func[i]->bpf_func; 3574ff1889fcSSong Liu if (put_user((u64) ksym_addr, 3575ff1889fcSSong Liu &user_ksyms[i])) 3576ff1889fcSSong Liu return -EFAULT; 3577ff1889fcSSong Liu } 3578ff1889fcSSong Liu } else { 3579ff1889fcSSong Liu ksym_addr = (unsigned long) prog->bpf_func; 3580ff1889fcSSong Liu if (put_user((u64) ksym_addr, &user_ksyms[0])) 3581dbecd738SSandipan Das return -EFAULT; 3582dbecd738SSandipan Das } 3583dbecd738SSandipan Das } else { 3584dbecd738SSandipan Das info.jited_ksyms = 0; 3585dbecd738SSandipan Das } 3586dbecd738SSandipan Das } 3587dbecd738SSandipan Das 3588815581c1SSandipan Das ulen = info.nr_jited_func_lens; 3589ff1889fcSSong Liu info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; 35907a5725ddSSong Liu if (ulen) { 359163960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 3592815581c1SSandipan Das u32 __user *user_lens; 3593815581c1SSandipan Das u32 func_len, i; 3594815581c1SSandipan Das 3595815581c1SSandipan Das /* copy the JITed image lengths for each function */ 3596815581c1SSandipan Das ulen = min_t(u32, info.nr_jited_func_lens, ulen); 3597815581c1SSandipan Das user_lens = u64_to_user_ptr(info.jited_func_lens); 3598ff1889fcSSong Liu if (prog->aux->func_cnt) { 3599815581c1SSandipan Das for (i = 0; i < ulen; i++) { 3600ff1889fcSSong Liu func_len = 3601ff1889fcSSong Liu prog->aux->func[i]->jited_len; 3602815581c1SSandipan Das if (put_user(func_len, &user_lens[i])) 3603815581c1SSandipan Das return -EFAULT; 3604815581c1SSandipan Das } 3605815581c1SSandipan Das } else { 3606ff1889fcSSong Liu func_len = prog->jited_len; 3607ff1889fcSSong Liu if (put_user(func_len, &user_lens[0])) 3608ff1889fcSSong Liu return -EFAULT; 3609ff1889fcSSong Liu } 3610ff1889fcSSong Liu } else { 3611815581c1SSandipan Das info.jited_func_lens = 0; 3612815581c1SSandipan Das } 3613815581c1SSandipan Das } 3614815581c1SSandipan Das 36157337224fSMartin KaFai Lau if (prog->aux->btf) 361622dc4a0fSAndrii Nakryiko info.btf_id = btf_obj_id(prog->aux->btf); 3617838e9690SYonghong Song 361811d8b82dSYonghong Song ulen = info.nr_func_info; 361911d8b82dSYonghong Song info.nr_func_info = prog->aux->func_info_cnt; 362011d8b82dSYonghong Song if (info.nr_func_info && ulen) { 3621838e9690SYonghong Song char __user *user_finfo; 3622838e9690SYonghong Song 3623838e9690SYonghong Song user_finfo = u64_to_user_ptr(info.func_info); 362411d8b82dSYonghong Song ulen = min_t(u32, info.nr_func_info, ulen); 3625ba64e7d8SYonghong Song if (copy_to_user(user_finfo, prog->aux->func_info, 36267337224fSMartin KaFai Lau info.func_info_rec_size * ulen)) 3627838e9690SYonghong Song return -EFAULT; 3628838e9690SYonghong Song } 3629838e9690SYonghong Song 363011d8b82dSYonghong Song ulen = info.nr_line_info; 363111d8b82dSYonghong Song info.nr_line_info = prog->aux->nr_linfo; 363211d8b82dSYonghong Song if (info.nr_line_info && ulen) { 3633c454a46bSMartin KaFai Lau __u8 __user *user_linfo; 3634c454a46bSMartin KaFai Lau 3635c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.line_info); 363611d8b82dSYonghong Song ulen = min_t(u32, info.nr_line_info, ulen); 3637c454a46bSMartin KaFai Lau if (copy_to_user(user_linfo, prog->aux->linfo, 3638c454a46bSMartin KaFai Lau info.line_info_rec_size * ulen)) 3639c454a46bSMartin KaFai Lau return -EFAULT; 3640c454a46bSMartin KaFai Lau } 3641c454a46bSMartin KaFai Lau 364211d8b82dSYonghong Song ulen = info.nr_jited_line_info; 3643c454a46bSMartin KaFai Lau if (prog->aux->jited_linfo) 364411d8b82dSYonghong Song info.nr_jited_line_info = prog->aux->nr_linfo; 3645c454a46bSMartin KaFai Lau else 364611d8b82dSYonghong Song info.nr_jited_line_info = 0; 364711d8b82dSYonghong Song if (info.nr_jited_line_info && ulen) { 364863960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 3649c454a46bSMartin KaFai Lau __u64 __user *user_linfo; 3650c454a46bSMartin KaFai Lau u32 i; 3651c454a46bSMartin KaFai Lau 3652c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.jited_line_info); 365311d8b82dSYonghong Song ulen = min_t(u32, info.nr_jited_line_info, ulen); 3654c454a46bSMartin KaFai Lau for (i = 0; i < ulen; i++) { 3655c454a46bSMartin KaFai Lau if (put_user((__u64)(long)prog->aux->jited_linfo[i], 3656c454a46bSMartin KaFai Lau &user_linfo[i])) 3657c454a46bSMartin KaFai Lau return -EFAULT; 3658c454a46bSMartin KaFai Lau } 3659c454a46bSMartin KaFai Lau } else { 3660c454a46bSMartin KaFai Lau info.jited_line_info = 0; 3661c454a46bSMartin KaFai Lau } 3662c454a46bSMartin KaFai Lau } 3663c454a46bSMartin KaFai Lau 3664c872bdb3SSong Liu ulen = info.nr_prog_tags; 3665c872bdb3SSong Liu info.nr_prog_tags = prog->aux->func_cnt ? : 1; 3666c872bdb3SSong Liu if (ulen) { 3667c872bdb3SSong Liu __u8 __user (*user_prog_tags)[BPF_TAG_SIZE]; 3668c872bdb3SSong Liu u32 i; 3669c872bdb3SSong Liu 3670c872bdb3SSong Liu user_prog_tags = u64_to_user_ptr(info.prog_tags); 3671c872bdb3SSong Liu ulen = min_t(u32, info.nr_prog_tags, ulen); 3672c872bdb3SSong Liu if (prog->aux->func_cnt) { 3673c872bdb3SSong Liu for (i = 0; i < ulen; i++) { 3674c872bdb3SSong Liu if (copy_to_user(user_prog_tags[i], 3675c872bdb3SSong Liu prog->aux->func[i]->tag, 3676c872bdb3SSong Liu BPF_TAG_SIZE)) 3677c872bdb3SSong Liu return -EFAULT; 3678c872bdb3SSong Liu } 3679c872bdb3SSong Liu } else { 3680c872bdb3SSong Liu if (copy_to_user(user_prog_tags[0], 3681c872bdb3SSong Liu prog->tag, BPF_TAG_SIZE)) 3682c872bdb3SSong Liu return -EFAULT; 3683c872bdb3SSong Liu } 3684c872bdb3SSong Liu } 3685c872bdb3SSong Liu 36861e270976SMartin KaFai Lau done: 36871e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 36881e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 36891e270976SMartin KaFai Lau return -EFAULT; 36901e270976SMartin KaFai Lau 36911e270976SMartin KaFai Lau return 0; 36921e270976SMartin KaFai Lau } 36931e270976SMartin KaFai Lau 369463960260SKees Cook static int bpf_map_get_info_by_fd(struct file *file, 369563960260SKees Cook struct bpf_map *map, 36961e270976SMartin KaFai Lau const union bpf_attr *attr, 36971e270976SMartin KaFai Lau union bpf_attr __user *uattr) 36981e270976SMartin KaFai Lau { 36991e270976SMartin KaFai Lau struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 37005c6f2588SGreg Kroah-Hartman struct bpf_map_info info; 37011e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 37021e270976SMartin KaFai Lau int err; 37031e270976SMartin KaFai Lau 3704dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 37051e270976SMartin KaFai Lau if (err) 37061e270976SMartin KaFai Lau return err; 37071e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 37081e270976SMartin KaFai Lau 37095c6f2588SGreg Kroah-Hartman memset(&info, 0, sizeof(info)); 37101e270976SMartin KaFai Lau info.type = map->map_type; 37111e270976SMartin KaFai Lau info.id = map->id; 37121e270976SMartin KaFai Lau info.key_size = map->key_size; 37131e270976SMartin KaFai Lau info.value_size = map->value_size; 37141e270976SMartin KaFai Lau info.max_entries = map->max_entries; 37151e270976SMartin KaFai Lau info.map_flags = map->map_flags; 3716ad5b177bSMartin KaFai Lau memcpy(info.name, map->name, sizeof(map->name)); 37171e270976SMartin KaFai Lau 371878958fcaSMartin KaFai Lau if (map->btf) { 371922dc4a0fSAndrii Nakryiko info.btf_id = btf_obj_id(map->btf); 37209b2cf328SMartin KaFai Lau info.btf_key_type_id = map->btf_key_type_id; 37219b2cf328SMartin KaFai Lau info.btf_value_type_id = map->btf_value_type_id; 372278958fcaSMartin KaFai Lau } 372385d33df3SMartin KaFai Lau info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; 372478958fcaSMartin KaFai Lau 372552775b33SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 372652775b33SJakub Kicinski err = bpf_map_offload_info_fill(&info, map); 372752775b33SJakub Kicinski if (err) 372852775b33SJakub Kicinski return err; 372952775b33SJakub Kicinski } 373052775b33SJakub Kicinski 37311e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 37321e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 37331e270976SMartin KaFai Lau return -EFAULT; 37341e270976SMartin KaFai Lau 37351e270976SMartin KaFai Lau return 0; 37361e270976SMartin KaFai Lau } 37371e270976SMartin KaFai Lau 373863960260SKees Cook static int bpf_btf_get_info_by_fd(struct file *file, 373963960260SKees Cook struct btf *btf, 374062dab84cSMartin KaFai Lau const union bpf_attr *attr, 374162dab84cSMartin KaFai Lau union bpf_attr __user *uattr) 374262dab84cSMartin KaFai Lau { 374362dab84cSMartin KaFai Lau struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info); 374462dab84cSMartin KaFai Lau u32 info_len = attr->info.info_len; 374562dab84cSMartin KaFai Lau int err; 374662dab84cSMartin KaFai Lau 3747dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(*uinfo), info_len); 374862dab84cSMartin KaFai Lau if (err) 374962dab84cSMartin KaFai Lau return err; 375062dab84cSMartin KaFai Lau 375162dab84cSMartin KaFai Lau return btf_get_info_by_fd(btf, attr, uattr); 375262dab84cSMartin KaFai Lau } 375362dab84cSMartin KaFai Lau 375463960260SKees Cook static int bpf_link_get_info_by_fd(struct file *file, 375563960260SKees Cook struct bpf_link *link, 3756f2e10bffSAndrii Nakryiko const union bpf_attr *attr, 3757f2e10bffSAndrii Nakryiko union bpf_attr __user *uattr) 3758f2e10bffSAndrii Nakryiko { 3759f2e10bffSAndrii Nakryiko struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info); 3760f2e10bffSAndrii Nakryiko struct bpf_link_info info; 3761f2e10bffSAndrii Nakryiko u32 info_len = attr->info.info_len; 3762f2e10bffSAndrii Nakryiko int err; 3763f2e10bffSAndrii Nakryiko 3764f2e10bffSAndrii Nakryiko err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 3765f2e10bffSAndrii Nakryiko if (err) 3766f2e10bffSAndrii Nakryiko return err; 3767f2e10bffSAndrii Nakryiko info_len = min_t(u32, sizeof(info), info_len); 3768f2e10bffSAndrii Nakryiko 3769f2e10bffSAndrii Nakryiko memset(&info, 0, sizeof(info)); 3770f2e10bffSAndrii Nakryiko if (copy_from_user(&info, uinfo, info_len)) 3771f2e10bffSAndrii Nakryiko return -EFAULT; 3772f2e10bffSAndrii Nakryiko 3773f2e10bffSAndrii Nakryiko info.type = link->type; 3774f2e10bffSAndrii Nakryiko info.id = link->id; 3775f2e10bffSAndrii Nakryiko info.prog_id = link->prog->aux->id; 3776f2e10bffSAndrii Nakryiko 3777f2e10bffSAndrii Nakryiko if (link->ops->fill_link_info) { 3778f2e10bffSAndrii Nakryiko err = link->ops->fill_link_info(link, &info); 3779f2e10bffSAndrii Nakryiko if (err) 3780f2e10bffSAndrii Nakryiko return err; 3781f2e10bffSAndrii Nakryiko } 3782f2e10bffSAndrii Nakryiko 3783f2e10bffSAndrii Nakryiko if (copy_to_user(uinfo, &info, info_len) || 3784f2e10bffSAndrii Nakryiko put_user(info_len, &uattr->info.info_len)) 3785f2e10bffSAndrii Nakryiko return -EFAULT; 3786f2e10bffSAndrii Nakryiko 3787f2e10bffSAndrii Nakryiko return 0; 3788f2e10bffSAndrii Nakryiko } 3789f2e10bffSAndrii Nakryiko 3790f2e10bffSAndrii Nakryiko 37911e270976SMartin KaFai Lau #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 37921e270976SMartin KaFai Lau 37931e270976SMartin KaFai Lau static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 37941e270976SMartin KaFai Lau union bpf_attr __user *uattr) 37951e270976SMartin KaFai Lau { 37961e270976SMartin KaFai Lau int ufd = attr->info.bpf_fd; 37971e270976SMartin KaFai Lau struct fd f; 37981e270976SMartin KaFai Lau int err; 37991e270976SMartin KaFai Lau 38001e270976SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 38011e270976SMartin KaFai Lau return -EINVAL; 38021e270976SMartin KaFai Lau 38031e270976SMartin KaFai Lau f = fdget(ufd); 38041e270976SMartin KaFai Lau if (!f.file) 38051e270976SMartin KaFai Lau return -EBADFD; 38061e270976SMartin KaFai Lau 38071e270976SMartin KaFai Lau if (f.file->f_op == &bpf_prog_fops) 380863960260SKees Cook err = bpf_prog_get_info_by_fd(f.file, f.file->private_data, attr, 38091e270976SMartin KaFai Lau uattr); 38101e270976SMartin KaFai Lau else if (f.file->f_op == &bpf_map_fops) 381163960260SKees Cook err = bpf_map_get_info_by_fd(f.file, f.file->private_data, attr, 38121e270976SMartin KaFai Lau uattr); 381360197cfbSMartin KaFai Lau else if (f.file->f_op == &btf_fops) 381463960260SKees Cook err = bpf_btf_get_info_by_fd(f.file, f.file->private_data, attr, uattr); 3815f2e10bffSAndrii Nakryiko else if (f.file->f_op == &bpf_link_fops) 381663960260SKees Cook err = bpf_link_get_info_by_fd(f.file, f.file->private_data, 3817f2e10bffSAndrii Nakryiko attr, uattr); 38181e270976SMartin KaFai Lau else 38191e270976SMartin KaFai Lau err = -EINVAL; 38201e270976SMartin KaFai Lau 38211e270976SMartin KaFai Lau fdput(f); 38221e270976SMartin KaFai Lau return err; 38231e270976SMartin KaFai Lau } 38241e270976SMartin KaFai Lau 3825f56a653cSMartin KaFai Lau #define BPF_BTF_LOAD_LAST_FIELD btf_log_level 3826f56a653cSMartin KaFai Lau 3827f56a653cSMartin KaFai Lau static int bpf_btf_load(const union bpf_attr *attr) 3828f56a653cSMartin KaFai Lau { 3829f56a653cSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_LOAD)) 3830f56a653cSMartin KaFai Lau return -EINVAL; 3831f56a653cSMartin KaFai Lau 38322c78ee89SAlexei Starovoitov if (!bpf_capable()) 3833f56a653cSMartin KaFai Lau return -EPERM; 3834f56a653cSMartin KaFai Lau 3835f56a653cSMartin KaFai Lau return btf_new_fd(attr); 3836f56a653cSMartin KaFai Lau } 3837f56a653cSMartin KaFai Lau 383878958fcaSMartin KaFai Lau #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id 383978958fcaSMartin KaFai Lau 384078958fcaSMartin KaFai Lau static int bpf_btf_get_fd_by_id(const union bpf_attr *attr) 384178958fcaSMartin KaFai Lau { 384278958fcaSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID)) 384378958fcaSMartin KaFai Lau return -EINVAL; 384478958fcaSMartin KaFai Lau 384578958fcaSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 384678958fcaSMartin KaFai Lau return -EPERM; 384778958fcaSMartin KaFai Lau 384878958fcaSMartin KaFai Lau return btf_get_fd_by_id(attr->btf_id); 384978958fcaSMartin KaFai Lau } 385078958fcaSMartin KaFai Lau 385141bdc4b4SYonghong Song static int bpf_task_fd_query_copy(const union bpf_attr *attr, 385241bdc4b4SYonghong Song union bpf_attr __user *uattr, 385341bdc4b4SYonghong Song u32 prog_id, u32 fd_type, 385441bdc4b4SYonghong Song const char *buf, u64 probe_offset, 385541bdc4b4SYonghong Song u64 probe_addr) 385641bdc4b4SYonghong Song { 385741bdc4b4SYonghong Song char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf); 385841bdc4b4SYonghong Song u32 len = buf ? strlen(buf) : 0, input_len; 385941bdc4b4SYonghong Song int err = 0; 386041bdc4b4SYonghong Song 386141bdc4b4SYonghong Song if (put_user(len, &uattr->task_fd_query.buf_len)) 386241bdc4b4SYonghong Song return -EFAULT; 386341bdc4b4SYonghong Song input_len = attr->task_fd_query.buf_len; 386441bdc4b4SYonghong Song if (input_len && ubuf) { 386541bdc4b4SYonghong Song if (!len) { 386641bdc4b4SYonghong Song /* nothing to copy, just make ubuf NULL terminated */ 386741bdc4b4SYonghong Song char zero = '\0'; 386841bdc4b4SYonghong Song 386941bdc4b4SYonghong Song if (put_user(zero, ubuf)) 387041bdc4b4SYonghong Song return -EFAULT; 387141bdc4b4SYonghong Song } else if (input_len >= len + 1) { 387241bdc4b4SYonghong Song /* ubuf can hold the string with NULL terminator */ 387341bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, len + 1)) 387441bdc4b4SYonghong Song return -EFAULT; 387541bdc4b4SYonghong Song } else { 387641bdc4b4SYonghong Song /* ubuf cannot hold the string with NULL terminator, 387741bdc4b4SYonghong Song * do a partial copy with NULL terminator. 387841bdc4b4SYonghong Song */ 387941bdc4b4SYonghong Song char zero = '\0'; 388041bdc4b4SYonghong Song 388141bdc4b4SYonghong Song err = -ENOSPC; 388241bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, input_len - 1)) 388341bdc4b4SYonghong Song return -EFAULT; 388441bdc4b4SYonghong Song if (put_user(zero, ubuf + input_len - 1)) 388541bdc4b4SYonghong Song return -EFAULT; 388641bdc4b4SYonghong Song } 388741bdc4b4SYonghong Song } 388841bdc4b4SYonghong Song 388941bdc4b4SYonghong Song if (put_user(prog_id, &uattr->task_fd_query.prog_id) || 389041bdc4b4SYonghong Song put_user(fd_type, &uattr->task_fd_query.fd_type) || 389141bdc4b4SYonghong Song put_user(probe_offset, &uattr->task_fd_query.probe_offset) || 389241bdc4b4SYonghong Song put_user(probe_addr, &uattr->task_fd_query.probe_addr)) 389341bdc4b4SYonghong Song return -EFAULT; 389441bdc4b4SYonghong Song 389541bdc4b4SYonghong Song return err; 389641bdc4b4SYonghong Song } 389741bdc4b4SYonghong Song 389841bdc4b4SYonghong Song #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr 389941bdc4b4SYonghong Song 390041bdc4b4SYonghong Song static int bpf_task_fd_query(const union bpf_attr *attr, 390141bdc4b4SYonghong Song union bpf_attr __user *uattr) 390241bdc4b4SYonghong Song { 390341bdc4b4SYonghong Song pid_t pid = attr->task_fd_query.pid; 390441bdc4b4SYonghong Song u32 fd = attr->task_fd_query.fd; 390541bdc4b4SYonghong Song const struct perf_event *event; 390641bdc4b4SYonghong Song struct task_struct *task; 390741bdc4b4SYonghong Song struct file *file; 390841bdc4b4SYonghong Song int err; 390941bdc4b4SYonghong Song 391041bdc4b4SYonghong Song if (CHECK_ATTR(BPF_TASK_FD_QUERY)) 391141bdc4b4SYonghong Song return -EINVAL; 391241bdc4b4SYonghong Song 391341bdc4b4SYonghong Song if (!capable(CAP_SYS_ADMIN)) 391441bdc4b4SYonghong Song return -EPERM; 391541bdc4b4SYonghong Song 391641bdc4b4SYonghong Song if (attr->task_fd_query.flags != 0) 391741bdc4b4SYonghong Song return -EINVAL; 391841bdc4b4SYonghong Song 391941bdc4b4SYonghong Song task = get_pid_task(find_vpid(pid), PIDTYPE_PID); 392041bdc4b4SYonghong Song if (!task) 392141bdc4b4SYonghong Song return -ENOENT; 392241bdc4b4SYonghong Song 392341bdc4b4SYonghong Song err = 0; 3924b48845afSEric W. Biederman file = fget_task(task, fd); 3925b48845afSEric W. Biederman put_task_struct(task); 392641bdc4b4SYonghong Song if (!file) 3927b48845afSEric W. Biederman return -EBADF; 392841bdc4b4SYonghong Song 392970ed506cSAndrii Nakryiko if (file->f_op == &bpf_link_fops) { 393070ed506cSAndrii Nakryiko struct bpf_link *link = file->private_data; 393170ed506cSAndrii Nakryiko 3932a3b80e10SAndrii Nakryiko if (link->ops == &bpf_raw_tp_link_lops) { 393370ed506cSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 393470ed506cSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 393541bdc4b4SYonghong Song struct bpf_raw_event_map *btp = raw_tp->btp; 393641bdc4b4SYonghong Song 393741bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, 393870ed506cSAndrii Nakryiko raw_tp->link.prog->aux->id, 393941bdc4b4SYonghong Song BPF_FD_TYPE_RAW_TRACEPOINT, 394041bdc4b4SYonghong Song btp->tp->name, 0, 0); 394141bdc4b4SYonghong Song goto put_file; 394241bdc4b4SYonghong Song } 394370ed506cSAndrii Nakryiko goto out_not_supp; 394470ed506cSAndrii Nakryiko } 394541bdc4b4SYonghong Song 394641bdc4b4SYonghong Song event = perf_get_event(file); 394741bdc4b4SYonghong Song if (!IS_ERR(event)) { 394841bdc4b4SYonghong Song u64 probe_offset, probe_addr; 394941bdc4b4SYonghong Song u32 prog_id, fd_type; 395041bdc4b4SYonghong Song const char *buf; 395141bdc4b4SYonghong Song 395241bdc4b4SYonghong Song err = bpf_get_perf_event_info(event, &prog_id, &fd_type, 395341bdc4b4SYonghong Song &buf, &probe_offset, 395441bdc4b4SYonghong Song &probe_addr); 395541bdc4b4SYonghong Song if (!err) 395641bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, prog_id, 395741bdc4b4SYonghong Song fd_type, buf, 395841bdc4b4SYonghong Song probe_offset, 395941bdc4b4SYonghong Song probe_addr); 396041bdc4b4SYonghong Song goto put_file; 396141bdc4b4SYonghong Song } 396241bdc4b4SYonghong Song 396370ed506cSAndrii Nakryiko out_not_supp: 396441bdc4b4SYonghong Song err = -ENOTSUPP; 396541bdc4b4SYonghong Song put_file: 396641bdc4b4SYonghong Song fput(file); 396741bdc4b4SYonghong Song return err; 396841bdc4b4SYonghong Song } 396941bdc4b4SYonghong Song 3970cb4d03abSBrian Vazquez #define BPF_MAP_BATCH_LAST_FIELD batch.flags 3971cb4d03abSBrian Vazquez 3972cb4d03abSBrian Vazquez #define BPF_DO_BATCH(fn) \ 3973cb4d03abSBrian Vazquez do { \ 3974cb4d03abSBrian Vazquez if (!fn) { \ 3975cb4d03abSBrian Vazquez err = -ENOTSUPP; \ 3976cb4d03abSBrian Vazquez goto err_put; \ 3977cb4d03abSBrian Vazquez } \ 3978cb4d03abSBrian Vazquez err = fn(map, attr, uattr); \ 3979cb4d03abSBrian Vazquez } while (0) 3980cb4d03abSBrian Vazquez 3981cb4d03abSBrian Vazquez static int bpf_map_do_batch(const union bpf_attr *attr, 3982cb4d03abSBrian Vazquez union bpf_attr __user *uattr, 3983cb4d03abSBrian Vazquez int cmd) 3984cb4d03abSBrian Vazquez { 3985cb4d03abSBrian Vazquez struct bpf_map *map; 3986cb4d03abSBrian Vazquez int err, ufd; 3987cb4d03abSBrian Vazquez struct fd f; 3988cb4d03abSBrian Vazquez 3989cb4d03abSBrian Vazquez if (CHECK_ATTR(BPF_MAP_BATCH)) 3990cb4d03abSBrian Vazquez return -EINVAL; 3991cb4d03abSBrian Vazquez 3992cb4d03abSBrian Vazquez ufd = attr->batch.map_fd; 3993cb4d03abSBrian Vazquez f = fdget(ufd); 3994cb4d03abSBrian Vazquez map = __bpf_map_get(f); 3995cb4d03abSBrian Vazquez if (IS_ERR(map)) 3996cb4d03abSBrian Vazquez return PTR_ERR(map); 3997cb4d03abSBrian Vazquez 399805799638SYonghong Song if ((cmd == BPF_MAP_LOOKUP_BATCH || 399905799638SYonghong Song cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) && 4000cb4d03abSBrian Vazquez !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 4001cb4d03abSBrian Vazquez err = -EPERM; 4002cb4d03abSBrian Vazquez goto err_put; 4003cb4d03abSBrian Vazquez } 4004cb4d03abSBrian Vazquez 4005cb4d03abSBrian Vazquez if (cmd != BPF_MAP_LOOKUP_BATCH && 4006cb4d03abSBrian Vazquez !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 4007cb4d03abSBrian Vazquez err = -EPERM; 4008cb4d03abSBrian Vazquez goto err_put; 4009cb4d03abSBrian Vazquez } 4010cb4d03abSBrian Vazquez 4011cb4d03abSBrian Vazquez if (cmd == BPF_MAP_LOOKUP_BATCH) 4012cb4d03abSBrian Vazquez BPF_DO_BATCH(map->ops->map_lookup_batch); 401305799638SYonghong Song else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) 401405799638SYonghong Song BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch); 4015aa2e93b8SBrian Vazquez else if (cmd == BPF_MAP_UPDATE_BATCH) 4016aa2e93b8SBrian Vazquez BPF_DO_BATCH(map->ops->map_update_batch); 4017aa2e93b8SBrian Vazquez else 4018aa2e93b8SBrian Vazquez BPF_DO_BATCH(map->ops->map_delete_batch); 4019cb4d03abSBrian Vazquez 4020cb4d03abSBrian Vazquez err_put: 4021cb4d03abSBrian Vazquez fdput(f); 4022cb4d03abSBrian Vazquez return err; 4023cb4d03abSBrian Vazquez } 4024cb4d03abSBrian Vazquez 4025de4e05caSYonghong Song static int tracing_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 4026de4e05caSYonghong Song { 40274a1e7c0cSToke Høiland-Jørgensen if (attr->link_create.attach_type != prog->expected_attach_type) 40284a1e7c0cSToke Høiland-Jørgensen return -EINVAL; 4029de4e05caSYonghong Song 40304a1e7c0cSToke Høiland-Jørgensen if (prog->expected_attach_type == BPF_TRACE_ITER) 40314a1e7c0cSToke Høiland-Jørgensen return bpf_iter_link_attach(attr, prog); 40324a1e7c0cSToke Høiland-Jørgensen else if (prog->type == BPF_PROG_TYPE_EXT) 40334a1e7c0cSToke Høiland-Jørgensen return bpf_tracing_prog_attach(prog, 40344a1e7c0cSToke Høiland-Jørgensen attr->link_create.target_fd, 40354a1e7c0cSToke Høiland-Jørgensen attr->link_create.target_btf_id); 4036de4e05caSYonghong Song return -EINVAL; 4037de4e05caSYonghong Song } 4038de4e05caSYonghong Song 40395e7b3020SYonghong Song #define BPF_LINK_CREATE_LAST_FIELD link_create.iter_info_len 4040af6eea57SAndrii Nakryiko static int link_create(union bpf_attr *attr) 4041af6eea57SAndrii Nakryiko { 4042af6eea57SAndrii Nakryiko enum bpf_prog_type ptype; 4043af6eea57SAndrii Nakryiko struct bpf_prog *prog; 4044af6eea57SAndrii Nakryiko int ret; 4045af6eea57SAndrii Nakryiko 4046af6eea57SAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_CREATE)) 4047af6eea57SAndrii Nakryiko return -EINVAL; 4048af6eea57SAndrii Nakryiko 40494a1e7c0cSToke Høiland-Jørgensen prog = bpf_prog_get(attr->link_create.prog_fd); 4050af6eea57SAndrii Nakryiko if (IS_ERR(prog)) 4051af6eea57SAndrii Nakryiko return PTR_ERR(prog); 4052af6eea57SAndrii Nakryiko 4053af6eea57SAndrii Nakryiko ret = bpf_prog_attach_check_attach_type(prog, 4054af6eea57SAndrii Nakryiko attr->link_create.attach_type); 4055af6eea57SAndrii Nakryiko if (ret) 40564a1e7c0cSToke Høiland-Jørgensen goto out; 40574a1e7c0cSToke Høiland-Jørgensen 40584a1e7c0cSToke Høiland-Jørgensen if (prog->type == BPF_PROG_TYPE_EXT) { 40594a1e7c0cSToke Høiland-Jørgensen ret = tracing_bpf_link_attach(attr, prog); 40604a1e7c0cSToke Høiland-Jørgensen goto out; 40614a1e7c0cSToke Høiland-Jørgensen } 40624a1e7c0cSToke Høiland-Jørgensen 40634a1e7c0cSToke Høiland-Jørgensen ptype = attach_type_to_prog_type(attr->link_create.attach_type); 40644a1e7c0cSToke Høiland-Jørgensen if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type) { 40654a1e7c0cSToke Høiland-Jørgensen ret = -EINVAL; 40664a1e7c0cSToke Høiland-Jørgensen goto out; 40674a1e7c0cSToke Høiland-Jørgensen } 4068af6eea57SAndrii Nakryiko 4069af6eea57SAndrii Nakryiko switch (ptype) { 4070af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 4071af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 4072af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 4073af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 4074af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 4075af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 4076af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 4077af6eea57SAndrii Nakryiko ret = cgroup_bpf_link_attach(attr, prog); 4078af6eea57SAndrii Nakryiko break; 4079de4e05caSYonghong Song case BPF_PROG_TYPE_TRACING: 4080de4e05caSYonghong Song ret = tracing_bpf_link_attach(attr, prog); 4081de4e05caSYonghong Song break; 40827f045a49SJakub Sitnicki case BPF_PROG_TYPE_FLOW_DISSECTOR: 4083e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP: 40847f045a49SJakub Sitnicki ret = netns_bpf_link_create(attr, prog); 40857f045a49SJakub Sitnicki break; 4086310ad797SAndrii Nakryiko #ifdef CONFIG_NET 4087aa8d3a71SAndrii Nakryiko case BPF_PROG_TYPE_XDP: 4088aa8d3a71SAndrii Nakryiko ret = bpf_xdp_link_attach(attr, prog); 4089aa8d3a71SAndrii Nakryiko break; 4090310ad797SAndrii Nakryiko #endif 4091af6eea57SAndrii Nakryiko default: 4092af6eea57SAndrii Nakryiko ret = -EINVAL; 4093af6eea57SAndrii Nakryiko } 4094af6eea57SAndrii Nakryiko 40954a1e7c0cSToke Høiland-Jørgensen out: 4096af6eea57SAndrii Nakryiko if (ret < 0) 4097af6eea57SAndrii Nakryiko bpf_prog_put(prog); 4098af6eea57SAndrii Nakryiko return ret; 4099af6eea57SAndrii Nakryiko } 4100af6eea57SAndrii Nakryiko 41010c991ebcSAndrii Nakryiko #define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd 41020c991ebcSAndrii Nakryiko 41030c991ebcSAndrii Nakryiko static int link_update(union bpf_attr *attr) 41040c991ebcSAndrii Nakryiko { 41050c991ebcSAndrii Nakryiko struct bpf_prog *old_prog = NULL, *new_prog; 41060c991ebcSAndrii Nakryiko struct bpf_link *link; 41070c991ebcSAndrii Nakryiko u32 flags; 41080c991ebcSAndrii Nakryiko int ret; 41090c991ebcSAndrii Nakryiko 41100c991ebcSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_UPDATE)) 41110c991ebcSAndrii Nakryiko return -EINVAL; 41120c991ebcSAndrii Nakryiko 41130c991ebcSAndrii Nakryiko flags = attr->link_update.flags; 41140c991ebcSAndrii Nakryiko if (flags & ~BPF_F_REPLACE) 41150c991ebcSAndrii Nakryiko return -EINVAL; 41160c991ebcSAndrii Nakryiko 41170c991ebcSAndrii Nakryiko link = bpf_link_get_from_fd(attr->link_update.link_fd); 41180c991ebcSAndrii Nakryiko if (IS_ERR(link)) 41190c991ebcSAndrii Nakryiko return PTR_ERR(link); 41200c991ebcSAndrii Nakryiko 41210c991ebcSAndrii Nakryiko new_prog = bpf_prog_get(attr->link_update.new_prog_fd); 41224adb7a4aSAndrii Nakryiko if (IS_ERR(new_prog)) { 41234adb7a4aSAndrii Nakryiko ret = PTR_ERR(new_prog); 41244adb7a4aSAndrii Nakryiko goto out_put_link; 41254adb7a4aSAndrii Nakryiko } 41260c991ebcSAndrii Nakryiko 41270c991ebcSAndrii Nakryiko if (flags & BPF_F_REPLACE) { 41280c991ebcSAndrii Nakryiko old_prog = bpf_prog_get(attr->link_update.old_prog_fd); 41290c991ebcSAndrii Nakryiko if (IS_ERR(old_prog)) { 41300c991ebcSAndrii Nakryiko ret = PTR_ERR(old_prog); 41310c991ebcSAndrii Nakryiko old_prog = NULL; 41320c991ebcSAndrii Nakryiko goto out_put_progs; 41330c991ebcSAndrii Nakryiko } 41344adb7a4aSAndrii Nakryiko } else if (attr->link_update.old_prog_fd) { 41354adb7a4aSAndrii Nakryiko ret = -EINVAL; 41364adb7a4aSAndrii Nakryiko goto out_put_progs; 41370c991ebcSAndrii Nakryiko } 41380c991ebcSAndrii Nakryiko 4139f9d04127SAndrii Nakryiko if (link->ops->update_prog) 4140f9d04127SAndrii Nakryiko ret = link->ops->update_prog(link, new_prog, old_prog); 4141f9d04127SAndrii Nakryiko else 41420c991ebcSAndrii Nakryiko ret = -EINVAL; 41430c991ebcSAndrii Nakryiko 41440c991ebcSAndrii Nakryiko out_put_progs: 41450c991ebcSAndrii Nakryiko if (old_prog) 41460c991ebcSAndrii Nakryiko bpf_prog_put(old_prog); 41470c991ebcSAndrii Nakryiko if (ret) 41480c991ebcSAndrii Nakryiko bpf_prog_put(new_prog); 41494adb7a4aSAndrii Nakryiko out_put_link: 41504adb7a4aSAndrii Nakryiko bpf_link_put(link); 41510c991ebcSAndrii Nakryiko return ret; 41520c991ebcSAndrii Nakryiko } 41530c991ebcSAndrii Nakryiko 415473b11c2aSAndrii Nakryiko #define BPF_LINK_DETACH_LAST_FIELD link_detach.link_fd 415573b11c2aSAndrii Nakryiko 415673b11c2aSAndrii Nakryiko static int link_detach(union bpf_attr *attr) 415773b11c2aSAndrii Nakryiko { 415873b11c2aSAndrii Nakryiko struct bpf_link *link; 415973b11c2aSAndrii Nakryiko int ret; 416073b11c2aSAndrii Nakryiko 416173b11c2aSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_DETACH)) 416273b11c2aSAndrii Nakryiko return -EINVAL; 416373b11c2aSAndrii Nakryiko 416473b11c2aSAndrii Nakryiko link = bpf_link_get_from_fd(attr->link_detach.link_fd); 416573b11c2aSAndrii Nakryiko if (IS_ERR(link)) 416673b11c2aSAndrii Nakryiko return PTR_ERR(link); 416773b11c2aSAndrii Nakryiko 416873b11c2aSAndrii Nakryiko if (link->ops->detach) 416973b11c2aSAndrii Nakryiko ret = link->ops->detach(link); 417073b11c2aSAndrii Nakryiko else 417173b11c2aSAndrii Nakryiko ret = -EOPNOTSUPP; 417273b11c2aSAndrii Nakryiko 417373b11c2aSAndrii Nakryiko bpf_link_put(link); 417473b11c2aSAndrii Nakryiko return ret; 417573b11c2aSAndrii Nakryiko } 417673b11c2aSAndrii Nakryiko 4177005142b8SAlexei Starovoitov static struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link) 41782d602c8cSAndrii Nakryiko { 4179005142b8SAlexei Starovoitov return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT); 4180005142b8SAlexei Starovoitov } 4181005142b8SAlexei Starovoitov 4182005142b8SAlexei Starovoitov struct bpf_link *bpf_link_by_id(u32 id) 4183005142b8SAlexei Starovoitov { 4184005142b8SAlexei Starovoitov struct bpf_link *link; 4185005142b8SAlexei Starovoitov 4186005142b8SAlexei Starovoitov if (!id) 4187005142b8SAlexei Starovoitov return ERR_PTR(-ENOENT); 4188005142b8SAlexei Starovoitov 4189005142b8SAlexei Starovoitov spin_lock_bh(&link_idr_lock); 4190005142b8SAlexei Starovoitov /* before link is "settled", ID is 0, pretend it doesn't exist yet */ 4191005142b8SAlexei Starovoitov link = idr_find(&link_idr, id); 4192005142b8SAlexei Starovoitov if (link) { 4193005142b8SAlexei Starovoitov if (link->id) 4194005142b8SAlexei Starovoitov link = bpf_link_inc_not_zero(link); 4195005142b8SAlexei Starovoitov else 4196005142b8SAlexei Starovoitov link = ERR_PTR(-EAGAIN); 4197005142b8SAlexei Starovoitov } else { 4198005142b8SAlexei Starovoitov link = ERR_PTR(-ENOENT); 4199005142b8SAlexei Starovoitov } 4200005142b8SAlexei Starovoitov spin_unlock_bh(&link_idr_lock); 4201005142b8SAlexei Starovoitov return link; 42022d602c8cSAndrii Nakryiko } 42032d602c8cSAndrii Nakryiko 42042d602c8cSAndrii Nakryiko #define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id 42052d602c8cSAndrii Nakryiko 42062d602c8cSAndrii Nakryiko static int bpf_link_get_fd_by_id(const union bpf_attr *attr) 42072d602c8cSAndrii Nakryiko { 42082d602c8cSAndrii Nakryiko struct bpf_link *link; 42092d602c8cSAndrii Nakryiko u32 id = attr->link_id; 4210005142b8SAlexei Starovoitov int fd; 42112d602c8cSAndrii Nakryiko 42122d602c8cSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID)) 42132d602c8cSAndrii Nakryiko return -EINVAL; 42142d602c8cSAndrii Nakryiko 42152d602c8cSAndrii Nakryiko if (!capable(CAP_SYS_ADMIN)) 42162d602c8cSAndrii Nakryiko return -EPERM; 42172d602c8cSAndrii Nakryiko 4218005142b8SAlexei Starovoitov link = bpf_link_by_id(id); 4219005142b8SAlexei Starovoitov if (IS_ERR(link)) 4220005142b8SAlexei Starovoitov return PTR_ERR(link); 42212d602c8cSAndrii Nakryiko 42222d602c8cSAndrii Nakryiko fd = bpf_link_new_fd(link); 42232d602c8cSAndrii Nakryiko if (fd < 0) 42242d602c8cSAndrii Nakryiko bpf_link_put(link); 42252d602c8cSAndrii Nakryiko 42262d602c8cSAndrii Nakryiko return fd; 42272d602c8cSAndrii Nakryiko } 42282d602c8cSAndrii Nakryiko 4229d46edd67SSong Liu DEFINE_MUTEX(bpf_stats_enabled_mutex); 4230d46edd67SSong Liu 4231d46edd67SSong Liu static int bpf_stats_release(struct inode *inode, struct file *file) 4232d46edd67SSong Liu { 4233d46edd67SSong Liu mutex_lock(&bpf_stats_enabled_mutex); 4234d46edd67SSong Liu static_key_slow_dec(&bpf_stats_enabled_key.key); 4235d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 4236d46edd67SSong Liu return 0; 4237d46edd67SSong Liu } 4238d46edd67SSong Liu 4239d46edd67SSong Liu static const struct file_operations bpf_stats_fops = { 4240d46edd67SSong Liu .release = bpf_stats_release, 4241d46edd67SSong Liu }; 4242d46edd67SSong Liu 4243d46edd67SSong Liu static int bpf_enable_runtime_stats(void) 4244d46edd67SSong Liu { 4245d46edd67SSong Liu int fd; 4246d46edd67SSong Liu 4247d46edd67SSong Liu mutex_lock(&bpf_stats_enabled_mutex); 4248d46edd67SSong Liu 4249d46edd67SSong Liu /* Set a very high limit to avoid overflow */ 4250d46edd67SSong Liu if (static_key_count(&bpf_stats_enabled_key.key) > INT_MAX / 2) { 4251d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 4252d46edd67SSong Liu return -EBUSY; 4253d46edd67SSong Liu } 4254d46edd67SSong Liu 4255d46edd67SSong Liu fd = anon_inode_getfd("bpf-stats", &bpf_stats_fops, NULL, O_CLOEXEC); 4256d46edd67SSong Liu if (fd >= 0) 4257d46edd67SSong Liu static_key_slow_inc(&bpf_stats_enabled_key.key); 4258d46edd67SSong Liu 4259d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 4260d46edd67SSong Liu return fd; 4261d46edd67SSong Liu } 4262d46edd67SSong Liu 4263d46edd67SSong Liu #define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type 4264d46edd67SSong Liu 4265d46edd67SSong Liu static int bpf_enable_stats(union bpf_attr *attr) 4266d46edd67SSong Liu { 4267d46edd67SSong Liu 4268d46edd67SSong Liu if (CHECK_ATTR(BPF_ENABLE_STATS)) 4269d46edd67SSong Liu return -EINVAL; 4270d46edd67SSong Liu 4271d46edd67SSong Liu if (!capable(CAP_SYS_ADMIN)) 4272d46edd67SSong Liu return -EPERM; 4273d46edd67SSong Liu 4274d46edd67SSong Liu switch (attr->enable_stats.type) { 4275d46edd67SSong Liu case BPF_STATS_RUN_TIME: 4276d46edd67SSong Liu return bpf_enable_runtime_stats(); 4277d46edd67SSong Liu default: 4278d46edd67SSong Liu break; 4279d46edd67SSong Liu } 4280d46edd67SSong Liu return -EINVAL; 4281d46edd67SSong Liu } 4282d46edd67SSong Liu 4283ac51d99bSYonghong Song #define BPF_ITER_CREATE_LAST_FIELD iter_create.flags 4284ac51d99bSYonghong Song 4285ac51d99bSYonghong Song static int bpf_iter_create(union bpf_attr *attr) 4286ac51d99bSYonghong Song { 4287ac51d99bSYonghong Song struct bpf_link *link; 4288ac51d99bSYonghong Song int err; 4289ac51d99bSYonghong Song 4290ac51d99bSYonghong Song if (CHECK_ATTR(BPF_ITER_CREATE)) 4291ac51d99bSYonghong Song return -EINVAL; 4292ac51d99bSYonghong Song 4293ac51d99bSYonghong Song if (attr->iter_create.flags) 4294ac51d99bSYonghong Song return -EINVAL; 4295ac51d99bSYonghong Song 4296ac51d99bSYonghong Song link = bpf_link_get_from_fd(attr->iter_create.link_fd); 4297ac51d99bSYonghong Song if (IS_ERR(link)) 4298ac51d99bSYonghong Song return PTR_ERR(link); 4299ac51d99bSYonghong Song 4300ac51d99bSYonghong Song err = bpf_iter_new_fd(link); 4301ac51d99bSYonghong Song bpf_link_put(link); 4302ac51d99bSYonghong Song 4303ac51d99bSYonghong Song return err; 4304ac51d99bSYonghong Song } 4305ac51d99bSYonghong Song 4306ef15314aSYiFei Zhu #define BPF_PROG_BIND_MAP_LAST_FIELD prog_bind_map.flags 4307ef15314aSYiFei Zhu 4308ef15314aSYiFei Zhu static int bpf_prog_bind_map(union bpf_attr *attr) 4309ef15314aSYiFei Zhu { 4310ef15314aSYiFei Zhu struct bpf_prog *prog; 4311ef15314aSYiFei Zhu struct bpf_map *map; 4312ef15314aSYiFei Zhu struct bpf_map **used_maps_old, **used_maps_new; 4313ef15314aSYiFei Zhu int i, ret = 0; 4314ef15314aSYiFei Zhu 4315ef15314aSYiFei Zhu if (CHECK_ATTR(BPF_PROG_BIND_MAP)) 4316ef15314aSYiFei Zhu return -EINVAL; 4317ef15314aSYiFei Zhu 4318ef15314aSYiFei Zhu if (attr->prog_bind_map.flags) 4319ef15314aSYiFei Zhu return -EINVAL; 4320ef15314aSYiFei Zhu 4321ef15314aSYiFei Zhu prog = bpf_prog_get(attr->prog_bind_map.prog_fd); 4322ef15314aSYiFei Zhu if (IS_ERR(prog)) 4323ef15314aSYiFei Zhu return PTR_ERR(prog); 4324ef15314aSYiFei Zhu 4325ef15314aSYiFei Zhu map = bpf_map_get(attr->prog_bind_map.map_fd); 4326ef15314aSYiFei Zhu if (IS_ERR(map)) { 4327ef15314aSYiFei Zhu ret = PTR_ERR(map); 4328ef15314aSYiFei Zhu goto out_prog_put; 4329ef15314aSYiFei Zhu } 4330ef15314aSYiFei Zhu 4331ef15314aSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 4332ef15314aSYiFei Zhu 4333ef15314aSYiFei Zhu used_maps_old = prog->aux->used_maps; 4334ef15314aSYiFei Zhu 4335ef15314aSYiFei Zhu for (i = 0; i < prog->aux->used_map_cnt; i++) 43361028ae40SStanislav Fomichev if (used_maps_old[i] == map) { 43371028ae40SStanislav Fomichev bpf_map_put(map); 4338ef15314aSYiFei Zhu goto out_unlock; 43391028ae40SStanislav Fomichev } 4340ef15314aSYiFei Zhu 4341ef15314aSYiFei Zhu used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1, 4342ef15314aSYiFei Zhu sizeof(used_maps_new[0]), 4343ef15314aSYiFei Zhu GFP_KERNEL); 4344ef15314aSYiFei Zhu if (!used_maps_new) { 4345ef15314aSYiFei Zhu ret = -ENOMEM; 4346ef15314aSYiFei Zhu goto out_unlock; 4347ef15314aSYiFei Zhu } 4348ef15314aSYiFei Zhu 4349ef15314aSYiFei Zhu memcpy(used_maps_new, used_maps_old, 4350ef15314aSYiFei Zhu sizeof(used_maps_old[0]) * prog->aux->used_map_cnt); 4351ef15314aSYiFei Zhu used_maps_new[prog->aux->used_map_cnt] = map; 4352ef15314aSYiFei Zhu 4353ef15314aSYiFei Zhu prog->aux->used_map_cnt++; 4354ef15314aSYiFei Zhu prog->aux->used_maps = used_maps_new; 4355ef15314aSYiFei Zhu 4356ef15314aSYiFei Zhu kfree(used_maps_old); 4357ef15314aSYiFei Zhu 4358ef15314aSYiFei Zhu out_unlock: 4359ef15314aSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 4360ef15314aSYiFei Zhu 4361ef15314aSYiFei Zhu if (ret) 4362ef15314aSYiFei Zhu bpf_map_put(map); 4363ef15314aSYiFei Zhu out_prog_put: 4364ef15314aSYiFei Zhu bpf_prog_put(prog); 4365ef15314aSYiFei Zhu return ret; 4366ef15314aSYiFei Zhu } 4367ef15314aSYiFei Zhu 436899c55f7dSAlexei Starovoitov SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 436999c55f7dSAlexei Starovoitov { 43708096f229SGreg Kroah-Hartman union bpf_attr attr; 437199c55f7dSAlexei Starovoitov int err; 437299c55f7dSAlexei Starovoitov 43732c78ee89SAlexei Starovoitov if (sysctl_unprivileged_bpf_disabled && !bpf_capable()) 437499c55f7dSAlexei Starovoitov return -EPERM; 437599c55f7dSAlexei Starovoitov 4376dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size); 437799c55f7dSAlexei Starovoitov if (err) 437899c55f7dSAlexei Starovoitov return err; 43791e270976SMartin KaFai Lau size = min_t(u32, size, sizeof(attr)); 438099c55f7dSAlexei Starovoitov 438199c55f7dSAlexei Starovoitov /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 43828096f229SGreg Kroah-Hartman memset(&attr, 0, sizeof(attr)); 438399c55f7dSAlexei Starovoitov if (copy_from_user(&attr, uattr, size) != 0) 438499c55f7dSAlexei Starovoitov return -EFAULT; 438599c55f7dSAlexei Starovoitov 4386afdb09c7SChenbo Feng err = security_bpf(cmd, &attr, size); 4387afdb09c7SChenbo Feng if (err < 0) 4388afdb09c7SChenbo Feng return err; 4389afdb09c7SChenbo Feng 439099c55f7dSAlexei Starovoitov switch (cmd) { 439199c55f7dSAlexei Starovoitov case BPF_MAP_CREATE: 439299c55f7dSAlexei Starovoitov err = map_create(&attr); 439399c55f7dSAlexei Starovoitov break; 4394db20fd2bSAlexei Starovoitov case BPF_MAP_LOOKUP_ELEM: 4395db20fd2bSAlexei Starovoitov err = map_lookup_elem(&attr); 4396db20fd2bSAlexei Starovoitov break; 4397db20fd2bSAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 4398db20fd2bSAlexei Starovoitov err = map_update_elem(&attr); 4399db20fd2bSAlexei Starovoitov break; 4400db20fd2bSAlexei Starovoitov case BPF_MAP_DELETE_ELEM: 4401db20fd2bSAlexei Starovoitov err = map_delete_elem(&attr); 4402db20fd2bSAlexei Starovoitov break; 4403db20fd2bSAlexei Starovoitov case BPF_MAP_GET_NEXT_KEY: 4404db20fd2bSAlexei Starovoitov err = map_get_next_key(&attr); 4405db20fd2bSAlexei Starovoitov break; 440687df15deSDaniel Borkmann case BPF_MAP_FREEZE: 440787df15deSDaniel Borkmann err = map_freeze(&attr); 440887df15deSDaniel Borkmann break; 440909756af4SAlexei Starovoitov case BPF_PROG_LOAD: 4410838e9690SYonghong Song err = bpf_prog_load(&attr, uattr); 441109756af4SAlexei Starovoitov break; 4412b2197755SDaniel Borkmann case BPF_OBJ_PIN: 4413b2197755SDaniel Borkmann err = bpf_obj_pin(&attr); 4414b2197755SDaniel Borkmann break; 4415b2197755SDaniel Borkmann case BPF_OBJ_GET: 4416b2197755SDaniel Borkmann err = bpf_obj_get(&attr); 4417b2197755SDaniel Borkmann break; 4418f4324551SDaniel Mack case BPF_PROG_ATTACH: 4419f4324551SDaniel Mack err = bpf_prog_attach(&attr); 4420f4324551SDaniel Mack break; 4421f4324551SDaniel Mack case BPF_PROG_DETACH: 4422f4324551SDaniel Mack err = bpf_prog_detach(&attr); 4423f4324551SDaniel Mack break; 4424468e2f64SAlexei Starovoitov case BPF_PROG_QUERY: 4425468e2f64SAlexei Starovoitov err = bpf_prog_query(&attr, uattr); 4426468e2f64SAlexei Starovoitov break; 44271cf1cae9SAlexei Starovoitov case BPF_PROG_TEST_RUN: 44281cf1cae9SAlexei Starovoitov err = bpf_prog_test_run(&attr, uattr); 44291cf1cae9SAlexei Starovoitov break; 443034ad5580SMartin KaFai Lau case BPF_PROG_GET_NEXT_ID: 443134ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 443234ad5580SMartin KaFai Lau &prog_idr, &prog_idr_lock); 443334ad5580SMartin KaFai Lau break; 443434ad5580SMartin KaFai Lau case BPF_MAP_GET_NEXT_ID: 443534ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 443634ad5580SMartin KaFai Lau &map_idr, &map_idr_lock); 443734ad5580SMartin KaFai Lau break; 44381b9ed84eSQuentin Monnet case BPF_BTF_GET_NEXT_ID: 44391b9ed84eSQuentin Monnet err = bpf_obj_get_next_id(&attr, uattr, 44401b9ed84eSQuentin Monnet &btf_idr, &btf_idr_lock); 44411b9ed84eSQuentin Monnet break; 4442b16d9aa4SMartin KaFai Lau case BPF_PROG_GET_FD_BY_ID: 4443b16d9aa4SMartin KaFai Lau err = bpf_prog_get_fd_by_id(&attr); 4444b16d9aa4SMartin KaFai Lau break; 4445bd5f5f4eSMartin KaFai Lau case BPF_MAP_GET_FD_BY_ID: 4446bd5f5f4eSMartin KaFai Lau err = bpf_map_get_fd_by_id(&attr); 4447bd5f5f4eSMartin KaFai Lau break; 44481e270976SMartin KaFai Lau case BPF_OBJ_GET_INFO_BY_FD: 44491e270976SMartin KaFai Lau err = bpf_obj_get_info_by_fd(&attr, uattr); 44501e270976SMartin KaFai Lau break; 4451c4f6699dSAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN: 4452c4f6699dSAlexei Starovoitov err = bpf_raw_tracepoint_open(&attr); 4453c4f6699dSAlexei Starovoitov break; 4454f56a653cSMartin KaFai Lau case BPF_BTF_LOAD: 4455f56a653cSMartin KaFai Lau err = bpf_btf_load(&attr); 4456f56a653cSMartin KaFai Lau break; 445778958fcaSMartin KaFai Lau case BPF_BTF_GET_FD_BY_ID: 445878958fcaSMartin KaFai Lau err = bpf_btf_get_fd_by_id(&attr); 445978958fcaSMartin KaFai Lau break; 446041bdc4b4SYonghong Song case BPF_TASK_FD_QUERY: 446141bdc4b4SYonghong Song err = bpf_task_fd_query(&attr, uattr); 446241bdc4b4SYonghong Song break; 4463bd513cd0SMauricio Vasquez B case BPF_MAP_LOOKUP_AND_DELETE_ELEM: 4464bd513cd0SMauricio Vasquez B err = map_lookup_and_delete_elem(&attr); 4465bd513cd0SMauricio Vasquez B break; 4466cb4d03abSBrian Vazquez case BPF_MAP_LOOKUP_BATCH: 4467cb4d03abSBrian Vazquez err = bpf_map_do_batch(&attr, uattr, BPF_MAP_LOOKUP_BATCH); 4468cb4d03abSBrian Vazquez break; 446905799638SYonghong Song case BPF_MAP_LOOKUP_AND_DELETE_BATCH: 447005799638SYonghong Song err = bpf_map_do_batch(&attr, uattr, 447105799638SYonghong Song BPF_MAP_LOOKUP_AND_DELETE_BATCH); 447205799638SYonghong Song break; 4473aa2e93b8SBrian Vazquez case BPF_MAP_UPDATE_BATCH: 4474aa2e93b8SBrian Vazquez err = bpf_map_do_batch(&attr, uattr, BPF_MAP_UPDATE_BATCH); 4475aa2e93b8SBrian Vazquez break; 4476aa2e93b8SBrian Vazquez case BPF_MAP_DELETE_BATCH: 4477aa2e93b8SBrian Vazquez err = bpf_map_do_batch(&attr, uattr, BPF_MAP_DELETE_BATCH); 4478aa2e93b8SBrian Vazquez break; 4479af6eea57SAndrii Nakryiko case BPF_LINK_CREATE: 4480af6eea57SAndrii Nakryiko err = link_create(&attr); 4481af6eea57SAndrii Nakryiko break; 44820c991ebcSAndrii Nakryiko case BPF_LINK_UPDATE: 44830c991ebcSAndrii Nakryiko err = link_update(&attr); 44840c991ebcSAndrii Nakryiko break; 44852d602c8cSAndrii Nakryiko case BPF_LINK_GET_FD_BY_ID: 44862d602c8cSAndrii Nakryiko err = bpf_link_get_fd_by_id(&attr); 44872d602c8cSAndrii Nakryiko break; 44882d602c8cSAndrii Nakryiko case BPF_LINK_GET_NEXT_ID: 44892d602c8cSAndrii Nakryiko err = bpf_obj_get_next_id(&attr, uattr, 44902d602c8cSAndrii Nakryiko &link_idr, &link_idr_lock); 44912d602c8cSAndrii Nakryiko break; 4492d46edd67SSong Liu case BPF_ENABLE_STATS: 4493d46edd67SSong Liu err = bpf_enable_stats(&attr); 4494d46edd67SSong Liu break; 4495ac51d99bSYonghong Song case BPF_ITER_CREATE: 4496ac51d99bSYonghong Song err = bpf_iter_create(&attr); 4497ac51d99bSYonghong Song break; 449873b11c2aSAndrii Nakryiko case BPF_LINK_DETACH: 449973b11c2aSAndrii Nakryiko err = link_detach(&attr); 450073b11c2aSAndrii Nakryiko break; 4501ef15314aSYiFei Zhu case BPF_PROG_BIND_MAP: 4502ef15314aSYiFei Zhu err = bpf_prog_bind_map(&attr); 4503ef15314aSYiFei Zhu break; 450499c55f7dSAlexei Starovoitov default: 450599c55f7dSAlexei Starovoitov err = -EINVAL; 450699c55f7dSAlexei Starovoitov break; 450799c55f7dSAlexei Starovoitov } 450899c55f7dSAlexei Starovoitov 450999c55f7dSAlexei Starovoitov return err; 451099c55f7dSAlexei Starovoitov } 4511