15b497af4SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 299c55f7dSAlexei Starovoitov /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 399c55f7dSAlexei Starovoitov */ 499c55f7dSAlexei Starovoitov #include <linux/bpf.h> 5a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 6f4364dcfSSean Young #include <linux/bpf_lirc.h> 74a1e7c0cSToke Høiland-Jørgensen #include <linux/bpf_verifier.h> 8f56a653cSMartin KaFai Lau #include <linux/btf.h> 999c55f7dSAlexei Starovoitov #include <linux/syscalls.h> 1099c55f7dSAlexei Starovoitov #include <linux/slab.h> 113f07c014SIngo Molnar #include <linux/sched/signal.h> 12d407bd25SDaniel Borkmann #include <linux/vmalloc.h> 13d407bd25SDaniel Borkmann #include <linux/mmzone.h> 1499c55f7dSAlexei Starovoitov #include <linux/anon_inodes.h> 1541bdc4b4SYonghong Song #include <linux/fdtable.h> 16db20fd2bSAlexei Starovoitov #include <linux/file.h> 1741bdc4b4SYonghong Song #include <linux/fs.h> 1809756af4SAlexei Starovoitov #include <linux/license.h> 1909756af4SAlexei Starovoitov #include <linux/filter.h> 20535e7b4bSMickaël Salaün #include <linux/kernel.h> 21dc4bb0e2SMartin KaFai Lau #include <linux/idr.h> 22cb4d2b3fSMartin KaFai Lau #include <linux/cred.h> 23cb4d2b3fSMartin KaFai Lau #include <linux/timekeeping.h> 24cb4d2b3fSMartin KaFai Lau #include <linux/ctype.h> 259ef09e35SMark Rutland #include <linux/nospec.h> 26bae141f5SDaniel Borkmann #include <linux/audit.h> 27ccfe29ebSAlexei Starovoitov #include <uapi/linux/btf.h> 28ca5999fdSMike Rapoport #include <linux/pgtable.h> 299e4e01dfSKP Singh #include <linux/bpf_lsm.h> 30457f4436SAndrii Nakryiko #include <linux/poll.h> 31a3fd7ceeSJakub Sitnicki #include <linux/bpf-netns.h> 321e6c62a8SAlexei Starovoitov #include <linux/rcupdate_trace.h> 3348edc1f7SRoman Gushchin #include <linux/memcontrol.h> 3499c55f7dSAlexei Starovoitov 35da765a2fSDaniel Borkmann #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 3614dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 3714dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 38da765a2fSDaniel Borkmann #define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY) 3914dc6f04SMartin KaFai Lau #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 40da765a2fSDaniel Borkmann #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \ 41da765a2fSDaniel Borkmann IS_FD_HASH(map)) 4214dc6f04SMartin KaFai Lau 436e71b04aSChenbo Feng #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) 446e71b04aSChenbo Feng 45b121d1e7SAlexei Starovoitov DEFINE_PER_CPU(int, bpf_prog_active); 46dc4bb0e2SMartin KaFai Lau static DEFINE_IDR(prog_idr); 47dc4bb0e2SMartin KaFai Lau static DEFINE_SPINLOCK(prog_idr_lock); 48f3f1c054SMartin KaFai Lau static DEFINE_IDR(map_idr); 49f3f1c054SMartin KaFai Lau static DEFINE_SPINLOCK(map_idr_lock); 50a3b80e10SAndrii Nakryiko static DEFINE_IDR(link_idr); 51a3b80e10SAndrii Nakryiko static DEFINE_SPINLOCK(link_idr_lock); 52b121d1e7SAlexei Starovoitov 531be7f75dSAlexei Starovoitov int sysctl_unprivileged_bpf_disabled __read_mostly; 541be7f75dSAlexei Starovoitov 5540077e0cSJohannes Berg static const struct bpf_map_ops * const bpf_map_types[] = { 5691cc1a99SAlexei Starovoitov #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) 5740077e0cSJohannes Berg #define BPF_MAP_TYPE(_id, _ops) \ 5840077e0cSJohannes Berg [_id] = &_ops, 59f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) 6040077e0cSJohannes Berg #include <linux/bpf_types.h> 6140077e0cSJohannes Berg #undef BPF_PROG_TYPE 6240077e0cSJohannes Berg #undef BPF_MAP_TYPE 63f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 6440077e0cSJohannes Berg }; 6599c55f7dSAlexei Starovoitov 66752ba56fSMickaël Salaün /* 67752ba56fSMickaël Salaün * If we're handed a bigger struct than we know of, ensure all the unknown bits 68752ba56fSMickaël Salaün * are 0 - i.e. new user-space does not rely on any kernel feature extensions 69752ba56fSMickaël Salaün * we don't know about yet. 70752ba56fSMickaël Salaün * 71752ba56fSMickaël Salaün * There is a ToCToU between this function call and the following 72752ba56fSMickaël Salaün * copy_from_user() call. However, this is not a concern since this function is 73752ba56fSMickaël Salaün * meant to be a future-proofing of bits. 74752ba56fSMickaël Salaün */ 75dcab51f1SMartin KaFai Lau int bpf_check_uarg_tail_zero(void __user *uaddr, 7658291a74SMickaël Salaün size_t expected_size, 7758291a74SMickaël Salaün size_t actual_size) 7858291a74SMickaël Salaün { 79b7e4b65fSAl Viro unsigned char __user *addr = uaddr + expected_size; 80b7e4b65fSAl Viro int res; 8158291a74SMickaël Salaün 82752ba56fSMickaël Salaün if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 83752ba56fSMickaël Salaün return -E2BIG; 84752ba56fSMickaël Salaün 8558291a74SMickaël Salaün if (actual_size <= expected_size) 8658291a74SMickaël Salaün return 0; 8758291a74SMickaël Salaün 88b7e4b65fSAl Viro res = check_zeroed_user(addr, actual_size - expected_size); 89b7e4b65fSAl Viro if (res < 0) 90b7e4b65fSAl Viro return res; 91b7e4b65fSAl Viro return res ? 0 : -E2BIG; 9258291a74SMickaël Salaün } 9358291a74SMickaël Salaün 94a3884572SJakub Kicinski const struct bpf_map_ops bpf_map_offload_ops = { 95f4d05259SMartin KaFai Lau .map_meta_equal = bpf_map_meta_equal, 96a3884572SJakub Kicinski .map_alloc = bpf_map_offload_map_alloc, 97a3884572SJakub Kicinski .map_free = bpf_map_offload_map_free, 98e8d2bec0SDaniel Borkmann .map_check_btf = map_check_no_btf, 99a3884572SJakub Kicinski }; 100a3884572SJakub Kicinski 10199c55f7dSAlexei Starovoitov static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 10299c55f7dSAlexei Starovoitov { 1031110f3a9SJakub Kicinski const struct bpf_map_ops *ops; 1049ef09e35SMark Rutland u32 type = attr->map_type; 10599c55f7dSAlexei Starovoitov struct bpf_map *map; 1061110f3a9SJakub Kicinski int err; 10799c55f7dSAlexei Starovoitov 1089ef09e35SMark Rutland if (type >= ARRAY_SIZE(bpf_map_types)) 1091110f3a9SJakub Kicinski return ERR_PTR(-EINVAL); 1109ef09e35SMark Rutland type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types)); 1119ef09e35SMark Rutland ops = bpf_map_types[type]; 1121110f3a9SJakub Kicinski if (!ops) 11340077e0cSJohannes Berg return ERR_PTR(-EINVAL); 11440077e0cSJohannes Berg 1151110f3a9SJakub Kicinski if (ops->map_alloc_check) { 1161110f3a9SJakub Kicinski err = ops->map_alloc_check(attr); 1171110f3a9SJakub Kicinski if (err) 1181110f3a9SJakub Kicinski return ERR_PTR(err); 1191110f3a9SJakub Kicinski } 120a3884572SJakub Kicinski if (attr->map_ifindex) 121a3884572SJakub Kicinski ops = &bpf_map_offload_ops; 1221110f3a9SJakub Kicinski map = ops->map_alloc(attr); 12399c55f7dSAlexei Starovoitov if (IS_ERR(map)) 12499c55f7dSAlexei Starovoitov return map; 1251110f3a9SJakub Kicinski map->ops = ops; 1269ef09e35SMark Rutland map->map_type = type; 12799c55f7dSAlexei Starovoitov return map; 12899c55f7dSAlexei Starovoitov } 12999c55f7dSAlexei Starovoitov 13080ee81e0SRoman Gushchin static u32 bpf_map_value_size(const struct bpf_map *map) 13115c14a3dSBrian Vazquez { 13215c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 13315c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 13415c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 13515c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 13615c14a3dSBrian Vazquez return round_up(map->value_size, 8) * num_possible_cpus(); 13715c14a3dSBrian Vazquez else if (IS_FD_MAP(map)) 13815c14a3dSBrian Vazquez return sizeof(u32); 13915c14a3dSBrian Vazquez else 14015c14a3dSBrian Vazquez return map->value_size; 14115c14a3dSBrian Vazquez } 14215c14a3dSBrian Vazquez 14315c14a3dSBrian Vazquez static void maybe_wait_bpf_programs(struct bpf_map *map) 14415c14a3dSBrian Vazquez { 14515c14a3dSBrian Vazquez /* Wait for any running BPF programs to complete so that 14615c14a3dSBrian Vazquez * userspace, when we return to it, knows that all programs 14715c14a3dSBrian Vazquez * that could be running use the new map value. 14815c14a3dSBrian Vazquez */ 14915c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS || 15015c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 15115c14a3dSBrian Vazquez synchronize_rcu(); 15215c14a3dSBrian Vazquez } 15315c14a3dSBrian Vazquez 15415c14a3dSBrian Vazquez static int bpf_map_update_value(struct bpf_map *map, struct fd f, void *key, 15515c14a3dSBrian Vazquez void *value, __u64 flags) 15615c14a3dSBrian Vazquez { 15715c14a3dSBrian Vazquez int err; 15815c14a3dSBrian Vazquez 15915c14a3dSBrian Vazquez /* Need to create a kthread, thus must support schedule */ 16015c14a3dSBrian Vazquez if (bpf_map_is_dev_bound(map)) { 16115c14a3dSBrian Vazquez return bpf_map_offload_update_elem(map, key, value, flags); 16215c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_CPUMAP || 16315c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 16415c14a3dSBrian Vazquez return map->ops->map_update_elem(map, key, value, flags); 16513b79d3fSLorenz Bauer } else if (map->map_type == BPF_MAP_TYPE_SOCKHASH || 16613b79d3fSLorenz Bauer map->map_type == BPF_MAP_TYPE_SOCKMAP) { 16713b79d3fSLorenz Bauer return sock_map_update_elem_sys(map, key, value, flags); 16815c14a3dSBrian Vazquez } else if (IS_FD_PROG_ARRAY(map)) { 16915c14a3dSBrian Vazquez return bpf_fd_array_map_update_elem(map, f.file, key, value, 17015c14a3dSBrian Vazquez flags); 17115c14a3dSBrian Vazquez } 17215c14a3dSBrian Vazquez 173b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 17415c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 17515c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 17615c14a3dSBrian Vazquez err = bpf_percpu_hash_update(map, key, value, flags); 17715c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 17815c14a3dSBrian Vazquez err = bpf_percpu_array_update(map, key, value, flags); 17915c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 18015c14a3dSBrian Vazquez err = bpf_percpu_cgroup_storage_update(map, key, value, 18115c14a3dSBrian Vazquez flags); 18215c14a3dSBrian Vazquez } else if (IS_FD_ARRAY(map)) { 18315c14a3dSBrian Vazquez rcu_read_lock(); 18415c14a3dSBrian Vazquez err = bpf_fd_array_map_update_elem(map, f.file, key, value, 18515c14a3dSBrian Vazquez flags); 18615c14a3dSBrian Vazquez rcu_read_unlock(); 18715c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 18815c14a3dSBrian Vazquez rcu_read_lock(); 18915c14a3dSBrian Vazquez err = bpf_fd_htab_map_update_elem(map, f.file, key, value, 19015c14a3dSBrian Vazquez flags); 19115c14a3dSBrian Vazquez rcu_read_unlock(); 19215c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 19315c14a3dSBrian Vazquez /* rcu_read_lock() is not needed */ 19415c14a3dSBrian Vazquez err = bpf_fd_reuseport_array_update_elem(map, key, value, 19515c14a3dSBrian Vazquez flags); 19615c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 19715c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_STACK) { 19815c14a3dSBrian Vazquez err = map->ops->map_push_elem(map, value, flags); 19915c14a3dSBrian Vazquez } else { 20015c14a3dSBrian Vazquez rcu_read_lock(); 20115c14a3dSBrian Vazquez err = map->ops->map_update_elem(map, key, value, flags); 20215c14a3dSBrian Vazquez rcu_read_unlock(); 20315c14a3dSBrian Vazquez } 204b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 20515c14a3dSBrian Vazquez maybe_wait_bpf_programs(map); 20615c14a3dSBrian Vazquez 20715c14a3dSBrian Vazquez return err; 20815c14a3dSBrian Vazquez } 20915c14a3dSBrian Vazquez 21015c14a3dSBrian Vazquez static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value, 21115c14a3dSBrian Vazquez __u64 flags) 21215c14a3dSBrian Vazquez { 21315c14a3dSBrian Vazquez void *ptr; 21415c14a3dSBrian Vazquez int err; 21515c14a3dSBrian Vazquez 216cb4d03abSBrian Vazquez if (bpf_map_is_dev_bound(map)) 217cb4d03abSBrian Vazquez return bpf_map_offload_lookup_elem(map, key, value); 21815c14a3dSBrian Vazquez 219b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 22015c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 22115c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 22215c14a3dSBrian Vazquez err = bpf_percpu_hash_copy(map, key, value); 22315c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 22415c14a3dSBrian Vazquez err = bpf_percpu_array_copy(map, key, value); 22515c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 22615c14a3dSBrian Vazquez err = bpf_percpu_cgroup_storage_copy(map, key, value); 22715c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 22815c14a3dSBrian Vazquez err = bpf_stackmap_copy(map, key, value); 22915c14a3dSBrian Vazquez } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) { 23015c14a3dSBrian Vazquez err = bpf_fd_array_map_lookup_elem(map, key, value); 23115c14a3dSBrian Vazquez } else if (IS_FD_HASH(map)) { 23215c14a3dSBrian Vazquez err = bpf_fd_htab_map_lookup_elem(map, key, value); 23315c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 23415c14a3dSBrian Vazquez err = bpf_fd_reuseport_array_lookup_elem(map, key, value); 23515c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 23615c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_STACK) { 23715c14a3dSBrian Vazquez err = map->ops->map_peek_elem(map, value); 23815c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 23915c14a3dSBrian Vazquez /* struct_ops map requires directly updating "value" */ 24015c14a3dSBrian Vazquez err = bpf_struct_ops_map_sys_lookup_elem(map, key, value); 24115c14a3dSBrian Vazquez } else { 24215c14a3dSBrian Vazquez rcu_read_lock(); 24315c14a3dSBrian Vazquez if (map->ops->map_lookup_elem_sys_only) 24415c14a3dSBrian Vazquez ptr = map->ops->map_lookup_elem_sys_only(map, key); 24515c14a3dSBrian Vazquez else 24615c14a3dSBrian Vazquez ptr = map->ops->map_lookup_elem(map, key); 24715c14a3dSBrian Vazquez if (IS_ERR(ptr)) { 24815c14a3dSBrian Vazquez err = PTR_ERR(ptr); 24915c14a3dSBrian Vazquez } else if (!ptr) { 25015c14a3dSBrian Vazquez err = -ENOENT; 25115c14a3dSBrian Vazquez } else { 25215c14a3dSBrian Vazquez err = 0; 25315c14a3dSBrian Vazquez if (flags & BPF_F_LOCK) 25415c14a3dSBrian Vazquez /* lock 'ptr' and copy everything but lock */ 25515c14a3dSBrian Vazquez copy_map_value_locked(map, value, ptr, true); 25615c14a3dSBrian Vazquez else 25715c14a3dSBrian Vazquez copy_map_value(map, value, ptr); 25815c14a3dSBrian Vazquez /* mask lock, since value wasn't zero inited */ 25915c14a3dSBrian Vazquez check_and_init_map_lock(map, value); 26015c14a3dSBrian Vazquez } 26115c14a3dSBrian Vazquez rcu_read_unlock(); 26215c14a3dSBrian Vazquez } 26315c14a3dSBrian Vazquez 264b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 26515c14a3dSBrian Vazquez maybe_wait_bpf_programs(map); 26615c14a3dSBrian Vazquez 26715c14a3dSBrian Vazquez return err; 26815c14a3dSBrian Vazquez } 26915c14a3dSBrian Vazquez 270d5299b67SRoman Gushchin /* Please, do not use this function outside from the map creation path 271d5299b67SRoman Gushchin * (e.g. in map update path) without taking care of setting the active 272d5299b67SRoman Gushchin * memory cgroup (see at bpf_map_kmalloc_node() for example). 273d5299b67SRoman Gushchin */ 274196e8ca7SDaniel Borkmann static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable) 275d407bd25SDaniel Borkmann { 276f01a7dbeSMartynas Pumputis /* We really just want to fail instead of triggering OOM killer 277f01a7dbeSMartynas Pumputis * under memory pressure, therefore we set __GFP_NORETRY to kmalloc, 278f01a7dbeSMartynas Pumputis * which is used for lower order allocation requests. 279f01a7dbeSMartynas Pumputis * 280f01a7dbeSMartynas Pumputis * It has been observed that higher order allocation requests done by 281f01a7dbeSMartynas Pumputis * vmalloc with __GFP_NORETRY being set might fail due to not trying 282f01a7dbeSMartynas Pumputis * to reclaim memory from the page cache, thus we set 283f01a7dbeSMartynas Pumputis * __GFP_RETRY_MAYFAIL to avoid such situations. 284d407bd25SDaniel Borkmann */ 285f01a7dbeSMartynas Pumputis 286d5299b67SRoman Gushchin const gfp_t gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_ACCOUNT; 287041de93fSChristoph Hellwig unsigned int flags = 0; 288041de93fSChristoph Hellwig unsigned long align = 1; 289d407bd25SDaniel Borkmann void *area; 290d407bd25SDaniel Borkmann 291196e8ca7SDaniel Borkmann if (size >= SIZE_MAX) 292196e8ca7SDaniel Borkmann return NULL; 293196e8ca7SDaniel Borkmann 294fc970227SAndrii Nakryiko /* kmalloc()'ed memory can't be mmap()'ed */ 295041de93fSChristoph Hellwig if (mmapable) { 296041de93fSChristoph Hellwig BUG_ON(!PAGE_ALIGNED(size)); 297041de93fSChristoph Hellwig align = SHMLBA; 298041de93fSChristoph Hellwig flags = VM_USERMAP; 299041de93fSChristoph Hellwig } else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 300041de93fSChristoph Hellwig area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY, 301f01a7dbeSMartynas Pumputis numa_node); 302d407bd25SDaniel Borkmann if (area != NULL) 303d407bd25SDaniel Borkmann return area; 304d407bd25SDaniel Borkmann } 305041de93fSChristoph Hellwig 306041de93fSChristoph Hellwig return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, 307041de93fSChristoph Hellwig gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL, 308041de93fSChristoph Hellwig flags, numa_node, __builtin_return_address(0)); 309d407bd25SDaniel Borkmann } 310d407bd25SDaniel Borkmann 311196e8ca7SDaniel Borkmann void *bpf_map_area_alloc(u64 size, int numa_node) 312fc970227SAndrii Nakryiko { 313fc970227SAndrii Nakryiko return __bpf_map_area_alloc(size, numa_node, false); 314fc970227SAndrii Nakryiko } 315fc970227SAndrii Nakryiko 316196e8ca7SDaniel Borkmann void *bpf_map_area_mmapable_alloc(u64 size, int numa_node) 317fc970227SAndrii Nakryiko { 318fc970227SAndrii Nakryiko return __bpf_map_area_alloc(size, numa_node, true); 319fc970227SAndrii Nakryiko } 320fc970227SAndrii Nakryiko 321d407bd25SDaniel Borkmann void bpf_map_area_free(void *area) 322d407bd25SDaniel Borkmann { 323d407bd25SDaniel Borkmann kvfree(area); 324d407bd25SDaniel Borkmann } 325d407bd25SDaniel Borkmann 326be70bcd5SDaniel Borkmann static u32 bpf_map_flags_retain_permanent(u32 flags) 327be70bcd5SDaniel Borkmann { 328be70bcd5SDaniel Borkmann /* Some map creation flags are not tied to the map object but 329be70bcd5SDaniel Borkmann * rather to the map fd instead, so they have no meaning upon 330be70bcd5SDaniel Borkmann * map object inspection since multiple file descriptors with 331be70bcd5SDaniel Borkmann * different (access) properties can exist here. Thus, given 332be70bcd5SDaniel Borkmann * this has zero meaning for the map itself, lets clear these 333be70bcd5SDaniel Borkmann * from here. 334be70bcd5SDaniel Borkmann */ 335be70bcd5SDaniel Borkmann return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY); 336be70bcd5SDaniel Borkmann } 337be70bcd5SDaniel Borkmann 338bd475643SJakub Kicinski void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) 339bd475643SJakub Kicinski { 340bd475643SJakub Kicinski map->map_type = attr->map_type; 341bd475643SJakub Kicinski map->key_size = attr->key_size; 342bd475643SJakub Kicinski map->value_size = attr->value_size; 343bd475643SJakub Kicinski map->max_entries = attr->max_entries; 344be70bcd5SDaniel Borkmann map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags); 345bd475643SJakub Kicinski map->numa_node = bpf_map_attr_numa_node(attr); 346bd475643SJakub Kicinski } 347bd475643SJakub Kicinski 348f3f1c054SMartin KaFai Lau static int bpf_map_alloc_id(struct bpf_map *map) 349f3f1c054SMartin KaFai Lau { 350f3f1c054SMartin KaFai Lau int id; 351f3f1c054SMartin KaFai Lau 352b76354cdSShaohua Li idr_preload(GFP_KERNEL); 353f3f1c054SMartin KaFai Lau spin_lock_bh(&map_idr_lock); 354f3f1c054SMartin KaFai Lau id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 355f3f1c054SMartin KaFai Lau if (id > 0) 356f3f1c054SMartin KaFai Lau map->id = id; 357f3f1c054SMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 358b76354cdSShaohua Li idr_preload_end(); 359f3f1c054SMartin KaFai Lau 360f3f1c054SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 361f3f1c054SMartin KaFai Lau return -ENOSPC; 362f3f1c054SMartin KaFai Lau 363f3f1c054SMartin KaFai Lau return id > 0 ? 0 : id; 364f3f1c054SMartin KaFai Lau } 365f3f1c054SMartin KaFai Lau 366a3884572SJakub Kicinski void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) 367f3f1c054SMartin KaFai Lau { 368930651a7SEric Dumazet unsigned long flags; 369930651a7SEric Dumazet 370a3884572SJakub Kicinski /* Offloaded maps are removed from the IDR store when their device 371a3884572SJakub Kicinski * disappears - even if someone holds an fd to them they are unusable, 372a3884572SJakub Kicinski * the memory is gone, all ops will fail; they are simply waiting for 373a3884572SJakub Kicinski * refcnt to drop to be freed. 374a3884572SJakub Kicinski */ 375a3884572SJakub Kicinski if (!map->id) 376a3884572SJakub Kicinski return; 377a3884572SJakub Kicinski 378bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 379930651a7SEric Dumazet spin_lock_irqsave(&map_idr_lock, flags); 380bd5f5f4eSMartin KaFai Lau else 381bd5f5f4eSMartin KaFai Lau __acquire(&map_idr_lock); 382bd5f5f4eSMartin KaFai Lau 383f3f1c054SMartin KaFai Lau idr_remove(&map_idr, map->id); 384a3884572SJakub Kicinski map->id = 0; 385bd5f5f4eSMartin KaFai Lau 386bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 387930651a7SEric Dumazet spin_unlock_irqrestore(&map_idr_lock, flags); 388bd5f5f4eSMartin KaFai Lau else 389bd5f5f4eSMartin KaFai Lau __release(&map_idr_lock); 390f3f1c054SMartin KaFai Lau } 391f3f1c054SMartin KaFai Lau 39248edc1f7SRoman Gushchin #ifdef CONFIG_MEMCG_KMEM 39348edc1f7SRoman Gushchin static void bpf_map_save_memcg(struct bpf_map *map) 39448edc1f7SRoman Gushchin { 39548edc1f7SRoman Gushchin map->memcg = get_mem_cgroup_from_mm(current->mm); 39648edc1f7SRoman Gushchin } 39748edc1f7SRoman Gushchin 39848edc1f7SRoman Gushchin static void bpf_map_release_memcg(struct bpf_map *map) 39948edc1f7SRoman Gushchin { 40048edc1f7SRoman Gushchin mem_cgroup_put(map->memcg); 40148edc1f7SRoman Gushchin } 40248edc1f7SRoman Gushchin 40348edc1f7SRoman Gushchin void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, 40448edc1f7SRoman Gushchin int node) 40548edc1f7SRoman Gushchin { 40648edc1f7SRoman Gushchin struct mem_cgroup *old_memcg; 40748edc1f7SRoman Gushchin void *ptr; 40848edc1f7SRoman Gushchin 40948edc1f7SRoman Gushchin old_memcg = set_active_memcg(map->memcg); 41048edc1f7SRoman Gushchin ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node); 41148edc1f7SRoman Gushchin set_active_memcg(old_memcg); 41248edc1f7SRoman Gushchin 41348edc1f7SRoman Gushchin return ptr; 41448edc1f7SRoman Gushchin } 41548edc1f7SRoman Gushchin 41648edc1f7SRoman Gushchin void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags) 41748edc1f7SRoman Gushchin { 41848edc1f7SRoman Gushchin struct mem_cgroup *old_memcg; 41948edc1f7SRoman Gushchin void *ptr; 42048edc1f7SRoman Gushchin 42148edc1f7SRoman Gushchin old_memcg = set_active_memcg(map->memcg); 42248edc1f7SRoman Gushchin ptr = kzalloc(size, flags | __GFP_ACCOUNT); 42348edc1f7SRoman Gushchin set_active_memcg(old_memcg); 42448edc1f7SRoman Gushchin 42548edc1f7SRoman Gushchin return ptr; 42648edc1f7SRoman Gushchin } 42748edc1f7SRoman Gushchin 42848edc1f7SRoman Gushchin void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, 42948edc1f7SRoman Gushchin size_t align, gfp_t flags) 43048edc1f7SRoman Gushchin { 43148edc1f7SRoman Gushchin struct mem_cgroup *old_memcg; 43248edc1f7SRoman Gushchin void __percpu *ptr; 43348edc1f7SRoman Gushchin 43448edc1f7SRoman Gushchin old_memcg = set_active_memcg(map->memcg); 43548edc1f7SRoman Gushchin ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT); 43648edc1f7SRoman Gushchin set_active_memcg(old_memcg); 43748edc1f7SRoman Gushchin 43848edc1f7SRoman Gushchin return ptr; 43948edc1f7SRoman Gushchin } 44048edc1f7SRoman Gushchin 44148edc1f7SRoman Gushchin #else 44248edc1f7SRoman Gushchin static void bpf_map_save_memcg(struct bpf_map *map) 44348edc1f7SRoman Gushchin { 44448edc1f7SRoman Gushchin } 44548edc1f7SRoman Gushchin 44648edc1f7SRoman Gushchin static void bpf_map_release_memcg(struct bpf_map *map) 44748edc1f7SRoman Gushchin { 44848edc1f7SRoman Gushchin } 44948edc1f7SRoman Gushchin #endif 45048edc1f7SRoman Gushchin 45199c55f7dSAlexei Starovoitov /* called from workqueue */ 45299c55f7dSAlexei Starovoitov static void bpf_map_free_deferred(struct work_struct *work) 45399c55f7dSAlexei Starovoitov { 45499c55f7dSAlexei Starovoitov struct bpf_map *map = container_of(work, struct bpf_map, work); 45599c55f7dSAlexei Starovoitov 456afdb09c7SChenbo Feng security_bpf_map_free(map); 45748edc1f7SRoman Gushchin bpf_map_release_memcg(map); 45899c55f7dSAlexei Starovoitov /* implementation dependent freeing */ 45999c55f7dSAlexei Starovoitov map->ops->map_free(map); 46099c55f7dSAlexei Starovoitov } 46199c55f7dSAlexei Starovoitov 462c9da161cSDaniel Borkmann static void bpf_map_put_uref(struct bpf_map *map) 463c9da161cSDaniel Borkmann { 4641e0bd5a0SAndrii Nakryiko if (atomic64_dec_and_test(&map->usercnt)) { 465ba6b8de4SJohn Fastabend if (map->ops->map_release_uref) 466ba6b8de4SJohn Fastabend map->ops->map_release_uref(map); 467c9da161cSDaniel Borkmann } 468c9da161cSDaniel Borkmann } 469c9da161cSDaniel Borkmann 47099c55f7dSAlexei Starovoitov /* decrement map refcnt and schedule it for freeing via workqueue 47199c55f7dSAlexei Starovoitov * (unrelying map implementation ops->map_free() might sleep) 47299c55f7dSAlexei Starovoitov */ 473bd5f5f4eSMartin KaFai Lau static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) 47499c55f7dSAlexei Starovoitov { 4751e0bd5a0SAndrii Nakryiko if (atomic64_dec_and_test(&map->refcnt)) { 47634ad5580SMartin KaFai Lau /* bpf_map_free_id() must be called first */ 477bd5f5f4eSMartin KaFai Lau bpf_map_free_id(map, do_idr_lock); 47878958fcaSMartin KaFai Lau btf_put(map->btf); 47999c55f7dSAlexei Starovoitov INIT_WORK(&map->work, bpf_map_free_deferred); 48099c55f7dSAlexei Starovoitov schedule_work(&map->work); 48199c55f7dSAlexei Starovoitov } 48299c55f7dSAlexei Starovoitov } 48399c55f7dSAlexei Starovoitov 484bd5f5f4eSMartin KaFai Lau void bpf_map_put(struct bpf_map *map) 485bd5f5f4eSMartin KaFai Lau { 486bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, true); 487bd5f5f4eSMartin KaFai Lau } 488630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_put); 489bd5f5f4eSMartin KaFai Lau 490c9da161cSDaniel Borkmann void bpf_map_put_with_uref(struct bpf_map *map) 491c9da161cSDaniel Borkmann { 492c9da161cSDaniel Borkmann bpf_map_put_uref(map); 493c9da161cSDaniel Borkmann bpf_map_put(map); 494c9da161cSDaniel Borkmann } 495c9da161cSDaniel Borkmann 49699c55f7dSAlexei Starovoitov static int bpf_map_release(struct inode *inode, struct file *filp) 49799c55f7dSAlexei Starovoitov { 49861d1b6a4SDaniel Borkmann struct bpf_map *map = filp->private_data; 49961d1b6a4SDaniel Borkmann 50061d1b6a4SDaniel Borkmann if (map->ops->map_release) 50161d1b6a4SDaniel Borkmann map->ops->map_release(map, filp); 50261d1b6a4SDaniel Borkmann 50361d1b6a4SDaniel Borkmann bpf_map_put_with_uref(map); 50499c55f7dSAlexei Starovoitov return 0; 50599c55f7dSAlexei Starovoitov } 50699c55f7dSAlexei Starovoitov 50787df15deSDaniel Borkmann static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f) 50887df15deSDaniel Borkmann { 50987df15deSDaniel Borkmann fmode_t mode = f.file->f_mode; 51087df15deSDaniel Borkmann 51187df15deSDaniel Borkmann /* Our file permissions may have been overridden by global 51287df15deSDaniel Borkmann * map permissions facing syscall side. 51387df15deSDaniel Borkmann */ 51487df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) 51587df15deSDaniel Borkmann mode &= ~FMODE_CAN_WRITE; 51687df15deSDaniel Borkmann return mode; 51787df15deSDaniel Borkmann } 51887df15deSDaniel Borkmann 519f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 52080ee81e0SRoman Gushchin /* Provides an approximation of the map's memory footprint. 52180ee81e0SRoman Gushchin * Used only to provide a backward compatibility and display 52280ee81e0SRoman Gushchin * a reasonable "memlock" info. 52380ee81e0SRoman Gushchin */ 52480ee81e0SRoman Gushchin static unsigned long bpf_map_memory_footprint(const struct bpf_map *map) 52580ee81e0SRoman Gushchin { 52680ee81e0SRoman Gushchin unsigned long size; 52780ee81e0SRoman Gushchin 52880ee81e0SRoman Gushchin size = round_up(map->key_size + bpf_map_value_size(map), 8); 52980ee81e0SRoman Gushchin 53080ee81e0SRoman Gushchin return round_up(map->max_entries * size, PAGE_SIZE); 53180ee81e0SRoman Gushchin } 53280ee81e0SRoman Gushchin 533f99bf205SDaniel Borkmann static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 534f99bf205SDaniel Borkmann { 535f99bf205SDaniel Borkmann const struct bpf_map *map = filp->private_data; 53621116b70SDaniel Borkmann const struct bpf_array *array; 5372beee5f5SDaniel Borkmann u32 type = 0, jited = 0; 53821116b70SDaniel Borkmann 53921116b70SDaniel Borkmann if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 54021116b70SDaniel Borkmann array = container_of(map, struct bpf_array, map); 5412beee5f5SDaniel Borkmann type = array->aux->type; 5422beee5f5SDaniel Borkmann jited = array->aux->jited; 54321116b70SDaniel Borkmann } 544f99bf205SDaniel Borkmann 545f99bf205SDaniel Borkmann seq_printf(m, 546f99bf205SDaniel Borkmann "map_type:\t%u\n" 547f99bf205SDaniel Borkmann "key_size:\t%u\n" 548f99bf205SDaniel Borkmann "value_size:\t%u\n" 549322cea2fSDaniel Borkmann "max_entries:\t%u\n" 55021116b70SDaniel Borkmann "map_flags:\t%#x\n" 55180ee81e0SRoman Gushchin "memlock:\t%lu\n" 55287df15deSDaniel Borkmann "map_id:\t%u\n" 55387df15deSDaniel Borkmann "frozen:\t%u\n", 554f99bf205SDaniel Borkmann map->map_type, 555f99bf205SDaniel Borkmann map->key_size, 556f99bf205SDaniel Borkmann map->value_size, 557322cea2fSDaniel Borkmann map->max_entries, 55821116b70SDaniel Borkmann map->map_flags, 55980ee81e0SRoman Gushchin bpf_map_memory_footprint(map), 56087df15deSDaniel Borkmann map->id, 56187df15deSDaniel Borkmann READ_ONCE(map->frozen)); 5622beee5f5SDaniel Borkmann if (type) { 5632beee5f5SDaniel Borkmann seq_printf(m, "owner_prog_type:\t%u\n", type); 5642beee5f5SDaniel Borkmann seq_printf(m, "owner_jited:\t%u\n", jited); 5659780c0abSDaniel Borkmann } 566f99bf205SDaniel Borkmann } 567f99bf205SDaniel Borkmann #endif 568f99bf205SDaniel Borkmann 5696e71b04aSChenbo Feng static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, 5706e71b04aSChenbo Feng loff_t *ppos) 5716e71b04aSChenbo Feng { 5726e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 5736e71b04aSChenbo Feng * f_mode with FMODE_CAN_READ. 5746e71b04aSChenbo Feng */ 5756e71b04aSChenbo Feng return -EINVAL; 5766e71b04aSChenbo Feng } 5776e71b04aSChenbo Feng 5786e71b04aSChenbo Feng static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, 5796e71b04aSChenbo Feng size_t siz, loff_t *ppos) 5806e71b04aSChenbo Feng { 5816e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 5826e71b04aSChenbo Feng * f_mode with FMODE_CAN_WRITE. 5836e71b04aSChenbo Feng */ 5846e71b04aSChenbo Feng return -EINVAL; 5856e71b04aSChenbo Feng } 5866e71b04aSChenbo Feng 587fc970227SAndrii Nakryiko /* called for any extra memory-mapped regions (except initial) */ 588fc970227SAndrii Nakryiko static void bpf_map_mmap_open(struct vm_area_struct *vma) 589fc970227SAndrii Nakryiko { 590fc970227SAndrii Nakryiko struct bpf_map *map = vma->vm_file->private_data; 591fc970227SAndrii Nakryiko 5921f6cb19bSAndrii Nakryiko if (vma->vm_flags & VM_MAYWRITE) { 593fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 594fc970227SAndrii Nakryiko map->writecnt++; 595fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 596fc970227SAndrii Nakryiko } 597fc970227SAndrii Nakryiko } 598fc970227SAndrii Nakryiko 599fc970227SAndrii Nakryiko /* called for all unmapped memory region (including initial) */ 600fc970227SAndrii Nakryiko static void bpf_map_mmap_close(struct vm_area_struct *vma) 601fc970227SAndrii Nakryiko { 602fc970227SAndrii Nakryiko struct bpf_map *map = vma->vm_file->private_data; 603fc970227SAndrii Nakryiko 6041f6cb19bSAndrii Nakryiko if (vma->vm_flags & VM_MAYWRITE) { 605fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 606fc970227SAndrii Nakryiko map->writecnt--; 607fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 608fc970227SAndrii Nakryiko } 609fc970227SAndrii Nakryiko } 610fc970227SAndrii Nakryiko 611fc970227SAndrii Nakryiko static const struct vm_operations_struct bpf_map_default_vmops = { 612fc970227SAndrii Nakryiko .open = bpf_map_mmap_open, 613fc970227SAndrii Nakryiko .close = bpf_map_mmap_close, 614fc970227SAndrii Nakryiko }; 615fc970227SAndrii Nakryiko 616fc970227SAndrii Nakryiko static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma) 617fc970227SAndrii Nakryiko { 618fc970227SAndrii Nakryiko struct bpf_map *map = filp->private_data; 619fc970227SAndrii Nakryiko int err; 620fc970227SAndrii Nakryiko 621fc970227SAndrii Nakryiko if (!map->ops->map_mmap || map_value_has_spin_lock(map)) 622fc970227SAndrii Nakryiko return -ENOTSUPP; 623fc970227SAndrii Nakryiko 624fc970227SAndrii Nakryiko if (!(vma->vm_flags & VM_SHARED)) 625fc970227SAndrii Nakryiko return -EINVAL; 626fc970227SAndrii Nakryiko 627fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 628fc970227SAndrii Nakryiko 629dfeb376dSAndrii Nakryiko if (vma->vm_flags & VM_WRITE) { 630dfeb376dSAndrii Nakryiko if (map->frozen) { 631fc970227SAndrii Nakryiko err = -EPERM; 632fc970227SAndrii Nakryiko goto out; 633fc970227SAndrii Nakryiko } 634dfeb376dSAndrii Nakryiko /* map is meant to be read-only, so do not allow mapping as 635dfeb376dSAndrii Nakryiko * writable, because it's possible to leak a writable page 636dfeb376dSAndrii Nakryiko * reference and allows user-space to still modify it after 637dfeb376dSAndrii Nakryiko * freezing, while verifier will assume contents do not change 638dfeb376dSAndrii Nakryiko */ 639dfeb376dSAndrii Nakryiko if (map->map_flags & BPF_F_RDONLY_PROG) { 640dfeb376dSAndrii Nakryiko err = -EACCES; 641dfeb376dSAndrii Nakryiko goto out; 642dfeb376dSAndrii Nakryiko } 643dfeb376dSAndrii Nakryiko } 644fc970227SAndrii Nakryiko 645fc970227SAndrii Nakryiko /* set default open/close callbacks */ 646fc970227SAndrii Nakryiko vma->vm_ops = &bpf_map_default_vmops; 647fc970227SAndrii Nakryiko vma->vm_private_data = map; 6481f6cb19bSAndrii Nakryiko vma->vm_flags &= ~VM_MAYEXEC; 6491f6cb19bSAndrii Nakryiko if (!(vma->vm_flags & VM_WRITE)) 6501f6cb19bSAndrii Nakryiko /* disallow re-mapping with PROT_WRITE */ 6511f6cb19bSAndrii Nakryiko vma->vm_flags &= ~VM_MAYWRITE; 652fc970227SAndrii Nakryiko 653fc970227SAndrii Nakryiko err = map->ops->map_mmap(map, vma); 654fc970227SAndrii Nakryiko if (err) 655fc970227SAndrii Nakryiko goto out; 656fc970227SAndrii Nakryiko 6571f6cb19bSAndrii Nakryiko if (vma->vm_flags & VM_MAYWRITE) 658fc970227SAndrii Nakryiko map->writecnt++; 659fc970227SAndrii Nakryiko out: 660fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 661fc970227SAndrii Nakryiko return err; 662fc970227SAndrii Nakryiko } 663fc970227SAndrii Nakryiko 664457f4436SAndrii Nakryiko static __poll_t bpf_map_poll(struct file *filp, struct poll_table_struct *pts) 665457f4436SAndrii Nakryiko { 666457f4436SAndrii Nakryiko struct bpf_map *map = filp->private_data; 667457f4436SAndrii Nakryiko 668457f4436SAndrii Nakryiko if (map->ops->map_poll) 669457f4436SAndrii Nakryiko return map->ops->map_poll(map, filp, pts); 670457f4436SAndrii Nakryiko 671457f4436SAndrii Nakryiko return EPOLLERR; 672457f4436SAndrii Nakryiko } 673457f4436SAndrii Nakryiko 674f66e448cSChenbo Feng const struct file_operations bpf_map_fops = { 675f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 676f99bf205SDaniel Borkmann .show_fdinfo = bpf_map_show_fdinfo, 677f99bf205SDaniel Borkmann #endif 67899c55f7dSAlexei Starovoitov .release = bpf_map_release, 6796e71b04aSChenbo Feng .read = bpf_dummy_read, 6806e71b04aSChenbo Feng .write = bpf_dummy_write, 681fc970227SAndrii Nakryiko .mmap = bpf_map_mmap, 682457f4436SAndrii Nakryiko .poll = bpf_map_poll, 68399c55f7dSAlexei Starovoitov }; 68499c55f7dSAlexei Starovoitov 6856e71b04aSChenbo Feng int bpf_map_new_fd(struct bpf_map *map, int flags) 686aa79781bSDaniel Borkmann { 687afdb09c7SChenbo Feng int ret; 688afdb09c7SChenbo Feng 689afdb09c7SChenbo Feng ret = security_bpf_map(map, OPEN_FMODE(flags)); 690afdb09c7SChenbo Feng if (ret < 0) 691afdb09c7SChenbo Feng return ret; 692afdb09c7SChenbo Feng 693aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 6946e71b04aSChenbo Feng flags | O_CLOEXEC); 6956e71b04aSChenbo Feng } 6966e71b04aSChenbo Feng 6976e71b04aSChenbo Feng int bpf_get_file_flag(int flags) 6986e71b04aSChenbo Feng { 6996e71b04aSChenbo Feng if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) 7006e71b04aSChenbo Feng return -EINVAL; 7016e71b04aSChenbo Feng if (flags & BPF_F_RDONLY) 7026e71b04aSChenbo Feng return O_RDONLY; 7036e71b04aSChenbo Feng if (flags & BPF_F_WRONLY) 7046e71b04aSChenbo Feng return O_WRONLY; 7056e71b04aSChenbo Feng return O_RDWR; 706aa79781bSDaniel Borkmann } 707aa79781bSDaniel Borkmann 70899c55f7dSAlexei Starovoitov /* helper macro to check that unused fields 'union bpf_attr' are zero */ 70999c55f7dSAlexei Starovoitov #define CHECK_ATTR(CMD) \ 71099c55f7dSAlexei Starovoitov memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 71199c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD), 0, \ 71299c55f7dSAlexei Starovoitov sizeof(*attr) - \ 71399c55f7dSAlexei Starovoitov offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 71499c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD)) != NULL 71599c55f7dSAlexei Starovoitov 7168e7ae251SMartin KaFai Lau /* dst and src must have at least "size" number of bytes. 7178e7ae251SMartin KaFai Lau * Return strlen on success and < 0 on error. 718cb4d2b3fSMartin KaFai Lau */ 7198e7ae251SMartin KaFai Lau int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size) 720cb4d2b3fSMartin KaFai Lau { 7218e7ae251SMartin KaFai Lau const char *end = src + size; 7228e7ae251SMartin KaFai Lau const char *orig_src = src; 723cb4d2b3fSMartin KaFai Lau 7248e7ae251SMartin KaFai Lau memset(dst, 0, size); 7253e0ddc4fSDaniel Borkmann /* Copy all isalnum(), '_' and '.' chars. */ 726cb4d2b3fSMartin KaFai Lau while (src < end && *src) { 7273e0ddc4fSDaniel Borkmann if (!isalnum(*src) && 7283e0ddc4fSDaniel Borkmann *src != '_' && *src != '.') 729cb4d2b3fSMartin KaFai Lau return -EINVAL; 730cb4d2b3fSMartin KaFai Lau *dst++ = *src++; 731cb4d2b3fSMartin KaFai Lau } 732cb4d2b3fSMartin KaFai Lau 7338e7ae251SMartin KaFai Lau /* No '\0' found in "size" number of bytes */ 734cb4d2b3fSMartin KaFai Lau if (src == end) 735cb4d2b3fSMartin KaFai Lau return -EINVAL; 736cb4d2b3fSMartin KaFai Lau 7378e7ae251SMartin KaFai Lau return src - orig_src; 738cb4d2b3fSMartin KaFai Lau } 739cb4d2b3fSMartin KaFai Lau 740e8d2bec0SDaniel Borkmann int map_check_no_btf(const struct bpf_map *map, 7411b2b234bSRoman Gushchin const struct btf *btf, 742e8d2bec0SDaniel Borkmann const struct btf_type *key_type, 743e8d2bec0SDaniel Borkmann const struct btf_type *value_type) 744e8d2bec0SDaniel Borkmann { 745e8d2bec0SDaniel Borkmann return -ENOTSUPP; 746e8d2bec0SDaniel Borkmann } 747e8d2bec0SDaniel Borkmann 748d83525caSAlexei Starovoitov static int map_check_btf(struct bpf_map *map, const struct btf *btf, 749e8d2bec0SDaniel Borkmann u32 btf_key_id, u32 btf_value_id) 750e8d2bec0SDaniel Borkmann { 751e8d2bec0SDaniel Borkmann const struct btf_type *key_type, *value_type; 752e8d2bec0SDaniel Borkmann u32 key_size, value_size; 753e8d2bec0SDaniel Borkmann int ret = 0; 754e8d2bec0SDaniel Borkmann 7552824ecb7SDaniel Borkmann /* Some maps allow key to be unspecified. */ 7562824ecb7SDaniel Borkmann if (btf_key_id) { 757e8d2bec0SDaniel Borkmann key_type = btf_type_id_size(btf, &btf_key_id, &key_size); 758e8d2bec0SDaniel Borkmann if (!key_type || key_size != map->key_size) 759e8d2bec0SDaniel Borkmann return -EINVAL; 7602824ecb7SDaniel Borkmann } else { 7612824ecb7SDaniel Borkmann key_type = btf_type_by_id(btf, 0); 7622824ecb7SDaniel Borkmann if (!map->ops->map_check_btf) 7632824ecb7SDaniel Borkmann return -EINVAL; 7642824ecb7SDaniel Borkmann } 765e8d2bec0SDaniel Borkmann 766e8d2bec0SDaniel Borkmann value_type = btf_type_id_size(btf, &btf_value_id, &value_size); 767e8d2bec0SDaniel Borkmann if (!value_type || value_size != map->value_size) 768e8d2bec0SDaniel Borkmann return -EINVAL; 769e8d2bec0SDaniel Borkmann 770d83525caSAlexei Starovoitov map->spin_lock_off = btf_find_spin_lock(btf, value_type); 771d83525caSAlexei Starovoitov 772d83525caSAlexei Starovoitov if (map_value_has_spin_lock(map)) { 773591fe988SDaniel Borkmann if (map->map_flags & BPF_F_RDONLY_PROG) 774591fe988SDaniel Borkmann return -EACCES; 775d83525caSAlexei Starovoitov if (map->map_type != BPF_MAP_TYPE_HASH && 776e16d2f1aSAlexei Starovoitov map->map_type != BPF_MAP_TYPE_ARRAY && 7776ac99e8fSMartin KaFai Lau map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 7788ea63684SKP Singh map->map_type != BPF_MAP_TYPE_SK_STORAGE && 7794cf1bc1fSKP Singh map->map_type != BPF_MAP_TYPE_INODE_STORAGE && 7804cf1bc1fSKP Singh map->map_type != BPF_MAP_TYPE_TASK_STORAGE) 781d83525caSAlexei Starovoitov return -ENOTSUPP; 782d83525caSAlexei Starovoitov if (map->spin_lock_off + sizeof(struct bpf_spin_lock) > 783d83525caSAlexei Starovoitov map->value_size) { 784d83525caSAlexei Starovoitov WARN_ONCE(1, 785d83525caSAlexei Starovoitov "verifier bug spin_lock_off %d value_size %d\n", 786d83525caSAlexei Starovoitov map->spin_lock_off, map->value_size); 787d83525caSAlexei Starovoitov return -EFAULT; 788d83525caSAlexei Starovoitov } 789d83525caSAlexei Starovoitov } 790d83525caSAlexei Starovoitov 791e8d2bec0SDaniel Borkmann if (map->ops->map_check_btf) 7921b2b234bSRoman Gushchin ret = map->ops->map_check_btf(map, btf, key_type, value_type); 793e8d2bec0SDaniel Borkmann 794e8d2bec0SDaniel Borkmann return ret; 795e8d2bec0SDaniel Borkmann } 796e8d2bec0SDaniel Borkmann 79785d33df3SMartin KaFai Lau #define BPF_MAP_CREATE_LAST_FIELD btf_vmlinux_value_type_id 79899c55f7dSAlexei Starovoitov /* called via syscall */ 79999c55f7dSAlexei Starovoitov static int map_create(union bpf_attr *attr) 80099c55f7dSAlexei Starovoitov { 80196eabe7aSMartin KaFai Lau int numa_node = bpf_map_attr_numa_node(attr); 80299c55f7dSAlexei Starovoitov struct bpf_map *map; 8036e71b04aSChenbo Feng int f_flags; 80499c55f7dSAlexei Starovoitov int err; 80599c55f7dSAlexei Starovoitov 80699c55f7dSAlexei Starovoitov err = CHECK_ATTR(BPF_MAP_CREATE); 80799c55f7dSAlexei Starovoitov if (err) 80899c55f7dSAlexei Starovoitov return -EINVAL; 80999c55f7dSAlexei Starovoitov 81085d33df3SMartin KaFai Lau if (attr->btf_vmlinux_value_type_id) { 81185d33df3SMartin KaFai Lau if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS || 81285d33df3SMartin KaFai Lau attr->btf_key_type_id || attr->btf_value_type_id) 81385d33df3SMartin KaFai Lau return -EINVAL; 81485d33df3SMartin KaFai Lau } else if (attr->btf_key_type_id && !attr->btf_value_type_id) { 81585d33df3SMartin KaFai Lau return -EINVAL; 81685d33df3SMartin KaFai Lau } 81785d33df3SMartin KaFai Lau 8186e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->map_flags); 8196e71b04aSChenbo Feng if (f_flags < 0) 8206e71b04aSChenbo Feng return f_flags; 8216e71b04aSChenbo Feng 82296eabe7aSMartin KaFai Lau if (numa_node != NUMA_NO_NODE && 82396e5ae4eSEric Dumazet ((unsigned int)numa_node >= nr_node_ids || 82496e5ae4eSEric Dumazet !node_online(numa_node))) 82596eabe7aSMartin KaFai Lau return -EINVAL; 82696eabe7aSMartin KaFai Lau 82799c55f7dSAlexei Starovoitov /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 82899c55f7dSAlexei Starovoitov map = find_and_alloc_map(attr); 82999c55f7dSAlexei Starovoitov if (IS_ERR(map)) 83099c55f7dSAlexei Starovoitov return PTR_ERR(map); 83199c55f7dSAlexei Starovoitov 8328e7ae251SMartin KaFai Lau err = bpf_obj_name_cpy(map->name, attr->map_name, 8338e7ae251SMartin KaFai Lau sizeof(attr->map_name)); 8348e7ae251SMartin KaFai Lau if (err < 0) 835b936ca64SRoman Gushchin goto free_map; 836ad5b177bSMartin KaFai Lau 8371e0bd5a0SAndrii Nakryiko atomic64_set(&map->refcnt, 1); 8381e0bd5a0SAndrii Nakryiko atomic64_set(&map->usercnt, 1); 839fc970227SAndrii Nakryiko mutex_init(&map->freeze_mutex); 84099c55f7dSAlexei Starovoitov 84185d33df3SMartin KaFai Lau map->spin_lock_off = -EINVAL; 84285d33df3SMartin KaFai Lau if (attr->btf_key_type_id || attr->btf_value_type_id || 84385d33df3SMartin KaFai Lau /* Even the map's value is a kernel's struct, 84485d33df3SMartin KaFai Lau * the bpf_prog.o must have BTF to begin with 84585d33df3SMartin KaFai Lau * to figure out the corresponding kernel's 84685d33df3SMartin KaFai Lau * counter part. Thus, attr->btf_fd has 84785d33df3SMartin KaFai Lau * to be valid also. 84885d33df3SMartin KaFai Lau */ 84985d33df3SMartin KaFai Lau attr->btf_vmlinux_value_type_id) { 850a26ca7c9SMartin KaFai Lau struct btf *btf; 851a26ca7c9SMartin KaFai Lau 852a26ca7c9SMartin KaFai Lau btf = btf_get_by_fd(attr->btf_fd); 853a26ca7c9SMartin KaFai Lau if (IS_ERR(btf)) { 854a26ca7c9SMartin KaFai Lau err = PTR_ERR(btf); 855b936ca64SRoman Gushchin goto free_map; 856a26ca7c9SMartin KaFai Lau } 857350a5c4dSAlexei Starovoitov if (btf_is_kernel(btf)) { 858350a5c4dSAlexei Starovoitov btf_put(btf); 859350a5c4dSAlexei Starovoitov err = -EACCES; 860350a5c4dSAlexei Starovoitov goto free_map; 861350a5c4dSAlexei Starovoitov } 86285d33df3SMartin KaFai Lau map->btf = btf; 863a26ca7c9SMartin KaFai Lau 86485d33df3SMartin KaFai Lau if (attr->btf_value_type_id) { 865e8d2bec0SDaniel Borkmann err = map_check_btf(map, btf, attr->btf_key_type_id, 8669b2cf328SMartin KaFai Lau attr->btf_value_type_id); 86785d33df3SMartin KaFai Lau if (err) 868b936ca64SRoman Gushchin goto free_map; 869a26ca7c9SMartin KaFai Lau } 870a26ca7c9SMartin KaFai Lau 8719b2cf328SMartin KaFai Lau map->btf_key_type_id = attr->btf_key_type_id; 8729b2cf328SMartin KaFai Lau map->btf_value_type_id = attr->btf_value_type_id; 87385d33df3SMartin KaFai Lau map->btf_vmlinux_value_type_id = 87485d33df3SMartin KaFai Lau attr->btf_vmlinux_value_type_id; 875a26ca7c9SMartin KaFai Lau } 876a26ca7c9SMartin KaFai Lau 877afdb09c7SChenbo Feng err = security_bpf_map_alloc(map); 878aaac3ba9SAlexei Starovoitov if (err) 879b936ca64SRoman Gushchin goto free_map; 880afdb09c7SChenbo Feng 881f3f1c054SMartin KaFai Lau err = bpf_map_alloc_id(map); 882f3f1c054SMartin KaFai Lau if (err) 883b936ca64SRoman Gushchin goto free_map_sec; 884f3f1c054SMartin KaFai Lau 88548edc1f7SRoman Gushchin bpf_map_save_memcg(map); 88648edc1f7SRoman Gushchin 8876e71b04aSChenbo Feng err = bpf_map_new_fd(map, f_flags); 888bd5f5f4eSMartin KaFai Lau if (err < 0) { 889bd5f5f4eSMartin KaFai Lau /* failed to allocate fd. 890352d20d6SPeng Sun * bpf_map_put_with_uref() is needed because the above 891bd5f5f4eSMartin KaFai Lau * bpf_map_alloc_id() has published the map 892bd5f5f4eSMartin KaFai Lau * to the userspace and the userspace may 893bd5f5f4eSMartin KaFai Lau * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 894bd5f5f4eSMartin KaFai Lau */ 895352d20d6SPeng Sun bpf_map_put_with_uref(map); 896bd5f5f4eSMartin KaFai Lau return err; 897bd5f5f4eSMartin KaFai Lau } 89899c55f7dSAlexei Starovoitov 89999c55f7dSAlexei Starovoitov return err; 90099c55f7dSAlexei Starovoitov 901afdb09c7SChenbo Feng free_map_sec: 902afdb09c7SChenbo Feng security_bpf_map_free(map); 903b936ca64SRoman Gushchin free_map: 904a26ca7c9SMartin KaFai Lau btf_put(map->btf); 90599c55f7dSAlexei Starovoitov map->ops->map_free(map); 90699c55f7dSAlexei Starovoitov return err; 90799c55f7dSAlexei Starovoitov } 90899c55f7dSAlexei Starovoitov 909db20fd2bSAlexei Starovoitov /* if error is returned, fd is released. 910db20fd2bSAlexei Starovoitov * On success caller should complete fd access with matching fdput() 911db20fd2bSAlexei Starovoitov */ 912c2101297SDaniel Borkmann struct bpf_map *__bpf_map_get(struct fd f) 913db20fd2bSAlexei Starovoitov { 914db20fd2bSAlexei Starovoitov if (!f.file) 915db20fd2bSAlexei Starovoitov return ERR_PTR(-EBADF); 916db20fd2bSAlexei Starovoitov if (f.file->f_op != &bpf_map_fops) { 917db20fd2bSAlexei Starovoitov fdput(f); 918db20fd2bSAlexei Starovoitov return ERR_PTR(-EINVAL); 919db20fd2bSAlexei Starovoitov } 920db20fd2bSAlexei Starovoitov 921c2101297SDaniel Borkmann return f.file->private_data; 922c2101297SDaniel Borkmann } 923c2101297SDaniel Borkmann 9241e0bd5a0SAndrii Nakryiko void bpf_map_inc(struct bpf_map *map) 925c9da161cSDaniel Borkmann { 9261e0bd5a0SAndrii Nakryiko atomic64_inc(&map->refcnt); 927c9da161cSDaniel Borkmann } 928630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_inc); 929c9da161cSDaniel Borkmann 9301e0bd5a0SAndrii Nakryiko void bpf_map_inc_with_uref(struct bpf_map *map) 9311e0bd5a0SAndrii Nakryiko { 9321e0bd5a0SAndrii Nakryiko atomic64_inc(&map->refcnt); 9331e0bd5a0SAndrii Nakryiko atomic64_inc(&map->usercnt); 9341e0bd5a0SAndrii Nakryiko } 9351e0bd5a0SAndrii Nakryiko EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref); 9361e0bd5a0SAndrii Nakryiko 9371ed4d924SMartin KaFai Lau struct bpf_map *bpf_map_get(u32 ufd) 9381ed4d924SMartin KaFai Lau { 9391ed4d924SMartin KaFai Lau struct fd f = fdget(ufd); 9401ed4d924SMartin KaFai Lau struct bpf_map *map; 9411ed4d924SMartin KaFai Lau 9421ed4d924SMartin KaFai Lau map = __bpf_map_get(f); 9431ed4d924SMartin KaFai Lau if (IS_ERR(map)) 9441ed4d924SMartin KaFai Lau return map; 9451ed4d924SMartin KaFai Lau 9461ed4d924SMartin KaFai Lau bpf_map_inc(map); 9471ed4d924SMartin KaFai Lau fdput(f); 9481ed4d924SMartin KaFai Lau 9491ed4d924SMartin KaFai Lau return map; 9501ed4d924SMartin KaFai Lau } 9511ed4d924SMartin KaFai Lau 952c9da161cSDaniel Borkmann struct bpf_map *bpf_map_get_with_uref(u32 ufd) 953c2101297SDaniel Borkmann { 954c2101297SDaniel Borkmann struct fd f = fdget(ufd); 955c2101297SDaniel Borkmann struct bpf_map *map; 956c2101297SDaniel Borkmann 957c2101297SDaniel Borkmann map = __bpf_map_get(f); 958c2101297SDaniel Borkmann if (IS_ERR(map)) 959c2101297SDaniel Borkmann return map; 960c2101297SDaniel Borkmann 9611e0bd5a0SAndrii Nakryiko bpf_map_inc_with_uref(map); 962c2101297SDaniel Borkmann fdput(f); 963db20fd2bSAlexei Starovoitov 964db20fd2bSAlexei Starovoitov return map; 965db20fd2bSAlexei Starovoitov } 966db20fd2bSAlexei Starovoitov 967bd5f5f4eSMartin KaFai Lau /* map_idr_lock should have been held */ 9681e0bd5a0SAndrii Nakryiko static struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref) 969bd5f5f4eSMartin KaFai Lau { 970bd5f5f4eSMartin KaFai Lau int refold; 971bd5f5f4eSMartin KaFai Lau 9721e0bd5a0SAndrii Nakryiko refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0); 973bd5f5f4eSMartin KaFai Lau if (!refold) 974bd5f5f4eSMartin KaFai Lau return ERR_PTR(-ENOENT); 975bd5f5f4eSMartin KaFai Lau if (uref) 9761e0bd5a0SAndrii Nakryiko atomic64_inc(&map->usercnt); 977bd5f5f4eSMartin KaFai Lau 978bd5f5f4eSMartin KaFai Lau return map; 979bd5f5f4eSMartin KaFai Lau } 980bd5f5f4eSMartin KaFai Lau 9811e0bd5a0SAndrii Nakryiko struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map) 982b0e4701cSStanislav Fomichev { 983b0e4701cSStanislav Fomichev spin_lock_bh(&map_idr_lock); 9841e0bd5a0SAndrii Nakryiko map = __bpf_map_inc_not_zero(map, false); 985b0e4701cSStanislav Fomichev spin_unlock_bh(&map_idr_lock); 986b0e4701cSStanislav Fomichev 987b0e4701cSStanislav Fomichev return map; 988b0e4701cSStanislav Fomichev } 989b0e4701cSStanislav Fomichev EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero); 990b0e4701cSStanislav Fomichev 991b8cdc051SAlexei Starovoitov int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 992b8cdc051SAlexei Starovoitov { 993b8cdc051SAlexei Starovoitov return -ENOTSUPP; 994b8cdc051SAlexei Starovoitov } 995b8cdc051SAlexei Starovoitov 996c9d29f46SMauricio Vasquez B static void *__bpf_copy_key(void __user *ukey, u64 key_size) 997c9d29f46SMauricio Vasquez B { 998c9d29f46SMauricio Vasquez B if (key_size) 999c9d29f46SMauricio Vasquez B return memdup_user(ukey, key_size); 1000c9d29f46SMauricio Vasquez B 1001c9d29f46SMauricio Vasquez B if (ukey) 1002c9d29f46SMauricio Vasquez B return ERR_PTR(-EINVAL); 1003c9d29f46SMauricio Vasquez B 1004c9d29f46SMauricio Vasquez B return NULL; 1005c9d29f46SMauricio Vasquez B } 1006c9d29f46SMauricio Vasquez B 1007db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 100896049f3aSAlexei Starovoitov #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags 1009db20fd2bSAlexei Starovoitov 1010db20fd2bSAlexei Starovoitov static int map_lookup_elem(union bpf_attr *attr) 1011db20fd2bSAlexei Starovoitov { 1012535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1013535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 1014db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1015db20fd2bSAlexei Starovoitov struct bpf_map *map; 101615c14a3dSBrian Vazquez void *key, *value; 101715a07b33SAlexei Starovoitov u32 value_size; 1018592867bfSDaniel Borkmann struct fd f; 1019db20fd2bSAlexei Starovoitov int err; 1020db20fd2bSAlexei Starovoitov 1021db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 1022db20fd2bSAlexei Starovoitov return -EINVAL; 1023db20fd2bSAlexei Starovoitov 102496049f3aSAlexei Starovoitov if (attr->flags & ~BPF_F_LOCK) 102596049f3aSAlexei Starovoitov return -EINVAL; 102696049f3aSAlexei Starovoitov 1027592867bfSDaniel Borkmann f = fdget(ufd); 1028c2101297SDaniel Borkmann map = __bpf_map_get(f); 1029db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1030db20fd2bSAlexei Starovoitov return PTR_ERR(map); 103187df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 10326e71b04aSChenbo Feng err = -EPERM; 10336e71b04aSChenbo Feng goto err_put; 10346e71b04aSChenbo Feng } 10356e71b04aSChenbo Feng 103696049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 103796049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 103896049f3aSAlexei Starovoitov err = -EINVAL; 103996049f3aSAlexei Starovoitov goto err_put; 104096049f3aSAlexei Starovoitov } 104196049f3aSAlexei Starovoitov 1042c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1043e4448ed8SAl Viro if (IS_ERR(key)) { 1044e4448ed8SAl Viro err = PTR_ERR(key); 1045db20fd2bSAlexei Starovoitov goto err_put; 1046e4448ed8SAl Viro } 1047db20fd2bSAlexei Starovoitov 104815c14a3dSBrian Vazquez value_size = bpf_map_value_size(map); 104915a07b33SAlexei Starovoitov 10508ebe667cSAlexei Starovoitov err = -ENOMEM; 105115a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1052db20fd2bSAlexei Starovoitov if (!value) 10538ebe667cSAlexei Starovoitov goto free_key; 10548ebe667cSAlexei Starovoitov 105515c14a3dSBrian Vazquez err = bpf_map_copy_value(map, key, value, attr->flags); 105615a07b33SAlexei Starovoitov if (err) 10578ebe667cSAlexei Starovoitov goto free_value; 1058db20fd2bSAlexei Starovoitov 1059db20fd2bSAlexei Starovoitov err = -EFAULT; 106015a07b33SAlexei Starovoitov if (copy_to_user(uvalue, value, value_size) != 0) 10618ebe667cSAlexei Starovoitov goto free_value; 1062db20fd2bSAlexei Starovoitov 1063db20fd2bSAlexei Starovoitov err = 0; 1064db20fd2bSAlexei Starovoitov 10658ebe667cSAlexei Starovoitov free_value: 10668ebe667cSAlexei Starovoitov kfree(value); 1067db20fd2bSAlexei Starovoitov free_key: 1068db20fd2bSAlexei Starovoitov kfree(key); 1069db20fd2bSAlexei Starovoitov err_put: 1070db20fd2bSAlexei Starovoitov fdput(f); 1071db20fd2bSAlexei Starovoitov return err; 1072db20fd2bSAlexei Starovoitov } 1073db20fd2bSAlexei Starovoitov 10741ae80cf3SDaniel Colascione 10753274f520SAlexei Starovoitov #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 1076db20fd2bSAlexei Starovoitov 1077db20fd2bSAlexei Starovoitov static int map_update_elem(union bpf_attr *attr) 1078db20fd2bSAlexei Starovoitov { 1079535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1080535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 1081db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1082db20fd2bSAlexei Starovoitov struct bpf_map *map; 1083db20fd2bSAlexei Starovoitov void *key, *value; 108415a07b33SAlexei Starovoitov u32 value_size; 1085592867bfSDaniel Borkmann struct fd f; 1086db20fd2bSAlexei Starovoitov int err; 1087db20fd2bSAlexei Starovoitov 1088db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 1089db20fd2bSAlexei Starovoitov return -EINVAL; 1090db20fd2bSAlexei Starovoitov 1091592867bfSDaniel Borkmann f = fdget(ufd); 1092c2101297SDaniel Borkmann map = __bpf_map_get(f); 1093db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1094db20fd2bSAlexei Starovoitov return PTR_ERR(map); 109587df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 10966e71b04aSChenbo Feng err = -EPERM; 10976e71b04aSChenbo Feng goto err_put; 10986e71b04aSChenbo Feng } 10996e71b04aSChenbo Feng 110096049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 110196049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 110296049f3aSAlexei Starovoitov err = -EINVAL; 110396049f3aSAlexei Starovoitov goto err_put; 110496049f3aSAlexei Starovoitov } 110596049f3aSAlexei Starovoitov 1106c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1107e4448ed8SAl Viro if (IS_ERR(key)) { 1108e4448ed8SAl Viro err = PTR_ERR(key); 1109db20fd2bSAlexei Starovoitov goto err_put; 1110e4448ed8SAl Viro } 1111db20fd2bSAlexei Starovoitov 111215a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 11138f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 1114b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 1115b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 111615a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 111715a07b33SAlexei Starovoitov else 111815a07b33SAlexei Starovoitov value_size = map->value_size; 111915a07b33SAlexei Starovoitov 1120db20fd2bSAlexei Starovoitov err = -ENOMEM; 112115a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1122db20fd2bSAlexei Starovoitov if (!value) 1123db20fd2bSAlexei Starovoitov goto free_key; 1124db20fd2bSAlexei Starovoitov 1125db20fd2bSAlexei Starovoitov err = -EFAULT; 112615a07b33SAlexei Starovoitov if (copy_from_user(value, uvalue, value_size) != 0) 1127db20fd2bSAlexei Starovoitov goto free_value; 1128db20fd2bSAlexei Starovoitov 112915c14a3dSBrian Vazquez err = bpf_map_update_value(map, f, key, value, attr->flags); 11306710e112SJesper Dangaard Brouer 1131db20fd2bSAlexei Starovoitov free_value: 1132db20fd2bSAlexei Starovoitov kfree(value); 1133db20fd2bSAlexei Starovoitov free_key: 1134db20fd2bSAlexei Starovoitov kfree(key); 1135db20fd2bSAlexei Starovoitov err_put: 1136db20fd2bSAlexei Starovoitov fdput(f); 1137db20fd2bSAlexei Starovoitov return err; 1138db20fd2bSAlexei Starovoitov } 1139db20fd2bSAlexei Starovoitov 1140db20fd2bSAlexei Starovoitov #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 1141db20fd2bSAlexei Starovoitov 1142db20fd2bSAlexei Starovoitov static int map_delete_elem(union bpf_attr *attr) 1143db20fd2bSAlexei Starovoitov { 1144535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1145db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1146db20fd2bSAlexei Starovoitov struct bpf_map *map; 1147592867bfSDaniel Borkmann struct fd f; 1148db20fd2bSAlexei Starovoitov void *key; 1149db20fd2bSAlexei Starovoitov int err; 1150db20fd2bSAlexei Starovoitov 1151db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 1152db20fd2bSAlexei Starovoitov return -EINVAL; 1153db20fd2bSAlexei Starovoitov 1154592867bfSDaniel Borkmann f = fdget(ufd); 1155c2101297SDaniel Borkmann map = __bpf_map_get(f); 1156db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1157db20fd2bSAlexei Starovoitov return PTR_ERR(map); 115887df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 11596e71b04aSChenbo Feng err = -EPERM; 11606e71b04aSChenbo Feng goto err_put; 11616e71b04aSChenbo Feng } 11626e71b04aSChenbo Feng 1163c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1164e4448ed8SAl Viro if (IS_ERR(key)) { 1165e4448ed8SAl Viro err = PTR_ERR(key); 1166db20fd2bSAlexei Starovoitov goto err_put; 1167e4448ed8SAl Viro } 1168db20fd2bSAlexei Starovoitov 1169a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1170a3884572SJakub Kicinski err = bpf_map_offload_delete_elem(map, key); 1171a3884572SJakub Kicinski goto out; 117285d33df3SMartin KaFai Lau } else if (IS_FD_PROG_ARRAY(map) || 117385d33df3SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 117485d33df3SMartin KaFai Lau /* These maps require sleepable context */ 1175da765a2fSDaniel Borkmann err = map->ops->map_delete_elem(map, key); 1176da765a2fSDaniel Borkmann goto out; 1177a3884572SJakub Kicinski } 1178a3884572SJakub Kicinski 1179b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 1180db20fd2bSAlexei Starovoitov rcu_read_lock(); 1181db20fd2bSAlexei Starovoitov err = map->ops->map_delete_elem(map, key); 1182db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1183b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 11841ae80cf3SDaniel Colascione maybe_wait_bpf_programs(map); 1185a3884572SJakub Kicinski out: 1186db20fd2bSAlexei Starovoitov kfree(key); 1187db20fd2bSAlexei Starovoitov err_put: 1188db20fd2bSAlexei Starovoitov fdput(f); 1189db20fd2bSAlexei Starovoitov return err; 1190db20fd2bSAlexei Starovoitov } 1191db20fd2bSAlexei Starovoitov 1192db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 1193db20fd2bSAlexei Starovoitov #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 1194db20fd2bSAlexei Starovoitov 1195db20fd2bSAlexei Starovoitov static int map_get_next_key(union bpf_attr *attr) 1196db20fd2bSAlexei Starovoitov { 1197535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1198535e7b4bSMickaël Salaün void __user *unext_key = u64_to_user_ptr(attr->next_key); 1199db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1200db20fd2bSAlexei Starovoitov struct bpf_map *map; 1201db20fd2bSAlexei Starovoitov void *key, *next_key; 1202592867bfSDaniel Borkmann struct fd f; 1203db20fd2bSAlexei Starovoitov int err; 1204db20fd2bSAlexei Starovoitov 1205db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 1206db20fd2bSAlexei Starovoitov return -EINVAL; 1207db20fd2bSAlexei Starovoitov 1208592867bfSDaniel Borkmann f = fdget(ufd); 1209c2101297SDaniel Borkmann map = __bpf_map_get(f); 1210db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1211db20fd2bSAlexei Starovoitov return PTR_ERR(map); 121287df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 12136e71b04aSChenbo Feng err = -EPERM; 12146e71b04aSChenbo Feng goto err_put; 12156e71b04aSChenbo Feng } 12166e71b04aSChenbo Feng 12178fe45924STeng Qin if (ukey) { 1218c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1219e4448ed8SAl Viro if (IS_ERR(key)) { 1220e4448ed8SAl Viro err = PTR_ERR(key); 1221db20fd2bSAlexei Starovoitov goto err_put; 1222e4448ed8SAl Viro } 12238fe45924STeng Qin } else { 12248fe45924STeng Qin key = NULL; 12258fe45924STeng Qin } 1226db20fd2bSAlexei Starovoitov 1227db20fd2bSAlexei Starovoitov err = -ENOMEM; 1228db20fd2bSAlexei Starovoitov next_key = kmalloc(map->key_size, GFP_USER); 1229db20fd2bSAlexei Starovoitov if (!next_key) 1230db20fd2bSAlexei Starovoitov goto free_key; 1231db20fd2bSAlexei Starovoitov 1232a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1233a3884572SJakub Kicinski err = bpf_map_offload_get_next_key(map, key, next_key); 1234a3884572SJakub Kicinski goto out; 1235a3884572SJakub Kicinski } 1236a3884572SJakub Kicinski 1237db20fd2bSAlexei Starovoitov rcu_read_lock(); 1238db20fd2bSAlexei Starovoitov err = map->ops->map_get_next_key(map, key, next_key); 1239db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1240a3884572SJakub Kicinski out: 1241db20fd2bSAlexei Starovoitov if (err) 1242db20fd2bSAlexei Starovoitov goto free_next_key; 1243db20fd2bSAlexei Starovoitov 1244db20fd2bSAlexei Starovoitov err = -EFAULT; 1245db20fd2bSAlexei Starovoitov if (copy_to_user(unext_key, next_key, map->key_size) != 0) 1246db20fd2bSAlexei Starovoitov goto free_next_key; 1247db20fd2bSAlexei Starovoitov 1248db20fd2bSAlexei Starovoitov err = 0; 1249db20fd2bSAlexei Starovoitov 1250db20fd2bSAlexei Starovoitov free_next_key: 1251db20fd2bSAlexei Starovoitov kfree(next_key); 1252db20fd2bSAlexei Starovoitov free_key: 1253db20fd2bSAlexei Starovoitov kfree(key); 1254db20fd2bSAlexei Starovoitov err_put: 1255db20fd2bSAlexei Starovoitov fdput(f); 1256db20fd2bSAlexei Starovoitov return err; 1257db20fd2bSAlexei Starovoitov } 1258db20fd2bSAlexei Starovoitov 1259aa2e93b8SBrian Vazquez int generic_map_delete_batch(struct bpf_map *map, 1260aa2e93b8SBrian Vazquez const union bpf_attr *attr, 1261aa2e93b8SBrian Vazquez union bpf_attr __user *uattr) 1262aa2e93b8SBrian Vazquez { 1263aa2e93b8SBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1264aa2e93b8SBrian Vazquez u32 cp, max_count; 1265aa2e93b8SBrian Vazquez int err = 0; 1266aa2e93b8SBrian Vazquez void *key; 1267aa2e93b8SBrian Vazquez 1268aa2e93b8SBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1269aa2e93b8SBrian Vazquez return -EINVAL; 1270aa2e93b8SBrian Vazquez 1271aa2e93b8SBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1272aa2e93b8SBrian Vazquez !map_value_has_spin_lock(map)) { 1273aa2e93b8SBrian Vazquez return -EINVAL; 1274aa2e93b8SBrian Vazquez } 1275aa2e93b8SBrian Vazquez 1276aa2e93b8SBrian Vazquez max_count = attr->batch.count; 1277aa2e93b8SBrian Vazquez if (!max_count) 1278aa2e93b8SBrian Vazquez return 0; 1279aa2e93b8SBrian Vazquez 12802e3a94aaSBrian Vazquez key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 12812e3a94aaSBrian Vazquez if (!key) 12822e3a94aaSBrian Vazquez return -ENOMEM; 12832e3a94aaSBrian Vazquez 1284aa2e93b8SBrian Vazquez for (cp = 0; cp < max_count; cp++) { 12852e3a94aaSBrian Vazquez err = -EFAULT; 12862e3a94aaSBrian Vazquez if (copy_from_user(key, keys + cp * map->key_size, 12872e3a94aaSBrian Vazquez map->key_size)) 1288aa2e93b8SBrian Vazquez break; 1289aa2e93b8SBrian Vazquez 1290aa2e93b8SBrian Vazquez if (bpf_map_is_dev_bound(map)) { 1291aa2e93b8SBrian Vazquez err = bpf_map_offload_delete_elem(map, key); 1292aa2e93b8SBrian Vazquez break; 1293aa2e93b8SBrian Vazquez } 1294aa2e93b8SBrian Vazquez 1295b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 1296aa2e93b8SBrian Vazquez rcu_read_lock(); 1297aa2e93b8SBrian Vazquez err = map->ops->map_delete_elem(map, key); 1298aa2e93b8SBrian Vazquez rcu_read_unlock(); 1299b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 1300aa2e93b8SBrian Vazquez maybe_wait_bpf_programs(map); 1301aa2e93b8SBrian Vazquez if (err) 1302aa2e93b8SBrian Vazquez break; 1303aa2e93b8SBrian Vazquez } 1304aa2e93b8SBrian Vazquez if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) 1305aa2e93b8SBrian Vazquez err = -EFAULT; 13062e3a94aaSBrian Vazquez 13072e3a94aaSBrian Vazquez kfree(key); 1308aa2e93b8SBrian Vazquez return err; 1309aa2e93b8SBrian Vazquez } 1310aa2e93b8SBrian Vazquez 1311aa2e93b8SBrian Vazquez int generic_map_update_batch(struct bpf_map *map, 1312aa2e93b8SBrian Vazquez const union bpf_attr *attr, 1313aa2e93b8SBrian Vazquez union bpf_attr __user *uattr) 1314aa2e93b8SBrian Vazquez { 1315aa2e93b8SBrian Vazquez void __user *values = u64_to_user_ptr(attr->batch.values); 1316aa2e93b8SBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1317aa2e93b8SBrian Vazquez u32 value_size, cp, max_count; 1318aa2e93b8SBrian Vazquez int ufd = attr->map_fd; 1319aa2e93b8SBrian Vazquez void *key, *value; 1320aa2e93b8SBrian Vazquez struct fd f; 1321aa2e93b8SBrian Vazquez int err = 0; 1322aa2e93b8SBrian Vazquez 1323aa2e93b8SBrian Vazquez f = fdget(ufd); 1324aa2e93b8SBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1325aa2e93b8SBrian Vazquez return -EINVAL; 1326aa2e93b8SBrian Vazquez 1327aa2e93b8SBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1328aa2e93b8SBrian Vazquez !map_value_has_spin_lock(map)) { 1329aa2e93b8SBrian Vazquez return -EINVAL; 1330aa2e93b8SBrian Vazquez } 1331aa2e93b8SBrian Vazquez 1332aa2e93b8SBrian Vazquez value_size = bpf_map_value_size(map); 1333aa2e93b8SBrian Vazquez 1334aa2e93b8SBrian Vazquez max_count = attr->batch.count; 1335aa2e93b8SBrian Vazquez if (!max_count) 1336aa2e93b8SBrian Vazquez return 0; 1337aa2e93b8SBrian Vazquez 13382e3a94aaSBrian Vazquez key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 13392e3a94aaSBrian Vazquez if (!key) 1340aa2e93b8SBrian Vazquez return -ENOMEM; 1341aa2e93b8SBrian Vazquez 13422e3a94aaSBrian Vazquez value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 13432e3a94aaSBrian Vazquez if (!value) { 13442e3a94aaSBrian Vazquez kfree(key); 13452e3a94aaSBrian Vazquez return -ENOMEM; 1346aa2e93b8SBrian Vazquez } 13472e3a94aaSBrian Vazquez 13482e3a94aaSBrian Vazquez for (cp = 0; cp < max_count; cp++) { 1349aa2e93b8SBrian Vazquez err = -EFAULT; 13502e3a94aaSBrian Vazquez if (copy_from_user(key, keys + cp * map->key_size, 13512e3a94aaSBrian Vazquez map->key_size) || 13522e3a94aaSBrian Vazquez copy_from_user(value, values + cp * value_size, value_size)) 1353aa2e93b8SBrian Vazquez break; 1354aa2e93b8SBrian Vazquez 1355aa2e93b8SBrian Vazquez err = bpf_map_update_value(map, f, key, value, 1356aa2e93b8SBrian Vazquez attr->batch.elem_flags); 1357aa2e93b8SBrian Vazquez 1358aa2e93b8SBrian Vazquez if (err) 1359aa2e93b8SBrian Vazquez break; 1360aa2e93b8SBrian Vazquez } 1361aa2e93b8SBrian Vazquez 1362aa2e93b8SBrian Vazquez if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) 1363aa2e93b8SBrian Vazquez err = -EFAULT; 1364aa2e93b8SBrian Vazquez 1365aa2e93b8SBrian Vazquez kfree(value); 1366aa2e93b8SBrian Vazquez kfree(key); 1367aa2e93b8SBrian Vazquez return err; 1368aa2e93b8SBrian Vazquez } 1369aa2e93b8SBrian Vazquez 1370cb4d03abSBrian Vazquez #define MAP_LOOKUP_RETRIES 3 1371cb4d03abSBrian Vazquez 1372cb4d03abSBrian Vazquez int generic_map_lookup_batch(struct bpf_map *map, 1373cb4d03abSBrian Vazquez const union bpf_attr *attr, 1374cb4d03abSBrian Vazquez union bpf_attr __user *uattr) 1375cb4d03abSBrian Vazquez { 1376cb4d03abSBrian Vazquez void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch); 1377cb4d03abSBrian Vazquez void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch); 1378cb4d03abSBrian Vazquez void __user *values = u64_to_user_ptr(attr->batch.values); 1379cb4d03abSBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1380cb4d03abSBrian Vazquez void *buf, *buf_prevkey, *prev_key, *key, *value; 1381cb4d03abSBrian Vazquez int err, retry = MAP_LOOKUP_RETRIES; 1382cb4d03abSBrian Vazquez u32 value_size, cp, max_count; 1383cb4d03abSBrian Vazquez 1384cb4d03abSBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1385cb4d03abSBrian Vazquez return -EINVAL; 1386cb4d03abSBrian Vazquez 1387cb4d03abSBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1388cb4d03abSBrian Vazquez !map_value_has_spin_lock(map)) 1389cb4d03abSBrian Vazquez return -EINVAL; 1390cb4d03abSBrian Vazquez 1391cb4d03abSBrian Vazquez value_size = bpf_map_value_size(map); 1392cb4d03abSBrian Vazquez 1393cb4d03abSBrian Vazquez max_count = attr->batch.count; 1394cb4d03abSBrian Vazquez if (!max_count) 1395cb4d03abSBrian Vazquez return 0; 1396cb4d03abSBrian Vazquez 1397cb4d03abSBrian Vazquez if (put_user(0, &uattr->batch.count)) 1398cb4d03abSBrian Vazquez return -EFAULT; 1399cb4d03abSBrian Vazquez 1400cb4d03abSBrian Vazquez buf_prevkey = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 1401cb4d03abSBrian Vazquez if (!buf_prevkey) 1402cb4d03abSBrian Vazquez return -ENOMEM; 1403cb4d03abSBrian Vazquez 1404cb4d03abSBrian Vazquez buf = kmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN); 1405cb4d03abSBrian Vazquez if (!buf) { 1406bb2359f4SDenis Efremov kfree(buf_prevkey); 1407cb4d03abSBrian Vazquez return -ENOMEM; 1408cb4d03abSBrian Vazquez } 1409cb4d03abSBrian Vazquez 1410cb4d03abSBrian Vazquez err = -EFAULT; 1411cb4d03abSBrian Vazquez prev_key = NULL; 1412cb4d03abSBrian Vazquez if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size)) 1413cb4d03abSBrian Vazquez goto free_buf; 1414cb4d03abSBrian Vazquez key = buf; 1415cb4d03abSBrian Vazquez value = key + map->key_size; 1416cb4d03abSBrian Vazquez if (ubatch) 1417cb4d03abSBrian Vazquez prev_key = buf_prevkey; 1418cb4d03abSBrian Vazquez 1419cb4d03abSBrian Vazquez for (cp = 0; cp < max_count;) { 1420cb4d03abSBrian Vazquez rcu_read_lock(); 1421cb4d03abSBrian Vazquez err = map->ops->map_get_next_key(map, prev_key, key); 1422cb4d03abSBrian Vazquez rcu_read_unlock(); 1423cb4d03abSBrian Vazquez if (err) 1424cb4d03abSBrian Vazquez break; 1425cb4d03abSBrian Vazquez err = bpf_map_copy_value(map, key, value, 1426cb4d03abSBrian Vazquez attr->batch.elem_flags); 1427cb4d03abSBrian Vazquez 1428cb4d03abSBrian Vazquez if (err == -ENOENT) { 1429cb4d03abSBrian Vazquez if (retry) { 1430cb4d03abSBrian Vazquez retry--; 1431cb4d03abSBrian Vazquez continue; 1432cb4d03abSBrian Vazquez } 1433cb4d03abSBrian Vazquez err = -EINTR; 1434cb4d03abSBrian Vazquez break; 1435cb4d03abSBrian Vazquez } 1436cb4d03abSBrian Vazquez 1437cb4d03abSBrian Vazquez if (err) 1438cb4d03abSBrian Vazquez goto free_buf; 1439cb4d03abSBrian Vazquez 1440cb4d03abSBrian Vazquez if (copy_to_user(keys + cp * map->key_size, key, 1441cb4d03abSBrian Vazquez map->key_size)) { 1442cb4d03abSBrian Vazquez err = -EFAULT; 1443cb4d03abSBrian Vazquez goto free_buf; 1444cb4d03abSBrian Vazquez } 1445cb4d03abSBrian Vazquez if (copy_to_user(values + cp * value_size, value, value_size)) { 1446cb4d03abSBrian Vazquez err = -EFAULT; 1447cb4d03abSBrian Vazquez goto free_buf; 1448cb4d03abSBrian Vazquez } 1449cb4d03abSBrian Vazquez 1450cb4d03abSBrian Vazquez if (!prev_key) 1451cb4d03abSBrian Vazquez prev_key = buf_prevkey; 1452cb4d03abSBrian Vazquez 1453cb4d03abSBrian Vazquez swap(prev_key, key); 1454cb4d03abSBrian Vazquez retry = MAP_LOOKUP_RETRIES; 1455cb4d03abSBrian Vazquez cp++; 1456cb4d03abSBrian Vazquez } 1457cb4d03abSBrian Vazquez 1458cb4d03abSBrian Vazquez if (err == -EFAULT) 1459cb4d03abSBrian Vazquez goto free_buf; 1460cb4d03abSBrian Vazquez 1461cb4d03abSBrian Vazquez if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) || 1462cb4d03abSBrian Vazquez (cp && copy_to_user(uobatch, prev_key, map->key_size)))) 1463cb4d03abSBrian Vazquez err = -EFAULT; 1464cb4d03abSBrian Vazquez 1465cb4d03abSBrian Vazquez free_buf: 1466cb4d03abSBrian Vazquez kfree(buf_prevkey); 1467cb4d03abSBrian Vazquez kfree(buf); 1468cb4d03abSBrian Vazquez return err; 1469cb4d03abSBrian Vazquez } 1470cb4d03abSBrian Vazquez 1471bd513cd0SMauricio Vasquez B #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value 1472bd513cd0SMauricio Vasquez B 1473bd513cd0SMauricio Vasquez B static int map_lookup_and_delete_elem(union bpf_attr *attr) 1474bd513cd0SMauricio Vasquez B { 1475bd513cd0SMauricio Vasquez B void __user *ukey = u64_to_user_ptr(attr->key); 1476bd513cd0SMauricio Vasquez B void __user *uvalue = u64_to_user_ptr(attr->value); 1477bd513cd0SMauricio Vasquez B int ufd = attr->map_fd; 1478bd513cd0SMauricio Vasquez B struct bpf_map *map; 1479540fefc0SAlexei Starovoitov void *key, *value; 1480bd513cd0SMauricio Vasquez B u32 value_size; 1481bd513cd0SMauricio Vasquez B struct fd f; 1482bd513cd0SMauricio Vasquez B int err; 1483bd513cd0SMauricio Vasquez B 1484bd513cd0SMauricio Vasquez B if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) 1485bd513cd0SMauricio Vasquez B return -EINVAL; 1486bd513cd0SMauricio Vasquez B 1487bd513cd0SMauricio Vasquez B f = fdget(ufd); 1488bd513cd0SMauricio Vasquez B map = __bpf_map_get(f); 1489bd513cd0SMauricio Vasquez B if (IS_ERR(map)) 1490bd513cd0SMauricio Vasquez B return PTR_ERR(map); 14911ea0f912SAnton Protopopov if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) || 14921ea0f912SAnton Protopopov !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 1493bd513cd0SMauricio Vasquez B err = -EPERM; 1494bd513cd0SMauricio Vasquez B goto err_put; 1495bd513cd0SMauricio Vasquez B } 1496bd513cd0SMauricio Vasquez B 1497bd513cd0SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1498bd513cd0SMauricio Vasquez B if (IS_ERR(key)) { 1499bd513cd0SMauricio Vasquez B err = PTR_ERR(key); 1500bd513cd0SMauricio Vasquez B goto err_put; 1501bd513cd0SMauricio Vasquez B } 1502bd513cd0SMauricio Vasquez B 1503bd513cd0SMauricio Vasquez B value_size = map->value_size; 1504bd513cd0SMauricio Vasquez B 1505bd513cd0SMauricio Vasquez B err = -ENOMEM; 1506bd513cd0SMauricio Vasquez B value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1507bd513cd0SMauricio Vasquez B if (!value) 1508bd513cd0SMauricio Vasquez B goto free_key; 1509bd513cd0SMauricio Vasquez B 1510bd513cd0SMauricio Vasquez B if (map->map_type == BPF_MAP_TYPE_QUEUE || 1511bd513cd0SMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 1512bd513cd0SMauricio Vasquez B err = map->ops->map_pop_elem(map, value); 1513bd513cd0SMauricio Vasquez B } else { 1514bd513cd0SMauricio Vasquez B err = -ENOTSUPP; 1515bd513cd0SMauricio Vasquez B } 1516bd513cd0SMauricio Vasquez B 1517bd513cd0SMauricio Vasquez B if (err) 1518bd513cd0SMauricio Vasquez B goto free_value; 1519bd513cd0SMauricio Vasquez B 15207f645462SWei Yongjun if (copy_to_user(uvalue, value, value_size) != 0) { 15217f645462SWei Yongjun err = -EFAULT; 1522bd513cd0SMauricio Vasquez B goto free_value; 15237f645462SWei Yongjun } 1524bd513cd0SMauricio Vasquez B 1525bd513cd0SMauricio Vasquez B err = 0; 1526bd513cd0SMauricio Vasquez B 1527bd513cd0SMauricio Vasquez B free_value: 1528bd513cd0SMauricio Vasquez B kfree(value); 1529bd513cd0SMauricio Vasquez B free_key: 1530bd513cd0SMauricio Vasquez B kfree(key); 1531bd513cd0SMauricio Vasquez B err_put: 1532bd513cd0SMauricio Vasquez B fdput(f); 1533bd513cd0SMauricio Vasquez B return err; 1534bd513cd0SMauricio Vasquez B } 1535bd513cd0SMauricio Vasquez B 153687df15deSDaniel Borkmann #define BPF_MAP_FREEZE_LAST_FIELD map_fd 153787df15deSDaniel Borkmann 153887df15deSDaniel Borkmann static int map_freeze(const union bpf_attr *attr) 153987df15deSDaniel Borkmann { 154087df15deSDaniel Borkmann int err = 0, ufd = attr->map_fd; 154187df15deSDaniel Borkmann struct bpf_map *map; 154287df15deSDaniel Borkmann struct fd f; 154387df15deSDaniel Borkmann 154487df15deSDaniel Borkmann if (CHECK_ATTR(BPF_MAP_FREEZE)) 154587df15deSDaniel Borkmann return -EINVAL; 154687df15deSDaniel Borkmann 154787df15deSDaniel Borkmann f = fdget(ufd); 154887df15deSDaniel Borkmann map = __bpf_map_get(f); 154987df15deSDaniel Borkmann if (IS_ERR(map)) 155087df15deSDaniel Borkmann return PTR_ERR(map); 1551fc970227SAndrii Nakryiko 1552849b4d94SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 1553849b4d94SMartin KaFai Lau fdput(f); 1554849b4d94SMartin KaFai Lau return -ENOTSUPP; 1555849b4d94SMartin KaFai Lau } 1556849b4d94SMartin KaFai Lau 1557fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 1558fc970227SAndrii Nakryiko 1559fc970227SAndrii Nakryiko if (map->writecnt) { 1560fc970227SAndrii Nakryiko err = -EBUSY; 1561fc970227SAndrii Nakryiko goto err_put; 1562fc970227SAndrii Nakryiko } 156387df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) { 156487df15deSDaniel Borkmann err = -EBUSY; 156587df15deSDaniel Borkmann goto err_put; 156687df15deSDaniel Borkmann } 15672c78ee89SAlexei Starovoitov if (!bpf_capable()) { 156887df15deSDaniel Borkmann err = -EPERM; 156987df15deSDaniel Borkmann goto err_put; 157087df15deSDaniel Borkmann } 157187df15deSDaniel Borkmann 157287df15deSDaniel Borkmann WRITE_ONCE(map->frozen, true); 157387df15deSDaniel Borkmann err_put: 1574fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 157587df15deSDaniel Borkmann fdput(f); 157687df15deSDaniel Borkmann return err; 157787df15deSDaniel Borkmann } 157887df15deSDaniel Borkmann 15797de16e3aSJakub Kicinski static const struct bpf_prog_ops * const bpf_prog_types[] = { 158091cc1a99SAlexei Starovoitov #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 15817de16e3aSJakub Kicinski [_id] = & _name ## _prog_ops, 15827de16e3aSJakub Kicinski #define BPF_MAP_TYPE(_id, _ops) 1583f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) 15847de16e3aSJakub Kicinski #include <linux/bpf_types.h> 15857de16e3aSJakub Kicinski #undef BPF_PROG_TYPE 15867de16e3aSJakub Kicinski #undef BPF_MAP_TYPE 1587f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 15887de16e3aSJakub Kicinski }; 15897de16e3aSJakub Kicinski 159009756af4SAlexei Starovoitov static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 159109756af4SAlexei Starovoitov { 1592d0f1a451SDaniel Borkmann const struct bpf_prog_ops *ops; 1593d0f1a451SDaniel Borkmann 1594d0f1a451SDaniel Borkmann if (type >= ARRAY_SIZE(bpf_prog_types)) 1595d0f1a451SDaniel Borkmann return -EINVAL; 1596d0f1a451SDaniel Borkmann type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types)); 1597d0f1a451SDaniel Borkmann ops = bpf_prog_types[type]; 1598d0f1a451SDaniel Borkmann if (!ops) 1599be9370a7SJohannes Berg return -EINVAL; 160009756af4SAlexei Starovoitov 1601ab3f0063SJakub Kicinski if (!bpf_prog_is_dev_bound(prog->aux)) 1602d0f1a451SDaniel Borkmann prog->aux->ops = ops; 1603ab3f0063SJakub Kicinski else 1604ab3f0063SJakub Kicinski prog->aux->ops = &bpf_offload_prog_ops; 160524701eceSDaniel Borkmann prog->type = type; 160609756af4SAlexei Starovoitov return 0; 160709756af4SAlexei Starovoitov } 160809756af4SAlexei Starovoitov 1609bae141f5SDaniel Borkmann enum bpf_audit { 1610bae141f5SDaniel Borkmann BPF_AUDIT_LOAD, 1611bae141f5SDaniel Borkmann BPF_AUDIT_UNLOAD, 1612bae141f5SDaniel Borkmann BPF_AUDIT_MAX, 1613bae141f5SDaniel Borkmann }; 1614bae141f5SDaniel Borkmann 1615bae141f5SDaniel Borkmann static const char * const bpf_audit_str[BPF_AUDIT_MAX] = { 1616bae141f5SDaniel Borkmann [BPF_AUDIT_LOAD] = "LOAD", 1617bae141f5SDaniel Borkmann [BPF_AUDIT_UNLOAD] = "UNLOAD", 1618bae141f5SDaniel Borkmann }; 1619bae141f5SDaniel Borkmann 1620bae141f5SDaniel Borkmann static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op) 1621bae141f5SDaniel Borkmann { 1622bae141f5SDaniel Borkmann struct audit_context *ctx = NULL; 1623bae141f5SDaniel Borkmann struct audit_buffer *ab; 1624bae141f5SDaniel Borkmann 1625bae141f5SDaniel Borkmann if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX)) 1626bae141f5SDaniel Borkmann return; 1627bae141f5SDaniel Borkmann if (audit_enabled == AUDIT_OFF) 1628bae141f5SDaniel Borkmann return; 1629bae141f5SDaniel Borkmann if (op == BPF_AUDIT_LOAD) 1630bae141f5SDaniel Borkmann ctx = audit_context(); 1631bae141f5SDaniel Borkmann ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF); 1632bae141f5SDaniel Borkmann if (unlikely(!ab)) 1633bae141f5SDaniel Borkmann return; 1634bae141f5SDaniel Borkmann audit_log_format(ab, "prog-id=%u op=%s", 1635bae141f5SDaniel Borkmann prog->aux->id, bpf_audit_str[op]); 1636bae141f5SDaniel Borkmann audit_log_end(ab); 1637bae141f5SDaniel Borkmann } 1638bae141f5SDaniel Borkmann 1639dc4bb0e2SMartin KaFai Lau static int bpf_prog_alloc_id(struct bpf_prog *prog) 1640dc4bb0e2SMartin KaFai Lau { 1641dc4bb0e2SMartin KaFai Lau int id; 1642dc4bb0e2SMartin KaFai Lau 1643b76354cdSShaohua Li idr_preload(GFP_KERNEL); 1644dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1645dc4bb0e2SMartin KaFai Lau id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 1646dc4bb0e2SMartin KaFai Lau if (id > 0) 1647dc4bb0e2SMartin KaFai Lau prog->aux->id = id; 1648dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1649b76354cdSShaohua Li idr_preload_end(); 1650dc4bb0e2SMartin KaFai Lau 1651dc4bb0e2SMartin KaFai Lau /* id is in [1, INT_MAX) */ 1652dc4bb0e2SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 1653dc4bb0e2SMartin KaFai Lau return -ENOSPC; 1654dc4bb0e2SMartin KaFai Lau 1655dc4bb0e2SMartin KaFai Lau return id > 0 ? 0 : id; 1656dc4bb0e2SMartin KaFai Lau } 1657dc4bb0e2SMartin KaFai Lau 1658ad8ad79fSJakub Kicinski void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) 1659dc4bb0e2SMartin KaFai Lau { 1660ad8ad79fSJakub Kicinski /* cBPF to eBPF migrations are currently not in the idr store. 1661ad8ad79fSJakub Kicinski * Offloaded programs are removed from the store when their device 1662ad8ad79fSJakub Kicinski * disappears - even if someone grabs an fd to them they are unusable, 1663ad8ad79fSJakub Kicinski * simply waiting for refcnt to drop to be freed. 1664ad8ad79fSJakub Kicinski */ 1665dc4bb0e2SMartin KaFai Lau if (!prog->aux->id) 1666dc4bb0e2SMartin KaFai Lau return; 1667dc4bb0e2SMartin KaFai Lau 1668b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1669dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1670b16d9aa4SMartin KaFai Lau else 1671b16d9aa4SMartin KaFai Lau __acquire(&prog_idr_lock); 1672b16d9aa4SMartin KaFai Lau 1673dc4bb0e2SMartin KaFai Lau idr_remove(&prog_idr, prog->aux->id); 1674ad8ad79fSJakub Kicinski prog->aux->id = 0; 1675b16d9aa4SMartin KaFai Lau 1676b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1677dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1678b16d9aa4SMartin KaFai Lau else 1679b16d9aa4SMartin KaFai Lau __release(&prog_idr_lock); 1680dc4bb0e2SMartin KaFai Lau } 1681dc4bb0e2SMartin KaFai Lau 16821aacde3dSDaniel Borkmann static void __bpf_prog_put_rcu(struct rcu_head *rcu) 1683abf2e7d6SAlexei Starovoitov { 1684abf2e7d6SAlexei Starovoitov struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 1685abf2e7d6SAlexei Starovoitov 16863b4d9eb2SDaniel Borkmann kvfree(aux->func_info); 16878c1b6e69SAlexei Starovoitov kfree(aux->func_info_aux); 16883ac1f01bSRoman Gushchin free_uid(aux->user); 1689afdb09c7SChenbo Feng security_bpf_prog_free(aux); 1690abf2e7d6SAlexei Starovoitov bpf_prog_free(aux->prog); 1691abf2e7d6SAlexei Starovoitov } 1692abf2e7d6SAlexei Starovoitov 1693cd7455f1SDaniel Borkmann static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) 1694cd7455f1SDaniel Borkmann { 1695cd7455f1SDaniel Borkmann bpf_prog_kallsyms_del_all(prog); 1696cd7455f1SDaniel Borkmann btf_put(prog->aux->btf); 1697e16301fbSMartin KaFai Lau kvfree(prog->aux->jited_linfo); 1698e16301fbSMartin KaFai Lau kvfree(prog->aux->linfo); 1699e6ac2450SMartin KaFai Lau kfree(prog->aux->kfunc_tab); 170022dc4a0fSAndrii Nakryiko if (prog->aux->attach_btf) 170122dc4a0fSAndrii Nakryiko btf_put(prog->aux->attach_btf); 1702cd7455f1SDaniel Borkmann 17031e6c62a8SAlexei Starovoitov if (deferred) { 17041e6c62a8SAlexei Starovoitov if (prog->aux->sleepable) 17051e6c62a8SAlexei Starovoitov call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); 1706cd7455f1SDaniel Borkmann else 17071e6c62a8SAlexei Starovoitov call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 17081e6c62a8SAlexei Starovoitov } else { 1709cd7455f1SDaniel Borkmann __bpf_prog_put_rcu(&prog->aux->rcu); 1710cd7455f1SDaniel Borkmann } 17111e6c62a8SAlexei Starovoitov } 1712cd7455f1SDaniel Borkmann 1713b16d9aa4SMartin KaFai Lau static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 171409756af4SAlexei Starovoitov { 171585192dbfSAndrii Nakryiko if (atomic64_dec_and_test(&prog->aux->refcnt)) { 17166ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0); 1717bae141f5SDaniel Borkmann bpf_audit_prog(prog, BPF_AUDIT_UNLOAD); 171834ad5580SMartin KaFai Lau /* bpf_prog_free_id() must be called first */ 1719b16d9aa4SMartin KaFai Lau bpf_prog_free_id(prog, do_idr_lock); 1720cd7455f1SDaniel Borkmann __bpf_prog_put_noref(prog, true); 172109756af4SAlexei Starovoitov } 1722a67edbf4SDaniel Borkmann } 1723b16d9aa4SMartin KaFai Lau 1724b16d9aa4SMartin KaFai Lau void bpf_prog_put(struct bpf_prog *prog) 1725b16d9aa4SMartin KaFai Lau { 1726b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, true); 1727b16d9aa4SMartin KaFai Lau } 1728e2e9b654SDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_put); 172909756af4SAlexei Starovoitov 173009756af4SAlexei Starovoitov static int bpf_prog_release(struct inode *inode, struct file *filp) 173109756af4SAlexei Starovoitov { 173209756af4SAlexei Starovoitov struct bpf_prog *prog = filp->private_data; 173309756af4SAlexei Starovoitov 17341aacde3dSDaniel Borkmann bpf_prog_put(prog); 173509756af4SAlexei Starovoitov return 0; 173609756af4SAlexei Starovoitov } 173709756af4SAlexei Starovoitov 1738492ecee8SAlexei Starovoitov static void bpf_prog_get_stats(const struct bpf_prog *prog, 1739492ecee8SAlexei Starovoitov struct bpf_prog_stats *stats) 1740492ecee8SAlexei Starovoitov { 17419ed9e9baSAlexei Starovoitov u64 nsecs = 0, cnt = 0, misses = 0; 1742492ecee8SAlexei Starovoitov int cpu; 1743492ecee8SAlexei Starovoitov 1744492ecee8SAlexei Starovoitov for_each_possible_cpu(cpu) { 1745492ecee8SAlexei Starovoitov const struct bpf_prog_stats *st; 1746492ecee8SAlexei Starovoitov unsigned int start; 17479ed9e9baSAlexei Starovoitov u64 tnsecs, tcnt, tmisses; 1748492ecee8SAlexei Starovoitov 1749700d4796SAlexei Starovoitov st = per_cpu_ptr(prog->stats, cpu); 1750492ecee8SAlexei Starovoitov do { 1751492ecee8SAlexei Starovoitov start = u64_stats_fetch_begin_irq(&st->syncp); 1752492ecee8SAlexei Starovoitov tnsecs = st->nsecs; 1753492ecee8SAlexei Starovoitov tcnt = st->cnt; 17549ed9e9baSAlexei Starovoitov tmisses = st->misses; 1755492ecee8SAlexei Starovoitov } while (u64_stats_fetch_retry_irq(&st->syncp, start)); 1756492ecee8SAlexei Starovoitov nsecs += tnsecs; 1757492ecee8SAlexei Starovoitov cnt += tcnt; 17589ed9e9baSAlexei Starovoitov misses += tmisses; 1759492ecee8SAlexei Starovoitov } 1760492ecee8SAlexei Starovoitov stats->nsecs = nsecs; 1761492ecee8SAlexei Starovoitov stats->cnt = cnt; 17629ed9e9baSAlexei Starovoitov stats->misses = misses; 1763492ecee8SAlexei Starovoitov } 1764492ecee8SAlexei Starovoitov 17657bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 17667bd509e3SDaniel Borkmann static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 17677bd509e3SDaniel Borkmann { 17687bd509e3SDaniel Borkmann const struct bpf_prog *prog = filp->private_data; 1769f1f7714eSDaniel Borkmann char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 1770492ecee8SAlexei Starovoitov struct bpf_prog_stats stats; 17717bd509e3SDaniel Borkmann 1772492ecee8SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 1773f1f7714eSDaniel Borkmann bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 17747bd509e3SDaniel Borkmann seq_printf(m, 17757bd509e3SDaniel Borkmann "prog_type:\t%u\n" 17767bd509e3SDaniel Borkmann "prog_jited:\t%u\n" 1777f1f7714eSDaniel Borkmann "prog_tag:\t%s\n" 17784316b409SDaniel Borkmann "memlock:\t%llu\n" 1779492ecee8SAlexei Starovoitov "prog_id:\t%u\n" 1780492ecee8SAlexei Starovoitov "run_time_ns:\t%llu\n" 17819ed9e9baSAlexei Starovoitov "run_cnt:\t%llu\n" 17829ed9e9baSAlexei Starovoitov "recursion_misses:\t%llu\n", 17837bd509e3SDaniel Borkmann prog->type, 17847bd509e3SDaniel Borkmann prog->jited, 1785f1f7714eSDaniel Borkmann prog_tag, 17864316b409SDaniel Borkmann prog->pages * 1ULL << PAGE_SHIFT, 1787492ecee8SAlexei Starovoitov prog->aux->id, 1788492ecee8SAlexei Starovoitov stats.nsecs, 17899ed9e9baSAlexei Starovoitov stats.cnt, 17909ed9e9baSAlexei Starovoitov stats.misses); 17917bd509e3SDaniel Borkmann } 17927bd509e3SDaniel Borkmann #endif 17937bd509e3SDaniel Borkmann 1794f66e448cSChenbo Feng const struct file_operations bpf_prog_fops = { 17957bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 17967bd509e3SDaniel Borkmann .show_fdinfo = bpf_prog_show_fdinfo, 17977bd509e3SDaniel Borkmann #endif 179809756af4SAlexei Starovoitov .release = bpf_prog_release, 17996e71b04aSChenbo Feng .read = bpf_dummy_read, 18006e71b04aSChenbo Feng .write = bpf_dummy_write, 180109756af4SAlexei Starovoitov }; 180209756af4SAlexei Starovoitov 1803b2197755SDaniel Borkmann int bpf_prog_new_fd(struct bpf_prog *prog) 1804aa79781bSDaniel Borkmann { 1805afdb09c7SChenbo Feng int ret; 1806afdb09c7SChenbo Feng 1807afdb09c7SChenbo Feng ret = security_bpf_prog(prog); 1808afdb09c7SChenbo Feng if (ret < 0) 1809afdb09c7SChenbo Feng return ret; 1810afdb09c7SChenbo Feng 1811aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 1812aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC); 1813aa79781bSDaniel Borkmann } 1814aa79781bSDaniel Borkmann 1815113214beSDaniel Borkmann static struct bpf_prog *____bpf_prog_get(struct fd f) 181609756af4SAlexei Starovoitov { 181709756af4SAlexei Starovoitov if (!f.file) 181809756af4SAlexei Starovoitov return ERR_PTR(-EBADF); 181909756af4SAlexei Starovoitov if (f.file->f_op != &bpf_prog_fops) { 182009756af4SAlexei Starovoitov fdput(f); 182109756af4SAlexei Starovoitov return ERR_PTR(-EINVAL); 182209756af4SAlexei Starovoitov } 182309756af4SAlexei Starovoitov 1824c2101297SDaniel Borkmann return f.file->private_data; 182509756af4SAlexei Starovoitov } 182609756af4SAlexei Starovoitov 182785192dbfSAndrii Nakryiko void bpf_prog_add(struct bpf_prog *prog, int i) 182892117d84SAlexei Starovoitov { 182985192dbfSAndrii Nakryiko atomic64_add(i, &prog->aux->refcnt); 183092117d84SAlexei Starovoitov } 183159d3656dSBrenden Blanco EXPORT_SYMBOL_GPL(bpf_prog_add); 183259d3656dSBrenden Blanco 1833c540594fSDaniel Borkmann void bpf_prog_sub(struct bpf_prog *prog, int i) 1834c540594fSDaniel Borkmann { 1835c540594fSDaniel Borkmann /* Only to be used for undoing previous bpf_prog_add() in some 1836c540594fSDaniel Borkmann * error path. We still know that another entity in our call 1837c540594fSDaniel Borkmann * path holds a reference to the program, thus atomic_sub() can 1838c540594fSDaniel Borkmann * be safely used in such cases! 1839c540594fSDaniel Borkmann */ 184085192dbfSAndrii Nakryiko WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0); 1841c540594fSDaniel Borkmann } 1842c540594fSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_sub); 1843c540594fSDaniel Borkmann 184485192dbfSAndrii Nakryiko void bpf_prog_inc(struct bpf_prog *prog) 184559d3656dSBrenden Blanco { 184685192dbfSAndrii Nakryiko atomic64_inc(&prog->aux->refcnt); 184759d3656dSBrenden Blanco } 184897bc402dSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_inc); 184992117d84SAlexei Starovoitov 1850b16d9aa4SMartin KaFai Lau /* prog_idr_lock should have been held */ 1851a6f6df69SJohn Fastabend struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 1852b16d9aa4SMartin KaFai Lau { 1853b16d9aa4SMartin KaFai Lau int refold; 1854b16d9aa4SMartin KaFai Lau 185585192dbfSAndrii Nakryiko refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0); 1856b16d9aa4SMartin KaFai Lau 1857b16d9aa4SMartin KaFai Lau if (!refold) 1858b16d9aa4SMartin KaFai Lau return ERR_PTR(-ENOENT); 1859b16d9aa4SMartin KaFai Lau 1860b16d9aa4SMartin KaFai Lau return prog; 1861b16d9aa4SMartin KaFai Lau } 1862a6f6df69SJohn Fastabend EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 1863b16d9aa4SMartin KaFai Lau 1864040ee692SAl Viro bool bpf_prog_get_ok(struct bpf_prog *prog, 1865288b3de5SJakub Kicinski enum bpf_prog_type *attach_type, bool attach_drv) 1866248f346fSJakub Kicinski { 1867288b3de5SJakub Kicinski /* not an attachment, just a refcount inc, always allow */ 1868288b3de5SJakub Kicinski if (!attach_type) 1869288b3de5SJakub Kicinski return true; 1870248f346fSJakub Kicinski 1871248f346fSJakub Kicinski if (prog->type != *attach_type) 1872248f346fSJakub Kicinski return false; 1873288b3de5SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv) 1874248f346fSJakub Kicinski return false; 1875248f346fSJakub Kicinski 1876248f346fSJakub Kicinski return true; 1877248f346fSJakub Kicinski } 1878248f346fSJakub Kicinski 1879248f346fSJakub Kicinski static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, 1880288b3de5SJakub Kicinski bool attach_drv) 188109756af4SAlexei Starovoitov { 188209756af4SAlexei Starovoitov struct fd f = fdget(ufd); 188309756af4SAlexei Starovoitov struct bpf_prog *prog; 188409756af4SAlexei Starovoitov 1885113214beSDaniel Borkmann prog = ____bpf_prog_get(f); 188609756af4SAlexei Starovoitov if (IS_ERR(prog)) 188709756af4SAlexei Starovoitov return prog; 1888288b3de5SJakub Kicinski if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) { 1889113214beSDaniel Borkmann prog = ERR_PTR(-EINVAL); 1890113214beSDaniel Borkmann goto out; 1891113214beSDaniel Borkmann } 189209756af4SAlexei Starovoitov 189385192dbfSAndrii Nakryiko bpf_prog_inc(prog); 1894113214beSDaniel Borkmann out: 189509756af4SAlexei Starovoitov fdput(f); 189609756af4SAlexei Starovoitov return prog; 189709756af4SAlexei Starovoitov } 1898113214beSDaniel Borkmann 1899113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get(u32 ufd) 1900113214beSDaniel Borkmann { 1901288b3de5SJakub Kicinski return __bpf_prog_get(ufd, NULL, false); 1902113214beSDaniel Borkmann } 1903113214beSDaniel Borkmann 1904248f346fSJakub Kicinski struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 1905288b3de5SJakub Kicinski bool attach_drv) 1906248f346fSJakub Kicinski { 19074d220ed0SAlexei Starovoitov return __bpf_prog_get(ufd, &type, attach_drv); 1908248f346fSJakub Kicinski } 19096c8dfe21SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev); 1910248f346fSJakub Kicinski 1911aac3fc32SAndrey Ignatov /* Initially all BPF programs could be loaded w/o specifying 1912aac3fc32SAndrey Ignatov * expected_attach_type. Later for some of them specifying expected_attach_type 1913aac3fc32SAndrey Ignatov * at load time became required so that program could be validated properly. 1914aac3fc32SAndrey Ignatov * Programs of types that are allowed to be loaded both w/ and w/o (for 1915aac3fc32SAndrey Ignatov * backward compatibility) expected_attach_type, should have the default attach 1916aac3fc32SAndrey Ignatov * type assigned to expected_attach_type for the latter case, so that it can be 1917aac3fc32SAndrey Ignatov * validated later at attach time. 1918aac3fc32SAndrey Ignatov * 1919aac3fc32SAndrey Ignatov * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if 1920aac3fc32SAndrey Ignatov * prog type requires it but has some attach types that have to be backward 1921aac3fc32SAndrey Ignatov * compatible. 1922aac3fc32SAndrey Ignatov */ 1923aac3fc32SAndrey Ignatov static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) 1924aac3fc32SAndrey Ignatov { 1925aac3fc32SAndrey Ignatov switch (attr->prog_type) { 1926aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1927aac3fc32SAndrey Ignatov /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't 1928aac3fc32SAndrey Ignatov * exist so checking for non-zero is the way to go here. 1929aac3fc32SAndrey Ignatov */ 1930aac3fc32SAndrey Ignatov if (!attr->expected_attach_type) 1931aac3fc32SAndrey Ignatov attr->expected_attach_type = 1932aac3fc32SAndrey Ignatov BPF_CGROUP_INET_SOCK_CREATE; 1933aac3fc32SAndrey Ignatov break; 1934aac3fc32SAndrey Ignatov } 1935aac3fc32SAndrey Ignatov } 1936aac3fc32SAndrey Ignatov 19375e43f899SAndrey Ignatov static int 1938ccfe29ebSAlexei Starovoitov bpf_prog_load_check_attach(enum bpf_prog_type prog_type, 1939ccfe29ebSAlexei Starovoitov enum bpf_attach_type expected_attach_type, 1940290248a5SAndrii Nakryiko struct btf *attach_btf, u32 btf_id, 1941290248a5SAndrii Nakryiko struct bpf_prog *dst_prog) 19425e43f899SAndrey Ignatov { 194327ae7997SMartin KaFai Lau if (btf_id) { 1944c108e3c1SAlexei Starovoitov if (btf_id > BTF_MAX_TYPE) 1945c108e3c1SAlexei Starovoitov return -EINVAL; 194627ae7997SMartin KaFai Lau 1947290248a5SAndrii Nakryiko if (!attach_btf && !dst_prog) 1948290248a5SAndrii Nakryiko return -EINVAL; 1949290248a5SAndrii Nakryiko 195027ae7997SMartin KaFai Lau switch (prog_type) { 195127ae7997SMartin KaFai Lau case BPF_PROG_TYPE_TRACING: 19529e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 195327ae7997SMartin KaFai Lau case BPF_PROG_TYPE_STRUCT_OPS: 1954be8704ffSAlexei Starovoitov case BPF_PROG_TYPE_EXT: 1955c108e3c1SAlexei Starovoitov break; 1956c108e3c1SAlexei Starovoitov default: 1957c108e3c1SAlexei Starovoitov return -EINVAL; 1958c108e3c1SAlexei Starovoitov } 195927ae7997SMartin KaFai Lau } 196027ae7997SMartin KaFai Lau 1961290248a5SAndrii Nakryiko if (attach_btf && (!btf_id || dst_prog)) 1962290248a5SAndrii Nakryiko return -EINVAL; 1963290248a5SAndrii Nakryiko 1964290248a5SAndrii Nakryiko if (dst_prog && prog_type != BPF_PROG_TYPE_TRACING && 1965be8704ffSAlexei Starovoitov prog_type != BPF_PROG_TYPE_EXT) 196627ae7997SMartin KaFai Lau return -EINVAL; 1967c108e3c1SAlexei Starovoitov 1968c108e3c1SAlexei Starovoitov switch (prog_type) { 1969aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1970aac3fc32SAndrey Ignatov switch (expected_attach_type) { 1971aac3fc32SAndrey Ignatov case BPF_CGROUP_INET_SOCK_CREATE: 1972f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 1973aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1974aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 1975aac3fc32SAndrey Ignatov return 0; 1976aac3fc32SAndrey Ignatov default: 1977aac3fc32SAndrey Ignatov return -EINVAL; 1978aac3fc32SAndrey Ignatov } 19794fbac77dSAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 19804fbac77dSAndrey Ignatov switch (expected_attach_type) { 19814fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 19824fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1983d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1984d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 19851b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 19861b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 19871b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 19881b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 19891cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 19901cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 1991983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG: 1992983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG: 19935e43f899SAndrey Ignatov return 0; 19944fbac77dSAndrey Ignatov default: 19954fbac77dSAndrey Ignatov return -EINVAL; 19964fbac77dSAndrey Ignatov } 19975cf1e914Sbrakmo case BPF_PROG_TYPE_CGROUP_SKB: 19985cf1e914Sbrakmo switch (expected_attach_type) { 19995cf1e914Sbrakmo case BPF_CGROUP_INET_INGRESS: 20005cf1e914Sbrakmo case BPF_CGROUP_INET_EGRESS: 20015cf1e914Sbrakmo return 0; 20025cf1e914Sbrakmo default: 20035cf1e914Sbrakmo return -EINVAL; 20045cf1e914Sbrakmo } 20050d01da6aSStanislav Fomichev case BPF_PROG_TYPE_CGROUP_SOCKOPT: 20060d01da6aSStanislav Fomichev switch (expected_attach_type) { 20070d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT: 20080d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT: 20090d01da6aSStanislav Fomichev return 0; 20100d01da6aSStanislav Fomichev default: 20110d01da6aSStanislav Fomichev return -EINVAL; 20120d01da6aSStanislav Fomichev } 2013e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP: 2014e9ddbb77SJakub Sitnicki if (expected_attach_type == BPF_SK_LOOKUP) 2015e9ddbb77SJakub Sitnicki return 0; 2016e9ddbb77SJakub Sitnicki return -EINVAL; 2017be8704ffSAlexei Starovoitov case BPF_PROG_TYPE_EXT: 2018be8704ffSAlexei Starovoitov if (expected_attach_type) 2019be8704ffSAlexei Starovoitov return -EINVAL; 2020df561f66SGustavo A. R. Silva fallthrough; 20214fbac77dSAndrey Ignatov default: 20224fbac77dSAndrey Ignatov return 0; 20234fbac77dSAndrey Ignatov } 20245e43f899SAndrey Ignatov } 20255e43f899SAndrey Ignatov 20262c78ee89SAlexei Starovoitov static bool is_net_admin_prog_type(enum bpf_prog_type prog_type) 20272c78ee89SAlexei Starovoitov { 20282c78ee89SAlexei Starovoitov switch (prog_type) { 20292c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SCHED_CLS: 20302c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SCHED_ACT: 20312c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_XDP: 20322c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_IN: 20332c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_OUT: 20342c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_XMIT: 20352c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_SEG6LOCAL: 20362c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_SKB: 20372c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_MSG: 20382c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LIRC_MODE2: 20392c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_FLOW_DISSECTOR: 20402c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_DEVICE: 20412c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCK: 20422c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 20432c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCKOPT: 20442c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SYSCTL: 20452c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SOCK_OPS: 20462c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_EXT: /* extends any prog */ 20472c78ee89SAlexei Starovoitov return true; 20482c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SKB: 20492c78ee89SAlexei Starovoitov /* always unpriv */ 20502c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_REUSEPORT: 20512c78ee89SAlexei Starovoitov /* equivalent to SOCKET_FILTER. need CAP_BPF only */ 20522c78ee89SAlexei Starovoitov default: 20532c78ee89SAlexei Starovoitov return false; 20542c78ee89SAlexei Starovoitov } 20552c78ee89SAlexei Starovoitov } 20562c78ee89SAlexei Starovoitov 20572c78ee89SAlexei Starovoitov static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) 20582c78ee89SAlexei Starovoitov { 20592c78ee89SAlexei Starovoitov switch (prog_type) { 20602c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_KPROBE: 20612c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_TRACEPOINT: 20622c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_PERF_EVENT: 20632c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_RAW_TRACEPOINT: 20642c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 20652c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_TRACING: 20662c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LSM: 20672c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_STRUCT_OPS: /* has access to struct sock */ 20682c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_EXT: /* extends any prog */ 20692c78ee89SAlexei Starovoitov return true; 20702c78ee89SAlexei Starovoitov default: 20712c78ee89SAlexei Starovoitov return false; 20722c78ee89SAlexei Starovoitov } 20732c78ee89SAlexei Starovoitov } 20742c78ee89SAlexei Starovoitov 207509756af4SAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 20765b92a28aSAlexei Starovoitov #define BPF_PROG_LOAD_LAST_FIELD attach_prog_fd 207709756af4SAlexei Starovoitov 2078838e9690SYonghong Song static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) 207909756af4SAlexei Starovoitov { 208009756af4SAlexei Starovoitov enum bpf_prog_type type = attr->prog_type; 2081290248a5SAndrii Nakryiko struct bpf_prog *prog, *dst_prog = NULL; 2082290248a5SAndrii Nakryiko struct btf *attach_btf = NULL; 208309756af4SAlexei Starovoitov int err; 208409756af4SAlexei Starovoitov char license[128]; 208509756af4SAlexei Starovoitov bool is_gpl; 208609756af4SAlexei Starovoitov 208709756af4SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_LOAD)) 208809756af4SAlexei Starovoitov return -EINVAL; 208909756af4SAlexei Starovoitov 2090c240eff6SJiong Wang if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | 2091c240eff6SJiong Wang BPF_F_ANY_ALIGNMENT | 209210d274e8SAlexei Starovoitov BPF_F_TEST_STATE_FREQ | 20931e6c62a8SAlexei Starovoitov BPF_F_SLEEPABLE | 2094c240eff6SJiong Wang BPF_F_TEST_RND_HI32)) 2095e07b98d9SDavid S. Miller return -EINVAL; 2096e07b98d9SDavid S. Miller 2097e9ee9efcSDavid Miller if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && 2098e9ee9efcSDavid Miller (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && 20992c78ee89SAlexei Starovoitov !bpf_capable()) 2100e9ee9efcSDavid Miller return -EPERM; 2101e9ee9efcSDavid Miller 210209756af4SAlexei Starovoitov /* copy eBPF program license from user space */ 2103535e7b4bSMickaël Salaün if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 210409756af4SAlexei Starovoitov sizeof(license) - 1) < 0) 210509756af4SAlexei Starovoitov return -EFAULT; 210609756af4SAlexei Starovoitov license[sizeof(license) - 1] = 0; 210709756af4SAlexei Starovoitov 210809756af4SAlexei Starovoitov /* eBPF programs must be GPL compatible to use GPL-ed functions */ 210909756af4SAlexei Starovoitov is_gpl = license_is_gpl_compatible(license); 211009756af4SAlexei Starovoitov 2111c04c0d2bSAlexei Starovoitov if (attr->insn_cnt == 0 || 21122c78ee89SAlexei Starovoitov attr->insn_cnt > (bpf_capable() ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) 2113ef0915caSDaniel Borkmann return -E2BIG; 211480b7d819SChenbo Feng if (type != BPF_PROG_TYPE_SOCKET_FILTER && 211580b7d819SChenbo Feng type != BPF_PROG_TYPE_CGROUP_SKB && 21162c78ee89SAlexei Starovoitov !bpf_capable()) 21172c78ee89SAlexei Starovoitov return -EPERM; 21182c78ee89SAlexei Starovoitov 2119b338cb92SMaciej Żenczykowski if (is_net_admin_prog_type(type) && !capable(CAP_NET_ADMIN) && !capable(CAP_SYS_ADMIN)) 21202c78ee89SAlexei Starovoitov return -EPERM; 21212c78ee89SAlexei Starovoitov if (is_perfmon_prog_type(type) && !perfmon_capable()) 21221be7f75dSAlexei Starovoitov return -EPERM; 21231be7f75dSAlexei Starovoitov 2124290248a5SAndrii Nakryiko /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog 2125290248a5SAndrii Nakryiko * or btf, we need to check which one it is 2126290248a5SAndrii Nakryiko */ 2127290248a5SAndrii Nakryiko if (attr->attach_prog_fd) { 2128290248a5SAndrii Nakryiko dst_prog = bpf_prog_get(attr->attach_prog_fd); 2129290248a5SAndrii Nakryiko if (IS_ERR(dst_prog)) { 2130290248a5SAndrii Nakryiko dst_prog = NULL; 2131290248a5SAndrii Nakryiko attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd); 2132290248a5SAndrii Nakryiko if (IS_ERR(attach_btf)) 2133290248a5SAndrii Nakryiko return -EINVAL; 2134290248a5SAndrii Nakryiko if (!btf_is_kernel(attach_btf)) { 21358bdd8e27SAndrii Nakryiko /* attaching through specifying bpf_prog's BTF 21368bdd8e27SAndrii Nakryiko * objects directly might be supported eventually 21378bdd8e27SAndrii Nakryiko */ 2138290248a5SAndrii Nakryiko btf_put(attach_btf); 21398bdd8e27SAndrii Nakryiko return -ENOTSUPP; 2140290248a5SAndrii Nakryiko } 2141290248a5SAndrii Nakryiko } 2142290248a5SAndrii Nakryiko } else if (attr->attach_btf_id) { 2143290248a5SAndrii Nakryiko /* fall back to vmlinux BTF, if BTF type ID is specified */ 2144290248a5SAndrii Nakryiko attach_btf = bpf_get_btf_vmlinux(); 2145290248a5SAndrii Nakryiko if (IS_ERR(attach_btf)) 2146290248a5SAndrii Nakryiko return PTR_ERR(attach_btf); 2147290248a5SAndrii Nakryiko if (!attach_btf) 2148290248a5SAndrii Nakryiko return -EINVAL; 2149290248a5SAndrii Nakryiko btf_get(attach_btf); 2150290248a5SAndrii Nakryiko } 2151290248a5SAndrii Nakryiko 2152aac3fc32SAndrey Ignatov bpf_prog_load_fixup_attach_type(attr); 2153ccfe29ebSAlexei Starovoitov if (bpf_prog_load_check_attach(type, attr->expected_attach_type, 2154290248a5SAndrii Nakryiko attach_btf, attr->attach_btf_id, 2155290248a5SAndrii Nakryiko dst_prog)) { 2156290248a5SAndrii Nakryiko if (dst_prog) 2157290248a5SAndrii Nakryiko bpf_prog_put(dst_prog); 2158290248a5SAndrii Nakryiko if (attach_btf) 2159290248a5SAndrii Nakryiko btf_put(attach_btf); 21605e43f899SAndrey Ignatov return -EINVAL; 2161290248a5SAndrii Nakryiko } 21625e43f899SAndrey Ignatov 216309756af4SAlexei Starovoitov /* plain bpf_prog allocation */ 216409756af4SAlexei Starovoitov prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 2165290248a5SAndrii Nakryiko if (!prog) { 2166290248a5SAndrii Nakryiko if (dst_prog) 2167290248a5SAndrii Nakryiko bpf_prog_put(dst_prog); 2168290248a5SAndrii Nakryiko if (attach_btf) 2169290248a5SAndrii Nakryiko btf_put(attach_btf); 217009756af4SAlexei Starovoitov return -ENOMEM; 2171290248a5SAndrii Nakryiko } 217209756af4SAlexei Starovoitov 21735e43f899SAndrey Ignatov prog->expected_attach_type = attr->expected_attach_type; 2174290248a5SAndrii Nakryiko prog->aux->attach_btf = attach_btf; 2175ccfe29ebSAlexei Starovoitov prog->aux->attach_btf_id = attr->attach_btf_id; 21763aac1eadSToke Høiland-Jørgensen prog->aux->dst_prog = dst_prog; 21779a18eedbSJakub Kicinski prog->aux->offload_requested = !!attr->prog_ifindex; 21781e6c62a8SAlexei Starovoitov prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE; 21799a18eedbSJakub Kicinski 2180afdb09c7SChenbo Feng err = security_bpf_prog_alloc(prog->aux); 2181aaac3ba9SAlexei Starovoitov if (err) 21823ac1f01bSRoman Gushchin goto free_prog; 2183aaac3ba9SAlexei Starovoitov 21843ac1f01bSRoman Gushchin prog->aux->user = get_current_user(); 218509756af4SAlexei Starovoitov prog->len = attr->insn_cnt; 218609756af4SAlexei Starovoitov 218709756af4SAlexei Starovoitov err = -EFAULT; 2188535e7b4bSMickaël Salaün if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 2189aafe6ae9SDaniel Borkmann bpf_prog_insn_size(prog)) != 0) 21903ac1f01bSRoman Gushchin goto free_prog_sec; 219109756af4SAlexei Starovoitov 219209756af4SAlexei Starovoitov prog->orig_prog = NULL; 2193a91263d5SDaniel Borkmann prog->jited = 0; 219409756af4SAlexei Starovoitov 219585192dbfSAndrii Nakryiko atomic64_set(&prog->aux->refcnt, 1); 2196a91263d5SDaniel Borkmann prog->gpl_compatible = is_gpl ? 1 : 0; 219709756af4SAlexei Starovoitov 21989a18eedbSJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 2199ab3f0063SJakub Kicinski err = bpf_prog_offload_init(prog, attr); 2200ab3f0063SJakub Kicinski if (err) 22013ac1f01bSRoman Gushchin goto free_prog_sec; 2202ab3f0063SJakub Kicinski } 2203ab3f0063SJakub Kicinski 220409756af4SAlexei Starovoitov /* find program type: socket_filter vs tracing_filter */ 220509756af4SAlexei Starovoitov err = find_prog_type(type, prog); 220609756af4SAlexei Starovoitov if (err < 0) 22073ac1f01bSRoman Gushchin goto free_prog_sec; 220809756af4SAlexei Starovoitov 22099285ec4cSJason A. Donenfeld prog->aux->load_time = ktime_get_boottime_ns(); 22108e7ae251SMartin KaFai Lau err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, 22118e7ae251SMartin KaFai Lau sizeof(attr->prog_name)); 22128e7ae251SMartin KaFai Lau if (err < 0) 22133ac1f01bSRoman Gushchin goto free_prog_sec; 2214cb4d2b3fSMartin KaFai Lau 221509756af4SAlexei Starovoitov /* run eBPF verifier */ 2216838e9690SYonghong Song err = bpf_check(&prog, attr, uattr); 221709756af4SAlexei Starovoitov if (err < 0) 221809756af4SAlexei Starovoitov goto free_used_maps; 221909756af4SAlexei Starovoitov 2220d1c55ab5SDaniel Borkmann prog = bpf_prog_select_runtime(prog, &err); 222104fd61abSAlexei Starovoitov if (err < 0) 222204fd61abSAlexei Starovoitov goto free_used_maps; 222309756af4SAlexei Starovoitov 2224dc4bb0e2SMartin KaFai Lau err = bpf_prog_alloc_id(prog); 2225dc4bb0e2SMartin KaFai Lau if (err) 2226dc4bb0e2SMartin KaFai Lau goto free_used_maps; 2227dc4bb0e2SMartin KaFai Lau 2228c751798aSDaniel Borkmann /* Upon success of bpf_prog_alloc_id(), the BPF prog is 2229c751798aSDaniel Borkmann * effectively publicly exposed. However, retrieving via 2230c751798aSDaniel Borkmann * bpf_prog_get_fd_by_id() will take another reference, 2231c751798aSDaniel Borkmann * therefore it cannot be gone underneath us. 2232c751798aSDaniel Borkmann * 2233c751798aSDaniel Borkmann * Only for the time /after/ successful bpf_prog_new_fd() 2234c751798aSDaniel Borkmann * and before returning to userspace, we might just hold 2235c751798aSDaniel Borkmann * one reference and any parallel close on that fd could 2236c751798aSDaniel Borkmann * rip everything out. Hence, below notifications must 2237c751798aSDaniel Borkmann * happen before bpf_prog_new_fd(). 2238c751798aSDaniel Borkmann * 2239c751798aSDaniel Borkmann * Also, any failure handling from this point onwards must 2240c751798aSDaniel Borkmann * be using bpf_prog_put() given the program is exposed. 2241b16d9aa4SMartin KaFai Lau */ 224274451e66SDaniel Borkmann bpf_prog_kallsyms_add(prog); 22436ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); 2244bae141f5SDaniel Borkmann bpf_audit_prog(prog, BPF_AUDIT_LOAD); 2245c751798aSDaniel Borkmann 2246c751798aSDaniel Borkmann err = bpf_prog_new_fd(prog); 2247c751798aSDaniel Borkmann if (err < 0) 2248c751798aSDaniel Borkmann bpf_prog_put(prog); 224909756af4SAlexei Starovoitov return err; 225009756af4SAlexei Starovoitov 225109756af4SAlexei Starovoitov free_used_maps: 2252cd7455f1SDaniel Borkmann /* In case we have subprogs, we need to wait for a grace 2253cd7455f1SDaniel Borkmann * period before we can tear down JIT memory since symbols 2254cd7455f1SDaniel Borkmann * are already exposed under kallsyms. 2255cd7455f1SDaniel Borkmann */ 2256cd7455f1SDaniel Borkmann __bpf_prog_put_noref(prog, prog->aux->func_cnt); 2257cd7455f1SDaniel Borkmann return err; 2258afdb09c7SChenbo Feng free_prog_sec: 22593ac1f01bSRoman Gushchin free_uid(prog->aux->user); 2260afdb09c7SChenbo Feng security_bpf_prog_free(prog->aux); 22613ac1f01bSRoman Gushchin free_prog: 226222dc4a0fSAndrii Nakryiko if (prog->aux->attach_btf) 226322dc4a0fSAndrii Nakryiko btf_put(prog->aux->attach_btf); 226409756af4SAlexei Starovoitov bpf_prog_free(prog); 226509756af4SAlexei Starovoitov return err; 226609756af4SAlexei Starovoitov } 226709756af4SAlexei Starovoitov 22686e71b04aSChenbo Feng #define BPF_OBJ_LAST_FIELD file_flags 2269b2197755SDaniel Borkmann 2270b2197755SDaniel Borkmann static int bpf_obj_pin(const union bpf_attr *attr) 2271b2197755SDaniel Borkmann { 22726e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0) 2273b2197755SDaniel Borkmann return -EINVAL; 2274b2197755SDaniel Borkmann 2275535e7b4bSMickaël Salaün return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 2276b2197755SDaniel Borkmann } 2277b2197755SDaniel Borkmann 2278b2197755SDaniel Borkmann static int bpf_obj_get(const union bpf_attr *attr) 2279b2197755SDaniel Borkmann { 22806e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || 22816e71b04aSChenbo Feng attr->file_flags & ~BPF_OBJ_FLAG_MASK) 2282b2197755SDaniel Borkmann return -EINVAL; 2283b2197755SDaniel Borkmann 22846e71b04aSChenbo Feng return bpf_obj_get_user(u64_to_user_ptr(attr->pathname), 22856e71b04aSChenbo Feng attr->file_flags); 2286b2197755SDaniel Borkmann } 2287b2197755SDaniel Borkmann 2288f2e10bffSAndrii Nakryiko void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 2289a3b80e10SAndrii Nakryiko const struct bpf_link_ops *ops, struct bpf_prog *prog) 229070ed506cSAndrii Nakryiko { 229170ed506cSAndrii Nakryiko atomic64_set(&link->refcnt, 1); 2292f2e10bffSAndrii Nakryiko link->type = type; 2293a3b80e10SAndrii Nakryiko link->id = 0; 229470ed506cSAndrii Nakryiko link->ops = ops; 229570ed506cSAndrii Nakryiko link->prog = prog; 229670ed506cSAndrii Nakryiko } 229770ed506cSAndrii Nakryiko 2298a3b80e10SAndrii Nakryiko static void bpf_link_free_id(int id) 2299a3b80e10SAndrii Nakryiko { 2300a3b80e10SAndrii Nakryiko if (!id) 2301a3b80e10SAndrii Nakryiko return; 2302a3b80e10SAndrii Nakryiko 2303a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 2304a3b80e10SAndrii Nakryiko idr_remove(&link_idr, id); 2305a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 2306a3b80e10SAndrii Nakryiko } 2307a3b80e10SAndrii Nakryiko 230898868668SAndrii Nakryiko /* Clean up bpf_link and corresponding anon_inode file and FD. After 230998868668SAndrii Nakryiko * anon_inode is created, bpf_link can't be just kfree()'d due to deferred 2310a3b80e10SAndrii Nakryiko * anon_inode's release() call. This helper marksbpf_link as 2311a3b80e10SAndrii Nakryiko * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt 2312a3b80e10SAndrii Nakryiko * is not decremented, it's the responsibility of a calling code that failed 2313a3b80e10SAndrii Nakryiko * to complete bpf_link initialization. 231498868668SAndrii Nakryiko */ 2315a3b80e10SAndrii Nakryiko void bpf_link_cleanup(struct bpf_link_primer *primer) 2316babf3164SAndrii Nakryiko { 2317a3b80e10SAndrii Nakryiko primer->link->prog = NULL; 2318a3b80e10SAndrii Nakryiko bpf_link_free_id(primer->id); 2319a3b80e10SAndrii Nakryiko fput(primer->file); 2320a3b80e10SAndrii Nakryiko put_unused_fd(primer->fd); 2321babf3164SAndrii Nakryiko } 2322babf3164SAndrii Nakryiko 232370ed506cSAndrii Nakryiko void bpf_link_inc(struct bpf_link *link) 232470ed506cSAndrii Nakryiko { 232570ed506cSAndrii Nakryiko atomic64_inc(&link->refcnt); 232670ed506cSAndrii Nakryiko } 232770ed506cSAndrii Nakryiko 232870ed506cSAndrii Nakryiko /* bpf_link_free is guaranteed to be called from process context */ 232970ed506cSAndrii Nakryiko static void bpf_link_free(struct bpf_link *link) 233070ed506cSAndrii Nakryiko { 2331a3b80e10SAndrii Nakryiko bpf_link_free_id(link->id); 2332babf3164SAndrii Nakryiko if (link->prog) { 2333babf3164SAndrii Nakryiko /* detach BPF program, clean up used resources */ 233470ed506cSAndrii Nakryiko link->ops->release(link); 2335babf3164SAndrii Nakryiko bpf_prog_put(link->prog); 2336babf3164SAndrii Nakryiko } 2337babf3164SAndrii Nakryiko /* free bpf_link and its containing memory */ 2338babf3164SAndrii Nakryiko link->ops->dealloc(link); 233970ed506cSAndrii Nakryiko } 234070ed506cSAndrii Nakryiko 234170ed506cSAndrii Nakryiko static void bpf_link_put_deferred(struct work_struct *work) 234270ed506cSAndrii Nakryiko { 234370ed506cSAndrii Nakryiko struct bpf_link *link = container_of(work, struct bpf_link, work); 234470ed506cSAndrii Nakryiko 234570ed506cSAndrii Nakryiko bpf_link_free(link); 234670ed506cSAndrii Nakryiko } 234770ed506cSAndrii Nakryiko 234870ed506cSAndrii Nakryiko /* bpf_link_put can be called from atomic context, but ensures that resources 234970ed506cSAndrii Nakryiko * are freed from process context 235070ed506cSAndrii Nakryiko */ 235170ed506cSAndrii Nakryiko void bpf_link_put(struct bpf_link *link) 235270ed506cSAndrii Nakryiko { 235370ed506cSAndrii Nakryiko if (!atomic64_dec_and_test(&link->refcnt)) 235470ed506cSAndrii Nakryiko return; 235570ed506cSAndrii Nakryiko 2356f00f2f7fSAlexei Starovoitov if (in_atomic()) { 235770ed506cSAndrii Nakryiko INIT_WORK(&link->work, bpf_link_put_deferred); 235870ed506cSAndrii Nakryiko schedule_work(&link->work); 2359f00f2f7fSAlexei Starovoitov } else { 2360f00f2f7fSAlexei Starovoitov bpf_link_free(link); 2361f00f2f7fSAlexei Starovoitov } 236270ed506cSAndrii Nakryiko } 236370ed506cSAndrii Nakryiko 236470ed506cSAndrii Nakryiko static int bpf_link_release(struct inode *inode, struct file *filp) 236570ed506cSAndrii Nakryiko { 236670ed506cSAndrii Nakryiko struct bpf_link *link = filp->private_data; 236770ed506cSAndrii Nakryiko 236870ed506cSAndrii Nakryiko bpf_link_put(link); 2369fec56f58SAlexei Starovoitov return 0; 2370fec56f58SAlexei Starovoitov } 2371fec56f58SAlexei Starovoitov 237270ed506cSAndrii Nakryiko #ifdef CONFIG_PROC_FS 2373f2e10bffSAndrii Nakryiko #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) 2374f2e10bffSAndrii Nakryiko #define BPF_MAP_TYPE(_id, _ops) 2375f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) [_id] = #_name, 2376f2e10bffSAndrii Nakryiko static const char *bpf_link_type_strs[] = { 2377f2e10bffSAndrii Nakryiko [BPF_LINK_TYPE_UNSPEC] = "<invalid>", 2378f2e10bffSAndrii Nakryiko #include <linux/bpf_types.h> 2379f2e10bffSAndrii Nakryiko }; 2380f2e10bffSAndrii Nakryiko #undef BPF_PROG_TYPE 2381f2e10bffSAndrii Nakryiko #undef BPF_MAP_TYPE 2382f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 238370ed506cSAndrii Nakryiko 238470ed506cSAndrii Nakryiko static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) 238570ed506cSAndrii Nakryiko { 238670ed506cSAndrii Nakryiko const struct bpf_link *link = filp->private_data; 238770ed506cSAndrii Nakryiko const struct bpf_prog *prog = link->prog; 238870ed506cSAndrii Nakryiko char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 238970ed506cSAndrii Nakryiko 239070ed506cSAndrii Nakryiko bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 239170ed506cSAndrii Nakryiko seq_printf(m, 239270ed506cSAndrii Nakryiko "link_type:\t%s\n" 2393a3b80e10SAndrii Nakryiko "link_id:\t%u\n" 239470ed506cSAndrii Nakryiko "prog_tag:\t%s\n" 239570ed506cSAndrii Nakryiko "prog_id:\t%u\n", 2396f2e10bffSAndrii Nakryiko bpf_link_type_strs[link->type], 2397a3b80e10SAndrii Nakryiko link->id, 239870ed506cSAndrii Nakryiko prog_tag, 239970ed506cSAndrii Nakryiko prog->aux->id); 2400f2e10bffSAndrii Nakryiko if (link->ops->show_fdinfo) 2401f2e10bffSAndrii Nakryiko link->ops->show_fdinfo(link, m); 240270ed506cSAndrii Nakryiko } 240370ed506cSAndrii Nakryiko #endif 240470ed506cSAndrii Nakryiko 24056f302bfbSZou Wei static const struct file_operations bpf_link_fops = { 240670ed506cSAndrii Nakryiko #ifdef CONFIG_PROC_FS 240770ed506cSAndrii Nakryiko .show_fdinfo = bpf_link_show_fdinfo, 240870ed506cSAndrii Nakryiko #endif 240970ed506cSAndrii Nakryiko .release = bpf_link_release, 2410fec56f58SAlexei Starovoitov .read = bpf_dummy_read, 2411fec56f58SAlexei Starovoitov .write = bpf_dummy_write, 2412fec56f58SAlexei Starovoitov }; 2413fec56f58SAlexei Starovoitov 2414a3b80e10SAndrii Nakryiko static int bpf_link_alloc_id(struct bpf_link *link) 241570ed506cSAndrii Nakryiko { 2416a3b80e10SAndrii Nakryiko int id; 2417a3b80e10SAndrii Nakryiko 2418a3b80e10SAndrii Nakryiko idr_preload(GFP_KERNEL); 2419a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 2420a3b80e10SAndrii Nakryiko id = idr_alloc_cyclic(&link_idr, link, 1, INT_MAX, GFP_ATOMIC); 2421a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 2422a3b80e10SAndrii Nakryiko idr_preload_end(); 2423a3b80e10SAndrii Nakryiko 2424a3b80e10SAndrii Nakryiko return id; 242570ed506cSAndrii Nakryiko } 242670ed506cSAndrii Nakryiko 2427a3b80e10SAndrii Nakryiko /* Prepare bpf_link to be exposed to user-space by allocating anon_inode file, 2428a3b80e10SAndrii Nakryiko * reserving unused FD and allocating ID from link_idr. This is to be paired 2429a3b80e10SAndrii Nakryiko * with bpf_link_settle() to install FD and ID and expose bpf_link to 2430a3b80e10SAndrii Nakryiko * user-space, if bpf_link is successfully attached. If not, bpf_link and 2431a3b80e10SAndrii Nakryiko * pre-allocated resources are to be freed with bpf_cleanup() call. All the 2432a3b80e10SAndrii Nakryiko * transient state is passed around in struct bpf_link_primer. 2433a3b80e10SAndrii Nakryiko * This is preferred way to create and initialize bpf_link, especially when 2434a3b80e10SAndrii Nakryiko * there are complicated and expensive operations inbetween creating bpf_link 2435a3b80e10SAndrii Nakryiko * itself and attaching it to BPF hook. By using bpf_link_prime() and 2436a3b80e10SAndrii Nakryiko * bpf_link_settle() kernel code using bpf_link doesn't have to perform 2437a3b80e10SAndrii Nakryiko * expensive (and potentially failing) roll back operations in a rare case 2438a3b80e10SAndrii Nakryiko * that file, FD, or ID can't be allocated. 2439babf3164SAndrii Nakryiko */ 2440a3b80e10SAndrii Nakryiko int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer) 2441babf3164SAndrii Nakryiko { 2442babf3164SAndrii Nakryiko struct file *file; 2443a3b80e10SAndrii Nakryiko int fd, id; 2444babf3164SAndrii Nakryiko 2445babf3164SAndrii Nakryiko fd = get_unused_fd_flags(O_CLOEXEC); 2446babf3164SAndrii Nakryiko if (fd < 0) 2447a3b80e10SAndrii Nakryiko return fd; 2448babf3164SAndrii Nakryiko 2449babf3164SAndrii Nakryiko 2450a3b80e10SAndrii Nakryiko id = bpf_link_alloc_id(link); 2451a3b80e10SAndrii Nakryiko if (id < 0) { 2452a3b80e10SAndrii Nakryiko put_unused_fd(fd); 2453a3b80e10SAndrii Nakryiko return id; 2454a3b80e10SAndrii Nakryiko } 2455babf3164SAndrii Nakryiko 2456babf3164SAndrii Nakryiko file = anon_inode_getfile("bpf_link", &bpf_link_fops, link, O_CLOEXEC); 2457babf3164SAndrii Nakryiko if (IS_ERR(file)) { 2458138c6767SAndrii Nakryiko bpf_link_free_id(id); 2459babf3164SAndrii Nakryiko put_unused_fd(fd); 2460138c6767SAndrii Nakryiko return PTR_ERR(file); 2461babf3164SAndrii Nakryiko } 2462babf3164SAndrii Nakryiko 2463a3b80e10SAndrii Nakryiko primer->link = link; 2464a3b80e10SAndrii Nakryiko primer->file = file; 2465a3b80e10SAndrii Nakryiko primer->fd = fd; 2466a3b80e10SAndrii Nakryiko primer->id = id; 2467a3b80e10SAndrii Nakryiko return 0; 2468a3b80e10SAndrii Nakryiko } 2469a3b80e10SAndrii Nakryiko 2470a3b80e10SAndrii Nakryiko int bpf_link_settle(struct bpf_link_primer *primer) 2471a3b80e10SAndrii Nakryiko { 2472a3b80e10SAndrii Nakryiko /* make bpf_link fetchable by ID */ 2473a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 2474a3b80e10SAndrii Nakryiko primer->link->id = primer->id; 2475a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 2476a3b80e10SAndrii Nakryiko /* make bpf_link fetchable by FD */ 2477a3b80e10SAndrii Nakryiko fd_install(primer->fd, primer->file); 2478a3b80e10SAndrii Nakryiko /* pass through installed FD */ 2479a3b80e10SAndrii Nakryiko return primer->fd; 2480a3b80e10SAndrii Nakryiko } 2481a3b80e10SAndrii Nakryiko 2482a3b80e10SAndrii Nakryiko int bpf_link_new_fd(struct bpf_link *link) 2483a3b80e10SAndrii Nakryiko { 2484a3b80e10SAndrii Nakryiko return anon_inode_getfd("bpf-link", &bpf_link_fops, link, O_CLOEXEC); 2485babf3164SAndrii Nakryiko } 2486babf3164SAndrii Nakryiko 248770ed506cSAndrii Nakryiko struct bpf_link *bpf_link_get_from_fd(u32 ufd) 248870ed506cSAndrii Nakryiko { 248970ed506cSAndrii Nakryiko struct fd f = fdget(ufd); 249070ed506cSAndrii Nakryiko struct bpf_link *link; 249170ed506cSAndrii Nakryiko 249270ed506cSAndrii Nakryiko if (!f.file) 249370ed506cSAndrii Nakryiko return ERR_PTR(-EBADF); 249470ed506cSAndrii Nakryiko if (f.file->f_op != &bpf_link_fops) { 249570ed506cSAndrii Nakryiko fdput(f); 249670ed506cSAndrii Nakryiko return ERR_PTR(-EINVAL); 249770ed506cSAndrii Nakryiko } 249870ed506cSAndrii Nakryiko 249970ed506cSAndrii Nakryiko link = f.file->private_data; 250070ed506cSAndrii Nakryiko bpf_link_inc(link); 250170ed506cSAndrii Nakryiko fdput(f); 250270ed506cSAndrii Nakryiko 250370ed506cSAndrii Nakryiko return link; 250470ed506cSAndrii Nakryiko } 250570ed506cSAndrii Nakryiko 250670ed506cSAndrii Nakryiko struct bpf_tracing_link { 250770ed506cSAndrii Nakryiko struct bpf_link link; 2508f2e10bffSAndrii Nakryiko enum bpf_attach_type attach_type; 25093aac1eadSToke Høiland-Jørgensen struct bpf_trampoline *trampoline; 25103aac1eadSToke Høiland-Jørgensen struct bpf_prog *tgt_prog; 251170ed506cSAndrii Nakryiko }; 251270ed506cSAndrii Nakryiko 251370ed506cSAndrii Nakryiko static void bpf_tracing_link_release(struct bpf_link *link) 251470ed506cSAndrii Nakryiko { 25153aac1eadSToke Høiland-Jørgensen struct bpf_tracing_link *tr_link = 25163aac1eadSToke Høiland-Jørgensen container_of(link, struct bpf_tracing_link, link); 25173aac1eadSToke Høiland-Jørgensen 25183aac1eadSToke Høiland-Jørgensen WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog, 25193aac1eadSToke Høiland-Jørgensen tr_link->trampoline)); 25203aac1eadSToke Høiland-Jørgensen 25213aac1eadSToke Høiland-Jørgensen bpf_trampoline_put(tr_link->trampoline); 25223aac1eadSToke Høiland-Jørgensen 25233aac1eadSToke Høiland-Jørgensen /* tgt_prog is NULL if target is a kernel function */ 25243aac1eadSToke Høiland-Jørgensen if (tr_link->tgt_prog) 25253aac1eadSToke Høiland-Jørgensen bpf_prog_put(tr_link->tgt_prog); 2526babf3164SAndrii Nakryiko } 2527babf3164SAndrii Nakryiko 2528babf3164SAndrii Nakryiko static void bpf_tracing_link_dealloc(struct bpf_link *link) 2529babf3164SAndrii Nakryiko { 253070ed506cSAndrii Nakryiko struct bpf_tracing_link *tr_link = 253170ed506cSAndrii Nakryiko container_of(link, struct bpf_tracing_link, link); 253270ed506cSAndrii Nakryiko 253370ed506cSAndrii Nakryiko kfree(tr_link); 253470ed506cSAndrii Nakryiko } 253570ed506cSAndrii Nakryiko 2536f2e10bffSAndrii Nakryiko static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link, 2537f2e10bffSAndrii Nakryiko struct seq_file *seq) 2538f2e10bffSAndrii Nakryiko { 2539f2e10bffSAndrii Nakryiko struct bpf_tracing_link *tr_link = 2540f2e10bffSAndrii Nakryiko container_of(link, struct bpf_tracing_link, link); 2541f2e10bffSAndrii Nakryiko 2542f2e10bffSAndrii Nakryiko seq_printf(seq, 2543f2e10bffSAndrii Nakryiko "attach_type:\t%d\n", 2544f2e10bffSAndrii Nakryiko tr_link->attach_type); 2545f2e10bffSAndrii Nakryiko } 2546f2e10bffSAndrii Nakryiko 2547f2e10bffSAndrii Nakryiko static int bpf_tracing_link_fill_link_info(const struct bpf_link *link, 2548f2e10bffSAndrii Nakryiko struct bpf_link_info *info) 2549f2e10bffSAndrii Nakryiko { 2550f2e10bffSAndrii Nakryiko struct bpf_tracing_link *tr_link = 2551f2e10bffSAndrii Nakryiko container_of(link, struct bpf_tracing_link, link); 2552f2e10bffSAndrii Nakryiko 2553f2e10bffSAndrii Nakryiko info->tracing.attach_type = tr_link->attach_type; 2554f2e10bffSAndrii Nakryiko 2555f2e10bffSAndrii Nakryiko return 0; 2556f2e10bffSAndrii Nakryiko } 2557f2e10bffSAndrii Nakryiko 255870ed506cSAndrii Nakryiko static const struct bpf_link_ops bpf_tracing_link_lops = { 255970ed506cSAndrii Nakryiko .release = bpf_tracing_link_release, 2560babf3164SAndrii Nakryiko .dealloc = bpf_tracing_link_dealloc, 2561f2e10bffSAndrii Nakryiko .show_fdinfo = bpf_tracing_link_show_fdinfo, 2562f2e10bffSAndrii Nakryiko .fill_link_info = bpf_tracing_link_fill_link_info, 256370ed506cSAndrii Nakryiko }; 256470ed506cSAndrii Nakryiko 25654a1e7c0cSToke Høiland-Jørgensen static int bpf_tracing_prog_attach(struct bpf_prog *prog, 25664a1e7c0cSToke Høiland-Jørgensen int tgt_prog_fd, 25674a1e7c0cSToke Høiland-Jørgensen u32 btf_id) 2568fec56f58SAlexei Starovoitov { 2569a3b80e10SAndrii Nakryiko struct bpf_link_primer link_primer; 25703aac1eadSToke Høiland-Jørgensen struct bpf_prog *tgt_prog = NULL; 25714a1e7c0cSToke Høiland-Jørgensen struct bpf_trampoline *tr = NULL; 257270ed506cSAndrii Nakryiko struct bpf_tracing_link *link; 25734a1e7c0cSToke Høiland-Jørgensen u64 key = 0; 2574a3b80e10SAndrii Nakryiko int err; 2575fec56f58SAlexei Starovoitov 25769e4e01dfSKP Singh switch (prog->type) { 25779e4e01dfSKP Singh case BPF_PROG_TYPE_TRACING: 2578fec56f58SAlexei Starovoitov if (prog->expected_attach_type != BPF_TRACE_FENTRY && 2579be8704ffSAlexei Starovoitov prog->expected_attach_type != BPF_TRACE_FEXIT && 25809e4e01dfSKP Singh prog->expected_attach_type != BPF_MODIFY_RETURN) { 25819e4e01dfSKP Singh err = -EINVAL; 25829e4e01dfSKP Singh goto out_put_prog; 25839e4e01dfSKP Singh } 25849e4e01dfSKP Singh break; 25859e4e01dfSKP Singh case BPF_PROG_TYPE_EXT: 25869e4e01dfSKP Singh if (prog->expected_attach_type != 0) { 25879e4e01dfSKP Singh err = -EINVAL; 25889e4e01dfSKP Singh goto out_put_prog; 25899e4e01dfSKP Singh } 25909e4e01dfSKP Singh break; 25919e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 25929e4e01dfSKP Singh if (prog->expected_attach_type != BPF_LSM_MAC) { 25939e4e01dfSKP Singh err = -EINVAL; 25949e4e01dfSKP Singh goto out_put_prog; 25959e4e01dfSKP Singh } 25969e4e01dfSKP Singh break; 25979e4e01dfSKP Singh default: 2598fec56f58SAlexei Starovoitov err = -EINVAL; 2599fec56f58SAlexei Starovoitov goto out_put_prog; 2600fec56f58SAlexei Starovoitov } 2601fec56f58SAlexei Starovoitov 26024a1e7c0cSToke Høiland-Jørgensen if (!!tgt_prog_fd != !!btf_id) { 26034a1e7c0cSToke Høiland-Jørgensen err = -EINVAL; 26044a1e7c0cSToke Høiland-Jørgensen goto out_put_prog; 26054a1e7c0cSToke Høiland-Jørgensen } 26064a1e7c0cSToke Høiland-Jørgensen 26074a1e7c0cSToke Høiland-Jørgensen if (tgt_prog_fd) { 26084a1e7c0cSToke Høiland-Jørgensen /* For now we only allow new targets for BPF_PROG_TYPE_EXT */ 26094a1e7c0cSToke Høiland-Jørgensen if (prog->type != BPF_PROG_TYPE_EXT) { 26104a1e7c0cSToke Høiland-Jørgensen err = -EINVAL; 26114a1e7c0cSToke Høiland-Jørgensen goto out_put_prog; 26124a1e7c0cSToke Høiland-Jørgensen } 26134a1e7c0cSToke Høiland-Jørgensen 26144a1e7c0cSToke Høiland-Jørgensen tgt_prog = bpf_prog_get(tgt_prog_fd); 26154a1e7c0cSToke Høiland-Jørgensen if (IS_ERR(tgt_prog)) { 26164a1e7c0cSToke Høiland-Jørgensen err = PTR_ERR(tgt_prog); 26174a1e7c0cSToke Høiland-Jørgensen tgt_prog = NULL; 26184a1e7c0cSToke Høiland-Jørgensen goto out_put_prog; 26194a1e7c0cSToke Høiland-Jørgensen } 26204a1e7c0cSToke Høiland-Jørgensen 262122dc4a0fSAndrii Nakryiko key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id); 26224a1e7c0cSToke Høiland-Jørgensen } 26234a1e7c0cSToke Høiland-Jørgensen 262470ed506cSAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER); 262570ed506cSAndrii Nakryiko if (!link) { 262670ed506cSAndrii Nakryiko err = -ENOMEM; 2627fec56f58SAlexei Starovoitov goto out_put_prog; 2628fec56f58SAlexei Starovoitov } 2629f2e10bffSAndrii Nakryiko bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING, 2630f2e10bffSAndrii Nakryiko &bpf_tracing_link_lops, prog); 2631f2e10bffSAndrii Nakryiko link->attach_type = prog->expected_attach_type; 2632fec56f58SAlexei Starovoitov 26333aac1eadSToke Høiland-Jørgensen mutex_lock(&prog->aux->dst_mutex); 2634babf3164SAndrii Nakryiko 26354a1e7c0cSToke Høiland-Jørgensen /* There are a few possible cases here: 26364a1e7c0cSToke Høiland-Jørgensen * 26374a1e7c0cSToke Høiland-Jørgensen * - if prog->aux->dst_trampoline is set, the program was just loaded 26384a1e7c0cSToke Høiland-Jørgensen * and not yet attached to anything, so we can use the values stored 26394a1e7c0cSToke Høiland-Jørgensen * in prog->aux 26404a1e7c0cSToke Høiland-Jørgensen * 26414a1e7c0cSToke Høiland-Jørgensen * - if prog->aux->dst_trampoline is NULL, the program has already been 26424a1e7c0cSToke Høiland-Jørgensen * attached to a target and its initial target was cleared (below) 26434a1e7c0cSToke Høiland-Jørgensen * 26444a1e7c0cSToke Høiland-Jørgensen * - if tgt_prog != NULL, the caller specified tgt_prog_fd + 26454a1e7c0cSToke Høiland-Jørgensen * target_btf_id using the link_create API. 26464a1e7c0cSToke Høiland-Jørgensen * 26474a1e7c0cSToke Høiland-Jørgensen * - if tgt_prog == NULL when this function was called using the old 26484a1e7c0cSToke Høiland-Jørgensen * raw_tracepoint_open API, and we need a target from prog->aux 26494a1e7c0cSToke Høiland-Jørgensen * 26504a1e7c0cSToke Høiland-Jørgensen * The combination of no saved target in prog->aux, and no target 26514a1e7c0cSToke Høiland-Jørgensen * specified on load is illegal, and we reject that here. 26524a1e7c0cSToke Høiland-Jørgensen */ 26534a1e7c0cSToke Høiland-Jørgensen if (!prog->aux->dst_trampoline && !tgt_prog) { 26543aac1eadSToke Høiland-Jørgensen err = -ENOENT; 26553aac1eadSToke Høiland-Jørgensen goto out_unlock; 26563aac1eadSToke Høiland-Jørgensen } 26574a1e7c0cSToke Høiland-Jørgensen 26584a1e7c0cSToke Høiland-Jørgensen if (!prog->aux->dst_trampoline || 26594a1e7c0cSToke Høiland-Jørgensen (key && key != prog->aux->dst_trampoline->key)) { 26604a1e7c0cSToke Høiland-Jørgensen /* If there is no saved target, or the specified target is 26614a1e7c0cSToke Høiland-Jørgensen * different from the destination specified at load time, we 26624a1e7c0cSToke Høiland-Jørgensen * need a new trampoline and a check for compatibility 26634a1e7c0cSToke Høiland-Jørgensen */ 26644a1e7c0cSToke Høiland-Jørgensen struct bpf_attach_target_info tgt_info = {}; 26654a1e7c0cSToke Høiland-Jørgensen 26664a1e7c0cSToke Høiland-Jørgensen err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id, 26674a1e7c0cSToke Høiland-Jørgensen &tgt_info); 26684a1e7c0cSToke Høiland-Jørgensen if (err) 26694a1e7c0cSToke Høiland-Jørgensen goto out_unlock; 26704a1e7c0cSToke Høiland-Jørgensen 26714a1e7c0cSToke Høiland-Jørgensen tr = bpf_trampoline_get(key, &tgt_info); 26724a1e7c0cSToke Høiland-Jørgensen if (!tr) { 26734a1e7c0cSToke Høiland-Jørgensen err = -ENOMEM; 26744a1e7c0cSToke Høiland-Jørgensen goto out_unlock; 26754a1e7c0cSToke Høiland-Jørgensen } 26764a1e7c0cSToke Høiland-Jørgensen } else { 26774a1e7c0cSToke Høiland-Jørgensen /* The caller didn't specify a target, or the target was the 26784a1e7c0cSToke Høiland-Jørgensen * same as the destination supplied during program load. This 26794a1e7c0cSToke Høiland-Jørgensen * means we can reuse the trampoline and reference from program 26804a1e7c0cSToke Høiland-Jørgensen * load time, and there is no need to allocate a new one. This 26814a1e7c0cSToke Høiland-Jørgensen * can only happen once for any program, as the saved values in 26824a1e7c0cSToke Høiland-Jørgensen * prog->aux are cleared below. 26834a1e7c0cSToke Høiland-Jørgensen */ 26843aac1eadSToke Høiland-Jørgensen tr = prog->aux->dst_trampoline; 26853aac1eadSToke Høiland-Jørgensen tgt_prog = prog->aux->dst_prog; 26864a1e7c0cSToke Høiland-Jørgensen } 26873aac1eadSToke Høiland-Jørgensen 26883aac1eadSToke Høiland-Jørgensen err = bpf_link_prime(&link->link, &link_primer); 26893aac1eadSToke Høiland-Jørgensen if (err) 26903aac1eadSToke Høiland-Jørgensen goto out_unlock; 26913aac1eadSToke Høiland-Jørgensen 26923aac1eadSToke Høiland-Jørgensen err = bpf_trampoline_link_prog(prog, tr); 2693babf3164SAndrii Nakryiko if (err) { 2694a3b80e10SAndrii Nakryiko bpf_link_cleanup(&link_primer); 26953aac1eadSToke Høiland-Jørgensen link = NULL; 26963aac1eadSToke Høiland-Jørgensen goto out_unlock; 2697babf3164SAndrii Nakryiko } 2698babf3164SAndrii Nakryiko 26993aac1eadSToke Høiland-Jørgensen link->tgt_prog = tgt_prog; 27003aac1eadSToke Høiland-Jørgensen link->trampoline = tr; 27013aac1eadSToke Høiland-Jørgensen 27024a1e7c0cSToke Høiland-Jørgensen /* Always clear the trampoline and target prog from prog->aux to make 27034a1e7c0cSToke Høiland-Jørgensen * sure the original attach destination is not kept alive after a 27044a1e7c0cSToke Høiland-Jørgensen * program is (re-)attached to another target. 27054a1e7c0cSToke Høiland-Jørgensen */ 27064a1e7c0cSToke Høiland-Jørgensen if (prog->aux->dst_prog && 27074a1e7c0cSToke Høiland-Jørgensen (tgt_prog_fd || tr != prog->aux->dst_trampoline)) 27084a1e7c0cSToke Høiland-Jørgensen /* got extra prog ref from syscall, or attaching to different prog */ 27094a1e7c0cSToke Høiland-Jørgensen bpf_prog_put(prog->aux->dst_prog); 27104a1e7c0cSToke Høiland-Jørgensen if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline) 27114a1e7c0cSToke Høiland-Jørgensen /* we allocated a new trampoline, so free the old one */ 27124a1e7c0cSToke Høiland-Jørgensen bpf_trampoline_put(prog->aux->dst_trampoline); 27134a1e7c0cSToke Høiland-Jørgensen 27143aac1eadSToke Høiland-Jørgensen prog->aux->dst_prog = NULL; 27153aac1eadSToke Høiland-Jørgensen prog->aux->dst_trampoline = NULL; 27163aac1eadSToke Høiland-Jørgensen mutex_unlock(&prog->aux->dst_mutex); 27173aac1eadSToke Høiland-Jørgensen 2718a3b80e10SAndrii Nakryiko return bpf_link_settle(&link_primer); 27193aac1eadSToke Høiland-Jørgensen out_unlock: 27204a1e7c0cSToke Høiland-Jørgensen if (tr && tr != prog->aux->dst_trampoline) 27214a1e7c0cSToke Høiland-Jørgensen bpf_trampoline_put(tr); 27223aac1eadSToke Høiland-Jørgensen mutex_unlock(&prog->aux->dst_mutex); 27233aac1eadSToke Høiland-Jørgensen kfree(link); 2724fec56f58SAlexei Starovoitov out_put_prog: 27254a1e7c0cSToke Høiland-Jørgensen if (tgt_prog_fd && tgt_prog) 27264a1e7c0cSToke Høiland-Jørgensen bpf_prog_put(tgt_prog); 2727fec56f58SAlexei Starovoitov return err; 2728fec56f58SAlexei Starovoitov } 2729fec56f58SAlexei Starovoitov 273070ed506cSAndrii Nakryiko struct bpf_raw_tp_link { 273170ed506cSAndrii Nakryiko struct bpf_link link; 2732c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 2733c4f6699dSAlexei Starovoitov }; 2734c4f6699dSAlexei Starovoitov 273570ed506cSAndrii Nakryiko static void bpf_raw_tp_link_release(struct bpf_link *link) 2736c4f6699dSAlexei Starovoitov { 273770ed506cSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 273870ed506cSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 2739c4f6699dSAlexei Starovoitov 274070ed506cSAndrii Nakryiko bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog); 2741a38d1107SMatt Mullins bpf_put_raw_tracepoint(raw_tp->btp); 2742babf3164SAndrii Nakryiko } 2743babf3164SAndrii Nakryiko 2744babf3164SAndrii Nakryiko static void bpf_raw_tp_link_dealloc(struct bpf_link *link) 2745babf3164SAndrii Nakryiko { 2746babf3164SAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 2747babf3164SAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 2748babf3164SAndrii Nakryiko 2749c4f6699dSAlexei Starovoitov kfree(raw_tp); 2750c4f6699dSAlexei Starovoitov } 2751c4f6699dSAlexei Starovoitov 2752f2e10bffSAndrii Nakryiko static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link, 2753f2e10bffSAndrii Nakryiko struct seq_file *seq) 2754f2e10bffSAndrii Nakryiko { 2755f2e10bffSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp_link = 2756f2e10bffSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 2757f2e10bffSAndrii Nakryiko 2758f2e10bffSAndrii Nakryiko seq_printf(seq, 2759f2e10bffSAndrii Nakryiko "tp_name:\t%s\n", 2760f2e10bffSAndrii Nakryiko raw_tp_link->btp->tp->name); 2761f2e10bffSAndrii Nakryiko } 2762f2e10bffSAndrii Nakryiko 2763f2e10bffSAndrii Nakryiko static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link, 2764f2e10bffSAndrii Nakryiko struct bpf_link_info *info) 2765f2e10bffSAndrii Nakryiko { 2766f2e10bffSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp_link = 2767f2e10bffSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 2768f2e10bffSAndrii Nakryiko char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name); 2769f2e10bffSAndrii Nakryiko const char *tp_name = raw_tp_link->btp->tp->name; 2770f2e10bffSAndrii Nakryiko u32 ulen = info->raw_tracepoint.tp_name_len; 2771f2e10bffSAndrii Nakryiko size_t tp_len = strlen(tp_name); 2772f2e10bffSAndrii Nakryiko 2773b474959dSYonghong Song if (!ulen ^ !ubuf) 2774f2e10bffSAndrii Nakryiko return -EINVAL; 2775f2e10bffSAndrii Nakryiko 2776f2e10bffSAndrii Nakryiko info->raw_tracepoint.tp_name_len = tp_len + 1; 2777f2e10bffSAndrii Nakryiko 2778f2e10bffSAndrii Nakryiko if (!ubuf) 2779f2e10bffSAndrii Nakryiko return 0; 2780f2e10bffSAndrii Nakryiko 2781f2e10bffSAndrii Nakryiko if (ulen >= tp_len + 1) { 2782f2e10bffSAndrii Nakryiko if (copy_to_user(ubuf, tp_name, tp_len + 1)) 2783f2e10bffSAndrii Nakryiko return -EFAULT; 2784f2e10bffSAndrii Nakryiko } else { 2785f2e10bffSAndrii Nakryiko char zero = '\0'; 2786f2e10bffSAndrii Nakryiko 2787f2e10bffSAndrii Nakryiko if (copy_to_user(ubuf, tp_name, ulen - 1)) 2788f2e10bffSAndrii Nakryiko return -EFAULT; 2789f2e10bffSAndrii Nakryiko if (put_user(zero, ubuf + ulen - 1)) 2790f2e10bffSAndrii Nakryiko return -EFAULT; 2791f2e10bffSAndrii Nakryiko return -ENOSPC; 2792f2e10bffSAndrii Nakryiko } 2793f2e10bffSAndrii Nakryiko 2794f2e10bffSAndrii Nakryiko return 0; 2795f2e10bffSAndrii Nakryiko } 2796f2e10bffSAndrii Nakryiko 2797a3b80e10SAndrii Nakryiko static const struct bpf_link_ops bpf_raw_tp_link_lops = { 279870ed506cSAndrii Nakryiko .release = bpf_raw_tp_link_release, 2799babf3164SAndrii Nakryiko .dealloc = bpf_raw_tp_link_dealloc, 2800f2e10bffSAndrii Nakryiko .show_fdinfo = bpf_raw_tp_link_show_fdinfo, 2801f2e10bffSAndrii Nakryiko .fill_link_info = bpf_raw_tp_link_fill_link_info, 2802c4f6699dSAlexei Starovoitov }; 2803c4f6699dSAlexei Starovoitov 2804c4f6699dSAlexei Starovoitov #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd 2805c4f6699dSAlexei Starovoitov 2806c4f6699dSAlexei Starovoitov static int bpf_raw_tracepoint_open(const union bpf_attr *attr) 2807c4f6699dSAlexei Starovoitov { 2808a3b80e10SAndrii Nakryiko struct bpf_link_primer link_primer; 2809babf3164SAndrii Nakryiko struct bpf_raw_tp_link *link; 2810c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 2811c4f6699dSAlexei Starovoitov struct bpf_prog *prog; 2812ac4414b5SAlexei Starovoitov const char *tp_name; 2813ac4414b5SAlexei Starovoitov char buf[128]; 2814a3b80e10SAndrii Nakryiko int err; 2815c4f6699dSAlexei Starovoitov 2816ac4414b5SAlexei Starovoitov if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN)) 2817ac4414b5SAlexei Starovoitov return -EINVAL; 2818ac4414b5SAlexei Starovoitov 2819ac4414b5SAlexei Starovoitov prog = bpf_prog_get(attr->raw_tracepoint.prog_fd); 2820ac4414b5SAlexei Starovoitov if (IS_ERR(prog)) 2821ac4414b5SAlexei Starovoitov return PTR_ERR(prog); 2822ac4414b5SAlexei Starovoitov 28239e4e01dfSKP Singh switch (prog->type) { 28249e4e01dfSKP Singh case BPF_PROG_TYPE_TRACING: 28259e4e01dfSKP Singh case BPF_PROG_TYPE_EXT: 28269e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 2827ac4414b5SAlexei Starovoitov if (attr->raw_tracepoint.name) { 2828fec56f58SAlexei Starovoitov /* The attach point for this category of programs 2829fec56f58SAlexei Starovoitov * should be specified via btf_id during program load. 2830ac4414b5SAlexei Starovoitov */ 2831ac4414b5SAlexei Starovoitov err = -EINVAL; 2832ac4414b5SAlexei Starovoitov goto out_put_prog; 2833ac4414b5SAlexei Starovoitov } 28349e4e01dfSKP Singh if (prog->type == BPF_PROG_TYPE_TRACING && 28359e4e01dfSKP Singh prog->expected_attach_type == BPF_TRACE_RAW_TP) { 283638207291SMartin KaFai Lau tp_name = prog->aux->attach_func_name; 28379e4e01dfSKP Singh break; 28389e4e01dfSKP Singh } 28395541075aSJiri Olsa err = bpf_tracing_prog_attach(prog, 0, 0); 28405541075aSJiri Olsa if (err >= 0) 28415541075aSJiri Olsa return err; 28425541075aSJiri Olsa goto out_put_prog; 28439e4e01dfSKP Singh case BPF_PROG_TYPE_RAW_TRACEPOINT: 28449e4e01dfSKP Singh case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 2845ac4414b5SAlexei Starovoitov if (strncpy_from_user(buf, 2846ac4414b5SAlexei Starovoitov u64_to_user_ptr(attr->raw_tracepoint.name), 2847ac4414b5SAlexei Starovoitov sizeof(buf) - 1) < 0) { 2848ac4414b5SAlexei Starovoitov err = -EFAULT; 2849ac4414b5SAlexei Starovoitov goto out_put_prog; 2850ac4414b5SAlexei Starovoitov } 2851ac4414b5SAlexei Starovoitov buf[sizeof(buf) - 1] = 0; 2852ac4414b5SAlexei Starovoitov tp_name = buf; 28539e4e01dfSKP Singh break; 28549e4e01dfSKP Singh default: 28559e4e01dfSKP Singh err = -EINVAL; 28569e4e01dfSKP Singh goto out_put_prog; 2857ac4414b5SAlexei Starovoitov } 2858c4f6699dSAlexei Starovoitov 2859a38d1107SMatt Mullins btp = bpf_get_raw_tracepoint(tp_name); 2860ac4414b5SAlexei Starovoitov if (!btp) { 2861ac4414b5SAlexei Starovoitov err = -ENOENT; 2862ac4414b5SAlexei Starovoitov goto out_put_prog; 2863ac4414b5SAlexei Starovoitov } 2864c4f6699dSAlexei Starovoitov 2865babf3164SAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER); 2866babf3164SAndrii Nakryiko if (!link) { 2867a38d1107SMatt Mullins err = -ENOMEM; 2868a38d1107SMatt Mullins goto out_put_btp; 2869a38d1107SMatt Mullins } 2870f2e10bffSAndrii Nakryiko bpf_link_init(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT, 2871f2e10bffSAndrii Nakryiko &bpf_raw_tp_link_lops, prog); 2872babf3164SAndrii Nakryiko link->btp = btp; 2873c4f6699dSAlexei Starovoitov 2874a3b80e10SAndrii Nakryiko err = bpf_link_prime(&link->link, &link_primer); 2875a3b80e10SAndrii Nakryiko if (err) { 2876babf3164SAndrii Nakryiko kfree(link); 2877babf3164SAndrii Nakryiko goto out_put_btp; 2878c4f6699dSAlexei Starovoitov } 2879babf3164SAndrii Nakryiko 2880babf3164SAndrii Nakryiko err = bpf_probe_register(link->btp, prog); 2881babf3164SAndrii Nakryiko if (err) { 2882a3b80e10SAndrii Nakryiko bpf_link_cleanup(&link_primer); 2883babf3164SAndrii Nakryiko goto out_put_btp; 2884babf3164SAndrii Nakryiko } 2885babf3164SAndrii Nakryiko 2886a3b80e10SAndrii Nakryiko return bpf_link_settle(&link_primer); 2887c4f6699dSAlexei Starovoitov 2888a38d1107SMatt Mullins out_put_btp: 2889a38d1107SMatt Mullins bpf_put_raw_tracepoint(btp); 2890ac4414b5SAlexei Starovoitov out_put_prog: 2891ac4414b5SAlexei Starovoitov bpf_prog_put(prog); 2892c4f6699dSAlexei Starovoitov return err; 2893c4f6699dSAlexei Starovoitov } 2894c4f6699dSAlexei Starovoitov 289533491588SAnders Roxell static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, 289633491588SAnders Roxell enum bpf_attach_type attach_type) 289733491588SAnders Roxell { 289833491588SAnders Roxell switch (prog->type) { 289933491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK: 290033491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 29010d01da6aSStanislav Fomichev case BPF_PROG_TYPE_CGROUP_SOCKOPT: 2902e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP: 290333491588SAnders Roxell return attach_type == prog->expected_attach_type ? 0 : -EINVAL; 29045cf1e914Sbrakmo case BPF_PROG_TYPE_CGROUP_SKB: 29052c78ee89SAlexei Starovoitov if (!capable(CAP_NET_ADMIN)) 29062c78ee89SAlexei Starovoitov /* cg-skb progs can be loaded by unpriv user. 29072c78ee89SAlexei Starovoitov * check permissions at attach time. 29082c78ee89SAlexei Starovoitov */ 29092c78ee89SAlexei Starovoitov return -EPERM; 29105cf1e914Sbrakmo return prog->enforce_expected_attach_type && 29115cf1e914Sbrakmo prog->expected_attach_type != attach_type ? 29125cf1e914Sbrakmo -EINVAL : 0; 291333491588SAnders Roxell default: 291433491588SAnders Roxell return 0; 291533491588SAnders Roxell } 291633491588SAnders Roxell } 291733491588SAnders Roxell 2918e28784e3SAndrii Nakryiko static enum bpf_prog_type 2919e28784e3SAndrii Nakryiko attach_type_to_prog_type(enum bpf_attach_type attach_type) 2920e28784e3SAndrii Nakryiko { 2921e28784e3SAndrii Nakryiko switch (attach_type) { 2922e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_INGRESS: 2923e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_EGRESS: 2924e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SKB; 2925e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_SOCK_CREATE: 2926f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 2927e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_POST_BIND: 2928e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_POST_BIND: 2929e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCK; 2930e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_BIND: 2931e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_BIND: 2932e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_CONNECT: 2933e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_CONNECT: 29341b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 29351b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 29361b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 29371b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 2938e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP4_SENDMSG: 2939e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP6_SENDMSG: 2940e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP4_RECVMSG: 2941e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP6_RECVMSG: 2942e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 2943e28784e3SAndrii Nakryiko case BPF_CGROUP_SOCK_OPS: 2944e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SOCK_OPS; 2945e28784e3SAndrii Nakryiko case BPF_CGROUP_DEVICE: 2946e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_DEVICE; 2947e28784e3SAndrii Nakryiko case BPF_SK_MSG_VERDICT: 2948e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SK_MSG; 2949e28784e3SAndrii Nakryiko case BPF_SK_SKB_STREAM_PARSER: 2950e28784e3SAndrii Nakryiko case BPF_SK_SKB_STREAM_VERDICT: 2951*a7ba4558SCong Wang case BPF_SK_SKB_VERDICT: 2952e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SK_SKB; 2953e28784e3SAndrii Nakryiko case BPF_LIRC_MODE2: 2954e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_LIRC_MODE2; 2955e28784e3SAndrii Nakryiko case BPF_FLOW_DISSECTOR: 2956e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_FLOW_DISSECTOR; 2957e28784e3SAndrii Nakryiko case BPF_CGROUP_SYSCTL: 2958e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SYSCTL; 2959e28784e3SAndrii Nakryiko case BPF_CGROUP_GETSOCKOPT: 2960e28784e3SAndrii Nakryiko case BPF_CGROUP_SETSOCKOPT: 2961e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCKOPT; 2962de4e05caSYonghong Song case BPF_TRACE_ITER: 2963de4e05caSYonghong Song return BPF_PROG_TYPE_TRACING; 2964e9ddbb77SJakub Sitnicki case BPF_SK_LOOKUP: 2965e9ddbb77SJakub Sitnicki return BPF_PROG_TYPE_SK_LOOKUP; 2966aa8d3a71SAndrii Nakryiko case BPF_XDP: 2967aa8d3a71SAndrii Nakryiko return BPF_PROG_TYPE_XDP; 2968e28784e3SAndrii Nakryiko default: 2969e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_UNSPEC; 2970e28784e3SAndrii Nakryiko } 2971e28784e3SAndrii Nakryiko } 2972e28784e3SAndrii Nakryiko 29737dd68b32SAndrey Ignatov #define BPF_PROG_ATTACH_LAST_FIELD replace_bpf_fd 2974174a79ffSJohn Fastabend 2975324bda9eSAlexei Starovoitov #define BPF_F_ATTACH_MASK \ 29767dd68b32SAndrey Ignatov (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI | BPF_F_REPLACE) 2977324bda9eSAlexei Starovoitov 2978f4324551SDaniel Mack static int bpf_prog_attach(const union bpf_attr *attr) 2979f4324551SDaniel Mack { 29807f677633SAlexei Starovoitov enum bpf_prog_type ptype; 2981f4324551SDaniel Mack struct bpf_prog *prog; 29827f677633SAlexei Starovoitov int ret; 2983f4324551SDaniel Mack 2984f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_ATTACH)) 2985f4324551SDaniel Mack return -EINVAL; 2986f4324551SDaniel Mack 2987324bda9eSAlexei Starovoitov if (attr->attach_flags & ~BPF_F_ATTACH_MASK) 29887f677633SAlexei Starovoitov return -EINVAL; 29897f677633SAlexei Starovoitov 2990e28784e3SAndrii Nakryiko ptype = attach_type_to_prog_type(attr->attach_type); 2991e28784e3SAndrii Nakryiko if (ptype == BPF_PROG_TYPE_UNSPEC) 2992b2cd1257SDavid Ahern return -EINVAL; 2993b2cd1257SDavid Ahern 2994b2cd1257SDavid Ahern prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 2995f4324551SDaniel Mack if (IS_ERR(prog)) 2996f4324551SDaniel Mack return PTR_ERR(prog); 2997f4324551SDaniel Mack 29985e43f899SAndrey Ignatov if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) { 29995e43f899SAndrey Ignatov bpf_prog_put(prog); 30005e43f899SAndrey Ignatov return -EINVAL; 30015e43f899SAndrey Ignatov } 30025e43f899SAndrey Ignatov 3003fdb5c453SSean Young switch (ptype) { 3004fdb5c453SSean Young case BPF_PROG_TYPE_SK_SKB: 3005fdb5c453SSean Young case BPF_PROG_TYPE_SK_MSG: 3006604326b4SDaniel Borkmann ret = sock_map_get_from_fd(attr, prog); 3007fdb5c453SSean Young break; 3008fdb5c453SSean Young case BPF_PROG_TYPE_LIRC_MODE2: 3009fdb5c453SSean Young ret = lirc_prog_attach(attr, prog); 3010fdb5c453SSean Young break; 3011d58e468bSPetar Penkov case BPF_PROG_TYPE_FLOW_DISSECTOR: 3012a3fd7ceeSJakub Sitnicki ret = netns_bpf_prog_attach(attr, prog); 3013d58e468bSPetar Penkov break; 3014e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 3015e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 3016e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 3017e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 3018e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 3019e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 3020e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 3021fdb5c453SSean Young ret = cgroup_bpf_prog_attach(attr, ptype, prog); 3022e28784e3SAndrii Nakryiko break; 3023e28784e3SAndrii Nakryiko default: 3024e28784e3SAndrii Nakryiko ret = -EINVAL; 3025f4324551SDaniel Mack } 3026f4324551SDaniel Mack 30277f677633SAlexei Starovoitov if (ret) 30287f677633SAlexei Starovoitov bpf_prog_put(prog); 30297f677633SAlexei Starovoitov return ret; 3030f4324551SDaniel Mack } 3031f4324551SDaniel Mack 3032f4324551SDaniel Mack #define BPF_PROG_DETACH_LAST_FIELD attach_type 3033f4324551SDaniel Mack 3034f4324551SDaniel Mack static int bpf_prog_detach(const union bpf_attr *attr) 3035f4324551SDaniel Mack { 3036324bda9eSAlexei Starovoitov enum bpf_prog_type ptype; 3037f4324551SDaniel Mack 3038f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_DETACH)) 3039f4324551SDaniel Mack return -EINVAL; 3040f4324551SDaniel Mack 3041e28784e3SAndrii Nakryiko ptype = attach_type_to_prog_type(attr->attach_type); 3042e28784e3SAndrii Nakryiko 3043e28784e3SAndrii Nakryiko switch (ptype) { 3044e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SK_MSG: 3045e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SK_SKB: 3046bb0de313SLorenz Bauer return sock_map_prog_detach(attr, ptype); 3047e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_LIRC_MODE2: 3048f4364dcfSSean Young return lirc_prog_detach(attr); 3049e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_FLOW_DISSECTOR: 30504ac2add6SLorenz Bauer return netns_bpf_prog_detach(attr, ptype); 3051e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 3052e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 3053e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 3054e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 3055e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 3056e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 3057e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 3058e28784e3SAndrii Nakryiko return cgroup_bpf_prog_detach(attr, ptype); 3059f4324551SDaniel Mack default: 3060f4324551SDaniel Mack return -EINVAL; 3061f4324551SDaniel Mack } 3062f4324551SDaniel Mack } 306340304b2aSLawrence Brakmo 3064468e2f64SAlexei Starovoitov #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt 3065468e2f64SAlexei Starovoitov 3066468e2f64SAlexei Starovoitov static int bpf_prog_query(const union bpf_attr *attr, 3067468e2f64SAlexei Starovoitov union bpf_attr __user *uattr) 3068468e2f64SAlexei Starovoitov { 3069468e2f64SAlexei Starovoitov if (!capable(CAP_NET_ADMIN)) 3070468e2f64SAlexei Starovoitov return -EPERM; 3071468e2f64SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_QUERY)) 3072468e2f64SAlexei Starovoitov return -EINVAL; 3073468e2f64SAlexei Starovoitov if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE) 3074468e2f64SAlexei Starovoitov return -EINVAL; 3075468e2f64SAlexei Starovoitov 3076468e2f64SAlexei Starovoitov switch (attr->query.attach_type) { 3077468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_INGRESS: 3078468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_EGRESS: 3079468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_SOCK_CREATE: 3080f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 30814fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 30824fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 3083aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 3084aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 3085d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 3086d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 30871b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 30881b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 30891b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 30901b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 30911cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 30921cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 3093983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG: 3094983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG: 3095468e2f64SAlexei Starovoitov case BPF_CGROUP_SOCK_OPS: 3096ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 30977b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 30980d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT: 30990d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT: 3100e28784e3SAndrii Nakryiko return cgroup_bpf_prog_query(attr, uattr); 3101f4364dcfSSean Young case BPF_LIRC_MODE2: 3102f4364dcfSSean Young return lirc_prog_query(attr, uattr); 3103118c8e9aSStanislav Fomichev case BPF_FLOW_DISSECTOR: 3104e9ddbb77SJakub Sitnicki case BPF_SK_LOOKUP: 3105a3fd7ceeSJakub Sitnicki return netns_bpf_prog_query(attr, uattr); 3106468e2f64SAlexei Starovoitov default: 3107468e2f64SAlexei Starovoitov return -EINVAL; 3108468e2f64SAlexei Starovoitov } 3109468e2f64SAlexei Starovoitov } 3110f4324551SDaniel Mack 31111b4d60ecSSong Liu #define BPF_PROG_TEST_RUN_LAST_FIELD test.cpu 31121cf1cae9SAlexei Starovoitov 31131cf1cae9SAlexei Starovoitov static int bpf_prog_test_run(const union bpf_attr *attr, 31141cf1cae9SAlexei Starovoitov union bpf_attr __user *uattr) 31151cf1cae9SAlexei Starovoitov { 31161cf1cae9SAlexei Starovoitov struct bpf_prog *prog; 31171cf1cae9SAlexei Starovoitov int ret = -ENOTSUPP; 31181cf1cae9SAlexei Starovoitov 31191cf1cae9SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 31201cf1cae9SAlexei Starovoitov return -EINVAL; 31211cf1cae9SAlexei Starovoitov 3122b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_in && !attr->test.ctx_in) || 3123b0b9395dSStanislav Fomichev (!attr->test.ctx_size_in && attr->test.ctx_in)) 3124b0b9395dSStanislav Fomichev return -EINVAL; 3125b0b9395dSStanislav Fomichev 3126b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_out && !attr->test.ctx_out) || 3127b0b9395dSStanislav Fomichev (!attr->test.ctx_size_out && attr->test.ctx_out)) 3128b0b9395dSStanislav Fomichev return -EINVAL; 3129b0b9395dSStanislav Fomichev 31301cf1cae9SAlexei Starovoitov prog = bpf_prog_get(attr->test.prog_fd); 31311cf1cae9SAlexei Starovoitov if (IS_ERR(prog)) 31321cf1cae9SAlexei Starovoitov return PTR_ERR(prog); 31331cf1cae9SAlexei Starovoitov 31341cf1cae9SAlexei Starovoitov if (prog->aux->ops->test_run) 31351cf1cae9SAlexei Starovoitov ret = prog->aux->ops->test_run(prog, attr, uattr); 31361cf1cae9SAlexei Starovoitov 31371cf1cae9SAlexei Starovoitov bpf_prog_put(prog); 31381cf1cae9SAlexei Starovoitov return ret; 31391cf1cae9SAlexei Starovoitov } 31401cf1cae9SAlexei Starovoitov 314134ad5580SMartin KaFai Lau #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 314234ad5580SMartin KaFai Lau 314334ad5580SMartin KaFai Lau static int bpf_obj_get_next_id(const union bpf_attr *attr, 314434ad5580SMartin KaFai Lau union bpf_attr __user *uattr, 314534ad5580SMartin KaFai Lau struct idr *idr, 314634ad5580SMartin KaFai Lau spinlock_t *lock) 314734ad5580SMartin KaFai Lau { 314834ad5580SMartin KaFai Lau u32 next_id = attr->start_id; 314934ad5580SMartin KaFai Lau int err = 0; 315034ad5580SMartin KaFai Lau 315134ad5580SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 315234ad5580SMartin KaFai Lau return -EINVAL; 315334ad5580SMartin KaFai Lau 315434ad5580SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 315534ad5580SMartin KaFai Lau return -EPERM; 315634ad5580SMartin KaFai Lau 315734ad5580SMartin KaFai Lau next_id++; 315834ad5580SMartin KaFai Lau spin_lock_bh(lock); 315934ad5580SMartin KaFai Lau if (!idr_get_next(idr, &next_id)) 316034ad5580SMartin KaFai Lau err = -ENOENT; 316134ad5580SMartin KaFai Lau spin_unlock_bh(lock); 316234ad5580SMartin KaFai Lau 316334ad5580SMartin KaFai Lau if (!err) 316434ad5580SMartin KaFai Lau err = put_user(next_id, &uattr->next_id); 316534ad5580SMartin KaFai Lau 316634ad5580SMartin KaFai Lau return err; 316734ad5580SMartin KaFai Lau } 316834ad5580SMartin KaFai Lau 31696086d29dSYonghong Song struct bpf_map *bpf_map_get_curr_or_next(u32 *id) 31706086d29dSYonghong Song { 31716086d29dSYonghong Song struct bpf_map *map; 31726086d29dSYonghong Song 31736086d29dSYonghong Song spin_lock_bh(&map_idr_lock); 31746086d29dSYonghong Song again: 31756086d29dSYonghong Song map = idr_get_next(&map_idr, id); 31766086d29dSYonghong Song if (map) { 31776086d29dSYonghong Song map = __bpf_map_inc_not_zero(map, false); 31786086d29dSYonghong Song if (IS_ERR(map)) { 31796086d29dSYonghong Song (*id)++; 31806086d29dSYonghong Song goto again; 31816086d29dSYonghong Song } 31826086d29dSYonghong Song } 31836086d29dSYonghong Song spin_unlock_bh(&map_idr_lock); 31846086d29dSYonghong Song 31856086d29dSYonghong Song return map; 31866086d29dSYonghong Song } 31876086d29dSYonghong Song 3188a228a64fSAlexei Starovoitov struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id) 3189a228a64fSAlexei Starovoitov { 3190a228a64fSAlexei Starovoitov struct bpf_prog *prog; 3191a228a64fSAlexei Starovoitov 3192a228a64fSAlexei Starovoitov spin_lock_bh(&prog_idr_lock); 3193a228a64fSAlexei Starovoitov again: 3194a228a64fSAlexei Starovoitov prog = idr_get_next(&prog_idr, id); 3195a228a64fSAlexei Starovoitov if (prog) { 3196a228a64fSAlexei Starovoitov prog = bpf_prog_inc_not_zero(prog); 3197a228a64fSAlexei Starovoitov if (IS_ERR(prog)) { 3198a228a64fSAlexei Starovoitov (*id)++; 3199a228a64fSAlexei Starovoitov goto again; 3200a228a64fSAlexei Starovoitov } 3201a228a64fSAlexei Starovoitov } 3202a228a64fSAlexei Starovoitov spin_unlock_bh(&prog_idr_lock); 3203a228a64fSAlexei Starovoitov 3204a228a64fSAlexei Starovoitov return prog; 3205a228a64fSAlexei Starovoitov } 3206a228a64fSAlexei Starovoitov 3207b16d9aa4SMartin KaFai Lau #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 3208b16d9aa4SMartin KaFai Lau 32097e6897f9SBjörn Töpel struct bpf_prog *bpf_prog_by_id(u32 id) 32107e6897f9SBjörn Töpel { 32117e6897f9SBjörn Töpel struct bpf_prog *prog; 32127e6897f9SBjörn Töpel 32137e6897f9SBjörn Töpel if (!id) 32147e6897f9SBjörn Töpel return ERR_PTR(-ENOENT); 32157e6897f9SBjörn Töpel 32167e6897f9SBjörn Töpel spin_lock_bh(&prog_idr_lock); 32177e6897f9SBjörn Töpel prog = idr_find(&prog_idr, id); 32187e6897f9SBjörn Töpel if (prog) 32197e6897f9SBjörn Töpel prog = bpf_prog_inc_not_zero(prog); 32207e6897f9SBjörn Töpel else 32217e6897f9SBjörn Töpel prog = ERR_PTR(-ENOENT); 32227e6897f9SBjörn Töpel spin_unlock_bh(&prog_idr_lock); 32237e6897f9SBjörn Töpel return prog; 32247e6897f9SBjörn Töpel } 32257e6897f9SBjörn Töpel 3226b16d9aa4SMartin KaFai Lau static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 3227b16d9aa4SMartin KaFai Lau { 3228b16d9aa4SMartin KaFai Lau struct bpf_prog *prog; 3229b16d9aa4SMartin KaFai Lau u32 id = attr->prog_id; 3230b16d9aa4SMartin KaFai Lau int fd; 3231b16d9aa4SMartin KaFai Lau 3232b16d9aa4SMartin KaFai Lau if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 3233b16d9aa4SMartin KaFai Lau return -EINVAL; 3234b16d9aa4SMartin KaFai Lau 3235b16d9aa4SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 3236b16d9aa4SMartin KaFai Lau return -EPERM; 3237b16d9aa4SMartin KaFai Lau 32387e6897f9SBjörn Töpel prog = bpf_prog_by_id(id); 3239b16d9aa4SMartin KaFai Lau if (IS_ERR(prog)) 3240b16d9aa4SMartin KaFai Lau return PTR_ERR(prog); 3241b16d9aa4SMartin KaFai Lau 3242b16d9aa4SMartin KaFai Lau fd = bpf_prog_new_fd(prog); 3243b16d9aa4SMartin KaFai Lau if (fd < 0) 3244b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 3245b16d9aa4SMartin KaFai Lau 3246b16d9aa4SMartin KaFai Lau return fd; 3247b16d9aa4SMartin KaFai Lau } 3248b16d9aa4SMartin KaFai Lau 32496e71b04aSChenbo Feng #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags 3250bd5f5f4eSMartin KaFai Lau 3251bd5f5f4eSMartin KaFai Lau static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 3252bd5f5f4eSMartin KaFai Lau { 3253bd5f5f4eSMartin KaFai Lau struct bpf_map *map; 3254bd5f5f4eSMartin KaFai Lau u32 id = attr->map_id; 32556e71b04aSChenbo Feng int f_flags; 3256bd5f5f4eSMartin KaFai Lau int fd; 3257bd5f5f4eSMartin KaFai Lau 32586e71b04aSChenbo Feng if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || 32596e71b04aSChenbo Feng attr->open_flags & ~BPF_OBJ_FLAG_MASK) 3260bd5f5f4eSMartin KaFai Lau return -EINVAL; 3261bd5f5f4eSMartin KaFai Lau 3262bd5f5f4eSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 3263bd5f5f4eSMartin KaFai Lau return -EPERM; 3264bd5f5f4eSMartin KaFai Lau 32656e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->open_flags); 32666e71b04aSChenbo Feng if (f_flags < 0) 32676e71b04aSChenbo Feng return f_flags; 32686e71b04aSChenbo Feng 3269bd5f5f4eSMartin KaFai Lau spin_lock_bh(&map_idr_lock); 3270bd5f5f4eSMartin KaFai Lau map = idr_find(&map_idr, id); 3271bd5f5f4eSMartin KaFai Lau if (map) 3272b0e4701cSStanislav Fomichev map = __bpf_map_inc_not_zero(map, true); 3273bd5f5f4eSMartin KaFai Lau else 3274bd5f5f4eSMartin KaFai Lau map = ERR_PTR(-ENOENT); 3275bd5f5f4eSMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 3276bd5f5f4eSMartin KaFai Lau 3277bd5f5f4eSMartin KaFai Lau if (IS_ERR(map)) 3278bd5f5f4eSMartin KaFai Lau return PTR_ERR(map); 3279bd5f5f4eSMartin KaFai Lau 32806e71b04aSChenbo Feng fd = bpf_map_new_fd(map, f_flags); 3281bd5f5f4eSMartin KaFai Lau if (fd < 0) 3282781e6282SPeng Sun bpf_map_put_with_uref(map); 3283bd5f5f4eSMartin KaFai Lau 3284bd5f5f4eSMartin KaFai Lau return fd; 3285bd5f5f4eSMartin KaFai Lau } 3286bd5f5f4eSMartin KaFai Lau 32877105e828SDaniel Borkmann static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, 3288d8eca5bbSDaniel Borkmann unsigned long addr, u32 *off, 3289d8eca5bbSDaniel Borkmann u32 *type) 32907105e828SDaniel Borkmann { 3291d8eca5bbSDaniel Borkmann const struct bpf_map *map; 32927105e828SDaniel Borkmann int i; 32937105e828SDaniel Borkmann 3294984fe94fSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 3295d8eca5bbSDaniel Borkmann for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { 3296d8eca5bbSDaniel Borkmann map = prog->aux->used_maps[i]; 3297d8eca5bbSDaniel Borkmann if (map == (void *)addr) { 3298d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_FD; 3299984fe94fSYiFei Zhu goto out; 3300d8eca5bbSDaniel Borkmann } 3301d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta) 3302d8eca5bbSDaniel Borkmann continue; 3303d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta(map, addr, off)) { 3304d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_VALUE; 3305984fe94fSYiFei Zhu goto out; 3306d8eca5bbSDaniel Borkmann } 3307d8eca5bbSDaniel Borkmann } 3308984fe94fSYiFei Zhu map = NULL; 3309d8eca5bbSDaniel Borkmann 3310984fe94fSYiFei Zhu out: 3311984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 3312984fe94fSYiFei Zhu return map; 33137105e828SDaniel Borkmann } 33147105e828SDaniel Borkmann 331563960260SKees Cook static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, 331663960260SKees Cook const struct cred *f_cred) 33177105e828SDaniel Borkmann { 33187105e828SDaniel Borkmann const struct bpf_map *map; 33197105e828SDaniel Borkmann struct bpf_insn *insns; 3320d8eca5bbSDaniel Borkmann u32 off, type; 33217105e828SDaniel Borkmann u64 imm; 332229fcb05bSAndrii Nakryiko u8 code; 33237105e828SDaniel Borkmann int i; 33247105e828SDaniel Borkmann 33257105e828SDaniel Borkmann insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), 33267105e828SDaniel Borkmann GFP_USER); 33277105e828SDaniel Borkmann if (!insns) 33287105e828SDaniel Borkmann return insns; 33297105e828SDaniel Borkmann 33307105e828SDaniel Borkmann for (i = 0; i < prog->len; i++) { 333129fcb05bSAndrii Nakryiko code = insns[i].code; 333229fcb05bSAndrii Nakryiko 333329fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_TAIL_CALL)) { 33347105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 33357105e828SDaniel Borkmann insns[i].imm = BPF_FUNC_tail_call; 33367105e828SDaniel Borkmann /* fall-through */ 33377105e828SDaniel Borkmann } 333829fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_CALL) || 333929fcb05bSAndrii Nakryiko code == (BPF_JMP | BPF_CALL_ARGS)) { 334029fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_CALL_ARGS)) 33417105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 334263960260SKees Cook if (!bpf_dump_raw_ok(f_cred)) 33437105e828SDaniel Borkmann insns[i].imm = 0; 33447105e828SDaniel Borkmann continue; 33457105e828SDaniel Borkmann } 334629fcb05bSAndrii Nakryiko if (BPF_CLASS(code) == BPF_LDX && BPF_MODE(code) == BPF_PROBE_MEM) { 334729fcb05bSAndrii Nakryiko insns[i].code = BPF_LDX | BPF_SIZE(code) | BPF_MEM; 334829fcb05bSAndrii Nakryiko continue; 334929fcb05bSAndrii Nakryiko } 33507105e828SDaniel Borkmann 335129fcb05bSAndrii Nakryiko if (code != (BPF_LD | BPF_IMM | BPF_DW)) 33527105e828SDaniel Borkmann continue; 33537105e828SDaniel Borkmann 33547105e828SDaniel Borkmann imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; 3355d8eca5bbSDaniel Borkmann map = bpf_map_from_imm(prog, imm, &off, &type); 33567105e828SDaniel Borkmann if (map) { 3357d8eca5bbSDaniel Borkmann insns[i].src_reg = type; 33587105e828SDaniel Borkmann insns[i].imm = map->id; 3359d8eca5bbSDaniel Borkmann insns[i + 1].imm = off; 33607105e828SDaniel Borkmann continue; 33617105e828SDaniel Borkmann } 33627105e828SDaniel Borkmann } 33637105e828SDaniel Borkmann 33647105e828SDaniel Borkmann return insns; 33657105e828SDaniel Borkmann } 33667105e828SDaniel Borkmann 3367c454a46bSMartin KaFai Lau static int set_info_rec_size(struct bpf_prog_info *info) 3368c454a46bSMartin KaFai Lau { 3369c454a46bSMartin KaFai Lau /* 3370c454a46bSMartin KaFai Lau * Ensure info.*_rec_size is the same as kernel expected size 3371c454a46bSMartin KaFai Lau * 3372c454a46bSMartin KaFai Lau * or 3373c454a46bSMartin KaFai Lau * 3374c454a46bSMartin KaFai Lau * Only allow zero *_rec_size if both _rec_size and _cnt are 3375c454a46bSMartin KaFai Lau * zero. In this case, the kernel will set the expected 3376c454a46bSMartin KaFai Lau * _rec_size back to the info. 3377c454a46bSMartin KaFai Lau */ 3378c454a46bSMartin KaFai Lau 337911d8b82dSYonghong Song if ((info->nr_func_info || info->func_info_rec_size) && 3380c454a46bSMartin KaFai Lau info->func_info_rec_size != sizeof(struct bpf_func_info)) 3381c454a46bSMartin KaFai Lau return -EINVAL; 3382c454a46bSMartin KaFai Lau 338311d8b82dSYonghong Song if ((info->nr_line_info || info->line_info_rec_size) && 3384c454a46bSMartin KaFai Lau info->line_info_rec_size != sizeof(struct bpf_line_info)) 3385c454a46bSMartin KaFai Lau return -EINVAL; 3386c454a46bSMartin KaFai Lau 338711d8b82dSYonghong Song if ((info->nr_jited_line_info || info->jited_line_info_rec_size) && 3388c454a46bSMartin KaFai Lau info->jited_line_info_rec_size != sizeof(__u64)) 3389c454a46bSMartin KaFai Lau return -EINVAL; 3390c454a46bSMartin KaFai Lau 3391c454a46bSMartin KaFai Lau info->func_info_rec_size = sizeof(struct bpf_func_info); 3392c454a46bSMartin KaFai Lau info->line_info_rec_size = sizeof(struct bpf_line_info); 3393c454a46bSMartin KaFai Lau info->jited_line_info_rec_size = sizeof(__u64); 3394c454a46bSMartin KaFai Lau 3395c454a46bSMartin KaFai Lau return 0; 3396c454a46bSMartin KaFai Lau } 3397c454a46bSMartin KaFai Lau 339863960260SKees Cook static int bpf_prog_get_info_by_fd(struct file *file, 339963960260SKees Cook struct bpf_prog *prog, 34001e270976SMartin KaFai Lau const union bpf_attr *attr, 34011e270976SMartin KaFai Lau union bpf_attr __user *uattr) 34021e270976SMartin KaFai Lau { 34031e270976SMartin KaFai Lau struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 34045c6f2588SGreg Kroah-Hartman struct bpf_prog_info info; 34051e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 34065f8f8b93SAlexei Starovoitov struct bpf_prog_stats stats; 34071e270976SMartin KaFai Lau char __user *uinsns; 34081e270976SMartin KaFai Lau u32 ulen; 34091e270976SMartin KaFai Lau int err; 34101e270976SMartin KaFai Lau 3411dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 34121e270976SMartin KaFai Lau if (err) 34131e270976SMartin KaFai Lau return err; 34141e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 34151e270976SMartin KaFai Lau 34165c6f2588SGreg Kroah-Hartman memset(&info, 0, sizeof(info)); 34171e270976SMartin KaFai Lau if (copy_from_user(&info, uinfo, info_len)) 341889b09689SDaniel Borkmann return -EFAULT; 34191e270976SMartin KaFai Lau 34201e270976SMartin KaFai Lau info.type = prog->type; 34211e270976SMartin KaFai Lau info.id = prog->aux->id; 3422cb4d2b3fSMartin KaFai Lau info.load_time = prog->aux->load_time; 3423cb4d2b3fSMartin KaFai Lau info.created_by_uid = from_kuid_munged(current_user_ns(), 3424cb4d2b3fSMartin KaFai Lau prog->aux->user->uid); 3425b85fab0eSJiri Olsa info.gpl_compatible = prog->gpl_compatible; 34261e270976SMartin KaFai Lau 34271e270976SMartin KaFai Lau memcpy(info.tag, prog->tag, sizeof(prog->tag)); 3428cb4d2b3fSMartin KaFai Lau memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); 3429cb4d2b3fSMartin KaFai Lau 3430984fe94fSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 3431cb4d2b3fSMartin KaFai Lau ulen = info.nr_map_ids; 3432cb4d2b3fSMartin KaFai Lau info.nr_map_ids = prog->aux->used_map_cnt; 3433cb4d2b3fSMartin KaFai Lau ulen = min_t(u32, info.nr_map_ids, ulen); 3434cb4d2b3fSMartin KaFai Lau if (ulen) { 3435721e08daSMartin KaFai Lau u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); 3436cb4d2b3fSMartin KaFai Lau u32 i; 3437cb4d2b3fSMartin KaFai Lau 3438cb4d2b3fSMartin KaFai Lau for (i = 0; i < ulen; i++) 3439cb4d2b3fSMartin KaFai Lau if (put_user(prog->aux->used_maps[i]->id, 3440984fe94fSYiFei Zhu &user_map_ids[i])) { 3441984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 3442cb4d2b3fSMartin KaFai Lau return -EFAULT; 3443cb4d2b3fSMartin KaFai Lau } 3444984fe94fSYiFei Zhu } 3445984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 34461e270976SMartin KaFai Lau 3447c454a46bSMartin KaFai Lau err = set_info_rec_size(&info); 3448c454a46bSMartin KaFai Lau if (err) 3449c454a46bSMartin KaFai Lau return err; 34507337224fSMartin KaFai Lau 34515f8f8b93SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 34525f8f8b93SAlexei Starovoitov info.run_time_ns = stats.nsecs; 34535f8f8b93SAlexei Starovoitov info.run_cnt = stats.cnt; 34549ed9e9baSAlexei Starovoitov info.recursion_misses = stats.misses; 34555f8f8b93SAlexei Starovoitov 34562c78ee89SAlexei Starovoitov if (!bpf_capable()) { 34571e270976SMartin KaFai Lau info.jited_prog_len = 0; 34581e270976SMartin KaFai Lau info.xlated_prog_len = 0; 3459dbecd738SSandipan Das info.nr_jited_ksyms = 0; 346028c2fae7SDaniel Borkmann info.nr_jited_func_lens = 0; 346111d8b82dSYonghong Song info.nr_func_info = 0; 346211d8b82dSYonghong Song info.nr_line_info = 0; 346311d8b82dSYonghong Song info.nr_jited_line_info = 0; 34641e270976SMartin KaFai Lau goto done; 34651e270976SMartin KaFai Lau } 34661e270976SMartin KaFai Lau 34671e270976SMartin KaFai Lau ulen = info.xlated_prog_len; 34689975a54bSDaniel Borkmann info.xlated_prog_len = bpf_prog_insn_size(prog); 34691e270976SMartin KaFai Lau if (info.xlated_prog_len && ulen) { 34707105e828SDaniel Borkmann struct bpf_insn *insns_sanitized; 34717105e828SDaniel Borkmann bool fault; 34727105e828SDaniel Borkmann 347363960260SKees Cook if (prog->blinded && !bpf_dump_raw_ok(file->f_cred)) { 34747105e828SDaniel Borkmann info.xlated_prog_insns = 0; 34757105e828SDaniel Borkmann goto done; 34767105e828SDaniel Borkmann } 347763960260SKees Cook insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred); 34787105e828SDaniel Borkmann if (!insns_sanitized) 34797105e828SDaniel Borkmann return -ENOMEM; 34801e270976SMartin KaFai Lau uinsns = u64_to_user_ptr(info.xlated_prog_insns); 34811e270976SMartin KaFai Lau ulen = min_t(u32, info.xlated_prog_len, ulen); 34827105e828SDaniel Borkmann fault = copy_to_user(uinsns, insns_sanitized, ulen); 34837105e828SDaniel Borkmann kfree(insns_sanitized); 34847105e828SDaniel Borkmann if (fault) 34851e270976SMartin KaFai Lau return -EFAULT; 34861e270976SMartin KaFai Lau } 34871e270976SMartin KaFai Lau 3488675fc275SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 3489675fc275SJakub Kicinski err = bpf_prog_offload_info_fill(&info, prog); 3490675fc275SJakub Kicinski if (err) 3491675fc275SJakub Kicinski return err; 3492fcfb126dSJiong Wang goto done; 3493fcfb126dSJiong Wang } 3494fcfb126dSJiong Wang 3495fcfb126dSJiong Wang /* NOTE: the following code is supposed to be skipped for offload. 3496fcfb126dSJiong Wang * bpf_prog_offload_info_fill() is the place to fill similar fields 3497fcfb126dSJiong Wang * for offload. 3498fcfb126dSJiong Wang */ 3499fcfb126dSJiong Wang ulen = info.jited_prog_len; 35004d56a76eSSandipan Das if (prog->aux->func_cnt) { 35014d56a76eSSandipan Das u32 i; 35024d56a76eSSandipan Das 35034d56a76eSSandipan Das info.jited_prog_len = 0; 35044d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) 35054d56a76eSSandipan Das info.jited_prog_len += prog->aux->func[i]->jited_len; 35064d56a76eSSandipan Das } else { 3507fcfb126dSJiong Wang info.jited_prog_len = prog->jited_len; 35084d56a76eSSandipan Das } 35094d56a76eSSandipan Das 3510fcfb126dSJiong Wang if (info.jited_prog_len && ulen) { 351163960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 3512fcfb126dSJiong Wang uinsns = u64_to_user_ptr(info.jited_prog_insns); 3513fcfb126dSJiong Wang ulen = min_t(u32, info.jited_prog_len, ulen); 35144d56a76eSSandipan Das 35154d56a76eSSandipan Das /* for multi-function programs, copy the JITed 35164d56a76eSSandipan Das * instructions for all the functions 35174d56a76eSSandipan Das */ 35184d56a76eSSandipan Das if (prog->aux->func_cnt) { 35194d56a76eSSandipan Das u32 len, free, i; 35204d56a76eSSandipan Das u8 *img; 35214d56a76eSSandipan Das 35224d56a76eSSandipan Das free = ulen; 35234d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) { 35244d56a76eSSandipan Das len = prog->aux->func[i]->jited_len; 35254d56a76eSSandipan Das len = min_t(u32, len, free); 35264d56a76eSSandipan Das img = (u8 *) prog->aux->func[i]->bpf_func; 35274d56a76eSSandipan Das if (copy_to_user(uinsns, img, len)) 35284d56a76eSSandipan Das return -EFAULT; 35294d56a76eSSandipan Das uinsns += len; 35304d56a76eSSandipan Das free -= len; 35314d56a76eSSandipan Das if (!free) 35324d56a76eSSandipan Das break; 35334d56a76eSSandipan Das } 35344d56a76eSSandipan Das } else { 3535fcfb126dSJiong Wang if (copy_to_user(uinsns, prog->bpf_func, ulen)) 3536fcfb126dSJiong Wang return -EFAULT; 35374d56a76eSSandipan Das } 3538fcfb126dSJiong Wang } else { 3539fcfb126dSJiong Wang info.jited_prog_insns = 0; 3540fcfb126dSJiong Wang } 3541675fc275SJakub Kicinski } 3542675fc275SJakub Kicinski 3543dbecd738SSandipan Das ulen = info.nr_jited_ksyms; 3544ff1889fcSSong Liu info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; 35457a5725ddSSong Liu if (ulen) { 354663960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 3547ff1889fcSSong Liu unsigned long ksym_addr; 3548dbecd738SSandipan Das u64 __user *user_ksyms; 3549dbecd738SSandipan Das u32 i; 3550dbecd738SSandipan Das 3551dbecd738SSandipan Das /* copy the address of the kernel symbol 3552dbecd738SSandipan Das * corresponding to each function 3553dbecd738SSandipan Das */ 3554dbecd738SSandipan Das ulen = min_t(u32, info.nr_jited_ksyms, ulen); 3555dbecd738SSandipan Das user_ksyms = u64_to_user_ptr(info.jited_ksyms); 3556ff1889fcSSong Liu if (prog->aux->func_cnt) { 3557dbecd738SSandipan Das for (i = 0; i < ulen; i++) { 3558ff1889fcSSong Liu ksym_addr = (unsigned long) 3559ff1889fcSSong Liu prog->aux->func[i]->bpf_func; 3560ff1889fcSSong Liu if (put_user((u64) ksym_addr, 3561ff1889fcSSong Liu &user_ksyms[i])) 3562ff1889fcSSong Liu return -EFAULT; 3563ff1889fcSSong Liu } 3564ff1889fcSSong Liu } else { 3565ff1889fcSSong Liu ksym_addr = (unsigned long) prog->bpf_func; 3566ff1889fcSSong Liu if (put_user((u64) ksym_addr, &user_ksyms[0])) 3567dbecd738SSandipan Das return -EFAULT; 3568dbecd738SSandipan Das } 3569dbecd738SSandipan Das } else { 3570dbecd738SSandipan Das info.jited_ksyms = 0; 3571dbecd738SSandipan Das } 3572dbecd738SSandipan Das } 3573dbecd738SSandipan Das 3574815581c1SSandipan Das ulen = info.nr_jited_func_lens; 3575ff1889fcSSong Liu info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; 35767a5725ddSSong Liu if (ulen) { 357763960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 3578815581c1SSandipan Das u32 __user *user_lens; 3579815581c1SSandipan Das u32 func_len, i; 3580815581c1SSandipan Das 3581815581c1SSandipan Das /* copy the JITed image lengths for each function */ 3582815581c1SSandipan Das ulen = min_t(u32, info.nr_jited_func_lens, ulen); 3583815581c1SSandipan Das user_lens = u64_to_user_ptr(info.jited_func_lens); 3584ff1889fcSSong Liu if (prog->aux->func_cnt) { 3585815581c1SSandipan Das for (i = 0; i < ulen; i++) { 3586ff1889fcSSong Liu func_len = 3587ff1889fcSSong Liu prog->aux->func[i]->jited_len; 3588815581c1SSandipan Das if (put_user(func_len, &user_lens[i])) 3589815581c1SSandipan Das return -EFAULT; 3590815581c1SSandipan Das } 3591815581c1SSandipan Das } else { 3592ff1889fcSSong Liu func_len = prog->jited_len; 3593ff1889fcSSong Liu if (put_user(func_len, &user_lens[0])) 3594ff1889fcSSong Liu return -EFAULT; 3595ff1889fcSSong Liu } 3596ff1889fcSSong Liu } else { 3597815581c1SSandipan Das info.jited_func_lens = 0; 3598815581c1SSandipan Das } 3599815581c1SSandipan Das } 3600815581c1SSandipan Das 36017337224fSMartin KaFai Lau if (prog->aux->btf) 360222dc4a0fSAndrii Nakryiko info.btf_id = btf_obj_id(prog->aux->btf); 3603838e9690SYonghong Song 360411d8b82dSYonghong Song ulen = info.nr_func_info; 360511d8b82dSYonghong Song info.nr_func_info = prog->aux->func_info_cnt; 360611d8b82dSYonghong Song if (info.nr_func_info && ulen) { 3607838e9690SYonghong Song char __user *user_finfo; 3608838e9690SYonghong Song 3609838e9690SYonghong Song user_finfo = u64_to_user_ptr(info.func_info); 361011d8b82dSYonghong Song ulen = min_t(u32, info.nr_func_info, ulen); 3611ba64e7d8SYonghong Song if (copy_to_user(user_finfo, prog->aux->func_info, 36127337224fSMartin KaFai Lau info.func_info_rec_size * ulen)) 3613838e9690SYonghong Song return -EFAULT; 3614838e9690SYonghong Song } 3615838e9690SYonghong Song 361611d8b82dSYonghong Song ulen = info.nr_line_info; 361711d8b82dSYonghong Song info.nr_line_info = prog->aux->nr_linfo; 361811d8b82dSYonghong Song if (info.nr_line_info && ulen) { 3619c454a46bSMartin KaFai Lau __u8 __user *user_linfo; 3620c454a46bSMartin KaFai Lau 3621c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.line_info); 362211d8b82dSYonghong Song ulen = min_t(u32, info.nr_line_info, ulen); 3623c454a46bSMartin KaFai Lau if (copy_to_user(user_linfo, prog->aux->linfo, 3624c454a46bSMartin KaFai Lau info.line_info_rec_size * ulen)) 3625c454a46bSMartin KaFai Lau return -EFAULT; 3626c454a46bSMartin KaFai Lau } 3627c454a46bSMartin KaFai Lau 362811d8b82dSYonghong Song ulen = info.nr_jited_line_info; 3629c454a46bSMartin KaFai Lau if (prog->aux->jited_linfo) 363011d8b82dSYonghong Song info.nr_jited_line_info = prog->aux->nr_linfo; 3631c454a46bSMartin KaFai Lau else 363211d8b82dSYonghong Song info.nr_jited_line_info = 0; 363311d8b82dSYonghong Song if (info.nr_jited_line_info && ulen) { 363463960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 3635c454a46bSMartin KaFai Lau __u64 __user *user_linfo; 3636c454a46bSMartin KaFai Lau u32 i; 3637c454a46bSMartin KaFai Lau 3638c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.jited_line_info); 363911d8b82dSYonghong Song ulen = min_t(u32, info.nr_jited_line_info, ulen); 3640c454a46bSMartin KaFai Lau for (i = 0; i < ulen; i++) { 3641c454a46bSMartin KaFai Lau if (put_user((__u64)(long)prog->aux->jited_linfo[i], 3642c454a46bSMartin KaFai Lau &user_linfo[i])) 3643c454a46bSMartin KaFai Lau return -EFAULT; 3644c454a46bSMartin KaFai Lau } 3645c454a46bSMartin KaFai Lau } else { 3646c454a46bSMartin KaFai Lau info.jited_line_info = 0; 3647c454a46bSMartin KaFai Lau } 3648c454a46bSMartin KaFai Lau } 3649c454a46bSMartin KaFai Lau 3650c872bdb3SSong Liu ulen = info.nr_prog_tags; 3651c872bdb3SSong Liu info.nr_prog_tags = prog->aux->func_cnt ? : 1; 3652c872bdb3SSong Liu if (ulen) { 3653c872bdb3SSong Liu __u8 __user (*user_prog_tags)[BPF_TAG_SIZE]; 3654c872bdb3SSong Liu u32 i; 3655c872bdb3SSong Liu 3656c872bdb3SSong Liu user_prog_tags = u64_to_user_ptr(info.prog_tags); 3657c872bdb3SSong Liu ulen = min_t(u32, info.nr_prog_tags, ulen); 3658c872bdb3SSong Liu if (prog->aux->func_cnt) { 3659c872bdb3SSong Liu for (i = 0; i < ulen; i++) { 3660c872bdb3SSong Liu if (copy_to_user(user_prog_tags[i], 3661c872bdb3SSong Liu prog->aux->func[i]->tag, 3662c872bdb3SSong Liu BPF_TAG_SIZE)) 3663c872bdb3SSong Liu return -EFAULT; 3664c872bdb3SSong Liu } 3665c872bdb3SSong Liu } else { 3666c872bdb3SSong Liu if (copy_to_user(user_prog_tags[0], 3667c872bdb3SSong Liu prog->tag, BPF_TAG_SIZE)) 3668c872bdb3SSong Liu return -EFAULT; 3669c872bdb3SSong Liu } 3670c872bdb3SSong Liu } 3671c872bdb3SSong Liu 36721e270976SMartin KaFai Lau done: 36731e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 36741e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 36751e270976SMartin KaFai Lau return -EFAULT; 36761e270976SMartin KaFai Lau 36771e270976SMartin KaFai Lau return 0; 36781e270976SMartin KaFai Lau } 36791e270976SMartin KaFai Lau 368063960260SKees Cook static int bpf_map_get_info_by_fd(struct file *file, 368163960260SKees Cook struct bpf_map *map, 36821e270976SMartin KaFai Lau const union bpf_attr *attr, 36831e270976SMartin KaFai Lau union bpf_attr __user *uattr) 36841e270976SMartin KaFai Lau { 36851e270976SMartin KaFai Lau struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 36865c6f2588SGreg Kroah-Hartman struct bpf_map_info info; 36871e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 36881e270976SMartin KaFai Lau int err; 36891e270976SMartin KaFai Lau 3690dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 36911e270976SMartin KaFai Lau if (err) 36921e270976SMartin KaFai Lau return err; 36931e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 36941e270976SMartin KaFai Lau 36955c6f2588SGreg Kroah-Hartman memset(&info, 0, sizeof(info)); 36961e270976SMartin KaFai Lau info.type = map->map_type; 36971e270976SMartin KaFai Lau info.id = map->id; 36981e270976SMartin KaFai Lau info.key_size = map->key_size; 36991e270976SMartin KaFai Lau info.value_size = map->value_size; 37001e270976SMartin KaFai Lau info.max_entries = map->max_entries; 37011e270976SMartin KaFai Lau info.map_flags = map->map_flags; 3702ad5b177bSMartin KaFai Lau memcpy(info.name, map->name, sizeof(map->name)); 37031e270976SMartin KaFai Lau 370478958fcaSMartin KaFai Lau if (map->btf) { 370522dc4a0fSAndrii Nakryiko info.btf_id = btf_obj_id(map->btf); 37069b2cf328SMartin KaFai Lau info.btf_key_type_id = map->btf_key_type_id; 37079b2cf328SMartin KaFai Lau info.btf_value_type_id = map->btf_value_type_id; 370878958fcaSMartin KaFai Lau } 370985d33df3SMartin KaFai Lau info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; 371078958fcaSMartin KaFai Lau 371152775b33SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 371252775b33SJakub Kicinski err = bpf_map_offload_info_fill(&info, map); 371352775b33SJakub Kicinski if (err) 371452775b33SJakub Kicinski return err; 371552775b33SJakub Kicinski } 371652775b33SJakub Kicinski 37171e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 37181e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 37191e270976SMartin KaFai Lau return -EFAULT; 37201e270976SMartin KaFai Lau 37211e270976SMartin KaFai Lau return 0; 37221e270976SMartin KaFai Lau } 37231e270976SMartin KaFai Lau 372463960260SKees Cook static int bpf_btf_get_info_by_fd(struct file *file, 372563960260SKees Cook struct btf *btf, 372662dab84cSMartin KaFai Lau const union bpf_attr *attr, 372762dab84cSMartin KaFai Lau union bpf_attr __user *uattr) 372862dab84cSMartin KaFai Lau { 372962dab84cSMartin KaFai Lau struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info); 373062dab84cSMartin KaFai Lau u32 info_len = attr->info.info_len; 373162dab84cSMartin KaFai Lau int err; 373262dab84cSMartin KaFai Lau 3733dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(*uinfo), info_len); 373462dab84cSMartin KaFai Lau if (err) 373562dab84cSMartin KaFai Lau return err; 373662dab84cSMartin KaFai Lau 373762dab84cSMartin KaFai Lau return btf_get_info_by_fd(btf, attr, uattr); 373862dab84cSMartin KaFai Lau } 373962dab84cSMartin KaFai Lau 374063960260SKees Cook static int bpf_link_get_info_by_fd(struct file *file, 374163960260SKees Cook struct bpf_link *link, 3742f2e10bffSAndrii Nakryiko const union bpf_attr *attr, 3743f2e10bffSAndrii Nakryiko union bpf_attr __user *uattr) 3744f2e10bffSAndrii Nakryiko { 3745f2e10bffSAndrii Nakryiko struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info); 3746f2e10bffSAndrii Nakryiko struct bpf_link_info info; 3747f2e10bffSAndrii Nakryiko u32 info_len = attr->info.info_len; 3748f2e10bffSAndrii Nakryiko int err; 3749f2e10bffSAndrii Nakryiko 3750f2e10bffSAndrii Nakryiko err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 3751f2e10bffSAndrii Nakryiko if (err) 3752f2e10bffSAndrii Nakryiko return err; 3753f2e10bffSAndrii Nakryiko info_len = min_t(u32, sizeof(info), info_len); 3754f2e10bffSAndrii Nakryiko 3755f2e10bffSAndrii Nakryiko memset(&info, 0, sizeof(info)); 3756f2e10bffSAndrii Nakryiko if (copy_from_user(&info, uinfo, info_len)) 3757f2e10bffSAndrii Nakryiko return -EFAULT; 3758f2e10bffSAndrii Nakryiko 3759f2e10bffSAndrii Nakryiko info.type = link->type; 3760f2e10bffSAndrii Nakryiko info.id = link->id; 3761f2e10bffSAndrii Nakryiko info.prog_id = link->prog->aux->id; 3762f2e10bffSAndrii Nakryiko 3763f2e10bffSAndrii Nakryiko if (link->ops->fill_link_info) { 3764f2e10bffSAndrii Nakryiko err = link->ops->fill_link_info(link, &info); 3765f2e10bffSAndrii Nakryiko if (err) 3766f2e10bffSAndrii Nakryiko return err; 3767f2e10bffSAndrii Nakryiko } 3768f2e10bffSAndrii Nakryiko 3769f2e10bffSAndrii Nakryiko if (copy_to_user(uinfo, &info, info_len) || 3770f2e10bffSAndrii Nakryiko put_user(info_len, &uattr->info.info_len)) 3771f2e10bffSAndrii Nakryiko return -EFAULT; 3772f2e10bffSAndrii Nakryiko 3773f2e10bffSAndrii Nakryiko return 0; 3774f2e10bffSAndrii Nakryiko } 3775f2e10bffSAndrii Nakryiko 3776f2e10bffSAndrii Nakryiko 37771e270976SMartin KaFai Lau #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 37781e270976SMartin KaFai Lau 37791e270976SMartin KaFai Lau static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 37801e270976SMartin KaFai Lau union bpf_attr __user *uattr) 37811e270976SMartin KaFai Lau { 37821e270976SMartin KaFai Lau int ufd = attr->info.bpf_fd; 37831e270976SMartin KaFai Lau struct fd f; 37841e270976SMartin KaFai Lau int err; 37851e270976SMartin KaFai Lau 37861e270976SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 37871e270976SMartin KaFai Lau return -EINVAL; 37881e270976SMartin KaFai Lau 37891e270976SMartin KaFai Lau f = fdget(ufd); 37901e270976SMartin KaFai Lau if (!f.file) 37911e270976SMartin KaFai Lau return -EBADFD; 37921e270976SMartin KaFai Lau 37931e270976SMartin KaFai Lau if (f.file->f_op == &bpf_prog_fops) 379463960260SKees Cook err = bpf_prog_get_info_by_fd(f.file, f.file->private_data, attr, 37951e270976SMartin KaFai Lau uattr); 37961e270976SMartin KaFai Lau else if (f.file->f_op == &bpf_map_fops) 379763960260SKees Cook err = bpf_map_get_info_by_fd(f.file, f.file->private_data, attr, 37981e270976SMartin KaFai Lau uattr); 379960197cfbSMartin KaFai Lau else if (f.file->f_op == &btf_fops) 380063960260SKees Cook err = bpf_btf_get_info_by_fd(f.file, f.file->private_data, attr, uattr); 3801f2e10bffSAndrii Nakryiko else if (f.file->f_op == &bpf_link_fops) 380263960260SKees Cook err = bpf_link_get_info_by_fd(f.file, f.file->private_data, 3803f2e10bffSAndrii Nakryiko attr, uattr); 38041e270976SMartin KaFai Lau else 38051e270976SMartin KaFai Lau err = -EINVAL; 38061e270976SMartin KaFai Lau 38071e270976SMartin KaFai Lau fdput(f); 38081e270976SMartin KaFai Lau return err; 38091e270976SMartin KaFai Lau } 38101e270976SMartin KaFai Lau 3811f56a653cSMartin KaFai Lau #define BPF_BTF_LOAD_LAST_FIELD btf_log_level 3812f56a653cSMartin KaFai Lau 3813f56a653cSMartin KaFai Lau static int bpf_btf_load(const union bpf_attr *attr) 3814f56a653cSMartin KaFai Lau { 3815f56a653cSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_LOAD)) 3816f56a653cSMartin KaFai Lau return -EINVAL; 3817f56a653cSMartin KaFai Lau 38182c78ee89SAlexei Starovoitov if (!bpf_capable()) 3819f56a653cSMartin KaFai Lau return -EPERM; 3820f56a653cSMartin KaFai Lau 3821f56a653cSMartin KaFai Lau return btf_new_fd(attr); 3822f56a653cSMartin KaFai Lau } 3823f56a653cSMartin KaFai Lau 382478958fcaSMartin KaFai Lau #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id 382578958fcaSMartin KaFai Lau 382678958fcaSMartin KaFai Lau static int bpf_btf_get_fd_by_id(const union bpf_attr *attr) 382778958fcaSMartin KaFai Lau { 382878958fcaSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID)) 382978958fcaSMartin KaFai Lau return -EINVAL; 383078958fcaSMartin KaFai Lau 383178958fcaSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 383278958fcaSMartin KaFai Lau return -EPERM; 383378958fcaSMartin KaFai Lau 383478958fcaSMartin KaFai Lau return btf_get_fd_by_id(attr->btf_id); 383578958fcaSMartin KaFai Lau } 383678958fcaSMartin KaFai Lau 383741bdc4b4SYonghong Song static int bpf_task_fd_query_copy(const union bpf_attr *attr, 383841bdc4b4SYonghong Song union bpf_attr __user *uattr, 383941bdc4b4SYonghong Song u32 prog_id, u32 fd_type, 384041bdc4b4SYonghong Song const char *buf, u64 probe_offset, 384141bdc4b4SYonghong Song u64 probe_addr) 384241bdc4b4SYonghong Song { 384341bdc4b4SYonghong Song char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf); 384441bdc4b4SYonghong Song u32 len = buf ? strlen(buf) : 0, input_len; 384541bdc4b4SYonghong Song int err = 0; 384641bdc4b4SYonghong Song 384741bdc4b4SYonghong Song if (put_user(len, &uattr->task_fd_query.buf_len)) 384841bdc4b4SYonghong Song return -EFAULT; 384941bdc4b4SYonghong Song input_len = attr->task_fd_query.buf_len; 385041bdc4b4SYonghong Song if (input_len && ubuf) { 385141bdc4b4SYonghong Song if (!len) { 385241bdc4b4SYonghong Song /* nothing to copy, just make ubuf NULL terminated */ 385341bdc4b4SYonghong Song char zero = '\0'; 385441bdc4b4SYonghong Song 385541bdc4b4SYonghong Song if (put_user(zero, ubuf)) 385641bdc4b4SYonghong Song return -EFAULT; 385741bdc4b4SYonghong Song } else if (input_len >= len + 1) { 385841bdc4b4SYonghong Song /* ubuf can hold the string with NULL terminator */ 385941bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, len + 1)) 386041bdc4b4SYonghong Song return -EFAULT; 386141bdc4b4SYonghong Song } else { 386241bdc4b4SYonghong Song /* ubuf cannot hold the string with NULL terminator, 386341bdc4b4SYonghong Song * do a partial copy with NULL terminator. 386441bdc4b4SYonghong Song */ 386541bdc4b4SYonghong Song char zero = '\0'; 386641bdc4b4SYonghong Song 386741bdc4b4SYonghong Song err = -ENOSPC; 386841bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, input_len - 1)) 386941bdc4b4SYonghong Song return -EFAULT; 387041bdc4b4SYonghong Song if (put_user(zero, ubuf + input_len - 1)) 387141bdc4b4SYonghong Song return -EFAULT; 387241bdc4b4SYonghong Song } 387341bdc4b4SYonghong Song } 387441bdc4b4SYonghong Song 387541bdc4b4SYonghong Song if (put_user(prog_id, &uattr->task_fd_query.prog_id) || 387641bdc4b4SYonghong Song put_user(fd_type, &uattr->task_fd_query.fd_type) || 387741bdc4b4SYonghong Song put_user(probe_offset, &uattr->task_fd_query.probe_offset) || 387841bdc4b4SYonghong Song put_user(probe_addr, &uattr->task_fd_query.probe_addr)) 387941bdc4b4SYonghong Song return -EFAULT; 388041bdc4b4SYonghong Song 388141bdc4b4SYonghong Song return err; 388241bdc4b4SYonghong Song } 388341bdc4b4SYonghong Song 388441bdc4b4SYonghong Song #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr 388541bdc4b4SYonghong Song 388641bdc4b4SYonghong Song static int bpf_task_fd_query(const union bpf_attr *attr, 388741bdc4b4SYonghong Song union bpf_attr __user *uattr) 388841bdc4b4SYonghong Song { 388941bdc4b4SYonghong Song pid_t pid = attr->task_fd_query.pid; 389041bdc4b4SYonghong Song u32 fd = attr->task_fd_query.fd; 389141bdc4b4SYonghong Song const struct perf_event *event; 389241bdc4b4SYonghong Song struct task_struct *task; 389341bdc4b4SYonghong Song struct file *file; 389441bdc4b4SYonghong Song int err; 389541bdc4b4SYonghong Song 389641bdc4b4SYonghong Song if (CHECK_ATTR(BPF_TASK_FD_QUERY)) 389741bdc4b4SYonghong Song return -EINVAL; 389841bdc4b4SYonghong Song 389941bdc4b4SYonghong Song if (!capable(CAP_SYS_ADMIN)) 390041bdc4b4SYonghong Song return -EPERM; 390141bdc4b4SYonghong Song 390241bdc4b4SYonghong Song if (attr->task_fd_query.flags != 0) 390341bdc4b4SYonghong Song return -EINVAL; 390441bdc4b4SYonghong Song 390541bdc4b4SYonghong Song task = get_pid_task(find_vpid(pid), PIDTYPE_PID); 390641bdc4b4SYonghong Song if (!task) 390741bdc4b4SYonghong Song return -ENOENT; 390841bdc4b4SYonghong Song 390941bdc4b4SYonghong Song err = 0; 3910b48845afSEric W. Biederman file = fget_task(task, fd); 3911b48845afSEric W. Biederman put_task_struct(task); 391241bdc4b4SYonghong Song if (!file) 3913b48845afSEric W. Biederman return -EBADF; 391441bdc4b4SYonghong Song 391570ed506cSAndrii Nakryiko if (file->f_op == &bpf_link_fops) { 391670ed506cSAndrii Nakryiko struct bpf_link *link = file->private_data; 391770ed506cSAndrii Nakryiko 3918a3b80e10SAndrii Nakryiko if (link->ops == &bpf_raw_tp_link_lops) { 391970ed506cSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 392070ed506cSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 392141bdc4b4SYonghong Song struct bpf_raw_event_map *btp = raw_tp->btp; 392241bdc4b4SYonghong Song 392341bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, 392470ed506cSAndrii Nakryiko raw_tp->link.prog->aux->id, 392541bdc4b4SYonghong Song BPF_FD_TYPE_RAW_TRACEPOINT, 392641bdc4b4SYonghong Song btp->tp->name, 0, 0); 392741bdc4b4SYonghong Song goto put_file; 392841bdc4b4SYonghong Song } 392970ed506cSAndrii Nakryiko goto out_not_supp; 393070ed506cSAndrii Nakryiko } 393141bdc4b4SYonghong Song 393241bdc4b4SYonghong Song event = perf_get_event(file); 393341bdc4b4SYonghong Song if (!IS_ERR(event)) { 393441bdc4b4SYonghong Song u64 probe_offset, probe_addr; 393541bdc4b4SYonghong Song u32 prog_id, fd_type; 393641bdc4b4SYonghong Song const char *buf; 393741bdc4b4SYonghong Song 393841bdc4b4SYonghong Song err = bpf_get_perf_event_info(event, &prog_id, &fd_type, 393941bdc4b4SYonghong Song &buf, &probe_offset, 394041bdc4b4SYonghong Song &probe_addr); 394141bdc4b4SYonghong Song if (!err) 394241bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, prog_id, 394341bdc4b4SYonghong Song fd_type, buf, 394441bdc4b4SYonghong Song probe_offset, 394541bdc4b4SYonghong Song probe_addr); 394641bdc4b4SYonghong Song goto put_file; 394741bdc4b4SYonghong Song } 394841bdc4b4SYonghong Song 394970ed506cSAndrii Nakryiko out_not_supp: 395041bdc4b4SYonghong Song err = -ENOTSUPP; 395141bdc4b4SYonghong Song put_file: 395241bdc4b4SYonghong Song fput(file); 395341bdc4b4SYonghong Song return err; 395441bdc4b4SYonghong Song } 395541bdc4b4SYonghong Song 3956cb4d03abSBrian Vazquez #define BPF_MAP_BATCH_LAST_FIELD batch.flags 3957cb4d03abSBrian Vazquez 3958cb4d03abSBrian Vazquez #define BPF_DO_BATCH(fn) \ 3959cb4d03abSBrian Vazquez do { \ 3960cb4d03abSBrian Vazquez if (!fn) { \ 3961cb4d03abSBrian Vazquez err = -ENOTSUPP; \ 3962cb4d03abSBrian Vazquez goto err_put; \ 3963cb4d03abSBrian Vazquez } \ 3964cb4d03abSBrian Vazquez err = fn(map, attr, uattr); \ 3965cb4d03abSBrian Vazquez } while (0) 3966cb4d03abSBrian Vazquez 3967cb4d03abSBrian Vazquez static int bpf_map_do_batch(const union bpf_attr *attr, 3968cb4d03abSBrian Vazquez union bpf_attr __user *uattr, 3969cb4d03abSBrian Vazquez int cmd) 3970cb4d03abSBrian Vazquez { 3971cb4d03abSBrian Vazquez struct bpf_map *map; 3972cb4d03abSBrian Vazquez int err, ufd; 3973cb4d03abSBrian Vazquez struct fd f; 3974cb4d03abSBrian Vazquez 3975cb4d03abSBrian Vazquez if (CHECK_ATTR(BPF_MAP_BATCH)) 3976cb4d03abSBrian Vazquez return -EINVAL; 3977cb4d03abSBrian Vazquez 3978cb4d03abSBrian Vazquez ufd = attr->batch.map_fd; 3979cb4d03abSBrian Vazquez f = fdget(ufd); 3980cb4d03abSBrian Vazquez map = __bpf_map_get(f); 3981cb4d03abSBrian Vazquez if (IS_ERR(map)) 3982cb4d03abSBrian Vazquez return PTR_ERR(map); 3983cb4d03abSBrian Vazquez 398405799638SYonghong Song if ((cmd == BPF_MAP_LOOKUP_BATCH || 398505799638SYonghong Song cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) && 3986cb4d03abSBrian Vazquez !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 3987cb4d03abSBrian Vazquez err = -EPERM; 3988cb4d03abSBrian Vazquez goto err_put; 3989cb4d03abSBrian Vazquez } 3990cb4d03abSBrian Vazquez 3991cb4d03abSBrian Vazquez if (cmd != BPF_MAP_LOOKUP_BATCH && 3992cb4d03abSBrian Vazquez !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 3993cb4d03abSBrian Vazquez err = -EPERM; 3994cb4d03abSBrian Vazquez goto err_put; 3995cb4d03abSBrian Vazquez } 3996cb4d03abSBrian Vazquez 3997cb4d03abSBrian Vazquez if (cmd == BPF_MAP_LOOKUP_BATCH) 3998cb4d03abSBrian Vazquez BPF_DO_BATCH(map->ops->map_lookup_batch); 399905799638SYonghong Song else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) 400005799638SYonghong Song BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch); 4001aa2e93b8SBrian Vazquez else if (cmd == BPF_MAP_UPDATE_BATCH) 4002aa2e93b8SBrian Vazquez BPF_DO_BATCH(map->ops->map_update_batch); 4003aa2e93b8SBrian Vazquez else 4004aa2e93b8SBrian Vazquez BPF_DO_BATCH(map->ops->map_delete_batch); 4005cb4d03abSBrian Vazquez 4006cb4d03abSBrian Vazquez err_put: 4007cb4d03abSBrian Vazquez fdput(f); 4008cb4d03abSBrian Vazquez return err; 4009cb4d03abSBrian Vazquez } 4010cb4d03abSBrian Vazquez 4011de4e05caSYonghong Song static int tracing_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 4012de4e05caSYonghong Song { 40134a1e7c0cSToke Høiland-Jørgensen if (attr->link_create.attach_type != prog->expected_attach_type) 40144a1e7c0cSToke Høiland-Jørgensen return -EINVAL; 4015de4e05caSYonghong Song 40164a1e7c0cSToke Høiland-Jørgensen if (prog->expected_attach_type == BPF_TRACE_ITER) 40174a1e7c0cSToke Høiland-Jørgensen return bpf_iter_link_attach(attr, prog); 40184a1e7c0cSToke Høiland-Jørgensen else if (prog->type == BPF_PROG_TYPE_EXT) 40194a1e7c0cSToke Høiland-Jørgensen return bpf_tracing_prog_attach(prog, 40204a1e7c0cSToke Høiland-Jørgensen attr->link_create.target_fd, 40214a1e7c0cSToke Høiland-Jørgensen attr->link_create.target_btf_id); 4022de4e05caSYonghong Song return -EINVAL; 4023de4e05caSYonghong Song } 4024de4e05caSYonghong Song 40255e7b3020SYonghong Song #define BPF_LINK_CREATE_LAST_FIELD link_create.iter_info_len 4026af6eea57SAndrii Nakryiko static int link_create(union bpf_attr *attr) 4027af6eea57SAndrii Nakryiko { 4028af6eea57SAndrii Nakryiko enum bpf_prog_type ptype; 4029af6eea57SAndrii Nakryiko struct bpf_prog *prog; 4030af6eea57SAndrii Nakryiko int ret; 4031af6eea57SAndrii Nakryiko 4032af6eea57SAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_CREATE)) 4033af6eea57SAndrii Nakryiko return -EINVAL; 4034af6eea57SAndrii Nakryiko 40354a1e7c0cSToke Høiland-Jørgensen prog = bpf_prog_get(attr->link_create.prog_fd); 4036af6eea57SAndrii Nakryiko if (IS_ERR(prog)) 4037af6eea57SAndrii Nakryiko return PTR_ERR(prog); 4038af6eea57SAndrii Nakryiko 4039af6eea57SAndrii Nakryiko ret = bpf_prog_attach_check_attach_type(prog, 4040af6eea57SAndrii Nakryiko attr->link_create.attach_type); 4041af6eea57SAndrii Nakryiko if (ret) 40424a1e7c0cSToke Høiland-Jørgensen goto out; 40434a1e7c0cSToke Høiland-Jørgensen 40444a1e7c0cSToke Høiland-Jørgensen if (prog->type == BPF_PROG_TYPE_EXT) { 40454a1e7c0cSToke Høiland-Jørgensen ret = tracing_bpf_link_attach(attr, prog); 40464a1e7c0cSToke Høiland-Jørgensen goto out; 40474a1e7c0cSToke Høiland-Jørgensen } 40484a1e7c0cSToke Høiland-Jørgensen 40494a1e7c0cSToke Høiland-Jørgensen ptype = attach_type_to_prog_type(attr->link_create.attach_type); 40504a1e7c0cSToke Høiland-Jørgensen if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type) { 40514a1e7c0cSToke Høiland-Jørgensen ret = -EINVAL; 40524a1e7c0cSToke Høiland-Jørgensen goto out; 40534a1e7c0cSToke Høiland-Jørgensen } 4054af6eea57SAndrii Nakryiko 4055af6eea57SAndrii Nakryiko switch (ptype) { 4056af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 4057af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 4058af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 4059af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 4060af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 4061af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 4062af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 4063af6eea57SAndrii Nakryiko ret = cgroup_bpf_link_attach(attr, prog); 4064af6eea57SAndrii Nakryiko break; 4065de4e05caSYonghong Song case BPF_PROG_TYPE_TRACING: 4066de4e05caSYonghong Song ret = tracing_bpf_link_attach(attr, prog); 4067de4e05caSYonghong Song break; 40687f045a49SJakub Sitnicki case BPF_PROG_TYPE_FLOW_DISSECTOR: 4069e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP: 40707f045a49SJakub Sitnicki ret = netns_bpf_link_create(attr, prog); 40717f045a49SJakub Sitnicki break; 4072310ad797SAndrii Nakryiko #ifdef CONFIG_NET 4073aa8d3a71SAndrii Nakryiko case BPF_PROG_TYPE_XDP: 4074aa8d3a71SAndrii Nakryiko ret = bpf_xdp_link_attach(attr, prog); 4075aa8d3a71SAndrii Nakryiko break; 4076310ad797SAndrii Nakryiko #endif 4077af6eea57SAndrii Nakryiko default: 4078af6eea57SAndrii Nakryiko ret = -EINVAL; 4079af6eea57SAndrii Nakryiko } 4080af6eea57SAndrii Nakryiko 40814a1e7c0cSToke Høiland-Jørgensen out: 4082af6eea57SAndrii Nakryiko if (ret < 0) 4083af6eea57SAndrii Nakryiko bpf_prog_put(prog); 4084af6eea57SAndrii Nakryiko return ret; 4085af6eea57SAndrii Nakryiko } 4086af6eea57SAndrii Nakryiko 40870c991ebcSAndrii Nakryiko #define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd 40880c991ebcSAndrii Nakryiko 40890c991ebcSAndrii Nakryiko static int link_update(union bpf_attr *attr) 40900c991ebcSAndrii Nakryiko { 40910c991ebcSAndrii Nakryiko struct bpf_prog *old_prog = NULL, *new_prog; 40920c991ebcSAndrii Nakryiko struct bpf_link *link; 40930c991ebcSAndrii Nakryiko u32 flags; 40940c991ebcSAndrii Nakryiko int ret; 40950c991ebcSAndrii Nakryiko 40960c991ebcSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_UPDATE)) 40970c991ebcSAndrii Nakryiko return -EINVAL; 40980c991ebcSAndrii Nakryiko 40990c991ebcSAndrii Nakryiko flags = attr->link_update.flags; 41000c991ebcSAndrii Nakryiko if (flags & ~BPF_F_REPLACE) 41010c991ebcSAndrii Nakryiko return -EINVAL; 41020c991ebcSAndrii Nakryiko 41030c991ebcSAndrii Nakryiko link = bpf_link_get_from_fd(attr->link_update.link_fd); 41040c991ebcSAndrii Nakryiko if (IS_ERR(link)) 41050c991ebcSAndrii Nakryiko return PTR_ERR(link); 41060c991ebcSAndrii Nakryiko 41070c991ebcSAndrii Nakryiko new_prog = bpf_prog_get(attr->link_update.new_prog_fd); 41084adb7a4aSAndrii Nakryiko if (IS_ERR(new_prog)) { 41094adb7a4aSAndrii Nakryiko ret = PTR_ERR(new_prog); 41104adb7a4aSAndrii Nakryiko goto out_put_link; 41114adb7a4aSAndrii Nakryiko } 41120c991ebcSAndrii Nakryiko 41130c991ebcSAndrii Nakryiko if (flags & BPF_F_REPLACE) { 41140c991ebcSAndrii Nakryiko old_prog = bpf_prog_get(attr->link_update.old_prog_fd); 41150c991ebcSAndrii Nakryiko if (IS_ERR(old_prog)) { 41160c991ebcSAndrii Nakryiko ret = PTR_ERR(old_prog); 41170c991ebcSAndrii Nakryiko old_prog = NULL; 41180c991ebcSAndrii Nakryiko goto out_put_progs; 41190c991ebcSAndrii Nakryiko } 41204adb7a4aSAndrii Nakryiko } else if (attr->link_update.old_prog_fd) { 41214adb7a4aSAndrii Nakryiko ret = -EINVAL; 41224adb7a4aSAndrii Nakryiko goto out_put_progs; 41230c991ebcSAndrii Nakryiko } 41240c991ebcSAndrii Nakryiko 4125f9d04127SAndrii Nakryiko if (link->ops->update_prog) 4126f9d04127SAndrii Nakryiko ret = link->ops->update_prog(link, new_prog, old_prog); 4127f9d04127SAndrii Nakryiko else 41280c991ebcSAndrii Nakryiko ret = -EINVAL; 41290c991ebcSAndrii Nakryiko 41300c991ebcSAndrii Nakryiko out_put_progs: 41310c991ebcSAndrii Nakryiko if (old_prog) 41320c991ebcSAndrii Nakryiko bpf_prog_put(old_prog); 41330c991ebcSAndrii Nakryiko if (ret) 41340c991ebcSAndrii Nakryiko bpf_prog_put(new_prog); 41354adb7a4aSAndrii Nakryiko out_put_link: 41364adb7a4aSAndrii Nakryiko bpf_link_put(link); 41370c991ebcSAndrii Nakryiko return ret; 41380c991ebcSAndrii Nakryiko } 41390c991ebcSAndrii Nakryiko 414073b11c2aSAndrii Nakryiko #define BPF_LINK_DETACH_LAST_FIELD link_detach.link_fd 414173b11c2aSAndrii Nakryiko 414273b11c2aSAndrii Nakryiko static int link_detach(union bpf_attr *attr) 414373b11c2aSAndrii Nakryiko { 414473b11c2aSAndrii Nakryiko struct bpf_link *link; 414573b11c2aSAndrii Nakryiko int ret; 414673b11c2aSAndrii Nakryiko 414773b11c2aSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_DETACH)) 414873b11c2aSAndrii Nakryiko return -EINVAL; 414973b11c2aSAndrii Nakryiko 415073b11c2aSAndrii Nakryiko link = bpf_link_get_from_fd(attr->link_detach.link_fd); 415173b11c2aSAndrii Nakryiko if (IS_ERR(link)) 415273b11c2aSAndrii Nakryiko return PTR_ERR(link); 415373b11c2aSAndrii Nakryiko 415473b11c2aSAndrii Nakryiko if (link->ops->detach) 415573b11c2aSAndrii Nakryiko ret = link->ops->detach(link); 415673b11c2aSAndrii Nakryiko else 415773b11c2aSAndrii Nakryiko ret = -EOPNOTSUPP; 415873b11c2aSAndrii Nakryiko 415973b11c2aSAndrii Nakryiko bpf_link_put(link); 416073b11c2aSAndrii Nakryiko return ret; 416173b11c2aSAndrii Nakryiko } 416273b11c2aSAndrii Nakryiko 4163005142b8SAlexei Starovoitov static struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link) 41642d602c8cSAndrii Nakryiko { 4165005142b8SAlexei Starovoitov return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT); 4166005142b8SAlexei Starovoitov } 4167005142b8SAlexei Starovoitov 4168005142b8SAlexei Starovoitov struct bpf_link *bpf_link_by_id(u32 id) 4169005142b8SAlexei Starovoitov { 4170005142b8SAlexei Starovoitov struct bpf_link *link; 4171005142b8SAlexei Starovoitov 4172005142b8SAlexei Starovoitov if (!id) 4173005142b8SAlexei Starovoitov return ERR_PTR(-ENOENT); 4174005142b8SAlexei Starovoitov 4175005142b8SAlexei Starovoitov spin_lock_bh(&link_idr_lock); 4176005142b8SAlexei Starovoitov /* before link is "settled", ID is 0, pretend it doesn't exist yet */ 4177005142b8SAlexei Starovoitov link = idr_find(&link_idr, id); 4178005142b8SAlexei Starovoitov if (link) { 4179005142b8SAlexei Starovoitov if (link->id) 4180005142b8SAlexei Starovoitov link = bpf_link_inc_not_zero(link); 4181005142b8SAlexei Starovoitov else 4182005142b8SAlexei Starovoitov link = ERR_PTR(-EAGAIN); 4183005142b8SAlexei Starovoitov } else { 4184005142b8SAlexei Starovoitov link = ERR_PTR(-ENOENT); 4185005142b8SAlexei Starovoitov } 4186005142b8SAlexei Starovoitov spin_unlock_bh(&link_idr_lock); 4187005142b8SAlexei Starovoitov return link; 41882d602c8cSAndrii Nakryiko } 41892d602c8cSAndrii Nakryiko 41902d602c8cSAndrii Nakryiko #define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id 41912d602c8cSAndrii Nakryiko 41922d602c8cSAndrii Nakryiko static int bpf_link_get_fd_by_id(const union bpf_attr *attr) 41932d602c8cSAndrii Nakryiko { 41942d602c8cSAndrii Nakryiko struct bpf_link *link; 41952d602c8cSAndrii Nakryiko u32 id = attr->link_id; 4196005142b8SAlexei Starovoitov int fd; 41972d602c8cSAndrii Nakryiko 41982d602c8cSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID)) 41992d602c8cSAndrii Nakryiko return -EINVAL; 42002d602c8cSAndrii Nakryiko 42012d602c8cSAndrii Nakryiko if (!capable(CAP_SYS_ADMIN)) 42022d602c8cSAndrii Nakryiko return -EPERM; 42032d602c8cSAndrii Nakryiko 4204005142b8SAlexei Starovoitov link = bpf_link_by_id(id); 4205005142b8SAlexei Starovoitov if (IS_ERR(link)) 4206005142b8SAlexei Starovoitov return PTR_ERR(link); 42072d602c8cSAndrii Nakryiko 42082d602c8cSAndrii Nakryiko fd = bpf_link_new_fd(link); 42092d602c8cSAndrii Nakryiko if (fd < 0) 42102d602c8cSAndrii Nakryiko bpf_link_put(link); 42112d602c8cSAndrii Nakryiko 42122d602c8cSAndrii Nakryiko return fd; 42132d602c8cSAndrii Nakryiko } 42142d602c8cSAndrii Nakryiko 4215d46edd67SSong Liu DEFINE_MUTEX(bpf_stats_enabled_mutex); 4216d46edd67SSong Liu 4217d46edd67SSong Liu static int bpf_stats_release(struct inode *inode, struct file *file) 4218d46edd67SSong Liu { 4219d46edd67SSong Liu mutex_lock(&bpf_stats_enabled_mutex); 4220d46edd67SSong Liu static_key_slow_dec(&bpf_stats_enabled_key.key); 4221d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 4222d46edd67SSong Liu return 0; 4223d46edd67SSong Liu } 4224d46edd67SSong Liu 4225d46edd67SSong Liu static const struct file_operations bpf_stats_fops = { 4226d46edd67SSong Liu .release = bpf_stats_release, 4227d46edd67SSong Liu }; 4228d46edd67SSong Liu 4229d46edd67SSong Liu static int bpf_enable_runtime_stats(void) 4230d46edd67SSong Liu { 4231d46edd67SSong Liu int fd; 4232d46edd67SSong Liu 4233d46edd67SSong Liu mutex_lock(&bpf_stats_enabled_mutex); 4234d46edd67SSong Liu 4235d46edd67SSong Liu /* Set a very high limit to avoid overflow */ 4236d46edd67SSong Liu if (static_key_count(&bpf_stats_enabled_key.key) > INT_MAX / 2) { 4237d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 4238d46edd67SSong Liu return -EBUSY; 4239d46edd67SSong Liu } 4240d46edd67SSong Liu 4241d46edd67SSong Liu fd = anon_inode_getfd("bpf-stats", &bpf_stats_fops, NULL, O_CLOEXEC); 4242d46edd67SSong Liu if (fd >= 0) 4243d46edd67SSong Liu static_key_slow_inc(&bpf_stats_enabled_key.key); 4244d46edd67SSong Liu 4245d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 4246d46edd67SSong Liu return fd; 4247d46edd67SSong Liu } 4248d46edd67SSong Liu 4249d46edd67SSong Liu #define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type 4250d46edd67SSong Liu 4251d46edd67SSong Liu static int bpf_enable_stats(union bpf_attr *attr) 4252d46edd67SSong Liu { 4253d46edd67SSong Liu 4254d46edd67SSong Liu if (CHECK_ATTR(BPF_ENABLE_STATS)) 4255d46edd67SSong Liu return -EINVAL; 4256d46edd67SSong Liu 4257d46edd67SSong Liu if (!capable(CAP_SYS_ADMIN)) 4258d46edd67SSong Liu return -EPERM; 4259d46edd67SSong Liu 4260d46edd67SSong Liu switch (attr->enable_stats.type) { 4261d46edd67SSong Liu case BPF_STATS_RUN_TIME: 4262d46edd67SSong Liu return bpf_enable_runtime_stats(); 4263d46edd67SSong Liu default: 4264d46edd67SSong Liu break; 4265d46edd67SSong Liu } 4266d46edd67SSong Liu return -EINVAL; 4267d46edd67SSong Liu } 4268d46edd67SSong Liu 4269ac51d99bSYonghong Song #define BPF_ITER_CREATE_LAST_FIELD iter_create.flags 4270ac51d99bSYonghong Song 4271ac51d99bSYonghong Song static int bpf_iter_create(union bpf_attr *attr) 4272ac51d99bSYonghong Song { 4273ac51d99bSYonghong Song struct bpf_link *link; 4274ac51d99bSYonghong Song int err; 4275ac51d99bSYonghong Song 4276ac51d99bSYonghong Song if (CHECK_ATTR(BPF_ITER_CREATE)) 4277ac51d99bSYonghong Song return -EINVAL; 4278ac51d99bSYonghong Song 4279ac51d99bSYonghong Song if (attr->iter_create.flags) 4280ac51d99bSYonghong Song return -EINVAL; 4281ac51d99bSYonghong Song 4282ac51d99bSYonghong Song link = bpf_link_get_from_fd(attr->iter_create.link_fd); 4283ac51d99bSYonghong Song if (IS_ERR(link)) 4284ac51d99bSYonghong Song return PTR_ERR(link); 4285ac51d99bSYonghong Song 4286ac51d99bSYonghong Song err = bpf_iter_new_fd(link); 4287ac51d99bSYonghong Song bpf_link_put(link); 4288ac51d99bSYonghong Song 4289ac51d99bSYonghong Song return err; 4290ac51d99bSYonghong Song } 4291ac51d99bSYonghong Song 4292ef15314aSYiFei Zhu #define BPF_PROG_BIND_MAP_LAST_FIELD prog_bind_map.flags 4293ef15314aSYiFei Zhu 4294ef15314aSYiFei Zhu static int bpf_prog_bind_map(union bpf_attr *attr) 4295ef15314aSYiFei Zhu { 4296ef15314aSYiFei Zhu struct bpf_prog *prog; 4297ef15314aSYiFei Zhu struct bpf_map *map; 4298ef15314aSYiFei Zhu struct bpf_map **used_maps_old, **used_maps_new; 4299ef15314aSYiFei Zhu int i, ret = 0; 4300ef15314aSYiFei Zhu 4301ef15314aSYiFei Zhu if (CHECK_ATTR(BPF_PROG_BIND_MAP)) 4302ef15314aSYiFei Zhu return -EINVAL; 4303ef15314aSYiFei Zhu 4304ef15314aSYiFei Zhu if (attr->prog_bind_map.flags) 4305ef15314aSYiFei Zhu return -EINVAL; 4306ef15314aSYiFei Zhu 4307ef15314aSYiFei Zhu prog = bpf_prog_get(attr->prog_bind_map.prog_fd); 4308ef15314aSYiFei Zhu if (IS_ERR(prog)) 4309ef15314aSYiFei Zhu return PTR_ERR(prog); 4310ef15314aSYiFei Zhu 4311ef15314aSYiFei Zhu map = bpf_map_get(attr->prog_bind_map.map_fd); 4312ef15314aSYiFei Zhu if (IS_ERR(map)) { 4313ef15314aSYiFei Zhu ret = PTR_ERR(map); 4314ef15314aSYiFei Zhu goto out_prog_put; 4315ef15314aSYiFei Zhu } 4316ef15314aSYiFei Zhu 4317ef15314aSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 4318ef15314aSYiFei Zhu 4319ef15314aSYiFei Zhu used_maps_old = prog->aux->used_maps; 4320ef15314aSYiFei Zhu 4321ef15314aSYiFei Zhu for (i = 0; i < prog->aux->used_map_cnt; i++) 43221028ae40SStanislav Fomichev if (used_maps_old[i] == map) { 43231028ae40SStanislav Fomichev bpf_map_put(map); 4324ef15314aSYiFei Zhu goto out_unlock; 43251028ae40SStanislav Fomichev } 4326ef15314aSYiFei Zhu 4327ef15314aSYiFei Zhu used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1, 4328ef15314aSYiFei Zhu sizeof(used_maps_new[0]), 4329ef15314aSYiFei Zhu GFP_KERNEL); 4330ef15314aSYiFei Zhu if (!used_maps_new) { 4331ef15314aSYiFei Zhu ret = -ENOMEM; 4332ef15314aSYiFei Zhu goto out_unlock; 4333ef15314aSYiFei Zhu } 4334ef15314aSYiFei Zhu 4335ef15314aSYiFei Zhu memcpy(used_maps_new, used_maps_old, 4336ef15314aSYiFei Zhu sizeof(used_maps_old[0]) * prog->aux->used_map_cnt); 4337ef15314aSYiFei Zhu used_maps_new[prog->aux->used_map_cnt] = map; 4338ef15314aSYiFei Zhu 4339ef15314aSYiFei Zhu prog->aux->used_map_cnt++; 4340ef15314aSYiFei Zhu prog->aux->used_maps = used_maps_new; 4341ef15314aSYiFei Zhu 4342ef15314aSYiFei Zhu kfree(used_maps_old); 4343ef15314aSYiFei Zhu 4344ef15314aSYiFei Zhu out_unlock: 4345ef15314aSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 4346ef15314aSYiFei Zhu 4347ef15314aSYiFei Zhu if (ret) 4348ef15314aSYiFei Zhu bpf_map_put(map); 4349ef15314aSYiFei Zhu out_prog_put: 4350ef15314aSYiFei Zhu bpf_prog_put(prog); 4351ef15314aSYiFei Zhu return ret; 4352ef15314aSYiFei Zhu } 4353ef15314aSYiFei Zhu 435499c55f7dSAlexei Starovoitov SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 435599c55f7dSAlexei Starovoitov { 43568096f229SGreg Kroah-Hartman union bpf_attr attr; 435799c55f7dSAlexei Starovoitov int err; 435899c55f7dSAlexei Starovoitov 43592c78ee89SAlexei Starovoitov if (sysctl_unprivileged_bpf_disabled && !bpf_capable()) 436099c55f7dSAlexei Starovoitov return -EPERM; 436199c55f7dSAlexei Starovoitov 4362dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size); 436399c55f7dSAlexei Starovoitov if (err) 436499c55f7dSAlexei Starovoitov return err; 43651e270976SMartin KaFai Lau size = min_t(u32, size, sizeof(attr)); 436699c55f7dSAlexei Starovoitov 436799c55f7dSAlexei Starovoitov /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 43688096f229SGreg Kroah-Hartman memset(&attr, 0, sizeof(attr)); 436999c55f7dSAlexei Starovoitov if (copy_from_user(&attr, uattr, size) != 0) 437099c55f7dSAlexei Starovoitov return -EFAULT; 437199c55f7dSAlexei Starovoitov 4372afdb09c7SChenbo Feng err = security_bpf(cmd, &attr, size); 4373afdb09c7SChenbo Feng if (err < 0) 4374afdb09c7SChenbo Feng return err; 4375afdb09c7SChenbo Feng 437699c55f7dSAlexei Starovoitov switch (cmd) { 437799c55f7dSAlexei Starovoitov case BPF_MAP_CREATE: 437899c55f7dSAlexei Starovoitov err = map_create(&attr); 437999c55f7dSAlexei Starovoitov break; 4380db20fd2bSAlexei Starovoitov case BPF_MAP_LOOKUP_ELEM: 4381db20fd2bSAlexei Starovoitov err = map_lookup_elem(&attr); 4382db20fd2bSAlexei Starovoitov break; 4383db20fd2bSAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 4384db20fd2bSAlexei Starovoitov err = map_update_elem(&attr); 4385db20fd2bSAlexei Starovoitov break; 4386db20fd2bSAlexei Starovoitov case BPF_MAP_DELETE_ELEM: 4387db20fd2bSAlexei Starovoitov err = map_delete_elem(&attr); 4388db20fd2bSAlexei Starovoitov break; 4389db20fd2bSAlexei Starovoitov case BPF_MAP_GET_NEXT_KEY: 4390db20fd2bSAlexei Starovoitov err = map_get_next_key(&attr); 4391db20fd2bSAlexei Starovoitov break; 439287df15deSDaniel Borkmann case BPF_MAP_FREEZE: 439387df15deSDaniel Borkmann err = map_freeze(&attr); 439487df15deSDaniel Borkmann break; 439509756af4SAlexei Starovoitov case BPF_PROG_LOAD: 4396838e9690SYonghong Song err = bpf_prog_load(&attr, uattr); 439709756af4SAlexei Starovoitov break; 4398b2197755SDaniel Borkmann case BPF_OBJ_PIN: 4399b2197755SDaniel Borkmann err = bpf_obj_pin(&attr); 4400b2197755SDaniel Borkmann break; 4401b2197755SDaniel Borkmann case BPF_OBJ_GET: 4402b2197755SDaniel Borkmann err = bpf_obj_get(&attr); 4403b2197755SDaniel Borkmann break; 4404f4324551SDaniel Mack case BPF_PROG_ATTACH: 4405f4324551SDaniel Mack err = bpf_prog_attach(&attr); 4406f4324551SDaniel Mack break; 4407f4324551SDaniel Mack case BPF_PROG_DETACH: 4408f4324551SDaniel Mack err = bpf_prog_detach(&attr); 4409f4324551SDaniel Mack break; 4410468e2f64SAlexei Starovoitov case BPF_PROG_QUERY: 4411468e2f64SAlexei Starovoitov err = bpf_prog_query(&attr, uattr); 4412468e2f64SAlexei Starovoitov break; 44131cf1cae9SAlexei Starovoitov case BPF_PROG_TEST_RUN: 44141cf1cae9SAlexei Starovoitov err = bpf_prog_test_run(&attr, uattr); 44151cf1cae9SAlexei Starovoitov break; 441634ad5580SMartin KaFai Lau case BPF_PROG_GET_NEXT_ID: 441734ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 441834ad5580SMartin KaFai Lau &prog_idr, &prog_idr_lock); 441934ad5580SMartin KaFai Lau break; 442034ad5580SMartin KaFai Lau case BPF_MAP_GET_NEXT_ID: 442134ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 442234ad5580SMartin KaFai Lau &map_idr, &map_idr_lock); 442334ad5580SMartin KaFai Lau break; 44241b9ed84eSQuentin Monnet case BPF_BTF_GET_NEXT_ID: 44251b9ed84eSQuentin Monnet err = bpf_obj_get_next_id(&attr, uattr, 44261b9ed84eSQuentin Monnet &btf_idr, &btf_idr_lock); 44271b9ed84eSQuentin Monnet break; 4428b16d9aa4SMartin KaFai Lau case BPF_PROG_GET_FD_BY_ID: 4429b16d9aa4SMartin KaFai Lau err = bpf_prog_get_fd_by_id(&attr); 4430b16d9aa4SMartin KaFai Lau break; 4431bd5f5f4eSMartin KaFai Lau case BPF_MAP_GET_FD_BY_ID: 4432bd5f5f4eSMartin KaFai Lau err = bpf_map_get_fd_by_id(&attr); 4433bd5f5f4eSMartin KaFai Lau break; 44341e270976SMartin KaFai Lau case BPF_OBJ_GET_INFO_BY_FD: 44351e270976SMartin KaFai Lau err = bpf_obj_get_info_by_fd(&attr, uattr); 44361e270976SMartin KaFai Lau break; 4437c4f6699dSAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN: 4438c4f6699dSAlexei Starovoitov err = bpf_raw_tracepoint_open(&attr); 4439c4f6699dSAlexei Starovoitov break; 4440f56a653cSMartin KaFai Lau case BPF_BTF_LOAD: 4441f56a653cSMartin KaFai Lau err = bpf_btf_load(&attr); 4442f56a653cSMartin KaFai Lau break; 444378958fcaSMartin KaFai Lau case BPF_BTF_GET_FD_BY_ID: 444478958fcaSMartin KaFai Lau err = bpf_btf_get_fd_by_id(&attr); 444578958fcaSMartin KaFai Lau break; 444641bdc4b4SYonghong Song case BPF_TASK_FD_QUERY: 444741bdc4b4SYonghong Song err = bpf_task_fd_query(&attr, uattr); 444841bdc4b4SYonghong Song break; 4449bd513cd0SMauricio Vasquez B case BPF_MAP_LOOKUP_AND_DELETE_ELEM: 4450bd513cd0SMauricio Vasquez B err = map_lookup_and_delete_elem(&attr); 4451bd513cd0SMauricio Vasquez B break; 4452cb4d03abSBrian Vazquez case BPF_MAP_LOOKUP_BATCH: 4453cb4d03abSBrian Vazquez err = bpf_map_do_batch(&attr, uattr, BPF_MAP_LOOKUP_BATCH); 4454cb4d03abSBrian Vazquez break; 445505799638SYonghong Song case BPF_MAP_LOOKUP_AND_DELETE_BATCH: 445605799638SYonghong Song err = bpf_map_do_batch(&attr, uattr, 445705799638SYonghong Song BPF_MAP_LOOKUP_AND_DELETE_BATCH); 445805799638SYonghong Song break; 4459aa2e93b8SBrian Vazquez case BPF_MAP_UPDATE_BATCH: 4460aa2e93b8SBrian Vazquez err = bpf_map_do_batch(&attr, uattr, BPF_MAP_UPDATE_BATCH); 4461aa2e93b8SBrian Vazquez break; 4462aa2e93b8SBrian Vazquez case BPF_MAP_DELETE_BATCH: 4463aa2e93b8SBrian Vazquez err = bpf_map_do_batch(&attr, uattr, BPF_MAP_DELETE_BATCH); 4464aa2e93b8SBrian Vazquez break; 4465af6eea57SAndrii Nakryiko case BPF_LINK_CREATE: 4466af6eea57SAndrii Nakryiko err = link_create(&attr); 4467af6eea57SAndrii Nakryiko break; 44680c991ebcSAndrii Nakryiko case BPF_LINK_UPDATE: 44690c991ebcSAndrii Nakryiko err = link_update(&attr); 44700c991ebcSAndrii Nakryiko break; 44712d602c8cSAndrii Nakryiko case BPF_LINK_GET_FD_BY_ID: 44722d602c8cSAndrii Nakryiko err = bpf_link_get_fd_by_id(&attr); 44732d602c8cSAndrii Nakryiko break; 44742d602c8cSAndrii Nakryiko case BPF_LINK_GET_NEXT_ID: 44752d602c8cSAndrii Nakryiko err = bpf_obj_get_next_id(&attr, uattr, 44762d602c8cSAndrii Nakryiko &link_idr, &link_idr_lock); 44772d602c8cSAndrii Nakryiko break; 4478d46edd67SSong Liu case BPF_ENABLE_STATS: 4479d46edd67SSong Liu err = bpf_enable_stats(&attr); 4480d46edd67SSong Liu break; 4481ac51d99bSYonghong Song case BPF_ITER_CREATE: 4482ac51d99bSYonghong Song err = bpf_iter_create(&attr); 4483ac51d99bSYonghong Song break; 448473b11c2aSAndrii Nakryiko case BPF_LINK_DETACH: 448573b11c2aSAndrii Nakryiko err = link_detach(&attr); 448673b11c2aSAndrii Nakryiko break; 4487ef15314aSYiFei Zhu case BPF_PROG_BIND_MAP: 4488ef15314aSYiFei Zhu err = bpf_prog_bind_map(&attr); 4489ef15314aSYiFei Zhu break; 449099c55f7dSAlexei Starovoitov default: 449199c55f7dSAlexei Starovoitov err = -EINVAL; 449299c55f7dSAlexei Starovoitov break; 449399c55f7dSAlexei Starovoitov } 449499c55f7dSAlexei Starovoitov 449599c55f7dSAlexei Starovoitov return err; 449699c55f7dSAlexei Starovoitov } 4497