15b497af4SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 299c55f7dSAlexei Starovoitov /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 399c55f7dSAlexei Starovoitov */ 499c55f7dSAlexei Starovoitov #include <linux/bpf.h> 5a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 6f4364dcfSSean Young #include <linux/bpf_lirc.h> 7f56a653cSMartin KaFai Lau #include <linux/btf.h> 899c55f7dSAlexei Starovoitov #include <linux/syscalls.h> 999c55f7dSAlexei Starovoitov #include <linux/slab.h> 103f07c014SIngo Molnar #include <linux/sched/signal.h> 11d407bd25SDaniel Borkmann #include <linux/vmalloc.h> 12d407bd25SDaniel Borkmann #include <linux/mmzone.h> 1399c55f7dSAlexei Starovoitov #include <linux/anon_inodes.h> 1441bdc4b4SYonghong Song #include <linux/fdtable.h> 15db20fd2bSAlexei Starovoitov #include <linux/file.h> 1641bdc4b4SYonghong Song #include <linux/fs.h> 1709756af4SAlexei Starovoitov #include <linux/license.h> 1809756af4SAlexei Starovoitov #include <linux/filter.h> 192541517cSAlexei Starovoitov #include <linux/version.h> 20535e7b4bSMickaël Salaün #include <linux/kernel.h> 21dc4bb0e2SMartin KaFai Lau #include <linux/idr.h> 22cb4d2b3fSMartin KaFai Lau #include <linux/cred.h> 23cb4d2b3fSMartin KaFai Lau #include <linux/timekeeping.h> 24cb4d2b3fSMartin KaFai Lau #include <linux/ctype.h> 259ef09e35SMark Rutland #include <linux/nospec.h> 26bae141f5SDaniel Borkmann #include <linux/audit.h> 27ccfe29ebSAlexei Starovoitov #include <uapi/linux/btf.h> 28ca5999fdSMike Rapoport #include <linux/pgtable.h> 299e4e01dfSKP Singh #include <linux/bpf_lsm.h> 30457f4436SAndrii Nakryiko #include <linux/poll.h> 31a3fd7ceeSJakub Sitnicki #include <linux/bpf-netns.h> 321e6c62a8SAlexei Starovoitov #include <linux/rcupdate_trace.h> 3399c55f7dSAlexei Starovoitov 34da765a2fSDaniel Borkmann #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 3514dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 3614dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 37da765a2fSDaniel Borkmann #define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY) 3814dc6f04SMartin KaFai Lau #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 39da765a2fSDaniel Borkmann #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \ 40da765a2fSDaniel Borkmann IS_FD_HASH(map)) 4114dc6f04SMartin KaFai Lau 426e71b04aSChenbo Feng #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) 436e71b04aSChenbo Feng 44b121d1e7SAlexei Starovoitov DEFINE_PER_CPU(int, bpf_prog_active); 45dc4bb0e2SMartin KaFai Lau static DEFINE_IDR(prog_idr); 46dc4bb0e2SMartin KaFai Lau static DEFINE_SPINLOCK(prog_idr_lock); 47f3f1c054SMartin KaFai Lau static DEFINE_IDR(map_idr); 48f3f1c054SMartin KaFai Lau static DEFINE_SPINLOCK(map_idr_lock); 49a3b80e10SAndrii Nakryiko static DEFINE_IDR(link_idr); 50a3b80e10SAndrii Nakryiko static DEFINE_SPINLOCK(link_idr_lock); 51b121d1e7SAlexei Starovoitov 521be7f75dSAlexei Starovoitov int sysctl_unprivileged_bpf_disabled __read_mostly; 531be7f75dSAlexei Starovoitov 5440077e0cSJohannes Berg static const struct bpf_map_ops * const bpf_map_types[] = { 5591cc1a99SAlexei Starovoitov #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) 5640077e0cSJohannes Berg #define BPF_MAP_TYPE(_id, _ops) \ 5740077e0cSJohannes Berg [_id] = &_ops, 58f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) 5940077e0cSJohannes Berg #include <linux/bpf_types.h> 6040077e0cSJohannes Berg #undef BPF_PROG_TYPE 6140077e0cSJohannes Berg #undef BPF_MAP_TYPE 62f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 6340077e0cSJohannes Berg }; 6499c55f7dSAlexei Starovoitov 65752ba56fSMickaël Salaün /* 66752ba56fSMickaël Salaün * If we're handed a bigger struct than we know of, ensure all the unknown bits 67752ba56fSMickaël Salaün * are 0 - i.e. new user-space does not rely on any kernel feature extensions 68752ba56fSMickaël Salaün * we don't know about yet. 69752ba56fSMickaël Salaün * 70752ba56fSMickaël Salaün * There is a ToCToU between this function call and the following 71752ba56fSMickaël Salaün * copy_from_user() call. However, this is not a concern since this function is 72752ba56fSMickaël Salaün * meant to be a future-proofing of bits. 73752ba56fSMickaël Salaün */ 74dcab51f1SMartin KaFai Lau int bpf_check_uarg_tail_zero(void __user *uaddr, 7558291a74SMickaël Salaün size_t expected_size, 7658291a74SMickaël Salaün size_t actual_size) 7758291a74SMickaël Salaün { 78b7e4b65fSAl Viro unsigned char __user *addr = uaddr + expected_size; 79b7e4b65fSAl Viro int res; 8058291a74SMickaël Salaün 81752ba56fSMickaël Salaün if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 82752ba56fSMickaël Salaün return -E2BIG; 83752ba56fSMickaël Salaün 8458291a74SMickaël Salaün if (actual_size <= expected_size) 8558291a74SMickaël Salaün return 0; 8658291a74SMickaël Salaün 87b7e4b65fSAl Viro res = check_zeroed_user(addr, actual_size - expected_size); 88b7e4b65fSAl Viro if (res < 0) 89b7e4b65fSAl Viro return res; 90b7e4b65fSAl Viro return res ? 0 : -E2BIG; 9158291a74SMickaël Salaün } 9258291a74SMickaël Salaün 93a3884572SJakub Kicinski const struct bpf_map_ops bpf_map_offload_ops = { 94f4d05259SMartin KaFai Lau .map_meta_equal = bpf_map_meta_equal, 95a3884572SJakub Kicinski .map_alloc = bpf_map_offload_map_alloc, 96a3884572SJakub Kicinski .map_free = bpf_map_offload_map_free, 97e8d2bec0SDaniel Borkmann .map_check_btf = map_check_no_btf, 98a3884572SJakub Kicinski }; 99a3884572SJakub Kicinski 10099c55f7dSAlexei Starovoitov static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 10199c55f7dSAlexei Starovoitov { 1021110f3a9SJakub Kicinski const struct bpf_map_ops *ops; 1039ef09e35SMark Rutland u32 type = attr->map_type; 10499c55f7dSAlexei Starovoitov struct bpf_map *map; 1051110f3a9SJakub Kicinski int err; 10699c55f7dSAlexei Starovoitov 1079ef09e35SMark Rutland if (type >= ARRAY_SIZE(bpf_map_types)) 1081110f3a9SJakub Kicinski return ERR_PTR(-EINVAL); 1099ef09e35SMark Rutland type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types)); 1109ef09e35SMark Rutland ops = bpf_map_types[type]; 1111110f3a9SJakub Kicinski if (!ops) 11240077e0cSJohannes Berg return ERR_PTR(-EINVAL); 11340077e0cSJohannes Berg 1141110f3a9SJakub Kicinski if (ops->map_alloc_check) { 1151110f3a9SJakub Kicinski err = ops->map_alloc_check(attr); 1161110f3a9SJakub Kicinski if (err) 1171110f3a9SJakub Kicinski return ERR_PTR(err); 1181110f3a9SJakub Kicinski } 119a3884572SJakub Kicinski if (attr->map_ifindex) 120a3884572SJakub Kicinski ops = &bpf_map_offload_ops; 1211110f3a9SJakub Kicinski map = ops->map_alloc(attr); 12299c55f7dSAlexei Starovoitov if (IS_ERR(map)) 12399c55f7dSAlexei Starovoitov return map; 1241110f3a9SJakub Kicinski map->ops = ops; 1259ef09e35SMark Rutland map->map_type = type; 12699c55f7dSAlexei Starovoitov return map; 12799c55f7dSAlexei Starovoitov } 12899c55f7dSAlexei Starovoitov 12915c14a3dSBrian Vazquez static u32 bpf_map_value_size(struct bpf_map *map) 13015c14a3dSBrian Vazquez { 13115c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 13215c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 13315c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 13415c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 13515c14a3dSBrian Vazquez return round_up(map->value_size, 8) * num_possible_cpus(); 13615c14a3dSBrian Vazquez else if (IS_FD_MAP(map)) 13715c14a3dSBrian Vazquez return sizeof(u32); 13815c14a3dSBrian Vazquez else 13915c14a3dSBrian Vazquez return map->value_size; 14015c14a3dSBrian Vazquez } 14115c14a3dSBrian Vazquez 14215c14a3dSBrian Vazquez static void maybe_wait_bpf_programs(struct bpf_map *map) 14315c14a3dSBrian Vazquez { 14415c14a3dSBrian Vazquez /* Wait for any running BPF programs to complete so that 14515c14a3dSBrian Vazquez * userspace, when we return to it, knows that all programs 14615c14a3dSBrian Vazquez * that could be running use the new map value. 14715c14a3dSBrian Vazquez */ 14815c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS || 14915c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 15015c14a3dSBrian Vazquez synchronize_rcu(); 15115c14a3dSBrian Vazquez } 15215c14a3dSBrian Vazquez 15315c14a3dSBrian Vazquez static int bpf_map_update_value(struct bpf_map *map, struct fd f, void *key, 15415c14a3dSBrian Vazquez void *value, __u64 flags) 15515c14a3dSBrian Vazquez { 15615c14a3dSBrian Vazquez int err; 15715c14a3dSBrian Vazquez 15815c14a3dSBrian Vazquez /* Need to create a kthread, thus must support schedule */ 15915c14a3dSBrian Vazquez if (bpf_map_is_dev_bound(map)) { 16015c14a3dSBrian Vazquez return bpf_map_offload_update_elem(map, key, value, flags); 16115c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_CPUMAP || 16215c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 16315c14a3dSBrian Vazquez return map->ops->map_update_elem(map, key, value, flags); 16413b79d3fSLorenz Bauer } else if (map->map_type == BPF_MAP_TYPE_SOCKHASH || 16513b79d3fSLorenz Bauer map->map_type == BPF_MAP_TYPE_SOCKMAP) { 16613b79d3fSLorenz Bauer return sock_map_update_elem_sys(map, key, value, flags); 16715c14a3dSBrian Vazquez } else if (IS_FD_PROG_ARRAY(map)) { 16815c14a3dSBrian Vazquez return bpf_fd_array_map_update_elem(map, f.file, key, value, 16915c14a3dSBrian Vazquez flags); 17015c14a3dSBrian Vazquez } 17115c14a3dSBrian Vazquez 172b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 17315c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 17415c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 17515c14a3dSBrian Vazquez err = bpf_percpu_hash_update(map, key, value, flags); 17615c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 17715c14a3dSBrian Vazquez err = bpf_percpu_array_update(map, key, value, flags); 17815c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 17915c14a3dSBrian Vazquez err = bpf_percpu_cgroup_storage_update(map, key, value, 18015c14a3dSBrian Vazquez flags); 18115c14a3dSBrian Vazquez } else if (IS_FD_ARRAY(map)) { 18215c14a3dSBrian Vazquez rcu_read_lock(); 18315c14a3dSBrian Vazquez err = bpf_fd_array_map_update_elem(map, f.file, key, value, 18415c14a3dSBrian Vazquez flags); 18515c14a3dSBrian Vazquez rcu_read_unlock(); 18615c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 18715c14a3dSBrian Vazquez rcu_read_lock(); 18815c14a3dSBrian Vazquez err = bpf_fd_htab_map_update_elem(map, f.file, key, value, 18915c14a3dSBrian Vazquez flags); 19015c14a3dSBrian Vazquez rcu_read_unlock(); 19115c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 19215c14a3dSBrian Vazquez /* rcu_read_lock() is not needed */ 19315c14a3dSBrian Vazquez err = bpf_fd_reuseport_array_update_elem(map, key, value, 19415c14a3dSBrian Vazquez flags); 19515c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 19615c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_STACK) { 19715c14a3dSBrian Vazquez err = map->ops->map_push_elem(map, value, flags); 19815c14a3dSBrian Vazquez } else { 19915c14a3dSBrian Vazquez rcu_read_lock(); 20015c14a3dSBrian Vazquez err = map->ops->map_update_elem(map, key, value, flags); 20115c14a3dSBrian Vazquez rcu_read_unlock(); 20215c14a3dSBrian Vazquez } 203b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 20415c14a3dSBrian Vazquez maybe_wait_bpf_programs(map); 20515c14a3dSBrian Vazquez 20615c14a3dSBrian Vazquez return err; 20715c14a3dSBrian Vazquez } 20815c14a3dSBrian Vazquez 20915c14a3dSBrian Vazquez static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value, 21015c14a3dSBrian Vazquez __u64 flags) 21115c14a3dSBrian Vazquez { 21215c14a3dSBrian Vazquez void *ptr; 21315c14a3dSBrian Vazquez int err; 21415c14a3dSBrian Vazquez 215cb4d03abSBrian Vazquez if (bpf_map_is_dev_bound(map)) 216cb4d03abSBrian Vazquez return bpf_map_offload_lookup_elem(map, key, value); 21715c14a3dSBrian Vazquez 218b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 21915c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 22015c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 22115c14a3dSBrian Vazquez err = bpf_percpu_hash_copy(map, key, value); 22215c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 22315c14a3dSBrian Vazquez err = bpf_percpu_array_copy(map, key, value); 22415c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 22515c14a3dSBrian Vazquez err = bpf_percpu_cgroup_storage_copy(map, key, value); 22615c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 22715c14a3dSBrian Vazquez err = bpf_stackmap_copy(map, key, value); 22815c14a3dSBrian Vazquez } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) { 22915c14a3dSBrian Vazquez err = bpf_fd_array_map_lookup_elem(map, key, value); 23015c14a3dSBrian Vazquez } else if (IS_FD_HASH(map)) { 23115c14a3dSBrian Vazquez err = bpf_fd_htab_map_lookup_elem(map, key, value); 23215c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 23315c14a3dSBrian Vazquez err = bpf_fd_reuseport_array_lookup_elem(map, key, value); 23415c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 23515c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_STACK) { 23615c14a3dSBrian Vazquez err = map->ops->map_peek_elem(map, value); 23715c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 23815c14a3dSBrian Vazquez /* struct_ops map requires directly updating "value" */ 23915c14a3dSBrian Vazquez err = bpf_struct_ops_map_sys_lookup_elem(map, key, value); 24015c14a3dSBrian Vazquez } else { 24115c14a3dSBrian Vazquez rcu_read_lock(); 24215c14a3dSBrian Vazquez if (map->ops->map_lookup_elem_sys_only) 24315c14a3dSBrian Vazquez ptr = map->ops->map_lookup_elem_sys_only(map, key); 24415c14a3dSBrian Vazquez else 24515c14a3dSBrian Vazquez ptr = map->ops->map_lookup_elem(map, key); 24615c14a3dSBrian Vazquez if (IS_ERR(ptr)) { 24715c14a3dSBrian Vazquez err = PTR_ERR(ptr); 24815c14a3dSBrian Vazquez } else if (!ptr) { 24915c14a3dSBrian Vazquez err = -ENOENT; 25015c14a3dSBrian Vazquez } else { 25115c14a3dSBrian Vazquez err = 0; 25215c14a3dSBrian Vazquez if (flags & BPF_F_LOCK) 25315c14a3dSBrian Vazquez /* lock 'ptr' and copy everything but lock */ 25415c14a3dSBrian Vazquez copy_map_value_locked(map, value, ptr, true); 25515c14a3dSBrian Vazquez else 25615c14a3dSBrian Vazquez copy_map_value(map, value, ptr); 25715c14a3dSBrian Vazquez /* mask lock, since value wasn't zero inited */ 25815c14a3dSBrian Vazquez check_and_init_map_lock(map, value); 25915c14a3dSBrian Vazquez } 26015c14a3dSBrian Vazquez rcu_read_unlock(); 26115c14a3dSBrian Vazquez } 26215c14a3dSBrian Vazquez 263b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 26415c14a3dSBrian Vazquez maybe_wait_bpf_programs(map); 26515c14a3dSBrian Vazquez 26615c14a3dSBrian Vazquez return err; 26715c14a3dSBrian Vazquez } 26815c14a3dSBrian Vazquez 269196e8ca7SDaniel Borkmann static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable) 270d407bd25SDaniel Borkmann { 271f01a7dbeSMartynas Pumputis /* We really just want to fail instead of triggering OOM killer 272f01a7dbeSMartynas Pumputis * under memory pressure, therefore we set __GFP_NORETRY to kmalloc, 273f01a7dbeSMartynas Pumputis * which is used for lower order allocation requests. 274f01a7dbeSMartynas Pumputis * 275f01a7dbeSMartynas Pumputis * It has been observed that higher order allocation requests done by 276f01a7dbeSMartynas Pumputis * vmalloc with __GFP_NORETRY being set might fail due to not trying 277f01a7dbeSMartynas Pumputis * to reclaim memory from the page cache, thus we set 278f01a7dbeSMartynas Pumputis * __GFP_RETRY_MAYFAIL to avoid such situations. 279d407bd25SDaniel Borkmann */ 280f01a7dbeSMartynas Pumputis 281041de93fSChristoph Hellwig const gfp_t gfp = __GFP_NOWARN | __GFP_ZERO; 282041de93fSChristoph Hellwig unsigned int flags = 0; 283041de93fSChristoph Hellwig unsigned long align = 1; 284d407bd25SDaniel Borkmann void *area; 285d407bd25SDaniel Borkmann 286196e8ca7SDaniel Borkmann if (size >= SIZE_MAX) 287196e8ca7SDaniel Borkmann return NULL; 288196e8ca7SDaniel Borkmann 289fc970227SAndrii Nakryiko /* kmalloc()'ed memory can't be mmap()'ed */ 290041de93fSChristoph Hellwig if (mmapable) { 291041de93fSChristoph Hellwig BUG_ON(!PAGE_ALIGNED(size)); 292041de93fSChristoph Hellwig align = SHMLBA; 293041de93fSChristoph Hellwig flags = VM_USERMAP; 294041de93fSChristoph Hellwig } else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 295041de93fSChristoph Hellwig area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY, 296f01a7dbeSMartynas Pumputis numa_node); 297d407bd25SDaniel Borkmann if (area != NULL) 298d407bd25SDaniel Borkmann return area; 299d407bd25SDaniel Borkmann } 300041de93fSChristoph Hellwig 301041de93fSChristoph Hellwig return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, 302041de93fSChristoph Hellwig gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL, 303041de93fSChristoph Hellwig flags, numa_node, __builtin_return_address(0)); 304d407bd25SDaniel Borkmann } 305d407bd25SDaniel Borkmann 306196e8ca7SDaniel Borkmann void *bpf_map_area_alloc(u64 size, int numa_node) 307fc970227SAndrii Nakryiko { 308fc970227SAndrii Nakryiko return __bpf_map_area_alloc(size, numa_node, false); 309fc970227SAndrii Nakryiko } 310fc970227SAndrii Nakryiko 311196e8ca7SDaniel Borkmann void *bpf_map_area_mmapable_alloc(u64 size, int numa_node) 312fc970227SAndrii Nakryiko { 313fc970227SAndrii Nakryiko return __bpf_map_area_alloc(size, numa_node, true); 314fc970227SAndrii Nakryiko } 315fc970227SAndrii Nakryiko 316d407bd25SDaniel Borkmann void bpf_map_area_free(void *area) 317d407bd25SDaniel Borkmann { 318d407bd25SDaniel Borkmann kvfree(area); 319d407bd25SDaniel Borkmann } 320d407bd25SDaniel Borkmann 321be70bcd5SDaniel Borkmann static u32 bpf_map_flags_retain_permanent(u32 flags) 322be70bcd5SDaniel Borkmann { 323be70bcd5SDaniel Borkmann /* Some map creation flags are not tied to the map object but 324be70bcd5SDaniel Borkmann * rather to the map fd instead, so they have no meaning upon 325be70bcd5SDaniel Borkmann * map object inspection since multiple file descriptors with 326be70bcd5SDaniel Borkmann * different (access) properties can exist here. Thus, given 327be70bcd5SDaniel Borkmann * this has zero meaning for the map itself, lets clear these 328be70bcd5SDaniel Borkmann * from here. 329be70bcd5SDaniel Borkmann */ 330be70bcd5SDaniel Borkmann return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY); 331be70bcd5SDaniel Borkmann } 332be70bcd5SDaniel Borkmann 333bd475643SJakub Kicinski void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) 334bd475643SJakub Kicinski { 335bd475643SJakub Kicinski map->map_type = attr->map_type; 336bd475643SJakub Kicinski map->key_size = attr->key_size; 337bd475643SJakub Kicinski map->value_size = attr->value_size; 338bd475643SJakub Kicinski map->max_entries = attr->max_entries; 339be70bcd5SDaniel Borkmann map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags); 340bd475643SJakub Kicinski map->numa_node = bpf_map_attr_numa_node(attr); 341bd475643SJakub Kicinski } 342bd475643SJakub Kicinski 3430a4c58f5SRoman Gushchin static int bpf_charge_memlock(struct user_struct *user, u32 pages) 344aaac3ba9SAlexei Starovoitov { 3450a4c58f5SRoman Gushchin unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 346aaac3ba9SAlexei Starovoitov 3470a4c58f5SRoman Gushchin if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) { 3480a4c58f5SRoman Gushchin atomic_long_sub(pages, &user->locked_vm); 349aaac3ba9SAlexei Starovoitov return -EPERM; 350aaac3ba9SAlexei Starovoitov } 351aaac3ba9SAlexei Starovoitov return 0; 352aaac3ba9SAlexei Starovoitov } 353aaac3ba9SAlexei Starovoitov 3540a4c58f5SRoman Gushchin static void bpf_uncharge_memlock(struct user_struct *user, u32 pages) 3550a4c58f5SRoman Gushchin { 356b936ca64SRoman Gushchin if (user) 3570a4c58f5SRoman Gushchin atomic_long_sub(pages, &user->locked_vm); 3580a4c58f5SRoman Gushchin } 3590a4c58f5SRoman Gushchin 360196e8ca7SDaniel Borkmann int bpf_map_charge_init(struct bpf_map_memory *mem, u64 size) 3610a4c58f5SRoman Gushchin { 362c85d6913SRoman Gushchin u32 pages = round_up(size, PAGE_SIZE) >> PAGE_SHIFT; 363c85d6913SRoman Gushchin struct user_struct *user; 3640a4c58f5SRoman Gushchin int ret; 3650a4c58f5SRoman Gushchin 366c85d6913SRoman Gushchin if (size >= U32_MAX - PAGE_SIZE) 367c85d6913SRoman Gushchin return -E2BIG; 368c85d6913SRoman Gushchin 369c85d6913SRoman Gushchin user = get_current_user(); 370b936ca64SRoman Gushchin ret = bpf_charge_memlock(user, pages); 3710a4c58f5SRoman Gushchin if (ret) { 3720a4c58f5SRoman Gushchin free_uid(user); 3730a4c58f5SRoman Gushchin return ret; 3740a4c58f5SRoman Gushchin } 375b936ca64SRoman Gushchin 376b936ca64SRoman Gushchin mem->pages = pages; 377b936ca64SRoman Gushchin mem->user = user; 378b936ca64SRoman Gushchin 379b936ca64SRoman Gushchin return 0; 3800a4c58f5SRoman Gushchin } 3810a4c58f5SRoman Gushchin 382b936ca64SRoman Gushchin void bpf_map_charge_finish(struct bpf_map_memory *mem) 383aaac3ba9SAlexei Starovoitov { 384b936ca64SRoman Gushchin bpf_uncharge_memlock(mem->user, mem->pages); 385b936ca64SRoman Gushchin free_uid(mem->user); 386b936ca64SRoman Gushchin } 3873539b96eSRoman Gushchin 388b936ca64SRoman Gushchin void bpf_map_charge_move(struct bpf_map_memory *dst, 389b936ca64SRoman Gushchin struct bpf_map_memory *src) 390b936ca64SRoman Gushchin { 391b936ca64SRoman Gushchin *dst = *src; 392b936ca64SRoman Gushchin 393b936ca64SRoman Gushchin /* Make sure src will not be used for the redundant uncharging. */ 394b936ca64SRoman Gushchin memset(src, 0, sizeof(struct bpf_map_memory)); 395aaac3ba9SAlexei Starovoitov } 396aaac3ba9SAlexei Starovoitov 3970a4c58f5SRoman Gushchin int bpf_map_charge_memlock(struct bpf_map *map, u32 pages) 3980a4c58f5SRoman Gushchin { 3990a4c58f5SRoman Gushchin int ret; 4000a4c58f5SRoman Gushchin 4013539b96eSRoman Gushchin ret = bpf_charge_memlock(map->memory.user, pages); 4020a4c58f5SRoman Gushchin if (ret) 4030a4c58f5SRoman Gushchin return ret; 4043539b96eSRoman Gushchin map->memory.pages += pages; 4050a4c58f5SRoman Gushchin return ret; 4060a4c58f5SRoman Gushchin } 4070a4c58f5SRoman Gushchin 4080a4c58f5SRoman Gushchin void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages) 4090a4c58f5SRoman Gushchin { 4103539b96eSRoman Gushchin bpf_uncharge_memlock(map->memory.user, pages); 4113539b96eSRoman Gushchin map->memory.pages -= pages; 4120a4c58f5SRoman Gushchin } 4130a4c58f5SRoman Gushchin 414f3f1c054SMartin KaFai Lau static int bpf_map_alloc_id(struct bpf_map *map) 415f3f1c054SMartin KaFai Lau { 416f3f1c054SMartin KaFai Lau int id; 417f3f1c054SMartin KaFai Lau 418b76354cdSShaohua Li idr_preload(GFP_KERNEL); 419f3f1c054SMartin KaFai Lau spin_lock_bh(&map_idr_lock); 420f3f1c054SMartin KaFai Lau id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 421f3f1c054SMartin KaFai Lau if (id > 0) 422f3f1c054SMartin KaFai Lau map->id = id; 423f3f1c054SMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 424b76354cdSShaohua Li idr_preload_end(); 425f3f1c054SMartin KaFai Lau 426f3f1c054SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 427f3f1c054SMartin KaFai Lau return -ENOSPC; 428f3f1c054SMartin KaFai Lau 429f3f1c054SMartin KaFai Lau return id > 0 ? 0 : id; 430f3f1c054SMartin KaFai Lau } 431f3f1c054SMartin KaFai Lau 432a3884572SJakub Kicinski void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) 433f3f1c054SMartin KaFai Lau { 434930651a7SEric Dumazet unsigned long flags; 435930651a7SEric Dumazet 436a3884572SJakub Kicinski /* Offloaded maps are removed from the IDR store when their device 437a3884572SJakub Kicinski * disappears - even if someone holds an fd to them they are unusable, 438a3884572SJakub Kicinski * the memory is gone, all ops will fail; they are simply waiting for 439a3884572SJakub Kicinski * refcnt to drop to be freed. 440a3884572SJakub Kicinski */ 441a3884572SJakub Kicinski if (!map->id) 442a3884572SJakub Kicinski return; 443a3884572SJakub Kicinski 444bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 445930651a7SEric Dumazet spin_lock_irqsave(&map_idr_lock, flags); 446bd5f5f4eSMartin KaFai Lau else 447bd5f5f4eSMartin KaFai Lau __acquire(&map_idr_lock); 448bd5f5f4eSMartin KaFai Lau 449f3f1c054SMartin KaFai Lau idr_remove(&map_idr, map->id); 450a3884572SJakub Kicinski map->id = 0; 451bd5f5f4eSMartin KaFai Lau 452bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 453930651a7SEric Dumazet spin_unlock_irqrestore(&map_idr_lock, flags); 454bd5f5f4eSMartin KaFai Lau else 455bd5f5f4eSMartin KaFai Lau __release(&map_idr_lock); 456f3f1c054SMartin KaFai Lau } 457f3f1c054SMartin KaFai Lau 45899c55f7dSAlexei Starovoitov /* called from workqueue */ 45999c55f7dSAlexei Starovoitov static void bpf_map_free_deferred(struct work_struct *work) 46099c55f7dSAlexei Starovoitov { 46199c55f7dSAlexei Starovoitov struct bpf_map *map = container_of(work, struct bpf_map, work); 462b936ca64SRoman Gushchin struct bpf_map_memory mem; 46399c55f7dSAlexei Starovoitov 464b936ca64SRoman Gushchin bpf_map_charge_move(&mem, &map->memory); 465afdb09c7SChenbo Feng security_bpf_map_free(map); 46699c55f7dSAlexei Starovoitov /* implementation dependent freeing */ 46799c55f7dSAlexei Starovoitov map->ops->map_free(map); 468b936ca64SRoman Gushchin bpf_map_charge_finish(&mem); 46999c55f7dSAlexei Starovoitov } 47099c55f7dSAlexei Starovoitov 471c9da161cSDaniel Borkmann static void bpf_map_put_uref(struct bpf_map *map) 472c9da161cSDaniel Borkmann { 4731e0bd5a0SAndrii Nakryiko if (atomic64_dec_and_test(&map->usercnt)) { 474ba6b8de4SJohn Fastabend if (map->ops->map_release_uref) 475ba6b8de4SJohn Fastabend map->ops->map_release_uref(map); 476c9da161cSDaniel Borkmann } 477c9da161cSDaniel Borkmann } 478c9da161cSDaniel Borkmann 47999c55f7dSAlexei Starovoitov /* decrement map refcnt and schedule it for freeing via workqueue 48099c55f7dSAlexei Starovoitov * (unrelying map implementation ops->map_free() might sleep) 48199c55f7dSAlexei Starovoitov */ 482bd5f5f4eSMartin KaFai Lau static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) 48399c55f7dSAlexei Starovoitov { 4841e0bd5a0SAndrii Nakryiko if (atomic64_dec_and_test(&map->refcnt)) { 48534ad5580SMartin KaFai Lau /* bpf_map_free_id() must be called first */ 486bd5f5f4eSMartin KaFai Lau bpf_map_free_id(map, do_idr_lock); 48778958fcaSMartin KaFai Lau btf_put(map->btf); 48899c55f7dSAlexei Starovoitov INIT_WORK(&map->work, bpf_map_free_deferred); 48999c55f7dSAlexei Starovoitov schedule_work(&map->work); 49099c55f7dSAlexei Starovoitov } 49199c55f7dSAlexei Starovoitov } 49299c55f7dSAlexei Starovoitov 493bd5f5f4eSMartin KaFai Lau void bpf_map_put(struct bpf_map *map) 494bd5f5f4eSMartin KaFai Lau { 495bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, true); 496bd5f5f4eSMartin KaFai Lau } 497630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_put); 498bd5f5f4eSMartin KaFai Lau 499c9da161cSDaniel Borkmann void bpf_map_put_with_uref(struct bpf_map *map) 500c9da161cSDaniel Borkmann { 501c9da161cSDaniel Borkmann bpf_map_put_uref(map); 502c9da161cSDaniel Borkmann bpf_map_put(map); 503c9da161cSDaniel Borkmann } 504c9da161cSDaniel Borkmann 50599c55f7dSAlexei Starovoitov static int bpf_map_release(struct inode *inode, struct file *filp) 50699c55f7dSAlexei Starovoitov { 50761d1b6a4SDaniel Borkmann struct bpf_map *map = filp->private_data; 50861d1b6a4SDaniel Borkmann 50961d1b6a4SDaniel Borkmann if (map->ops->map_release) 51061d1b6a4SDaniel Borkmann map->ops->map_release(map, filp); 51161d1b6a4SDaniel Borkmann 51261d1b6a4SDaniel Borkmann bpf_map_put_with_uref(map); 51399c55f7dSAlexei Starovoitov return 0; 51499c55f7dSAlexei Starovoitov } 51599c55f7dSAlexei Starovoitov 51687df15deSDaniel Borkmann static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f) 51787df15deSDaniel Borkmann { 51887df15deSDaniel Borkmann fmode_t mode = f.file->f_mode; 51987df15deSDaniel Borkmann 52087df15deSDaniel Borkmann /* Our file permissions may have been overridden by global 52187df15deSDaniel Borkmann * map permissions facing syscall side. 52287df15deSDaniel Borkmann */ 52387df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) 52487df15deSDaniel Borkmann mode &= ~FMODE_CAN_WRITE; 52587df15deSDaniel Borkmann return mode; 52687df15deSDaniel Borkmann } 52787df15deSDaniel Borkmann 528f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 529f99bf205SDaniel Borkmann static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 530f99bf205SDaniel Borkmann { 531f99bf205SDaniel Borkmann const struct bpf_map *map = filp->private_data; 53221116b70SDaniel Borkmann const struct bpf_array *array; 5332beee5f5SDaniel Borkmann u32 type = 0, jited = 0; 53421116b70SDaniel Borkmann 53521116b70SDaniel Borkmann if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 53621116b70SDaniel Borkmann array = container_of(map, struct bpf_array, map); 5372beee5f5SDaniel Borkmann type = array->aux->type; 5382beee5f5SDaniel Borkmann jited = array->aux->jited; 53921116b70SDaniel Borkmann } 540f99bf205SDaniel Borkmann 541f99bf205SDaniel Borkmann seq_printf(m, 542f99bf205SDaniel Borkmann "map_type:\t%u\n" 543f99bf205SDaniel Borkmann "key_size:\t%u\n" 544f99bf205SDaniel Borkmann "value_size:\t%u\n" 545322cea2fSDaniel Borkmann "max_entries:\t%u\n" 54621116b70SDaniel Borkmann "map_flags:\t%#x\n" 5474316b409SDaniel Borkmann "memlock:\t%llu\n" 54887df15deSDaniel Borkmann "map_id:\t%u\n" 54987df15deSDaniel Borkmann "frozen:\t%u\n", 550f99bf205SDaniel Borkmann map->map_type, 551f99bf205SDaniel Borkmann map->key_size, 552f99bf205SDaniel Borkmann map->value_size, 553322cea2fSDaniel Borkmann map->max_entries, 55421116b70SDaniel Borkmann map->map_flags, 5553539b96eSRoman Gushchin map->memory.pages * 1ULL << PAGE_SHIFT, 55687df15deSDaniel Borkmann map->id, 55787df15deSDaniel Borkmann READ_ONCE(map->frozen)); 5582beee5f5SDaniel Borkmann if (type) { 5592beee5f5SDaniel Borkmann seq_printf(m, "owner_prog_type:\t%u\n", type); 5602beee5f5SDaniel Borkmann seq_printf(m, "owner_jited:\t%u\n", jited); 5619780c0abSDaniel Borkmann } 562f99bf205SDaniel Borkmann } 563f99bf205SDaniel Borkmann #endif 564f99bf205SDaniel Borkmann 5656e71b04aSChenbo Feng static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, 5666e71b04aSChenbo Feng loff_t *ppos) 5676e71b04aSChenbo Feng { 5686e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 5696e71b04aSChenbo Feng * f_mode with FMODE_CAN_READ. 5706e71b04aSChenbo Feng */ 5716e71b04aSChenbo Feng return -EINVAL; 5726e71b04aSChenbo Feng } 5736e71b04aSChenbo Feng 5746e71b04aSChenbo Feng static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, 5756e71b04aSChenbo Feng size_t siz, loff_t *ppos) 5766e71b04aSChenbo Feng { 5776e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 5786e71b04aSChenbo Feng * f_mode with FMODE_CAN_WRITE. 5796e71b04aSChenbo Feng */ 5806e71b04aSChenbo Feng return -EINVAL; 5816e71b04aSChenbo Feng } 5826e71b04aSChenbo Feng 583fc970227SAndrii Nakryiko /* called for any extra memory-mapped regions (except initial) */ 584fc970227SAndrii Nakryiko static void bpf_map_mmap_open(struct vm_area_struct *vma) 585fc970227SAndrii Nakryiko { 586fc970227SAndrii Nakryiko struct bpf_map *map = vma->vm_file->private_data; 587fc970227SAndrii Nakryiko 5881f6cb19bSAndrii Nakryiko if (vma->vm_flags & VM_MAYWRITE) { 589fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 590fc970227SAndrii Nakryiko map->writecnt++; 591fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 592fc970227SAndrii Nakryiko } 593fc970227SAndrii Nakryiko } 594fc970227SAndrii Nakryiko 595fc970227SAndrii Nakryiko /* called for all unmapped memory region (including initial) */ 596fc970227SAndrii Nakryiko static void bpf_map_mmap_close(struct vm_area_struct *vma) 597fc970227SAndrii Nakryiko { 598fc970227SAndrii Nakryiko struct bpf_map *map = vma->vm_file->private_data; 599fc970227SAndrii Nakryiko 6001f6cb19bSAndrii Nakryiko if (vma->vm_flags & VM_MAYWRITE) { 601fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 602fc970227SAndrii Nakryiko map->writecnt--; 603fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 604fc970227SAndrii Nakryiko } 605fc970227SAndrii Nakryiko } 606fc970227SAndrii Nakryiko 607fc970227SAndrii Nakryiko static const struct vm_operations_struct bpf_map_default_vmops = { 608fc970227SAndrii Nakryiko .open = bpf_map_mmap_open, 609fc970227SAndrii Nakryiko .close = bpf_map_mmap_close, 610fc970227SAndrii Nakryiko }; 611fc970227SAndrii Nakryiko 612fc970227SAndrii Nakryiko static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma) 613fc970227SAndrii Nakryiko { 614fc970227SAndrii Nakryiko struct bpf_map *map = filp->private_data; 615fc970227SAndrii Nakryiko int err; 616fc970227SAndrii Nakryiko 617fc970227SAndrii Nakryiko if (!map->ops->map_mmap || map_value_has_spin_lock(map)) 618fc970227SAndrii Nakryiko return -ENOTSUPP; 619fc970227SAndrii Nakryiko 620fc970227SAndrii Nakryiko if (!(vma->vm_flags & VM_SHARED)) 621fc970227SAndrii Nakryiko return -EINVAL; 622fc970227SAndrii Nakryiko 623fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 624fc970227SAndrii Nakryiko 625dfeb376dSAndrii Nakryiko if (vma->vm_flags & VM_WRITE) { 626dfeb376dSAndrii Nakryiko if (map->frozen) { 627fc970227SAndrii Nakryiko err = -EPERM; 628fc970227SAndrii Nakryiko goto out; 629fc970227SAndrii Nakryiko } 630dfeb376dSAndrii Nakryiko /* map is meant to be read-only, so do not allow mapping as 631dfeb376dSAndrii Nakryiko * writable, because it's possible to leak a writable page 632dfeb376dSAndrii Nakryiko * reference and allows user-space to still modify it after 633dfeb376dSAndrii Nakryiko * freezing, while verifier will assume contents do not change 634dfeb376dSAndrii Nakryiko */ 635dfeb376dSAndrii Nakryiko if (map->map_flags & BPF_F_RDONLY_PROG) { 636dfeb376dSAndrii Nakryiko err = -EACCES; 637dfeb376dSAndrii Nakryiko goto out; 638dfeb376dSAndrii Nakryiko } 639dfeb376dSAndrii Nakryiko } 640fc970227SAndrii Nakryiko 641fc970227SAndrii Nakryiko /* set default open/close callbacks */ 642fc970227SAndrii Nakryiko vma->vm_ops = &bpf_map_default_vmops; 643fc970227SAndrii Nakryiko vma->vm_private_data = map; 6441f6cb19bSAndrii Nakryiko vma->vm_flags &= ~VM_MAYEXEC; 6451f6cb19bSAndrii Nakryiko if (!(vma->vm_flags & VM_WRITE)) 6461f6cb19bSAndrii Nakryiko /* disallow re-mapping with PROT_WRITE */ 6471f6cb19bSAndrii Nakryiko vma->vm_flags &= ~VM_MAYWRITE; 648fc970227SAndrii Nakryiko 649fc970227SAndrii Nakryiko err = map->ops->map_mmap(map, vma); 650fc970227SAndrii Nakryiko if (err) 651fc970227SAndrii Nakryiko goto out; 652fc970227SAndrii Nakryiko 6531f6cb19bSAndrii Nakryiko if (vma->vm_flags & VM_MAYWRITE) 654fc970227SAndrii Nakryiko map->writecnt++; 655fc970227SAndrii Nakryiko out: 656fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 657fc970227SAndrii Nakryiko return err; 658fc970227SAndrii Nakryiko } 659fc970227SAndrii Nakryiko 660457f4436SAndrii Nakryiko static __poll_t bpf_map_poll(struct file *filp, struct poll_table_struct *pts) 661457f4436SAndrii Nakryiko { 662457f4436SAndrii Nakryiko struct bpf_map *map = filp->private_data; 663457f4436SAndrii Nakryiko 664457f4436SAndrii Nakryiko if (map->ops->map_poll) 665457f4436SAndrii Nakryiko return map->ops->map_poll(map, filp, pts); 666457f4436SAndrii Nakryiko 667457f4436SAndrii Nakryiko return EPOLLERR; 668457f4436SAndrii Nakryiko } 669457f4436SAndrii Nakryiko 670f66e448cSChenbo Feng const struct file_operations bpf_map_fops = { 671f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 672f99bf205SDaniel Borkmann .show_fdinfo = bpf_map_show_fdinfo, 673f99bf205SDaniel Borkmann #endif 67499c55f7dSAlexei Starovoitov .release = bpf_map_release, 6756e71b04aSChenbo Feng .read = bpf_dummy_read, 6766e71b04aSChenbo Feng .write = bpf_dummy_write, 677fc970227SAndrii Nakryiko .mmap = bpf_map_mmap, 678457f4436SAndrii Nakryiko .poll = bpf_map_poll, 67999c55f7dSAlexei Starovoitov }; 68099c55f7dSAlexei Starovoitov 6816e71b04aSChenbo Feng int bpf_map_new_fd(struct bpf_map *map, int flags) 682aa79781bSDaniel Borkmann { 683afdb09c7SChenbo Feng int ret; 684afdb09c7SChenbo Feng 685afdb09c7SChenbo Feng ret = security_bpf_map(map, OPEN_FMODE(flags)); 686afdb09c7SChenbo Feng if (ret < 0) 687afdb09c7SChenbo Feng return ret; 688afdb09c7SChenbo Feng 689aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 6906e71b04aSChenbo Feng flags | O_CLOEXEC); 6916e71b04aSChenbo Feng } 6926e71b04aSChenbo Feng 6936e71b04aSChenbo Feng int bpf_get_file_flag(int flags) 6946e71b04aSChenbo Feng { 6956e71b04aSChenbo Feng if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) 6966e71b04aSChenbo Feng return -EINVAL; 6976e71b04aSChenbo Feng if (flags & BPF_F_RDONLY) 6986e71b04aSChenbo Feng return O_RDONLY; 6996e71b04aSChenbo Feng if (flags & BPF_F_WRONLY) 7006e71b04aSChenbo Feng return O_WRONLY; 7016e71b04aSChenbo Feng return O_RDWR; 702aa79781bSDaniel Borkmann } 703aa79781bSDaniel Borkmann 70499c55f7dSAlexei Starovoitov /* helper macro to check that unused fields 'union bpf_attr' are zero */ 70599c55f7dSAlexei Starovoitov #define CHECK_ATTR(CMD) \ 70699c55f7dSAlexei Starovoitov memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 70799c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD), 0, \ 70899c55f7dSAlexei Starovoitov sizeof(*attr) - \ 70999c55f7dSAlexei Starovoitov offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 71099c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD)) != NULL 71199c55f7dSAlexei Starovoitov 7128e7ae251SMartin KaFai Lau /* dst and src must have at least "size" number of bytes. 7138e7ae251SMartin KaFai Lau * Return strlen on success and < 0 on error. 714cb4d2b3fSMartin KaFai Lau */ 7158e7ae251SMartin KaFai Lau int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size) 716cb4d2b3fSMartin KaFai Lau { 7178e7ae251SMartin KaFai Lau const char *end = src + size; 7188e7ae251SMartin KaFai Lau const char *orig_src = src; 719cb4d2b3fSMartin KaFai Lau 7208e7ae251SMartin KaFai Lau memset(dst, 0, size); 7213e0ddc4fSDaniel Borkmann /* Copy all isalnum(), '_' and '.' chars. */ 722cb4d2b3fSMartin KaFai Lau while (src < end && *src) { 7233e0ddc4fSDaniel Borkmann if (!isalnum(*src) && 7243e0ddc4fSDaniel Borkmann *src != '_' && *src != '.') 725cb4d2b3fSMartin KaFai Lau return -EINVAL; 726cb4d2b3fSMartin KaFai Lau *dst++ = *src++; 727cb4d2b3fSMartin KaFai Lau } 728cb4d2b3fSMartin KaFai Lau 7298e7ae251SMartin KaFai Lau /* No '\0' found in "size" number of bytes */ 730cb4d2b3fSMartin KaFai Lau if (src == end) 731cb4d2b3fSMartin KaFai Lau return -EINVAL; 732cb4d2b3fSMartin KaFai Lau 7338e7ae251SMartin KaFai Lau return src - orig_src; 734cb4d2b3fSMartin KaFai Lau } 735cb4d2b3fSMartin KaFai Lau 736e8d2bec0SDaniel Borkmann int map_check_no_btf(const struct bpf_map *map, 7371b2b234bSRoman Gushchin const struct btf *btf, 738e8d2bec0SDaniel Borkmann const struct btf_type *key_type, 739e8d2bec0SDaniel Borkmann const struct btf_type *value_type) 740e8d2bec0SDaniel Borkmann { 741e8d2bec0SDaniel Borkmann return -ENOTSUPP; 742e8d2bec0SDaniel Borkmann } 743e8d2bec0SDaniel Borkmann 744d83525caSAlexei Starovoitov static int map_check_btf(struct bpf_map *map, const struct btf *btf, 745e8d2bec0SDaniel Borkmann u32 btf_key_id, u32 btf_value_id) 746e8d2bec0SDaniel Borkmann { 747e8d2bec0SDaniel Borkmann const struct btf_type *key_type, *value_type; 748e8d2bec0SDaniel Borkmann u32 key_size, value_size; 749e8d2bec0SDaniel Borkmann int ret = 0; 750e8d2bec0SDaniel Borkmann 7512824ecb7SDaniel Borkmann /* Some maps allow key to be unspecified. */ 7522824ecb7SDaniel Borkmann if (btf_key_id) { 753e8d2bec0SDaniel Borkmann key_type = btf_type_id_size(btf, &btf_key_id, &key_size); 754e8d2bec0SDaniel Borkmann if (!key_type || key_size != map->key_size) 755e8d2bec0SDaniel Borkmann return -EINVAL; 7562824ecb7SDaniel Borkmann } else { 7572824ecb7SDaniel Borkmann key_type = btf_type_by_id(btf, 0); 7582824ecb7SDaniel Borkmann if (!map->ops->map_check_btf) 7592824ecb7SDaniel Borkmann return -EINVAL; 7602824ecb7SDaniel Borkmann } 761e8d2bec0SDaniel Borkmann 762e8d2bec0SDaniel Borkmann value_type = btf_type_id_size(btf, &btf_value_id, &value_size); 763e8d2bec0SDaniel Borkmann if (!value_type || value_size != map->value_size) 764e8d2bec0SDaniel Borkmann return -EINVAL; 765e8d2bec0SDaniel Borkmann 766d83525caSAlexei Starovoitov map->spin_lock_off = btf_find_spin_lock(btf, value_type); 767d83525caSAlexei Starovoitov 768d83525caSAlexei Starovoitov if (map_value_has_spin_lock(map)) { 769591fe988SDaniel Borkmann if (map->map_flags & BPF_F_RDONLY_PROG) 770591fe988SDaniel Borkmann return -EACCES; 771d83525caSAlexei Starovoitov if (map->map_type != BPF_MAP_TYPE_HASH && 772e16d2f1aSAlexei Starovoitov map->map_type != BPF_MAP_TYPE_ARRAY && 7736ac99e8fSMartin KaFai Lau map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 7748ea63684SKP Singh map->map_type != BPF_MAP_TYPE_SK_STORAGE && 7758ea63684SKP Singh map->map_type != BPF_MAP_TYPE_INODE_STORAGE) 776d83525caSAlexei Starovoitov return -ENOTSUPP; 777d83525caSAlexei Starovoitov if (map->spin_lock_off + sizeof(struct bpf_spin_lock) > 778d83525caSAlexei Starovoitov map->value_size) { 779d83525caSAlexei Starovoitov WARN_ONCE(1, 780d83525caSAlexei Starovoitov "verifier bug spin_lock_off %d value_size %d\n", 781d83525caSAlexei Starovoitov map->spin_lock_off, map->value_size); 782d83525caSAlexei Starovoitov return -EFAULT; 783d83525caSAlexei Starovoitov } 784d83525caSAlexei Starovoitov } 785d83525caSAlexei Starovoitov 786e8d2bec0SDaniel Borkmann if (map->ops->map_check_btf) 7871b2b234bSRoman Gushchin ret = map->ops->map_check_btf(map, btf, key_type, value_type); 788e8d2bec0SDaniel Borkmann 789e8d2bec0SDaniel Borkmann return ret; 790e8d2bec0SDaniel Borkmann } 791e8d2bec0SDaniel Borkmann 79285d33df3SMartin KaFai Lau #define BPF_MAP_CREATE_LAST_FIELD btf_vmlinux_value_type_id 79399c55f7dSAlexei Starovoitov /* called via syscall */ 79499c55f7dSAlexei Starovoitov static int map_create(union bpf_attr *attr) 79599c55f7dSAlexei Starovoitov { 79696eabe7aSMartin KaFai Lau int numa_node = bpf_map_attr_numa_node(attr); 797b936ca64SRoman Gushchin struct bpf_map_memory mem; 79899c55f7dSAlexei Starovoitov struct bpf_map *map; 7996e71b04aSChenbo Feng int f_flags; 80099c55f7dSAlexei Starovoitov int err; 80199c55f7dSAlexei Starovoitov 80299c55f7dSAlexei Starovoitov err = CHECK_ATTR(BPF_MAP_CREATE); 80399c55f7dSAlexei Starovoitov if (err) 80499c55f7dSAlexei Starovoitov return -EINVAL; 80599c55f7dSAlexei Starovoitov 80685d33df3SMartin KaFai Lau if (attr->btf_vmlinux_value_type_id) { 80785d33df3SMartin KaFai Lau if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS || 80885d33df3SMartin KaFai Lau attr->btf_key_type_id || attr->btf_value_type_id) 80985d33df3SMartin KaFai Lau return -EINVAL; 81085d33df3SMartin KaFai Lau } else if (attr->btf_key_type_id && !attr->btf_value_type_id) { 81185d33df3SMartin KaFai Lau return -EINVAL; 81285d33df3SMartin KaFai Lau } 81385d33df3SMartin KaFai Lau 8146e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->map_flags); 8156e71b04aSChenbo Feng if (f_flags < 0) 8166e71b04aSChenbo Feng return f_flags; 8176e71b04aSChenbo Feng 81896eabe7aSMartin KaFai Lau if (numa_node != NUMA_NO_NODE && 81996e5ae4eSEric Dumazet ((unsigned int)numa_node >= nr_node_ids || 82096e5ae4eSEric Dumazet !node_online(numa_node))) 82196eabe7aSMartin KaFai Lau return -EINVAL; 82296eabe7aSMartin KaFai Lau 82399c55f7dSAlexei Starovoitov /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 82499c55f7dSAlexei Starovoitov map = find_and_alloc_map(attr); 82599c55f7dSAlexei Starovoitov if (IS_ERR(map)) 82699c55f7dSAlexei Starovoitov return PTR_ERR(map); 82799c55f7dSAlexei Starovoitov 8288e7ae251SMartin KaFai Lau err = bpf_obj_name_cpy(map->name, attr->map_name, 8298e7ae251SMartin KaFai Lau sizeof(attr->map_name)); 8308e7ae251SMartin KaFai Lau if (err < 0) 831b936ca64SRoman Gushchin goto free_map; 832ad5b177bSMartin KaFai Lau 8331e0bd5a0SAndrii Nakryiko atomic64_set(&map->refcnt, 1); 8341e0bd5a0SAndrii Nakryiko atomic64_set(&map->usercnt, 1); 835fc970227SAndrii Nakryiko mutex_init(&map->freeze_mutex); 83699c55f7dSAlexei Starovoitov 83785d33df3SMartin KaFai Lau map->spin_lock_off = -EINVAL; 83885d33df3SMartin KaFai Lau if (attr->btf_key_type_id || attr->btf_value_type_id || 83985d33df3SMartin KaFai Lau /* Even the map's value is a kernel's struct, 84085d33df3SMartin KaFai Lau * the bpf_prog.o must have BTF to begin with 84185d33df3SMartin KaFai Lau * to figure out the corresponding kernel's 84285d33df3SMartin KaFai Lau * counter part. Thus, attr->btf_fd has 84385d33df3SMartin KaFai Lau * to be valid also. 84485d33df3SMartin KaFai Lau */ 84585d33df3SMartin KaFai Lau attr->btf_vmlinux_value_type_id) { 846a26ca7c9SMartin KaFai Lau struct btf *btf; 847a26ca7c9SMartin KaFai Lau 848a26ca7c9SMartin KaFai Lau btf = btf_get_by_fd(attr->btf_fd); 849a26ca7c9SMartin KaFai Lau if (IS_ERR(btf)) { 850a26ca7c9SMartin KaFai Lau err = PTR_ERR(btf); 851b936ca64SRoman Gushchin goto free_map; 852a26ca7c9SMartin KaFai Lau } 85385d33df3SMartin KaFai Lau map->btf = btf; 854a26ca7c9SMartin KaFai Lau 85585d33df3SMartin KaFai Lau if (attr->btf_value_type_id) { 856e8d2bec0SDaniel Borkmann err = map_check_btf(map, btf, attr->btf_key_type_id, 8579b2cf328SMartin KaFai Lau attr->btf_value_type_id); 85885d33df3SMartin KaFai Lau if (err) 859b936ca64SRoman Gushchin goto free_map; 860a26ca7c9SMartin KaFai Lau } 861a26ca7c9SMartin KaFai Lau 8629b2cf328SMartin KaFai Lau map->btf_key_type_id = attr->btf_key_type_id; 8639b2cf328SMartin KaFai Lau map->btf_value_type_id = attr->btf_value_type_id; 86485d33df3SMartin KaFai Lau map->btf_vmlinux_value_type_id = 86585d33df3SMartin KaFai Lau attr->btf_vmlinux_value_type_id; 866a26ca7c9SMartin KaFai Lau } 867a26ca7c9SMartin KaFai Lau 868afdb09c7SChenbo Feng err = security_bpf_map_alloc(map); 869aaac3ba9SAlexei Starovoitov if (err) 870b936ca64SRoman Gushchin goto free_map; 871afdb09c7SChenbo Feng 872f3f1c054SMartin KaFai Lau err = bpf_map_alloc_id(map); 873f3f1c054SMartin KaFai Lau if (err) 874b936ca64SRoman Gushchin goto free_map_sec; 875f3f1c054SMartin KaFai Lau 8766e71b04aSChenbo Feng err = bpf_map_new_fd(map, f_flags); 877bd5f5f4eSMartin KaFai Lau if (err < 0) { 878bd5f5f4eSMartin KaFai Lau /* failed to allocate fd. 879352d20d6SPeng Sun * bpf_map_put_with_uref() is needed because the above 880bd5f5f4eSMartin KaFai Lau * bpf_map_alloc_id() has published the map 881bd5f5f4eSMartin KaFai Lau * to the userspace and the userspace may 882bd5f5f4eSMartin KaFai Lau * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 883bd5f5f4eSMartin KaFai Lau */ 884352d20d6SPeng Sun bpf_map_put_with_uref(map); 885bd5f5f4eSMartin KaFai Lau return err; 886bd5f5f4eSMartin KaFai Lau } 88799c55f7dSAlexei Starovoitov 88899c55f7dSAlexei Starovoitov return err; 88999c55f7dSAlexei Starovoitov 890afdb09c7SChenbo Feng free_map_sec: 891afdb09c7SChenbo Feng security_bpf_map_free(map); 892b936ca64SRoman Gushchin free_map: 893a26ca7c9SMartin KaFai Lau btf_put(map->btf); 894b936ca64SRoman Gushchin bpf_map_charge_move(&mem, &map->memory); 89599c55f7dSAlexei Starovoitov map->ops->map_free(map); 896b936ca64SRoman Gushchin bpf_map_charge_finish(&mem); 89799c55f7dSAlexei Starovoitov return err; 89899c55f7dSAlexei Starovoitov } 89999c55f7dSAlexei Starovoitov 900db20fd2bSAlexei Starovoitov /* if error is returned, fd is released. 901db20fd2bSAlexei Starovoitov * On success caller should complete fd access with matching fdput() 902db20fd2bSAlexei Starovoitov */ 903c2101297SDaniel Borkmann struct bpf_map *__bpf_map_get(struct fd f) 904db20fd2bSAlexei Starovoitov { 905db20fd2bSAlexei Starovoitov if (!f.file) 906db20fd2bSAlexei Starovoitov return ERR_PTR(-EBADF); 907db20fd2bSAlexei Starovoitov if (f.file->f_op != &bpf_map_fops) { 908db20fd2bSAlexei Starovoitov fdput(f); 909db20fd2bSAlexei Starovoitov return ERR_PTR(-EINVAL); 910db20fd2bSAlexei Starovoitov } 911db20fd2bSAlexei Starovoitov 912c2101297SDaniel Borkmann return f.file->private_data; 913c2101297SDaniel Borkmann } 914c2101297SDaniel Borkmann 9151e0bd5a0SAndrii Nakryiko void bpf_map_inc(struct bpf_map *map) 916c9da161cSDaniel Borkmann { 9171e0bd5a0SAndrii Nakryiko atomic64_inc(&map->refcnt); 918c9da161cSDaniel Borkmann } 919630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_inc); 920c9da161cSDaniel Borkmann 9211e0bd5a0SAndrii Nakryiko void bpf_map_inc_with_uref(struct bpf_map *map) 9221e0bd5a0SAndrii Nakryiko { 9231e0bd5a0SAndrii Nakryiko atomic64_inc(&map->refcnt); 9241e0bd5a0SAndrii Nakryiko atomic64_inc(&map->usercnt); 9251e0bd5a0SAndrii Nakryiko } 9261e0bd5a0SAndrii Nakryiko EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref); 9271e0bd5a0SAndrii Nakryiko 9281ed4d924SMartin KaFai Lau struct bpf_map *bpf_map_get(u32 ufd) 9291ed4d924SMartin KaFai Lau { 9301ed4d924SMartin KaFai Lau struct fd f = fdget(ufd); 9311ed4d924SMartin KaFai Lau struct bpf_map *map; 9321ed4d924SMartin KaFai Lau 9331ed4d924SMartin KaFai Lau map = __bpf_map_get(f); 9341ed4d924SMartin KaFai Lau if (IS_ERR(map)) 9351ed4d924SMartin KaFai Lau return map; 9361ed4d924SMartin KaFai Lau 9371ed4d924SMartin KaFai Lau bpf_map_inc(map); 9381ed4d924SMartin KaFai Lau fdput(f); 9391ed4d924SMartin KaFai Lau 9401ed4d924SMartin KaFai Lau return map; 9411ed4d924SMartin KaFai Lau } 9421ed4d924SMartin KaFai Lau 943c9da161cSDaniel Borkmann struct bpf_map *bpf_map_get_with_uref(u32 ufd) 944c2101297SDaniel Borkmann { 945c2101297SDaniel Borkmann struct fd f = fdget(ufd); 946c2101297SDaniel Borkmann struct bpf_map *map; 947c2101297SDaniel Borkmann 948c2101297SDaniel Borkmann map = __bpf_map_get(f); 949c2101297SDaniel Borkmann if (IS_ERR(map)) 950c2101297SDaniel Borkmann return map; 951c2101297SDaniel Borkmann 9521e0bd5a0SAndrii Nakryiko bpf_map_inc_with_uref(map); 953c2101297SDaniel Borkmann fdput(f); 954db20fd2bSAlexei Starovoitov 955db20fd2bSAlexei Starovoitov return map; 956db20fd2bSAlexei Starovoitov } 957db20fd2bSAlexei Starovoitov 958bd5f5f4eSMartin KaFai Lau /* map_idr_lock should have been held */ 9591e0bd5a0SAndrii Nakryiko static struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref) 960bd5f5f4eSMartin KaFai Lau { 961bd5f5f4eSMartin KaFai Lau int refold; 962bd5f5f4eSMartin KaFai Lau 9631e0bd5a0SAndrii Nakryiko refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0); 964bd5f5f4eSMartin KaFai Lau if (!refold) 965bd5f5f4eSMartin KaFai Lau return ERR_PTR(-ENOENT); 966bd5f5f4eSMartin KaFai Lau if (uref) 9671e0bd5a0SAndrii Nakryiko atomic64_inc(&map->usercnt); 968bd5f5f4eSMartin KaFai Lau 969bd5f5f4eSMartin KaFai Lau return map; 970bd5f5f4eSMartin KaFai Lau } 971bd5f5f4eSMartin KaFai Lau 9721e0bd5a0SAndrii Nakryiko struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map) 973b0e4701cSStanislav Fomichev { 974b0e4701cSStanislav Fomichev spin_lock_bh(&map_idr_lock); 9751e0bd5a0SAndrii Nakryiko map = __bpf_map_inc_not_zero(map, false); 976b0e4701cSStanislav Fomichev spin_unlock_bh(&map_idr_lock); 977b0e4701cSStanislav Fomichev 978b0e4701cSStanislav Fomichev return map; 979b0e4701cSStanislav Fomichev } 980b0e4701cSStanislav Fomichev EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero); 981b0e4701cSStanislav Fomichev 982b8cdc051SAlexei Starovoitov int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 983b8cdc051SAlexei Starovoitov { 984b8cdc051SAlexei Starovoitov return -ENOTSUPP; 985b8cdc051SAlexei Starovoitov } 986b8cdc051SAlexei Starovoitov 987c9d29f46SMauricio Vasquez B static void *__bpf_copy_key(void __user *ukey, u64 key_size) 988c9d29f46SMauricio Vasquez B { 989c9d29f46SMauricio Vasquez B if (key_size) 990c9d29f46SMauricio Vasquez B return memdup_user(ukey, key_size); 991c9d29f46SMauricio Vasquez B 992c9d29f46SMauricio Vasquez B if (ukey) 993c9d29f46SMauricio Vasquez B return ERR_PTR(-EINVAL); 994c9d29f46SMauricio Vasquez B 995c9d29f46SMauricio Vasquez B return NULL; 996c9d29f46SMauricio Vasquez B } 997c9d29f46SMauricio Vasquez B 998db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 99996049f3aSAlexei Starovoitov #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags 1000db20fd2bSAlexei Starovoitov 1001db20fd2bSAlexei Starovoitov static int map_lookup_elem(union bpf_attr *attr) 1002db20fd2bSAlexei Starovoitov { 1003535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1004535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 1005db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1006db20fd2bSAlexei Starovoitov struct bpf_map *map; 100715c14a3dSBrian Vazquez void *key, *value; 100815a07b33SAlexei Starovoitov u32 value_size; 1009592867bfSDaniel Borkmann struct fd f; 1010db20fd2bSAlexei Starovoitov int err; 1011db20fd2bSAlexei Starovoitov 1012db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 1013db20fd2bSAlexei Starovoitov return -EINVAL; 1014db20fd2bSAlexei Starovoitov 101596049f3aSAlexei Starovoitov if (attr->flags & ~BPF_F_LOCK) 101696049f3aSAlexei Starovoitov return -EINVAL; 101796049f3aSAlexei Starovoitov 1018592867bfSDaniel Borkmann f = fdget(ufd); 1019c2101297SDaniel Borkmann map = __bpf_map_get(f); 1020db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1021db20fd2bSAlexei Starovoitov return PTR_ERR(map); 102287df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 10236e71b04aSChenbo Feng err = -EPERM; 10246e71b04aSChenbo Feng goto err_put; 10256e71b04aSChenbo Feng } 10266e71b04aSChenbo Feng 102796049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 102896049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 102996049f3aSAlexei Starovoitov err = -EINVAL; 103096049f3aSAlexei Starovoitov goto err_put; 103196049f3aSAlexei Starovoitov } 103296049f3aSAlexei Starovoitov 1033c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1034e4448ed8SAl Viro if (IS_ERR(key)) { 1035e4448ed8SAl Viro err = PTR_ERR(key); 1036db20fd2bSAlexei Starovoitov goto err_put; 1037e4448ed8SAl Viro } 1038db20fd2bSAlexei Starovoitov 103915c14a3dSBrian Vazquez value_size = bpf_map_value_size(map); 104015a07b33SAlexei Starovoitov 10418ebe667cSAlexei Starovoitov err = -ENOMEM; 104215a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1043db20fd2bSAlexei Starovoitov if (!value) 10448ebe667cSAlexei Starovoitov goto free_key; 10458ebe667cSAlexei Starovoitov 104615c14a3dSBrian Vazquez err = bpf_map_copy_value(map, key, value, attr->flags); 104715a07b33SAlexei Starovoitov if (err) 10488ebe667cSAlexei Starovoitov goto free_value; 1049db20fd2bSAlexei Starovoitov 1050db20fd2bSAlexei Starovoitov err = -EFAULT; 105115a07b33SAlexei Starovoitov if (copy_to_user(uvalue, value, value_size) != 0) 10528ebe667cSAlexei Starovoitov goto free_value; 1053db20fd2bSAlexei Starovoitov 1054db20fd2bSAlexei Starovoitov err = 0; 1055db20fd2bSAlexei Starovoitov 10568ebe667cSAlexei Starovoitov free_value: 10578ebe667cSAlexei Starovoitov kfree(value); 1058db20fd2bSAlexei Starovoitov free_key: 1059db20fd2bSAlexei Starovoitov kfree(key); 1060db20fd2bSAlexei Starovoitov err_put: 1061db20fd2bSAlexei Starovoitov fdput(f); 1062db20fd2bSAlexei Starovoitov return err; 1063db20fd2bSAlexei Starovoitov } 1064db20fd2bSAlexei Starovoitov 10651ae80cf3SDaniel Colascione 10663274f520SAlexei Starovoitov #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 1067db20fd2bSAlexei Starovoitov 1068db20fd2bSAlexei Starovoitov static int map_update_elem(union bpf_attr *attr) 1069db20fd2bSAlexei Starovoitov { 1070535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1071535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 1072db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1073db20fd2bSAlexei Starovoitov struct bpf_map *map; 1074db20fd2bSAlexei Starovoitov void *key, *value; 107515a07b33SAlexei Starovoitov u32 value_size; 1076592867bfSDaniel Borkmann struct fd f; 1077db20fd2bSAlexei Starovoitov int err; 1078db20fd2bSAlexei Starovoitov 1079db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 1080db20fd2bSAlexei Starovoitov return -EINVAL; 1081db20fd2bSAlexei Starovoitov 1082592867bfSDaniel Borkmann f = fdget(ufd); 1083c2101297SDaniel Borkmann map = __bpf_map_get(f); 1084db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1085db20fd2bSAlexei Starovoitov return PTR_ERR(map); 108687df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 10876e71b04aSChenbo Feng err = -EPERM; 10886e71b04aSChenbo Feng goto err_put; 10896e71b04aSChenbo Feng } 10906e71b04aSChenbo Feng 109196049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 109296049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 109396049f3aSAlexei Starovoitov err = -EINVAL; 109496049f3aSAlexei Starovoitov goto err_put; 109596049f3aSAlexei Starovoitov } 109696049f3aSAlexei Starovoitov 1097c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1098e4448ed8SAl Viro if (IS_ERR(key)) { 1099e4448ed8SAl Viro err = PTR_ERR(key); 1100db20fd2bSAlexei Starovoitov goto err_put; 1101e4448ed8SAl Viro } 1102db20fd2bSAlexei Starovoitov 110315a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 11048f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 1105b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 1106b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 110715a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 110815a07b33SAlexei Starovoitov else 110915a07b33SAlexei Starovoitov value_size = map->value_size; 111015a07b33SAlexei Starovoitov 1111db20fd2bSAlexei Starovoitov err = -ENOMEM; 111215a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1113db20fd2bSAlexei Starovoitov if (!value) 1114db20fd2bSAlexei Starovoitov goto free_key; 1115db20fd2bSAlexei Starovoitov 1116db20fd2bSAlexei Starovoitov err = -EFAULT; 111715a07b33SAlexei Starovoitov if (copy_from_user(value, uvalue, value_size) != 0) 1118db20fd2bSAlexei Starovoitov goto free_value; 1119db20fd2bSAlexei Starovoitov 112015c14a3dSBrian Vazquez err = bpf_map_update_value(map, f, key, value, attr->flags); 11216710e112SJesper Dangaard Brouer 1122db20fd2bSAlexei Starovoitov free_value: 1123db20fd2bSAlexei Starovoitov kfree(value); 1124db20fd2bSAlexei Starovoitov free_key: 1125db20fd2bSAlexei Starovoitov kfree(key); 1126db20fd2bSAlexei Starovoitov err_put: 1127db20fd2bSAlexei Starovoitov fdput(f); 1128db20fd2bSAlexei Starovoitov return err; 1129db20fd2bSAlexei Starovoitov } 1130db20fd2bSAlexei Starovoitov 1131db20fd2bSAlexei Starovoitov #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 1132db20fd2bSAlexei Starovoitov 1133db20fd2bSAlexei Starovoitov static int map_delete_elem(union bpf_attr *attr) 1134db20fd2bSAlexei Starovoitov { 1135535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1136db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1137db20fd2bSAlexei Starovoitov struct bpf_map *map; 1138592867bfSDaniel Borkmann struct fd f; 1139db20fd2bSAlexei Starovoitov void *key; 1140db20fd2bSAlexei Starovoitov int err; 1141db20fd2bSAlexei Starovoitov 1142db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 1143db20fd2bSAlexei Starovoitov return -EINVAL; 1144db20fd2bSAlexei Starovoitov 1145592867bfSDaniel Borkmann f = fdget(ufd); 1146c2101297SDaniel Borkmann map = __bpf_map_get(f); 1147db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1148db20fd2bSAlexei Starovoitov return PTR_ERR(map); 114987df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 11506e71b04aSChenbo Feng err = -EPERM; 11516e71b04aSChenbo Feng goto err_put; 11526e71b04aSChenbo Feng } 11536e71b04aSChenbo Feng 1154c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1155e4448ed8SAl Viro if (IS_ERR(key)) { 1156e4448ed8SAl Viro err = PTR_ERR(key); 1157db20fd2bSAlexei Starovoitov goto err_put; 1158e4448ed8SAl Viro } 1159db20fd2bSAlexei Starovoitov 1160a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1161a3884572SJakub Kicinski err = bpf_map_offload_delete_elem(map, key); 1162a3884572SJakub Kicinski goto out; 116385d33df3SMartin KaFai Lau } else if (IS_FD_PROG_ARRAY(map) || 116485d33df3SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 116585d33df3SMartin KaFai Lau /* These maps require sleepable context */ 1166da765a2fSDaniel Borkmann err = map->ops->map_delete_elem(map, key); 1167da765a2fSDaniel Borkmann goto out; 1168a3884572SJakub Kicinski } 1169a3884572SJakub Kicinski 1170b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 1171db20fd2bSAlexei Starovoitov rcu_read_lock(); 1172db20fd2bSAlexei Starovoitov err = map->ops->map_delete_elem(map, key); 1173db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1174b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 11751ae80cf3SDaniel Colascione maybe_wait_bpf_programs(map); 1176a3884572SJakub Kicinski out: 1177db20fd2bSAlexei Starovoitov kfree(key); 1178db20fd2bSAlexei Starovoitov err_put: 1179db20fd2bSAlexei Starovoitov fdput(f); 1180db20fd2bSAlexei Starovoitov return err; 1181db20fd2bSAlexei Starovoitov } 1182db20fd2bSAlexei Starovoitov 1183db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 1184db20fd2bSAlexei Starovoitov #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 1185db20fd2bSAlexei Starovoitov 1186db20fd2bSAlexei Starovoitov static int map_get_next_key(union bpf_attr *attr) 1187db20fd2bSAlexei Starovoitov { 1188535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1189535e7b4bSMickaël Salaün void __user *unext_key = u64_to_user_ptr(attr->next_key); 1190db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1191db20fd2bSAlexei Starovoitov struct bpf_map *map; 1192db20fd2bSAlexei Starovoitov void *key, *next_key; 1193592867bfSDaniel Borkmann struct fd f; 1194db20fd2bSAlexei Starovoitov int err; 1195db20fd2bSAlexei Starovoitov 1196db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 1197db20fd2bSAlexei Starovoitov return -EINVAL; 1198db20fd2bSAlexei Starovoitov 1199592867bfSDaniel Borkmann f = fdget(ufd); 1200c2101297SDaniel Borkmann map = __bpf_map_get(f); 1201db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1202db20fd2bSAlexei Starovoitov return PTR_ERR(map); 120387df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 12046e71b04aSChenbo Feng err = -EPERM; 12056e71b04aSChenbo Feng goto err_put; 12066e71b04aSChenbo Feng } 12076e71b04aSChenbo Feng 12088fe45924STeng Qin if (ukey) { 1209c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1210e4448ed8SAl Viro if (IS_ERR(key)) { 1211e4448ed8SAl Viro err = PTR_ERR(key); 1212db20fd2bSAlexei Starovoitov goto err_put; 1213e4448ed8SAl Viro } 12148fe45924STeng Qin } else { 12158fe45924STeng Qin key = NULL; 12168fe45924STeng Qin } 1217db20fd2bSAlexei Starovoitov 1218db20fd2bSAlexei Starovoitov err = -ENOMEM; 1219db20fd2bSAlexei Starovoitov next_key = kmalloc(map->key_size, GFP_USER); 1220db20fd2bSAlexei Starovoitov if (!next_key) 1221db20fd2bSAlexei Starovoitov goto free_key; 1222db20fd2bSAlexei Starovoitov 1223a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1224a3884572SJakub Kicinski err = bpf_map_offload_get_next_key(map, key, next_key); 1225a3884572SJakub Kicinski goto out; 1226a3884572SJakub Kicinski } 1227a3884572SJakub Kicinski 1228db20fd2bSAlexei Starovoitov rcu_read_lock(); 1229db20fd2bSAlexei Starovoitov err = map->ops->map_get_next_key(map, key, next_key); 1230db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1231a3884572SJakub Kicinski out: 1232db20fd2bSAlexei Starovoitov if (err) 1233db20fd2bSAlexei Starovoitov goto free_next_key; 1234db20fd2bSAlexei Starovoitov 1235db20fd2bSAlexei Starovoitov err = -EFAULT; 1236db20fd2bSAlexei Starovoitov if (copy_to_user(unext_key, next_key, map->key_size) != 0) 1237db20fd2bSAlexei Starovoitov goto free_next_key; 1238db20fd2bSAlexei Starovoitov 1239db20fd2bSAlexei Starovoitov err = 0; 1240db20fd2bSAlexei Starovoitov 1241db20fd2bSAlexei Starovoitov free_next_key: 1242db20fd2bSAlexei Starovoitov kfree(next_key); 1243db20fd2bSAlexei Starovoitov free_key: 1244db20fd2bSAlexei Starovoitov kfree(key); 1245db20fd2bSAlexei Starovoitov err_put: 1246db20fd2bSAlexei Starovoitov fdput(f); 1247db20fd2bSAlexei Starovoitov return err; 1248db20fd2bSAlexei Starovoitov } 1249db20fd2bSAlexei Starovoitov 1250aa2e93b8SBrian Vazquez int generic_map_delete_batch(struct bpf_map *map, 1251aa2e93b8SBrian Vazquez const union bpf_attr *attr, 1252aa2e93b8SBrian Vazquez union bpf_attr __user *uattr) 1253aa2e93b8SBrian Vazquez { 1254aa2e93b8SBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1255aa2e93b8SBrian Vazquez u32 cp, max_count; 1256aa2e93b8SBrian Vazquez int err = 0; 1257aa2e93b8SBrian Vazquez void *key; 1258aa2e93b8SBrian Vazquez 1259aa2e93b8SBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1260aa2e93b8SBrian Vazquez return -EINVAL; 1261aa2e93b8SBrian Vazquez 1262aa2e93b8SBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1263aa2e93b8SBrian Vazquez !map_value_has_spin_lock(map)) { 1264aa2e93b8SBrian Vazquez return -EINVAL; 1265aa2e93b8SBrian Vazquez } 1266aa2e93b8SBrian Vazquez 1267aa2e93b8SBrian Vazquez max_count = attr->batch.count; 1268aa2e93b8SBrian Vazquez if (!max_count) 1269aa2e93b8SBrian Vazquez return 0; 1270aa2e93b8SBrian Vazquez 12712e3a94aaSBrian Vazquez key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 12722e3a94aaSBrian Vazquez if (!key) 12732e3a94aaSBrian Vazquez return -ENOMEM; 12742e3a94aaSBrian Vazquez 1275aa2e93b8SBrian Vazquez for (cp = 0; cp < max_count; cp++) { 12762e3a94aaSBrian Vazquez err = -EFAULT; 12772e3a94aaSBrian Vazquez if (copy_from_user(key, keys + cp * map->key_size, 12782e3a94aaSBrian Vazquez map->key_size)) 1279aa2e93b8SBrian Vazquez break; 1280aa2e93b8SBrian Vazquez 1281aa2e93b8SBrian Vazquez if (bpf_map_is_dev_bound(map)) { 1282aa2e93b8SBrian Vazquez err = bpf_map_offload_delete_elem(map, key); 1283aa2e93b8SBrian Vazquez break; 1284aa2e93b8SBrian Vazquez } 1285aa2e93b8SBrian Vazquez 1286b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 1287aa2e93b8SBrian Vazquez rcu_read_lock(); 1288aa2e93b8SBrian Vazquez err = map->ops->map_delete_elem(map, key); 1289aa2e93b8SBrian Vazquez rcu_read_unlock(); 1290b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 1291aa2e93b8SBrian Vazquez maybe_wait_bpf_programs(map); 1292aa2e93b8SBrian Vazquez if (err) 1293aa2e93b8SBrian Vazquez break; 1294aa2e93b8SBrian Vazquez } 1295aa2e93b8SBrian Vazquez if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) 1296aa2e93b8SBrian Vazquez err = -EFAULT; 12972e3a94aaSBrian Vazquez 12982e3a94aaSBrian Vazquez kfree(key); 1299aa2e93b8SBrian Vazquez return err; 1300aa2e93b8SBrian Vazquez } 1301aa2e93b8SBrian Vazquez 1302aa2e93b8SBrian Vazquez int generic_map_update_batch(struct bpf_map *map, 1303aa2e93b8SBrian Vazquez const union bpf_attr *attr, 1304aa2e93b8SBrian Vazquez union bpf_attr __user *uattr) 1305aa2e93b8SBrian Vazquez { 1306aa2e93b8SBrian Vazquez void __user *values = u64_to_user_ptr(attr->batch.values); 1307aa2e93b8SBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1308aa2e93b8SBrian Vazquez u32 value_size, cp, max_count; 1309aa2e93b8SBrian Vazquez int ufd = attr->map_fd; 1310aa2e93b8SBrian Vazquez void *key, *value; 1311aa2e93b8SBrian Vazquez struct fd f; 1312aa2e93b8SBrian Vazquez int err = 0; 1313aa2e93b8SBrian Vazquez 1314aa2e93b8SBrian Vazquez f = fdget(ufd); 1315aa2e93b8SBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1316aa2e93b8SBrian Vazquez return -EINVAL; 1317aa2e93b8SBrian Vazquez 1318aa2e93b8SBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1319aa2e93b8SBrian Vazquez !map_value_has_spin_lock(map)) { 1320aa2e93b8SBrian Vazquez return -EINVAL; 1321aa2e93b8SBrian Vazquez } 1322aa2e93b8SBrian Vazquez 1323aa2e93b8SBrian Vazquez value_size = bpf_map_value_size(map); 1324aa2e93b8SBrian Vazquez 1325aa2e93b8SBrian Vazquez max_count = attr->batch.count; 1326aa2e93b8SBrian Vazquez if (!max_count) 1327aa2e93b8SBrian Vazquez return 0; 1328aa2e93b8SBrian Vazquez 13292e3a94aaSBrian Vazquez key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 13302e3a94aaSBrian Vazquez if (!key) 1331aa2e93b8SBrian Vazquez return -ENOMEM; 1332aa2e93b8SBrian Vazquez 13332e3a94aaSBrian Vazquez value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 13342e3a94aaSBrian Vazquez if (!value) { 13352e3a94aaSBrian Vazquez kfree(key); 13362e3a94aaSBrian Vazquez return -ENOMEM; 1337aa2e93b8SBrian Vazquez } 13382e3a94aaSBrian Vazquez 13392e3a94aaSBrian Vazquez for (cp = 0; cp < max_count; cp++) { 1340aa2e93b8SBrian Vazquez err = -EFAULT; 13412e3a94aaSBrian Vazquez if (copy_from_user(key, keys + cp * map->key_size, 13422e3a94aaSBrian Vazquez map->key_size) || 13432e3a94aaSBrian Vazquez copy_from_user(value, values + cp * value_size, value_size)) 1344aa2e93b8SBrian Vazquez break; 1345aa2e93b8SBrian Vazquez 1346aa2e93b8SBrian Vazquez err = bpf_map_update_value(map, f, key, value, 1347aa2e93b8SBrian Vazquez attr->batch.elem_flags); 1348aa2e93b8SBrian Vazquez 1349aa2e93b8SBrian Vazquez if (err) 1350aa2e93b8SBrian Vazquez break; 1351aa2e93b8SBrian Vazquez } 1352aa2e93b8SBrian Vazquez 1353aa2e93b8SBrian Vazquez if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) 1354aa2e93b8SBrian Vazquez err = -EFAULT; 1355aa2e93b8SBrian Vazquez 1356aa2e93b8SBrian Vazquez kfree(value); 1357aa2e93b8SBrian Vazquez kfree(key); 1358aa2e93b8SBrian Vazquez return err; 1359aa2e93b8SBrian Vazquez } 1360aa2e93b8SBrian Vazquez 1361cb4d03abSBrian Vazquez #define MAP_LOOKUP_RETRIES 3 1362cb4d03abSBrian Vazquez 1363cb4d03abSBrian Vazquez int generic_map_lookup_batch(struct bpf_map *map, 1364cb4d03abSBrian Vazquez const union bpf_attr *attr, 1365cb4d03abSBrian Vazquez union bpf_attr __user *uattr) 1366cb4d03abSBrian Vazquez { 1367cb4d03abSBrian Vazquez void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch); 1368cb4d03abSBrian Vazquez void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch); 1369cb4d03abSBrian Vazquez void __user *values = u64_to_user_ptr(attr->batch.values); 1370cb4d03abSBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1371cb4d03abSBrian Vazquez void *buf, *buf_prevkey, *prev_key, *key, *value; 1372cb4d03abSBrian Vazquez int err, retry = MAP_LOOKUP_RETRIES; 1373cb4d03abSBrian Vazquez u32 value_size, cp, max_count; 1374cb4d03abSBrian Vazquez 1375cb4d03abSBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1376cb4d03abSBrian Vazquez return -EINVAL; 1377cb4d03abSBrian Vazquez 1378cb4d03abSBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1379cb4d03abSBrian Vazquez !map_value_has_spin_lock(map)) 1380cb4d03abSBrian Vazquez return -EINVAL; 1381cb4d03abSBrian Vazquez 1382cb4d03abSBrian Vazquez value_size = bpf_map_value_size(map); 1383cb4d03abSBrian Vazquez 1384cb4d03abSBrian Vazquez max_count = attr->batch.count; 1385cb4d03abSBrian Vazquez if (!max_count) 1386cb4d03abSBrian Vazquez return 0; 1387cb4d03abSBrian Vazquez 1388cb4d03abSBrian Vazquez if (put_user(0, &uattr->batch.count)) 1389cb4d03abSBrian Vazquez return -EFAULT; 1390cb4d03abSBrian Vazquez 1391cb4d03abSBrian Vazquez buf_prevkey = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 1392cb4d03abSBrian Vazquez if (!buf_prevkey) 1393cb4d03abSBrian Vazquez return -ENOMEM; 1394cb4d03abSBrian Vazquez 1395cb4d03abSBrian Vazquez buf = kmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN); 1396cb4d03abSBrian Vazquez if (!buf) { 1397bb2359f4SDenis Efremov kfree(buf_prevkey); 1398cb4d03abSBrian Vazquez return -ENOMEM; 1399cb4d03abSBrian Vazquez } 1400cb4d03abSBrian Vazquez 1401cb4d03abSBrian Vazquez err = -EFAULT; 1402cb4d03abSBrian Vazquez prev_key = NULL; 1403cb4d03abSBrian Vazquez if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size)) 1404cb4d03abSBrian Vazquez goto free_buf; 1405cb4d03abSBrian Vazquez key = buf; 1406cb4d03abSBrian Vazquez value = key + map->key_size; 1407cb4d03abSBrian Vazquez if (ubatch) 1408cb4d03abSBrian Vazquez prev_key = buf_prevkey; 1409cb4d03abSBrian Vazquez 1410cb4d03abSBrian Vazquez for (cp = 0; cp < max_count;) { 1411cb4d03abSBrian Vazquez rcu_read_lock(); 1412cb4d03abSBrian Vazquez err = map->ops->map_get_next_key(map, prev_key, key); 1413cb4d03abSBrian Vazquez rcu_read_unlock(); 1414cb4d03abSBrian Vazquez if (err) 1415cb4d03abSBrian Vazquez break; 1416cb4d03abSBrian Vazquez err = bpf_map_copy_value(map, key, value, 1417cb4d03abSBrian Vazquez attr->batch.elem_flags); 1418cb4d03abSBrian Vazquez 1419cb4d03abSBrian Vazquez if (err == -ENOENT) { 1420cb4d03abSBrian Vazquez if (retry) { 1421cb4d03abSBrian Vazquez retry--; 1422cb4d03abSBrian Vazquez continue; 1423cb4d03abSBrian Vazquez } 1424cb4d03abSBrian Vazquez err = -EINTR; 1425cb4d03abSBrian Vazquez break; 1426cb4d03abSBrian Vazquez } 1427cb4d03abSBrian Vazquez 1428cb4d03abSBrian Vazquez if (err) 1429cb4d03abSBrian Vazquez goto free_buf; 1430cb4d03abSBrian Vazquez 1431cb4d03abSBrian Vazquez if (copy_to_user(keys + cp * map->key_size, key, 1432cb4d03abSBrian Vazquez map->key_size)) { 1433cb4d03abSBrian Vazquez err = -EFAULT; 1434cb4d03abSBrian Vazquez goto free_buf; 1435cb4d03abSBrian Vazquez } 1436cb4d03abSBrian Vazquez if (copy_to_user(values + cp * value_size, value, value_size)) { 1437cb4d03abSBrian Vazquez err = -EFAULT; 1438cb4d03abSBrian Vazquez goto free_buf; 1439cb4d03abSBrian Vazquez } 1440cb4d03abSBrian Vazquez 1441cb4d03abSBrian Vazquez if (!prev_key) 1442cb4d03abSBrian Vazquez prev_key = buf_prevkey; 1443cb4d03abSBrian Vazquez 1444cb4d03abSBrian Vazquez swap(prev_key, key); 1445cb4d03abSBrian Vazquez retry = MAP_LOOKUP_RETRIES; 1446cb4d03abSBrian Vazquez cp++; 1447cb4d03abSBrian Vazquez } 1448cb4d03abSBrian Vazquez 1449cb4d03abSBrian Vazquez if (err == -EFAULT) 1450cb4d03abSBrian Vazquez goto free_buf; 1451cb4d03abSBrian Vazquez 1452cb4d03abSBrian Vazquez if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) || 1453cb4d03abSBrian Vazquez (cp && copy_to_user(uobatch, prev_key, map->key_size)))) 1454cb4d03abSBrian Vazquez err = -EFAULT; 1455cb4d03abSBrian Vazquez 1456cb4d03abSBrian Vazquez free_buf: 1457cb4d03abSBrian Vazquez kfree(buf_prevkey); 1458cb4d03abSBrian Vazquez kfree(buf); 1459cb4d03abSBrian Vazquez return err; 1460cb4d03abSBrian Vazquez } 1461cb4d03abSBrian Vazquez 1462bd513cd0SMauricio Vasquez B #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value 1463bd513cd0SMauricio Vasquez B 1464bd513cd0SMauricio Vasquez B static int map_lookup_and_delete_elem(union bpf_attr *attr) 1465bd513cd0SMauricio Vasquez B { 1466bd513cd0SMauricio Vasquez B void __user *ukey = u64_to_user_ptr(attr->key); 1467bd513cd0SMauricio Vasquez B void __user *uvalue = u64_to_user_ptr(attr->value); 1468bd513cd0SMauricio Vasquez B int ufd = attr->map_fd; 1469bd513cd0SMauricio Vasquez B struct bpf_map *map; 1470540fefc0SAlexei Starovoitov void *key, *value; 1471bd513cd0SMauricio Vasquez B u32 value_size; 1472bd513cd0SMauricio Vasquez B struct fd f; 1473bd513cd0SMauricio Vasquez B int err; 1474bd513cd0SMauricio Vasquez B 1475bd513cd0SMauricio Vasquez B if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) 1476bd513cd0SMauricio Vasquez B return -EINVAL; 1477bd513cd0SMauricio Vasquez B 1478bd513cd0SMauricio Vasquez B f = fdget(ufd); 1479bd513cd0SMauricio Vasquez B map = __bpf_map_get(f); 1480bd513cd0SMauricio Vasquez B if (IS_ERR(map)) 1481bd513cd0SMauricio Vasquez B return PTR_ERR(map); 14821ea0f912SAnton Protopopov if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) || 14831ea0f912SAnton Protopopov !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 1484bd513cd0SMauricio Vasquez B err = -EPERM; 1485bd513cd0SMauricio Vasquez B goto err_put; 1486bd513cd0SMauricio Vasquez B } 1487bd513cd0SMauricio Vasquez B 1488bd513cd0SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1489bd513cd0SMauricio Vasquez B if (IS_ERR(key)) { 1490bd513cd0SMauricio Vasquez B err = PTR_ERR(key); 1491bd513cd0SMauricio Vasquez B goto err_put; 1492bd513cd0SMauricio Vasquez B } 1493bd513cd0SMauricio Vasquez B 1494bd513cd0SMauricio Vasquez B value_size = map->value_size; 1495bd513cd0SMauricio Vasquez B 1496bd513cd0SMauricio Vasquez B err = -ENOMEM; 1497bd513cd0SMauricio Vasquez B value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1498bd513cd0SMauricio Vasquez B if (!value) 1499bd513cd0SMauricio Vasquez B goto free_key; 1500bd513cd0SMauricio Vasquez B 1501bd513cd0SMauricio Vasquez B if (map->map_type == BPF_MAP_TYPE_QUEUE || 1502bd513cd0SMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 1503bd513cd0SMauricio Vasquez B err = map->ops->map_pop_elem(map, value); 1504bd513cd0SMauricio Vasquez B } else { 1505bd513cd0SMauricio Vasquez B err = -ENOTSUPP; 1506bd513cd0SMauricio Vasquez B } 1507bd513cd0SMauricio Vasquez B 1508bd513cd0SMauricio Vasquez B if (err) 1509bd513cd0SMauricio Vasquez B goto free_value; 1510bd513cd0SMauricio Vasquez B 15117f645462SWei Yongjun if (copy_to_user(uvalue, value, value_size) != 0) { 15127f645462SWei Yongjun err = -EFAULT; 1513bd513cd0SMauricio Vasquez B goto free_value; 15147f645462SWei Yongjun } 1515bd513cd0SMauricio Vasquez B 1516bd513cd0SMauricio Vasquez B err = 0; 1517bd513cd0SMauricio Vasquez B 1518bd513cd0SMauricio Vasquez B free_value: 1519bd513cd0SMauricio Vasquez B kfree(value); 1520bd513cd0SMauricio Vasquez B free_key: 1521bd513cd0SMauricio Vasquez B kfree(key); 1522bd513cd0SMauricio Vasquez B err_put: 1523bd513cd0SMauricio Vasquez B fdput(f); 1524bd513cd0SMauricio Vasquez B return err; 1525bd513cd0SMauricio Vasquez B } 1526bd513cd0SMauricio Vasquez B 152787df15deSDaniel Borkmann #define BPF_MAP_FREEZE_LAST_FIELD map_fd 152887df15deSDaniel Borkmann 152987df15deSDaniel Borkmann static int map_freeze(const union bpf_attr *attr) 153087df15deSDaniel Borkmann { 153187df15deSDaniel Borkmann int err = 0, ufd = attr->map_fd; 153287df15deSDaniel Borkmann struct bpf_map *map; 153387df15deSDaniel Borkmann struct fd f; 153487df15deSDaniel Borkmann 153587df15deSDaniel Borkmann if (CHECK_ATTR(BPF_MAP_FREEZE)) 153687df15deSDaniel Borkmann return -EINVAL; 153787df15deSDaniel Borkmann 153887df15deSDaniel Borkmann f = fdget(ufd); 153987df15deSDaniel Borkmann map = __bpf_map_get(f); 154087df15deSDaniel Borkmann if (IS_ERR(map)) 154187df15deSDaniel Borkmann return PTR_ERR(map); 1542fc970227SAndrii Nakryiko 1543849b4d94SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 1544849b4d94SMartin KaFai Lau fdput(f); 1545849b4d94SMartin KaFai Lau return -ENOTSUPP; 1546849b4d94SMartin KaFai Lau } 1547849b4d94SMartin KaFai Lau 1548fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 1549fc970227SAndrii Nakryiko 1550fc970227SAndrii Nakryiko if (map->writecnt) { 1551fc970227SAndrii Nakryiko err = -EBUSY; 1552fc970227SAndrii Nakryiko goto err_put; 1553fc970227SAndrii Nakryiko } 155487df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) { 155587df15deSDaniel Borkmann err = -EBUSY; 155687df15deSDaniel Borkmann goto err_put; 155787df15deSDaniel Borkmann } 15582c78ee89SAlexei Starovoitov if (!bpf_capable()) { 155987df15deSDaniel Borkmann err = -EPERM; 156087df15deSDaniel Borkmann goto err_put; 156187df15deSDaniel Borkmann } 156287df15deSDaniel Borkmann 156387df15deSDaniel Borkmann WRITE_ONCE(map->frozen, true); 156487df15deSDaniel Borkmann err_put: 1565fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 156687df15deSDaniel Borkmann fdput(f); 156787df15deSDaniel Borkmann return err; 156887df15deSDaniel Borkmann } 156987df15deSDaniel Borkmann 15707de16e3aSJakub Kicinski static const struct bpf_prog_ops * const bpf_prog_types[] = { 157191cc1a99SAlexei Starovoitov #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 15727de16e3aSJakub Kicinski [_id] = & _name ## _prog_ops, 15737de16e3aSJakub Kicinski #define BPF_MAP_TYPE(_id, _ops) 1574f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) 15757de16e3aSJakub Kicinski #include <linux/bpf_types.h> 15767de16e3aSJakub Kicinski #undef BPF_PROG_TYPE 15777de16e3aSJakub Kicinski #undef BPF_MAP_TYPE 1578f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 15797de16e3aSJakub Kicinski }; 15807de16e3aSJakub Kicinski 158109756af4SAlexei Starovoitov static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 158209756af4SAlexei Starovoitov { 1583d0f1a451SDaniel Borkmann const struct bpf_prog_ops *ops; 1584d0f1a451SDaniel Borkmann 1585d0f1a451SDaniel Borkmann if (type >= ARRAY_SIZE(bpf_prog_types)) 1586d0f1a451SDaniel Borkmann return -EINVAL; 1587d0f1a451SDaniel Borkmann type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types)); 1588d0f1a451SDaniel Borkmann ops = bpf_prog_types[type]; 1589d0f1a451SDaniel Borkmann if (!ops) 1590be9370a7SJohannes Berg return -EINVAL; 159109756af4SAlexei Starovoitov 1592ab3f0063SJakub Kicinski if (!bpf_prog_is_dev_bound(prog->aux)) 1593d0f1a451SDaniel Borkmann prog->aux->ops = ops; 1594ab3f0063SJakub Kicinski else 1595ab3f0063SJakub Kicinski prog->aux->ops = &bpf_offload_prog_ops; 159624701eceSDaniel Borkmann prog->type = type; 159709756af4SAlexei Starovoitov return 0; 159809756af4SAlexei Starovoitov } 159909756af4SAlexei Starovoitov 1600bae141f5SDaniel Borkmann enum bpf_audit { 1601bae141f5SDaniel Borkmann BPF_AUDIT_LOAD, 1602bae141f5SDaniel Borkmann BPF_AUDIT_UNLOAD, 1603bae141f5SDaniel Borkmann BPF_AUDIT_MAX, 1604bae141f5SDaniel Borkmann }; 1605bae141f5SDaniel Borkmann 1606bae141f5SDaniel Borkmann static const char * const bpf_audit_str[BPF_AUDIT_MAX] = { 1607bae141f5SDaniel Borkmann [BPF_AUDIT_LOAD] = "LOAD", 1608bae141f5SDaniel Borkmann [BPF_AUDIT_UNLOAD] = "UNLOAD", 1609bae141f5SDaniel Borkmann }; 1610bae141f5SDaniel Borkmann 1611bae141f5SDaniel Borkmann static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op) 1612bae141f5SDaniel Borkmann { 1613bae141f5SDaniel Borkmann struct audit_context *ctx = NULL; 1614bae141f5SDaniel Borkmann struct audit_buffer *ab; 1615bae141f5SDaniel Borkmann 1616bae141f5SDaniel Borkmann if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX)) 1617bae141f5SDaniel Borkmann return; 1618bae141f5SDaniel Borkmann if (audit_enabled == AUDIT_OFF) 1619bae141f5SDaniel Borkmann return; 1620bae141f5SDaniel Borkmann if (op == BPF_AUDIT_LOAD) 1621bae141f5SDaniel Borkmann ctx = audit_context(); 1622bae141f5SDaniel Borkmann ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF); 1623bae141f5SDaniel Borkmann if (unlikely(!ab)) 1624bae141f5SDaniel Borkmann return; 1625bae141f5SDaniel Borkmann audit_log_format(ab, "prog-id=%u op=%s", 1626bae141f5SDaniel Borkmann prog->aux->id, bpf_audit_str[op]); 1627bae141f5SDaniel Borkmann audit_log_end(ab); 1628bae141f5SDaniel Borkmann } 1629bae141f5SDaniel Borkmann 16305ccb071eSDaniel Borkmann int __bpf_prog_charge(struct user_struct *user, u32 pages) 16315ccb071eSDaniel Borkmann { 16325ccb071eSDaniel Borkmann unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 16335ccb071eSDaniel Borkmann unsigned long user_bufs; 16345ccb071eSDaniel Borkmann 16355ccb071eSDaniel Borkmann if (user) { 16365ccb071eSDaniel Borkmann user_bufs = atomic_long_add_return(pages, &user->locked_vm); 16375ccb071eSDaniel Borkmann if (user_bufs > memlock_limit) { 16385ccb071eSDaniel Borkmann atomic_long_sub(pages, &user->locked_vm); 16395ccb071eSDaniel Borkmann return -EPERM; 16405ccb071eSDaniel Borkmann } 16415ccb071eSDaniel Borkmann } 16425ccb071eSDaniel Borkmann 16435ccb071eSDaniel Borkmann return 0; 16445ccb071eSDaniel Borkmann } 16455ccb071eSDaniel Borkmann 16465ccb071eSDaniel Borkmann void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 16475ccb071eSDaniel Borkmann { 16485ccb071eSDaniel Borkmann if (user) 16495ccb071eSDaniel Borkmann atomic_long_sub(pages, &user->locked_vm); 16505ccb071eSDaniel Borkmann } 16515ccb071eSDaniel Borkmann 1652aaac3ba9SAlexei Starovoitov static int bpf_prog_charge_memlock(struct bpf_prog *prog) 1653aaac3ba9SAlexei Starovoitov { 1654aaac3ba9SAlexei Starovoitov struct user_struct *user = get_current_user(); 16555ccb071eSDaniel Borkmann int ret; 1656aaac3ba9SAlexei Starovoitov 16575ccb071eSDaniel Borkmann ret = __bpf_prog_charge(user, prog->pages); 16585ccb071eSDaniel Borkmann if (ret) { 1659aaac3ba9SAlexei Starovoitov free_uid(user); 16605ccb071eSDaniel Borkmann return ret; 1661aaac3ba9SAlexei Starovoitov } 16625ccb071eSDaniel Borkmann 1663aaac3ba9SAlexei Starovoitov prog->aux->user = user; 1664aaac3ba9SAlexei Starovoitov return 0; 1665aaac3ba9SAlexei Starovoitov } 1666aaac3ba9SAlexei Starovoitov 1667aaac3ba9SAlexei Starovoitov static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 1668aaac3ba9SAlexei Starovoitov { 1669aaac3ba9SAlexei Starovoitov struct user_struct *user = prog->aux->user; 1670aaac3ba9SAlexei Starovoitov 16715ccb071eSDaniel Borkmann __bpf_prog_uncharge(user, prog->pages); 1672aaac3ba9SAlexei Starovoitov free_uid(user); 1673aaac3ba9SAlexei Starovoitov } 1674aaac3ba9SAlexei Starovoitov 1675dc4bb0e2SMartin KaFai Lau static int bpf_prog_alloc_id(struct bpf_prog *prog) 1676dc4bb0e2SMartin KaFai Lau { 1677dc4bb0e2SMartin KaFai Lau int id; 1678dc4bb0e2SMartin KaFai Lau 1679b76354cdSShaohua Li idr_preload(GFP_KERNEL); 1680dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1681dc4bb0e2SMartin KaFai Lau id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 1682dc4bb0e2SMartin KaFai Lau if (id > 0) 1683dc4bb0e2SMartin KaFai Lau prog->aux->id = id; 1684dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1685b76354cdSShaohua Li idr_preload_end(); 1686dc4bb0e2SMartin KaFai Lau 1687dc4bb0e2SMartin KaFai Lau /* id is in [1, INT_MAX) */ 1688dc4bb0e2SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 1689dc4bb0e2SMartin KaFai Lau return -ENOSPC; 1690dc4bb0e2SMartin KaFai Lau 1691dc4bb0e2SMartin KaFai Lau return id > 0 ? 0 : id; 1692dc4bb0e2SMartin KaFai Lau } 1693dc4bb0e2SMartin KaFai Lau 1694ad8ad79fSJakub Kicinski void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) 1695dc4bb0e2SMartin KaFai Lau { 1696ad8ad79fSJakub Kicinski /* cBPF to eBPF migrations are currently not in the idr store. 1697ad8ad79fSJakub Kicinski * Offloaded programs are removed from the store when their device 1698ad8ad79fSJakub Kicinski * disappears - even if someone grabs an fd to them they are unusable, 1699ad8ad79fSJakub Kicinski * simply waiting for refcnt to drop to be freed. 1700ad8ad79fSJakub Kicinski */ 1701dc4bb0e2SMartin KaFai Lau if (!prog->aux->id) 1702dc4bb0e2SMartin KaFai Lau return; 1703dc4bb0e2SMartin KaFai Lau 1704b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1705dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1706b16d9aa4SMartin KaFai Lau else 1707b16d9aa4SMartin KaFai Lau __acquire(&prog_idr_lock); 1708b16d9aa4SMartin KaFai Lau 1709dc4bb0e2SMartin KaFai Lau idr_remove(&prog_idr, prog->aux->id); 1710ad8ad79fSJakub Kicinski prog->aux->id = 0; 1711b16d9aa4SMartin KaFai Lau 1712b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1713dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1714b16d9aa4SMartin KaFai Lau else 1715b16d9aa4SMartin KaFai Lau __release(&prog_idr_lock); 1716dc4bb0e2SMartin KaFai Lau } 1717dc4bb0e2SMartin KaFai Lau 17181aacde3dSDaniel Borkmann static void __bpf_prog_put_rcu(struct rcu_head *rcu) 1719abf2e7d6SAlexei Starovoitov { 1720abf2e7d6SAlexei Starovoitov struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 1721abf2e7d6SAlexei Starovoitov 17223b4d9eb2SDaniel Borkmann kvfree(aux->func_info); 17238c1b6e69SAlexei Starovoitov kfree(aux->func_info_aux); 1724aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(aux->prog); 1725afdb09c7SChenbo Feng security_bpf_prog_free(aux); 1726abf2e7d6SAlexei Starovoitov bpf_prog_free(aux->prog); 1727abf2e7d6SAlexei Starovoitov } 1728abf2e7d6SAlexei Starovoitov 1729cd7455f1SDaniel Borkmann static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) 1730cd7455f1SDaniel Borkmann { 1731cd7455f1SDaniel Borkmann bpf_prog_kallsyms_del_all(prog); 1732cd7455f1SDaniel Borkmann btf_put(prog->aux->btf); 1733cd7455f1SDaniel Borkmann bpf_prog_free_linfo(prog); 1734cd7455f1SDaniel Borkmann 17351e6c62a8SAlexei Starovoitov if (deferred) { 17361e6c62a8SAlexei Starovoitov if (prog->aux->sleepable) 17371e6c62a8SAlexei Starovoitov call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); 1738cd7455f1SDaniel Borkmann else 17391e6c62a8SAlexei Starovoitov call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 17401e6c62a8SAlexei Starovoitov } else { 1741cd7455f1SDaniel Borkmann __bpf_prog_put_rcu(&prog->aux->rcu); 1742cd7455f1SDaniel Borkmann } 17431e6c62a8SAlexei Starovoitov } 1744cd7455f1SDaniel Borkmann 1745b16d9aa4SMartin KaFai Lau static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 174609756af4SAlexei Starovoitov { 174785192dbfSAndrii Nakryiko if (atomic64_dec_and_test(&prog->aux->refcnt)) { 17486ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0); 1749bae141f5SDaniel Borkmann bpf_audit_prog(prog, BPF_AUDIT_UNLOAD); 175034ad5580SMartin KaFai Lau /* bpf_prog_free_id() must be called first */ 1751b16d9aa4SMartin KaFai Lau bpf_prog_free_id(prog, do_idr_lock); 1752cd7455f1SDaniel Borkmann __bpf_prog_put_noref(prog, true); 175309756af4SAlexei Starovoitov } 1754a67edbf4SDaniel Borkmann } 1755b16d9aa4SMartin KaFai Lau 1756b16d9aa4SMartin KaFai Lau void bpf_prog_put(struct bpf_prog *prog) 1757b16d9aa4SMartin KaFai Lau { 1758b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, true); 1759b16d9aa4SMartin KaFai Lau } 1760e2e9b654SDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_put); 176109756af4SAlexei Starovoitov 176209756af4SAlexei Starovoitov static int bpf_prog_release(struct inode *inode, struct file *filp) 176309756af4SAlexei Starovoitov { 176409756af4SAlexei Starovoitov struct bpf_prog *prog = filp->private_data; 176509756af4SAlexei Starovoitov 17661aacde3dSDaniel Borkmann bpf_prog_put(prog); 176709756af4SAlexei Starovoitov return 0; 176809756af4SAlexei Starovoitov } 176909756af4SAlexei Starovoitov 1770492ecee8SAlexei Starovoitov static void bpf_prog_get_stats(const struct bpf_prog *prog, 1771492ecee8SAlexei Starovoitov struct bpf_prog_stats *stats) 1772492ecee8SAlexei Starovoitov { 1773492ecee8SAlexei Starovoitov u64 nsecs = 0, cnt = 0; 1774492ecee8SAlexei Starovoitov int cpu; 1775492ecee8SAlexei Starovoitov 1776492ecee8SAlexei Starovoitov for_each_possible_cpu(cpu) { 1777492ecee8SAlexei Starovoitov const struct bpf_prog_stats *st; 1778492ecee8SAlexei Starovoitov unsigned int start; 1779492ecee8SAlexei Starovoitov u64 tnsecs, tcnt; 1780492ecee8SAlexei Starovoitov 1781492ecee8SAlexei Starovoitov st = per_cpu_ptr(prog->aux->stats, cpu); 1782492ecee8SAlexei Starovoitov do { 1783492ecee8SAlexei Starovoitov start = u64_stats_fetch_begin_irq(&st->syncp); 1784492ecee8SAlexei Starovoitov tnsecs = st->nsecs; 1785492ecee8SAlexei Starovoitov tcnt = st->cnt; 1786492ecee8SAlexei Starovoitov } while (u64_stats_fetch_retry_irq(&st->syncp, start)); 1787492ecee8SAlexei Starovoitov nsecs += tnsecs; 1788492ecee8SAlexei Starovoitov cnt += tcnt; 1789492ecee8SAlexei Starovoitov } 1790492ecee8SAlexei Starovoitov stats->nsecs = nsecs; 1791492ecee8SAlexei Starovoitov stats->cnt = cnt; 1792492ecee8SAlexei Starovoitov } 1793492ecee8SAlexei Starovoitov 17947bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 17957bd509e3SDaniel Borkmann static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 17967bd509e3SDaniel Borkmann { 17977bd509e3SDaniel Borkmann const struct bpf_prog *prog = filp->private_data; 1798f1f7714eSDaniel Borkmann char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 1799492ecee8SAlexei Starovoitov struct bpf_prog_stats stats; 18007bd509e3SDaniel Borkmann 1801492ecee8SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 1802f1f7714eSDaniel Borkmann bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 18037bd509e3SDaniel Borkmann seq_printf(m, 18047bd509e3SDaniel Borkmann "prog_type:\t%u\n" 18057bd509e3SDaniel Borkmann "prog_jited:\t%u\n" 1806f1f7714eSDaniel Borkmann "prog_tag:\t%s\n" 18074316b409SDaniel Borkmann "memlock:\t%llu\n" 1808492ecee8SAlexei Starovoitov "prog_id:\t%u\n" 1809492ecee8SAlexei Starovoitov "run_time_ns:\t%llu\n" 1810492ecee8SAlexei Starovoitov "run_cnt:\t%llu\n", 18117bd509e3SDaniel Borkmann prog->type, 18127bd509e3SDaniel Borkmann prog->jited, 1813f1f7714eSDaniel Borkmann prog_tag, 18144316b409SDaniel Borkmann prog->pages * 1ULL << PAGE_SHIFT, 1815492ecee8SAlexei Starovoitov prog->aux->id, 1816492ecee8SAlexei Starovoitov stats.nsecs, 1817492ecee8SAlexei Starovoitov stats.cnt); 18187bd509e3SDaniel Borkmann } 18197bd509e3SDaniel Borkmann #endif 18207bd509e3SDaniel Borkmann 1821f66e448cSChenbo Feng const struct file_operations bpf_prog_fops = { 18227bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 18237bd509e3SDaniel Borkmann .show_fdinfo = bpf_prog_show_fdinfo, 18247bd509e3SDaniel Borkmann #endif 182509756af4SAlexei Starovoitov .release = bpf_prog_release, 18266e71b04aSChenbo Feng .read = bpf_dummy_read, 18276e71b04aSChenbo Feng .write = bpf_dummy_write, 182809756af4SAlexei Starovoitov }; 182909756af4SAlexei Starovoitov 1830b2197755SDaniel Borkmann int bpf_prog_new_fd(struct bpf_prog *prog) 1831aa79781bSDaniel Borkmann { 1832afdb09c7SChenbo Feng int ret; 1833afdb09c7SChenbo Feng 1834afdb09c7SChenbo Feng ret = security_bpf_prog(prog); 1835afdb09c7SChenbo Feng if (ret < 0) 1836afdb09c7SChenbo Feng return ret; 1837afdb09c7SChenbo Feng 1838aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 1839aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC); 1840aa79781bSDaniel Borkmann } 1841aa79781bSDaniel Borkmann 1842113214beSDaniel Borkmann static struct bpf_prog *____bpf_prog_get(struct fd f) 184309756af4SAlexei Starovoitov { 184409756af4SAlexei Starovoitov if (!f.file) 184509756af4SAlexei Starovoitov return ERR_PTR(-EBADF); 184609756af4SAlexei Starovoitov if (f.file->f_op != &bpf_prog_fops) { 184709756af4SAlexei Starovoitov fdput(f); 184809756af4SAlexei Starovoitov return ERR_PTR(-EINVAL); 184909756af4SAlexei Starovoitov } 185009756af4SAlexei Starovoitov 1851c2101297SDaniel Borkmann return f.file->private_data; 185209756af4SAlexei Starovoitov } 185309756af4SAlexei Starovoitov 185485192dbfSAndrii Nakryiko void bpf_prog_add(struct bpf_prog *prog, int i) 185592117d84SAlexei Starovoitov { 185685192dbfSAndrii Nakryiko atomic64_add(i, &prog->aux->refcnt); 185792117d84SAlexei Starovoitov } 185859d3656dSBrenden Blanco EXPORT_SYMBOL_GPL(bpf_prog_add); 185959d3656dSBrenden Blanco 1860c540594fSDaniel Borkmann void bpf_prog_sub(struct bpf_prog *prog, int i) 1861c540594fSDaniel Borkmann { 1862c540594fSDaniel Borkmann /* Only to be used for undoing previous bpf_prog_add() in some 1863c540594fSDaniel Borkmann * error path. We still know that another entity in our call 1864c540594fSDaniel Borkmann * path holds a reference to the program, thus atomic_sub() can 1865c540594fSDaniel Borkmann * be safely used in such cases! 1866c540594fSDaniel Borkmann */ 186785192dbfSAndrii Nakryiko WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0); 1868c540594fSDaniel Borkmann } 1869c540594fSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_sub); 1870c540594fSDaniel Borkmann 187185192dbfSAndrii Nakryiko void bpf_prog_inc(struct bpf_prog *prog) 187259d3656dSBrenden Blanco { 187385192dbfSAndrii Nakryiko atomic64_inc(&prog->aux->refcnt); 187459d3656dSBrenden Blanco } 187597bc402dSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_inc); 187692117d84SAlexei Starovoitov 1877b16d9aa4SMartin KaFai Lau /* prog_idr_lock should have been held */ 1878a6f6df69SJohn Fastabend struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 1879b16d9aa4SMartin KaFai Lau { 1880b16d9aa4SMartin KaFai Lau int refold; 1881b16d9aa4SMartin KaFai Lau 188285192dbfSAndrii Nakryiko refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0); 1883b16d9aa4SMartin KaFai Lau 1884b16d9aa4SMartin KaFai Lau if (!refold) 1885b16d9aa4SMartin KaFai Lau return ERR_PTR(-ENOENT); 1886b16d9aa4SMartin KaFai Lau 1887b16d9aa4SMartin KaFai Lau return prog; 1888b16d9aa4SMartin KaFai Lau } 1889a6f6df69SJohn Fastabend EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 1890b16d9aa4SMartin KaFai Lau 1891040ee692SAl Viro bool bpf_prog_get_ok(struct bpf_prog *prog, 1892288b3de5SJakub Kicinski enum bpf_prog_type *attach_type, bool attach_drv) 1893248f346fSJakub Kicinski { 1894288b3de5SJakub Kicinski /* not an attachment, just a refcount inc, always allow */ 1895288b3de5SJakub Kicinski if (!attach_type) 1896288b3de5SJakub Kicinski return true; 1897248f346fSJakub Kicinski 1898248f346fSJakub Kicinski if (prog->type != *attach_type) 1899248f346fSJakub Kicinski return false; 1900288b3de5SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv) 1901248f346fSJakub Kicinski return false; 1902248f346fSJakub Kicinski 1903248f346fSJakub Kicinski return true; 1904248f346fSJakub Kicinski } 1905248f346fSJakub Kicinski 1906248f346fSJakub Kicinski static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, 1907288b3de5SJakub Kicinski bool attach_drv) 190809756af4SAlexei Starovoitov { 190909756af4SAlexei Starovoitov struct fd f = fdget(ufd); 191009756af4SAlexei Starovoitov struct bpf_prog *prog; 191109756af4SAlexei Starovoitov 1912113214beSDaniel Borkmann prog = ____bpf_prog_get(f); 191309756af4SAlexei Starovoitov if (IS_ERR(prog)) 191409756af4SAlexei Starovoitov return prog; 1915288b3de5SJakub Kicinski if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) { 1916113214beSDaniel Borkmann prog = ERR_PTR(-EINVAL); 1917113214beSDaniel Borkmann goto out; 1918113214beSDaniel Borkmann } 191909756af4SAlexei Starovoitov 192085192dbfSAndrii Nakryiko bpf_prog_inc(prog); 1921113214beSDaniel Borkmann out: 192209756af4SAlexei Starovoitov fdput(f); 192309756af4SAlexei Starovoitov return prog; 192409756af4SAlexei Starovoitov } 1925113214beSDaniel Borkmann 1926113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get(u32 ufd) 1927113214beSDaniel Borkmann { 1928288b3de5SJakub Kicinski return __bpf_prog_get(ufd, NULL, false); 1929113214beSDaniel Borkmann } 1930113214beSDaniel Borkmann 1931248f346fSJakub Kicinski struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 1932288b3de5SJakub Kicinski bool attach_drv) 1933248f346fSJakub Kicinski { 19344d220ed0SAlexei Starovoitov return __bpf_prog_get(ufd, &type, attach_drv); 1935248f346fSJakub Kicinski } 19366c8dfe21SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev); 1937248f346fSJakub Kicinski 1938aac3fc32SAndrey Ignatov /* Initially all BPF programs could be loaded w/o specifying 1939aac3fc32SAndrey Ignatov * expected_attach_type. Later for some of them specifying expected_attach_type 1940aac3fc32SAndrey Ignatov * at load time became required so that program could be validated properly. 1941aac3fc32SAndrey Ignatov * Programs of types that are allowed to be loaded both w/ and w/o (for 1942aac3fc32SAndrey Ignatov * backward compatibility) expected_attach_type, should have the default attach 1943aac3fc32SAndrey Ignatov * type assigned to expected_attach_type for the latter case, so that it can be 1944aac3fc32SAndrey Ignatov * validated later at attach time. 1945aac3fc32SAndrey Ignatov * 1946aac3fc32SAndrey Ignatov * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if 1947aac3fc32SAndrey Ignatov * prog type requires it but has some attach types that have to be backward 1948aac3fc32SAndrey Ignatov * compatible. 1949aac3fc32SAndrey Ignatov */ 1950aac3fc32SAndrey Ignatov static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) 1951aac3fc32SAndrey Ignatov { 1952aac3fc32SAndrey Ignatov switch (attr->prog_type) { 1953aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1954aac3fc32SAndrey Ignatov /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't 1955aac3fc32SAndrey Ignatov * exist so checking for non-zero is the way to go here. 1956aac3fc32SAndrey Ignatov */ 1957aac3fc32SAndrey Ignatov if (!attr->expected_attach_type) 1958aac3fc32SAndrey Ignatov attr->expected_attach_type = 1959aac3fc32SAndrey Ignatov BPF_CGROUP_INET_SOCK_CREATE; 1960aac3fc32SAndrey Ignatov break; 1961aac3fc32SAndrey Ignatov } 1962aac3fc32SAndrey Ignatov } 1963aac3fc32SAndrey Ignatov 19645e43f899SAndrey Ignatov static int 1965ccfe29ebSAlexei Starovoitov bpf_prog_load_check_attach(enum bpf_prog_type prog_type, 1966ccfe29ebSAlexei Starovoitov enum bpf_attach_type expected_attach_type, 19675b92a28aSAlexei Starovoitov u32 btf_id, u32 prog_fd) 19685e43f899SAndrey Ignatov { 196927ae7997SMartin KaFai Lau if (btf_id) { 1970c108e3c1SAlexei Starovoitov if (btf_id > BTF_MAX_TYPE) 1971c108e3c1SAlexei Starovoitov return -EINVAL; 197227ae7997SMartin KaFai Lau 197327ae7997SMartin KaFai Lau switch (prog_type) { 197427ae7997SMartin KaFai Lau case BPF_PROG_TYPE_TRACING: 19759e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 197627ae7997SMartin KaFai Lau case BPF_PROG_TYPE_STRUCT_OPS: 1977be8704ffSAlexei Starovoitov case BPF_PROG_TYPE_EXT: 1978c108e3c1SAlexei Starovoitov break; 1979c108e3c1SAlexei Starovoitov default: 1980c108e3c1SAlexei Starovoitov return -EINVAL; 1981c108e3c1SAlexei Starovoitov } 198227ae7997SMartin KaFai Lau } 198327ae7997SMartin KaFai Lau 1984be8704ffSAlexei Starovoitov if (prog_fd && prog_type != BPF_PROG_TYPE_TRACING && 1985be8704ffSAlexei Starovoitov prog_type != BPF_PROG_TYPE_EXT) 198627ae7997SMartin KaFai Lau return -EINVAL; 1987c108e3c1SAlexei Starovoitov 1988c108e3c1SAlexei Starovoitov switch (prog_type) { 1989aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1990aac3fc32SAndrey Ignatov switch (expected_attach_type) { 1991aac3fc32SAndrey Ignatov case BPF_CGROUP_INET_SOCK_CREATE: 1992f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 1993aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1994aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 1995aac3fc32SAndrey Ignatov return 0; 1996aac3fc32SAndrey Ignatov default: 1997aac3fc32SAndrey Ignatov return -EINVAL; 1998aac3fc32SAndrey Ignatov } 19994fbac77dSAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 20004fbac77dSAndrey Ignatov switch (expected_attach_type) { 20014fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 20024fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 2003d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 2004d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 20051b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 20061b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 20071b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 20081b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 20091cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 20101cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 2011983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG: 2012983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG: 20135e43f899SAndrey Ignatov return 0; 20144fbac77dSAndrey Ignatov default: 20154fbac77dSAndrey Ignatov return -EINVAL; 20164fbac77dSAndrey Ignatov } 20175cf1e914Sbrakmo case BPF_PROG_TYPE_CGROUP_SKB: 20185cf1e914Sbrakmo switch (expected_attach_type) { 20195cf1e914Sbrakmo case BPF_CGROUP_INET_INGRESS: 20205cf1e914Sbrakmo case BPF_CGROUP_INET_EGRESS: 20215cf1e914Sbrakmo return 0; 20225cf1e914Sbrakmo default: 20235cf1e914Sbrakmo return -EINVAL; 20245cf1e914Sbrakmo } 20250d01da6aSStanislav Fomichev case BPF_PROG_TYPE_CGROUP_SOCKOPT: 20260d01da6aSStanislav Fomichev switch (expected_attach_type) { 20270d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT: 20280d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT: 20290d01da6aSStanislav Fomichev return 0; 20300d01da6aSStanislav Fomichev default: 20310d01da6aSStanislav Fomichev return -EINVAL; 20320d01da6aSStanislav Fomichev } 2033e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP: 2034e9ddbb77SJakub Sitnicki if (expected_attach_type == BPF_SK_LOOKUP) 2035e9ddbb77SJakub Sitnicki return 0; 2036e9ddbb77SJakub Sitnicki return -EINVAL; 2037be8704ffSAlexei Starovoitov case BPF_PROG_TYPE_EXT: 2038be8704ffSAlexei Starovoitov if (expected_attach_type) 2039be8704ffSAlexei Starovoitov return -EINVAL; 2040be8704ffSAlexei Starovoitov /* fallthrough */ 20414fbac77dSAndrey Ignatov default: 20424fbac77dSAndrey Ignatov return 0; 20434fbac77dSAndrey Ignatov } 20445e43f899SAndrey Ignatov } 20455e43f899SAndrey Ignatov 20462c78ee89SAlexei Starovoitov static bool is_net_admin_prog_type(enum bpf_prog_type prog_type) 20472c78ee89SAlexei Starovoitov { 20482c78ee89SAlexei Starovoitov switch (prog_type) { 20492c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SCHED_CLS: 20502c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SCHED_ACT: 20512c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_XDP: 20522c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_IN: 20532c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_OUT: 20542c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_XMIT: 20552c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_SEG6LOCAL: 20562c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_SKB: 20572c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_MSG: 20582c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LIRC_MODE2: 20592c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_FLOW_DISSECTOR: 20602c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_DEVICE: 20612c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCK: 20622c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 20632c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCKOPT: 20642c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SYSCTL: 20652c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SOCK_OPS: 20662c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_EXT: /* extends any prog */ 20672c78ee89SAlexei Starovoitov return true; 20682c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SKB: 20692c78ee89SAlexei Starovoitov /* always unpriv */ 20702c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_REUSEPORT: 20712c78ee89SAlexei Starovoitov /* equivalent to SOCKET_FILTER. need CAP_BPF only */ 20722c78ee89SAlexei Starovoitov default: 20732c78ee89SAlexei Starovoitov return false; 20742c78ee89SAlexei Starovoitov } 20752c78ee89SAlexei Starovoitov } 20762c78ee89SAlexei Starovoitov 20772c78ee89SAlexei Starovoitov static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) 20782c78ee89SAlexei Starovoitov { 20792c78ee89SAlexei Starovoitov switch (prog_type) { 20802c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_KPROBE: 20812c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_TRACEPOINT: 20822c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_PERF_EVENT: 20832c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_RAW_TRACEPOINT: 20842c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 20852c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_TRACING: 20862c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LSM: 20872c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_STRUCT_OPS: /* has access to struct sock */ 20882c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_EXT: /* extends any prog */ 20892c78ee89SAlexei Starovoitov return true; 20902c78ee89SAlexei Starovoitov default: 20912c78ee89SAlexei Starovoitov return false; 20922c78ee89SAlexei Starovoitov } 20932c78ee89SAlexei Starovoitov } 20942c78ee89SAlexei Starovoitov 209509756af4SAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 20965b92a28aSAlexei Starovoitov #define BPF_PROG_LOAD_LAST_FIELD attach_prog_fd 209709756af4SAlexei Starovoitov 2098838e9690SYonghong Song static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) 209909756af4SAlexei Starovoitov { 210009756af4SAlexei Starovoitov enum bpf_prog_type type = attr->prog_type; 210109756af4SAlexei Starovoitov struct bpf_prog *prog; 210209756af4SAlexei Starovoitov int err; 210309756af4SAlexei Starovoitov char license[128]; 210409756af4SAlexei Starovoitov bool is_gpl; 210509756af4SAlexei Starovoitov 210609756af4SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_LOAD)) 210709756af4SAlexei Starovoitov return -EINVAL; 210809756af4SAlexei Starovoitov 2109c240eff6SJiong Wang if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | 2110c240eff6SJiong Wang BPF_F_ANY_ALIGNMENT | 211110d274e8SAlexei Starovoitov BPF_F_TEST_STATE_FREQ | 21121e6c62a8SAlexei Starovoitov BPF_F_SLEEPABLE | 2113c240eff6SJiong Wang BPF_F_TEST_RND_HI32)) 2114e07b98d9SDavid S. Miller return -EINVAL; 2115e07b98d9SDavid S. Miller 2116e9ee9efcSDavid Miller if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && 2117e9ee9efcSDavid Miller (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && 21182c78ee89SAlexei Starovoitov !bpf_capable()) 2119e9ee9efcSDavid Miller return -EPERM; 2120e9ee9efcSDavid Miller 212109756af4SAlexei Starovoitov /* copy eBPF program license from user space */ 2122535e7b4bSMickaël Salaün if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 212309756af4SAlexei Starovoitov sizeof(license) - 1) < 0) 212409756af4SAlexei Starovoitov return -EFAULT; 212509756af4SAlexei Starovoitov license[sizeof(license) - 1] = 0; 212609756af4SAlexei Starovoitov 212709756af4SAlexei Starovoitov /* eBPF programs must be GPL compatible to use GPL-ed functions */ 212809756af4SAlexei Starovoitov is_gpl = license_is_gpl_compatible(license); 212909756af4SAlexei Starovoitov 2130c04c0d2bSAlexei Starovoitov if (attr->insn_cnt == 0 || 21312c78ee89SAlexei Starovoitov attr->insn_cnt > (bpf_capable() ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) 2132ef0915caSDaniel Borkmann return -E2BIG; 213380b7d819SChenbo Feng if (type != BPF_PROG_TYPE_SOCKET_FILTER && 213480b7d819SChenbo Feng type != BPF_PROG_TYPE_CGROUP_SKB && 21352c78ee89SAlexei Starovoitov !bpf_capable()) 21362c78ee89SAlexei Starovoitov return -EPERM; 21372c78ee89SAlexei Starovoitov 2138b338cb92SMaciej Żenczykowski if (is_net_admin_prog_type(type) && !capable(CAP_NET_ADMIN) && !capable(CAP_SYS_ADMIN)) 21392c78ee89SAlexei Starovoitov return -EPERM; 21402c78ee89SAlexei Starovoitov if (is_perfmon_prog_type(type) && !perfmon_capable()) 21411be7f75dSAlexei Starovoitov return -EPERM; 21421be7f75dSAlexei Starovoitov 2143aac3fc32SAndrey Ignatov bpf_prog_load_fixup_attach_type(attr); 2144ccfe29ebSAlexei Starovoitov if (bpf_prog_load_check_attach(type, attr->expected_attach_type, 21455b92a28aSAlexei Starovoitov attr->attach_btf_id, 21465b92a28aSAlexei Starovoitov attr->attach_prog_fd)) 21475e43f899SAndrey Ignatov return -EINVAL; 21485e43f899SAndrey Ignatov 214909756af4SAlexei Starovoitov /* plain bpf_prog allocation */ 215009756af4SAlexei Starovoitov prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 215109756af4SAlexei Starovoitov if (!prog) 215209756af4SAlexei Starovoitov return -ENOMEM; 215309756af4SAlexei Starovoitov 21545e43f899SAndrey Ignatov prog->expected_attach_type = attr->expected_attach_type; 2155ccfe29ebSAlexei Starovoitov prog->aux->attach_btf_id = attr->attach_btf_id; 21565b92a28aSAlexei Starovoitov if (attr->attach_prog_fd) { 21575b92a28aSAlexei Starovoitov struct bpf_prog *tgt_prog; 21585b92a28aSAlexei Starovoitov 21595b92a28aSAlexei Starovoitov tgt_prog = bpf_prog_get(attr->attach_prog_fd); 21605b92a28aSAlexei Starovoitov if (IS_ERR(tgt_prog)) { 21615b92a28aSAlexei Starovoitov err = PTR_ERR(tgt_prog); 21625b92a28aSAlexei Starovoitov goto free_prog_nouncharge; 21635b92a28aSAlexei Starovoitov } 21645b92a28aSAlexei Starovoitov prog->aux->linked_prog = tgt_prog; 21655b92a28aSAlexei Starovoitov } 21665e43f899SAndrey Ignatov 21679a18eedbSJakub Kicinski prog->aux->offload_requested = !!attr->prog_ifindex; 21681e6c62a8SAlexei Starovoitov prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE; 21699a18eedbSJakub Kicinski 2170afdb09c7SChenbo Feng err = security_bpf_prog_alloc(prog->aux); 2171aaac3ba9SAlexei Starovoitov if (err) 2172aaac3ba9SAlexei Starovoitov goto free_prog_nouncharge; 2173aaac3ba9SAlexei Starovoitov 2174afdb09c7SChenbo Feng err = bpf_prog_charge_memlock(prog); 2175afdb09c7SChenbo Feng if (err) 2176afdb09c7SChenbo Feng goto free_prog_sec; 2177afdb09c7SChenbo Feng 217809756af4SAlexei Starovoitov prog->len = attr->insn_cnt; 217909756af4SAlexei Starovoitov 218009756af4SAlexei Starovoitov err = -EFAULT; 2181535e7b4bSMickaël Salaün if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 2182aafe6ae9SDaniel Borkmann bpf_prog_insn_size(prog)) != 0) 218309756af4SAlexei Starovoitov goto free_prog; 218409756af4SAlexei Starovoitov 218509756af4SAlexei Starovoitov prog->orig_prog = NULL; 2186a91263d5SDaniel Borkmann prog->jited = 0; 218709756af4SAlexei Starovoitov 218885192dbfSAndrii Nakryiko atomic64_set(&prog->aux->refcnt, 1); 2189a91263d5SDaniel Borkmann prog->gpl_compatible = is_gpl ? 1 : 0; 219009756af4SAlexei Starovoitov 21919a18eedbSJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 2192ab3f0063SJakub Kicinski err = bpf_prog_offload_init(prog, attr); 2193ab3f0063SJakub Kicinski if (err) 2194ab3f0063SJakub Kicinski goto free_prog; 2195ab3f0063SJakub Kicinski } 2196ab3f0063SJakub Kicinski 219709756af4SAlexei Starovoitov /* find program type: socket_filter vs tracing_filter */ 219809756af4SAlexei Starovoitov err = find_prog_type(type, prog); 219909756af4SAlexei Starovoitov if (err < 0) 220009756af4SAlexei Starovoitov goto free_prog; 220109756af4SAlexei Starovoitov 22029285ec4cSJason A. Donenfeld prog->aux->load_time = ktime_get_boottime_ns(); 22038e7ae251SMartin KaFai Lau err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, 22048e7ae251SMartin KaFai Lau sizeof(attr->prog_name)); 22058e7ae251SMartin KaFai Lau if (err < 0) 2206cb4d2b3fSMartin KaFai Lau goto free_prog; 2207cb4d2b3fSMartin KaFai Lau 220809756af4SAlexei Starovoitov /* run eBPF verifier */ 2209838e9690SYonghong Song err = bpf_check(&prog, attr, uattr); 221009756af4SAlexei Starovoitov if (err < 0) 221109756af4SAlexei Starovoitov goto free_used_maps; 221209756af4SAlexei Starovoitov 2213d1c55ab5SDaniel Borkmann prog = bpf_prog_select_runtime(prog, &err); 221404fd61abSAlexei Starovoitov if (err < 0) 221504fd61abSAlexei Starovoitov goto free_used_maps; 221609756af4SAlexei Starovoitov 2217dc4bb0e2SMartin KaFai Lau err = bpf_prog_alloc_id(prog); 2218dc4bb0e2SMartin KaFai Lau if (err) 2219dc4bb0e2SMartin KaFai Lau goto free_used_maps; 2220dc4bb0e2SMartin KaFai Lau 2221c751798aSDaniel Borkmann /* Upon success of bpf_prog_alloc_id(), the BPF prog is 2222c751798aSDaniel Borkmann * effectively publicly exposed. However, retrieving via 2223c751798aSDaniel Borkmann * bpf_prog_get_fd_by_id() will take another reference, 2224c751798aSDaniel Borkmann * therefore it cannot be gone underneath us. 2225c751798aSDaniel Borkmann * 2226c751798aSDaniel Borkmann * Only for the time /after/ successful bpf_prog_new_fd() 2227c751798aSDaniel Borkmann * and before returning to userspace, we might just hold 2228c751798aSDaniel Borkmann * one reference and any parallel close on that fd could 2229c751798aSDaniel Borkmann * rip everything out. Hence, below notifications must 2230c751798aSDaniel Borkmann * happen before bpf_prog_new_fd(). 2231c751798aSDaniel Borkmann * 2232c751798aSDaniel Borkmann * Also, any failure handling from this point onwards must 2233c751798aSDaniel Borkmann * be using bpf_prog_put() given the program is exposed. 2234b16d9aa4SMartin KaFai Lau */ 223574451e66SDaniel Borkmann bpf_prog_kallsyms_add(prog); 22366ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); 2237bae141f5SDaniel Borkmann bpf_audit_prog(prog, BPF_AUDIT_LOAD); 2238c751798aSDaniel Borkmann 2239c751798aSDaniel Borkmann err = bpf_prog_new_fd(prog); 2240c751798aSDaniel Borkmann if (err < 0) 2241c751798aSDaniel Borkmann bpf_prog_put(prog); 224209756af4SAlexei Starovoitov return err; 224309756af4SAlexei Starovoitov 224409756af4SAlexei Starovoitov free_used_maps: 2245cd7455f1SDaniel Borkmann /* In case we have subprogs, we need to wait for a grace 2246cd7455f1SDaniel Borkmann * period before we can tear down JIT memory since symbols 2247cd7455f1SDaniel Borkmann * are already exposed under kallsyms. 2248cd7455f1SDaniel Borkmann */ 2249cd7455f1SDaniel Borkmann __bpf_prog_put_noref(prog, prog->aux->func_cnt); 2250cd7455f1SDaniel Borkmann return err; 225109756af4SAlexei Starovoitov free_prog: 2252aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(prog); 2253afdb09c7SChenbo Feng free_prog_sec: 2254afdb09c7SChenbo Feng security_bpf_prog_free(prog->aux); 2255aaac3ba9SAlexei Starovoitov free_prog_nouncharge: 225609756af4SAlexei Starovoitov bpf_prog_free(prog); 225709756af4SAlexei Starovoitov return err; 225809756af4SAlexei Starovoitov } 225909756af4SAlexei Starovoitov 22606e71b04aSChenbo Feng #define BPF_OBJ_LAST_FIELD file_flags 2261b2197755SDaniel Borkmann 2262b2197755SDaniel Borkmann static int bpf_obj_pin(const union bpf_attr *attr) 2263b2197755SDaniel Borkmann { 22646e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0) 2265b2197755SDaniel Borkmann return -EINVAL; 2266b2197755SDaniel Borkmann 2267535e7b4bSMickaël Salaün return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 2268b2197755SDaniel Borkmann } 2269b2197755SDaniel Borkmann 2270b2197755SDaniel Borkmann static int bpf_obj_get(const union bpf_attr *attr) 2271b2197755SDaniel Borkmann { 22726e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || 22736e71b04aSChenbo Feng attr->file_flags & ~BPF_OBJ_FLAG_MASK) 2274b2197755SDaniel Borkmann return -EINVAL; 2275b2197755SDaniel Borkmann 22766e71b04aSChenbo Feng return bpf_obj_get_user(u64_to_user_ptr(attr->pathname), 22776e71b04aSChenbo Feng attr->file_flags); 2278b2197755SDaniel Borkmann } 2279b2197755SDaniel Borkmann 2280f2e10bffSAndrii Nakryiko void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 2281a3b80e10SAndrii Nakryiko const struct bpf_link_ops *ops, struct bpf_prog *prog) 228270ed506cSAndrii Nakryiko { 228370ed506cSAndrii Nakryiko atomic64_set(&link->refcnt, 1); 2284f2e10bffSAndrii Nakryiko link->type = type; 2285a3b80e10SAndrii Nakryiko link->id = 0; 228670ed506cSAndrii Nakryiko link->ops = ops; 228770ed506cSAndrii Nakryiko link->prog = prog; 228870ed506cSAndrii Nakryiko } 228970ed506cSAndrii Nakryiko 2290a3b80e10SAndrii Nakryiko static void bpf_link_free_id(int id) 2291a3b80e10SAndrii Nakryiko { 2292a3b80e10SAndrii Nakryiko if (!id) 2293a3b80e10SAndrii Nakryiko return; 2294a3b80e10SAndrii Nakryiko 2295a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 2296a3b80e10SAndrii Nakryiko idr_remove(&link_idr, id); 2297a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 2298a3b80e10SAndrii Nakryiko } 2299a3b80e10SAndrii Nakryiko 230098868668SAndrii Nakryiko /* Clean up bpf_link and corresponding anon_inode file and FD. After 230198868668SAndrii Nakryiko * anon_inode is created, bpf_link can't be just kfree()'d due to deferred 2302a3b80e10SAndrii Nakryiko * anon_inode's release() call. This helper marksbpf_link as 2303a3b80e10SAndrii Nakryiko * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt 2304a3b80e10SAndrii Nakryiko * is not decremented, it's the responsibility of a calling code that failed 2305a3b80e10SAndrii Nakryiko * to complete bpf_link initialization. 230698868668SAndrii Nakryiko */ 2307a3b80e10SAndrii Nakryiko void bpf_link_cleanup(struct bpf_link_primer *primer) 2308babf3164SAndrii Nakryiko { 2309a3b80e10SAndrii Nakryiko primer->link->prog = NULL; 2310a3b80e10SAndrii Nakryiko bpf_link_free_id(primer->id); 2311a3b80e10SAndrii Nakryiko fput(primer->file); 2312a3b80e10SAndrii Nakryiko put_unused_fd(primer->fd); 2313babf3164SAndrii Nakryiko } 2314babf3164SAndrii Nakryiko 231570ed506cSAndrii Nakryiko void bpf_link_inc(struct bpf_link *link) 231670ed506cSAndrii Nakryiko { 231770ed506cSAndrii Nakryiko atomic64_inc(&link->refcnt); 231870ed506cSAndrii Nakryiko } 231970ed506cSAndrii Nakryiko 232070ed506cSAndrii Nakryiko /* bpf_link_free is guaranteed to be called from process context */ 232170ed506cSAndrii Nakryiko static void bpf_link_free(struct bpf_link *link) 232270ed506cSAndrii Nakryiko { 2323a3b80e10SAndrii Nakryiko bpf_link_free_id(link->id); 2324babf3164SAndrii Nakryiko if (link->prog) { 2325babf3164SAndrii Nakryiko /* detach BPF program, clean up used resources */ 232670ed506cSAndrii Nakryiko link->ops->release(link); 2327babf3164SAndrii Nakryiko bpf_prog_put(link->prog); 2328babf3164SAndrii Nakryiko } 2329babf3164SAndrii Nakryiko /* free bpf_link and its containing memory */ 2330babf3164SAndrii Nakryiko link->ops->dealloc(link); 233170ed506cSAndrii Nakryiko } 233270ed506cSAndrii Nakryiko 233370ed506cSAndrii Nakryiko static void bpf_link_put_deferred(struct work_struct *work) 233470ed506cSAndrii Nakryiko { 233570ed506cSAndrii Nakryiko struct bpf_link *link = container_of(work, struct bpf_link, work); 233670ed506cSAndrii Nakryiko 233770ed506cSAndrii Nakryiko bpf_link_free(link); 233870ed506cSAndrii Nakryiko } 233970ed506cSAndrii Nakryiko 234070ed506cSAndrii Nakryiko /* bpf_link_put can be called from atomic context, but ensures that resources 234170ed506cSAndrii Nakryiko * are freed from process context 234270ed506cSAndrii Nakryiko */ 234370ed506cSAndrii Nakryiko void bpf_link_put(struct bpf_link *link) 234470ed506cSAndrii Nakryiko { 234570ed506cSAndrii Nakryiko if (!atomic64_dec_and_test(&link->refcnt)) 234670ed506cSAndrii Nakryiko return; 234770ed506cSAndrii Nakryiko 234870ed506cSAndrii Nakryiko if (in_atomic()) { 234970ed506cSAndrii Nakryiko INIT_WORK(&link->work, bpf_link_put_deferred); 235070ed506cSAndrii Nakryiko schedule_work(&link->work); 235170ed506cSAndrii Nakryiko } else { 235270ed506cSAndrii Nakryiko bpf_link_free(link); 235370ed506cSAndrii Nakryiko } 235470ed506cSAndrii Nakryiko } 235570ed506cSAndrii Nakryiko 235670ed506cSAndrii Nakryiko static int bpf_link_release(struct inode *inode, struct file *filp) 235770ed506cSAndrii Nakryiko { 235870ed506cSAndrii Nakryiko struct bpf_link *link = filp->private_data; 235970ed506cSAndrii Nakryiko 236070ed506cSAndrii Nakryiko bpf_link_put(link); 2361fec56f58SAlexei Starovoitov return 0; 2362fec56f58SAlexei Starovoitov } 2363fec56f58SAlexei Starovoitov 236470ed506cSAndrii Nakryiko #ifdef CONFIG_PROC_FS 2365f2e10bffSAndrii Nakryiko #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) 2366f2e10bffSAndrii Nakryiko #define BPF_MAP_TYPE(_id, _ops) 2367f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) [_id] = #_name, 2368f2e10bffSAndrii Nakryiko static const char *bpf_link_type_strs[] = { 2369f2e10bffSAndrii Nakryiko [BPF_LINK_TYPE_UNSPEC] = "<invalid>", 2370f2e10bffSAndrii Nakryiko #include <linux/bpf_types.h> 2371f2e10bffSAndrii Nakryiko }; 2372f2e10bffSAndrii Nakryiko #undef BPF_PROG_TYPE 2373f2e10bffSAndrii Nakryiko #undef BPF_MAP_TYPE 2374f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 237570ed506cSAndrii Nakryiko 237670ed506cSAndrii Nakryiko static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) 237770ed506cSAndrii Nakryiko { 237870ed506cSAndrii Nakryiko const struct bpf_link *link = filp->private_data; 237970ed506cSAndrii Nakryiko const struct bpf_prog *prog = link->prog; 238070ed506cSAndrii Nakryiko char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 238170ed506cSAndrii Nakryiko 238270ed506cSAndrii Nakryiko bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 238370ed506cSAndrii Nakryiko seq_printf(m, 238470ed506cSAndrii Nakryiko "link_type:\t%s\n" 2385a3b80e10SAndrii Nakryiko "link_id:\t%u\n" 238670ed506cSAndrii Nakryiko "prog_tag:\t%s\n" 238770ed506cSAndrii Nakryiko "prog_id:\t%u\n", 2388f2e10bffSAndrii Nakryiko bpf_link_type_strs[link->type], 2389a3b80e10SAndrii Nakryiko link->id, 239070ed506cSAndrii Nakryiko prog_tag, 239170ed506cSAndrii Nakryiko prog->aux->id); 2392f2e10bffSAndrii Nakryiko if (link->ops->show_fdinfo) 2393f2e10bffSAndrii Nakryiko link->ops->show_fdinfo(link, m); 239470ed506cSAndrii Nakryiko } 239570ed506cSAndrii Nakryiko #endif 239670ed506cSAndrii Nakryiko 23976f302bfbSZou Wei static const struct file_operations bpf_link_fops = { 239870ed506cSAndrii Nakryiko #ifdef CONFIG_PROC_FS 239970ed506cSAndrii Nakryiko .show_fdinfo = bpf_link_show_fdinfo, 240070ed506cSAndrii Nakryiko #endif 240170ed506cSAndrii Nakryiko .release = bpf_link_release, 2402fec56f58SAlexei Starovoitov .read = bpf_dummy_read, 2403fec56f58SAlexei Starovoitov .write = bpf_dummy_write, 2404fec56f58SAlexei Starovoitov }; 2405fec56f58SAlexei Starovoitov 2406a3b80e10SAndrii Nakryiko static int bpf_link_alloc_id(struct bpf_link *link) 240770ed506cSAndrii Nakryiko { 2408a3b80e10SAndrii Nakryiko int id; 2409a3b80e10SAndrii Nakryiko 2410a3b80e10SAndrii Nakryiko idr_preload(GFP_KERNEL); 2411a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 2412a3b80e10SAndrii Nakryiko id = idr_alloc_cyclic(&link_idr, link, 1, INT_MAX, GFP_ATOMIC); 2413a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 2414a3b80e10SAndrii Nakryiko idr_preload_end(); 2415a3b80e10SAndrii Nakryiko 2416a3b80e10SAndrii Nakryiko return id; 241770ed506cSAndrii Nakryiko } 241870ed506cSAndrii Nakryiko 2419a3b80e10SAndrii Nakryiko /* Prepare bpf_link to be exposed to user-space by allocating anon_inode file, 2420a3b80e10SAndrii Nakryiko * reserving unused FD and allocating ID from link_idr. This is to be paired 2421a3b80e10SAndrii Nakryiko * with bpf_link_settle() to install FD and ID and expose bpf_link to 2422a3b80e10SAndrii Nakryiko * user-space, if bpf_link is successfully attached. If not, bpf_link and 2423a3b80e10SAndrii Nakryiko * pre-allocated resources are to be freed with bpf_cleanup() call. All the 2424a3b80e10SAndrii Nakryiko * transient state is passed around in struct bpf_link_primer. 2425a3b80e10SAndrii Nakryiko * This is preferred way to create and initialize bpf_link, especially when 2426a3b80e10SAndrii Nakryiko * there are complicated and expensive operations inbetween creating bpf_link 2427a3b80e10SAndrii Nakryiko * itself and attaching it to BPF hook. By using bpf_link_prime() and 2428a3b80e10SAndrii Nakryiko * bpf_link_settle() kernel code using bpf_link doesn't have to perform 2429a3b80e10SAndrii Nakryiko * expensive (and potentially failing) roll back operations in a rare case 2430a3b80e10SAndrii Nakryiko * that file, FD, or ID can't be allocated. 2431babf3164SAndrii Nakryiko */ 2432a3b80e10SAndrii Nakryiko int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer) 2433babf3164SAndrii Nakryiko { 2434babf3164SAndrii Nakryiko struct file *file; 2435a3b80e10SAndrii Nakryiko int fd, id; 2436babf3164SAndrii Nakryiko 2437babf3164SAndrii Nakryiko fd = get_unused_fd_flags(O_CLOEXEC); 2438babf3164SAndrii Nakryiko if (fd < 0) 2439a3b80e10SAndrii Nakryiko return fd; 2440babf3164SAndrii Nakryiko 2441babf3164SAndrii Nakryiko 2442a3b80e10SAndrii Nakryiko id = bpf_link_alloc_id(link); 2443a3b80e10SAndrii Nakryiko if (id < 0) { 2444a3b80e10SAndrii Nakryiko put_unused_fd(fd); 2445a3b80e10SAndrii Nakryiko return id; 2446a3b80e10SAndrii Nakryiko } 2447babf3164SAndrii Nakryiko 2448babf3164SAndrii Nakryiko file = anon_inode_getfile("bpf_link", &bpf_link_fops, link, O_CLOEXEC); 2449babf3164SAndrii Nakryiko if (IS_ERR(file)) { 2450138c6767SAndrii Nakryiko bpf_link_free_id(id); 2451babf3164SAndrii Nakryiko put_unused_fd(fd); 2452138c6767SAndrii Nakryiko return PTR_ERR(file); 2453babf3164SAndrii Nakryiko } 2454babf3164SAndrii Nakryiko 2455a3b80e10SAndrii Nakryiko primer->link = link; 2456a3b80e10SAndrii Nakryiko primer->file = file; 2457a3b80e10SAndrii Nakryiko primer->fd = fd; 2458a3b80e10SAndrii Nakryiko primer->id = id; 2459a3b80e10SAndrii Nakryiko return 0; 2460a3b80e10SAndrii Nakryiko } 2461a3b80e10SAndrii Nakryiko 2462a3b80e10SAndrii Nakryiko int bpf_link_settle(struct bpf_link_primer *primer) 2463a3b80e10SAndrii Nakryiko { 2464a3b80e10SAndrii Nakryiko /* make bpf_link fetchable by ID */ 2465a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 2466a3b80e10SAndrii Nakryiko primer->link->id = primer->id; 2467a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 2468a3b80e10SAndrii Nakryiko /* make bpf_link fetchable by FD */ 2469a3b80e10SAndrii Nakryiko fd_install(primer->fd, primer->file); 2470a3b80e10SAndrii Nakryiko /* pass through installed FD */ 2471a3b80e10SAndrii Nakryiko return primer->fd; 2472a3b80e10SAndrii Nakryiko } 2473a3b80e10SAndrii Nakryiko 2474a3b80e10SAndrii Nakryiko int bpf_link_new_fd(struct bpf_link *link) 2475a3b80e10SAndrii Nakryiko { 2476a3b80e10SAndrii Nakryiko return anon_inode_getfd("bpf-link", &bpf_link_fops, link, O_CLOEXEC); 2477babf3164SAndrii Nakryiko } 2478babf3164SAndrii Nakryiko 247970ed506cSAndrii Nakryiko struct bpf_link *bpf_link_get_from_fd(u32 ufd) 248070ed506cSAndrii Nakryiko { 248170ed506cSAndrii Nakryiko struct fd f = fdget(ufd); 248270ed506cSAndrii Nakryiko struct bpf_link *link; 248370ed506cSAndrii Nakryiko 248470ed506cSAndrii Nakryiko if (!f.file) 248570ed506cSAndrii Nakryiko return ERR_PTR(-EBADF); 248670ed506cSAndrii Nakryiko if (f.file->f_op != &bpf_link_fops) { 248770ed506cSAndrii Nakryiko fdput(f); 248870ed506cSAndrii Nakryiko return ERR_PTR(-EINVAL); 248970ed506cSAndrii Nakryiko } 249070ed506cSAndrii Nakryiko 249170ed506cSAndrii Nakryiko link = f.file->private_data; 249270ed506cSAndrii Nakryiko bpf_link_inc(link); 249370ed506cSAndrii Nakryiko fdput(f); 249470ed506cSAndrii Nakryiko 249570ed506cSAndrii Nakryiko return link; 249670ed506cSAndrii Nakryiko } 249770ed506cSAndrii Nakryiko 249870ed506cSAndrii Nakryiko struct bpf_tracing_link { 249970ed506cSAndrii Nakryiko struct bpf_link link; 2500f2e10bffSAndrii Nakryiko enum bpf_attach_type attach_type; 250170ed506cSAndrii Nakryiko }; 250270ed506cSAndrii Nakryiko 250370ed506cSAndrii Nakryiko static void bpf_tracing_link_release(struct bpf_link *link) 250470ed506cSAndrii Nakryiko { 2505babf3164SAndrii Nakryiko WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog)); 2506babf3164SAndrii Nakryiko } 2507babf3164SAndrii Nakryiko 2508babf3164SAndrii Nakryiko static void bpf_tracing_link_dealloc(struct bpf_link *link) 2509babf3164SAndrii Nakryiko { 251070ed506cSAndrii Nakryiko struct bpf_tracing_link *tr_link = 251170ed506cSAndrii Nakryiko container_of(link, struct bpf_tracing_link, link); 251270ed506cSAndrii Nakryiko 251370ed506cSAndrii Nakryiko kfree(tr_link); 251470ed506cSAndrii Nakryiko } 251570ed506cSAndrii Nakryiko 2516f2e10bffSAndrii Nakryiko static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link, 2517f2e10bffSAndrii Nakryiko struct seq_file *seq) 2518f2e10bffSAndrii Nakryiko { 2519f2e10bffSAndrii Nakryiko struct bpf_tracing_link *tr_link = 2520f2e10bffSAndrii Nakryiko container_of(link, struct bpf_tracing_link, link); 2521f2e10bffSAndrii Nakryiko 2522f2e10bffSAndrii Nakryiko seq_printf(seq, 2523f2e10bffSAndrii Nakryiko "attach_type:\t%d\n", 2524f2e10bffSAndrii Nakryiko tr_link->attach_type); 2525f2e10bffSAndrii Nakryiko } 2526f2e10bffSAndrii Nakryiko 2527f2e10bffSAndrii Nakryiko static int bpf_tracing_link_fill_link_info(const struct bpf_link *link, 2528f2e10bffSAndrii Nakryiko struct bpf_link_info *info) 2529f2e10bffSAndrii Nakryiko { 2530f2e10bffSAndrii Nakryiko struct bpf_tracing_link *tr_link = 2531f2e10bffSAndrii Nakryiko container_of(link, struct bpf_tracing_link, link); 2532f2e10bffSAndrii Nakryiko 2533f2e10bffSAndrii Nakryiko info->tracing.attach_type = tr_link->attach_type; 2534f2e10bffSAndrii Nakryiko 2535f2e10bffSAndrii Nakryiko return 0; 2536f2e10bffSAndrii Nakryiko } 2537f2e10bffSAndrii Nakryiko 253870ed506cSAndrii Nakryiko static const struct bpf_link_ops bpf_tracing_link_lops = { 253970ed506cSAndrii Nakryiko .release = bpf_tracing_link_release, 2540babf3164SAndrii Nakryiko .dealloc = bpf_tracing_link_dealloc, 2541f2e10bffSAndrii Nakryiko .show_fdinfo = bpf_tracing_link_show_fdinfo, 2542f2e10bffSAndrii Nakryiko .fill_link_info = bpf_tracing_link_fill_link_info, 254370ed506cSAndrii Nakryiko }; 254470ed506cSAndrii Nakryiko 2545fec56f58SAlexei Starovoitov static int bpf_tracing_prog_attach(struct bpf_prog *prog) 2546fec56f58SAlexei Starovoitov { 2547a3b80e10SAndrii Nakryiko struct bpf_link_primer link_primer; 254870ed506cSAndrii Nakryiko struct bpf_tracing_link *link; 2549a3b80e10SAndrii Nakryiko int err; 2550fec56f58SAlexei Starovoitov 25519e4e01dfSKP Singh switch (prog->type) { 25529e4e01dfSKP Singh case BPF_PROG_TYPE_TRACING: 2553fec56f58SAlexei Starovoitov if (prog->expected_attach_type != BPF_TRACE_FENTRY && 2554be8704ffSAlexei Starovoitov prog->expected_attach_type != BPF_TRACE_FEXIT && 25559e4e01dfSKP Singh prog->expected_attach_type != BPF_MODIFY_RETURN) { 25569e4e01dfSKP Singh err = -EINVAL; 25579e4e01dfSKP Singh goto out_put_prog; 25589e4e01dfSKP Singh } 25599e4e01dfSKP Singh break; 25609e4e01dfSKP Singh case BPF_PROG_TYPE_EXT: 25619e4e01dfSKP Singh if (prog->expected_attach_type != 0) { 25629e4e01dfSKP Singh err = -EINVAL; 25639e4e01dfSKP Singh goto out_put_prog; 25649e4e01dfSKP Singh } 25659e4e01dfSKP Singh break; 25669e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 25679e4e01dfSKP Singh if (prog->expected_attach_type != BPF_LSM_MAC) { 25689e4e01dfSKP Singh err = -EINVAL; 25699e4e01dfSKP Singh goto out_put_prog; 25709e4e01dfSKP Singh } 25719e4e01dfSKP Singh break; 25729e4e01dfSKP Singh default: 2573fec56f58SAlexei Starovoitov err = -EINVAL; 2574fec56f58SAlexei Starovoitov goto out_put_prog; 2575fec56f58SAlexei Starovoitov } 2576fec56f58SAlexei Starovoitov 257770ed506cSAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER); 257870ed506cSAndrii Nakryiko if (!link) { 257970ed506cSAndrii Nakryiko err = -ENOMEM; 2580fec56f58SAlexei Starovoitov goto out_put_prog; 2581fec56f58SAlexei Starovoitov } 2582f2e10bffSAndrii Nakryiko bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING, 2583f2e10bffSAndrii Nakryiko &bpf_tracing_link_lops, prog); 2584f2e10bffSAndrii Nakryiko link->attach_type = prog->expected_attach_type; 2585fec56f58SAlexei Starovoitov 2586a3b80e10SAndrii Nakryiko err = bpf_link_prime(&link->link, &link_primer); 2587a3b80e10SAndrii Nakryiko if (err) { 2588babf3164SAndrii Nakryiko kfree(link); 2589babf3164SAndrii Nakryiko goto out_put_prog; 259070ed506cSAndrii Nakryiko } 2591babf3164SAndrii Nakryiko 2592babf3164SAndrii Nakryiko err = bpf_trampoline_link_prog(prog); 2593babf3164SAndrii Nakryiko if (err) { 2594a3b80e10SAndrii Nakryiko bpf_link_cleanup(&link_primer); 2595babf3164SAndrii Nakryiko goto out_put_prog; 2596babf3164SAndrii Nakryiko } 2597babf3164SAndrii Nakryiko 2598a3b80e10SAndrii Nakryiko return bpf_link_settle(&link_primer); 2599fec56f58SAlexei Starovoitov out_put_prog: 2600fec56f58SAlexei Starovoitov bpf_prog_put(prog); 2601fec56f58SAlexei Starovoitov return err; 2602fec56f58SAlexei Starovoitov } 2603fec56f58SAlexei Starovoitov 260470ed506cSAndrii Nakryiko struct bpf_raw_tp_link { 260570ed506cSAndrii Nakryiko struct bpf_link link; 2606c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 2607c4f6699dSAlexei Starovoitov }; 2608c4f6699dSAlexei Starovoitov 260970ed506cSAndrii Nakryiko static void bpf_raw_tp_link_release(struct bpf_link *link) 2610c4f6699dSAlexei Starovoitov { 261170ed506cSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 261270ed506cSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 2613c4f6699dSAlexei Starovoitov 261470ed506cSAndrii Nakryiko bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog); 2615a38d1107SMatt Mullins bpf_put_raw_tracepoint(raw_tp->btp); 2616babf3164SAndrii Nakryiko } 2617babf3164SAndrii Nakryiko 2618babf3164SAndrii Nakryiko static void bpf_raw_tp_link_dealloc(struct bpf_link *link) 2619babf3164SAndrii Nakryiko { 2620babf3164SAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 2621babf3164SAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 2622babf3164SAndrii Nakryiko 2623c4f6699dSAlexei Starovoitov kfree(raw_tp); 2624c4f6699dSAlexei Starovoitov } 2625c4f6699dSAlexei Starovoitov 2626f2e10bffSAndrii Nakryiko static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link, 2627f2e10bffSAndrii Nakryiko struct seq_file *seq) 2628f2e10bffSAndrii Nakryiko { 2629f2e10bffSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp_link = 2630f2e10bffSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 2631f2e10bffSAndrii Nakryiko 2632f2e10bffSAndrii Nakryiko seq_printf(seq, 2633f2e10bffSAndrii Nakryiko "tp_name:\t%s\n", 2634f2e10bffSAndrii Nakryiko raw_tp_link->btp->tp->name); 2635f2e10bffSAndrii Nakryiko } 2636f2e10bffSAndrii Nakryiko 2637f2e10bffSAndrii Nakryiko static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link, 2638f2e10bffSAndrii Nakryiko struct bpf_link_info *info) 2639f2e10bffSAndrii Nakryiko { 2640f2e10bffSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp_link = 2641f2e10bffSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 2642f2e10bffSAndrii Nakryiko char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name); 2643f2e10bffSAndrii Nakryiko const char *tp_name = raw_tp_link->btp->tp->name; 2644f2e10bffSAndrii Nakryiko u32 ulen = info->raw_tracepoint.tp_name_len; 2645f2e10bffSAndrii Nakryiko size_t tp_len = strlen(tp_name); 2646f2e10bffSAndrii Nakryiko 2647f2e10bffSAndrii Nakryiko if (ulen && !ubuf) 2648f2e10bffSAndrii Nakryiko return -EINVAL; 2649f2e10bffSAndrii Nakryiko 2650f2e10bffSAndrii Nakryiko info->raw_tracepoint.tp_name_len = tp_len + 1; 2651f2e10bffSAndrii Nakryiko 2652f2e10bffSAndrii Nakryiko if (!ubuf) 2653f2e10bffSAndrii Nakryiko return 0; 2654f2e10bffSAndrii Nakryiko 2655f2e10bffSAndrii Nakryiko if (ulen >= tp_len + 1) { 2656f2e10bffSAndrii Nakryiko if (copy_to_user(ubuf, tp_name, tp_len + 1)) 2657f2e10bffSAndrii Nakryiko return -EFAULT; 2658f2e10bffSAndrii Nakryiko } else { 2659f2e10bffSAndrii Nakryiko char zero = '\0'; 2660f2e10bffSAndrii Nakryiko 2661f2e10bffSAndrii Nakryiko if (copy_to_user(ubuf, tp_name, ulen - 1)) 2662f2e10bffSAndrii Nakryiko return -EFAULT; 2663f2e10bffSAndrii Nakryiko if (put_user(zero, ubuf + ulen - 1)) 2664f2e10bffSAndrii Nakryiko return -EFAULT; 2665f2e10bffSAndrii Nakryiko return -ENOSPC; 2666f2e10bffSAndrii Nakryiko } 2667f2e10bffSAndrii Nakryiko 2668f2e10bffSAndrii Nakryiko return 0; 2669f2e10bffSAndrii Nakryiko } 2670f2e10bffSAndrii Nakryiko 2671a3b80e10SAndrii Nakryiko static const struct bpf_link_ops bpf_raw_tp_link_lops = { 267270ed506cSAndrii Nakryiko .release = bpf_raw_tp_link_release, 2673babf3164SAndrii Nakryiko .dealloc = bpf_raw_tp_link_dealloc, 2674f2e10bffSAndrii Nakryiko .show_fdinfo = bpf_raw_tp_link_show_fdinfo, 2675f2e10bffSAndrii Nakryiko .fill_link_info = bpf_raw_tp_link_fill_link_info, 2676c4f6699dSAlexei Starovoitov }; 2677c4f6699dSAlexei Starovoitov 2678c4f6699dSAlexei Starovoitov #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd 2679c4f6699dSAlexei Starovoitov 2680c4f6699dSAlexei Starovoitov static int bpf_raw_tracepoint_open(const union bpf_attr *attr) 2681c4f6699dSAlexei Starovoitov { 2682a3b80e10SAndrii Nakryiko struct bpf_link_primer link_primer; 2683babf3164SAndrii Nakryiko struct bpf_raw_tp_link *link; 2684c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 2685c4f6699dSAlexei Starovoitov struct bpf_prog *prog; 2686ac4414b5SAlexei Starovoitov const char *tp_name; 2687ac4414b5SAlexei Starovoitov char buf[128]; 2688a3b80e10SAndrii Nakryiko int err; 2689c4f6699dSAlexei Starovoitov 2690ac4414b5SAlexei Starovoitov if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN)) 2691ac4414b5SAlexei Starovoitov return -EINVAL; 2692ac4414b5SAlexei Starovoitov 2693ac4414b5SAlexei Starovoitov prog = bpf_prog_get(attr->raw_tracepoint.prog_fd); 2694ac4414b5SAlexei Starovoitov if (IS_ERR(prog)) 2695ac4414b5SAlexei Starovoitov return PTR_ERR(prog); 2696ac4414b5SAlexei Starovoitov 26979e4e01dfSKP Singh switch (prog->type) { 26989e4e01dfSKP Singh case BPF_PROG_TYPE_TRACING: 26999e4e01dfSKP Singh case BPF_PROG_TYPE_EXT: 27009e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 2701ac4414b5SAlexei Starovoitov if (attr->raw_tracepoint.name) { 2702fec56f58SAlexei Starovoitov /* The attach point for this category of programs 2703fec56f58SAlexei Starovoitov * should be specified via btf_id during program load. 2704ac4414b5SAlexei Starovoitov */ 2705ac4414b5SAlexei Starovoitov err = -EINVAL; 2706ac4414b5SAlexei Starovoitov goto out_put_prog; 2707ac4414b5SAlexei Starovoitov } 27089e4e01dfSKP Singh if (prog->type == BPF_PROG_TYPE_TRACING && 27099e4e01dfSKP Singh prog->expected_attach_type == BPF_TRACE_RAW_TP) { 271038207291SMartin KaFai Lau tp_name = prog->aux->attach_func_name; 27119e4e01dfSKP Singh break; 27129e4e01dfSKP Singh } 2713fec56f58SAlexei Starovoitov return bpf_tracing_prog_attach(prog); 27149e4e01dfSKP Singh case BPF_PROG_TYPE_RAW_TRACEPOINT: 27159e4e01dfSKP Singh case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 2716ac4414b5SAlexei Starovoitov if (strncpy_from_user(buf, 2717ac4414b5SAlexei Starovoitov u64_to_user_ptr(attr->raw_tracepoint.name), 2718ac4414b5SAlexei Starovoitov sizeof(buf) - 1) < 0) { 2719ac4414b5SAlexei Starovoitov err = -EFAULT; 2720ac4414b5SAlexei Starovoitov goto out_put_prog; 2721ac4414b5SAlexei Starovoitov } 2722ac4414b5SAlexei Starovoitov buf[sizeof(buf) - 1] = 0; 2723ac4414b5SAlexei Starovoitov tp_name = buf; 27249e4e01dfSKP Singh break; 27259e4e01dfSKP Singh default: 27269e4e01dfSKP Singh err = -EINVAL; 27279e4e01dfSKP Singh goto out_put_prog; 2728ac4414b5SAlexei Starovoitov } 2729c4f6699dSAlexei Starovoitov 2730a38d1107SMatt Mullins btp = bpf_get_raw_tracepoint(tp_name); 2731ac4414b5SAlexei Starovoitov if (!btp) { 2732ac4414b5SAlexei Starovoitov err = -ENOENT; 2733ac4414b5SAlexei Starovoitov goto out_put_prog; 2734ac4414b5SAlexei Starovoitov } 2735c4f6699dSAlexei Starovoitov 2736babf3164SAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER); 2737babf3164SAndrii Nakryiko if (!link) { 2738a38d1107SMatt Mullins err = -ENOMEM; 2739a38d1107SMatt Mullins goto out_put_btp; 2740a38d1107SMatt Mullins } 2741f2e10bffSAndrii Nakryiko bpf_link_init(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT, 2742f2e10bffSAndrii Nakryiko &bpf_raw_tp_link_lops, prog); 2743babf3164SAndrii Nakryiko link->btp = btp; 2744c4f6699dSAlexei Starovoitov 2745a3b80e10SAndrii Nakryiko err = bpf_link_prime(&link->link, &link_primer); 2746a3b80e10SAndrii Nakryiko if (err) { 2747babf3164SAndrii Nakryiko kfree(link); 2748babf3164SAndrii Nakryiko goto out_put_btp; 2749c4f6699dSAlexei Starovoitov } 2750babf3164SAndrii Nakryiko 2751babf3164SAndrii Nakryiko err = bpf_probe_register(link->btp, prog); 2752babf3164SAndrii Nakryiko if (err) { 2753a3b80e10SAndrii Nakryiko bpf_link_cleanup(&link_primer); 2754babf3164SAndrii Nakryiko goto out_put_btp; 2755babf3164SAndrii Nakryiko } 2756babf3164SAndrii Nakryiko 2757a3b80e10SAndrii Nakryiko return bpf_link_settle(&link_primer); 2758c4f6699dSAlexei Starovoitov 2759a38d1107SMatt Mullins out_put_btp: 2760a38d1107SMatt Mullins bpf_put_raw_tracepoint(btp); 2761ac4414b5SAlexei Starovoitov out_put_prog: 2762ac4414b5SAlexei Starovoitov bpf_prog_put(prog); 2763c4f6699dSAlexei Starovoitov return err; 2764c4f6699dSAlexei Starovoitov } 2765c4f6699dSAlexei Starovoitov 276633491588SAnders Roxell static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, 276733491588SAnders Roxell enum bpf_attach_type attach_type) 276833491588SAnders Roxell { 276933491588SAnders Roxell switch (prog->type) { 277033491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK: 277133491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 27720d01da6aSStanislav Fomichev case BPF_PROG_TYPE_CGROUP_SOCKOPT: 2773e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP: 277433491588SAnders Roxell return attach_type == prog->expected_attach_type ? 0 : -EINVAL; 27755cf1e914Sbrakmo case BPF_PROG_TYPE_CGROUP_SKB: 27762c78ee89SAlexei Starovoitov if (!capable(CAP_NET_ADMIN)) 27772c78ee89SAlexei Starovoitov /* cg-skb progs can be loaded by unpriv user. 27782c78ee89SAlexei Starovoitov * check permissions at attach time. 27792c78ee89SAlexei Starovoitov */ 27802c78ee89SAlexei Starovoitov return -EPERM; 27815cf1e914Sbrakmo return prog->enforce_expected_attach_type && 27825cf1e914Sbrakmo prog->expected_attach_type != attach_type ? 27835cf1e914Sbrakmo -EINVAL : 0; 278433491588SAnders Roxell default: 278533491588SAnders Roxell return 0; 278633491588SAnders Roxell } 278733491588SAnders Roxell } 278833491588SAnders Roxell 2789e28784e3SAndrii Nakryiko static enum bpf_prog_type 2790e28784e3SAndrii Nakryiko attach_type_to_prog_type(enum bpf_attach_type attach_type) 2791e28784e3SAndrii Nakryiko { 2792e28784e3SAndrii Nakryiko switch (attach_type) { 2793e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_INGRESS: 2794e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_EGRESS: 2795e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SKB; 2796e28784e3SAndrii Nakryiko break; 2797e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_SOCK_CREATE: 2798f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 2799e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_POST_BIND: 2800e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_POST_BIND: 2801e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCK; 2802e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_BIND: 2803e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_BIND: 2804e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_CONNECT: 2805e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_CONNECT: 28061b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 28071b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 28081b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 28091b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 2810e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP4_SENDMSG: 2811e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP6_SENDMSG: 2812e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP4_RECVMSG: 2813e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP6_RECVMSG: 2814e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 2815e28784e3SAndrii Nakryiko case BPF_CGROUP_SOCK_OPS: 2816e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SOCK_OPS; 2817e28784e3SAndrii Nakryiko case BPF_CGROUP_DEVICE: 2818e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_DEVICE; 2819e28784e3SAndrii Nakryiko case BPF_SK_MSG_VERDICT: 2820e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SK_MSG; 2821e28784e3SAndrii Nakryiko case BPF_SK_SKB_STREAM_PARSER: 2822e28784e3SAndrii Nakryiko case BPF_SK_SKB_STREAM_VERDICT: 2823e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SK_SKB; 2824e28784e3SAndrii Nakryiko case BPF_LIRC_MODE2: 2825e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_LIRC_MODE2; 2826e28784e3SAndrii Nakryiko case BPF_FLOW_DISSECTOR: 2827e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_FLOW_DISSECTOR; 2828e28784e3SAndrii Nakryiko case BPF_CGROUP_SYSCTL: 2829e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SYSCTL; 2830e28784e3SAndrii Nakryiko case BPF_CGROUP_GETSOCKOPT: 2831e28784e3SAndrii Nakryiko case BPF_CGROUP_SETSOCKOPT: 2832e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCKOPT; 2833de4e05caSYonghong Song case BPF_TRACE_ITER: 2834de4e05caSYonghong Song return BPF_PROG_TYPE_TRACING; 2835e9ddbb77SJakub Sitnicki case BPF_SK_LOOKUP: 2836e9ddbb77SJakub Sitnicki return BPF_PROG_TYPE_SK_LOOKUP; 2837aa8d3a71SAndrii Nakryiko case BPF_XDP: 2838aa8d3a71SAndrii Nakryiko return BPF_PROG_TYPE_XDP; 2839e28784e3SAndrii Nakryiko default: 2840e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_UNSPEC; 2841e28784e3SAndrii Nakryiko } 2842e28784e3SAndrii Nakryiko } 2843e28784e3SAndrii Nakryiko 28447dd68b32SAndrey Ignatov #define BPF_PROG_ATTACH_LAST_FIELD replace_bpf_fd 2845174a79ffSJohn Fastabend 2846324bda9eSAlexei Starovoitov #define BPF_F_ATTACH_MASK \ 28477dd68b32SAndrey Ignatov (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI | BPF_F_REPLACE) 2848324bda9eSAlexei Starovoitov 2849f4324551SDaniel Mack static int bpf_prog_attach(const union bpf_attr *attr) 2850f4324551SDaniel Mack { 28517f677633SAlexei Starovoitov enum bpf_prog_type ptype; 2852f4324551SDaniel Mack struct bpf_prog *prog; 28537f677633SAlexei Starovoitov int ret; 2854f4324551SDaniel Mack 2855f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_ATTACH)) 2856f4324551SDaniel Mack return -EINVAL; 2857f4324551SDaniel Mack 2858324bda9eSAlexei Starovoitov if (attr->attach_flags & ~BPF_F_ATTACH_MASK) 28597f677633SAlexei Starovoitov return -EINVAL; 28607f677633SAlexei Starovoitov 2861e28784e3SAndrii Nakryiko ptype = attach_type_to_prog_type(attr->attach_type); 2862e28784e3SAndrii Nakryiko if (ptype == BPF_PROG_TYPE_UNSPEC) 2863b2cd1257SDavid Ahern return -EINVAL; 2864b2cd1257SDavid Ahern 2865b2cd1257SDavid Ahern prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 2866f4324551SDaniel Mack if (IS_ERR(prog)) 2867f4324551SDaniel Mack return PTR_ERR(prog); 2868f4324551SDaniel Mack 28695e43f899SAndrey Ignatov if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) { 28705e43f899SAndrey Ignatov bpf_prog_put(prog); 28715e43f899SAndrey Ignatov return -EINVAL; 28725e43f899SAndrey Ignatov } 28735e43f899SAndrey Ignatov 2874fdb5c453SSean Young switch (ptype) { 2875fdb5c453SSean Young case BPF_PROG_TYPE_SK_SKB: 2876fdb5c453SSean Young case BPF_PROG_TYPE_SK_MSG: 2877604326b4SDaniel Borkmann ret = sock_map_get_from_fd(attr, prog); 2878fdb5c453SSean Young break; 2879fdb5c453SSean Young case BPF_PROG_TYPE_LIRC_MODE2: 2880fdb5c453SSean Young ret = lirc_prog_attach(attr, prog); 2881fdb5c453SSean Young break; 2882d58e468bSPetar Penkov case BPF_PROG_TYPE_FLOW_DISSECTOR: 2883a3fd7ceeSJakub Sitnicki ret = netns_bpf_prog_attach(attr, prog); 2884d58e468bSPetar Penkov break; 2885e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 2886e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 2887e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 2888e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 2889e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 2890e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 2891e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 2892fdb5c453SSean Young ret = cgroup_bpf_prog_attach(attr, ptype, prog); 2893e28784e3SAndrii Nakryiko break; 2894e28784e3SAndrii Nakryiko default: 2895e28784e3SAndrii Nakryiko ret = -EINVAL; 2896f4324551SDaniel Mack } 2897f4324551SDaniel Mack 28987f677633SAlexei Starovoitov if (ret) 28997f677633SAlexei Starovoitov bpf_prog_put(prog); 29007f677633SAlexei Starovoitov return ret; 2901f4324551SDaniel Mack } 2902f4324551SDaniel Mack 2903f4324551SDaniel Mack #define BPF_PROG_DETACH_LAST_FIELD attach_type 2904f4324551SDaniel Mack 2905f4324551SDaniel Mack static int bpf_prog_detach(const union bpf_attr *attr) 2906f4324551SDaniel Mack { 2907324bda9eSAlexei Starovoitov enum bpf_prog_type ptype; 2908f4324551SDaniel Mack 2909f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_DETACH)) 2910f4324551SDaniel Mack return -EINVAL; 2911f4324551SDaniel Mack 2912e28784e3SAndrii Nakryiko ptype = attach_type_to_prog_type(attr->attach_type); 2913e28784e3SAndrii Nakryiko 2914e28784e3SAndrii Nakryiko switch (ptype) { 2915e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SK_MSG: 2916e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SK_SKB: 2917bb0de313SLorenz Bauer return sock_map_prog_detach(attr, ptype); 2918e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_LIRC_MODE2: 2919f4364dcfSSean Young return lirc_prog_detach(attr); 2920e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_FLOW_DISSECTOR: 29214ac2add6SLorenz Bauer return netns_bpf_prog_detach(attr, ptype); 2922e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 2923e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 2924e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 2925e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 2926e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 2927e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 2928e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 2929e28784e3SAndrii Nakryiko return cgroup_bpf_prog_detach(attr, ptype); 2930f4324551SDaniel Mack default: 2931f4324551SDaniel Mack return -EINVAL; 2932f4324551SDaniel Mack } 2933f4324551SDaniel Mack } 293440304b2aSLawrence Brakmo 2935468e2f64SAlexei Starovoitov #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt 2936468e2f64SAlexei Starovoitov 2937468e2f64SAlexei Starovoitov static int bpf_prog_query(const union bpf_attr *attr, 2938468e2f64SAlexei Starovoitov union bpf_attr __user *uattr) 2939468e2f64SAlexei Starovoitov { 2940468e2f64SAlexei Starovoitov if (!capable(CAP_NET_ADMIN)) 2941468e2f64SAlexei Starovoitov return -EPERM; 2942468e2f64SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_QUERY)) 2943468e2f64SAlexei Starovoitov return -EINVAL; 2944468e2f64SAlexei Starovoitov if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE) 2945468e2f64SAlexei Starovoitov return -EINVAL; 2946468e2f64SAlexei Starovoitov 2947468e2f64SAlexei Starovoitov switch (attr->query.attach_type) { 2948468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_INGRESS: 2949468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_EGRESS: 2950468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_SOCK_CREATE: 2951f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 29524fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 29534fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 2954aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 2955aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 2956d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 2957d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 29581b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 29591b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 29601b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 29611b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 29621cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 29631cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 2964983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG: 2965983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG: 2966468e2f64SAlexei Starovoitov case BPF_CGROUP_SOCK_OPS: 2967ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 29687b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 29690d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT: 29700d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT: 2971e28784e3SAndrii Nakryiko return cgroup_bpf_prog_query(attr, uattr); 2972f4364dcfSSean Young case BPF_LIRC_MODE2: 2973f4364dcfSSean Young return lirc_prog_query(attr, uattr); 2974118c8e9aSStanislav Fomichev case BPF_FLOW_DISSECTOR: 2975e9ddbb77SJakub Sitnicki case BPF_SK_LOOKUP: 2976a3fd7ceeSJakub Sitnicki return netns_bpf_prog_query(attr, uattr); 2977468e2f64SAlexei Starovoitov default: 2978468e2f64SAlexei Starovoitov return -EINVAL; 2979468e2f64SAlexei Starovoitov } 2980468e2f64SAlexei Starovoitov } 2981f4324551SDaniel Mack 2982b0b9395dSStanislav Fomichev #define BPF_PROG_TEST_RUN_LAST_FIELD test.ctx_out 29831cf1cae9SAlexei Starovoitov 29841cf1cae9SAlexei Starovoitov static int bpf_prog_test_run(const union bpf_attr *attr, 29851cf1cae9SAlexei Starovoitov union bpf_attr __user *uattr) 29861cf1cae9SAlexei Starovoitov { 29871cf1cae9SAlexei Starovoitov struct bpf_prog *prog; 29881cf1cae9SAlexei Starovoitov int ret = -ENOTSUPP; 29891cf1cae9SAlexei Starovoitov 29901cf1cae9SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 29911cf1cae9SAlexei Starovoitov return -EINVAL; 29921cf1cae9SAlexei Starovoitov 2993b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_in && !attr->test.ctx_in) || 2994b0b9395dSStanislav Fomichev (!attr->test.ctx_size_in && attr->test.ctx_in)) 2995b0b9395dSStanislav Fomichev return -EINVAL; 2996b0b9395dSStanislav Fomichev 2997b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_out && !attr->test.ctx_out) || 2998b0b9395dSStanislav Fomichev (!attr->test.ctx_size_out && attr->test.ctx_out)) 2999b0b9395dSStanislav Fomichev return -EINVAL; 3000b0b9395dSStanislav Fomichev 30011cf1cae9SAlexei Starovoitov prog = bpf_prog_get(attr->test.prog_fd); 30021cf1cae9SAlexei Starovoitov if (IS_ERR(prog)) 30031cf1cae9SAlexei Starovoitov return PTR_ERR(prog); 30041cf1cae9SAlexei Starovoitov 30051cf1cae9SAlexei Starovoitov if (prog->aux->ops->test_run) 30061cf1cae9SAlexei Starovoitov ret = prog->aux->ops->test_run(prog, attr, uattr); 30071cf1cae9SAlexei Starovoitov 30081cf1cae9SAlexei Starovoitov bpf_prog_put(prog); 30091cf1cae9SAlexei Starovoitov return ret; 30101cf1cae9SAlexei Starovoitov } 30111cf1cae9SAlexei Starovoitov 301234ad5580SMartin KaFai Lau #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 301334ad5580SMartin KaFai Lau 301434ad5580SMartin KaFai Lau static int bpf_obj_get_next_id(const union bpf_attr *attr, 301534ad5580SMartin KaFai Lau union bpf_attr __user *uattr, 301634ad5580SMartin KaFai Lau struct idr *idr, 301734ad5580SMartin KaFai Lau spinlock_t *lock) 301834ad5580SMartin KaFai Lau { 301934ad5580SMartin KaFai Lau u32 next_id = attr->start_id; 302034ad5580SMartin KaFai Lau int err = 0; 302134ad5580SMartin KaFai Lau 302234ad5580SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 302334ad5580SMartin KaFai Lau return -EINVAL; 302434ad5580SMartin KaFai Lau 302534ad5580SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 302634ad5580SMartin KaFai Lau return -EPERM; 302734ad5580SMartin KaFai Lau 302834ad5580SMartin KaFai Lau next_id++; 302934ad5580SMartin KaFai Lau spin_lock_bh(lock); 303034ad5580SMartin KaFai Lau if (!idr_get_next(idr, &next_id)) 303134ad5580SMartin KaFai Lau err = -ENOENT; 303234ad5580SMartin KaFai Lau spin_unlock_bh(lock); 303334ad5580SMartin KaFai Lau 303434ad5580SMartin KaFai Lau if (!err) 303534ad5580SMartin KaFai Lau err = put_user(next_id, &uattr->next_id); 303634ad5580SMartin KaFai Lau 303734ad5580SMartin KaFai Lau return err; 303834ad5580SMartin KaFai Lau } 303934ad5580SMartin KaFai Lau 30406086d29dSYonghong Song struct bpf_map *bpf_map_get_curr_or_next(u32 *id) 30416086d29dSYonghong Song { 30426086d29dSYonghong Song struct bpf_map *map; 30436086d29dSYonghong Song 30446086d29dSYonghong Song spin_lock_bh(&map_idr_lock); 30456086d29dSYonghong Song again: 30466086d29dSYonghong Song map = idr_get_next(&map_idr, id); 30476086d29dSYonghong Song if (map) { 30486086d29dSYonghong Song map = __bpf_map_inc_not_zero(map, false); 30496086d29dSYonghong Song if (IS_ERR(map)) { 30506086d29dSYonghong Song (*id)++; 30516086d29dSYonghong Song goto again; 30526086d29dSYonghong Song } 30536086d29dSYonghong Song } 30546086d29dSYonghong Song spin_unlock_bh(&map_idr_lock); 30556086d29dSYonghong Song 30566086d29dSYonghong Song return map; 30576086d29dSYonghong Song } 30586086d29dSYonghong Song 3059a228a64fSAlexei Starovoitov struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id) 3060a228a64fSAlexei Starovoitov { 3061a228a64fSAlexei Starovoitov struct bpf_prog *prog; 3062a228a64fSAlexei Starovoitov 3063a228a64fSAlexei Starovoitov spin_lock_bh(&prog_idr_lock); 3064a228a64fSAlexei Starovoitov again: 3065a228a64fSAlexei Starovoitov prog = idr_get_next(&prog_idr, id); 3066a228a64fSAlexei Starovoitov if (prog) { 3067a228a64fSAlexei Starovoitov prog = bpf_prog_inc_not_zero(prog); 3068a228a64fSAlexei Starovoitov if (IS_ERR(prog)) { 3069a228a64fSAlexei Starovoitov (*id)++; 3070a228a64fSAlexei Starovoitov goto again; 3071a228a64fSAlexei Starovoitov } 3072a228a64fSAlexei Starovoitov } 3073a228a64fSAlexei Starovoitov spin_unlock_bh(&prog_idr_lock); 3074a228a64fSAlexei Starovoitov 3075a228a64fSAlexei Starovoitov return prog; 3076a228a64fSAlexei Starovoitov } 3077a228a64fSAlexei Starovoitov 3078b16d9aa4SMartin KaFai Lau #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 3079b16d9aa4SMartin KaFai Lau 30807e6897f9SBjörn Töpel struct bpf_prog *bpf_prog_by_id(u32 id) 30817e6897f9SBjörn Töpel { 30827e6897f9SBjörn Töpel struct bpf_prog *prog; 30837e6897f9SBjörn Töpel 30847e6897f9SBjörn Töpel if (!id) 30857e6897f9SBjörn Töpel return ERR_PTR(-ENOENT); 30867e6897f9SBjörn Töpel 30877e6897f9SBjörn Töpel spin_lock_bh(&prog_idr_lock); 30887e6897f9SBjörn Töpel prog = idr_find(&prog_idr, id); 30897e6897f9SBjörn Töpel if (prog) 30907e6897f9SBjörn Töpel prog = bpf_prog_inc_not_zero(prog); 30917e6897f9SBjörn Töpel else 30927e6897f9SBjörn Töpel prog = ERR_PTR(-ENOENT); 30937e6897f9SBjörn Töpel spin_unlock_bh(&prog_idr_lock); 30947e6897f9SBjörn Töpel return prog; 30957e6897f9SBjörn Töpel } 30967e6897f9SBjörn Töpel 3097b16d9aa4SMartin KaFai Lau static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 3098b16d9aa4SMartin KaFai Lau { 3099b16d9aa4SMartin KaFai Lau struct bpf_prog *prog; 3100b16d9aa4SMartin KaFai Lau u32 id = attr->prog_id; 3101b16d9aa4SMartin KaFai Lau int fd; 3102b16d9aa4SMartin KaFai Lau 3103b16d9aa4SMartin KaFai Lau if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 3104b16d9aa4SMartin KaFai Lau return -EINVAL; 3105b16d9aa4SMartin KaFai Lau 3106b16d9aa4SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 3107b16d9aa4SMartin KaFai Lau return -EPERM; 3108b16d9aa4SMartin KaFai Lau 31097e6897f9SBjörn Töpel prog = bpf_prog_by_id(id); 3110b16d9aa4SMartin KaFai Lau if (IS_ERR(prog)) 3111b16d9aa4SMartin KaFai Lau return PTR_ERR(prog); 3112b16d9aa4SMartin KaFai Lau 3113b16d9aa4SMartin KaFai Lau fd = bpf_prog_new_fd(prog); 3114b16d9aa4SMartin KaFai Lau if (fd < 0) 3115b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 3116b16d9aa4SMartin KaFai Lau 3117b16d9aa4SMartin KaFai Lau return fd; 3118b16d9aa4SMartin KaFai Lau } 3119b16d9aa4SMartin KaFai Lau 31206e71b04aSChenbo Feng #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags 3121bd5f5f4eSMartin KaFai Lau 3122bd5f5f4eSMartin KaFai Lau static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 3123bd5f5f4eSMartin KaFai Lau { 3124bd5f5f4eSMartin KaFai Lau struct bpf_map *map; 3125bd5f5f4eSMartin KaFai Lau u32 id = attr->map_id; 31266e71b04aSChenbo Feng int f_flags; 3127bd5f5f4eSMartin KaFai Lau int fd; 3128bd5f5f4eSMartin KaFai Lau 31296e71b04aSChenbo Feng if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || 31306e71b04aSChenbo Feng attr->open_flags & ~BPF_OBJ_FLAG_MASK) 3131bd5f5f4eSMartin KaFai Lau return -EINVAL; 3132bd5f5f4eSMartin KaFai Lau 3133bd5f5f4eSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 3134bd5f5f4eSMartin KaFai Lau return -EPERM; 3135bd5f5f4eSMartin KaFai Lau 31366e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->open_flags); 31376e71b04aSChenbo Feng if (f_flags < 0) 31386e71b04aSChenbo Feng return f_flags; 31396e71b04aSChenbo Feng 3140bd5f5f4eSMartin KaFai Lau spin_lock_bh(&map_idr_lock); 3141bd5f5f4eSMartin KaFai Lau map = idr_find(&map_idr, id); 3142bd5f5f4eSMartin KaFai Lau if (map) 3143b0e4701cSStanislav Fomichev map = __bpf_map_inc_not_zero(map, true); 3144bd5f5f4eSMartin KaFai Lau else 3145bd5f5f4eSMartin KaFai Lau map = ERR_PTR(-ENOENT); 3146bd5f5f4eSMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 3147bd5f5f4eSMartin KaFai Lau 3148bd5f5f4eSMartin KaFai Lau if (IS_ERR(map)) 3149bd5f5f4eSMartin KaFai Lau return PTR_ERR(map); 3150bd5f5f4eSMartin KaFai Lau 31516e71b04aSChenbo Feng fd = bpf_map_new_fd(map, f_flags); 3152bd5f5f4eSMartin KaFai Lau if (fd < 0) 3153781e6282SPeng Sun bpf_map_put_with_uref(map); 3154bd5f5f4eSMartin KaFai Lau 3155bd5f5f4eSMartin KaFai Lau return fd; 3156bd5f5f4eSMartin KaFai Lau } 3157bd5f5f4eSMartin KaFai Lau 31587105e828SDaniel Borkmann static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, 3159d8eca5bbSDaniel Borkmann unsigned long addr, u32 *off, 3160d8eca5bbSDaniel Borkmann u32 *type) 31617105e828SDaniel Borkmann { 3162d8eca5bbSDaniel Borkmann const struct bpf_map *map; 31637105e828SDaniel Borkmann int i; 31647105e828SDaniel Borkmann 3165984fe94fSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 3166d8eca5bbSDaniel Borkmann for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { 3167d8eca5bbSDaniel Borkmann map = prog->aux->used_maps[i]; 3168d8eca5bbSDaniel Borkmann if (map == (void *)addr) { 3169d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_FD; 3170984fe94fSYiFei Zhu goto out; 3171d8eca5bbSDaniel Borkmann } 3172d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta) 3173d8eca5bbSDaniel Borkmann continue; 3174d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta(map, addr, off)) { 3175d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_VALUE; 3176984fe94fSYiFei Zhu goto out; 3177d8eca5bbSDaniel Borkmann } 3178d8eca5bbSDaniel Borkmann } 3179984fe94fSYiFei Zhu map = NULL; 3180d8eca5bbSDaniel Borkmann 3181984fe94fSYiFei Zhu out: 3182984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 3183984fe94fSYiFei Zhu return map; 31847105e828SDaniel Borkmann } 31857105e828SDaniel Borkmann 318663960260SKees Cook static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, 318763960260SKees Cook const struct cred *f_cred) 31887105e828SDaniel Borkmann { 31897105e828SDaniel Borkmann const struct bpf_map *map; 31907105e828SDaniel Borkmann struct bpf_insn *insns; 3191d8eca5bbSDaniel Borkmann u32 off, type; 31927105e828SDaniel Borkmann u64 imm; 319329fcb05bSAndrii Nakryiko u8 code; 31947105e828SDaniel Borkmann int i; 31957105e828SDaniel Borkmann 31967105e828SDaniel Borkmann insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), 31977105e828SDaniel Borkmann GFP_USER); 31987105e828SDaniel Borkmann if (!insns) 31997105e828SDaniel Borkmann return insns; 32007105e828SDaniel Borkmann 32017105e828SDaniel Borkmann for (i = 0; i < prog->len; i++) { 320229fcb05bSAndrii Nakryiko code = insns[i].code; 320329fcb05bSAndrii Nakryiko 320429fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_TAIL_CALL)) { 32057105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 32067105e828SDaniel Borkmann insns[i].imm = BPF_FUNC_tail_call; 32077105e828SDaniel Borkmann /* fall-through */ 32087105e828SDaniel Borkmann } 320929fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_CALL) || 321029fcb05bSAndrii Nakryiko code == (BPF_JMP | BPF_CALL_ARGS)) { 321129fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_CALL_ARGS)) 32127105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 321363960260SKees Cook if (!bpf_dump_raw_ok(f_cred)) 32147105e828SDaniel Borkmann insns[i].imm = 0; 32157105e828SDaniel Borkmann continue; 32167105e828SDaniel Borkmann } 321729fcb05bSAndrii Nakryiko if (BPF_CLASS(code) == BPF_LDX && BPF_MODE(code) == BPF_PROBE_MEM) { 321829fcb05bSAndrii Nakryiko insns[i].code = BPF_LDX | BPF_SIZE(code) | BPF_MEM; 321929fcb05bSAndrii Nakryiko continue; 322029fcb05bSAndrii Nakryiko } 32217105e828SDaniel Borkmann 322229fcb05bSAndrii Nakryiko if (code != (BPF_LD | BPF_IMM | BPF_DW)) 32237105e828SDaniel Borkmann continue; 32247105e828SDaniel Borkmann 32257105e828SDaniel Borkmann imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; 3226d8eca5bbSDaniel Borkmann map = bpf_map_from_imm(prog, imm, &off, &type); 32277105e828SDaniel Borkmann if (map) { 3228d8eca5bbSDaniel Borkmann insns[i].src_reg = type; 32297105e828SDaniel Borkmann insns[i].imm = map->id; 3230d8eca5bbSDaniel Borkmann insns[i + 1].imm = off; 32317105e828SDaniel Borkmann continue; 32327105e828SDaniel Borkmann } 32337105e828SDaniel Borkmann } 32347105e828SDaniel Borkmann 32357105e828SDaniel Borkmann return insns; 32367105e828SDaniel Borkmann } 32377105e828SDaniel Borkmann 3238c454a46bSMartin KaFai Lau static int set_info_rec_size(struct bpf_prog_info *info) 3239c454a46bSMartin KaFai Lau { 3240c454a46bSMartin KaFai Lau /* 3241c454a46bSMartin KaFai Lau * Ensure info.*_rec_size is the same as kernel expected size 3242c454a46bSMartin KaFai Lau * 3243c454a46bSMartin KaFai Lau * or 3244c454a46bSMartin KaFai Lau * 3245c454a46bSMartin KaFai Lau * Only allow zero *_rec_size if both _rec_size and _cnt are 3246c454a46bSMartin KaFai Lau * zero. In this case, the kernel will set the expected 3247c454a46bSMartin KaFai Lau * _rec_size back to the info. 3248c454a46bSMartin KaFai Lau */ 3249c454a46bSMartin KaFai Lau 325011d8b82dSYonghong Song if ((info->nr_func_info || info->func_info_rec_size) && 3251c454a46bSMartin KaFai Lau info->func_info_rec_size != sizeof(struct bpf_func_info)) 3252c454a46bSMartin KaFai Lau return -EINVAL; 3253c454a46bSMartin KaFai Lau 325411d8b82dSYonghong Song if ((info->nr_line_info || info->line_info_rec_size) && 3255c454a46bSMartin KaFai Lau info->line_info_rec_size != sizeof(struct bpf_line_info)) 3256c454a46bSMartin KaFai Lau return -EINVAL; 3257c454a46bSMartin KaFai Lau 325811d8b82dSYonghong Song if ((info->nr_jited_line_info || info->jited_line_info_rec_size) && 3259c454a46bSMartin KaFai Lau info->jited_line_info_rec_size != sizeof(__u64)) 3260c454a46bSMartin KaFai Lau return -EINVAL; 3261c454a46bSMartin KaFai Lau 3262c454a46bSMartin KaFai Lau info->func_info_rec_size = sizeof(struct bpf_func_info); 3263c454a46bSMartin KaFai Lau info->line_info_rec_size = sizeof(struct bpf_line_info); 3264c454a46bSMartin KaFai Lau info->jited_line_info_rec_size = sizeof(__u64); 3265c454a46bSMartin KaFai Lau 3266c454a46bSMartin KaFai Lau return 0; 3267c454a46bSMartin KaFai Lau } 3268c454a46bSMartin KaFai Lau 326963960260SKees Cook static int bpf_prog_get_info_by_fd(struct file *file, 327063960260SKees Cook struct bpf_prog *prog, 32711e270976SMartin KaFai Lau const union bpf_attr *attr, 32721e270976SMartin KaFai Lau union bpf_attr __user *uattr) 32731e270976SMartin KaFai Lau { 32741e270976SMartin KaFai Lau struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 32755c6f2588SGreg Kroah-Hartman struct bpf_prog_info info; 32761e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 32775f8f8b93SAlexei Starovoitov struct bpf_prog_stats stats; 32781e270976SMartin KaFai Lau char __user *uinsns; 32791e270976SMartin KaFai Lau u32 ulen; 32801e270976SMartin KaFai Lau int err; 32811e270976SMartin KaFai Lau 3282dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 32831e270976SMartin KaFai Lau if (err) 32841e270976SMartin KaFai Lau return err; 32851e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 32861e270976SMartin KaFai Lau 32875c6f2588SGreg Kroah-Hartman memset(&info, 0, sizeof(info)); 32881e270976SMartin KaFai Lau if (copy_from_user(&info, uinfo, info_len)) 328989b09689SDaniel Borkmann return -EFAULT; 32901e270976SMartin KaFai Lau 32911e270976SMartin KaFai Lau info.type = prog->type; 32921e270976SMartin KaFai Lau info.id = prog->aux->id; 3293cb4d2b3fSMartin KaFai Lau info.load_time = prog->aux->load_time; 3294cb4d2b3fSMartin KaFai Lau info.created_by_uid = from_kuid_munged(current_user_ns(), 3295cb4d2b3fSMartin KaFai Lau prog->aux->user->uid); 3296b85fab0eSJiri Olsa info.gpl_compatible = prog->gpl_compatible; 32971e270976SMartin KaFai Lau 32981e270976SMartin KaFai Lau memcpy(info.tag, prog->tag, sizeof(prog->tag)); 3299cb4d2b3fSMartin KaFai Lau memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); 3300cb4d2b3fSMartin KaFai Lau 3301984fe94fSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 3302cb4d2b3fSMartin KaFai Lau ulen = info.nr_map_ids; 3303cb4d2b3fSMartin KaFai Lau info.nr_map_ids = prog->aux->used_map_cnt; 3304cb4d2b3fSMartin KaFai Lau ulen = min_t(u32, info.nr_map_ids, ulen); 3305cb4d2b3fSMartin KaFai Lau if (ulen) { 3306721e08daSMartin KaFai Lau u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); 3307cb4d2b3fSMartin KaFai Lau u32 i; 3308cb4d2b3fSMartin KaFai Lau 3309cb4d2b3fSMartin KaFai Lau for (i = 0; i < ulen; i++) 3310cb4d2b3fSMartin KaFai Lau if (put_user(prog->aux->used_maps[i]->id, 3311984fe94fSYiFei Zhu &user_map_ids[i])) { 3312984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 3313cb4d2b3fSMartin KaFai Lau return -EFAULT; 3314cb4d2b3fSMartin KaFai Lau } 3315984fe94fSYiFei Zhu } 3316984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 33171e270976SMartin KaFai Lau 3318c454a46bSMartin KaFai Lau err = set_info_rec_size(&info); 3319c454a46bSMartin KaFai Lau if (err) 3320c454a46bSMartin KaFai Lau return err; 33217337224fSMartin KaFai Lau 33225f8f8b93SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 33235f8f8b93SAlexei Starovoitov info.run_time_ns = stats.nsecs; 33245f8f8b93SAlexei Starovoitov info.run_cnt = stats.cnt; 33255f8f8b93SAlexei Starovoitov 33262c78ee89SAlexei Starovoitov if (!bpf_capable()) { 33271e270976SMartin KaFai Lau info.jited_prog_len = 0; 33281e270976SMartin KaFai Lau info.xlated_prog_len = 0; 3329dbecd738SSandipan Das info.nr_jited_ksyms = 0; 333028c2fae7SDaniel Borkmann info.nr_jited_func_lens = 0; 333111d8b82dSYonghong Song info.nr_func_info = 0; 333211d8b82dSYonghong Song info.nr_line_info = 0; 333311d8b82dSYonghong Song info.nr_jited_line_info = 0; 33341e270976SMartin KaFai Lau goto done; 33351e270976SMartin KaFai Lau } 33361e270976SMartin KaFai Lau 33371e270976SMartin KaFai Lau ulen = info.xlated_prog_len; 33389975a54bSDaniel Borkmann info.xlated_prog_len = bpf_prog_insn_size(prog); 33391e270976SMartin KaFai Lau if (info.xlated_prog_len && ulen) { 33407105e828SDaniel Borkmann struct bpf_insn *insns_sanitized; 33417105e828SDaniel Borkmann bool fault; 33427105e828SDaniel Borkmann 334363960260SKees Cook if (prog->blinded && !bpf_dump_raw_ok(file->f_cred)) { 33447105e828SDaniel Borkmann info.xlated_prog_insns = 0; 33457105e828SDaniel Borkmann goto done; 33467105e828SDaniel Borkmann } 334763960260SKees Cook insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred); 33487105e828SDaniel Borkmann if (!insns_sanitized) 33497105e828SDaniel Borkmann return -ENOMEM; 33501e270976SMartin KaFai Lau uinsns = u64_to_user_ptr(info.xlated_prog_insns); 33511e270976SMartin KaFai Lau ulen = min_t(u32, info.xlated_prog_len, ulen); 33527105e828SDaniel Borkmann fault = copy_to_user(uinsns, insns_sanitized, ulen); 33537105e828SDaniel Borkmann kfree(insns_sanitized); 33547105e828SDaniel Borkmann if (fault) 33551e270976SMartin KaFai Lau return -EFAULT; 33561e270976SMartin KaFai Lau } 33571e270976SMartin KaFai Lau 3358675fc275SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 3359675fc275SJakub Kicinski err = bpf_prog_offload_info_fill(&info, prog); 3360675fc275SJakub Kicinski if (err) 3361675fc275SJakub Kicinski return err; 3362fcfb126dSJiong Wang goto done; 3363fcfb126dSJiong Wang } 3364fcfb126dSJiong Wang 3365fcfb126dSJiong Wang /* NOTE: the following code is supposed to be skipped for offload. 3366fcfb126dSJiong Wang * bpf_prog_offload_info_fill() is the place to fill similar fields 3367fcfb126dSJiong Wang * for offload. 3368fcfb126dSJiong Wang */ 3369fcfb126dSJiong Wang ulen = info.jited_prog_len; 33704d56a76eSSandipan Das if (prog->aux->func_cnt) { 33714d56a76eSSandipan Das u32 i; 33724d56a76eSSandipan Das 33734d56a76eSSandipan Das info.jited_prog_len = 0; 33744d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) 33754d56a76eSSandipan Das info.jited_prog_len += prog->aux->func[i]->jited_len; 33764d56a76eSSandipan Das } else { 3377fcfb126dSJiong Wang info.jited_prog_len = prog->jited_len; 33784d56a76eSSandipan Das } 33794d56a76eSSandipan Das 3380fcfb126dSJiong Wang if (info.jited_prog_len && ulen) { 338163960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 3382fcfb126dSJiong Wang uinsns = u64_to_user_ptr(info.jited_prog_insns); 3383fcfb126dSJiong Wang ulen = min_t(u32, info.jited_prog_len, ulen); 33844d56a76eSSandipan Das 33854d56a76eSSandipan Das /* for multi-function programs, copy the JITed 33864d56a76eSSandipan Das * instructions for all the functions 33874d56a76eSSandipan Das */ 33884d56a76eSSandipan Das if (prog->aux->func_cnt) { 33894d56a76eSSandipan Das u32 len, free, i; 33904d56a76eSSandipan Das u8 *img; 33914d56a76eSSandipan Das 33924d56a76eSSandipan Das free = ulen; 33934d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) { 33944d56a76eSSandipan Das len = prog->aux->func[i]->jited_len; 33954d56a76eSSandipan Das len = min_t(u32, len, free); 33964d56a76eSSandipan Das img = (u8 *) prog->aux->func[i]->bpf_func; 33974d56a76eSSandipan Das if (copy_to_user(uinsns, img, len)) 33984d56a76eSSandipan Das return -EFAULT; 33994d56a76eSSandipan Das uinsns += len; 34004d56a76eSSandipan Das free -= len; 34014d56a76eSSandipan Das if (!free) 34024d56a76eSSandipan Das break; 34034d56a76eSSandipan Das } 34044d56a76eSSandipan Das } else { 3405fcfb126dSJiong Wang if (copy_to_user(uinsns, prog->bpf_func, ulen)) 3406fcfb126dSJiong Wang return -EFAULT; 34074d56a76eSSandipan Das } 3408fcfb126dSJiong Wang } else { 3409fcfb126dSJiong Wang info.jited_prog_insns = 0; 3410fcfb126dSJiong Wang } 3411675fc275SJakub Kicinski } 3412675fc275SJakub Kicinski 3413dbecd738SSandipan Das ulen = info.nr_jited_ksyms; 3414ff1889fcSSong Liu info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; 34157a5725ddSSong Liu if (ulen) { 341663960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 3417ff1889fcSSong Liu unsigned long ksym_addr; 3418dbecd738SSandipan Das u64 __user *user_ksyms; 3419dbecd738SSandipan Das u32 i; 3420dbecd738SSandipan Das 3421dbecd738SSandipan Das /* copy the address of the kernel symbol 3422dbecd738SSandipan Das * corresponding to each function 3423dbecd738SSandipan Das */ 3424dbecd738SSandipan Das ulen = min_t(u32, info.nr_jited_ksyms, ulen); 3425dbecd738SSandipan Das user_ksyms = u64_to_user_ptr(info.jited_ksyms); 3426ff1889fcSSong Liu if (prog->aux->func_cnt) { 3427dbecd738SSandipan Das for (i = 0; i < ulen; i++) { 3428ff1889fcSSong Liu ksym_addr = (unsigned long) 3429ff1889fcSSong Liu prog->aux->func[i]->bpf_func; 3430ff1889fcSSong Liu if (put_user((u64) ksym_addr, 3431ff1889fcSSong Liu &user_ksyms[i])) 3432ff1889fcSSong Liu return -EFAULT; 3433ff1889fcSSong Liu } 3434ff1889fcSSong Liu } else { 3435ff1889fcSSong Liu ksym_addr = (unsigned long) prog->bpf_func; 3436ff1889fcSSong Liu if (put_user((u64) ksym_addr, &user_ksyms[0])) 3437dbecd738SSandipan Das return -EFAULT; 3438dbecd738SSandipan Das } 3439dbecd738SSandipan Das } else { 3440dbecd738SSandipan Das info.jited_ksyms = 0; 3441dbecd738SSandipan Das } 3442dbecd738SSandipan Das } 3443dbecd738SSandipan Das 3444815581c1SSandipan Das ulen = info.nr_jited_func_lens; 3445ff1889fcSSong Liu info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; 34467a5725ddSSong Liu if (ulen) { 344763960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 3448815581c1SSandipan Das u32 __user *user_lens; 3449815581c1SSandipan Das u32 func_len, i; 3450815581c1SSandipan Das 3451815581c1SSandipan Das /* copy the JITed image lengths for each function */ 3452815581c1SSandipan Das ulen = min_t(u32, info.nr_jited_func_lens, ulen); 3453815581c1SSandipan Das user_lens = u64_to_user_ptr(info.jited_func_lens); 3454ff1889fcSSong Liu if (prog->aux->func_cnt) { 3455815581c1SSandipan Das for (i = 0; i < ulen; i++) { 3456ff1889fcSSong Liu func_len = 3457ff1889fcSSong Liu prog->aux->func[i]->jited_len; 3458815581c1SSandipan Das if (put_user(func_len, &user_lens[i])) 3459815581c1SSandipan Das return -EFAULT; 3460815581c1SSandipan Das } 3461815581c1SSandipan Das } else { 3462ff1889fcSSong Liu func_len = prog->jited_len; 3463ff1889fcSSong Liu if (put_user(func_len, &user_lens[0])) 3464ff1889fcSSong Liu return -EFAULT; 3465ff1889fcSSong Liu } 3466ff1889fcSSong Liu } else { 3467815581c1SSandipan Das info.jited_func_lens = 0; 3468815581c1SSandipan Das } 3469815581c1SSandipan Das } 3470815581c1SSandipan Das 34717337224fSMartin KaFai Lau if (prog->aux->btf) 3472838e9690SYonghong Song info.btf_id = btf_id(prog->aux->btf); 3473838e9690SYonghong Song 347411d8b82dSYonghong Song ulen = info.nr_func_info; 347511d8b82dSYonghong Song info.nr_func_info = prog->aux->func_info_cnt; 347611d8b82dSYonghong Song if (info.nr_func_info && ulen) { 3477838e9690SYonghong Song char __user *user_finfo; 3478838e9690SYonghong Song 3479838e9690SYonghong Song user_finfo = u64_to_user_ptr(info.func_info); 348011d8b82dSYonghong Song ulen = min_t(u32, info.nr_func_info, ulen); 3481ba64e7d8SYonghong Song if (copy_to_user(user_finfo, prog->aux->func_info, 34827337224fSMartin KaFai Lau info.func_info_rec_size * ulen)) 3483838e9690SYonghong Song return -EFAULT; 3484838e9690SYonghong Song } 3485838e9690SYonghong Song 348611d8b82dSYonghong Song ulen = info.nr_line_info; 348711d8b82dSYonghong Song info.nr_line_info = prog->aux->nr_linfo; 348811d8b82dSYonghong Song if (info.nr_line_info && ulen) { 3489c454a46bSMartin KaFai Lau __u8 __user *user_linfo; 3490c454a46bSMartin KaFai Lau 3491c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.line_info); 349211d8b82dSYonghong Song ulen = min_t(u32, info.nr_line_info, ulen); 3493c454a46bSMartin KaFai Lau if (copy_to_user(user_linfo, prog->aux->linfo, 3494c454a46bSMartin KaFai Lau info.line_info_rec_size * ulen)) 3495c454a46bSMartin KaFai Lau return -EFAULT; 3496c454a46bSMartin KaFai Lau } 3497c454a46bSMartin KaFai Lau 349811d8b82dSYonghong Song ulen = info.nr_jited_line_info; 3499c454a46bSMartin KaFai Lau if (prog->aux->jited_linfo) 350011d8b82dSYonghong Song info.nr_jited_line_info = prog->aux->nr_linfo; 3501c454a46bSMartin KaFai Lau else 350211d8b82dSYonghong Song info.nr_jited_line_info = 0; 350311d8b82dSYonghong Song if (info.nr_jited_line_info && ulen) { 350463960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 3505c454a46bSMartin KaFai Lau __u64 __user *user_linfo; 3506c454a46bSMartin KaFai Lau u32 i; 3507c454a46bSMartin KaFai Lau 3508c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.jited_line_info); 350911d8b82dSYonghong Song ulen = min_t(u32, info.nr_jited_line_info, ulen); 3510c454a46bSMartin KaFai Lau for (i = 0; i < ulen; i++) { 3511c454a46bSMartin KaFai Lau if (put_user((__u64)(long)prog->aux->jited_linfo[i], 3512c454a46bSMartin KaFai Lau &user_linfo[i])) 3513c454a46bSMartin KaFai Lau return -EFAULT; 3514c454a46bSMartin KaFai Lau } 3515c454a46bSMartin KaFai Lau } else { 3516c454a46bSMartin KaFai Lau info.jited_line_info = 0; 3517c454a46bSMartin KaFai Lau } 3518c454a46bSMartin KaFai Lau } 3519c454a46bSMartin KaFai Lau 3520c872bdb3SSong Liu ulen = info.nr_prog_tags; 3521c872bdb3SSong Liu info.nr_prog_tags = prog->aux->func_cnt ? : 1; 3522c872bdb3SSong Liu if (ulen) { 3523c872bdb3SSong Liu __u8 __user (*user_prog_tags)[BPF_TAG_SIZE]; 3524c872bdb3SSong Liu u32 i; 3525c872bdb3SSong Liu 3526c872bdb3SSong Liu user_prog_tags = u64_to_user_ptr(info.prog_tags); 3527c872bdb3SSong Liu ulen = min_t(u32, info.nr_prog_tags, ulen); 3528c872bdb3SSong Liu if (prog->aux->func_cnt) { 3529c872bdb3SSong Liu for (i = 0; i < ulen; i++) { 3530c872bdb3SSong Liu if (copy_to_user(user_prog_tags[i], 3531c872bdb3SSong Liu prog->aux->func[i]->tag, 3532c872bdb3SSong Liu BPF_TAG_SIZE)) 3533c872bdb3SSong Liu return -EFAULT; 3534c872bdb3SSong Liu } 3535c872bdb3SSong Liu } else { 3536c872bdb3SSong Liu if (copy_to_user(user_prog_tags[0], 3537c872bdb3SSong Liu prog->tag, BPF_TAG_SIZE)) 3538c872bdb3SSong Liu return -EFAULT; 3539c872bdb3SSong Liu } 3540c872bdb3SSong Liu } 3541c872bdb3SSong Liu 35421e270976SMartin KaFai Lau done: 35431e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 35441e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 35451e270976SMartin KaFai Lau return -EFAULT; 35461e270976SMartin KaFai Lau 35471e270976SMartin KaFai Lau return 0; 35481e270976SMartin KaFai Lau } 35491e270976SMartin KaFai Lau 355063960260SKees Cook static int bpf_map_get_info_by_fd(struct file *file, 355163960260SKees Cook struct bpf_map *map, 35521e270976SMartin KaFai Lau const union bpf_attr *attr, 35531e270976SMartin KaFai Lau union bpf_attr __user *uattr) 35541e270976SMartin KaFai Lau { 35551e270976SMartin KaFai Lau struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 35565c6f2588SGreg Kroah-Hartman struct bpf_map_info info; 35571e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 35581e270976SMartin KaFai Lau int err; 35591e270976SMartin KaFai Lau 3560dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 35611e270976SMartin KaFai Lau if (err) 35621e270976SMartin KaFai Lau return err; 35631e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 35641e270976SMartin KaFai Lau 35655c6f2588SGreg Kroah-Hartman memset(&info, 0, sizeof(info)); 35661e270976SMartin KaFai Lau info.type = map->map_type; 35671e270976SMartin KaFai Lau info.id = map->id; 35681e270976SMartin KaFai Lau info.key_size = map->key_size; 35691e270976SMartin KaFai Lau info.value_size = map->value_size; 35701e270976SMartin KaFai Lau info.max_entries = map->max_entries; 35711e270976SMartin KaFai Lau info.map_flags = map->map_flags; 3572ad5b177bSMartin KaFai Lau memcpy(info.name, map->name, sizeof(map->name)); 35731e270976SMartin KaFai Lau 357478958fcaSMartin KaFai Lau if (map->btf) { 357578958fcaSMartin KaFai Lau info.btf_id = btf_id(map->btf); 35769b2cf328SMartin KaFai Lau info.btf_key_type_id = map->btf_key_type_id; 35779b2cf328SMartin KaFai Lau info.btf_value_type_id = map->btf_value_type_id; 357878958fcaSMartin KaFai Lau } 357985d33df3SMartin KaFai Lau info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; 358078958fcaSMartin KaFai Lau 358152775b33SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 358252775b33SJakub Kicinski err = bpf_map_offload_info_fill(&info, map); 358352775b33SJakub Kicinski if (err) 358452775b33SJakub Kicinski return err; 358552775b33SJakub Kicinski } 358652775b33SJakub Kicinski 35871e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 35881e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 35891e270976SMartin KaFai Lau return -EFAULT; 35901e270976SMartin KaFai Lau 35911e270976SMartin KaFai Lau return 0; 35921e270976SMartin KaFai Lau } 35931e270976SMartin KaFai Lau 359463960260SKees Cook static int bpf_btf_get_info_by_fd(struct file *file, 359563960260SKees Cook struct btf *btf, 359662dab84cSMartin KaFai Lau const union bpf_attr *attr, 359762dab84cSMartin KaFai Lau union bpf_attr __user *uattr) 359862dab84cSMartin KaFai Lau { 359962dab84cSMartin KaFai Lau struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info); 360062dab84cSMartin KaFai Lau u32 info_len = attr->info.info_len; 360162dab84cSMartin KaFai Lau int err; 360262dab84cSMartin KaFai Lau 3603dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(*uinfo), info_len); 360462dab84cSMartin KaFai Lau if (err) 360562dab84cSMartin KaFai Lau return err; 360662dab84cSMartin KaFai Lau 360762dab84cSMartin KaFai Lau return btf_get_info_by_fd(btf, attr, uattr); 360862dab84cSMartin KaFai Lau } 360962dab84cSMartin KaFai Lau 361063960260SKees Cook static int bpf_link_get_info_by_fd(struct file *file, 361163960260SKees Cook struct bpf_link *link, 3612f2e10bffSAndrii Nakryiko const union bpf_attr *attr, 3613f2e10bffSAndrii Nakryiko union bpf_attr __user *uattr) 3614f2e10bffSAndrii Nakryiko { 3615f2e10bffSAndrii Nakryiko struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info); 3616f2e10bffSAndrii Nakryiko struct bpf_link_info info; 3617f2e10bffSAndrii Nakryiko u32 info_len = attr->info.info_len; 3618f2e10bffSAndrii Nakryiko int err; 3619f2e10bffSAndrii Nakryiko 3620f2e10bffSAndrii Nakryiko err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 3621f2e10bffSAndrii Nakryiko if (err) 3622f2e10bffSAndrii Nakryiko return err; 3623f2e10bffSAndrii Nakryiko info_len = min_t(u32, sizeof(info), info_len); 3624f2e10bffSAndrii Nakryiko 3625f2e10bffSAndrii Nakryiko memset(&info, 0, sizeof(info)); 3626f2e10bffSAndrii Nakryiko if (copy_from_user(&info, uinfo, info_len)) 3627f2e10bffSAndrii Nakryiko return -EFAULT; 3628f2e10bffSAndrii Nakryiko 3629f2e10bffSAndrii Nakryiko info.type = link->type; 3630f2e10bffSAndrii Nakryiko info.id = link->id; 3631f2e10bffSAndrii Nakryiko info.prog_id = link->prog->aux->id; 3632f2e10bffSAndrii Nakryiko 3633f2e10bffSAndrii Nakryiko if (link->ops->fill_link_info) { 3634f2e10bffSAndrii Nakryiko err = link->ops->fill_link_info(link, &info); 3635f2e10bffSAndrii Nakryiko if (err) 3636f2e10bffSAndrii Nakryiko return err; 3637f2e10bffSAndrii Nakryiko } 3638f2e10bffSAndrii Nakryiko 3639f2e10bffSAndrii Nakryiko if (copy_to_user(uinfo, &info, info_len) || 3640f2e10bffSAndrii Nakryiko put_user(info_len, &uattr->info.info_len)) 3641f2e10bffSAndrii Nakryiko return -EFAULT; 3642f2e10bffSAndrii Nakryiko 3643f2e10bffSAndrii Nakryiko return 0; 3644f2e10bffSAndrii Nakryiko } 3645f2e10bffSAndrii Nakryiko 3646f2e10bffSAndrii Nakryiko 36471e270976SMartin KaFai Lau #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 36481e270976SMartin KaFai Lau 36491e270976SMartin KaFai Lau static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 36501e270976SMartin KaFai Lau union bpf_attr __user *uattr) 36511e270976SMartin KaFai Lau { 36521e270976SMartin KaFai Lau int ufd = attr->info.bpf_fd; 36531e270976SMartin KaFai Lau struct fd f; 36541e270976SMartin KaFai Lau int err; 36551e270976SMartin KaFai Lau 36561e270976SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 36571e270976SMartin KaFai Lau return -EINVAL; 36581e270976SMartin KaFai Lau 36591e270976SMartin KaFai Lau f = fdget(ufd); 36601e270976SMartin KaFai Lau if (!f.file) 36611e270976SMartin KaFai Lau return -EBADFD; 36621e270976SMartin KaFai Lau 36631e270976SMartin KaFai Lau if (f.file->f_op == &bpf_prog_fops) 366463960260SKees Cook err = bpf_prog_get_info_by_fd(f.file, f.file->private_data, attr, 36651e270976SMartin KaFai Lau uattr); 36661e270976SMartin KaFai Lau else if (f.file->f_op == &bpf_map_fops) 366763960260SKees Cook err = bpf_map_get_info_by_fd(f.file, f.file->private_data, attr, 36681e270976SMartin KaFai Lau uattr); 366960197cfbSMartin KaFai Lau else if (f.file->f_op == &btf_fops) 367063960260SKees Cook err = bpf_btf_get_info_by_fd(f.file, f.file->private_data, attr, uattr); 3671f2e10bffSAndrii Nakryiko else if (f.file->f_op == &bpf_link_fops) 367263960260SKees Cook err = bpf_link_get_info_by_fd(f.file, f.file->private_data, 3673f2e10bffSAndrii Nakryiko attr, uattr); 36741e270976SMartin KaFai Lau else 36751e270976SMartin KaFai Lau err = -EINVAL; 36761e270976SMartin KaFai Lau 36771e270976SMartin KaFai Lau fdput(f); 36781e270976SMartin KaFai Lau return err; 36791e270976SMartin KaFai Lau } 36801e270976SMartin KaFai Lau 3681f56a653cSMartin KaFai Lau #define BPF_BTF_LOAD_LAST_FIELD btf_log_level 3682f56a653cSMartin KaFai Lau 3683f56a653cSMartin KaFai Lau static int bpf_btf_load(const union bpf_attr *attr) 3684f56a653cSMartin KaFai Lau { 3685f56a653cSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_LOAD)) 3686f56a653cSMartin KaFai Lau return -EINVAL; 3687f56a653cSMartin KaFai Lau 36882c78ee89SAlexei Starovoitov if (!bpf_capable()) 3689f56a653cSMartin KaFai Lau return -EPERM; 3690f56a653cSMartin KaFai Lau 3691f56a653cSMartin KaFai Lau return btf_new_fd(attr); 3692f56a653cSMartin KaFai Lau } 3693f56a653cSMartin KaFai Lau 369478958fcaSMartin KaFai Lau #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id 369578958fcaSMartin KaFai Lau 369678958fcaSMartin KaFai Lau static int bpf_btf_get_fd_by_id(const union bpf_attr *attr) 369778958fcaSMartin KaFai Lau { 369878958fcaSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID)) 369978958fcaSMartin KaFai Lau return -EINVAL; 370078958fcaSMartin KaFai Lau 370178958fcaSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 370278958fcaSMartin KaFai Lau return -EPERM; 370378958fcaSMartin KaFai Lau 370478958fcaSMartin KaFai Lau return btf_get_fd_by_id(attr->btf_id); 370578958fcaSMartin KaFai Lau } 370678958fcaSMartin KaFai Lau 370741bdc4b4SYonghong Song static int bpf_task_fd_query_copy(const union bpf_attr *attr, 370841bdc4b4SYonghong Song union bpf_attr __user *uattr, 370941bdc4b4SYonghong Song u32 prog_id, u32 fd_type, 371041bdc4b4SYonghong Song const char *buf, u64 probe_offset, 371141bdc4b4SYonghong Song u64 probe_addr) 371241bdc4b4SYonghong Song { 371341bdc4b4SYonghong Song char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf); 371441bdc4b4SYonghong Song u32 len = buf ? strlen(buf) : 0, input_len; 371541bdc4b4SYonghong Song int err = 0; 371641bdc4b4SYonghong Song 371741bdc4b4SYonghong Song if (put_user(len, &uattr->task_fd_query.buf_len)) 371841bdc4b4SYonghong Song return -EFAULT; 371941bdc4b4SYonghong Song input_len = attr->task_fd_query.buf_len; 372041bdc4b4SYonghong Song if (input_len && ubuf) { 372141bdc4b4SYonghong Song if (!len) { 372241bdc4b4SYonghong Song /* nothing to copy, just make ubuf NULL terminated */ 372341bdc4b4SYonghong Song char zero = '\0'; 372441bdc4b4SYonghong Song 372541bdc4b4SYonghong Song if (put_user(zero, ubuf)) 372641bdc4b4SYonghong Song return -EFAULT; 372741bdc4b4SYonghong Song } else if (input_len >= len + 1) { 372841bdc4b4SYonghong Song /* ubuf can hold the string with NULL terminator */ 372941bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, len + 1)) 373041bdc4b4SYonghong Song return -EFAULT; 373141bdc4b4SYonghong Song } else { 373241bdc4b4SYonghong Song /* ubuf cannot hold the string with NULL terminator, 373341bdc4b4SYonghong Song * do a partial copy with NULL terminator. 373441bdc4b4SYonghong Song */ 373541bdc4b4SYonghong Song char zero = '\0'; 373641bdc4b4SYonghong Song 373741bdc4b4SYonghong Song err = -ENOSPC; 373841bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, input_len - 1)) 373941bdc4b4SYonghong Song return -EFAULT; 374041bdc4b4SYonghong Song if (put_user(zero, ubuf + input_len - 1)) 374141bdc4b4SYonghong Song return -EFAULT; 374241bdc4b4SYonghong Song } 374341bdc4b4SYonghong Song } 374441bdc4b4SYonghong Song 374541bdc4b4SYonghong Song if (put_user(prog_id, &uattr->task_fd_query.prog_id) || 374641bdc4b4SYonghong Song put_user(fd_type, &uattr->task_fd_query.fd_type) || 374741bdc4b4SYonghong Song put_user(probe_offset, &uattr->task_fd_query.probe_offset) || 374841bdc4b4SYonghong Song put_user(probe_addr, &uattr->task_fd_query.probe_addr)) 374941bdc4b4SYonghong Song return -EFAULT; 375041bdc4b4SYonghong Song 375141bdc4b4SYonghong Song return err; 375241bdc4b4SYonghong Song } 375341bdc4b4SYonghong Song 375441bdc4b4SYonghong Song #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr 375541bdc4b4SYonghong Song 375641bdc4b4SYonghong Song static int bpf_task_fd_query(const union bpf_attr *attr, 375741bdc4b4SYonghong Song union bpf_attr __user *uattr) 375841bdc4b4SYonghong Song { 375941bdc4b4SYonghong Song pid_t pid = attr->task_fd_query.pid; 376041bdc4b4SYonghong Song u32 fd = attr->task_fd_query.fd; 376141bdc4b4SYonghong Song const struct perf_event *event; 376241bdc4b4SYonghong Song struct files_struct *files; 376341bdc4b4SYonghong Song struct task_struct *task; 376441bdc4b4SYonghong Song struct file *file; 376541bdc4b4SYonghong Song int err; 376641bdc4b4SYonghong Song 376741bdc4b4SYonghong Song if (CHECK_ATTR(BPF_TASK_FD_QUERY)) 376841bdc4b4SYonghong Song return -EINVAL; 376941bdc4b4SYonghong Song 377041bdc4b4SYonghong Song if (!capable(CAP_SYS_ADMIN)) 377141bdc4b4SYonghong Song return -EPERM; 377241bdc4b4SYonghong Song 377341bdc4b4SYonghong Song if (attr->task_fd_query.flags != 0) 377441bdc4b4SYonghong Song return -EINVAL; 377541bdc4b4SYonghong Song 377641bdc4b4SYonghong Song task = get_pid_task(find_vpid(pid), PIDTYPE_PID); 377741bdc4b4SYonghong Song if (!task) 377841bdc4b4SYonghong Song return -ENOENT; 377941bdc4b4SYonghong Song 378041bdc4b4SYonghong Song files = get_files_struct(task); 378141bdc4b4SYonghong Song put_task_struct(task); 378241bdc4b4SYonghong Song if (!files) 378341bdc4b4SYonghong Song return -ENOENT; 378441bdc4b4SYonghong Song 378541bdc4b4SYonghong Song err = 0; 378641bdc4b4SYonghong Song spin_lock(&files->file_lock); 378741bdc4b4SYonghong Song file = fcheck_files(files, fd); 378841bdc4b4SYonghong Song if (!file) 378941bdc4b4SYonghong Song err = -EBADF; 379041bdc4b4SYonghong Song else 379141bdc4b4SYonghong Song get_file(file); 379241bdc4b4SYonghong Song spin_unlock(&files->file_lock); 379341bdc4b4SYonghong Song put_files_struct(files); 379441bdc4b4SYonghong Song 379541bdc4b4SYonghong Song if (err) 379641bdc4b4SYonghong Song goto out; 379741bdc4b4SYonghong Song 379870ed506cSAndrii Nakryiko if (file->f_op == &bpf_link_fops) { 379970ed506cSAndrii Nakryiko struct bpf_link *link = file->private_data; 380070ed506cSAndrii Nakryiko 3801a3b80e10SAndrii Nakryiko if (link->ops == &bpf_raw_tp_link_lops) { 380270ed506cSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 380370ed506cSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 380441bdc4b4SYonghong Song struct bpf_raw_event_map *btp = raw_tp->btp; 380541bdc4b4SYonghong Song 380641bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, 380770ed506cSAndrii Nakryiko raw_tp->link.prog->aux->id, 380841bdc4b4SYonghong Song BPF_FD_TYPE_RAW_TRACEPOINT, 380941bdc4b4SYonghong Song btp->tp->name, 0, 0); 381041bdc4b4SYonghong Song goto put_file; 381141bdc4b4SYonghong Song } 381270ed506cSAndrii Nakryiko goto out_not_supp; 381370ed506cSAndrii Nakryiko } 381441bdc4b4SYonghong Song 381541bdc4b4SYonghong Song event = perf_get_event(file); 381641bdc4b4SYonghong Song if (!IS_ERR(event)) { 381741bdc4b4SYonghong Song u64 probe_offset, probe_addr; 381841bdc4b4SYonghong Song u32 prog_id, fd_type; 381941bdc4b4SYonghong Song const char *buf; 382041bdc4b4SYonghong Song 382141bdc4b4SYonghong Song err = bpf_get_perf_event_info(event, &prog_id, &fd_type, 382241bdc4b4SYonghong Song &buf, &probe_offset, 382341bdc4b4SYonghong Song &probe_addr); 382441bdc4b4SYonghong Song if (!err) 382541bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, prog_id, 382641bdc4b4SYonghong Song fd_type, buf, 382741bdc4b4SYonghong Song probe_offset, 382841bdc4b4SYonghong Song probe_addr); 382941bdc4b4SYonghong Song goto put_file; 383041bdc4b4SYonghong Song } 383141bdc4b4SYonghong Song 383270ed506cSAndrii Nakryiko out_not_supp: 383341bdc4b4SYonghong Song err = -ENOTSUPP; 383441bdc4b4SYonghong Song put_file: 383541bdc4b4SYonghong Song fput(file); 383641bdc4b4SYonghong Song out: 383741bdc4b4SYonghong Song return err; 383841bdc4b4SYonghong Song } 383941bdc4b4SYonghong Song 3840cb4d03abSBrian Vazquez #define BPF_MAP_BATCH_LAST_FIELD batch.flags 3841cb4d03abSBrian Vazquez 3842cb4d03abSBrian Vazquez #define BPF_DO_BATCH(fn) \ 3843cb4d03abSBrian Vazquez do { \ 3844cb4d03abSBrian Vazquez if (!fn) { \ 3845cb4d03abSBrian Vazquez err = -ENOTSUPP; \ 3846cb4d03abSBrian Vazquez goto err_put; \ 3847cb4d03abSBrian Vazquez } \ 3848cb4d03abSBrian Vazquez err = fn(map, attr, uattr); \ 3849cb4d03abSBrian Vazquez } while (0) 3850cb4d03abSBrian Vazquez 3851cb4d03abSBrian Vazquez static int bpf_map_do_batch(const union bpf_attr *attr, 3852cb4d03abSBrian Vazquez union bpf_attr __user *uattr, 3853cb4d03abSBrian Vazquez int cmd) 3854cb4d03abSBrian Vazquez { 3855cb4d03abSBrian Vazquez struct bpf_map *map; 3856cb4d03abSBrian Vazquez int err, ufd; 3857cb4d03abSBrian Vazquez struct fd f; 3858cb4d03abSBrian Vazquez 3859cb4d03abSBrian Vazquez if (CHECK_ATTR(BPF_MAP_BATCH)) 3860cb4d03abSBrian Vazquez return -EINVAL; 3861cb4d03abSBrian Vazquez 3862cb4d03abSBrian Vazquez ufd = attr->batch.map_fd; 3863cb4d03abSBrian Vazquez f = fdget(ufd); 3864cb4d03abSBrian Vazquez map = __bpf_map_get(f); 3865cb4d03abSBrian Vazquez if (IS_ERR(map)) 3866cb4d03abSBrian Vazquez return PTR_ERR(map); 3867cb4d03abSBrian Vazquez 386805799638SYonghong Song if ((cmd == BPF_MAP_LOOKUP_BATCH || 386905799638SYonghong Song cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) && 3870cb4d03abSBrian Vazquez !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 3871cb4d03abSBrian Vazquez err = -EPERM; 3872cb4d03abSBrian Vazquez goto err_put; 3873cb4d03abSBrian Vazquez } 3874cb4d03abSBrian Vazquez 3875cb4d03abSBrian Vazquez if (cmd != BPF_MAP_LOOKUP_BATCH && 3876cb4d03abSBrian Vazquez !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 3877cb4d03abSBrian Vazquez err = -EPERM; 3878cb4d03abSBrian Vazquez goto err_put; 3879cb4d03abSBrian Vazquez } 3880cb4d03abSBrian Vazquez 3881cb4d03abSBrian Vazquez if (cmd == BPF_MAP_LOOKUP_BATCH) 3882cb4d03abSBrian Vazquez BPF_DO_BATCH(map->ops->map_lookup_batch); 388305799638SYonghong Song else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) 388405799638SYonghong Song BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch); 3885aa2e93b8SBrian Vazquez else if (cmd == BPF_MAP_UPDATE_BATCH) 3886aa2e93b8SBrian Vazquez BPF_DO_BATCH(map->ops->map_update_batch); 3887aa2e93b8SBrian Vazquez else 3888aa2e93b8SBrian Vazquez BPF_DO_BATCH(map->ops->map_delete_batch); 3889cb4d03abSBrian Vazquez 3890cb4d03abSBrian Vazquez err_put: 3891cb4d03abSBrian Vazquez fdput(f); 3892cb4d03abSBrian Vazquez return err; 3893cb4d03abSBrian Vazquez } 3894cb4d03abSBrian Vazquez 3895de4e05caSYonghong Song static int tracing_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 3896de4e05caSYonghong Song { 3897de4e05caSYonghong Song if (attr->link_create.attach_type == BPF_TRACE_ITER && 3898de4e05caSYonghong Song prog->expected_attach_type == BPF_TRACE_ITER) 3899de4e05caSYonghong Song return bpf_iter_link_attach(attr, prog); 3900de4e05caSYonghong Song 3901de4e05caSYonghong Song return -EINVAL; 3902de4e05caSYonghong Song } 3903de4e05caSYonghong Song 39045e7b3020SYonghong Song #define BPF_LINK_CREATE_LAST_FIELD link_create.iter_info_len 3905af6eea57SAndrii Nakryiko static int link_create(union bpf_attr *attr) 3906af6eea57SAndrii Nakryiko { 3907af6eea57SAndrii Nakryiko enum bpf_prog_type ptype; 3908af6eea57SAndrii Nakryiko struct bpf_prog *prog; 3909af6eea57SAndrii Nakryiko int ret; 3910af6eea57SAndrii Nakryiko 3911af6eea57SAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_CREATE)) 3912af6eea57SAndrii Nakryiko return -EINVAL; 3913af6eea57SAndrii Nakryiko 3914af6eea57SAndrii Nakryiko ptype = attach_type_to_prog_type(attr->link_create.attach_type); 3915af6eea57SAndrii Nakryiko if (ptype == BPF_PROG_TYPE_UNSPEC) 3916af6eea57SAndrii Nakryiko return -EINVAL; 3917af6eea57SAndrii Nakryiko 3918af6eea57SAndrii Nakryiko prog = bpf_prog_get_type(attr->link_create.prog_fd, ptype); 3919af6eea57SAndrii Nakryiko if (IS_ERR(prog)) 3920af6eea57SAndrii Nakryiko return PTR_ERR(prog); 3921af6eea57SAndrii Nakryiko 3922af6eea57SAndrii Nakryiko ret = bpf_prog_attach_check_attach_type(prog, 3923af6eea57SAndrii Nakryiko attr->link_create.attach_type); 3924af6eea57SAndrii Nakryiko if (ret) 3925af6eea57SAndrii Nakryiko goto err_out; 3926af6eea57SAndrii Nakryiko 3927af6eea57SAndrii Nakryiko switch (ptype) { 3928af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 3929af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 3930af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 3931af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 3932af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 3933af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 3934af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 3935af6eea57SAndrii Nakryiko ret = cgroup_bpf_link_attach(attr, prog); 3936af6eea57SAndrii Nakryiko break; 3937de4e05caSYonghong Song case BPF_PROG_TYPE_TRACING: 3938de4e05caSYonghong Song ret = tracing_bpf_link_attach(attr, prog); 3939de4e05caSYonghong Song break; 39407f045a49SJakub Sitnicki case BPF_PROG_TYPE_FLOW_DISSECTOR: 3941e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP: 39427f045a49SJakub Sitnicki ret = netns_bpf_link_create(attr, prog); 39437f045a49SJakub Sitnicki break; 3944310ad797SAndrii Nakryiko #ifdef CONFIG_NET 3945aa8d3a71SAndrii Nakryiko case BPF_PROG_TYPE_XDP: 3946aa8d3a71SAndrii Nakryiko ret = bpf_xdp_link_attach(attr, prog); 3947aa8d3a71SAndrii Nakryiko break; 3948310ad797SAndrii Nakryiko #endif 3949af6eea57SAndrii Nakryiko default: 3950af6eea57SAndrii Nakryiko ret = -EINVAL; 3951af6eea57SAndrii Nakryiko } 3952af6eea57SAndrii Nakryiko 3953af6eea57SAndrii Nakryiko err_out: 3954af6eea57SAndrii Nakryiko if (ret < 0) 3955af6eea57SAndrii Nakryiko bpf_prog_put(prog); 3956af6eea57SAndrii Nakryiko return ret; 3957af6eea57SAndrii Nakryiko } 3958af6eea57SAndrii Nakryiko 39590c991ebcSAndrii Nakryiko #define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd 39600c991ebcSAndrii Nakryiko 39610c991ebcSAndrii Nakryiko static int link_update(union bpf_attr *attr) 39620c991ebcSAndrii Nakryiko { 39630c991ebcSAndrii Nakryiko struct bpf_prog *old_prog = NULL, *new_prog; 39640c991ebcSAndrii Nakryiko struct bpf_link *link; 39650c991ebcSAndrii Nakryiko u32 flags; 39660c991ebcSAndrii Nakryiko int ret; 39670c991ebcSAndrii Nakryiko 39680c991ebcSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_UPDATE)) 39690c991ebcSAndrii Nakryiko return -EINVAL; 39700c991ebcSAndrii Nakryiko 39710c991ebcSAndrii Nakryiko flags = attr->link_update.flags; 39720c991ebcSAndrii Nakryiko if (flags & ~BPF_F_REPLACE) 39730c991ebcSAndrii Nakryiko return -EINVAL; 39740c991ebcSAndrii Nakryiko 39750c991ebcSAndrii Nakryiko link = bpf_link_get_from_fd(attr->link_update.link_fd); 39760c991ebcSAndrii Nakryiko if (IS_ERR(link)) 39770c991ebcSAndrii Nakryiko return PTR_ERR(link); 39780c991ebcSAndrii Nakryiko 39790c991ebcSAndrii Nakryiko new_prog = bpf_prog_get(attr->link_update.new_prog_fd); 39804adb7a4aSAndrii Nakryiko if (IS_ERR(new_prog)) { 39814adb7a4aSAndrii Nakryiko ret = PTR_ERR(new_prog); 39824adb7a4aSAndrii Nakryiko goto out_put_link; 39834adb7a4aSAndrii Nakryiko } 39840c991ebcSAndrii Nakryiko 39850c991ebcSAndrii Nakryiko if (flags & BPF_F_REPLACE) { 39860c991ebcSAndrii Nakryiko old_prog = bpf_prog_get(attr->link_update.old_prog_fd); 39870c991ebcSAndrii Nakryiko if (IS_ERR(old_prog)) { 39880c991ebcSAndrii Nakryiko ret = PTR_ERR(old_prog); 39890c991ebcSAndrii Nakryiko old_prog = NULL; 39900c991ebcSAndrii Nakryiko goto out_put_progs; 39910c991ebcSAndrii Nakryiko } 39924adb7a4aSAndrii Nakryiko } else if (attr->link_update.old_prog_fd) { 39934adb7a4aSAndrii Nakryiko ret = -EINVAL; 39944adb7a4aSAndrii Nakryiko goto out_put_progs; 39950c991ebcSAndrii Nakryiko } 39960c991ebcSAndrii Nakryiko 3997f9d04127SAndrii Nakryiko if (link->ops->update_prog) 3998f9d04127SAndrii Nakryiko ret = link->ops->update_prog(link, new_prog, old_prog); 3999f9d04127SAndrii Nakryiko else 40000c991ebcSAndrii Nakryiko ret = -EINVAL; 40010c991ebcSAndrii Nakryiko 40020c991ebcSAndrii Nakryiko out_put_progs: 40030c991ebcSAndrii Nakryiko if (old_prog) 40040c991ebcSAndrii Nakryiko bpf_prog_put(old_prog); 40050c991ebcSAndrii Nakryiko if (ret) 40060c991ebcSAndrii Nakryiko bpf_prog_put(new_prog); 40074adb7a4aSAndrii Nakryiko out_put_link: 40084adb7a4aSAndrii Nakryiko bpf_link_put(link); 40090c991ebcSAndrii Nakryiko return ret; 40100c991ebcSAndrii Nakryiko } 40110c991ebcSAndrii Nakryiko 401273b11c2aSAndrii Nakryiko #define BPF_LINK_DETACH_LAST_FIELD link_detach.link_fd 401373b11c2aSAndrii Nakryiko 401473b11c2aSAndrii Nakryiko static int link_detach(union bpf_attr *attr) 401573b11c2aSAndrii Nakryiko { 401673b11c2aSAndrii Nakryiko struct bpf_link *link; 401773b11c2aSAndrii Nakryiko int ret; 401873b11c2aSAndrii Nakryiko 401973b11c2aSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_DETACH)) 402073b11c2aSAndrii Nakryiko return -EINVAL; 402173b11c2aSAndrii Nakryiko 402273b11c2aSAndrii Nakryiko link = bpf_link_get_from_fd(attr->link_detach.link_fd); 402373b11c2aSAndrii Nakryiko if (IS_ERR(link)) 402473b11c2aSAndrii Nakryiko return PTR_ERR(link); 402573b11c2aSAndrii Nakryiko 402673b11c2aSAndrii Nakryiko if (link->ops->detach) 402773b11c2aSAndrii Nakryiko ret = link->ops->detach(link); 402873b11c2aSAndrii Nakryiko else 402973b11c2aSAndrii Nakryiko ret = -EOPNOTSUPP; 403073b11c2aSAndrii Nakryiko 403173b11c2aSAndrii Nakryiko bpf_link_put(link); 403273b11c2aSAndrii Nakryiko return ret; 403373b11c2aSAndrii Nakryiko } 403473b11c2aSAndrii Nakryiko 4035005142b8SAlexei Starovoitov static struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link) 40362d602c8cSAndrii Nakryiko { 4037005142b8SAlexei Starovoitov return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT); 4038005142b8SAlexei Starovoitov } 4039005142b8SAlexei Starovoitov 4040005142b8SAlexei Starovoitov struct bpf_link *bpf_link_by_id(u32 id) 4041005142b8SAlexei Starovoitov { 4042005142b8SAlexei Starovoitov struct bpf_link *link; 4043005142b8SAlexei Starovoitov 4044005142b8SAlexei Starovoitov if (!id) 4045005142b8SAlexei Starovoitov return ERR_PTR(-ENOENT); 4046005142b8SAlexei Starovoitov 4047005142b8SAlexei Starovoitov spin_lock_bh(&link_idr_lock); 4048005142b8SAlexei Starovoitov /* before link is "settled", ID is 0, pretend it doesn't exist yet */ 4049005142b8SAlexei Starovoitov link = idr_find(&link_idr, id); 4050005142b8SAlexei Starovoitov if (link) { 4051005142b8SAlexei Starovoitov if (link->id) 4052005142b8SAlexei Starovoitov link = bpf_link_inc_not_zero(link); 4053005142b8SAlexei Starovoitov else 4054005142b8SAlexei Starovoitov link = ERR_PTR(-EAGAIN); 4055005142b8SAlexei Starovoitov } else { 4056005142b8SAlexei Starovoitov link = ERR_PTR(-ENOENT); 4057005142b8SAlexei Starovoitov } 4058005142b8SAlexei Starovoitov spin_unlock_bh(&link_idr_lock); 4059005142b8SAlexei Starovoitov return link; 40602d602c8cSAndrii Nakryiko } 40612d602c8cSAndrii Nakryiko 40622d602c8cSAndrii Nakryiko #define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id 40632d602c8cSAndrii Nakryiko 40642d602c8cSAndrii Nakryiko static int bpf_link_get_fd_by_id(const union bpf_attr *attr) 40652d602c8cSAndrii Nakryiko { 40662d602c8cSAndrii Nakryiko struct bpf_link *link; 40672d602c8cSAndrii Nakryiko u32 id = attr->link_id; 4068005142b8SAlexei Starovoitov int fd; 40692d602c8cSAndrii Nakryiko 40702d602c8cSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID)) 40712d602c8cSAndrii Nakryiko return -EINVAL; 40722d602c8cSAndrii Nakryiko 40732d602c8cSAndrii Nakryiko if (!capable(CAP_SYS_ADMIN)) 40742d602c8cSAndrii Nakryiko return -EPERM; 40752d602c8cSAndrii Nakryiko 4076005142b8SAlexei Starovoitov link = bpf_link_by_id(id); 4077005142b8SAlexei Starovoitov if (IS_ERR(link)) 4078005142b8SAlexei Starovoitov return PTR_ERR(link); 40792d602c8cSAndrii Nakryiko 40802d602c8cSAndrii Nakryiko fd = bpf_link_new_fd(link); 40812d602c8cSAndrii Nakryiko if (fd < 0) 40822d602c8cSAndrii Nakryiko bpf_link_put(link); 40832d602c8cSAndrii Nakryiko 40842d602c8cSAndrii Nakryiko return fd; 40852d602c8cSAndrii Nakryiko } 40862d602c8cSAndrii Nakryiko 4087d46edd67SSong Liu DEFINE_MUTEX(bpf_stats_enabled_mutex); 4088d46edd67SSong Liu 4089d46edd67SSong Liu static int bpf_stats_release(struct inode *inode, struct file *file) 4090d46edd67SSong Liu { 4091d46edd67SSong Liu mutex_lock(&bpf_stats_enabled_mutex); 4092d46edd67SSong Liu static_key_slow_dec(&bpf_stats_enabled_key.key); 4093d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 4094d46edd67SSong Liu return 0; 4095d46edd67SSong Liu } 4096d46edd67SSong Liu 4097d46edd67SSong Liu static const struct file_operations bpf_stats_fops = { 4098d46edd67SSong Liu .release = bpf_stats_release, 4099d46edd67SSong Liu }; 4100d46edd67SSong Liu 4101d46edd67SSong Liu static int bpf_enable_runtime_stats(void) 4102d46edd67SSong Liu { 4103d46edd67SSong Liu int fd; 4104d46edd67SSong Liu 4105d46edd67SSong Liu mutex_lock(&bpf_stats_enabled_mutex); 4106d46edd67SSong Liu 4107d46edd67SSong Liu /* Set a very high limit to avoid overflow */ 4108d46edd67SSong Liu if (static_key_count(&bpf_stats_enabled_key.key) > INT_MAX / 2) { 4109d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 4110d46edd67SSong Liu return -EBUSY; 4111d46edd67SSong Liu } 4112d46edd67SSong Liu 4113d46edd67SSong Liu fd = anon_inode_getfd("bpf-stats", &bpf_stats_fops, NULL, O_CLOEXEC); 4114d46edd67SSong Liu if (fd >= 0) 4115d46edd67SSong Liu static_key_slow_inc(&bpf_stats_enabled_key.key); 4116d46edd67SSong Liu 4117d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 4118d46edd67SSong Liu return fd; 4119d46edd67SSong Liu } 4120d46edd67SSong Liu 4121d46edd67SSong Liu #define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type 4122d46edd67SSong Liu 4123d46edd67SSong Liu static int bpf_enable_stats(union bpf_attr *attr) 4124d46edd67SSong Liu { 4125d46edd67SSong Liu 4126d46edd67SSong Liu if (CHECK_ATTR(BPF_ENABLE_STATS)) 4127d46edd67SSong Liu return -EINVAL; 4128d46edd67SSong Liu 4129d46edd67SSong Liu if (!capable(CAP_SYS_ADMIN)) 4130d46edd67SSong Liu return -EPERM; 4131d46edd67SSong Liu 4132d46edd67SSong Liu switch (attr->enable_stats.type) { 4133d46edd67SSong Liu case BPF_STATS_RUN_TIME: 4134d46edd67SSong Liu return bpf_enable_runtime_stats(); 4135d46edd67SSong Liu default: 4136d46edd67SSong Liu break; 4137d46edd67SSong Liu } 4138d46edd67SSong Liu return -EINVAL; 4139d46edd67SSong Liu } 4140d46edd67SSong Liu 4141ac51d99bSYonghong Song #define BPF_ITER_CREATE_LAST_FIELD iter_create.flags 4142ac51d99bSYonghong Song 4143ac51d99bSYonghong Song static int bpf_iter_create(union bpf_attr *attr) 4144ac51d99bSYonghong Song { 4145ac51d99bSYonghong Song struct bpf_link *link; 4146ac51d99bSYonghong Song int err; 4147ac51d99bSYonghong Song 4148ac51d99bSYonghong Song if (CHECK_ATTR(BPF_ITER_CREATE)) 4149ac51d99bSYonghong Song return -EINVAL; 4150ac51d99bSYonghong Song 4151ac51d99bSYonghong Song if (attr->iter_create.flags) 4152ac51d99bSYonghong Song return -EINVAL; 4153ac51d99bSYonghong Song 4154ac51d99bSYonghong Song link = bpf_link_get_from_fd(attr->iter_create.link_fd); 4155ac51d99bSYonghong Song if (IS_ERR(link)) 4156ac51d99bSYonghong Song return PTR_ERR(link); 4157ac51d99bSYonghong Song 4158ac51d99bSYonghong Song err = bpf_iter_new_fd(link); 4159ac51d99bSYonghong Song bpf_link_put(link); 4160ac51d99bSYonghong Song 4161ac51d99bSYonghong Song return err; 4162ac51d99bSYonghong Song } 4163ac51d99bSYonghong Song 4164*ef15314aSYiFei Zhu #define BPF_PROG_BIND_MAP_LAST_FIELD prog_bind_map.flags 4165*ef15314aSYiFei Zhu 4166*ef15314aSYiFei Zhu static int bpf_prog_bind_map(union bpf_attr *attr) 4167*ef15314aSYiFei Zhu { 4168*ef15314aSYiFei Zhu struct bpf_prog *prog; 4169*ef15314aSYiFei Zhu struct bpf_map *map; 4170*ef15314aSYiFei Zhu struct bpf_map **used_maps_old, **used_maps_new; 4171*ef15314aSYiFei Zhu int i, ret = 0; 4172*ef15314aSYiFei Zhu 4173*ef15314aSYiFei Zhu if (CHECK_ATTR(BPF_PROG_BIND_MAP)) 4174*ef15314aSYiFei Zhu return -EINVAL; 4175*ef15314aSYiFei Zhu 4176*ef15314aSYiFei Zhu if (attr->prog_bind_map.flags) 4177*ef15314aSYiFei Zhu return -EINVAL; 4178*ef15314aSYiFei Zhu 4179*ef15314aSYiFei Zhu prog = bpf_prog_get(attr->prog_bind_map.prog_fd); 4180*ef15314aSYiFei Zhu if (IS_ERR(prog)) 4181*ef15314aSYiFei Zhu return PTR_ERR(prog); 4182*ef15314aSYiFei Zhu 4183*ef15314aSYiFei Zhu map = bpf_map_get(attr->prog_bind_map.map_fd); 4184*ef15314aSYiFei Zhu if (IS_ERR(map)) { 4185*ef15314aSYiFei Zhu ret = PTR_ERR(map); 4186*ef15314aSYiFei Zhu goto out_prog_put; 4187*ef15314aSYiFei Zhu } 4188*ef15314aSYiFei Zhu 4189*ef15314aSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 4190*ef15314aSYiFei Zhu 4191*ef15314aSYiFei Zhu used_maps_old = prog->aux->used_maps; 4192*ef15314aSYiFei Zhu 4193*ef15314aSYiFei Zhu for (i = 0; i < prog->aux->used_map_cnt; i++) 4194*ef15314aSYiFei Zhu if (used_maps_old[i] == map) 4195*ef15314aSYiFei Zhu goto out_unlock; 4196*ef15314aSYiFei Zhu 4197*ef15314aSYiFei Zhu used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1, 4198*ef15314aSYiFei Zhu sizeof(used_maps_new[0]), 4199*ef15314aSYiFei Zhu GFP_KERNEL); 4200*ef15314aSYiFei Zhu if (!used_maps_new) { 4201*ef15314aSYiFei Zhu ret = -ENOMEM; 4202*ef15314aSYiFei Zhu goto out_unlock; 4203*ef15314aSYiFei Zhu } 4204*ef15314aSYiFei Zhu 4205*ef15314aSYiFei Zhu memcpy(used_maps_new, used_maps_old, 4206*ef15314aSYiFei Zhu sizeof(used_maps_old[0]) * prog->aux->used_map_cnt); 4207*ef15314aSYiFei Zhu used_maps_new[prog->aux->used_map_cnt] = map; 4208*ef15314aSYiFei Zhu 4209*ef15314aSYiFei Zhu prog->aux->used_map_cnt++; 4210*ef15314aSYiFei Zhu prog->aux->used_maps = used_maps_new; 4211*ef15314aSYiFei Zhu 4212*ef15314aSYiFei Zhu kfree(used_maps_old); 4213*ef15314aSYiFei Zhu 4214*ef15314aSYiFei Zhu out_unlock: 4215*ef15314aSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 4216*ef15314aSYiFei Zhu 4217*ef15314aSYiFei Zhu if (ret) 4218*ef15314aSYiFei Zhu bpf_map_put(map); 4219*ef15314aSYiFei Zhu out_prog_put: 4220*ef15314aSYiFei Zhu bpf_prog_put(prog); 4221*ef15314aSYiFei Zhu return ret; 4222*ef15314aSYiFei Zhu } 4223*ef15314aSYiFei Zhu 422499c55f7dSAlexei Starovoitov SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 422599c55f7dSAlexei Starovoitov { 42268096f229SGreg Kroah-Hartman union bpf_attr attr; 422799c55f7dSAlexei Starovoitov int err; 422899c55f7dSAlexei Starovoitov 42292c78ee89SAlexei Starovoitov if (sysctl_unprivileged_bpf_disabled && !bpf_capable()) 423099c55f7dSAlexei Starovoitov return -EPERM; 423199c55f7dSAlexei Starovoitov 4232dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size); 423399c55f7dSAlexei Starovoitov if (err) 423499c55f7dSAlexei Starovoitov return err; 42351e270976SMartin KaFai Lau size = min_t(u32, size, sizeof(attr)); 423699c55f7dSAlexei Starovoitov 423799c55f7dSAlexei Starovoitov /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 42388096f229SGreg Kroah-Hartman memset(&attr, 0, sizeof(attr)); 423999c55f7dSAlexei Starovoitov if (copy_from_user(&attr, uattr, size) != 0) 424099c55f7dSAlexei Starovoitov return -EFAULT; 424199c55f7dSAlexei Starovoitov 4242afdb09c7SChenbo Feng err = security_bpf(cmd, &attr, size); 4243afdb09c7SChenbo Feng if (err < 0) 4244afdb09c7SChenbo Feng return err; 4245afdb09c7SChenbo Feng 424699c55f7dSAlexei Starovoitov switch (cmd) { 424799c55f7dSAlexei Starovoitov case BPF_MAP_CREATE: 424899c55f7dSAlexei Starovoitov err = map_create(&attr); 424999c55f7dSAlexei Starovoitov break; 4250db20fd2bSAlexei Starovoitov case BPF_MAP_LOOKUP_ELEM: 4251db20fd2bSAlexei Starovoitov err = map_lookup_elem(&attr); 4252db20fd2bSAlexei Starovoitov break; 4253db20fd2bSAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 4254db20fd2bSAlexei Starovoitov err = map_update_elem(&attr); 4255db20fd2bSAlexei Starovoitov break; 4256db20fd2bSAlexei Starovoitov case BPF_MAP_DELETE_ELEM: 4257db20fd2bSAlexei Starovoitov err = map_delete_elem(&attr); 4258db20fd2bSAlexei Starovoitov break; 4259db20fd2bSAlexei Starovoitov case BPF_MAP_GET_NEXT_KEY: 4260db20fd2bSAlexei Starovoitov err = map_get_next_key(&attr); 4261db20fd2bSAlexei Starovoitov break; 426287df15deSDaniel Borkmann case BPF_MAP_FREEZE: 426387df15deSDaniel Borkmann err = map_freeze(&attr); 426487df15deSDaniel Borkmann break; 426509756af4SAlexei Starovoitov case BPF_PROG_LOAD: 4266838e9690SYonghong Song err = bpf_prog_load(&attr, uattr); 426709756af4SAlexei Starovoitov break; 4268b2197755SDaniel Borkmann case BPF_OBJ_PIN: 4269b2197755SDaniel Borkmann err = bpf_obj_pin(&attr); 4270b2197755SDaniel Borkmann break; 4271b2197755SDaniel Borkmann case BPF_OBJ_GET: 4272b2197755SDaniel Borkmann err = bpf_obj_get(&attr); 4273b2197755SDaniel Borkmann break; 4274f4324551SDaniel Mack case BPF_PROG_ATTACH: 4275f4324551SDaniel Mack err = bpf_prog_attach(&attr); 4276f4324551SDaniel Mack break; 4277f4324551SDaniel Mack case BPF_PROG_DETACH: 4278f4324551SDaniel Mack err = bpf_prog_detach(&attr); 4279f4324551SDaniel Mack break; 4280468e2f64SAlexei Starovoitov case BPF_PROG_QUERY: 4281468e2f64SAlexei Starovoitov err = bpf_prog_query(&attr, uattr); 4282468e2f64SAlexei Starovoitov break; 42831cf1cae9SAlexei Starovoitov case BPF_PROG_TEST_RUN: 42841cf1cae9SAlexei Starovoitov err = bpf_prog_test_run(&attr, uattr); 42851cf1cae9SAlexei Starovoitov break; 428634ad5580SMartin KaFai Lau case BPF_PROG_GET_NEXT_ID: 428734ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 428834ad5580SMartin KaFai Lau &prog_idr, &prog_idr_lock); 428934ad5580SMartin KaFai Lau break; 429034ad5580SMartin KaFai Lau case BPF_MAP_GET_NEXT_ID: 429134ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 429234ad5580SMartin KaFai Lau &map_idr, &map_idr_lock); 429334ad5580SMartin KaFai Lau break; 42941b9ed84eSQuentin Monnet case BPF_BTF_GET_NEXT_ID: 42951b9ed84eSQuentin Monnet err = bpf_obj_get_next_id(&attr, uattr, 42961b9ed84eSQuentin Monnet &btf_idr, &btf_idr_lock); 42971b9ed84eSQuentin Monnet break; 4298b16d9aa4SMartin KaFai Lau case BPF_PROG_GET_FD_BY_ID: 4299b16d9aa4SMartin KaFai Lau err = bpf_prog_get_fd_by_id(&attr); 4300b16d9aa4SMartin KaFai Lau break; 4301bd5f5f4eSMartin KaFai Lau case BPF_MAP_GET_FD_BY_ID: 4302bd5f5f4eSMartin KaFai Lau err = bpf_map_get_fd_by_id(&attr); 4303bd5f5f4eSMartin KaFai Lau break; 43041e270976SMartin KaFai Lau case BPF_OBJ_GET_INFO_BY_FD: 43051e270976SMartin KaFai Lau err = bpf_obj_get_info_by_fd(&attr, uattr); 43061e270976SMartin KaFai Lau break; 4307c4f6699dSAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN: 4308c4f6699dSAlexei Starovoitov err = bpf_raw_tracepoint_open(&attr); 4309c4f6699dSAlexei Starovoitov break; 4310f56a653cSMartin KaFai Lau case BPF_BTF_LOAD: 4311f56a653cSMartin KaFai Lau err = bpf_btf_load(&attr); 4312f56a653cSMartin KaFai Lau break; 431378958fcaSMartin KaFai Lau case BPF_BTF_GET_FD_BY_ID: 431478958fcaSMartin KaFai Lau err = bpf_btf_get_fd_by_id(&attr); 431578958fcaSMartin KaFai Lau break; 431641bdc4b4SYonghong Song case BPF_TASK_FD_QUERY: 431741bdc4b4SYonghong Song err = bpf_task_fd_query(&attr, uattr); 431841bdc4b4SYonghong Song break; 4319bd513cd0SMauricio Vasquez B case BPF_MAP_LOOKUP_AND_DELETE_ELEM: 4320bd513cd0SMauricio Vasquez B err = map_lookup_and_delete_elem(&attr); 4321bd513cd0SMauricio Vasquez B break; 4322cb4d03abSBrian Vazquez case BPF_MAP_LOOKUP_BATCH: 4323cb4d03abSBrian Vazquez err = bpf_map_do_batch(&attr, uattr, BPF_MAP_LOOKUP_BATCH); 4324cb4d03abSBrian Vazquez break; 432505799638SYonghong Song case BPF_MAP_LOOKUP_AND_DELETE_BATCH: 432605799638SYonghong Song err = bpf_map_do_batch(&attr, uattr, 432705799638SYonghong Song BPF_MAP_LOOKUP_AND_DELETE_BATCH); 432805799638SYonghong Song break; 4329aa2e93b8SBrian Vazquez case BPF_MAP_UPDATE_BATCH: 4330aa2e93b8SBrian Vazquez err = bpf_map_do_batch(&attr, uattr, BPF_MAP_UPDATE_BATCH); 4331aa2e93b8SBrian Vazquez break; 4332aa2e93b8SBrian Vazquez case BPF_MAP_DELETE_BATCH: 4333aa2e93b8SBrian Vazquez err = bpf_map_do_batch(&attr, uattr, BPF_MAP_DELETE_BATCH); 4334aa2e93b8SBrian Vazquez break; 4335af6eea57SAndrii Nakryiko case BPF_LINK_CREATE: 4336af6eea57SAndrii Nakryiko err = link_create(&attr); 4337af6eea57SAndrii Nakryiko break; 43380c991ebcSAndrii Nakryiko case BPF_LINK_UPDATE: 43390c991ebcSAndrii Nakryiko err = link_update(&attr); 43400c991ebcSAndrii Nakryiko break; 43412d602c8cSAndrii Nakryiko case BPF_LINK_GET_FD_BY_ID: 43422d602c8cSAndrii Nakryiko err = bpf_link_get_fd_by_id(&attr); 43432d602c8cSAndrii Nakryiko break; 43442d602c8cSAndrii Nakryiko case BPF_LINK_GET_NEXT_ID: 43452d602c8cSAndrii Nakryiko err = bpf_obj_get_next_id(&attr, uattr, 43462d602c8cSAndrii Nakryiko &link_idr, &link_idr_lock); 43472d602c8cSAndrii Nakryiko break; 4348d46edd67SSong Liu case BPF_ENABLE_STATS: 4349d46edd67SSong Liu err = bpf_enable_stats(&attr); 4350d46edd67SSong Liu break; 4351ac51d99bSYonghong Song case BPF_ITER_CREATE: 4352ac51d99bSYonghong Song err = bpf_iter_create(&attr); 4353ac51d99bSYonghong Song break; 435473b11c2aSAndrii Nakryiko case BPF_LINK_DETACH: 435573b11c2aSAndrii Nakryiko err = link_detach(&attr); 435673b11c2aSAndrii Nakryiko break; 4357*ef15314aSYiFei Zhu case BPF_PROG_BIND_MAP: 4358*ef15314aSYiFei Zhu err = bpf_prog_bind_map(&attr); 4359*ef15314aSYiFei Zhu break; 436099c55f7dSAlexei Starovoitov default: 436199c55f7dSAlexei Starovoitov err = -EINVAL; 436299c55f7dSAlexei Starovoitov break; 436399c55f7dSAlexei Starovoitov } 436499c55f7dSAlexei Starovoitov 436599c55f7dSAlexei Starovoitov return err; 436699c55f7dSAlexei Starovoitov } 4367