15b497af4SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 299c55f7dSAlexei Starovoitov /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 399c55f7dSAlexei Starovoitov */ 499c55f7dSAlexei Starovoitov #include <linux/bpf.h> 5a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 6f4364dcfSSean Young #include <linux/bpf_lirc.h> 7f56a653cSMartin KaFai Lau #include <linux/btf.h> 899c55f7dSAlexei Starovoitov #include <linux/syscalls.h> 999c55f7dSAlexei Starovoitov #include <linux/slab.h> 103f07c014SIngo Molnar #include <linux/sched/signal.h> 11d407bd25SDaniel Borkmann #include <linux/vmalloc.h> 12d407bd25SDaniel Borkmann #include <linux/mmzone.h> 1399c55f7dSAlexei Starovoitov #include <linux/anon_inodes.h> 1441bdc4b4SYonghong Song #include <linux/fdtable.h> 15db20fd2bSAlexei Starovoitov #include <linux/file.h> 1641bdc4b4SYonghong Song #include <linux/fs.h> 1709756af4SAlexei Starovoitov #include <linux/license.h> 1809756af4SAlexei Starovoitov #include <linux/filter.h> 192541517cSAlexei Starovoitov #include <linux/version.h> 20535e7b4bSMickaël Salaün #include <linux/kernel.h> 21dc4bb0e2SMartin KaFai Lau #include <linux/idr.h> 22cb4d2b3fSMartin KaFai Lau #include <linux/cred.h> 23cb4d2b3fSMartin KaFai Lau #include <linux/timekeeping.h> 24cb4d2b3fSMartin KaFai Lau #include <linux/ctype.h> 259ef09e35SMark Rutland #include <linux/nospec.h> 26bae141f5SDaniel Borkmann #include <linux/audit.h> 27ccfe29ebSAlexei Starovoitov #include <uapi/linux/btf.h> 2899c55f7dSAlexei Starovoitov 29da765a2fSDaniel Borkmann #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 3014dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 3114dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 32da765a2fSDaniel Borkmann #define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY) 3314dc6f04SMartin KaFai Lau #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 34da765a2fSDaniel Borkmann #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \ 35da765a2fSDaniel Borkmann IS_FD_HASH(map)) 3614dc6f04SMartin KaFai Lau 376e71b04aSChenbo Feng #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) 386e71b04aSChenbo Feng 39b121d1e7SAlexei Starovoitov DEFINE_PER_CPU(int, bpf_prog_active); 40dc4bb0e2SMartin KaFai Lau static DEFINE_IDR(prog_idr); 41dc4bb0e2SMartin KaFai Lau static DEFINE_SPINLOCK(prog_idr_lock); 42f3f1c054SMartin KaFai Lau static DEFINE_IDR(map_idr); 43f3f1c054SMartin KaFai Lau static DEFINE_SPINLOCK(map_idr_lock); 44b121d1e7SAlexei Starovoitov 451be7f75dSAlexei Starovoitov int sysctl_unprivileged_bpf_disabled __read_mostly; 461be7f75dSAlexei Starovoitov 4740077e0cSJohannes Berg static const struct bpf_map_ops * const bpf_map_types[] = { 4891cc1a99SAlexei Starovoitov #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) 4940077e0cSJohannes Berg #define BPF_MAP_TYPE(_id, _ops) \ 5040077e0cSJohannes Berg [_id] = &_ops, 5140077e0cSJohannes Berg #include <linux/bpf_types.h> 5240077e0cSJohannes Berg #undef BPF_PROG_TYPE 5340077e0cSJohannes Berg #undef BPF_MAP_TYPE 5440077e0cSJohannes Berg }; 5599c55f7dSAlexei Starovoitov 56752ba56fSMickaël Salaün /* 57752ba56fSMickaël Salaün * If we're handed a bigger struct than we know of, ensure all the unknown bits 58752ba56fSMickaël Salaün * are 0 - i.e. new user-space does not rely on any kernel feature extensions 59752ba56fSMickaël Salaün * we don't know about yet. 60752ba56fSMickaël Salaün * 61752ba56fSMickaël Salaün * There is a ToCToU between this function call and the following 62752ba56fSMickaël Salaün * copy_from_user() call. However, this is not a concern since this function is 63752ba56fSMickaël Salaün * meant to be a future-proofing of bits. 64752ba56fSMickaël Salaün */ 65dcab51f1SMartin KaFai Lau int bpf_check_uarg_tail_zero(void __user *uaddr, 6658291a74SMickaël Salaün size_t expected_size, 6758291a74SMickaël Salaün size_t actual_size) 6858291a74SMickaël Salaün { 6958291a74SMickaël Salaün unsigned char __user *addr; 7058291a74SMickaël Salaün unsigned char __user *end; 7158291a74SMickaël Salaün unsigned char val; 7258291a74SMickaël Salaün int err; 7358291a74SMickaël Salaün 74752ba56fSMickaël Salaün if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 75752ba56fSMickaël Salaün return -E2BIG; 76752ba56fSMickaël Salaün 7796d4f267SLinus Torvalds if (unlikely(!access_ok(uaddr, actual_size))) 78752ba56fSMickaël Salaün return -EFAULT; 79752ba56fSMickaël Salaün 8058291a74SMickaël Salaün if (actual_size <= expected_size) 8158291a74SMickaël Salaün return 0; 8258291a74SMickaël Salaün 8358291a74SMickaël Salaün addr = uaddr + expected_size; 8458291a74SMickaël Salaün end = uaddr + actual_size; 8558291a74SMickaël Salaün 8658291a74SMickaël Salaün for (; addr < end; addr++) { 8758291a74SMickaël Salaün err = get_user(val, addr); 8858291a74SMickaël Salaün if (err) 8958291a74SMickaël Salaün return err; 9058291a74SMickaël Salaün if (val) 9158291a74SMickaël Salaün return -E2BIG; 9258291a74SMickaël Salaün } 9358291a74SMickaël Salaün 9458291a74SMickaël Salaün return 0; 9558291a74SMickaël Salaün } 9658291a74SMickaël Salaün 97a3884572SJakub Kicinski const struct bpf_map_ops bpf_map_offload_ops = { 98a3884572SJakub Kicinski .map_alloc = bpf_map_offload_map_alloc, 99a3884572SJakub Kicinski .map_free = bpf_map_offload_map_free, 100e8d2bec0SDaniel Borkmann .map_check_btf = map_check_no_btf, 101a3884572SJakub Kicinski }; 102a3884572SJakub Kicinski 10399c55f7dSAlexei Starovoitov static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 10499c55f7dSAlexei Starovoitov { 1051110f3a9SJakub Kicinski const struct bpf_map_ops *ops; 1069ef09e35SMark Rutland u32 type = attr->map_type; 10799c55f7dSAlexei Starovoitov struct bpf_map *map; 1081110f3a9SJakub Kicinski int err; 10999c55f7dSAlexei Starovoitov 1109ef09e35SMark Rutland if (type >= ARRAY_SIZE(bpf_map_types)) 1111110f3a9SJakub Kicinski return ERR_PTR(-EINVAL); 1129ef09e35SMark Rutland type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types)); 1139ef09e35SMark Rutland ops = bpf_map_types[type]; 1141110f3a9SJakub Kicinski if (!ops) 11540077e0cSJohannes Berg return ERR_PTR(-EINVAL); 11640077e0cSJohannes Berg 1171110f3a9SJakub Kicinski if (ops->map_alloc_check) { 1181110f3a9SJakub Kicinski err = ops->map_alloc_check(attr); 1191110f3a9SJakub Kicinski if (err) 1201110f3a9SJakub Kicinski return ERR_PTR(err); 1211110f3a9SJakub Kicinski } 122a3884572SJakub Kicinski if (attr->map_ifindex) 123a3884572SJakub Kicinski ops = &bpf_map_offload_ops; 1241110f3a9SJakub Kicinski map = ops->map_alloc(attr); 12599c55f7dSAlexei Starovoitov if (IS_ERR(map)) 12699c55f7dSAlexei Starovoitov return map; 1271110f3a9SJakub Kicinski map->ops = ops; 1289ef09e35SMark Rutland map->map_type = type; 12999c55f7dSAlexei Starovoitov return map; 13099c55f7dSAlexei Starovoitov } 13199c55f7dSAlexei Starovoitov 13215c14a3dSBrian Vazquez static u32 bpf_map_value_size(struct bpf_map *map) 13315c14a3dSBrian Vazquez { 13415c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 13515c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 13615c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 13715c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 13815c14a3dSBrian Vazquez return round_up(map->value_size, 8) * num_possible_cpus(); 13915c14a3dSBrian Vazquez else if (IS_FD_MAP(map)) 14015c14a3dSBrian Vazquez return sizeof(u32); 14115c14a3dSBrian Vazquez else 14215c14a3dSBrian Vazquez return map->value_size; 14315c14a3dSBrian Vazquez } 14415c14a3dSBrian Vazquez 14515c14a3dSBrian Vazquez static void maybe_wait_bpf_programs(struct bpf_map *map) 14615c14a3dSBrian Vazquez { 14715c14a3dSBrian Vazquez /* Wait for any running BPF programs to complete so that 14815c14a3dSBrian Vazquez * userspace, when we return to it, knows that all programs 14915c14a3dSBrian Vazquez * that could be running use the new map value. 15015c14a3dSBrian Vazquez */ 15115c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS || 15215c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 15315c14a3dSBrian Vazquez synchronize_rcu(); 15415c14a3dSBrian Vazquez } 15515c14a3dSBrian Vazquez 15615c14a3dSBrian Vazquez static int bpf_map_update_value(struct bpf_map *map, struct fd f, void *key, 15715c14a3dSBrian Vazquez void *value, __u64 flags) 15815c14a3dSBrian Vazquez { 15915c14a3dSBrian Vazquez int err; 16015c14a3dSBrian Vazquez 16115c14a3dSBrian Vazquez /* Need to create a kthread, thus must support schedule */ 16215c14a3dSBrian Vazquez if (bpf_map_is_dev_bound(map)) { 16315c14a3dSBrian Vazquez return bpf_map_offload_update_elem(map, key, value, flags); 16415c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_CPUMAP || 16515c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_SOCKHASH || 16615c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_SOCKMAP || 16715c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 16815c14a3dSBrian Vazquez return map->ops->map_update_elem(map, key, value, flags); 16915c14a3dSBrian Vazquez } else if (IS_FD_PROG_ARRAY(map)) { 17015c14a3dSBrian Vazquez return bpf_fd_array_map_update_elem(map, f.file, key, value, 17115c14a3dSBrian Vazquez flags); 17215c14a3dSBrian Vazquez } 17315c14a3dSBrian Vazquez 17415c14a3dSBrian Vazquez /* must increment bpf_prog_active to avoid kprobe+bpf triggering from 17515c14a3dSBrian Vazquez * inside bpf map update or delete otherwise deadlocks are possible 17615c14a3dSBrian Vazquez */ 17715c14a3dSBrian Vazquez preempt_disable(); 17815c14a3dSBrian Vazquez __this_cpu_inc(bpf_prog_active); 17915c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 18015c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 18115c14a3dSBrian Vazquez err = bpf_percpu_hash_update(map, key, value, flags); 18215c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 18315c14a3dSBrian Vazquez err = bpf_percpu_array_update(map, key, value, flags); 18415c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 18515c14a3dSBrian Vazquez err = bpf_percpu_cgroup_storage_update(map, key, value, 18615c14a3dSBrian Vazquez flags); 18715c14a3dSBrian Vazquez } else if (IS_FD_ARRAY(map)) { 18815c14a3dSBrian Vazquez rcu_read_lock(); 18915c14a3dSBrian Vazquez err = bpf_fd_array_map_update_elem(map, f.file, key, value, 19015c14a3dSBrian Vazquez flags); 19115c14a3dSBrian Vazquez rcu_read_unlock(); 19215c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 19315c14a3dSBrian Vazquez rcu_read_lock(); 19415c14a3dSBrian Vazquez err = bpf_fd_htab_map_update_elem(map, f.file, key, value, 19515c14a3dSBrian Vazquez flags); 19615c14a3dSBrian Vazquez rcu_read_unlock(); 19715c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 19815c14a3dSBrian Vazquez /* rcu_read_lock() is not needed */ 19915c14a3dSBrian Vazquez err = bpf_fd_reuseport_array_update_elem(map, key, value, 20015c14a3dSBrian Vazquez flags); 20115c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 20215c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_STACK) { 20315c14a3dSBrian Vazquez err = map->ops->map_push_elem(map, value, flags); 20415c14a3dSBrian Vazquez } else { 20515c14a3dSBrian Vazquez rcu_read_lock(); 20615c14a3dSBrian Vazquez err = map->ops->map_update_elem(map, key, value, flags); 20715c14a3dSBrian Vazquez rcu_read_unlock(); 20815c14a3dSBrian Vazquez } 20915c14a3dSBrian Vazquez __this_cpu_dec(bpf_prog_active); 21015c14a3dSBrian Vazquez preempt_enable(); 21115c14a3dSBrian Vazquez maybe_wait_bpf_programs(map); 21215c14a3dSBrian Vazquez 21315c14a3dSBrian Vazquez return err; 21415c14a3dSBrian Vazquez } 21515c14a3dSBrian Vazquez 21615c14a3dSBrian Vazquez static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value, 21715c14a3dSBrian Vazquez __u64 flags) 21815c14a3dSBrian Vazquez { 21915c14a3dSBrian Vazquez void *ptr; 22015c14a3dSBrian Vazquez int err; 22115c14a3dSBrian Vazquez 222cb4d03abSBrian Vazquez if (bpf_map_is_dev_bound(map)) 223cb4d03abSBrian Vazquez return bpf_map_offload_lookup_elem(map, key, value); 22415c14a3dSBrian Vazquez 22515c14a3dSBrian Vazquez preempt_disable(); 22615c14a3dSBrian Vazquez this_cpu_inc(bpf_prog_active); 22715c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 22815c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 22915c14a3dSBrian Vazquez err = bpf_percpu_hash_copy(map, key, value); 23015c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 23115c14a3dSBrian Vazquez err = bpf_percpu_array_copy(map, key, value); 23215c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 23315c14a3dSBrian Vazquez err = bpf_percpu_cgroup_storage_copy(map, key, value); 23415c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 23515c14a3dSBrian Vazquez err = bpf_stackmap_copy(map, key, value); 23615c14a3dSBrian Vazquez } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) { 23715c14a3dSBrian Vazquez err = bpf_fd_array_map_lookup_elem(map, key, value); 23815c14a3dSBrian Vazquez } else if (IS_FD_HASH(map)) { 23915c14a3dSBrian Vazquez err = bpf_fd_htab_map_lookup_elem(map, key, value); 24015c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 24115c14a3dSBrian Vazquez err = bpf_fd_reuseport_array_lookup_elem(map, key, value); 24215c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 24315c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_STACK) { 24415c14a3dSBrian Vazquez err = map->ops->map_peek_elem(map, value); 24515c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 24615c14a3dSBrian Vazquez /* struct_ops map requires directly updating "value" */ 24715c14a3dSBrian Vazquez err = bpf_struct_ops_map_sys_lookup_elem(map, key, value); 24815c14a3dSBrian Vazquez } else { 24915c14a3dSBrian Vazquez rcu_read_lock(); 25015c14a3dSBrian Vazquez if (map->ops->map_lookup_elem_sys_only) 25115c14a3dSBrian Vazquez ptr = map->ops->map_lookup_elem_sys_only(map, key); 25215c14a3dSBrian Vazquez else 25315c14a3dSBrian Vazquez ptr = map->ops->map_lookup_elem(map, key); 25415c14a3dSBrian Vazquez if (IS_ERR(ptr)) { 25515c14a3dSBrian Vazquez err = PTR_ERR(ptr); 25615c14a3dSBrian Vazquez } else if (!ptr) { 25715c14a3dSBrian Vazquez err = -ENOENT; 25815c14a3dSBrian Vazquez } else { 25915c14a3dSBrian Vazquez err = 0; 26015c14a3dSBrian Vazquez if (flags & BPF_F_LOCK) 26115c14a3dSBrian Vazquez /* lock 'ptr' and copy everything but lock */ 26215c14a3dSBrian Vazquez copy_map_value_locked(map, value, ptr, true); 26315c14a3dSBrian Vazquez else 26415c14a3dSBrian Vazquez copy_map_value(map, value, ptr); 26515c14a3dSBrian Vazquez /* mask lock, since value wasn't zero inited */ 26615c14a3dSBrian Vazquez check_and_init_map_lock(map, value); 26715c14a3dSBrian Vazquez } 26815c14a3dSBrian Vazquez rcu_read_unlock(); 26915c14a3dSBrian Vazquez } 27015c14a3dSBrian Vazquez 27115c14a3dSBrian Vazquez this_cpu_dec(bpf_prog_active); 27215c14a3dSBrian Vazquez preempt_enable(); 27315c14a3dSBrian Vazquez maybe_wait_bpf_programs(map); 27415c14a3dSBrian Vazquez 27515c14a3dSBrian Vazquez return err; 27615c14a3dSBrian Vazquez } 27715c14a3dSBrian Vazquez 278196e8ca7SDaniel Borkmann static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable) 279d407bd25SDaniel Borkmann { 280f01a7dbeSMartynas Pumputis /* We really just want to fail instead of triggering OOM killer 281f01a7dbeSMartynas Pumputis * under memory pressure, therefore we set __GFP_NORETRY to kmalloc, 282f01a7dbeSMartynas Pumputis * which is used for lower order allocation requests. 283f01a7dbeSMartynas Pumputis * 284f01a7dbeSMartynas Pumputis * It has been observed that higher order allocation requests done by 285f01a7dbeSMartynas Pumputis * vmalloc with __GFP_NORETRY being set might fail due to not trying 286f01a7dbeSMartynas Pumputis * to reclaim memory from the page cache, thus we set 287f01a7dbeSMartynas Pumputis * __GFP_RETRY_MAYFAIL to avoid such situations. 288d407bd25SDaniel Borkmann */ 289f01a7dbeSMartynas Pumputis 290f01a7dbeSMartynas Pumputis const gfp_t flags = __GFP_NOWARN | __GFP_ZERO; 291d407bd25SDaniel Borkmann void *area; 292d407bd25SDaniel Borkmann 293196e8ca7SDaniel Borkmann if (size >= SIZE_MAX) 294196e8ca7SDaniel Borkmann return NULL; 295196e8ca7SDaniel Borkmann 296fc970227SAndrii Nakryiko /* kmalloc()'ed memory can't be mmap()'ed */ 297fc970227SAndrii Nakryiko if (!mmapable && size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 298f01a7dbeSMartynas Pumputis area = kmalloc_node(size, GFP_USER | __GFP_NORETRY | flags, 299f01a7dbeSMartynas Pumputis numa_node); 300d407bd25SDaniel Borkmann if (area != NULL) 301d407bd25SDaniel Borkmann return area; 302d407bd25SDaniel Borkmann } 303fc970227SAndrii Nakryiko if (mmapable) { 304fc970227SAndrii Nakryiko BUG_ON(!PAGE_ALIGNED(size)); 305fc970227SAndrii Nakryiko return vmalloc_user_node_flags(size, numa_node, GFP_KERNEL | 306fc970227SAndrii Nakryiko __GFP_RETRY_MAYFAIL | flags); 307fc970227SAndrii Nakryiko } 308f01a7dbeSMartynas Pumputis return __vmalloc_node_flags_caller(size, numa_node, 309f01a7dbeSMartynas Pumputis GFP_KERNEL | __GFP_RETRY_MAYFAIL | 310f01a7dbeSMartynas Pumputis flags, __builtin_return_address(0)); 311d407bd25SDaniel Borkmann } 312d407bd25SDaniel Borkmann 313196e8ca7SDaniel Borkmann void *bpf_map_area_alloc(u64 size, int numa_node) 314fc970227SAndrii Nakryiko { 315fc970227SAndrii Nakryiko return __bpf_map_area_alloc(size, numa_node, false); 316fc970227SAndrii Nakryiko } 317fc970227SAndrii Nakryiko 318196e8ca7SDaniel Borkmann void *bpf_map_area_mmapable_alloc(u64 size, int numa_node) 319fc970227SAndrii Nakryiko { 320fc970227SAndrii Nakryiko return __bpf_map_area_alloc(size, numa_node, true); 321fc970227SAndrii Nakryiko } 322fc970227SAndrii Nakryiko 323d407bd25SDaniel Borkmann void bpf_map_area_free(void *area) 324d407bd25SDaniel Borkmann { 325d407bd25SDaniel Borkmann kvfree(area); 326d407bd25SDaniel Borkmann } 327d407bd25SDaniel Borkmann 328be70bcd5SDaniel Borkmann static u32 bpf_map_flags_retain_permanent(u32 flags) 329be70bcd5SDaniel Borkmann { 330be70bcd5SDaniel Borkmann /* Some map creation flags are not tied to the map object but 331be70bcd5SDaniel Borkmann * rather to the map fd instead, so they have no meaning upon 332be70bcd5SDaniel Borkmann * map object inspection since multiple file descriptors with 333be70bcd5SDaniel Borkmann * different (access) properties can exist here. Thus, given 334be70bcd5SDaniel Borkmann * this has zero meaning for the map itself, lets clear these 335be70bcd5SDaniel Borkmann * from here. 336be70bcd5SDaniel Borkmann */ 337be70bcd5SDaniel Borkmann return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY); 338be70bcd5SDaniel Borkmann } 339be70bcd5SDaniel Borkmann 340bd475643SJakub Kicinski void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) 341bd475643SJakub Kicinski { 342bd475643SJakub Kicinski map->map_type = attr->map_type; 343bd475643SJakub Kicinski map->key_size = attr->key_size; 344bd475643SJakub Kicinski map->value_size = attr->value_size; 345bd475643SJakub Kicinski map->max_entries = attr->max_entries; 346be70bcd5SDaniel Borkmann map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags); 347bd475643SJakub Kicinski map->numa_node = bpf_map_attr_numa_node(attr); 348bd475643SJakub Kicinski } 349bd475643SJakub Kicinski 3500a4c58f5SRoman Gushchin static int bpf_charge_memlock(struct user_struct *user, u32 pages) 351aaac3ba9SAlexei Starovoitov { 3520a4c58f5SRoman Gushchin unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 353aaac3ba9SAlexei Starovoitov 3540a4c58f5SRoman Gushchin if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) { 3550a4c58f5SRoman Gushchin atomic_long_sub(pages, &user->locked_vm); 356aaac3ba9SAlexei Starovoitov return -EPERM; 357aaac3ba9SAlexei Starovoitov } 358aaac3ba9SAlexei Starovoitov return 0; 359aaac3ba9SAlexei Starovoitov } 360aaac3ba9SAlexei Starovoitov 3610a4c58f5SRoman Gushchin static void bpf_uncharge_memlock(struct user_struct *user, u32 pages) 3620a4c58f5SRoman Gushchin { 363b936ca64SRoman Gushchin if (user) 3640a4c58f5SRoman Gushchin atomic_long_sub(pages, &user->locked_vm); 3650a4c58f5SRoman Gushchin } 3660a4c58f5SRoman Gushchin 367196e8ca7SDaniel Borkmann int bpf_map_charge_init(struct bpf_map_memory *mem, u64 size) 3680a4c58f5SRoman Gushchin { 369c85d6913SRoman Gushchin u32 pages = round_up(size, PAGE_SIZE) >> PAGE_SHIFT; 370c85d6913SRoman Gushchin struct user_struct *user; 3710a4c58f5SRoman Gushchin int ret; 3720a4c58f5SRoman Gushchin 373c85d6913SRoman Gushchin if (size >= U32_MAX - PAGE_SIZE) 374c85d6913SRoman Gushchin return -E2BIG; 375c85d6913SRoman Gushchin 376c85d6913SRoman Gushchin user = get_current_user(); 377b936ca64SRoman Gushchin ret = bpf_charge_memlock(user, pages); 3780a4c58f5SRoman Gushchin if (ret) { 3790a4c58f5SRoman Gushchin free_uid(user); 3800a4c58f5SRoman Gushchin return ret; 3810a4c58f5SRoman Gushchin } 382b936ca64SRoman Gushchin 383b936ca64SRoman Gushchin mem->pages = pages; 384b936ca64SRoman Gushchin mem->user = user; 385b936ca64SRoman Gushchin 386b936ca64SRoman Gushchin return 0; 3870a4c58f5SRoman Gushchin } 3880a4c58f5SRoman Gushchin 389b936ca64SRoman Gushchin void bpf_map_charge_finish(struct bpf_map_memory *mem) 390aaac3ba9SAlexei Starovoitov { 391b936ca64SRoman Gushchin bpf_uncharge_memlock(mem->user, mem->pages); 392b936ca64SRoman Gushchin free_uid(mem->user); 393b936ca64SRoman Gushchin } 3943539b96eSRoman Gushchin 395b936ca64SRoman Gushchin void bpf_map_charge_move(struct bpf_map_memory *dst, 396b936ca64SRoman Gushchin struct bpf_map_memory *src) 397b936ca64SRoman Gushchin { 398b936ca64SRoman Gushchin *dst = *src; 399b936ca64SRoman Gushchin 400b936ca64SRoman Gushchin /* Make sure src will not be used for the redundant uncharging. */ 401b936ca64SRoman Gushchin memset(src, 0, sizeof(struct bpf_map_memory)); 402aaac3ba9SAlexei Starovoitov } 403aaac3ba9SAlexei Starovoitov 4040a4c58f5SRoman Gushchin int bpf_map_charge_memlock(struct bpf_map *map, u32 pages) 4050a4c58f5SRoman Gushchin { 4060a4c58f5SRoman Gushchin int ret; 4070a4c58f5SRoman Gushchin 4083539b96eSRoman Gushchin ret = bpf_charge_memlock(map->memory.user, pages); 4090a4c58f5SRoman Gushchin if (ret) 4100a4c58f5SRoman Gushchin return ret; 4113539b96eSRoman Gushchin map->memory.pages += pages; 4120a4c58f5SRoman Gushchin return ret; 4130a4c58f5SRoman Gushchin } 4140a4c58f5SRoman Gushchin 4150a4c58f5SRoman Gushchin void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages) 4160a4c58f5SRoman Gushchin { 4173539b96eSRoman Gushchin bpf_uncharge_memlock(map->memory.user, pages); 4183539b96eSRoman Gushchin map->memory.pages -= pages; 4190a4c58f5SRoman Gushchin } 4200a4c58f5SRoman Gushchin 421f3f1c054SMartin KaFai Lau static int bpf_map_alloc_id(struct bpf_map *map) 422f3f1c054SMartin KaFai Lau { 423f3f1c054SMartin KaFai Lau int id; 424f3f1c054SMartin KaFai Lau 425b76354cdSShaohua Li idr_preload(GFP_KERNEL); 426f3f1c054SMartin KaFai Lau spin_lock_bh(&map_idr_lock); 427f3f1c054SMartin KaFai Lau id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 428f3f1c054SMartin KaFai Lau if (id > 0) 429f3f1c054SMartin KaFai Lau map->id = id; 430f3f1c054SMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 431b76354cdSShaohua Li idr_preload_end(); 432f3f1c054SMartin KaFai Lau 433f3f1c054SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 434f3f1c054SMartin KaFai Lau return -ENOSPC; 435f3f1c054SMartin KaFai Lau 436f3f1c054SMartin KaFai Lau return id > 0 ? 0 : id; 437f3f1c054SMartin KaFai Lau } 438f3f1c054SMartin KaFai Lau 439a3884572SJakub Kicinski void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) 440f3f1c054SMartin KaFai Lau { 441930651a7SEric Dumazet unsigned long flags; 442930651a7SEric Dumazet 443a3884572SJakub Kicinski /* Offloaded maps are removed from the IDR store when their device 444a3884572SJakub Kicinski * disappears - even if someone holds an fd to them they are unusable, 445a3884572SJakub Kicinski * the memory is gone, all ops will fail; they are simply waiting for 446a3884572SJakub Kicinski * refcnt to drop to be freed. 447a3884572SJakub Kicinski */ 448a3884572SJakub Kicinski if (!map->id) 449a3884572SJakub Kicinski return; 450a3884572SJakub Kicinski 451bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 452930651a7SEric Dumazet spin_lock_irqsave(&map_idr_lock, flags); 453bd5f5f4eSMartin KaFai Lau else 454bd5f5f4eSMartin KaFai Lau __acquire(&map_idr_lock); 455bd5f5f4eSMartin KaFai Lau 456f3f1c054SMartin KaFai Lau idr_remove(&map_idr, map->id); 457a3884572SJakub Kicinski map->id = 0; 458bd5f5f4eSMartin KaFai Lau 459bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 460930651a7SEric Dumazet spin_unlock_irqrestore(&map_idr_lock, flags); 461bd5f5f4eSMartin KaFai Lau else 462bd5f5f4eSMartin KaFai Lau __release(&map_idr_lock); 463f3f1c054SMartin KaFai Lau } 464f3f1c054SMartin KaFai Lau 46599c55f7dSAlexei Starovoitov /* called from workqueue */ 46699c55f7dSAlexei Starovoitov static void bpf_map_free_deferred(struct work_struct *work) 46799c55f7dSAlexei Starovoitov { 46899c55f7dSAlexei Starovoitov struct bpf_map *map = container_of(work, struct bpf_map, work); 469b936ca64SRoman Gushchin struct bpf_map_memory mem; 47099c55f7dSAlexei Starovoitov 471b936ca64SRoman Gushchin bpf_map_charge_move(&mem, &map->memory); 472afdb09c7SChenbo Feng security_bpf_map_free(map); 47399c55f7dSAlexei Starovoitov /* implementation dependent freeing */ 47499c55f7dSAlexei Starovoitov map->ops->map_free(map); 475b936ca64SRoman Gushchin bpf_map_charge_finish(&mem); 47699c55f7dSAlexei Starovoitov } 47799c55f7dSAlexei Starovoitov 478c9da161cSDaniel Borkmann static void bpf_map_put_uref(struct bpf_map *map) 479c9da161cSDaniel Borkmann { 4801e0bd5a0SAndrii Nakryiko if (atomic64_dec_and_test(&map->usercnt)) { 481ba6b8de4SJohn Fastabend if (map->ops->map_release_uref) 482ba6b8de4SJohn Fastabend map->ops->map_release_uref(map); 483c9da161cSDaniel Borkmann } 484c9da161cSDaniel Borkmann } 485c9da161cSDaniel Borkmann 48699c55f7dSAlexei Starovoitov /* decrement map refcnt and schedule it for freeing via workqueue 48799c55f7dSAlexei Starovoitov * (unrelying map implementation ops->map_free() might sleep) 48899c55f7dSAlexei Starovoitov */ 489bd5f5f4eSMartin KaFai Lau static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) 49099c55f7dSAlexei Starovoitov { 4911e0bd5a0SAndrii Nakryiko if (atomic64_dec_and_test(&map->refcnt)) { 49234ad5580SMartin KaFai Lau /* bpf_map_free_id() must be called first */ 493bd5f5f4eSMartin KaFai Lau bpf_map_free_id(map, do_idr_lock); 49478958fcaSMartin KaFai Lau btf_put(map->btf); 49599c55f7dSAlexei Starovoitov INIT_WORK(&map->work, bpf_map_free_deferred); 49699c55f7dSAlexei Starovoitov schedule_work(&map->work); 49799c55f7dSAlexei Starovoitov } 49899c55f7dSAlexei Starovoitov } 49999c55f7dSAlexei Starovoitov 500bd5f5f4eSMartin KaFai Lau void bpf_map_put(struct bpf_map *map) 501bd5f5f4eSMartin KaFai Lau { 502bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, true); 503bd5f5f4eSMartin KaFai Lau } 504630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_put); 505bd5f5f4eSMartin KaFai Lau 506c9da161cSDaniel Borkmann void bpf_map_put_with_uref(struct bpf_map *map) 507c9da161cSDaniel Borkmann { 508c9da161cSDaniel Borkmann bpf_map_put_uref(map); 509c9da161cSDaniel Borkmann bpf_map_put(map); 510c9da161cSDaniel Borkmann } 511c9da161cSDaniel Borkmann 51299c55f7dSAlexei Starovoitov static int bpf_map_release(struct inode *inode, struct file *filp) 51399c55f7dSAlexei Starovoitov { 51461d1b6a4SDaniel Borkmann struct bpf_map *map = filp->private_data; 51561d1b6a4SDaniel Borkmann 51661d1b6a4SDaniel Borkmann if (map->ops->map_release) 51761d1b6a4SDaniel Borkmann map->ops->map_release(map, filp); 51861d1b6a4SDaniel Borkmann 51961d1b6a4SDaniel Borkmann bpf_map_put_with_uref(map); 52099c55f7dSAlexei Starovoitov return 0; 52199c55f7dSAlexei Starovoitov } 52299c55f7dSAlexei Starovoitov 52387df15deSDaniel Borkmann static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f) 52487df15deSDaniel Borkmann { 52587df15deSDaniel Borkmann fmode_t mode = f.file->f_mode; 52687df15deSDaniel Borkmann 52787df15deSDaniel Borkmann /* Our file permissions may have been overridden by global 52887df15deSDaniel Borkmann * map permissions facing syscall side. 52987df15deSDaniel Borkmann */ 53087df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) 53187df15deSDaniel Borkmann mode &= ~FMODE_CAN_WRITE; 53287df15deSDaniel Borkmann return mode; 53387df15deSDaniel Borkmann } 53487df15deSDaniel Borkmann 535f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 536f99bf205SDaniel Borkmann static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 537f99bf205SDaniel Borkmann { 538f99bf205SDaniel Borkmann const struct bpf_map *map = filp->private_data; 53921116b70SDaniel Borkmann const struct bpf_array *array; 5402beee5f5SDaniel Borkmann u32 type = 0, jited = 0; 54121116b70SDaniel Borkmann 54221116b70SDaniel Borkmann if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 54321116b70SDaniel Borkmann array = container_of(map, struct bpf_array, map); 5442beee5f5SDaniel Borkmann type = array->aux->type; 5452beee5f5SDaniel Borkmann jited = array->aux->jited; 54621116b70SDaniel Borkmann } 547f99bf205SDaniel Borkmann 548f99bf205SDaniel Borkmann seq_printf(m, 549f99bf205SDaniel Borkmann "map_type:\t%u\n" 550f99bf205SDaniel Borkmann "key_size:\t%u\n" 551f99bf205SDaniel Borkmann "value_size:\t%u\n" 552322cea2fSDaniel Borkmann "max_entries:\t%u\n" 55321116b70SDaniel Borkmann "map_flags:\t%#x\n" 5544316b409SDaniel Borkmann "memlock:\t%llu\n" 55587df15deSDaniel Borkmann "map_id:\t%u\n" 55687df15deSDaniel Borkmann "frozen:\t%u\n", 557f99bf205SDaniel Borkmann map->map_type, 558f99bf205SDaniel Borkmann map->key_size, 559f99bf205SDaniel Borkmann map->value_size, 560322cea2fSDaniel Borkmann map->max_entries, 56121116b70SDaniel Borkmann map->map_flags, 5623539b96eSRoman Gushchin map->memory.pages * 1ULL << PAGE_SHIFT, 56387df15deSDaniel Borkmann map->id, 56487df15deSDaniel Borkmann READ_ONCE(map->frozen)); 5652beee5f5SDaniel Borkmann if (type) { 5662beee5f5SDaniel Borkmann seq_printf(m, "owner_prog_type:\t%u\n", type); 5672beee5f5SDaniel Borkmann seq_printf(m, "owner_jited:\t%u\n", jited); 5689780c0abSDaniel Borkmann } 569f99bf205SDaniel Borkmann } 570f99bf205SDaniel Borkmann #endif 571f99bf205SDaniel Borkmann 5726e71b04aSChenbo Feng static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, 5736e71b04aSChenbo Feng loff_t *ppos) 5746e71b04aSChenbo Feng { 5756e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 5766e71b04aSChenbo Feng * f_mode with FMODE_CAN_READ. 5776e71b04aSChenbo Feng */ 5786e71b04aSChenbo Feng return -EINVAL; 5796e71b04aSChenbo Feng } 5806e71b04aSChenbo Feng 5816e71b04aSChenbo Feng static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, 5826e71b04aSChenbo Feng size_t siz, loff_t *ppos) 5836e71b04aSChenbo Feng { 5846e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 5856e71b04aSChenbo Feng * f_mode with FMODE_CAN_WRITE. 5866e71b04aSChenbo Feng */ 5876e71b04aSChenbo Feng return -EINVAL; 5886e71b04aSChenbo Feng } 5896e71b04aSChenbo Feng 590fc970227SAndrii Nakryiko /* called for any extra memory-mapped regions (except initial) */ 591fc970227SAndrii Nakryiko static void bpf_map_mmap_open(struct vm_area_struct *vma) 592fc970227SAndrii Nakryiko { 593fc970227SAndrii Nakryiko struct bpf_map *map = vma->vm_file->private_data; 594fc970227SAndrii Nakryiko 595fc970227SAndrii Nakryiko bpf_map_inc_with_uref(map); 596fc970227SAndrii Nakryiko 597fc970227SAndrii Nakryiko if (vma->vm_flags & VM_WRITE) { 598fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 599fc970227SAndrii Nakryiko map->writecnt++; 600fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 601fc970227SAndrii Nakryiko } 602fc970227SAndrii Nakryiko } 603fc970227SAndrii Nakryiko 604fc970227SAndrii Nakryiko /* called for all unmapped memory region (including initial) */ 605fc970227SAndrii Nakryiko static void bpf_map_mmap_close(struct vm_area_struct *vma) 606fc970227SAndrii Nakryiko { 607fc970227SAndrii Nakryiko struct bpf_map *map = vma->vm_file->private_data; 608fc970227SAndrii Nakryiko 609fc970227SAndrii Nakryiko if (vma->vm_flags & VM_WRITE) { 610fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 611fc970227SAndrii Nakryiko map->writecnt--; 612fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 613fc970227SAndrii Nakryiko } 614fc970227SAndrii Nakryiko 615fc970227SAndrii Nakryiko bpf_map_put_with_uref(map); 616fc970227SAndrii Nakryiko } 617fc970227SAndrii Nakryiko 618fc970227SAndrii Nakryiko static const struct vm_operations_struct bpf_map_default_vmops = { 619fc970227SAndrii Nakryiko .open = bpf_map_mmap_open, 620fc970227SAndrii Nakryiko .close = bpf_map_mmap_close, 621fc970227SAndrii Nakryiko }; 622fc970227SAndrii Nakryiko 623fc970227SAndrii Nakryiko static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma) 624fc970227SAndrii Nakryiko { 625fc970227SAndrii Nakryiko struct bpf_map *map = filp->private_data; 626fc970227SAndrii Nakryiko int err; 627fc970227SAndrii Nakryiko 628fc970227SAndrii Nakryiko if (!map->ops->map_mmap || map_value_has_spin_lock(map)) 629fc970227SAndrii Nakryiko return -ENOTSUPP; 630fc970227SAndrii Nakryiko 631fc970227SAndrii Nakryiko if (!(vma->vm_flags & VM_SHARED)) 632fc970227SAndrii Nakryiko return -EINVAL; 633fc970227SAndrii Nakryiko 634fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 635fc970227SAndrii Nakryiko 636fc970227SAndrii Nakryiko if ((vma->vm_flags & VM_WRITE) && map->frozen) { 637fc970227SAndrii Nakryiko err = -EPERM; 638fc970227SAndrii Nakryiko goto out; 639fc970227SAndrii Nakryiko } 640fc970227SAndrii Nakryiko 641fc970227SAndrii Nakryiko /* set default open/close callbacks */ 642fc970227SAndrii Nakryiko vma->vm_ops = &bpf_map_default_vmops; 643fc970227SAndrii Nakryiko vma->vm_private_data = map; 644fc970227SAndrii Nakryiko 645fc970227SAndrii Nakryiko err = map->ops->map_mmap(map, vma); 646fc970227SAndrii Nakryiko if (err) 647fc970227SAndrii Nakryiko goto out; 648fc970227SAndrii Nakryiko 649fc970227SAndrii Nakryiko bpf_map_inc_with_uref(map); 650fc970227SAndrii Nakryiko 651fc970227SAndrii Nakryiko if (vma->vm_flags & VM_WRITE) 652fc970227SAndrii Nakryiko map->writecnt++; 653fc970227SAndrii Nakryiko out: 654fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 655fc970227SAndrii Nakryiko return err; 656fc970227SAndrii Nakryiko } 657fc970227SAndrii Nakryiko 658f66e448cSChenbo Feng const struct file_operations bpf_map_fops = { 659f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 660f99bf205SDaniel Borkmann .show_fdinfo = bpf_map_show_fdinfo, 661f99bf205SDaniel Borkmann #endif 66299c55f7dSAlexei Starovoitov .release = bpf_map_release, 6636e71b04aSChenbo Feng .read = bpf_dummy_read, 6646e71b04aSChenbo Feng .write = bpf_dummy_write, 665fc970227SAndrii Nakryiko .mmap = bpf_map_mmap, 66699c55f7dSAlexei Starovoitov }; 66799c55f7dSAlexei Starovoitov 6686e71b04aSChenbo Feng int bpf_map_new_fd(struct bpf_map *map, int flags) 669aa79781bSDaniel Borkmann { 670afdb09c7SChenbo Feng int ret; 671afdb09c7SChenbo Feng 672afdb09c7SChenbo Feng ret = security_bpf_map(map, OPEN_FMODE(flags)); 673afdb09c7SChenbo Feng if (ret < 0) 674afdb09c7SChenbo Feng return ret; 675afdb09c7SChenbo Feng 676aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 6776e71b04aSChenbo Feng flags | O_CLOEXEC); 6786e71b04aSChenbo Feng } 6796e71b04aSChenbo Feng 6806e71b04aSChenbo Feng int bpf_get_file_flag(int flags) 6816e71b04aSChenbo Feng { 6826e71b04aSChenbo Feng if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) 6836e71b04aSChenbo Feng return -EINVAL; 6846e71b04aSChenbo Feng if (flags & BPF_F_RDONLY) 6856e71b04aSChenbo Feng return O_RDONLY; 6866e71b04aSChenbo Feng if (flags & BPF_F_WRONLY) 6876e71b04aSChenbo Feng return O_WRONLY; 6886e71b04aSChenbo Feng return O_RDWR; 689aa79781bSDaniel Borkmann } 690aa79781bSDaniel Borkmann 69199c55f7dSAlexei Starovoitov /* helper macro to check that unused fields 'union bpf_attr' are zero */ 69299c55f7dSAlexei Starovoitov #define CHECK_ATTR(CMD) \ 69399c55f7dSAlexei Starovoitov memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 69499c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD), 0, \ 69599c55f7dSAlexei Starovoitov sizeof(*attr) - \ 69699c55f7dSAlexei Starovoitov offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 69799c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD)) != NULL 69899c55f7dSAlexei Starovoitov 699cb4d2b3fSMartin KaFai Lau /* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes. 700cb4d2b3fSMartin KaFai Lau * Return 0 on success and < 0 on error. 701cb4d2b3fSMartin KaFai Lau */ 702cb4d2b3fSMartin KaFai Lau static int bpf_obj_name_cpy(char *dst, const char *src) 703cb4d2b3fSMartin KaFai Lau { 704cb4d2b3fSMartin KaFai Lau const char *end = src + BPF_OBJ_NAME_LEN; 705cb4d2b3fSMartin KaFai Lau 706473d9734SMartin KaFai Lau memset(dst, 0, BPF_OBJ_NAME_LEN); 7073e0ddc4fSDaniel Borkmann /* Copy all isalnum(), '_' and '.' chars. */ 708cb4d2b3fSMartin KaFai Lau while (src < end && *src) { 7093e0ddc4fSDaniel Borkmann if (!isalnum(*src) && 7103e0ddc4fSDaniel Borkmann *src != '_' && *src != '.') 711cb4d2b3fSMartin KaFai Lau return -EINVAL; 712cb4d2b3fSMartin KaFai Lau *dst++ = *src++; 713cb4d2b3fSMartin KaFai Lau } 714cb4d2b3fSMartin KaFai Lau 715cb4d2b3fSMartin KaFai Lau /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */ 716cb4d2b3fSMartin KaFai Lau if (src == end) 717cb4d2b3fSMartin KaFai Lau return -EINVAL; 718cb4d2b3fSMartin KaFai Lau 719cb4d2b3fSMartin KaFai Lau return 0; 720cb4d2b3fSMartin KaFai Lau } 721cb4d2b3fSMartin KaFai Lau 722e8d2bec0SDaniel Borkmann int map_check_no_btf(const struct bpf_map *map, 7231b2b234bSRoman Gushchin const struct btf *btf, 724e8d2bec0SDaniel Borkmann const struct btf_type *key_type, 725e8d2bec0SDaniel Borkmann const struct btf_type *value_type) 726e8d2bec0SDaniel Borkmann { 727e8d2bec0SDaniel Borkmann return -ENOTSUPP; 728e8d2bec0SDaniel Borkmann } 729e8d2bec0SDaniel Borkmann 730d83525caSAlexei Starovoitov static int map_check_btf(struct bpf_map *map, const struct btf *btf, 731e8d2bec0SDaniel Borkmann u32 btf_key_id, u32 btf_value_id) 732e8d2bec0SDaniel Borkmann { 733e8d2bec0SDaniel Borkmann const struct btf_type *key_type, *value_type; 734e8d2bec0SDaniel Borkmann u32 key_size, value_size; 735e8d2bec0SDaniel Borkmann int ret = 0; 736e8d2bec0SDaniel Borkmann 7372824ecb7SDaniel Borkmann /* Some maps allow key to be unspecified. */ 7382824ecb7SDaniel Borkmann if (btf_key_id) { 739e8d2bec0SDaniel Borkmann key_type = btf_type_id_size(btf, &btf_key_id, &key_size); 740e8d2bec0SDaniel Borkmann if (!key_type || key_size != map->key_size) 741e8d2bec0SDaniel Borkmann return -EINVAL; 7422824ecb7SDaniel Borkmann } else { 7432824ecb7SDaniel Borkmann key_type = btf_type_by_id(btf, 0); 7442824ecb7SDaniel Borkmann if (!map->ops->map_check_btf) 7452824ecb7SDaniel Borkmann return -EINVAL; 7462824ecb7SDaniel Borkmann } 747e8d2bec0SDaniel Borkmann 748e8d2bec0SDaniel Borkmann value_type = btf_type_id_size(btf, &btf_value_id, &value_size); 749e8d2bec0SDaniel Borkmann if (!value_type || value_size != map->value_size) 750e8d2bec0SDaniel Borkmann return -EINVAL; 751e8d2bec0SDaniel Borkmann 752d83525caSAlexei Starovoitov map->spin_lock_off = btf_find_spin_lock(btf, value_type); 753d83525caSAlexei Starovoitov 754d83525caSAlexei Starovoitov if (map_value_has_spin_lock(map)) { 755591fe988SDaniel Borkmann if (map->map_flags & BPF_F_RDONLY_PROG) 756591fe988SDaniel Borkmann return -EACCES; 757d83525caSAlexei Starovoitov if (map->map_type != BPF_MAP_TYPE_HASH && 758e16d2f1aSAlexei Starovoitov map->map_type != BPF_MAP_TYPE_ARRAY && 7596ac99e8fSMartin KaFai Lau map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 7606ac99e8fSMartin KaFai Lau map->map_type != BPF_MAP_TYPE_SK_STORAGE) 761d83525caSAlexei Starovoitov return -ENOTSUPP; 762d83525caSAlexei Starovoitov if (map->spin_lock_off + sizeof(struct bpf_spin_lock) > 763d83525caSAlexei Starovoitov map->value_size) { 764d83525caSAlexei Starovoitov WARN_ONCE(1, 765d83525caSAlexei Starovoitov "verifier bug spin_lock_off %d value_size %d\n", 766d83525caSAlexei Starovoitov map->spin_lock_off, map->value_size); 767d83525caSAlexei Starovoitov return -EFAULT; 768d83525caSAlexei Starovoitov } 769d83525caSAlexei Starovoitov } 770d83525caSAlexei Starovoitov 771e8d2bec0SDaniel Borkmann if (map->ops->map_check_btf) 7721b2b234bSRoman Gushchin ret = map->ops->map_check_btf(map, btf, key_type, value_type); 773e8d2bec0SDaniel Borkmann 774e8d2bec0SDaniel Borkmann return ret; 775e8d2bec0SDaniel Borkmann } 776e8d2bec0SDaniel Borkmann 77785d33df3SMartin KaFai Lau #define BPF_MAP_CREATE_LAST_FIELD btf_vmlinux_value_type_id 77899c55f7dSAlexei Starovoitov /* called via syscall */ 77999c55f7dSAlexei Starovoitov static int map_create(union bpf_attr *attr) 78099c55f7dSAlexei Starovoitov { 78196eabe7aSMartin KaFai Lau int numa_node = bpf_map_attr_numa_node(attr); 782b936ca64SRoman Gushchin struct bpf_map_memory mem; 78399c55f7dSAlexei Starovoitov struct bpf_map *map; 7846e71b04aSChenbo Feng int f_flags; 78599c55f7dSAlexei Starovoitov int err; 78699c55f7dSAlexei Starovoitov 78799c55f7dSAlexei Starovoitov err = CHECK_ATTR(BPF_MAP_CREATE); 78899c55f7dSAlexei Starovoitov if (err) 78999c55f7dSAlexei Starovoitov return -EINVAL; 79099c55f7dSAlexei Starovoitov 79185d33df3SMartin KaFai Lau if (attr->btf_vmlinux_value_type_id) { 79285d33df3SMartin KaFai Lau if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS || 79385d33df3SMartin KaFai Lau attr->btf_key_type_id || attr->btf_value_type_id) 79485d33df3SMartin KaFai Lau return -EINVAL; 79585d33df3SMartin KaFai Lau } else if (attr->btf_key_type_id && !attr->btf_value_type_id) { 79685d33df3SMartin KaFai Lau return -EINVAL; 79785d33df3SMartin KaFai Lau } 79885d33df3SMartin KaFai Lau 7996e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->map_flags); 8006e71b04aSChenbo Feng if (f_flags < 0) 8016e71b04aSChenbo Feng return f_flags; 8026e71b04aSChenbo Feng 80396eabe7aSMartin KaFai Lau if (numa_node != NUMA_NO_NODE && 80496e5ae4eSEric Dumazet ((unsigned int)numa_node >= nr_node_ids || 80596e5ae4eSEric Dumazet !node_online(numa_node))) 80696eabe7aSMartin KaFai Lau return -EINVAL; 80796eabe7aSMartin KaFai Lau 80899c55f7dSAlexei Starovoitov /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 80999c55f7dSAlexei Starovoitov map = find_and_alloc_map(attr); 81099c55f7dSAlexei Starovoitov if (IS_ERR(map)) 81199c55f7dSAlexei Starovoitov return PTR_ERR(map); 81299c55f7dSAlexei Starovoitov 813ad5b177bSMartin KaFai Lau err = bpf_obj_name_cpy(map->name, attr->map_name); 814ad5b177bSMartin KaFai Lau if (err) 815b936ca64SRoman Gushchin goto free_map; 816ad5b177bSMartin KaFai Lau 8171e0bd5a0SAndrii Nakryiko atomic64_set(&map->refcnt, 1); 8181e0bd5a0SAndrii Nakryiko atomic64_set(&map->usercnt, 1); 819fc970227SAndrii Nakryiko mutex_init(&map->freeze_mutex); 82099c55f7dSAlexei Starovoitov 82185d33df3SMartin KaFai Lau map->spin_lock_off = -EINVAL; 82285d33df3SMartin KaFai Lau if (attr->btf_key_type_id || attr->btf_value_type_id || 82385d33df3SMartin KaFai Lau /* Even the map's value is a kernel's struct, 82485d33df3SMartin KaFai Lau * the bpf_prog.o must have BTF to begin with 82585d33df3SMartin KaFai Lau * to figure out the corresponding kernel's 82685d33df3SMartin KaFai Lau * counter part. Thus, attr->btf_fd has 82785d33df3SMartin KaFai Lau * to be valid also. 82885d33df3SMartin KaFai Lau */ 82985d33df3SMartin KaFai Lau attr->btf_vmlinux_value_type_id) { 830a26ca7c9SMartin KaFai Lau struct btf *btf; 831a26ca7c9SMartin KaFai Lau 832a26ca7c9SMartin KaFai Lau btf = btf_get_by_fd(attr->btf_fd); 833a26ca7c9SMartin KaFai Lau if (IS_ERR(btf)) { 834a26ca7c9SMartin KaFai Lau err = PTR_ERR(btf); 835b936ca64SRoman Gushchin goto free_map; 836a26ca7c9SMartin KaFai Lau } 83785d33df3SMartin KaFai Lau map->btf = btf; 838a26ca7c9SMartin KaFai Lau 83985d33df3SMartin KaFai Lau if (attr->btf_value_type_id) { 840e8d2bec0SDaniel Borkmann err = map_check_btf(map, btf, attr->btf_key_type_id, 8419b2cf328SMartin KaFai Lau attr->btf_value_type_id); 84285d33df3SMartin KaFai Lau if (err) 843b936ca64SRoman Gushchin goto free_map; 844a26ca7c9SMartin KaFai Lau } 845a26ca7c9SMartin KaFai Lau 8469b2cf328SMartin KaFai Lau map->btf_key_type_id = attr->btf_key_type_id; 8479b2cf328SMartin KaFai Lau map->btf_value_type_id = attr->btf_value_type_id; 84885d33df3SMartin KaFai Lau map->btf_vmlinux_value_type_id = 84985d33df3SMartin KaFai Lau attr->btf_vmlinux_value_type_id; 850a26ca7c9SMartin KaFai Lau } 851a26ca7c9SMartin KaFai Lau 852afdb09c7SChenbo Feng err = security_bpf_map_alloc(map); 853aaac3ba9SAlexei Starovoitov if (err) 854b936ca64SRoman Gushchin goto free_map; 855afdb09c7SChenbo Feng 856f3f1c054SMartin KaFai Lau err = bpf_map_alloc_id(map); 857f3f1c054SMartin KaFai Lau if (err) 858b936ca64SRoman Gushchin goto free_map_sec; 859f3f1c054SMartin KaFai Lau 8606e71b04aSChenbo Feng err = bpf_map_new_fd(map, f_flags); 861bd5f5f4eSMartin KaFai Lau if (err < 0) { 862bd5f5f4eSMartin KaFai Lau /* failed to allocate fd. 863352d20d6SPeng Sun * bpf_map_put_with_uref() is needed because the above 864bd5f5f4eSMartin KaFai Lau * bpf_map_alloc_id() has published the map 865bd5f5f4eSMartin KaFai Lau * to the userspace and the userspace may 866bd5f5f4eSMartin KaFai Lau * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 867bd5f5f4eSMartin KaFai Lau */ 868352d20d6SPeng Sun bpf_map_put_with_uref(map); 869bd5f5f4eSMartin KaFai Lau return err; 870bd5f5f4eSMartin KaFai Lau } 87199c55f7dSAlexei Starovoitov 87299c55f7dSAlexei Starovoitov return err; 87399c55f7dSAlexei Starovoitov 874afdb09c7SChenbo Feng free_map_sec: 875afdb09c7SChenbo Feng security_bpf_map_free(map); 876b936ca64SRoman Gushchin free_map: 877a26ca7c9SMartin KaFai Lau btf_put(map->btf); 878b936ca64SRoman Gushchin bpf_map_charge_move(&mem, &map->memory); 87999c55f7dSAlexei Starovoitov map->ops->map_free(map); 880b936ca64SRoman Gushchin bpf_map_charge_finish(&mem); 88199c55f7dSAlexei Starovoitov return err; 88299c55f7dSAlexei Starovoitov } 88399c55f7dSAlexei Starovoitov 884db20fd2bSAlexei Starovoitov /* if error is returned, fd is released. 885db20fd2bSAlexei Starovoitov * On success caller should complete fd access with matching fdput() 886db20fd2bSAlexei Starovoitov */ 887c2101297SDaniel Borkmann struct bpf_map *__bpf_map_get(struct fd f) 888db20fd2bSAlexei Starovoitov { 889db20fd2bSAlexei Starovoitov if (!f.file) 890db20fd2bSAlexei Starovoitov return ERR_PTR(-EBADF); 891db20fd2bSAlexei Starovoitov if (f.file->f_op != &bpf_map_fops) { 892db20fd2bSAlexei Starovoitov fdput(f); 893db20fd2bSAlexei Starovoitov return ERR_PTR(-EINVAL); 894db20fd2bSAlexei Starovoitov } 895db20fd2bSAlexei Starovoitov 896c2101297SDaniel Borkmann return f.file->private_data; 897c2101297SDaniel Borkmann } 898c2101297SDaniel Borkmann 8991e0bd5a0SAndrii Nakryiko void bpf_map_inc(struct bpf_map *map) 900c9da161cSDaniel Borkmann { 9011e0bd5a0SAndrii Nakryiko atomic64_inc(&map->refcnt); 902c9da161cSDaniel Borkmann } 903630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_inc); 904c9da161cSDaniel Borkmann 9051e0bd5a0SAndrii Nakryiko void bpf_map_inc_with_uref(struct bpf_map *map) 9061e0bd5a0SAndrii Nakryiko { 9071e0bd5a0SAndrii Nakryiko atomic64_inc(&map->refcnt); 9081e0bd5a0SAndrii Nakryiko atomic64_inc(&map->usercnt); 9091e0bd5a0SAndrii Nakryiko } 9101e0bd5a0SAndrii Nakryiko EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref); 9111e0bd5a0SAndrii Nakryiko 912c9da161cSDaniel Borkmann struct bpf_map *bpf_map_get_with_uref(u32 ufd) 913c2101297SDaniel Borkmann { 914c2101297SDaniel Borkmann struct fd f = fdget(ufd); 915c2101297SDaniel Borkmann struct bpf_map *map; 916c2101297SDaniel Borkmann 917c2101297SDaniel Borkmann map = __bpf_map_get(f); 918c2101297SDaniel Borkmann if (IS_ERR(map)) 919c2101297SDaniel Borkmann return map; 920c2101297SDaniel Borkmann 9211e0bd5a0SAndrii Nakryiko bpf_map_inc_with_uref(map); 922c2101297SDaniel Borkmann fdput(f); 923db20fd2bSAlexei Starovoitov 924db20fd2bSAlexei Starovoitov return map; 925db20fd2bSAlexei Starovoitov } 926db20fd2bSAlexei Starovoitov 927bd5f5f4eSMartin KaFai Lau /* map_idr_lock should have been held */ 9281e0bd5a0SAndrii Nakryiko static struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref) 929bd5f5f4eSMartin KaFai Lau { 930bd5f5f4eSMartin KaFai Lau int refold; 931bd5f5f4eSMartin KaFai Lau 9321e0bd5a0SAndrii Nakryiko refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0); 933bd5f5f4eSMartin KaFai Lau if (!refold) 934bd5f5f4eSMartin KaFai Lau return ERR_PTR(-ENOENT); 935bd5f5f4eSMartin KaFai Lau if (uref) 9361e0bd5a0SAndrii Nakryiko atomic64_inc(&map->usercnt); 937bd5f5f4eSMartin KaFai Lau 938bd5f5f4eSMartin KaFai Lau return map; 939bd5f5f4eSMartin KaFai Lau } 940bd5f5f4eSMartin KaFai Lau 9411e0bd5a0SAndrii Nakryiko struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map) 942b0e4701cSStanislav Fomichev { 943b0e4701cSStanislav Fomichev spin_lock_bh(&map_idr_lock); 9441e0bd5a0SAndrii Nakryiko map = __bpf_map_inc_not_zero(map, false); 945b0e4701cSStanislav Fomichev spin_unlock_bh(&map_idr_lock); 946b0e4701cSStanislav Fomichev 947b0e4701cSStanislav Fomichev return map; 948b0e4701cSStanislav Fomichev } 949b0e4701cSStanislav Fomichev EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero); 950b0e4701cSStanislav Fomichev 951b8cdc051SAlexei Starovoitov int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 952b8cdc051SAlexei Starovoitov { 953b8cdc051SAlexei Starovoitov return -ENOTSUPP; 954b8cdc051SAlexei Starovoitov } 955b8cdc051SAlexei Starovoitov 956c9d29f46SMauricio Vasquez B static void *__bpf_copy_key(void __user *ukey, u64 key_size) 957c9d29f46SMauricio Vasquez B { 958c9d29f46SMauricio Vasquez B if (key_size) 959c9d29f46SMauricio Vasquez B return memdup_user(ukey, key_size); 960c9d29f46SMauricio Vasquez B 961c9d29f46SMauricio Vasquez B if (ukey) 962c9d29f46SMauricio Vasquez B return ERR_PTR(-EINVAL); 963c9d29f46SMauricio Vasquez B 964c9d29f46SMauricio Vasquez B return NULL; 965c9d29f46SMauricio Vasquez B } 966c9d29f46SMauricio Vasquez B 967db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 96896049f3aSAlexei Starovoitov #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags 969db20fd2bSAlexei Starovoitov 970db20fd2bSAlexei Starovoitov static int map_lookup_elem(union bpf_attr *attr) 971db20fd2bSAlexei Starovoitov { 972535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 973535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 974db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 975db20fd2bSAlexei Starovoitov struct bpf_map *map; 97615c14a3dSBrian Vazquez void *key, *value; 97715a07b33SAlexei Starovoitov u32 value_size; 978592867bfSDaniel Borkmann struct fd f; 979db20fd2bSAlexei Starovoitov int err; 980db20fd2bSAlexei Starovoitov 981db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 982db20fd2bSAlexei Starovoitov return -EINVAL; 983db20fd2bSAlexei Starovoitov 98496049f3aSAlexei Starovoitov if (attr->flags & ~BPF_F_LOCK) 98596049f3aSAlexei Starovoitov return -EINVAL; 98696049f3aSAlexei Starovoitov 987592867bfSDaniel Borkmann f = fdget(ufd); 988c2101297SDaniel Borkmann map = __bpf_map_get(f); 989db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 990db20fd2bSAlexei Starovoitov return PTR_ERR(map); 99187df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 9926e71b04aSChenbo Feng err = -EPERM; 9936e71b04aSChenbo Feng goto err_put; 9946e71b04aSChenbo Feng } 9956e71b04aSChenbo Feng 99696049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 99796049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 99896049f3aSAlexei Starovoitov err = -EINVAL; 99996049f3aSAlexei Starovoitov goto err_put; 100096049f3aSAlexei Starovoitov } 100196049f3aSAlexei Starovoitov 1002c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1003e4448ed8SAl Viro if (IS_ERR(key)) { 1004e4448ed8SAl Viro err = PTR_ERR(key); 1005db20fd2bSAlexei Starovoitov goto err_put; 1006e4448ed8SAl Viro } 1007db20fd2bSAlexei Starovoitov 100815c14a3dSBrian Vazquez value_size = bpf_map_value_size(map); 100915a07b33SAlexei Starovoitov 10108ebe667cSAlexei Starovoitov err = -ENOMEM; 101115a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1012db20fd2bSAlexei Starovoitov if (!value) 10138ebe667cSAlexei Starovoitov goto free_key; 10148ebe667cSAlexei Starovoitov 101515c14a3dSBrian Vazquez err = bpf_map_copy_value(map, key, value, attr->flags); 101615a07b33SAlexei Starovoitov if (err) 10178ebe667cSAlexei Starovoitov goto free_value; 1018db20fd2bSAlexei Starovoitov 1019db20fd2bSAlexei Starovoitov err = -EFAULT; 102015a07b33SAlexei Starovoitov if (copy_to_user(uvalue, value, value_size) != 0) 10218ebe667cSAlexei Starovoitov goto free_value; 1022db20fd2bSAlexei Starovoitov 1023db20fd2bSAlexei Starovoitov err = 0; 1024db20fd2bSAlexei Starovoitov 10258ebe667cSAlexei Starovoitov free_value: 10268ebe667cSAlexei Starovoitov kfree(value); 1027db20fd2bSAlexei Starovoitov free_key: 1028db20fd2bSAlexei Starovoitov kfree(key); 1029db20fd2bSAlexei Starovoitov err_put: 1030db20fd2bSAlexei Starovoitov fdput(f); 1031db20fd2bSAlexei Starovoitov return err; 1032db20fd2bSAlexei Starovoitov } 1033db20fd2bSAlexei Starovoitov 10341ae80cf3SDaniel Colascione 10353274f520SAlexei Starovoitov #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 1036db20fd2bSAlexei Starovoitov 1037db20fd2bSAlexei Starovoitov static int map_update_elem(union bpf_attr *attr) 1038db20fd2bSAlexei Starovoitov { 1039535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1040535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 1041db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1042db20fd2bSAlexei Starovoitov struct bpf_map *map; 1043db20fd2bSAlexei Starovoitov void *key, *value; 104415a07b33SAlexei Starovoitov u32 value_size; 1045592867bfSDaniel Borkmann struct fd f; 1046db20fd2bSAlexei Starovoitov int err; 1047db20fd2bSAlexei Starovoitov 1048db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 1049db20fd2bSAlexei Starovoitov return -EINVAL; 1050db20fd2bSAlexei Starovoitov 1051592867bfSDaniel Borkmann f = fdget(ufd); 1052c2101297SDaniel Borkmann map = __bpf_map_get(f); 1053db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1054db20fd2bSAlexei Starovoitov return PTR_ERR(map); 105587df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 10566e71b04aSChenbo Feng err = -EPERM; 10576e71b04aSChenbo Feng goto err_put; 10586e71b04aSChenbo Feng } 10596e71b04aSChenbo Feng 106096049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 106196049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 106296049f3aSAlexei Starovoitov err = -EINVAL; 106396049f3aSAlexei Starovoitov goto err_put; 106496049f3aSAlexei Starovoitov } 106596049f3aSAlexei Starovoitov 1066c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1067e4448ed8SAl Viro if (IS_ERR(key)) { 1068e4448ed8SAl Viro err = PTR_ERR(key); 1069db20fd2bSAlexei Starovoitov goto err_put; 1070e4448ed8SAl Viro } 1071db20fd2bSAlexei Starovoitov 107215a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 10738f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 1074b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 1075b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 107615a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 107715a07b33SAlexei Starovoitov else 107815a07b33SAlexei Starovoitov value_size = map->value_size; 107915a07b33SAlexei Starovoitov 1080db20fd2bSAlexei Starovoitov err = -ENOMEM; 108115a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1082db20fd2bSAlexei Starovoitov if (!value) 1083db20fd2bSAlexei Starovoitov goto free_key; 1084db20fd2bSAlexei Starovoitov 1085db20fd2bSAlexei Starovoitov err = -EFAULT; 108615a07b33SAlexei Starovoitov if (copy_from_user(value, uvalue, value_size) != 0) 1087db20fd2bSAlexei Starovoitov goto free_value; 1088db20fd2bSAlexei Starovoitov 108915c14a3dSBrian Vazquez err = bpf_map_update_value(map, f, key, value, attr->flags); 10906710e112SJesper Dangaard Brouer 1091db20fd2bSAlexei Starovoitov free_value: 1092db20fd2bSAlexei Starovoitov kfree(value); 1093db20fd2bSAlexei Starovoitov free_key: 1094db20fd2bSAlexei Starovoitov kfree(key); 1095db20fd2bSAlexei Starovoitov err_put: 1096db20fd2bSAlexei Starovoitov fdput(f); 1097db20fd2bSAlexei Starovoitov return err; 1098db20fd2bSAlexei Starovoitov } 1099db20fd2bSAlexei Starovoitov 1100db20fd2bSAlexei Starovoitov #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 1101db20fd2bSAlexei Starovoitov 1102db20fd2bSAlexei Starovoitov static int map_delete_elem(union bpf_attr *attr) 1103db20fd2bSAlexei Starovoitov { 1104535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1105db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1106db20fd2bSAlexei Starovoitov struct bpf_map *map; 1107592867bfSDaniel Borkmann struct fd f; 1108db20fd2bSAlexei Starovoitov void *key; 1109db20fd2bSAlexei Starovoitov int err; 1110db20fd2bSAlexei Starovoitov 1111db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 1112db20fd2bSAlexei Starovoitov return -EINVAL; 1113db20fd2bSAlexei Starovoitov 1114592867bfSDaniel Borkmann f = fdget(ufd); 1115c2101297SDaniel Borkmann map = __bpf_map_get(f); 1116db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1117db20fd2bSAlexei Starovoitov return PTR_ERR(map); 111887df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 11196e71b04aSChenbo Feng err = -EPERM; 11206e71b04aSChenbo Feng goto err_put; 11216e71b04aSChenbo Feng } 11226e71b04aSChenbo Feng 1123c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1124e4448ed8SAl Viro if (IS_ERR(key)) { 1125e4448ed8SAl Viro err = PTR_ERR(key); 1126db20fd2bSAlexei Starovoitov goto err_put; 1127e4448ed8SAl Viro } 1128db20fd2bSAlexei Starovoitov 1129a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1130a3884572SJakub Kicinski err = bpf_map_offload_delete_elem(map, key); 1131a3884572SJakub Kicinski goto out; 113285d33df3SMartin KaFai Lau } else if (IS_FD_PROG_ARRAY(map) || 113385d33df3SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 113485d33df3SMartin KaFai Lau /* These maps require sleepable context */ 1135da765a2fSDaniel Borkmann err = map->ops->map_delete_elem(map, key); 1136da765a2fSDaniel Borkmann goto out; 1137a3884572SJakub Kicinski } 1138a3884572SJakub Kicinski 1139b121d1e7SAlexei Starovoitov preempt_disable(); 1140b121d1e7SAlexei Starovoitov __this_cpu_inc(bpf_prog_active); 1141db20fd2bSAlexei Starovoitov rcu_read_lock(); 1142db20fd2bSAlexei Starovoitov err = map->ops->map_delete_elem(map, key); 1143db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1144b121d1e7SAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 1145b121d1e7SAlexei Starovoitov preempt_enable(); 11461ae80cf3SDaniel Colascione maybe_wait_bpf_programs(map); 1147a3884572SJakub Kicinski out: 1148db20fd2bSAlexei Starovoitov kfree(key); 1149db20fd2bSAlexei Starovoitov err_put: 1150db20fd2bSAlexei Starovoitov fdput(f); 1151db20fd2bSAlexei Starovoitov return err; 1152db20fd2bSAlexei Starovoitov } 1153db20fd2bSAlexei Starovoitov 1154db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 1155db20fd2bSAlexei Starovoitov #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 1156db20fd2bSAlexei Starovoitov 1157db20fd2bSAlexei Starovoitov static int map_get_next_key(union bpf_attr *attr) 1158db20fd2bSAlexei Starovoitov { 1159535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1160535e7b4bSMickaël Salaün void __user *unext_key = u64_to_user_ptr(attr->next_key); 1161db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1162db20fd2bSAlexei Starovoitov struct bpf_map *map; 1163db20fd2bSAlexei Starovoitov void *key, *next_key; 1164592867bfSDaniel Borkmann struct fd f; 1165db20fd2bSAlexei Starovoitov int err; 1166db20fd2bSAlexei Starovoitov 1167db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 1168db20fd2bSAlexei Starovoitov return -EINVAL; 1169db20fd2bSAlexei Starovoitov 1170592867bfSDaniel Borkmann f = fdget(ufd); 1171c2101297SDaniel Borkmann map = __bpf_map_get(f); 1172db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1173db20fd2bSAlexei Starovoitov return PTR_ERR(map); 117487df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 11756e71b04aSChenbo Feng err = -EPERM; 11766e71b04aSChenbo Feng goto err_put; 11776e71b04aSChenbo Feng } 11786e71b04aSChenbo Feng 11798fe45924STeng Qin if (ukey) { 1180c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1181e4448ed8SAl Viro if (IS_ERR(key)) { 1182e4448ed8SAl Viro err = PTR_ERR(key); 1183db20fd2bSAlexei Starovoitov goto err_put; 1184e4448ed8SAl Viro } 11858fe45924STeng Qin } else { 11868fe45924STeng Qin key = NULL; 11878fe45924STeng Qin } 1188db20fd2bSAlexei Starovoitov 1189db20fd2bSAlexei Starovoitov err = -ENOMEM; 1190db20fd2bSAlexei Starovoitov next_key = kmalloc(map->key_size, GFP_USER); 1191db20fd2bSAlexei Starovoitov if (!next_key) 1192db20fd2bSAlexei Starovoitov goto free_key; 1193db20fd2bSAlexei Starovoitov 1194a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1195a3884572SJakub Kicinski err = bpf_map_offload_get_next_key(map, key, next_key); 1196a3884572SJakub Kicinski goto out; 1197a3884572SJakub Kicinski } 1198a3884572SJakub Kicinski 1199db20fd2bSAlexei Starovoitov rcu_read_lock(); 1200db20fd2bSAlexei Starovoitov err = map->ops->map_get_next_key(map, key, next_key); 1201db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1202a3884572SJakub Kicinski out: 1203db20fd2bSAlexei Starovoitov if (err) 1204db20fd2bSAlexei Starovoitov goto free_next_key; 1205db20fd2bSAlexei Starovoitov 1206db20fd2bSAlexei Starovoitov err = -EFAULT; 1207db20fd2bSAlexei Starovoitov if (copy_to_user(unext_key, next_key, map->key_size) != 0) 1208db20fd2bSAlexei Starovoitov goto free_next_key; 1209db20fd2bSAlexei Starovoitov 1210db20fd2bSAlexei Starovoitov err = 0; 1211db20fd2bSAlexei Starovoitov 1212db20fd2bSAlexei Starovoitov free_next_key: 1213db20fd2bSAlexei Starovoitov kfree(next_key); 1214db20fd2bSAlexei Starovoitov free_key: 1215db20fd2bSAlexei Starovoitov kfree(key); 1216db20fd2bSAlexei Starovoitov err_put: 1217db20fd2bSAlexei Starovoitov fdput(f); 1218db20fd2bSAlexei Starovoitov return err; 1219db20fd2bSAlexei Starovoitov } 1220db20fd2bSAlexei Starovoitov 1221aa2e93b8SBrian Vazquez int generic_map_delete_batch(struct bpf_map *map, 1222aa2e93b8SBrian Vazquez const union bpf_attr *attr, 1223aa2e93b8SBrian Vazquez union bpf_attr __user *uattr) 1224aa2e93b8SBrian Vazquez { 1225aa2e93b8SBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1226aa2e93b8SBrian Vazquez u32 cp, max_count; 1227aa2e93b8SBrian Vazquez int err = 0; 1228aa2e93b8SBrian Vazquez void *key; 1229aa2e93b8SBrian Vazquez 1230aa2e93b8SBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1231aa2e93b8SBrian Vazquez return -EINVAL; 1232aa2e93b8SBrian Vazquez 1233aa2e93b8SBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1234aa2e93b8SBrian Vazquez !map_value_has_spin_lock(map)) { 1235aa2e93b8SBrian Vazquez return -EINVAL; 1236aa2e93b8SBrian Vazquez } 1237aa2e93b8SBrian Vazquez 1238aa2e93b8SBrian Vazquez max_count = attr->batch.count; 1239aa2e93b8SBrian Vazquez if (!max_count) 1240aa2e93b8SBrian Vazquez return 0; 1241aa2e93b8SBrian Vazquez 12422e3a94aaSBrian Vazquez key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 12432e3a94aaSBrian Vazquez if (!key) 12442e3a94aaSBrian Vazquez return -ENOMEM; 12452e3a94aaSBrian Vazquez 1246aa2e93b8SBrian Vazquez for (cp = 0; cp < max_count; cp++) { 12472e3a94aaSBrian Vazquez err = -EFAULT; 12482e3a94aaSBrian Vazquez if (copy_from_user(key, keys + cp * map->key_size, 12492e3a94aaSBrian Vazquez map->key_size)) 1250aa2e93b8SBrian Vazquez break; 1251aa2e93b8SBrian Vazquez 1252aa2e93b8SBrian Vazquez if (bpf_map_is_dev_bound(map)) { 1253aa2e93b8SBrian Vazquez err = bpf_map_offload_delete_elem(map, key); 1254aa2e93b8SBrian Vazquez break; 1255aa2e93b8SBrian Vazquez } 1256aa2e93b8SBrian Vazquez 1257aa2e93b8SBrian Vazquez preempt_disable(); 1258aa2e93b8SBrian Vazquez __this_cpu_inc(bpf_prog_active); 1259aa2e93b8SBrian Vazquez rcu_read_lock(); 1260aa2e93b8SBrian Vazquez err = map->ops->map_delete_elem(map, key); 1261aa2e93b8SBrian Vazquez rcu_read_unlock(); 1262aa2e93b8SBrian Vazquez __this_cpu_dec(bpf_prog_active); 1263aa2e93b8SBrian Vazquez preempt_enable(); 1264aa2e93b8SBrian Vazquez maybe_wait_bpf_programs(map); 1265aa2e93b8SBrian Vazquez if (err) 1266aa2e93b8SBrian Vazquez break; 1267aa2e93b8SBrian Vazquez } 1268aa2e93b8SBrian Vazquez if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) 1269aa2e93b8SBrian Vazquez err = -EFAULT; 12702e3a94aaSBrian Vazquez 12712e3a94aaSBrian Vazquez kfree(key); 1272aa2e93b8SBrian Vazquez return err; 1273aa2e93b8SBrian Vazquez } 1274aa2e93b8SBrian Vazquez 1275aa2e93b8SBrian Vazquez int generic_map_update_batch(struct bpf_map *map, 1276aa2e93b8SBrian Vazquez const union bpf_attr *attr, 1277aa2e93b8SBrian Vazquez union bpf_attr __user *uattr) 1278aa2e93b8SBrian Vazquez { 1279aa2e93b8SBrian Vazquez void __user *values = u64_to_user_ptr(attr->batch.values); 1280aa2e93b8SBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1281aa2e93b8SBrian Vazquez u32 value_size, cp, max_count; 1282aa2e93b8SBrian Vazquez int ufd = attr->map_fd; 1283aa2e93b8SBrian Vazquez void *key, *value; 1284aa2e93b8SBrian Vazquez struct fd f; 1285aa2e93b8SBrian Vazquez int err = 0; 1286aa2e93b8SBrian Vazquez 1287aa2e93b8SBrian Vazquez f = fdget(ufd); 1288aa2e93b8SBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1289aa2e93b8SBrian Vazquez return -EINVAL; 1290aa2e93b8SBrian Vazquez 1291aa2e93b8SBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1292aa2e93b8SBrian Vazquez !map_value_has_spin_lock(map)) { 1293aa2e93b8SBrian Vazquez return -EINVAL; 1294aa2e93b8SBrian Vazquez } 1295aa2e93b8SBrian Vazquez 1296aa2e93b8SBrian Vazquez value_size = bpf_map_value_size(map); 1297aa2e93b8SBrian Vazquez 1298aa2e93b8SBrian Vazquez max_count = attr->batch.count; 1299aa2e93b8SBrian Vazquez if (!max_count) 1300aa2e93b8SBrian Vazquez return 0; 1301aa2e93b8SBrian Vazquez 13022e3a94aaSBrian Vazquez key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 13032e3a94aaSBrian Vazquez if (!key) 1304aa2e93b8SBrian Vazquez return -ENOMEM; 1305aa2e93b8SBrian Vazquez 13062e3a94aaSBrian Vazquez value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 13072e3a94aaSBrian Vazquez if (!value) { 13082e3a94aaSBrian Vazquez kfree(key); 13092e3a94aaSBrian Vazquez return -ENOMEM; 1310aa2e93b8SBrian Vazquez } 13112e3a94aaSBrian Vazquez 13122e3a94aaSBrian Vazquez for (cp = 0; cp < max_count; cp++) { 1313aa2e93b8SBrian Vazquez err = -EFAULT; 13142e3a94aaSBrian Vazquez if (copy_from_user(key, keys + cp * map->key_size, 13152e3a94aaSBrian Vazquez map->key_size) || 13162e3a94aaSBrian Vazquez copy_from_user(value, values + cp * value_size, value_size)) 1317aa2e93b8SBrian Vazquez break; 1318aa2e93b8SBrian Vazquez 1319aa2e93b8SBrian Vazquez err = bpf_map_update_value(map, f, key, value, 1320aa2e93b8SBrian Vazquez attr->batch.elem_flags); 1321aa2e93b8SBrian Vazquez 1322aa2e93b8SBrian Vazquez if (err) 1323aa2e93b8SBrian Vazquez break; 1324aa2e93b8SBrian Vazquez } 1325aa2e93b8SBrian Vazquez 1326aa2e93b8SBrian Vazquez if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) 1327aa2e93b8SBrian Vazquez err = -EFAULT; 1328aa2e93b8SBrian Vazquez 1329aa2e93b8SBrian Vazquez kfree(value); 1330aa2e93b8SBrian Vazquez kfree(key); 1331aa2e93b8SBrian Vazquez return err; 1332aa2e93b8SBrian Vazquez } 1333aa2e93b8SBrian Vazquez 1334cb4d03abSBrian Vazquez #define MAP_LOOKUP_RETRIES 3 1335cb4d03abSBrian Vazquez 1336cb4d03abSBrian Vazquez int generic_map_lookup_batch(struct bpf_map *map, 1337cb4d03abSBrian Vazquez const union bpf_attr *attr, 1338cb4d03abSBrian Vazquez union bpf_attr __user *uattr) 1339cb4d03abSBrian Vazquez { 1340cb4d03abSBrian Vazquez void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch); 1341cb4d03abSBrian Vazquez void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch); 1342cb4d03abSBrian Vazquez void __user *values = u64_to_user_ptr(attr->batch.values); 1343cb4d03abSBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1344cb4d03abSBrian Vazquez void *buf, *buf_prevkey, *prev_key, *key, *value; 1345cb4d03abSBrian Vazquez int err, retry = MAP_LOOKUP_RETRIES; 1346cb4d03abSBrian Vazquez u32 value_size, cp, max_count; 1347cb4d03abSBrian Vazquez 1348cb4d03abSBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1349cb4d03abSBrian Vazquez return -EINVAL; 1350cb4d03abSBrian Vazquez 1351cb4d03abSBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1352cb4d03abSBrian Vazquez !map_value_has_spin_lock(map)) 1353cb4d03abSBrian Vazquez return -EINVAL; 1354cb4d03abSBrian Vazquez 1355cb4d03abSBrian Vazquez value_size = bpf_map_value_size(map); 1356cb4d03abSBrian Vazquez 1357cb4d03abSBrian Vazquez max_count = attr->batch.count; 1358cb4d03abSBrian Vazquez if (!max_count) 1359cb4d03abSBrian Vazquez return 0; 1360cb4d03abSBrian Vazquez 1361cb4d03abSBrian Vazquez if (put_user(0, &uattr->batch.count)) 1362cb4d03abSBrian Vazquez return -EFAULT; 1363cb4d03abSBrian Vazquez 1364cb4d03abSBrian Vazquez buf_prevkey = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 1365cb4d03abSBrian Vazquez if (!buf_prevkey) 1366cb4d03abSBrian Vazquez return -ENOMEM; 1367cb4d03abSBrian Vazquez 1368cb4d03abSBrian Vazquez buf = kmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN); 1369cb4d03abSBrian Vazquez if (!buf) { 1370cb4d03abSBrian Vazquez kvfree(buf_prevkey); 1371cb4d03abSBrian Vazquez return -ENOMEM; 1372cb4d03abSBrian Vazquez } 1373cb4d03abSBrian Vazquez 1374cb4d03abSBrian Vazquez err = -EFAULT; 1375cb4d03abSBrian Vazquez prev_key = NULL; 1376cb4d03abSBrian Vazquez if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size)) 1377cb4d03abSBrian Vazquez goto free_buf; 1378cb4d03abSBrian Vazquez key = buf; 1379cb4d03abSBrian Vazquez value = key + map->key_size; 1380cb4d03abSBrian Vazquez if (ubatch) 1381cb4d03abSBrian Vazquez prev_key = buf_prevkey; 1382cb4d03abSBrian Vazquez 1383cb4d03abSBrian Vazquez for (cp = 0; cp < max_count;) { 1384cb4d03abSBrian Vazquez rcu_read_lock(); 1385cb4d03abSBrian Vazquez err = map->ops->map_get_next_key(map, prev_key, key); 1386cb4d03abSBrian Vazquez rcu_read_unlock(); 1387cb4d03abSBrian Vazquez if (err) 1388cb4d03abSBrian Vazquez break; 1389cb4d03abSBrian Vazquez err = bpf_map_copy_value(map, key, value, 1390cb4d03abSBrian Vazquez attr->batch.elem_flags); 1391cb4d03abSBrian Vazquez 1392cb4d03abSBrian Vazquez if (err == -ENOENT) { 1393cb4d03abSBrian Vazquez if (retry) { 1394cb4d03abSBrian Vazquez retry--; 1395cb4d03abSBrian Vazquez continue; 1396cb4d03abSBrian Vazquez } 1397cb4d03abSBrian Vazquez err = -EINTR; 1398cb4d03abSBrian Vazquez break; 1399cb4d03abSBrian Vazquez } 1400cb4d03abSBrian Vazquez 1401cb4d03abSBrian Vazquez if (err) 1402cb4d03abSBrian Vazquez goto free_buf; 1403cb4d03abSBrian Vazquez 1404cb4d03abSBrian Vazquez if (copy_to_user(keys + cp * map->key_size, key, 1405cb4d03abSBrian Vazquez map->key_size)) { 1406cb4d03abSBrian Vazquez err = -EFAULT; 1407cb4d03abSBrian Vazquez goto free_buf; 1408cb4d03abSBrian Vazquez } 1409cb4d03abSBrian Vazquez if (copy_to_user(values + cp * value_size, value, value_size)) { 1410cb4d03abSBrian Vazquez err = -EFAULT; 1411cb4d03abSBrian Vazquez goto free_buf; 1412cb4d03abSBrian Vazquez } 1413cb4d03abSBrian Vazquez 1414cb4d03abSBrian Vazquez if (!prev_key) 1415cb4d03abSBrian Vazquez prev_key = buf_prevkey; 1416cb4d03abSBrian Vazquez 1417cb4d03abSBrian Vazquez swap(prev_key, key); 1418cb4d03abSBrian Vazquez retry = MAP_LOOKUP_RETRIES; 1419cb4d03abSBrian Vazquez cp++; 1420cb4d03abSBrian Vazquez } 1421cb4d03abSBrian Vazquez 1422cb4d03abSBrian Vazquez if (err == -EFAULT) 1423cb4d03abSBrian Vazquez goto free_buf; 1424cb4d03abSBrian Vazquez 1425cb4d03abSBrian Vazquez if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) || 1426cb4d03abSBrian Vazquez (cp && copy_to_user(uobatch, prev_key, map->key_size)))) 1427cb4d03abSBrian Vazquez err = -EFAULT; 1428cb4d03abSBrian Vazquez 1429cb4d03abSBrian Vazquez free_buf: 1430cb4d03abSBrian Vazquez kfree(buf_prevkey); 1431cb4d03abSBrian Vazquez kfree(buf); 1432cb4d03abSBrian Vazquez return err; 1433cb4d03abSBrian Vazquez } 1434cb4d03abSBrian Vazquez 1435bd513cd0SMauricio Vasquez B #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value 1436bd513cd0SMauricio Vasquez B 1437bd513cd0SMauricio Vasquez B static int map_lookup_and_delete_elem(union bpf_attr *attr) 1438bd513cd0SMauricio Vasquez B { 1439bd513cd0SMauricio Vasquez B void __user *ukey = u64_to_user_ptr(attr->key); 1440bd513cd0SMauricio Vasquez B void __user *uvalue = u64_to_user_ptr(attr->value); 1441bd513cd0SMauricio Vasquez B int ufd = attr->map_fd; 1442bd513cd0SMauricio Vasquez B struct bpf_map *map; 1443540fefc0SAlexei Starovoitov void *key, *value; 1444bd513cd0SMauricio Vasquez B u32 value_size; 1445bd513cd0SMauricio Vasquez B struct fd f; 1446bd513cd0SMauricio Vasquez B int err; 1447bd513cd0SMauricio Vasquez B 1448bd513cd0SMauricio Vasquez B if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) 1449bd513cd0SMauricio Vasquez B return -EINVAL; 1450bd513cd0SMauricio Vasquez B 1451bd513cd0SMauricio Vasquez B f = fdget(ufd); 1452bd513cd0SMauricio Vasquez B map = __bpf_map_get(f); 1453bd513cd0SMauricio Vasquez B if (IS_ERR(map)) 1454bd513cd0SMauricio Vasquez B return PTR_ERR(map); 145587df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 1456bd513cd0SMauricio Vasquez B err = -EPERM; 1457bd513cd0SMauricio Vasquez B goto err_put; 1458bd513cd0SMauricio Vasquez B } 1459bd513cd0SMauricio Vasquez B 1460bd513cd0SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1461bd513cd0SMauricio Vasquez B if (IS_ERR(key)) { 1462bd513cd0SMauricio Vasquez B err = PTR_ERR(key); 1463bd513cd0SMauricio Vasquez B goto err_put; 1464bd513cd0SMauricio Vasquez B } 1465bd513cd0SMauricio Vasquez B 1466bd513cd0SMauricio Vasquez B value_size = map->value_size; 1467bd513cd0SMauricio Vasquez B 1468bd513cd0SMauricio Vasquez B err = -ENOMEM; 1469bd513cd0SMauricio Vasquez B value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1470bd513cd0SMauricio Vasquez B if (!value) 1471bd513cd0SMauricio Vasquez B goto free_key; 1472bd513cd0SMauricio Vasquez B 1473bd513cd0SMauricio Vasquez B if (map->map_type == BPF_MAP_TYPE_QUEUE || 1474bd513cd0SMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 1475bd513cd0SMauricio Vasquez B err = map->ops->map_pop_elem(map, value); 1476bd513cd0SMauricio Vasquez B } else { 1477bd513cd0SMauricio Vasquez B err = -ENOTSUPP; 1478bd513cd0SMauricio Vasquez B } 1479bd513cd0SMauricio Vasquez B 1480bd513cd0SMauricio Vasquez B if (err) 1481bd513cd0SMauricio Vasquez B goto free_value; 1482bd513cd0SMauricio Vasquez B 1483bd513cd0SMauricio Vasquez B if (copy_to_user(uvalue, value, value_size) != 0) 1484bd513cd0SMauricio Vasquez B goto free_value; 1485bd513cd0SMauricio Vasquez B 1486bd513cd0SMauricio Vasquez B err = 0; 1487bd513cd0SMauricio Vasquez B 1488bd513cd0SMauricio Vasquez B free_value: 1489bd513cd0SMauricio Vasquez B kfree(value); 1490bd513cd0SMauricio Vasquez B free_key: 1491bd513cd0SMauricio Vasquez B kfree(key); 1492bd513cd0SMauricio Vasquez B err_put: 1493bd513cd0SMauricio Vasquez B fdput(f); 1494bd513cd0SMauricio Vasquez B return err; 1495bd513cd0SMauricio Vasquez B } 1496bd513cd0SMauricio Vasquez B 149787df15deSDaniel Borkmann #define BPF_MAP_FREEZE_LAST_FIELD map_fd 149887df15deSDaniel Borkmann 149987df15deSDaniel Borkmann static int map_freeze(const union bpf_attr *attr) 150087df15deSDaniel Borkmann { 150187df15deSDaniel Borkmann int err = 0, ufd = attr->map_fd; 150287df15deSDaniel Borkmann struct bpf_map *map; 150387df15deSDaniel Borkmann struct fd f; 150487df15deSDaniel Borkmann 150587df15deSDaniel Borkmann if (CHECK_ATTR(BPF_MAP_FREEZE)) 150687df15deSDaniel Borkmann return -EINVAL; 150787df15deSDaniel Borkmann 150887df15deSDaniel Borkmann f = fdget(ufd); 150987df15deSDaniel Borkmann map = __bpf_map_get(f); 151087df15deSDaniel Borkmann if (IS_ERR(map)) 151187df15deSDaniel Borkmann return PTR_ERR(map); 1512fc970227SAndrii Nakryiko 1513*849b4d94SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 1514*849b4d94SMartin KaFai Lau fdput(f); 1515*849b4d94SMartin KaFai Lau return -ENOTSUPP; 1516*849b4d94SMartin KaFai Lau } 1517*849b4d94SMartin KaFai Lau 1518fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 1519fc970227SAndrii Nakryiko 1520fc970227SAndrii Nakryiko if (map->writecnt) { 1521fc970227SAndrii Nakryiko err = -EBUSY; 1522fc970227SAndrii Nakryiko goto err_put; 1523fc970227SAndrii Nakryiko } 152487df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) { 152587df15deSDaniel Borkmann err = -EBUSY; 152687df15deSDaniel Borkmann goto err_put; 152787df15deSDaniel Borkmann } 152887df15deSDaniel Borkmann if (!capable(CAP_SYS_ADMIN)) { 152987df15deSDaniel Borkmann err = -EPERM; 153087df15deSDaniel Borkmann goto err_put; 153187df15deSDaniel Borkmann } 153287df15deSDaniel Borkmann 153387df15deSDaniel Borkmann WRITE_ONCE(map->frozen, true); 153487df15deSDaniel Borkmann err_put: 1535fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 153687df15deSDaniel Borkmann fdput(f); 153787df15deSDaniel Borkmann return err; 153887df15deSDaniel Borkmann } 153987df15deSDaniel Borkmann 15407de16e3aSJakub Kicinski static const struct bpf_prog_ops * const bpf_prog_types[] = { 154191cc1a99SAlexei Starovoitov #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 15427de16e3aSJakub Kicinski [_id] = & _name ## _prog_ops, 15437de16e3aSJakub Kicinski #define BPF_MAP_TYPE(_id, _ops) 15447de16e3aSJakub Kicinski #include <linux/bpf_types.h> 15457de16e3aSJakub Kicinski #undef BPF_PROG_TYPE 15467de16e3aSJakub Kicinski #undef BPF_MAP_TYPE 15477de16e3aSJakub Kicinski }; 15487de16e3aSJakub Kicinski 154909756af4SAlexei Starovoitov static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 155009756af4SAlexei Starovoitov { 1551d0f1a451SDaniel Borkmann const struct bpf_prog_ops *ops; 1552d0f1a451SDaniel Borkmann 1553d0f1a451SDaniel Borkmann if (type >= ARRAY_SIZE(bpf_prog_types)) 1554d0f1a451SDaniel Borkmann return -EINVAL; 1555d0f1a451SDaniel Borkmann type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types)); 1556d0f1a451SDaniel Borkmann ops = bpf_prog_types[type]; 1557d0f1a451SDaniel Borkmann if (!ops) 1558be9370a7SJohannes Berg return -EINVAL; 155909756af4SAlexei Starovoitov 1560ab3f0063SJakub Kicinski if (!bpf_prog_is_dev_bound(prog->aux)) 1561d0f1a451SDaniel Borkmann prog->aux->ops = ops; 1562ab3f0063SJakub Kicinski else 1563ab3f0063SJakub Kicinski prog->aux->ops = &bpf_offload_prog_ops; 156424701eceSDaniel Borkmann prog->type = type; 156509756af4SAlexei Starovoitov return 0; 156609756af4SAlexei Starovoitov } 156709756af4SAlexei Starovoitov 1568bae141f5SDaniel Borkmann enum bpf_audit { 1569bae141f5SDaniel Borkmann BPF_AUDIT_LOAD, 1570bae141f5SDaniel Borkmann BPF_AUDIT_UNLOAD, 1571bae141f5SDaniel Borkmann BPF_AUDIT_MAX, 1572bae141f5SDaniel Borkmann }; 1573bae141f5SDaniel Borkmann 1574bae141f5SDaniel Borkmann static const char * const bpf_audit_str[BPF_AUDIT_MAX] = { 1575bae141f5SDaniel Borkmann [BPF_AUDIT_LOAD] = "LOAD", 1576bae141f5SDaniel Borkmann [BPF_AUDIT_UNLOAD] = "UNLOAD", 1577bae141f5SDaniel Borkmann }; 1578bae141f5SDaniel Borkmann 1579bae141f5SDaniel Borkmann static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op) 1580bae141f5SDaniel Borkmann { 1581bae141f5SDaniel Borkmann struct audit_context *ctx = NULL; 1582bae141f5SDaniel Borkmann struct audit_buffer *ab; 1583bae141f5SDaniel Borkmann 1584bae141f5SDaniel Borkmann if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX)) 1585bae141f5SDaniel Borkmann return; 1586bae141f5SDaniel Borkmann if (audit_enabled == AUDIT_OFF) 1587bae141f5SDaniel Borkmann return; 1588bae141f5SDaniel Borkmann if (op == BPF_AUDIT_LOAD) 1589bae141f5SDaniel Borkmann ctx = audit_context(); 1590bae141f5SDaniel Borkmann ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF); 1591bae141f5SDaniel Borkmann if (unlikely(!ab)) 1592bae141f5SDaniel Borkmann return; 1593bae141f5SDaniel Borkmann audit_log_format(ab, "prog-id=%u op=%s", 1594bae141f5SDaniel Borkmann prog->aux->id, bpf_audit_str[op]); 1595bae141f5SDaniel Borkmann audit_log_end(ab); 1596bae141f5SDaniel Borkmann } 1597bae141f5SDaniel Borkmann 15985ccb071eSDaniel Borkmann int __bpf_prog_charge(struct user_struct *user, u32 pages) 15995ccb071eSDaniel Borkmann { 16005ccb071eSDaniel Borkmann unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 16015ccb071eSDaniel Borkmann unsigned long user_bufs; 16025ccb071eSDaniel Borkmann 16035ccb071eSDaniel Borkmann if (user) { 16045ccb071eSDaniel Borkmann user_bufs = atomic_long_add_return(pages, &user->locked_vm); 16055ccb071eSDaniel Borkmann if (user_bufs > memlock_limit) { 16065ccb071eSDaniel Borkmann atomic_long_sub(pages, &user->locked_vm); 16075ccb071eSDaniel Borkmann return -EPERM; 16085ccb071eSDaniel Borkmann } 16095ccb071eSDaniel Borkmann } 16105ccb071eSDaniel Borkmann 16115ccb071eSDaniel Borkmann return 0; 16125ccb071eSDaniel Borkmann } 16135ccb071eSDaniel Borkmann 16145ccb071eSDaniel Borkmann void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 16155ccb071eSDaniel Borkmann { 16165ccb071eSDaniel Borkmann if (user) 16175ccb071eSDaniel Borkmann atomic_long_sub(pages, &user->locked_vm); 16185ccb071eSDaniel Borkmann } 16195ccb071eSDaniel Borkmann 1620aaac3ba9SAlexei Starovoitov static int bpf_prog_charge_memlock(struct bpf_prog *prog) 1621aaac3ba9SAlexei Starovoitov { 1622aaac3ba9SAlexei Starovoitov struct user_struct *user = get_current_user(); 16235ccb071eSDaniel Borkmann int ret; 1624aaac3ba9SAlexei Starovoitov 16255ccb071eSDaniel Borkmann ret = __bpf_prog_charge(user, prog->pages); 16265ccb071eSDaniel Borkmann if (ret) { 1627aaac3ba9SAlexei Starovoitov free_uid(user); 16285ccb071eSDaniel Borkmann return ret; 1629aaac3ba9SAlexei Starovoitov } 16305ccb071eSDaniel Borkmann 1631aaac3ba9SAlexei Starovoitov prog->aux->user = user; 1632aaac3ba9SAlexei Starovoitov return 0; 1633aaac3ba9SAlexei Starovoitov } 1634aaac3ba9SAlexei Starovoitov 1635aaac3ba9SAlexei Starovoitov static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 1636aaac3ba9SAlexei Starovoitov { 1637aaac3ba9SAlexei Starovoitov struct user_struct *user = prog->aux->user; 1638aaac3ba9SAlexei Starovoitov 16395ccb071eSDaniel Borkmann __bpf_prog_uncharge(user, prog->pages); 1640aaac3ba9SAlexei Starovoitov free_uid(user); 1641aaac3ba9SAlexei Starovoitov } 1642aaac3ba9SAlexei Starovoitov 1643dc4bb0e2SMartin KaFai Lau static int bpf_prog_alloc_id(struct bpf_prog *prog) 1644dc4bb0e2SMartin KaFai Lau { 1645dc4bb0e2SMartin KaFai Lau int id; 1646dc4bb0e2SMartin KaFai Lau 1647b76354cdSShaohua Li idr_preload(GFP_KERNEL); 1648dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1649dc4bb0e2SMartin KaFai Lau id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 1650dc4bb0e2SMartin KaFai Lau if (id > 0) 1651dc4bb0e2SMartin KaFai Lau prog->aux->id = id; 1652dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1653b76354cdSShaohua Li idr_preload_end(); 1654dc4bb0e2SMartin KaFai Lau 1655dc4bb0e2SMartin KaFai Lau /* id is in [1, INT_MAX) */ 1656dc4bb0e2SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 1657dc4bb0e2SMartin KaFai Lau return -ENOSPC; 1658dc4bb0e2SMartin KaFai Lau 1659dc4bb0e2SMartin KaFai Lau return id > 0 ? 0 : id; 1660dc4bb0e2SMartin KaFai Lau } 1661dc4bb0e2SMartin KaFai Lau 1662ad8ad79fSJakub Kicinski void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) 1663dc4bb0e2SMartin KaFai Lau { 1664ad8ad79fSJakub Kicinski /* cBPF to eBPF migrations are currently not in the idr store. 1665ad8ad79fSJakub Kicinski * Offloaded programs are removed from the store when their device 1666ad8ad79fSJakub Kicinski * disappears - even if someone grabs an fd to them they are unusable, 1667ad8ad79fSJakub Kicinski * simply waiting for refcnt to drop to be freed. 1668ad8ad79fSJakub Kicinski */ 1669dc4bb0e2SMartin KaFai Lau if (!prog->aux->id) 1670dc4bb0e2SMartin KaFai Lau return; 1671dc4bb0e2SMartin KaFai Lau 1672b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1673dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1674b16d9aa4SMartin KaFai Lau else 1675b16d9aa4SMartin KaFai Lau __acquire(&prog_idr_lock); 1676b16d9aa4SMartin KaFai Lau 1677dc4bb0e2SMartin KaFai Lau idr_remove(&prog_idr, prog->aux->id); 1678ad8ad79fSJakub Kicinski prog->aux->id = 0; 1679b16d9aa4SMartin KaFai Lau 1680b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1681dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1682b16d9aa4SMartin KaFai Lau else 1683b16d9aa4SMartin KaFai Lau __release(&prog_idr_lock); 1684dc4bb0e2SMartin KaFai Lau } 1685dc4bb0e2SMartin KaFai Lau 16861aacde3dSDaniel Borkmann static void __bpf_prog_put_rcu(struct rcu_head *rcu) 1687abf2e7d6SAlexei Starovoitov { 1688abf2e7d6SAlexei Starovoitov struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 1689abf2e7d6SAlexei Starovoitov 16903b4d9eb2SDaniel Borkmann kvfree(aux->func_info); 16918c1b6e69SAlexei Starovoitov kfree(aux->func_info_aux); 1692aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(aux->prog); 1693afdb09c7SChenbo Feng security_bpf_prog_free(aux); 1694abf2e7d6SAlexei Starovoitov bpf_prog_free(aux->prog); 1695abf2e7d6SAlexei Starovoitov } 1696abf2e7d6SAlexei Starovoitov 1697cd7455f1SDaniel Borkmann static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) 1698cd7455f1SDaniel Borkmann { 1699cd7455f1SDaniel Borkmann bpf_prog_kallsyms_del_all(prog); 1700cd7455f1SDaniel Borkmann btf_put(prog->aux->btf); 1701cd7455f1SDaniel Borkmann bpf_prog_free_linfo(prog); 1702cd7455f1SDaniel Borkmann 1703cd7455f1SDaniel Borkmann if (deferred) 1704cd7455f1SDaniel Borkmann call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 1705cd7455f1SDaniel Borkmann else 1706cd7455f1SDaniel Borkmann __bpf_prog_put_rcu(&prog->aux->rcu); 1707cd7455f1SDaniel Borkmann } 1708cd7455f1SDaniel Borkmann 1709b16d9aa4SMartin KaFai Lau static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 171009756af4SAlexei Starovoitov { 171185192dbfSAndrii Nakryiko if (atomic64_dec_and_test(&prog->aux->refcnt)) { 17126ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0); 1713bae141f5SDaniel Borkmann bpf_audit_prog(prog, BPF_AUDIT_UNLOAD); 171434ad5580SMartin KaFai Lau /* bpf_prog_free_id() must be called first */ 1715b16d9aa4SMartin KaFai Lau bpf_prog_free_id(prog, do_idr_lock); 1716cd7455f1SDaniel Borkmann __bpf_prog_put_noref(prog, true); 171709756af4SAlexei Starovoitov } 1718a67edbf4SDaniel Borkmann } 1719b16d9aa4SMartin KaFai Lau 1720b16d9aa4SMartin KaFai Lau void bpf_prog_put(struct bpf_prog *prog) 1721b16d9aa4SMartin KaFai Lau { 1722b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, true); 1723b16d9aa4SMartin KaFai Lau } 1724e2e9b654SDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_put); 172509756af4SAlexei Starovoitov 172609756af4SAlexei Starovoitov static int bpf_prog_release(struct inode *inode, struct file *filp) 172709756af4SAlexei Starovoitov { 172809756af4SAlexei Starovoitov struct bpf_prog *prog = filp->private_data; 172909756af4SAlexei Starovoitov 17301aacde3dSDaniel Borkmann bpf_prog_put(prog); 173109756af4SAlexei Starovoitov return 0; 173209756af4SAlexei Starovoitov } 173309756af4SAlexei Starovoitov 1734492ecee8SAlexei Starovoitov static void bpf_prog_get_stats(const struct bpf_prog *prog, 1735492ecee8SAlexei Starovoitov struct bpf_prog_stats *stats) 1736492ecee8SAlexei Starovoitov { 1737492ecee8SAlexei Starovoitov u64 nsecs = 0, cnt = 0; 1738492ecee8SAlexei Starovoitov int cpu; 1739492ecee8SAlexei Starovoitov 1740492ecee8SAlexei Starovoitov for_each_possible_cpu(cpu) { 1741492ecee8SAlexei Starovoitov const struct bpf_prog_stats *st; 1742492ecee8SAlexei Starovoitov unsigned int start; 1743492ecee8SAlexei Starovoitov u64 tnsecs, tcnt; 1744492ecee8SAlexei Starovoitov 1745492ecee8SAlexei Starovoitov st = per_cpu_ptr(prog->aux->stats, cpu); 1746492ecee8SAlexei Starovoitov do { 1747492ecee8SAlexei Starovoitov start = u64_stats_fetch_begin_irq(&st->syncp); 1748492ecee8SAlexei Starovoitov tnsecs = st->nsecs; 1749492ecee8SAlexei Starovoitov tcnt = st->cnt; 1750492ecee8SAlexei Starovoitov } while (u64_stats_fetch_retry_irq(&st->syncp, start)); 1751492ecee8SAlexei Starovoitov nsecs += tnsecs; 1752492ecee8SAlexei Starovoitov cnt += tcnt; 1753492ecee8SAlexei Starovoitov } 1754492ecee8SAlexei Starovoitov stats->nsecs = nsecs; 1755492ecee8SAlexei Starovoitov stats->cnt = cnt; 1756492ecee8SAlexei Starovoitov } 1757492ecee8SAlexei Starovoitov 17587bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 17597bd509e3SDaniel Borkmann static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 17607bd509e3SDaniel Borkmann { 17617bd509e3SDaniel Borkmann const struct bpf_prog *prog = filp->private_data; 1762f1f7714eSDaniel Borkmann char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 1763492ecee8SAlexei Starovoitov struct bpf_prog_stats stats; 17647bd509e3SDaniel Borkmann 1765492ecee8SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 1766f1f7714eSDaniel Borkmann bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 17677bd509e3SDaniel Borkmann seq_printf(m, 17687bd509e3SDaniel Borkmann "prog_type:\t%u\n" 17697bd509e3SDaniel Borkmann "prog_jited:\t%u\n" 1770f1f7714eSDaniel Borkmann "prog_tag:\t%s\n" 17714316b409SDaniel Borkmann "memlock:\t%llu\n" 1772492ecee8SAlexei Starovoitov "prog_id:\t%u\n" 1773492ecee8SAlexei Starovoitov "run_time_ns:\t%llu\n" 1774492ecee8SAlexei Starovoitov "run_cnt:\t%llu\n", 17757bd509e3SDaniel Borkmann prog->type, 17767bd509e3SDaniel Borkmann prog->jited, 1777f1f7714eSDaniel Borkmann prog_tag, 17784316b409SDaniel Borkmann prog->pages * 1ULL << PAGE_SHIFT, 1779492ecee8SAlexei Starovoitov prog->aux->id, 1780492ecee8SAlexei Starovoitov stats.nsecs, 1781492ecee8SAlexei Starovoitov stats.cnt); 17827bd509e3SDaniel Borkmann } 17837bd509e3SDaniel Borkmann #endif 17847bd509e3SDaniel Borkmann 1785f66e448cSChenbo Feng const struct file_operations bpf_prog_fops = { 17867bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 17877bd509e3SDaniel Borkmann .show_fdinfo = bpf_prog_show_fdinfo, 17887bd509e3SDaniel Borkmann #endif 178909756af4SAlexei Starovoitov .release = bpf_prog_release, 17906e71b04aSChenbo Feng .read = bpf_dummy_read, 17916e71b04aSChenbo Feng .write = bpf_dummy_write, 179209756af4SAlexei Starovoitov }; 179309756af4SAlexei Starovoitov 1794b2197755SDaniel Borkmann int bpf_prog_new_fd(struct bpf_prog *prog) 1795aa79781bSDaniel Borkmann { 1796afdb09c7SChenbo Feng int ret; 1797afdb09c7SChenbo Feng 1798afdb09c7SChenbo Feng ret = security_bpf_prog(prog); 1799afdb09c7SChenbo Feng if (ret < 0) 1800afdb09c7SChenbo Feng return ret; 1801afdb09c7SChenbo Feng 1802aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 1803aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC); 1804aa79781bSDaniel Borkmann } 1805aa79781bSDaniel Borkmann 1806113214beSDaniel Borkmann static struct bpf_prog *____bpf_prog_get(struct fd f) 180709756af4SAlexei Starovoitov { 180809756af4SAlexei Starovoitov if (!f.file) 180909756af4SAlexei Starovoitov return ERR_PTR(-EBADF); 181009756af4SAlexei Starovoitov if (f.file->f_op != &bpf_prog_fops) { 181109756af4SAlexei Starovoitov fdput(f); 181209756af4SAlexei Starovoitov return ERR_PTR(-EINVAL); 181309756af4SAlexei Starovoitov } 181409756af4SAlexei Starovoitov 1815c2101297SDaniel Borkmann return f.file->private_data; 181609756af4SAlexei Starovoitov } 181709756af4SAlexei Starovoitov 181885192dbfSAndrii Nakryiko void bpf_prog_add(struct bpf_prog *prog, int i) 181992117d84SAlexei Starovoitov { 182085192dbfSAndrii Nakryiko atomic64_add(i, &prog->aux->refcnt); 182192117d84SAlexei Starovoitov } 182259d3656dSBrenden Blanco EXPORT_SYMBOL_GPL(bpf_prog_add); 182359d3656dSBrenden Blanco 1824c540594fSDaniel Borkmann void bpf_prog_sub(struct bpf_prog *prog, int i) 1825c540594fSDaniel Borkmann { 1826c540594fSDaniel Borkmann /* Only to be used for undoing previous bpf_prog_add() in some 1827c540594fSDaniel Borkmann * error path. We still know that another entity in our call 1828c540594fSDaniel Borkmann * path holds a reference to the program, thus atomic_sub() can 1829c540594fSDaniel Borkmann * be safely used in such cases! 1830c540594fSDaniel Borkmann */ 183185192dbfSAndrii Nakryiko WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0); 1832c540594fSDaniel Borkmann } 1833c540594fSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_sub); 1834c540594fSDaniel Borkmann 183585192dbfSAndrii Nakryiko void bpf_prog_inc(struct bpf_prog *prog) 183659d3656dSBrenden Blanco { 183785192dbfSAndrii Nakryiko atomic64_inc(&prog->aux->refcnt); 183859d3656dSBrenden Blanco } 183997bc402dSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_inc); 184092117d84SAlexei Starovoitov 1841b16d9aa4SMartin KaFai Lau /* prog_idr_lock should have been held */ 1842a6f6df69SJohn Fastabend struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 1843b16d9aa4SMartin KaFai Lau { 1844b16d9aa4SMartin KaFai Lau int refold; 1845b16d9aa4SMartin KaFai Lau 184685192dbfSAndrii Nakryiko refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0); 1847b16d9aa4SMartin KaFai Lau 1848b16d9aa4SMartin KaFai Lau if (!refold) 1849b16d9aa4SMartin KaFai Lau return ERR_PTR(-ENOENT); 1850b16d9aa4SMartin KaFai Lau 1851b16d9aa4SMartin KaFai Lau return prog; 1852b16d9aa4SMartin KaFai Lau } 1853a6f6df69SJohn Fastabend EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 1854b16d9aa4SMartin KaFai Lau 1855040ee692SAl Viro bool bpf_prog_get_ok(struct bpf_prog *prog, 1856288b3de5SJakub Kicinski enum bpf_prog_type *attach_type, bool attach_drv) 1857248f346fSJakub Kicinski { 1858288b3de5SJakub Kicinski /* not an attachment, just a refcount inc, always allow */ 1859288b3de5SJakub Kicinski if (!attach_type) 1860288b3de5SJakub Kicinski return true; 1861248f346fSJakub Kicinski 1862248f346fSJakub Kicinski if (prog->type != *attach_type) 1863248f346fSJakub Kicinski return false; 1864288b3de5SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv) 1865248f346fSJakub Kicinski return false; 1866248f346fSJakub Kicinski 1867248f346fSJakub Kicinski return true; 1868248f346fSJakub Kicinski } 1869248f346fSJakub Kicinski 1870248f346fSJakub Kicinski static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, 1871288b3de5SJakub Kicinski bool attach_drv) 187209756af4SAlexei Starovoitov { 187309756af4SAlexei Starovoitov struct fd f = fdget(ufd); 187409756af4SAlexei Starovoitov struct bpf_prog *prog; 187509756af4SAlexei Starovoitov 1876113214beSDaniel Borkmann prog = ____bpf_prog_get(f); 187709756af4SAlexei Starovoitov if (IS_ERR(prog)) 187809756af4SAlexei Starovoitov return prog; 1879288b3de5SJakub Kicinski if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) { 1880113214beSDaniel Borkmann prog = ERR_PTR(-EINVAL); 1881113214beSDaniel Borkmann goto out; 1882113214beSDaniel Borkmann } 188309756af4SAlexei Starovoitov 188485192dbfSAndrii Nakryiko bpf_prog_inc(prog); 1885113214beSDaniel Borkmann out: 188609756af4SAlexei Starovoitov fdput(f); 188709756af4SAlexei Starovoitov return prog; 188809756af4SAlexei Starovoitov } 1889113214beSDaniel Borkmann 1890113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get(u32 ufd) 1891113214beSDaniel Borkmann { 1892288b3de5SJakub Kicinski return __bpf_prog_get(ufd, NULL, false); 1893113214beSDaniel Borkmann } 1894113214beSDaniel Borkmann 1895248f346fSJakub Kicinski struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 1896288b3de5SJakub Kicinski bool attach_drv) 1897248f346fSJakub Kicinski { 18984d220ed0SAlexei Starovoitov return __bpf_prog_get(ufd, &type, attach_drv); 1899248f346fSJakub Kicinski } 19006c8dfe21SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev); 1901248f346fSJakub Kicinski 1902aac3fc32SAndrey Ignatov /* Initially all BPF programs could be loaded w/o specifying 1903aac3fc32SAndrey Ignatov * expected_attach_type. Later for some of them specifying expected_attach_type 1904aac3fc32SAndrey Ignatov * at load time became required so that program could be validated properly. 1905aac3fc32SAndrey Ignatov * Programs of types that are allowed to be loaded both w/ and w/o (for 1906aac3fc32SAndrey Ignatov * backward compatibility) expected_attach_type, should have the default attach 1907aac3fc32SAndrey Ignatov * type assigned to expected_attach_type for the latter case, so that it can be 1908aac3fc32SAndrey Ignatov * validated later at attach time. 1909aac3fc32SAndrey Ignatov * 1910aac3fc32SAndrey Ignatov * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if 1911aac3fc32SAndrey Ignatov * prog type requires it but has some attach types that have to be backward 1912aac3fc32SAndrey Ignatov * compatible. 1913aac3fc32SAndrey Ignatov */ 1914aac3fc32SAndrey Ignatov static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) 1915aac3fc32SAndrey Ignatov { 1916aac3fc32SAndrey Ignatov switch (attr->prog_type) { 1917aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1918aac3fc32SAndrey Ignatov /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't 1919aac3fc32SAndrey Ignatov * exist so checking for non-zero is the way to go here. 1920aac3fc32SAndrey Ignatov */ 1921aac3fc32SAndrey Ignatov if (!attr->expected_attach_type) 1922aac3fc32SAndrey Ignatov attr->expected_attach_type = 1923aac3fc32SAndrey Ignatov BPF_CGROUP_INET_SOCK_CREATE; 1924aac3fc32SAndrey Ignatov break; 1925aac3fc32SAndrey Ignatov } 1926aac3fc32SAndrey Ignatov } 1927aac3fc32SAndrey Ignatov 19285e43f899SAndrey Ignatov static int 1929ccfe29ebSAlexei Starovoitov bpf_prog_load_check_attach(enum bpf_prog_type prog_type, 1930ccfe29ebSAlexei Starovoitov enum bpf_attach_type expected_attach_type, 19315b92a28aSAlexei Starovoitov u32 btf_id, u32 prog_fd) 19325e43f899SAndrey Ignatov { 193327ae7997SMartin KaFai Lau if (btf_id) { 1934c108e3c1SAlexei Starovoitov if (btf_id > BTF_MAX_TYPE) 1935c108e3c1SAlexei Starovoitov return -EINVAL; 193627ae7997SMartin KaFai Lau 193727ae7997SMartin KaFai Lau switch (prog_type) { 193827ae7997SMartin KaFai Lau case BPF_PROG_TYPE_TRACING: 193927ae7997SMartin KaFai Lau case BPF_PROG_TYPE_STRUCT_OPS: 1940be8704ffSAlexei Starovoitov case BPF_PROG_TYPE_EXT: 1941c108e3c1SAlexei Starovoitov break; 1942c108e3c1SAlexei Starovoitov default: 1943c108e3c1SAlexei Starovoitov return -EINVAL; 1944c108e3c1SAlexei Starovoitov } 194527ae7997SMartin KaFai Lau } 194627ae7997SMartin KaFai Lau 1947be8704ffSAlexei Starovoitov if (prog_fd && prog_type != BPF_PROG_TYPE_TRACING && 1948be8704ffSAlexei Starovoitov prog_type != BPF_PROG_TYPE_EXT) 194927ae7997SMartin KaFai Lau return -EINVAL; 1950c108e3c1SAlexei Starovoitov 1951c108e3c1SAlexei Starovoitov switch (prog_type) { 1952aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1953aac3fc32SAndrey Ignatov switch (expected_attach_type) { 1954aac3fc32SAndrey Ignatov case BPF_CGROUP_INET_SOCK_CREATE: 1955aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1956aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 1957aac3fc32SAndrey Ignatov return 0; 1958aac3fc32SAndrey Ignatov default: 1959aac3fc32SAndrey Ignatov return -EINVAL; 1960aac3fc32SAndrey Ignatov } 19614fbac77dSAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 19624fbac77dSAndrey Ignatov switch (expected_attach_type) { 19634fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 19644fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1965d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1966d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 19671cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 19681cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 1969983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG: 1970983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG: 19715e43f899SAndrey Ignatov return 0; 19724fbac77dSAndrey Ignatov default: 19734fbac77dSAndrey Ignatov return -EINVAL; 19744fbac77dSAndrey Ignatov } 19755cf1e914Sbrakmo case BPF_PROG_TYPE_CGROUP_SKB: 19765cf1e914Sbrakmo switch (expected_attach_type) { 19775cf1e914Sbrakmo case BPF_CGROUP_INET_INGRESS: 19785cf1e914Sbrakmo case BPF_CGROUP_INET_EGRESS: 19795cf1e914Sbrakmo return 0; 19805cf1e914Sbrakmo default: 19815cf1e914Sbrakmo return -EINVAL; 19825cf1e914Sbrakmo } 19830d01da6aSStanislav Fomichev case BPF_PROG_TYPE_CGROUP_SOCKOPT: 19840d01da6aSStanislav Fomichev switch (expected_attach_type) { 19850d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT: 19860d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT: 19870d01da6aSStanislav Fomichev return 0; 19880d01da6aSStanislav Fomichev default: 19890d01da6aSStanislav Fomichev return -EINVAL; 19900d01da6aSStanislav Fomichev } 1991be8704ffSAlexei Starovoitov case BPF_PROG_TYPE_EXT: 1992be8704ffSAlexei Starovoitov if (expected_attach_type) 1993be8704ffSAlexei Starovoitov return -EINVAL; 1994be8704ffSAlexei Starovoitov /* fallthrough */ 19954fbac77dSAndrey Ignatov default: 19964fbac77dSAndrey Ignatov return 0; 19974fbac77dSAndrey Ignatov } 19985e43f899SAndrey Ignatov } 19995e43f899SAndrey Ignatov 200009756af4SAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 20015b92a28aSAlexei Starovoitov #define BPF_PROG_LOAD_LAST_FIELD attach_prog_fd 200209756af4SAlexei Starovoitov 2003838e9690SYonghong Song static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) 200409756af4SAlexei Starovoitov { 200509756af4SAlexei Starovoitov enum bpf_prog_type type = attr->prog_type; 200609756af4SAlexei Starovoitov struct bpf_prog *prog; 200709756af4SAlexei Starovoitov int err; 200809756af4SAlexei Starovoitov char license[128]; 200909756af4SAlexei Starovoitov bool is_gpl; 201009756af4SAlexei Starovoitov 201109756af4SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_LOAD)) 201209756af4SAlexei Starovoitov return -EINVAL; 201309756af4SAlexei Starovoitov 2014c240eff6SJiong Wang if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | 2015c240eff6SJiong Wang BPF_F_ANY_ALIGNMENT | 201610d274e8SAlexei Starovoitov BPF_F_TEST_STATE_FREQ | 2017c240eff6SJiong Wang BPF_F_TEST_RND_HI32)) 2018e07b98d9SDavid S. Miller return -EINVAL; 2019e07b98d9SDavid S. Miller 2020e9ee9efcSDavid Miller if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && 2021e9ee9efcSDavid Miller (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && 2022e9ee9efcSDavid Miller !capable(CAP_SYS_ADMIN)) 2023e9ee9efcSDavid Miller return -EPERM; 2024e9ee9efcSDavid Miller 202509756af4SAlexei Starovoitov /* copy eBPF program license from user space */ 2026535e7b4bSMickaël Salaün if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 202709756af4SAlexei Starovoitov sizeof(license) - 1) < 0) 202809756af4SAlexei Starovoitov return -EFAULT; 202909756af4SAlexei Starovoitov license[sizeof(license) - 1] = 0; 203009756af4SAlexei Starovoitov 203109756af4SAlexei Starovoitov /* eBPF programs must be GPL compatible to use GPL-ed functions */ 203209756af4SAlexei Starovoitov is_gpl = license_is_gpl_compatible(license); 203309756af4SAlexei Starovoitov 2034c04c0d2bSAlexei Starovoitov if (attr->insn_cnt == 0 || 2035c04c0d2bSAlexei Starovoitov attr->insn_cnt > (capable(CAP_SYS_ADMIN) ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) 2036ef0915caSDaniel Borkmann return -E2BIG; 203780b7d819SChenbo Feng if (type != BPF_PROG_TYPE_SOCKET_FILTER && 203880b7d819SChenbo Feng type != BPF_PROG_TYPE_CGROUP_SKB && 203980b7d819SChenbo Feng !capable(CAP_SYS_ADMIN)) 20401be7f75dSAlexei Starovoitov return -EPERM; 20411be7f75dSAlexei Starovoitov 2042aac3fc32SAndrey Ignatov bpf_prog_load_fixup_attach_type(attr); 2043ccfe29ebSAlexei Starovoitov if (bpf_prog_load_check_attach(type, attr->expected_attach_type, 20445b92a28aSAlexei Starovoitov attr->attach_btf_id, 20455b92a28aSAlexei Starovoitov attr->attach_prog_fd)) 20465e43f899SAndrey Ignatov return -EINVAL; 20475e43f899SAndrey Ignatov 204809756af4SAlexei Starovoitov /* plain bpf_prog allocation */ 204909756af4SAlexei Starovoitov prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 205009756af4SAlexei Starovoitov if (!prog) 205109756af4SAlexei Starovoitov return -ENOMEM; 205209756af4SAlexei Starovoitov 20535e43f899SAndrey Ignatov prog->expected_attach_type = attr->expected_attach_type; 2054ccfe29ebSAlexei Starovoitov prog->aux->attach_btf_id = attr->attach_btf_id; 20555b92a28aSAlexei Starovoitov if (attr->attach_prog_fd) { 20565b92a28aSAlexei Starovoitov struct bpf_prog *tgt_prog; 20575b92a28aSAlexei Starovoitov 20585b92a28aSAlexei Starovoitov tgt_prog = bpf_prog_get(attr->attach_prog_fd); 20595b92a28aSAlexei Starovoitov if (IS_ERR(tgt_prog)) { 20605b92a28aSAlexei Starovoitov err = PTR_ERR(tgt_prog); 20615b92a28aSAlexei Starovoitov goto free_prog_nouncharge; 20625b92a28aSAlexei Starovoitov } 20635b92a28aSAlexei Starovoitov prog->aux->linked_prog = tgt_prog; 20645b92a28aSAlexei Starovoitov } 20655e43f899SAndrey Ignatov 20669a18eedbSJakub Kicinski prog->aux->offload_requested = !!attr->prog_ifindex; 20679a18eedbSJakub Kicinski 2068afdb09c7SChenbo Feng err = security_bpf_prog_alloc(prog->aux); 2069aaac3ba9SAlexei Starovoitov if (err) 2070aaac3ba9SAlexei Starovoitov goto free_prog_nouncharge; 2071aaac3ba9SAlexei Starovoitov 2072afdb09c7SChenbo Feng err = bpf_prog_charge_memlock(prog); 2073afdb09c7SChenbo Feng if (err) 2074afdb09c7SChenbo Feng goto free_prog_sec; 2075afdb09c7SChenbo Feng 207609756af4SAlexei Starovoitov prog->len = attr->insn_cnt; 207709756af4SAlexei Starovoitov 207809756af4SAlexei Starovoitov err = -EFAULT; 2079535e7b4bSMickaël Salaün if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 2080aafe6ae9SDaniel Borkmann bpf_prog_insn_size(prog)) != 0) 208109756af4SAlexei Starovoitov goto free_prog; 208209756af4SAlexei Starovoitov 208309756af4SAlexei Starovoitov prog->orig_prog = NULL; 2084a91263d5SDaniel Borkmann prog->jited = 0; 208509756af4SAlexei Starovoitov 208685192dbfSAndrii Nakryiko atomic64_set(&prog->aux->refcnt, 1); 2087a91263d5SDaniel Borkmann prog->gpl_compatible = is_gpl ? 1 : 0; 208809756af4SAlexei Starovoitov 20899a18eedbSJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 2090ab3f0063SJakub Kicinski err = bpf_prog_offload_init(prog, attr); 2091ab3f0063SJakub Kicinski if (err) 2092ab3f0063SJakub Kicinski goto free_prog; 2093ab3f0063SJakub Kicinski } 2094ab3f0063SJakub Kicinski 209509756af4SAlexei Starovoitov /* find program type: socket_filter vs tracing_filter */ 209609756af4SAlexei Starovoitov err = find_prog_type(type, prog); 209709756af4SAlexei Starovoitov if (err < 0) 209809756af4SAlexei Starovoitov goto free_prog; 209909756af4SAlexei Starovoitov 21009285ec4cSJason A. Donenfeld prog->aux->load_time = ktime_get_boottime_ns(); 2101cb4d2b3fSMartin KaFai Lau err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name); 2102cb4d2b3fSMartin KaFai Lau if (err) 2103cb4d2b3fSMartin KaFai Lau goto free_prog; 2104cb4d2b3fSMartin KaFai Lau 210509756af4SAlexei Starovoitov /* run eBPF verifier */ 2106838e9690SYonghong Song err = bpf_check(&prog, attr, uattr); 210709756af4SAlexei Starovoitov if (err < 0) 210809756af4SAlexei Starovoitov goto free_used_maps; 210909756af4SAlexei Starovoitov 2110d1c55ab5SDaniel Borkmann prog = bpf_prog_select_runtime(prog, &err); 211104fd61abSAlexei Starovoitov if (err < 0) 211204fd61abSAlexei Starovoitov goto free_used_maps; 211309756af4SAlexei Starovoitov 2114dc4bb0e2SMartin KaFai Lau err = bpf_prog_alloc_id(prog); 2115dc4bb0e2SMartin KaFai Lau if (err) 2116dc4bb0e2SMartin KaFai Lau goto free_used_maps; 2117dc4bb0e2SMartin KaFai Lau 2118c751798aSDaniel Borkmann /* Upon success of bpf_prog_alloc_id(), the BPF prog is 2119c751798aSDaniel Borkmann * effectively publicly exposed. However, retrieving via 2120c751798aSDaniel Borkmann * bpf_prog_get_fd_by_id() will take another reference, 2121c751798aSDaniel Borkmann * therefore it cannot be gone underneath us. 2122c751798aSDaniel Borkmann * 2123c751798aSDaniel Borkmann * Only for the time /after/ successful bpf_prog_new_fd() 2124c751798aSDaniel Borkmann * and before returning to userspace, we might just hold 2125c751798aSDaniel Borkmann * one reference and any parallel close on that fd could 2126c751798aSDaniel Borkmann * rip everything out. Hence, below notifications must 2127c751798aSDaniel Borkmann * happen before bpf_prog_new_fd(). 2128c751798aSDaniel Borkmann * 2129c751798aSDaniel Borkmann * Also, any failure handling from this point onwards must 2130c751798aSDaniel Borkmann * be using bpf_prog_put() given the program is exposed. 2131b16d9aa4SMartin KaFai Lau */ 213274451e66SDaniel Borkmann bpf_prog_kallsyms_add(prog); 21336ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); 2134bae141f5SDaniel Borkmann bpf_audit_prog(prog, BPF_AUDIT_LOAD); 2135c751798aSDaniel Borkmann 2136c751798aSDaniel Borkmann err = bpf_prog_new_fd(prog); 2137c751798aSDaniel Borkmann if (err < 0) 2138c751798aSDaniel Borkmann bpf_prog_put(prog); 213909756af4SAlexei Starovoitov return err; 214009756af4SAlexei Starovoitov 214109756af4SAlexei Starovoitov free_used_maps: 2142cd7455f1SDaniel Borkmann /* In case we have subprogs, we need to wait for a grace 2143cd7455f1SDaniel Borkmann * period before we can tear down JIT memory since symbols 2144cd7455f1SDaniel Borkmann * are already exposed under kallsyms. 2145cd7455f1SDaniel Borkmann */ 2146cd7455f1SDaniel Borkmann __bpf_prog_put_noref(prog, prog->aux->func_cnt); 2147cd7455f1SDaniel Borkmann return err; 214809756af4SAlexei Starovoitov free_prog: 2149aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(prog); 2150afdb09c7SChenbo Feng free_prog_sec: 2151afdb09c7SChenbo Feng security_bpf_prog_free(prog->aux); 2152aaac3ba9SAlexei Starovoitov free_prog_nouncharge: 215309756af4SAlexei Starovoitov bpf_prog_free(prog); 215409756af4SAlexei Starovoitov return err; 215509756af4SAlexei Starovoitov } 215609756af4SAlexei Starovoitov 21576e71b04aSChenbo Feng #define BPF_OBJ_LAST_FIELD file_flags 2158b2197755SDaniel Borkmann 2159b2197755SDaniel Borkmann static int bpf_obj_pin(const union bpf_attr *attr) 2160b2197755SDaniel Borkmann { 21616e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0) 2162b2197755SDaniel Borkmann return -EINVAL; 2163b2197755SDaniel Borkmann 2164535e7b4bSMickaël Salaün return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 2165b2197755SDaniel Borkmann } 2166b2197755SDaniel Borkmann 2167b2197755SDaniel Borkmann static int bpf_obj_get(const union bpf_attr *attr) 2168b2197755SDaniel Borkmann { 21696e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || 21706e71b04aSChenbo Feng attr->file_flags & ~BPF_OBJ_FLAG_MASK) 2171b2197755SDaniel Borkmann return -EINVAL; 2172b2197755SDaniel Borkmann 21736e71b04aSChenbo Feng return bpf_obj_get_user(u64_to_user_ptr(attr->pathname), 21746e71b04aSChenbo Feng attr->file_flags); 2175b2197755SDaniel Borkmann } 2176b2197755SDaniel Borkmann 2177fec56f58SAlexei Starovoitov static int bpf_tracing_prog_release(struct inode *inode, struct file *filp) 2178fec56f58SAlexei Starovoitov { 2179fec56f58SAlexei Starovoitov struct bpf_prog *prog = filp->private_data; 2180fec56f58SAlexei Starovoitov 2181fec56f58SAlexei Starovoitov WARN_ON_ONCE(bpf_trampoline_unlink_prog(prog)); 2182fec56f58SAlexei Starovoitov bpf_prog_put(prog); 2183fec56f58SAlexei Starovoitov return 0; 2184fec56f58SAlexei Starovoitov } 2185fec56f58SAlexei Starovoitov 2186fec56f58SAlexei Starovoitov static const struct file_operations bpf_tracing_prog_fops = { 2187fec56f58SAlexei Starovoitov .release = bpf_tracing_prog_release, 2188fec56f58SAlexei Starovoitov .read = bpf_dummy_read, 2189fec56f58SAlexei Starovoitov .write = bpf_dummy_write, 2190fec56f58SAlexei Starovoitov }; 2191fec56f58SAlexei Starovoitov 2192fec56f58SAlexei Starovoitov static int bpf_tracing_prog_attach(struct bpf_prog *prog) 2193fec56f58SAlexei Starovoitov { 2194fec56f58SAlexei Starovoitov int tr_fd, err; 2195fec56f58SAlexei Starovoitov 2196fec56f58SAlexei Starovoitov if (prog->expected_attach_type != BPF_TRACE_FENTRY && 2197be8704ffSAlexei Starovoitov prog->expected_attach_type != BPF_TRACE_FEXIT && 2198be8704ffSAlexei Starovoitov prog->type != BPF_PROG_TYPE_EXT) { 2199fec56f58SAlexei Starovoitov err = -EINVAL; 2200fec56f58SAlexei Starovoitov goto out_put_prog; 2201fec56f58SAlexei Starovoitov } 2202fec56f58SAlexei Starovoitov 2203fec56f58SAlexei Starovoitov err = bpf_trampoline_link_prog(prog); 2204fec56f58SAlexei Starovoitov if (err) 2205fec56f58SAlexei Starovoitov goto out_put_prog; 2206fec56f58SAlexei Starovoitov 2207fec56f58SAlexei Starovoitov tr_fd = anon_inode_getfd("bpf-tracing-prog", &bpf_tracing_prog_fops, 2208fec56f58SAlexei Starovoitov prog, O_CLOEXEC); 2209fec56f58SAlexei Starovoitov if (tr_fd < 0) { 2210fec56f58SAlexei Starovoitov WARN_ON_ONCE(bpf_trampoline_unlink_prog(prog)); 2211fec56f58SAlexei Starovoitov err = tr_fd; 2212fec56f58SAlexei Starovoitov goto out_put_prog; 2213fec56f58SAlexei Starovoitov } 2214fec56f58SAlexei Starovoitov return tr_fd; 2215fec56f58SAlexei Starovoitov 2216fec56f58SAlexei Starovoitov out_put_prog: 2217fec56f58SAlexei Starovoitov bpf_prog_put(prog); 2218fec56f58SAlexei Starovoitov return err; 2219fec56f58SAlexei Starovoitov } 2220fec56f58SAlexei Starovoitov 2221c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint { 2222c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 2223c4f6699dSAlexei Starovoitov struct bpf_prog *prog; 2224c4f6699dSAlexei Starovoitov }; 2225c4f6699dSAlexei Starovoitov 2226c4f6699dSAlexei Starovoitov static int bpf_raw_tracepoint_release(struct inode *inode, struct file *filp) 2227c4f6699dSAlexei Starovoitov { 2228c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint *raw_tp = filp->private_data; 2229c4f6699dSAlexei Starovoitov 2230c4f6699dSAlexei Starovoitov if (raw_tp->prog) { 2231c4f6699dSAlexei Starovoitov bpf_probe_unregister(raw_tp->btp, raw_tp->prog); 2232c4f6699dSAlexei Starovoitov bpf_prog_put(raw_tp->prog); 2233c4f6699dSAlexei Starovoitov } 2234a38d1107SMatt Mullins bpf_put_raw_tracepoint(raw_tp->btp); 2235c4f6699dSAlexei Starovoitov kfree(raw_tp); 2236c4f6699dSAlexei Starovoitov return 0; 2237c4f6699dSAlexei Starovoitov } 2238c4f6699dSAlexei Starovoitov 2239c4f6699dSAlexei Starovoitov static const struct file_operations bpf_raw_tp_fops = { 2240c4f6699dSAlexei Starovoitov .release = bpf_raw_tracepoint_release, 2241c4f6699dSAlexei Starovoitov .read = bpf_dummy_read, 2242c4f6699dSAlexei Starovoitov .write = bpf_dummy_write, 2243c4f6699dSAlexei Starovoitov }; 2244c4f6699dSAlexei Starovoitov 2245c4f6699dSAlexei Starovoitov #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd 2246c4f6699dSAlexei Starovoitov 2247c4f6699dSAlexei Starovoitov static int bpf_raw_tracepoint_open(const union bpf_attr *attr) 2248c4f6699dSAlexei Starovoitov { 2249c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint *raw_tp; 2250c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 2251c4f6699dSAlexei Starovoitov struct bpf_prog *prog; 2252ac4414b5SAlexei Starovoitov const char *tp_name; 2253ac4414b5SAlexei Starovoitov char buf[128]; 2254c4f6699dSAlexei Starovoitov int tp_fd, err; 2255c4f6699dSAlexei Starovoitov 2256ac4414b5SAlexei Starovoitov if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN)) 2257ac4414b5SAlexei Starovoitov return -EINVAL; 2258ac4414b5SAlexei Starovoitov 2259ac4414b5SAlexei Starovoitov prog = bpf_prog_get(attr->raw_tracepoint.prog_fd); 2260ac4414b5SAlexei Starovoitov if (IS_ERR(prog)) 2261ac4414b5SAlexei Starovoitov return PTR_ERR(prog); 2262ac4414b5SAlexei Starovoitov 2263ac4414b5SAlexei Starovoitov if (prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT && 2264f1b9509cSAlexei Starovoitov prog->type != BPF_PROG_TYPE_TRACING && 2265be8704ffSAlexei Starovoitov prog->type != BPF_PROG_TYPE_EXT && 2266ac4414b5SAlexei Starovoitov prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) { 2267ac4414b5SAlexei Starovoitov err = -EINVAL; 2268ac4414b5SAlexei Starovoitov goto out_put_prog; 2269ac4414b5SAlexei Starovoitov } 2270ac4414b5SAlexei Starovoitov 2271be8704ffSAlexei Starovoitov if (prog->type == BPF_PROG_TYPE_TRACING || 2272be8704ffSAlexei Starovoitov prog->type == BPF_PROG_TYPE_EXT) { 2273ac4414b5SAlexei Starovoitov if (attr->raw_tracepoint.name) { 2274fec56f58SAlexei Starovoitov /* The attach point for this category of programs 2275fec56f58SAlexei Starovoitov * should be specified via btf_id during program load. 2276ac4414b5SAlexei Starovoitov */ 2277ac4414b5SAlexei Starovoitov err = -EINVAL; 2278ac4414b5SAlexei Starovoitov goto out_put_prog; 2279ac4414b5SAlexei Starovoitov } 2280fec56f58SAlexei Starovoitov if (prog->expected_attach_type == BPF_TRACE_RAW_TP) 228138207291SMartin KaFai Lau tp_name = prog->aux->attach_func_name; 2282fec56f58SAlexei Starovoitov else 2283fec56f58SAlexei Starovoitov return bpf_tracing_prog_attach(prog); 2284ac4414b5SAlexei Starovoitov } else { 2285ac4414b5SAlexei Starovoitov if (strncpy_from_user(buf, 2286ac4414b5SAlexei Starovoitov u64_to_user_ptr(attr->raw_tracepoint.name), 2287ac4414b5SAlexei Starovoitov sizeof(buf) - 1) < 0) { 2288ac4414b5SAlexei Starovoitov err = -EFAULT; 2289ac4414b5SAlexei Starovoitov goto out_put_prog; 2290ac4414b5SAlexei Starovoitov } 2291ac4414b5SAlexei Starovoitov buf[sizeof(buf) - 1] = 0; 2292ac4414b5SAlexei Starovoitov tp_name = buf; 2293ac4414b5SAlexei Starovoitov } 2294c4f6699dSAlexei Starovoitov 2295a38d1107SMatt Mullins btp = bpf_get_raw_tracepoint(tp_name); 2296ac4414b5SAlexei Starovoitov if (!btp) { 2297ac4414b5SAlexei Starovoitov err = -ENOENT; 2298ac4414b5SAlexei Starovoitov goto out_put_prog; 2299ac4414b5SAlexei Starovoitov } 2300c4f6699dSAlexei Starovoitov 2301c4f6699dSAlexei Starovoitov raw_tp = kzalloc(sizeof(*raw_tp), GFP_USER); 2302a38d1107SMatt Mullins if (!raw_tp) { 2303a38d1107SMatt Mullins err = -ENOMEM; 2304a38d1107SMatt Mullins goto out_put_btp; 2305a38d1107SMatt Mullins } 2306c4f6699dSAlexei Starovoitov raw_tp->btp = btp; 2307ac4414b5SAlexei Starovoitov raw_tp->prog = prog; 2308c4f6699dSAlexei Starovoitov 2309c4f6699dSAlexei Starovoitov err = bpf_probe_register(raw_tp->btp, prog); 2310c4f6699dSAlexei Starovoitov if (err) 2311ac4414b5SAlexei Starovoitov goto out_free_tp; 2312c4f6699dSAlexei Starovoitov 2313c4f6699dSAlexei Starovoitov tp_fd = anon_inode_getfd("bpf-raw-tracepoint", &bpf_raw_tp_fops, raw_tp, 2314c4f6699dSAlexei Starovoitov O_CLOEXEC); 2315c4f6699dSAlexei Starovoitov if (tp_fd < 0) { 2316c4f6699dSAlexei Starovoitov bpf_probe_unregister(raw_tp->btp, prog); 2317c4f6699dSAlexei Starovoitov err = tp_fd; 2318ac4414b5SAlexei Starovoitov goto out_free_tp; 2319c4f6699dSAlexei Starovoitov } 2320c4f6699dSAlexei Starovoitov return tp_fd; 2321c4f6699dSAlexei Starovoitov 2322c4f6699dSAlexei Starovoitov out_free_tp: 2323c4f6699dSAlexei Starovoitov kfree(raw_tp); 2324a38d1107SMatt Mullins out_put_btp: 2325a38d1107SMatt Mullins bpf_put_raw_tracepoint(btp); 2326ac4414b5SAlexei Starovoitov out_put_prog: 2327ac4414b5SAlexei Starovoitov bpf_prog_put(prog); 2328c4f6699dSAlexei Starovoitov return err; 2329c4f6699dSAlexei Starovoitov } 2330c4f6699dSAlexei Starovoitov 233133491588SAnders Roxell static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, 233233491588SAnders Roxell enum bpf_attach_type attach_type) 233333491588SAnders Roxell { 233433491588SAnders Roxell switch (prog->type) { 233533491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK: 233633491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 23370d01da6aSStanislav Fomichev case BPF_PROG_TYPE_CGROUP_SOCKOPT: 233833491588SAnders Roxell return attach_type == prog->expected_attach_type ? 0 : -EINVAL; 23395cf1e914Sbrakmo case BPF_PROG_TYPE_CGROUP_SKB: 23405cf1e914Sbrakmo return prog->enforce_expected_attach_type && 23415cf1e914Sbrakmo prog->expected_attach_type != attach_type ? 23425cf1e914Sbrakmo -EINVAL : 0; 234333491588SAnders Roxell default: 234433491588SAnders Roxell return 0; 234533491588SAnders Roxell } 234633491588SAnders Roxell } 234733491588SAnders Roxell 23487dd68b32SAndrey Ignatov #define BPF_PROG_ATTACH_LAST_FIELD replace_bpf_fd 2349174a79ffSJohn Fastabend 2350324bda9eSAlexei Starovoitov #define BPF_F_ATTACH_MASK \ 23517dd68b32SAndrey Ignatov (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI | BPF_F_REPLACE) 2352324bda9eSAlexei Starovoitov 2353f4324551SDaniel Mack static int bpf_prog_attach(const union bpf_attr *attr) 2354f4324551SDaniel Mack { 23557f677633SAlexei Starovoitov enum bpf_prog_type ptype; 2356f4324551SDaniel Mack struct bpf_prog *prog; 23577f677633SAlexei Starovoitov int ret; 2358f4324551SDaniel Mack 2359f4324551SDaniel Mack if (!capable(CAP_NET_ADMIN)) 2360f4324551SDaniel Mack return -EPERM; 2361f4324551SDaniel Mack 2362f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_ATTACH)) 2363f4324551SDaniel Mack return -EINVAL; 2364f4324551SDaniel Mack 2365324bda9eSAlexei Starovoitov if (attr->attach_flags & ~BPF_F_ATTACH_MASK) 23667f677633SAlexei Starovoitov return -EINVAL; 23677f677633SAlexei Starovoitov 2368f4324551SDaniel Mack switch (attr->attach_type) { 2369f4324551SDaniel Mack case BPF_CGROUP_INET_INGRESS: 2370f4324551SDaniel Mack case BPF_CGROUP_INET_EGRESS: 2371b2cd1257SDavid Ahern ptype = BPF_PROG_TYPE_CGROUP_SKB; 2372b2cd1257SDavid Ahern break; 237361023658SDavid Ahern case BPF_CGROUP_INET_SOCK_CREATE: 2374aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 2375aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 237661023658SDavid Ahern ptype = BPF_PROG_TYPE_CGROUP_SOCK; 237761023658SDavid Ahern break; 23784fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 23794fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 2380d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 2381d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 23821cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 23831cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 2384983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG: 2385983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG: 23864fbac77dSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 23874fbac77dSAndrey Ignatov break; 238840304b2aSLawrence Brakmo case BPF_CGROUP_SOCK_OPS: 238940304b2aSLawrence Brakmo ptype = BPF_PROG_TYPE_SOCK_OPS; 239040304b2aSLawrence Brakmo break; 2391ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 2392ebc614f6SRoman Gushchin ptype = BPF_PROG_TYPE_CGROUP_DEVICE; 2393ebc614f6SRoman Gushchin break; 23944f738adbSJohn Fastabend case BPF_SK_MSG_VERDICT: 2395fdb5c453SSean Young ptype = BPF_PROG_TYPE_SK_MSG; 2396fdb5c453SSean Young break; 2397464bc0fdSJohn Fastabend case BPF_SK_SKB_STREAM_PARSER: 2398464bc0fdSJohn Fastabend case BPF_SK_SKB_STREAM_VERDICT: 2399fdb5c453SSean Young ptype = BPF_PROG_TYPE_SK_SKB; 2400fdb5c453SSean Young break; 2401f4364dcfSSean Young case BPF_LIRC_MODE2: 2402fdb5c453SSean Young ptype = BPF_PROG_TYPE_LIRC_MODE2; 2403fdb5c453SSean Young break; 2404d58e468bSPetar Penkov case BPF_FLOW_DISSECTOR: 2405d58e468bSPetar Penkov ptype = BPF_PROG_TYPE_FLOW_DISSECTOR; 2406d58e468bSPetar Penkov break; 24077b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 24087b146cebSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SYSCTL; 24097b146cebSAndrey Ignatov break; 24100d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT: 24110d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT: 24120d01da6aSStanislav Fomichev ptype = BPF_PROG_TYPE_CGROUP_SOCKOPT; 24130d01da6aSStanislav Fomichev break; 2414b2cd1257SDavid Ahern default: 2415b2cd1257SDavid Ahern return -EINVAL; 2416b2cd1257SDavid Ahern } 2417b2cd1257SDavid Ahern 2418b2cd1257SDavid Ahern prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 2419f4324551SDaniel Mack if (IS_ERR(prog)) 2420f4324551SDaniel Mack return PTR_ERR(prog); 2421f4324551SDaniel Mack 24225e43f899SAndrey Ignatov if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) { 24235e43f899SAndrey Ignatov bpf_prog_put(prog); 24245e43f899SAndrey Ignatov return -EINVAL; 24255e43f899SAndrey Ignatov } 24265e43f899SAndrey Ignatov 2427fdb5c453SSean Young switch (ptype) { 2428fdb5c453SSean Young case BPF_PROG_TYPE_SK_SKB: 2429fdb5c453SSean Young case BPF_PROG_TYPE_SK_MSG: 2430604326b4SDaniel Borkmann ret = sock_map_get_from_fd(attr, prog); 2431fdb5c453SSean Young break; 2432fdb5c453SSean Young case BPF_PROG_TYPE_LIRC_MODE2: 2433fdb5c453SSean Young ret = lirc_prog_attach(attr, prog); 2434fdb5c453SSean Young break; 2435d58e468bSPetar Penkov case BPF_PROG_TYPE_FLOW_DISSECTOR: 2436d58e468bSPetar Penkov ret = skb_flow_dissector_bpf_prog_attach(attr, prog); 2437d58e468bSPetar Penkov break; 2438fdb5c453SSean Young default: 2439fdb5c453SSean Young ret = cgroup_bpf_prog_attach(attr, ptype, prog); 2440f4324551SDaniel Mack } 2441f4324551SDaniel Mack 24427f677633SAlexei Starovoitov if (ret) 24437f677633SAlexei Starovoitov bpf_prog_put(prog); 24447f677633SAlexei Starovoitov return ret; 2445f4324551SDaniel Mack } 2446f4324551SDaniel Mack 2447f4324551SDaniel Mack #define BPF_PROG_DETACH_LAST_FIELD attach_type 2448f4324551SDaniel Mack 2449f4324551SDaniel Mack static int bpf_prog_detach(const union bpf_attr *attr) 2450f4324551SDaniel Mack { 2451324bda9eSAlexei Starovoitov enum bpf_prog_type ptype; 2452f4324551SDaniel Mack 2453f4324551SDaniel Mack if (!capable(CAP_NET_ADMIN)) 2454f4324551SDaniel Mack return -EPERM; 2455f4324551SDaniel Mack 2456f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_DETACH)) 2457f4324551SDaniel Mack return -EINVAL; 2458f4324551SDaniel Mack 2459f4324551SDaniel Mack switch (attr->attach_type) { 2460f4324551SDaniel Mack case BPF_CGROUP_INET_INGRESS: 2461f4324551SDaniel Mack case BPF_CGROUP_INET_EGRESS: 2462324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_CGROUP_SKB; 2463324bda9eSAlexei Starovoitov break; 246461023658SDavid Ahern case BPF_CGROUP_INET_SOCK_CREATE: 2465aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 2466aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 2467324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_CGROUP_SOCK; 2468324bda9eSAlexei Starovoitov break; 24694fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 24704fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 2471d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 2472d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 24731cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 24741cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 2475983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG: 2476983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG: 24774fbac77dSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 24784fbac77dSAndrey Ignatov break; 247940304b2aSLawrence Brakmo case BPF_CGROUP_SOCK_OPS: 2480324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_SOCK_OPS; 2481f4324551SDaniel Mack break; 2482ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 2483ebc614f6SRoman Gushchin ptype = BPF_PROG_TYPE_CGROUP_DEVICE; 2484ebc614f6SRoman Gushchin break; 24854f738adbSJohn Fastabend case BPF_SK_MSG_VERDICT: 2486604326b4SDaniel Borkmann return sock_map_get_from_fd(attr, NULL); 24875a67da2aSJohn Fastabend case BPF_SK_SKB_STREAM_PARSER: 24885a67da2aSJohn Fastabend case BPF_SK_SKB_STREAM_VERDICT: 2489604326b4SDaniel Borkmann return sock_map_get_from_fd(attr, NULL); 2490f4364dcfSSean Young case BPF_LIRC_MODE2: 2491f4364dcfSSean Young return lirc_prog_detach(attr); 2492d58e468bSPetar Penkov case BPF_FLOW_DISSECTOR: 2493d58e468bSPetar Penkov return skb_flow_dissector_bpf_prog_detach(attr); 24947b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 24957b146cebSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SYSCTL; 24967b146cebSAndrey Ignatov break; 24970d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT: 24980d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT: 24990d01da6aSStanislav Fomichev ptype = BPF_PROG_TYPE_CGROUP_SOCKOPT; 25000d01da6aSStanislav Fomichev break; 2501f4324551SDaniel Mack default: 2502f4324551SDaniel Mack return -EINVAL; 2503f4324551SDaniel Mack } 2504f4324551SDaniel Mack 2505fdb5c453SSean Young return cgroup_bpf_prog_detach(attr, ptype); 2506f4324551SDaniel Mack } 250740304b2aSLawrence Brakmo 2508468e2f64SAlexei Starovoitov #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt 2509468e2f64SAlexei Starovoitov 2510468e2f64SAlexei Starovoitov static int bpf_prog_query(const union bpf_attr *attr, 2511468e2f64SAlexei Starovoitov union bpf_attr __user *uattr) 2512468e2f64SAlexei Starovoitov { 2513468e2f64SAlexei Starovoitov if (!capable(CAP_NET_ADMIN)) 2514468e2f64SAlexei Starovoitov return -EPERM; 2515468e2f64SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_QUERY)) 2516468e2f64SAlexei Starovoitov return -EINVAL; 2517468e2f64SAlexei Starovoitov if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE) 2518468e2f64SAlexei Starovoitov return -EINVAL; 2519468e2f64SAlexei Starovoitov 2520468e2f64SAlexei Starovoitov switch (attr->query.attach_type) { 2521468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_INGRESS: 2522468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_EGRESS: 2523468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_SOCK_CREATE: 25244fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 25254fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 2526aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 2527aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 2528d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 2529d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 25301cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 25311cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 2532983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG: 2533983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG: 2534468e2f64SAlexei Starovoitov case BPF_CGROUP_SOCK_OPS: 2535ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 25367b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 25370d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT: 25380d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT: 2539468e2f64SAlexei Starovoitov break; 2540f4364dcfSSean Young case BPF_LIRC_MODE2: 2541f4364dcfSSean Young return lirc_prog_query(attr, uattr); 2542118c8e9aSStanislav Fomichev case BPF_FLOW_DISSECTOR: 2543118c8e9aSStanislav Fomichev return skb_flow_dissector_prog_query(attr, uattr); 2544468e2f64SAlexei Starovoitov default: 2545468e2f64SAlexei Starovoitov return -EINVAL; 2546468e2f64SAlexei Starovoitov } 2547fdb5c453SSean Young 2548fdb5c453SSean Young return cgroup_bpf_prog_query(attr, uattr); 2549468e2f64SAlexei Starovoitov } 2550f4324551SDaniel Mack 2551b0b9395dSStanislav Fomichev #define BPF_PROG_TEST_RUN_LAST_FIELD test.ctx_out 25521cf1cae9SAlexei Starovoitov 25531cf1cae9SAlexei Starovoitov static int bpf_prog_test_run(const union bpf_attr *attr, 25541cf1cae9SAlexei Starovoitov union bpf_attr __user *uattr) 25551cf1cae9SAlexei Starovoitov { 25561cf1cae9SAlexei Starovoitov struct bpf_prog *prog; 25571cf1cae9SAlexei Starovoitov int ret = -ENOTSUPP; 25581cf1cae9SAlexei Starovoitov 255961f3c964SAlexei Starovoitov if (!capable(CAP_SYS_ADMIN)) 256061f3c964SAlexei Starovoitov return -EPERM; 25611cf1cae9SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 25621cf1cae9SAlexei Starovoitov return -EINVAL; 25631cf1cae9SAlexei Starovoitov 2564b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_in && !attr->test.ctx_in) || 2565b0b9395dSStanislav Fomichev (!attr->test.ctx_size_in && attr->test.ctx_in)) 2566b0b9395dSStanislav Fomichev return -EINVAL; 2567b0b9395dSStanislav Fomichev 2568b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_out && !attr->test.ctx_out) || 2569b0b9395dSStanislav Fomichev (!attr->test.ctx_size_out && attr->test.ctx_out)) 2570b0b9395dSStanislav Fomichev return -EINVAL; 2571b0b9395dSStanislav Fomichev 25721cf1cae9SAlexei Starovoitov prog = bpf_prog_get(attr->test.prog_fd); 25731cf1cae9SAlexei Starovoitov if (IS_ERR(prog)) 25741cf1cae9SAlexei Starovoitov return PTR_ERR(prog); 25751cf1cae9SAlexei Starovoitov 25761cf1cae9SAlexei Starovoitov if (prog->aux->ops->test_run) 25771cf1cae9SAlexei Starovoitov ret = prog->aux->ops->test_run(prog, attr, uattr); 25781cf1cae9SAlexei Starovoitov 25791cf1cae9SAlexei Starovoitov bpf_prog_put(prog); 25801cf1cae9SAlexei Starovoitov return ret; 25811cf1cae9SAlexei Starovoitov } 25821cf1cae9SAlexei Starovoitov 258334ad5580SMartin KaFai Lau #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 258434ad5580SMartin KaFai Lau 258534ad5580SMartin KaFai Lau static int bpf_obj_get_next_id(const union bpf_attr *attr, 258634ad5580SMartin KaFai Lau union bpf_attr __user *uattr, 258734ad5580SMartin KaFai Lau struct idr *idr, 258834ad5580SMartin KaFai Lau spinlock_t *lock) 258934ad5580SMartin KaFai Lau { 259034ad5580SMartin KaFai Lau u32 next_id = attr->start_id; 259134ad5580SMartin KaFai Lau int err = 0; 259234ad5580SMartin KaFai Lau 259334ad5580SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 259434ad5580SMartin KaFai Lau return -EINVAL; 259534ad5580SMartin KaFai Lau 259634ad5580SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 259734ad5580SMartin KaFai Lau return -EPERM; 259834ad5580SMartin KaFai Lau 259934ad5580SMartin KaFai Lau next_id++; 260034ad5580SMartin KaFai Lau spin_lock_bh(lock); 260134ad5580SMartin KaFai Lau if (!idr_get_next(idr, &next_id)) 260234ad5580SMartin KaFai Lau err = -ENOENT; 260334ad5580SMartin KaFai Lau spin_unlock_bh(lock); 260434ad5580SMartin KaFai Lau 260534ad5580SMartin KaFai Lau if (!err) 260634ad5580SMartin KaFai Lau err = put_user(next_id, &uattr->next_id); 260734ad5580SMartin KaFai Lau 260834ad5580SMartin KaFai Lau return err; 260934ad5580SMartin KaFai Lau } 261034ad5580SMartin KaFai Lau 2611b16d9aa4SMartin KaFai Lau #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 2612b16d9aa4SMartin KaFai Lau 26137e6897f9SBjörn Töpel struct bpf_prog *bpf_prog_by_id(u32 id) 26147e6897f9SBjörn Töpel { 26157e6897f9SBjörn Töpel struct bpf_prog *prog; 26167e6897f9SBjörn Töpel 26177e6897f9SBjörn Töpel if (!id) 26187e6897f9SBjörn Töpel return ERR_PTR(-ENOENT); 26197e6897f9SBjörn Töpel 26207e6897f9SBjörn Töpel spin_lock_bh(&prog_idr_lock); 26217e6897f9SBjörn Töpel prog = idr_find(&prog_idr, id); 26227e6897f9SBjörn Töpel if (prog) 26237e6897f9SBjörn Töpel prog = bpf_prog_inc_not_zero(prog); 26247e6897f9SBjörn Töpel else 26257e6897f9SBjörn Töpel prog = ERR_PTR(-ENOENT); 26267e6897f9SBjörn Töpel spin_unlock_bh(&prog_idr_lock); 26277e6897f9SBjörn Töpel return prog; 26287e6897f9SBjörn Töpel } 26297e6897f9SBjörn Töpel 2630b16d9aa4SMartin KaFai Lau static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 2631b16d9aa4SMartin KaFai Lau { 2632b16d9aa4SMartin KaFai Lau struct bpf_prog *prog; 2633b16d9aa4SMartin KaFai Lau u32 id = attr->prog_id; 2634b16d9aa4SMartin KaFai Lau int fd; 2635b16d9aa4SMartin KaFai Lau 2636b16d9aa4SMartin KaFai Lau if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 2637b16d9aa4SMartin KaFai Lau return -EINVAL; 2638b16d9aa4SMartin KaFai Lau 2639b16d9aa4SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 2640b16d9aa4SMartin KaFai Lau return -EPERM; 2641b16d9aa4SMartin KaFai Lau 26427e6897f9SBjörn Töpel prog = bpf_prog_by_id(id); 2643b16d9aa4SMartin KaFai Lau if (IS_ERR(prog)) 2644b16d9aa4SMartin KaFai Lau return PTR_ERR(prog); 2645b16d9aa4SMartin KaFai Lau 2646b16d9aa4SMartin KaFai Lau fd = bpf_prog_new_fd(prog); 2647b16d9aa4SMartin KaFai Lau if (fd < 0) 2648b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 2649b16d9aa4SMartin KaFai Lau 2650b16d9aa4SMartin KaFai Lau return fd; 2651b16d9aa4SMartin KaFai Lau } 2652b16d9aa4SMartin KaFai Lau 26536e71b04aSChenbo Feng #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags 2654bd5f5f4eSMartin KaFai Lau 2655bd5f5f4eSMartin KaFai Lau static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 2656bd5f5f4eSMartin KaFai Lau { 2657bd5f5f4eSMartin KaFai Lau struct bpf_map *map; 2658bd5f5f4eSMartin KaFai Lau u32 id = attr->map_id; 26596e71b04aSChenbo Feng int f_flags; 2660bd5f5f4eSMartin KaFai Lau int fd; 2661bd5f5f4eSMartin KaFai Lau 26626e71b04aSChenbo Feng if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || 26636e71b04aSChenbo Feng attr->open_flags & ~BPF_OBJ_FLAG_MASK) 2664bd5f5f4eSMartin KaFai Lau return -EINVAL; 2665bd5f5f4eSMartin KaFai Lau 2666bd5f5f4eSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 2667bd5f5f4eSMartin KaFai Lau return -EPERM; 2668bd5f5f4eSMartin KaFai Lau 26696e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->open_flags); 26706e71b04aSChenbo Feng if (f_flags < 0) 26716e71b04aSChenbo Feng return f_flags; 26726e71b04aSChenbo Feng 2673bd5f5f4eSMartin KaFai Lau spin_lock_bh(&map_idr_lock); 2674bd5f5f4eSMartin KaFai Lau map = idr_find(&map_idr, id); 2675bd5f5f4eSMartin KaFai Lau if (map) 2676b0e4701cSStanislav Fomichev map = __bpf_map_inc_not_zero(map, true); 2677bd5f5f4eSMartin KaFai Lau else 2678bd5f5f4eSMartin KaFai Lau map = ERR_PTR(-ENOENT); 2679bd5f5f4eSMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 2680bd5f5f4eSMartin KaFai Lau 2681bd5f5f4eSMartin KaFai Lau if (IS_ERR(map)) 2682bd5f5f4eSMartin KaFai Lau return PTR_ERR(map); 2683bd5f5f4eSMartin KaFai Lau 26846e71b04aSChenbo Feng fd = bpf_map_new_fd(map, f_flags); 2685bd5f5f4eSMartin KaFai Lau if (fd < 0) 2686781e6282SPeng Sun bpf_map_put_with_uref(map); 2687bd5f5f4eSMartin KaFai Lau 2688bd5f5f4eSMartin KaFai Lau return fd; 2689bd5f5f4eSMartin KaFai Lau } 2690bd5f5f4eSMartin KaFai Lau 26917105e828SDaniel Borkmann static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, 2692d8eca5bbSDaniel Borkmann unsigned long addr, u32 *off, 2693d8eca5bbSDaniel Borkmann u32 *type) 26947105e828SDaniel Borkmann { 2695d8eca5bbSDaniel Borkmann const struct bpf_map *map; 26967105e828SDaniel Borkmann int i; 26977105e828SDaniel Borkmann 2698d8eca5bbSDaniel Borkmann for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { 2699d8eca5bbSDaniel Borkmann map = prog->aux->used_maps[i]; 2700d8eca5bbSDaniel Borkmann if (map == (void *)addr) { 2701d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_FD; 2702d8eca5bbSDaniel Borkmann return map; 2703d8eca5bbSDaniel Borkmann } 2704d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta) 2705d8eca5bbSDaniel Borkmann continue; 2706d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta(map, addr, off)) { 2707d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_VALUE; 2708d8eca5bbSDaniel Borkmann return map; 2709d8eca5bbSDaniel Borkmann } 2710d8eca5bbSDaniel Borkmann } 2711d8eca5bbSDaniel Borkmann 27127105e828SDaniel Borkmann return NULL; 27137105e828SDaniel Borkmann } 27147105e828SDaniel Borkmann 27157105e828SDaniel Borkmann static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog) 27167105e828SDaniel Borkmann { 27177105e828SDaniel Borkmann const struct bpf_map *map; 27187105e828SDaniel Borkmann struct bpf_insn *insns; 2719d8eca5bbSDaniel Borkmann u32 off, type; 27207105e828SDaniel Borkmann u64 imm; 27217105e828SDaniel Borkmann int i; 27227105e828SDaniel Borkmann 27237105e828SDaniel Borkmann insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), 27247105e828SDaniel Borkmann GFP_USER); 27257105e828SDaniel Borkmann if (!insns) 27267105e828SDaniel Borkmann return insns; 27277105e828SDaniel Borkmann 27287105e828SDaniel Borkmann for (i = 0; i < prog->len; i++) { 27297105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_TAIL_CALL)) { 27307105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 27317105e828SDaniel Borkmann insns[i].imm = BPF_FUNC_tail_call; 27327105e828SDaniel Borkmann /* fall-through */ 27337105e828SDaniel Borkmann } 27347105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_CALL) || 27357105e828SDaniel Borkmann insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) { 27367105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) 27377105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 27387105e828SDaniel Borkmann if (!bpf_dump_raw_ok()) 27397105e828SDaniel Borkmann insns[i].imm = 0; 27407105e828SDaniel Borkmann continue; 27417105e828SDaniel Borkmann } 27427105e828SDaniel Borkmann 27437105e828SDaniel Borkmann if (insns[i].code != (BPF_LD | BPF_IMM | BPF_DW)) 27447105e828SDaniel Borkmann continue; 27457105e828SDaniel Borkmann 27467105e828SDaniel Borkmann imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; 2747d8eca5bbSDaniel Borkmann map = bpf_map_from_imm(prog, imm, &off, &type); 27487105e828SDaniel Borkmann if (map) { 2749d8eca5bbSDaniel Borkmann insns[i].src_reg = type; 27507105e828SDaniel Borkmann insns[i].imm = map->id; 2751d8eca5bbSDaniel Borkmann insns[i + 1].imm = off; 27527105e828SDaniel Borkmann continue; 27537105e828SDaniel Borkmann } 27547105e828SDaniel Borkmann } 27557105e828SDaniel Borkmann 27567105e828SDaniel Borkmann return insns; 27577105e828SDaniel Borkmann } 27587105e828SDaniel Borkmann 2759c454a46bSMartin KaFai Lau static int set_info_rec_size(struct bpf_prog_info *info) 2760c454a46bSMartin KaFai Lau { 2761c454a46bSMartin KaFai Lau /* 2762c454a46bSMartin KaFai Lau * Ensure info.*_rec_size is the same as kernel expected size 2763c454a46bSMartin KaFai Lau * 2764c454a46bSMartin KaFai Lau * or 2765c454a46bSMartin KaFai Lau * 2766c454a46bSMartin KaFai Lau * Only allow zero *_rec_size if both _rec_size and _cnt are 2767c454a46bSMartin KaFai Lau * zero. In this case, the kernel will set the expected 2768c454a46bSMartin KaFai Lau * _rec_size back to the info. 2769c454a46bSMartin KaFai Lau */ 2770c454a46bSMartin KaFai Lau 277111d8b82dSYonghong Song if ((info->nr_func_info || info->func_info_rec_size) && 2772c454a46bSMartin KaFai Lau info->func_info_rec_size != sizeof(struct bpf_func_info)) 2773c454a46bSMartin KaFai Lau return -EINVAL; 2774c454a46bSMartin KaFai Lau 277511d8b82dSYonghong Song if ((info->nr_line_info || info->line_info_rec_size) && 2776c454a46bSMartin KaFai Lau info->line_info_rec_size != sizeof(struct bpf_line_info)) 2777c454a46bSMartin KaFai Lau return -EINVAL; 2778c454a46bSMartin KaFai Lau 277911d8b82dSYonghong Song if ((info->nr_jited_line_info || info->jited_line_info_rec_size) && 2780c454a46bSMartin KaFai Lau info->jited_line_info_rec_size != sizeof(__u64)) 2781c454a46bSMartin KaFai Lau return -EINVAL; 2782c454a46bSMartin KaFai Lau 2783c454a46bSMartin KaFai Lau info->func_info_rec_size = sizeof(struct bpf_func_info); 2784c454a46bSMartin KaFai Lau info->line_info_rec_size = sizeof(struct bpf_line_info); 2785c454a46bSMartin KaFai Lau info->jited_line_info_rec_size = sizeof(__u64); 2786c454a46bSMartin KaFai Lau 2787c454a46bSMartin KaFai Lau return 0; 2788c454a46bSMartin KaFai Lau } 2789c454a46bSMartin KaFai Lau 27901e270976SMartin KaFai Lau static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, 27911e270976SMartin KaFai Lau const union bpf_attr *attr, 27921e270976SMartin KaFai Lau union bpf_attr __user *uattr) 27931e270976SMartin KaFai Lau { 27941e270976SMartin KaFai Lau struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 27951e270976SMartin KaFai Lau struct bpf_prog_info info = {}; 27961e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 27975f8f8b93SAlexei Starovoitov struct bpf_prog_stats stats; 27981e270976SMartin KaFai Lau char __user *uinsns; 27991e270976SMartin KaFai Lau u32 ulen; 28001e270976SMartin KaFai Lau int err; 28011e270976SMartin KaFai Lau 2802dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 28031e270976SMartin KaFai Lau if (err) 28041e270976SMartin KaFai Lau return err; 28051e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 28061e270976SMartin KaFai Lau 28071e270976SMartin KaFai Lau if (copy_from_user(&info, uinfo, info_len)) 280889b09689SDaniel Borkmann return -EFAULT; 28091e270976SMartin KaFai Lau 28101e270976SMartin KaFai Lau info.type = prog->type; 28111e270976SMartin KaFai Lau info.id = prog->aux->id; 2812cb4d2b3fSMartin KaFai Lau info.load_time = prog->aux->load_time; 2813cb4d2b3fSMartin KaFai Lau info.created_by_uid = from_kuid_munged(current_user_ns(), 2814cb4d2b3fSMartin KaFai Lau prog->aux->user->uid); 2815b85fab0eSJiri Olsa info.gpl_compatible = prog->gpl_compatible; 28161e270976SMartin KaFai Lau 28171e270976SMartin KaFai Lau memcpy(info.tag, prog->tag, sizeof(prog->tag)); 2818cb4d2b3fSMartin KaFai Lau memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); 2819cb4d2b3fSMartin KaFai Lau 2820cb4d2b3fSMartin KaFai Lau ulen = info.nr_map_ids; 2821cb4d2b3fSMartin KaFai Lau info.nr_map_ids = prog->aux->used_map_cnt; 2822cb4d2b3fSMartin KaFai Lau ulen = min_t(u32, info.nr_map_ids, ulen); 2823cb4d2b3fSMartin KaFai Lau if (ulen) { 2824721e08daSMartin KaFai Lau u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); 2825cb4d2b3fSMartin KaFai Lau u32 i; 2826cb4d2b3fSMartin KaFai Lau 2827cb4d2b3fSMartin KaFai Lau for (i = 0; i < ulen; i++) 2828cb4d2b3fSMartin KaFai Lau if (put_user(prog->aux->used_maps[i]->id, 2829cb4d2b3fSMartin KaFai Lau &user_map_ids[i])) 2830cb4d2b3fSMartin KaFai Lau return -EFAULT; 2831cb4d2b3fSMartin KaFai Lau } 28321e270976SMartin KaFai Lau 2833c454a46bSMartin KaFai Lau err = set_info_rec_size(&info); 2834c454a46bSMartin KaFai Lau if (err) 2835c454a46bSMartin KaFai Lau return err; 28367337224fSMartin KaFai Lau 28375f8f8b93SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 28385f8f8b93SAlexei Starovoitov info.run_time_ns = stats.nsecs; 28395f8f8b93SAlexei Starovoitov info.run_cnt = stats.cnt; 28405f8f8b93SAlexei Starovoitov 28411e270976SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) { 28421e270976SMartin KaFai Lau info.jited_prog_len = 0; 28431e270976SMartin KaFai Lau info.xlated_prog_len = 0; 2844dbecd738SSandipan Das info.nr_jited_ksyms = 0; 284528c2fae7SDaniel Borkmann info.nr_jited_func_lens = 0; 284611d8b82dSYonghong Song info.nr_func_info = 0; 284711d8b82dSYonghong Song info.nr_line_info = 0; 284811d8b82dSYonghong Song info.nr_jited_line_info = 0; 28491e270976SMartin KaFai Lau goto done; 28501e270976SMartin KaFai Lau } 28511e270976SMartin KaFai Lau 28521e270976SMartin KaFai Lau ulen = info.xlated_prog_len; 28539975a54bSDaniel Borkmann info.xlated_prog_len = bpf_prog_insn_size(prog); 28541e270976SMartin KaFai Lau if (info.xlated_prog_len && ulen) { 28557105e828SDaniel Borkmann struct bpf_insn *insns_sanitized; 28567105e828SDaniel Borkmann bool fault; 28577105e828SDaniel Borkmann 28587105e828SDaniel Borkmann if (prog->blinded && !bpf_dump_raw_ok()) { 28597105e828SDaniel Borkmann info.xlated_prog_insns = 0; 28607105e828SDaniel Borkmann goto done; 28617105e828SDaniel Borkmann } 28627105e828SDaniel Borkmann insns_sanitized = bpf_insn_prepare_dump(prog); 28637105e828SDaniel Borkmann if (!insns_sanitized) 28647105e828SDaniel Borkmann return -ENOMEM; 28651e270976SMartin KaFai Lau uinsns = u64_to_user_ptr(info.xlated_prog_insns); 28661e270976SMartin KaFai Lau ulen = min_t(u32, info.xlated_prog_len, ulen); 28677105e828SDaniel Borkmann fault = copy_to_user(uinsns, insns_sanitized, ulen); 28687105e828SDaniel Borkmann kfree(insns_sanitized); 28697105e828SDaniel Borkmann if (fault) 28701e270976SMartin KaFai Lau return -EFAULT; 28711e270976SMartin KaFai Lau } 28721e270976SMartin KaFai Lau 2873675fc275SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 2874675fc275SJakub Kicinski err = bpf_prog_offload_info_fill(&info, prog); 2875675fc275SJakub Kicinski if (err) 2876675fc275SJakub Kicinski return err; 2877fcfb126dSJiong Wang goto done; 2878fcfb126dSJiong Wang } 2879fcfb126dSJiong Wang 2880fcfb126dSJiong Wang /* NOTE: the following code is supposed to be skipped for offload. 2881fcfb126dSJiong Wang * bpf_prog_offload_info_fill() is the place to fill similar fields 2882fcfb126dSJiong Wang * for offload. 2883fcfb126dSJiong Wang */ 2884fcfb126dSJiong Wang ulen = info.jited_prog_len; 28854d56a76eSSandipan Das if (prog->aux->func_cnt) { 28864d56a76eSSandipan Das u32 i; 28874d56a76eSSandipan Das 28884d56a76eSSandipan Das info.jited_prog_len = 0; 28894d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) 28904d56a76eSSandipan Das info.jited_prog_len += prog->aux->func[i]->jited_len; 28914d56a76eSSandipan Das } else { 2892fcfb126dSJiong Wang info.jited_prog_len = prog->jited_len; 28934d56a76eSSandipan Das } 28944d56a76eSSandipan Das 2895fcfb126dSJiong Wang if (info.jited_prog_len && ulen) { 2896fcfb126dSJiong Wang if (bpf_dump_raw_ok()) { 2897fcfb126dSJiong Wang uinsns = u64_to_user_ptr(info.jited_prog_insns); 2898fcfb126dSJiong Wang ulen = min_t(u32, info.jited_prog_len, ulen); 28994d56a76eSSandipan Das 29004d56a76eSSandipan Das /* for multi-function programs, copy the JITed 29014d56a76eSSandipan Das * instructions for all the functions 29024d56a76eSSandipan Das */ 29034d56a76eSSandipan Das if (prog->aux->func_cnt) { 29044d56a76eSSandipan Das u32 len, free, i; 29054d56a76eSSandipan Das u8 *img; 29064d56a76eSSandipan Das 29074d56a76eSSandipan Das free = ulen; 29084d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) { 29094d56a76eSSandipan Das len = prog->aux->func[i]->jited_len; 29104d56a76eSSandipan Das len = min_t(u32, len, free); 29114d56a76eSSandipan Das img = (u8 *) prog->aux->func[i]->bpf_func; 29124d56a76eSSandipan Das if (copy_to_user(uinsns, img, len)) 29134d56a76eSSandipan Das return -EFAULT; 29144d56a76eSSandipan Das uinsns += len; 29154d56a76eSSandipan Das free -= len; 29164d56a76eSSandipan Das if (!free) 29174d56a76eSSandipan Das break; 29184d56a76eSSandipan Das } 29194d56a76eSSandipan Das } else { 2920fcfb126dSJiong Wang if (copy_to_user(uinsns, prog->bpf_func, ulen)) 2921fcfb126dSJiong Wang return -EFAULT; 29224d56a76eSSandipan Das } 2923fcfb126dSJiong Wang } else { 2924fcfb126dSJiong Wang info.jited_prog_insns = 0; 2925fcfb126dSJiong Wang } 2926675fc275SJakub Kicinski } 2927675fc275SJakub Kicinski 2928dbecd738SSandipan Das ulen = info.nr_jited_ksyms; 2929ff1889fcSSong Liu info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; 29307a5725ddSSong Liu if (ulen) { 2931dbecd738SSandipan Das if (bpf_dump_raw_ok()) { 2932ff1889fcSSong Liu unsigned long ksym_addr; 2933dbecd738SSandipan Das u64 __user *user_ksyms; 2934dbecd738SSandipan Das u32 i; 2935dbecd738SSandipan Das 2936dbecd738SSandipan Das /* copy the address of the kernel symbol 2937dbecd738SSandipan Das * corresponding to each function 2938dbecd738SSandipan Das */ 2939dbecd738SSandipan Das ulen = min_t(u32, info.nr_jited_ksyms, ulen); 2940dbecd738SSandipan Das user_ksyms = u64_to_user_ptr(info.jited_ksyms); 2941ff1889fcSSong Liu if (prog->aux->func_cnt) { 2942dbecd738SSandipan Das for (i = 0; i < ulen; i++) { 2943ff1889fcSSong Liu ksym_addr = (unsigned long) 2944ff1889fcSSong Liu prog->aux->func[i]->bpf_func; 2945ff1889fcSSong Liu if (put_user((u64) ksym_addr, 2946ff1889fcSSong Liu &user_ksyms[i])) 2947ff1889fcSSong Liu return -EFAULT; 2948ff1889fcSSong Liu } 2949ff1889fcSSong Liu } else { 2950ff1889fcSSong Liu ksym_addr = (unsigned long) prog->bpf_func; 2951ff1889fcSSong Liu if (put_user((u64) ksym_addr, &user_ksyms[0])) 2952dbecd738SSandipan Das return -EFAULT; 2953dbecd738SSandipan Das } 2954dbecd738SSandipan Das } else { 2955dbecd738SSandipan Das info.jited_ksyms = 0; 2956dbecd738SSandipan Das } 2957dbecd738SSandipan Das } 2958dbecd738SSandipan Das 2959815581c1SSandipan Das ulen = info.nr_jited_func_lens; 2960ff1889fcSSong Liu info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; 29617a5725ddSSong Liu if (ulen) { 2962815581c1SSandipan Das if (bpf_dump_raw_ok()) { 2963815581c1SSandipan Das u32 __user *user_lens; 2964815581c1SSandipan Das u32 func_len, i; 2965815581c1SSandipan Das 2966815581c1SSandipan Das /* copy the JITed image lengths for each function */ 2967815581c1SSandipan Das ulen = min_t(u32, info.nr_jited_func_lens, ulen); 2968815581c1SSandipan Das user_lens = u64_to_user_ptr(info.jited_func_lens); 2969ff1889fcSSong Liu if (prog->aux->func_cnt) { 2970815581c1SSandipan Das for (i = 0; i < ulen; i++) { 2971ff1889fcSSong Liu func_len = 2972ff1889fcSSong Liu prog->aux->func[i]->jited_len; 2973815581c1SSandipan Das if (put_user(func_len, &user_lens[i])) 2974815581c1SSandipan Das return -EFAULT; 2975815581c1SSandipan Das } 2976815581c1SSandipan Das } else { 2977ff1889fcSSong Liu func_len = prog->jited_len; 2978ff1889fcSSong Liu if (put_user(func_len, &user_lens[0])) 2979ff1889fcSSong Liu return -EFAULT; 2980ff1889fcSSong Liu } 2981ff1889fcSSong Liu } else { 2982815581c1SSandipan Das info.jited_func_lens = 0; 2983815581c1SSandipan Das } 2984815581c1SSandipan Das } 2985815581c1SSandipan Das 29867337224fSMartin KaFai Lau if (prog->aux->btf) 2987838e9690SYonghong Song info.btf_id = btf_id(prog->aux->btf); 2988838e9690SYonghong Song 298911d8b82dSYonghong Song ulen = info.nr_func_info; 299011d8b82dSYonghong Song info.nr_func_info = prog->aux->func_info_cnt; 299111d8b82dSYonghong Song if (info.nr_func_info && ulen) { 2992838e9690SYonghong Song char __user *user_finfo; 2993838e9690SYonghong Song 2994838e9690SYonghong Song user_finfo = u64_to_user_ptr(info.func_info); 299511d8b82dSYonghong Song ulen = min_t(u32, info.nr_func_info, ulen); 2996ba64e7d8SYonghong Song if (copy_to_user(user_finfo, prog->aux->func_info, 29977337224fSMartin KaFai Lau info.func_info_rec_size * ulen)) 2998838e9690SYonghong Song return -EFAULT; 2999838e9690SYonghong Song } 3000838e9690SYonghong Song 300111d8b82dSYonghong Song ulen = info.nr_line_info; 300211d8b82dSYonghong Song info.nr_line_info = prog->aux->nr_linfo; 300311d8b82dSYonghong Song if (info.nr_line_info && ulen) { 3004c454a46bSMartin KaFai Lau __u8 __user *user_linfo; 3005c454a46bSMartin KaFai Lau 3006c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.line_info); 300711d8b82dSYonghong Song ulen = min_t(u32, info.nr_line_info, ulen); 3008c454a46bSMartin KaFai Lau if (copy_to_user(user_linfo, prog->aux->linfo, 3009c454a46bSMartin KaFai Lau info.line_info_rec_size * ulen)) 3010c454a46bSMartin KaFai Lau return -EFAULT; 3011c454a46bSMartin KaFai Lau } 3012c454a46bSMartin KaFai Lau 301311d8b82dSYonghong Song ulen = info.nr_jited_line_info; 3014c454a46bSMartin KaFai Lau if (prog->aux->jited_linfo) 301511d8b82dSYonghong Song info.nr_jited_line_info = prog->aux->nr_linfo; 3016c454a46bSMartin KaFai Lau else 301711d8b82dSYonghong Song info.nr_jited_line_info = 0; 301811d8b82dSYonghong Song if (info.nr_jited_line_info && ulen) { 3019c454a46bSMartin KaFai Lau if (bpf_dump_raw_ok()) { 3020c454a46bSMartin KaFai Lau __u64 __user *user_linfo; 3021c454a46bSMartin KaFai Lau u32 i; 3022c454a46bSMartin KaFai Lau 3023c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.jited_line_info); 302411d8b82dSYonghong Song ulen = min_t(u32, info.nr_jited_line_info, ulen); 3025c454a46bSMartin KaFai Lau for (i = 0; i < ulen; i++) { 3026c454a46bSMartin KaFai Lau if (put_user((__u64)(long)prog->aux->jited_linfo[i], 3027c454a46bSMartin KaFai Lau &user_linfo[i])) 3028c454a46bSMartin KaFai Lau return -EFAULT; 3029c454a46bSMartin KaFai Lau } 3030c454a46bSMartin KaFai Lau } else { 3031c454a46bSMartin KaFai Lau info.jited_line_info = 0; 3032c454a46bSMartin KaFai Lau } 3033c454a46bSMartin KaFai Lau } 3034c454a46bSMartin KaFai Lau 3035c872bdb3SSong Liu ulen = info.nr_prog_tags; 3036c872bdb3SSong Liu info.nr_prog_tags = prog->aux->func_cnt ? : 1; 3037c872bdb3SSong Liu if (ulen) { 3038c872bdb3SSong Liu __u8 __user (*user_prog_tags)[BPF_TAG_SIZE]; 3039c872bdb3SSong Liu u32 i; 3040c872bdb3SSong Liu 3041c872bdb3SSong Liu user_prog_tags = u64_to_user_ptr(info.prog_tags); 3042c872bdb3SSong Liu ulen = min_t(u32, info.nr_prog_tags, ulen); 3043c872bdb3SSong Liu if (prog->aux->func_cnt) { 3044c872bdb3SSong Liu for (i = 0; i < ulen; i++) { 3045c872bdb3SSong Liu if (copy_to_user(user_prog_tags[i], 3046c872bdb3SSong Liu prog->aux->func[i]->tag, 3047c872bdb3SSong Liu BPF_TAG_SIZE)) 3048c872bdb3SSong Liu return -EFAULT; 3049c872bdb3SSong Liu } 3050c872bdb3SSong Liu } else { 3051c872bdb3SSong Liu if (copy_to_user(user_prog_tags[0], 3052c872bdb3SSong Liu prog->tag, BPF_TAG_SIZE)) 3053c872bdb3SSong Liu return -EFAULT; 3054c872bdb3SSong Liu } 3055c872bdb3SSong Liu } 3056c872bdb3SSong Liu 30571e270976SMartin KaFai Lau done: 30581e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 30591e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 30601e270976SMartin KaFai Lau return -EFAULT; 30611e270976SMartin KaFai Lau 30621e270976SMartin KaFai Lau return 0; 30631e270976SMartin KaFai Lau } 30641e270976SMartin KaFai Lau 30651e270976SMartin KaFai Lau static int bpf_map_get_info_by_fd(struct bpf_map *map, 30661e270976SMartin KaFai Lau const union bpf_attr *attr, 30671e270976SMartin KaFai Lau union bpf_attr __user *uattr) 30681e270976SMartin KaFai Lau { 30691e270976SMartin KaFai Lau struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 30701e270976SMartin KaFai Lau struct bpf_map_info info = {}; 30711e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 30721e270976SMartin KaFai Lau int err; 30731e270976SMartin KaFai Lau 3074dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 30751e270976SMartin KaFai Lau if (err) 30761e270976SMartin KaFai Lau return err; 30771e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 30781e270976SMartin KaFai Lau 30791e270976SMartin KaFai Lau info.type = map->map_type; 30801e270976SMartin KaFai Lau info.id = map->id; 30811e270976SMartin KaFai Lau info.key_size = map->key_size; 30821e270976SMartin KaFai Lau info.value_size = map->value_size; 30831e270976SMartin KaFai Lau info.max_entries = map->max_entries; 30841e270976SMartin KaFai Lau info.map_flags = map->map_flags; 3085ad5b177bSMartin KaFai Lau memcpy(info.name, map->name, sizeof(map->name)); 30861e270976SMartin KaFai Lau 308778958fcaSMartin KaFai Lau if (map->btf) { 308878958fcaSMartin KaFai Lau info.btf_id = btf_id(map->btf); 30899b2cf328SMartin KaFai Lau info.btf_key_type_id = map->btf_key_type_id; 30909b2cf328SMartin KaFai Lau info.btf_value_type_id = map->btf_value_type_id; 309178958fcaSMartin KaFai Lau } 309285d33df3SMartin KaFai Lau info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; 309378958fcaSMartin KaFai Lau 309452775b33SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 309552775b33SJakub Kicinski err = bpf_map_offload_info_fill(&info, map); 309652775b33SJakub Kicinski if (err) 309752775b33SJakub Kicinski return err; 309852775b33SJakub Kicinski } 309952775b33SJakub Kicinski 31001e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 31011e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 31021e270976SMartin KaFai Lau return -EFAULT; 31031e270976SMartin KaFai Lau 31041e270976SMartin KaFai Lau return 0; 31051e270976SMartin KaFai Lau } 31061e270976SMartin KaFai Lau 310762dab84cSMartin KaFai Lau static int bpf_btf_get_info_by_fd(struct btf *btf, 310862dab84cSMartin KaFai Lau const union bpf_attr *attr, 310962dab84cSMartin KaFai Lau union bpf_attr __user *uattr) 311062dab84cSMartin KaFai Lau { 311162dab84cSMartin KaFai Lau struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info); 311262dab84cSMartin KaFai Lau u32 info_len = attr->info.info_len; 311362dab84cSMartin KaFai Lau int err; 311462dab84cSMartin KaFai Lau 3115dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(*uinfo), info_len); 311662dab84cSMartin KaFai Lau if (err) 311762dab84cSMartin KaFai Lau return err; 311862dab84cSMartin KaFai Lau 311962dab84cSMartin KaFai Lau return btf_get_info_by_fd(btf, attr, uattr); 312062dab84cSMartin KaFai Lau } 312162dab84cSMartin KaFai Lau 31221e270976SMartin KaFai Lau #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 31231e270976SMartin KaFai Lau 31241e270976SMartin KaFai Lau static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 31251e270976SMartin KaFai Lau union bpf_attr __user *uattr) 31261e270976SMartin KaFai Lau { 31271e270976SMartin KaFai Lau int ufd = attr->info.bpf_fd; 31281e270976SMartin KaFai Lau struct fd f; 31291e270976SMartin KaFai Lau int err; 31301e270976SMartin KaFai Lau 31311e270976SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 31321e270976SMartin KaFai Lau return -EINVAL; 31331e270976SMartin KaFai Lau 31341e270976SMartin KaFai Lau f = fdget(ufd); 31351e270976SMartin KaFai Lau if (!f.file) 31361e270976SMartin KaFai Lau return -EBADFD; 31371e270976SMartin KaFai Lau 31381e270976SMartin KaFai Lau if (f.file->f_op == &bpf_prog_fops) 31391e270976SMartin KaFai Lau err = bpf_prog_get_info_by_fd(f.file->private_data, attr, 31401e270976SMartin KaFai Lau uattr); 31411e270976SMartin KaFai Lau else if (f.file->f_op == &bpf_map_fops) 31421e270976SMartin KaFai Lau err = bpf_map_get_info_by_fd(f.file->private_data, attr, 31431e270976SMartin KaFai Lau uattr); 314460197cfbSMartin KaFai Lau else if (f.file->f_op == &btf_fops) 314562dab84cSMartin KaFai Lau err = bpf_btf_get_info_by_fd(f.file->private_data, attr, uattr); 31461e270976SMartin KaFai Lau else 31471e270976SMartin KaFai Lau err = -EINVAL; 31481e270976SMartin KaFai Lau 31491e270976SMartin KaFai Lau fdput(f); 31501e270976SMartin KaFai Lau return err; 31511e270976SMartin KaFai Lau } 31521e270976SMartin KaFai Lau 3153f56a653cSMartin KaFai Lau #define BPF_BTF_LOAD_LAST_FIELD btf_log_level 3154f56a653cSMartin KaFai Lau 3155f56a653cSMartin KaFai Lau static int bpf_btf_load(const union bpf_attr *attr) 3156f56a653cSMartin KaFai Lau { 3157f56a653cSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_LOAD)) 3158f56a653cSMartin KaFai Lau return -EINVAL; 3159f56a653cSMartin KaFai Lau 3160f56a653cSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 3161f56a653cSMartin KaFai Lau return -EPERM; 3162f56a653cSMartin KaFai Lau 3163f56a653cSMartin KaFai Lau return btf_new_fd(attr); 3164f56a653cSMartin KaFai Lau } 3165f56a653cSMartin KaFai Lau 316678958fcaSMartin KaFai Lau #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id 316778958fcaSMartin KaFai Lau 316878958fcaSMartin KaFai Lau static int bpf_btf_get_fd_by_id(const union bpf_attr *attr) 316978958fcaSMartin KaFai Lau { 317078958fcaSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID)) 317178958fcaSMartin KaFai Lau return -EINVAL; 317278958fcaSMartin KaFai Lau 317378958fcaSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 317478958fcaSMartin KaFai Lau return -EPERM; 317578958fcaSMartin KaFai Lau 317678958fcaSMartin KaFai Lau return btf_get_fd_by_id(attr->btf_id); 317778958fcaSMartin KaFai Lau } 317878958fcaSMartin KaFai Lau 317941bdc4b4SYonghong Song static int bpf_task_fd_query_copy(const union bpf_attr *attr, 318041bdc4b4SYonghong Song union bpf_attr __user *uattr, 318141bdc4b4SYonghong Song u32 prog_id, u32 fd_type, 318241bdc4b4SYonghong Song const char *buf, u64 probe_offset, 318341bdc4b4SYonghong Song u64 probe_addr) 318441bdc4b4SYonghong Song { 318541bdc4b4SYonghong Song char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf); 318641bdc4b4SYonghong Song u32 len = buf ? strlen(buf) : 0, input_len; 318741bdc4b4SYonghong Song int err = 0; 318841bdc4b4SYonghong Song 318941bdc4b4SYonghong Song if (put_user(len, &uattr->task_fd_query.buf_len)) 319041bdc4b4SYonghong Song return -EFAULT; 319141bdc4b4SYonghong Song input_len = attr->task_fd_query.buf_len; 319241bdc4b4SYonghong Song if (input_len && ubuf) { 319341bdc4b4SYonghong Song if (!len) { 319441bdc4b4SYonghong Song /* nothing to copy, just make ubuf NULL terminated */ 319541bdc4b4SYonghong Song char zero = '\0'; 319641bdc4b4SYonghong Song 319741bdc4b4SYonghong Song if (put_user(zero, ubuf)) 319841bdc4b4SYonghong Song return -EFAULT; 319941bdc4b4SYonghong Song } else if (input_len >= len + 1) { 320041bdc4b4SYonghong Song /* ubuf can hold the string with NULL terminator */ 320141bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, len + 1)) 320241bdc4b4SYonghong Song return -EFAULT; 320341bdc4b4SYonghong Song } else { 320441bdc4b4SYonghong Song /* ubuf cannot hold the string with NULL terminator, 320541bdc4b4SYonghong Song * do a partial copy with NULL terminator. 320641bdc4b4SYonghong Song */ 320741bdc4b4SYonghong Song char zero = '\0'; 320841bdc4b4SYonghong Song 320941bdc4b4SYonghong Song err = -ENOSPC; 321041bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, input_len - 1)) 321141bdc4b4SYonghong Song return -EFAULT; 321241bdc4b4SYonghong Song if (put_user(zero, ubuf + input_len - 1)) 321341bdc4b4SYonghong Song return -EFAULT; 321441bdc4b4SYonghong Song } 321541bdc4b4SYonghong Song } 321641bdc4b4SYonghong Song 321741bdc4b4SYonghong Song if (put_user(prog_id, &uattr->task_fd_query.prog_id) || 321841bdc4b4SYonghong Song put_user(fd_type, &uattr->task_fd_query.fd_type) || 321941bdc4b4SYonghong Song put_user(probe_offset, &uattr->task_fd_query.probe_offset) || 322041bdc4b4SYonghong Song put_user(probe_addr, &uattr->task_fd_query.probe_addr)) 322141bdc4b4SYonghong Song return -EFAULT; 322241bdc4b4SYonghong Song 322341bdc4b4SYonghong Song return err; 322441bdc4b4SYonghong Song } 322541bdc4b4SYonghong Song 322641bdc4b4SYonghong Song #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr 322741bdc4b4SYonghong Song 322841bdc4b4SYonghong Song static int bpf_task_fd_query(const union bpf_attr *attr, 322941bdc4b4SYonghong Song union bpf_attr __user *uattr) 323041bdc4b4SYonghong Song { 323141bdc4b4SYonghong Song pid_t pid = attr->task_fd_query.pid; 323241bdc4b4SYonghong Song u32 fd = attr->task_fd_query.fd; 323341bdc4b4SYonghong Song const struct perf_event *event; 323441bdc4b4SYonghong Song struct files_struct *files; 323541bdc4b4SYonghong Song struct task_struct *task; 323641bdc4b4SYonghong Song struct file *file; 323741bdc4b4SYonghong Song int err; 323841bdc4b4SYonghong Song 323941bdc4b4SYonghong Song if (CHECK_ATTR(BPF_TASK_FD_QUERY)) 324041bdc4b4SYonghong Song return -EINVAL; 324141bdc4b4SYonghong Song 324241bdc4b4SYonghong Song if (!capable(CAP_SYS_ADMIN)) 324341bdc4b4SYonghong Song return -EPERM; 324441bdc4b4SYonghong Song 324541bdc4b4SYonghong Song if (attr->task_fd_query.flags != 0) 324641bdc4b4SYonghong Song return -EINVAL; 324741bdc4b4SYonghong Song 324841bdc4b4SYonghong Song task = get_pid_task(find_vpid(pid), PIDTYPE_PID); 324941bdc4b4SYonghong Song if (!task) 325041bdc4b4SYonghong Song return -ENOENT; 325141bdc4b4SYonghong Song 325241bdc4b4SYonghong Song files = get_files_struct(task); 325341bdc4b4SYonghong Song put_task_struct(task); 325441bdc4b4SYonghong Song if (!files) 325541bdc4b4SYonghong Song return -ENOENT; 325641bdc4b4SYonghong Song 325741bdc4b4SYonghong Song err = 0; 325841bdc4b4SYonghong Song spin_lock(&files->file_lock); 325941bdc4b4SYonghong Song file = fcheck_files(files, fd); 326041bdc4b4SYonghong Song if (!file) 326141bdc4b4SYonghong Song err = -EBADF; 326241bdc4b4SYonghong Song else 326341bdc4b4SYonghong Song get_file(file); 326441bdc4b4SYonghong Song spin_unlock(&files->file_lock); 326541bdc4b4SYonghong Song put_files_struct(files); 326641bdc4b4SYonghong Song 326741bdc4b4SYonghong Song if (err) 326841bdc4b4SYonghong Song goto out; 326941bdc4b4SYonghong Song 327041bdc4b4SYonghong Song if (file->f_op == &bpf_raw_tp_fops) { 327141bdc4b4SYonghong Song struct bpf_raw_tracepoint *raw_tp = file->private_data; 327241bdc4b4SYonghong Song struct bpf_raw_event_map *btp = raw_tp->btp; 327341bdc4b4SYonghong Song 327441bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, 327541bdc4b4SYonghong Song raw_tp->prog->aux->id, 327641bdc4b4SYonghong Song BPF_FD_TYPE_RAW_TRACEPOINT, 327741bdc4b4SYonghong Song btp->tp->name, 0, 0); 327841bdc4b4SYonghong Song goto put_file; 327941bdc4b4SYonghong Song } 328041bdc4b4SYonghong Song 328141bdc4b4SYonghong Song event = perf_get_event(file); 328241bdc4b4SYonghong Song if (!IS_ERR(event)) { 328341bdc4b4SYonghong Song u64 probe_offset, probe_addr; 328441bdc4b4SYonghong Song u32 prog_id, fd_type; 328541bdc4b4SYonghong Song const char *buf; 328641bdc4b4SYonghong Song 328741bdc4b4SYonghong Song err = bpf_get_perf_event_info(event, &prog_id, &fd_type, 328841bdc4b4SYonghong Song &buf, &probe_offset, 328941bdc4b4SYonghong Song &probe_addr); 329041bdc4b4SYonghong Song if (!err) 329141bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, prog_id, 329241bdc4b4SYonghong Song fd_type, buf, 329341bdc4b4SYonghong Song probe_offset, 329441bdc4b4SYonghong Song probe_addr); 329541bdc4b4SYonghong Song goto put_file; 329641bdc4b4SYonghong Song } 329741bdc4b4SYonghong Song 329841bdc4b4SYonghong Song err = -ENOTSUPP; 329941bdc4b4SYonghong Song put_file: 330041bdc4b4SYonghong Song fput(file); 330141bdc4b4SYonghong Song out: 330241bdc4b4SYonghong Song return err; 330341bdc4b4SYonghong Song } 330441bdc4b4SYonghong Song 3305cb4d03abSBrian Vazquez #define BPF_MAP_BATCH_LAST_FIELD batch.flags 3306cb4d03abSBrian Vazquez 3307cb4d03abSBrian Vazquez #define BPF_DO_BATCH(fn) \ 3308cb4d03abSBrian Vazquez do { \ 3309cb4d03abSBrian Vazquez if (!fn) { \ 3310cb4d03abSBrian Vazquez err = -ENOTSUPP; \ 3311cb4d03abSBrian Vazquez goto err_put; \ 3312cb4d03abSBrian Vazquez } \ 3313cb4d03abSBrian Vazquez err = fn(map, attr, uattr); \ 3314cb4d03abSBrian Vazquez } while (0) 3315cb4d03abSBrian Vazquez 3316cb4d03abSBrian Vazquez static int bpf_map_do_batch(const union bpf_attr *attr, 3317cb4d03abSBrian Vazquez union bpf_attr __user *uattr, 3318cb4d03abSBrian Vazquez int cmd) 3319cb4d03abSBrian Vazquez { 3320cb4d03abSBrian Vazquez struct bpf_map *map; 3321cb4d03abSBrian Vazquez int err, ufd; 3322cb4d03abSBrian Vazquez struct fd f; 3323cb4d03abSBrian Vazquez 3324cb4d03abSBrian Vazquez if (CHECK_ATTR(BPF_MAP_BATCH)) 3325cb4d03abSBrian Vazquez return -EINVAL; 3326cb4d03abSBrian Vazquez 3327cb4d03abSBrian Vazquez ufd = attr->batch.map_fd; 3328cb4d03abSBrian Vazquez f = fdget(ufd); 3329cb4d03abSBrian Vazquez map = __bpf_map_get(f); 3330cb4d03abSBrian Vazquez if (IS_ERR(map)) 3331cb4d03abSBrian Vazquez return PTR_ERR(map); 3332cb4d03abSBrian Vazquez 333305799638SYonghong Song if ((cmd == BPF_MAP_LOOKUP_BATCH || 333405799638SYonghong Song cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) && 3335cb4d03abSBrian Vazquez !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 3336cb4d03abSBrian Vazquez err = -EPERM; 3337cb4d03abSBrian Vazquez goto err_put; 3338cb4d03abSBrian Vazquez } 3339cb4d03abSBrian Vazquez 3340cb4d03abSBrian Vazquez if (cmd != BPF_MAP_LOOKUP_BATCH && 3341cb4d03abSBrian Vazquez !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 3342cb4d03abSBrian Vazquez err = -EPERM; 3343cb4d03abSBrian Vazquez goto err_put; 3344cb4d03abSBrian Vazquez } 3345cb4d03abSBrian Vazquez 3346cb4d03abSBrian Vazquez if (cmd == BPF_MAP_LOOKUP_BATCH) 3347cb4d03abSBrian Vazquez BPF_DO_BATCH(map->ops->map_lookup_batch); 334805799638SYonghong Song else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) 334905799638SYonghong Song BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch); 3350aa2e93b8SBrian Vazquez else if (cmd == BPF_MAP_UPDATE_BATCH) 3351aa2e93b8SBrian Vazquez BPF_DO_BATCH(map->ops->map_update_batch); 3352aa2e93b8SBrian Vazquez else 3353aa2e93b8SBrian Vazquez BPF_DO_BATCH(map->ops->map_delete_batch); 3354cb4d03abSBrian Vazquez 3355cb4d03abSBrian Vazquez err_put: 3356cb4d03abSBrian Vazquez fdput(f); 3357cb4d03abSBrian Vazquez return err; 3358cb4d03abSBrian Vazquez } 3359cb4d03abSBrian Vazquez 336099c55f7dSAlexei Starovoitov SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 336199c55f7dSAlexei Starovoitov { 336299c55f7dSAlexei Starovoitov union bpf_attr attr = {}; 336399c55f7dSAlexei Starovoitov int err; 336499c55f7dSAlexei Starovoitov 33650fa4fe85SChenbo Feng if (sysctl_unprivileged_bpf_disabled && !capable(CAP_SYS_ADMIN)) 336699c55f7dSAlexei Starovoitov return -EPERM; 336799c55f7dSAlexei Starovoitov 3368dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size); 336999c55f7dSAlexei Starovoitov if (err) 337099c55f7dSAlexei Starovoitov return err; 33711e270976SMartin KaFai Lau size = min_t(u32, size, sizeof(attr)); 337299c55f7dSAlexei Starovoitov 337399c55f7dSAlexei Starovoitov /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 337499c55f7dSAlexei Starovoitov if (copy_from_user(&attr, uattr, size) != 0) 337599c55f7dSAlexei Starovoitov return -EFAULT; 337699c55f7dSAlexei Starovoitov 3377afdb09c7SChenbo Feng err = security_bpf(cmd, &attr, size); 3378afdb09c7SChenbo Feng if (err < 0) 3379afdb09c7SChenbo Feng return err; 3380afdb09c7SChenbo Feng 338199c55f7dSAlexei Starovoitov switch (cmd) { 338299c55f7dSAlexei Starovoitov case BPF_MAP_CREATE: 338399c55f7dSAlexei Starovoitov err = map_create(&attr); 338499c55f7dSAlexei Starovoitov break; 3385db20fd2bSAlexei Starovoitov case BPF_MAP_LOOKUP_ELEM: 3386db20fd2bSAlexei Starovoitov err = map_lookup_elem(&attr); 3387db20fd2bSAlexei Starovoitov break; 3388db20fd2bSAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 3389db20fd2bSAlexei Starovoitov err = map_update_elem(&attr); 3390db20fd2bSAlexei Starovoitov break; 3391db20fd2bSAlexei Starovoitov case BPF_MAP_DELETE_ELEM: 3392db20fd2bSAlexei Starovoitov err = map_delete_elem(&attr); 3393db20fd2bSAlexei Starovoitov break; 3394db20fd2bSAlexei Starovoitov case BPF_MAP_GET_NEXT_KEY: 3395db20fd2bSAlexei Starovoitov err = map_get_next_key(&attr); 3396db20fd2bSAlexei Starovoitov break; 339787df15deSDaniel Borkmann case BPF_MAP_FREEZE: 339887df15deSDaniel Borkmann err = map_freeze(&attr); 339987df15deSDaniel Borkmann break; 340009756af4SAlexei Starovoitov case BPF_PROG_LOAD: 3401838e9690SYonghong Song err = bpf_prog_load(&attr, uattr); 340209756af4SAlexei Starovoitov break; 3403b2197755SDaniel Borkmann case BPF_OBJ_PIN: 3404b2197755SDaniel Borkmann err = bpf_obj_pin(&attr); 3405b2197755SDaniel Borkmann break; 3406b2197755SDaniel Borkmann case BPF_OBJ_GET: 3407b2197755SDaniel Borkmann err = bpf_obj_get(&attr); 3408b2197755SDaniel Borkmann break; 3409f4324551SDaniel Mack case BPF_PROG_ATTACH: 3410f4324551SDaniel Mack err = bpf_prog_attach(&attr); 3411f4324551SDaniel Mack break; 3412f4324551SDaniel Mack case BPF_PROG_DETACH: 3413f4324551SDaniel Mack err = bpf_prog_detach(&attr); 3414f4324551SDaniel Mack break; 3415468e2f64SAlexei Starovoitov case BPF_PROG_QUERY: 3416468e2f64SAlexei Starovoitov err = bpf_prog_query(&attr, uattr); 3417468e2f64SAlexei Starovoitov break; 34181cf1cae9SAlexei Starovoitov case BPF_PROG_TEST_RUN: 34191cf1cae9SAlexei Starovoitov err = bpf_prog_test_run(&attr, uattr); 34201cf1cae9SAlexei Starovoitov break; 342134ad5580SMartin KaFai Lau case BPF_PROG_GET_NEXT_ID: 342234ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 342334ad5580SMartin KaFai Lau &prog_idr, &prog_idr_lock); 342434ad5580SMartin KaFai Lau break; 342534ad5580SMartin KaFai Lau case BPF_MAP_GET_NEXT_ID: 342634ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 342734ad5580SMartin KaFai Lau &map_idr, &map_idr_lock); 342834ad5580SMartin KaFai Lau break; 34291b9ed84eSQuentin Monnet case BPF_BTF_GET_NEXT_ID: 34301b9ed84eSQuentin Monnet err = bpf_obj_get_next_id(&attr, uattr, 34311b9ed84eSQuentin Monnet &btf_idr, &btf_idr_lock); 34321b9ed84eSQuentin Monnet break; 3433b16d9aa4SMartin KaFai Lau case BPF_PROG_GET_FD_BY_ID: 3434b16d9aa4SMartin KaFai Lau err = bpf_prog_get_fd_by_id(&attr); 3435b16d9aa4SMartin KaFai Lau break; 3436bd5f5f4eSMartin KaFai Lau case BPF_MAP_GET_FD_BY_ID: 3437bd5f5f4eSMartin KaFai Lau err = bpf_map_get_fd_by_id(&attr); 3438bd5f5f4eSMartin KaFai Lau break; 34391e270976SMartin KaFai Lau case BPF_OBJ_GET_INFO_BY_FD: 34401e270976SMartin KaFai Lau err = bpf_obj_get_info_by_fd(&attr, uattr); 34411e270976SMartin KaFai Lau break; 3442c4f6699dSAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN: 3443c4f6699dSAlexei Starovoitov err = bpf_raw_tracepoint_open(&attr); 3444c4f6699dSAlexei Starovoitov break; 3445f56a653cSMartin KaFai Lau case BPF_BTF_LOAD: 3446f56a653cSMartin KaFai Lau err = bpf_btf_load(&attr); 3447f56a653cSMartin KaFai Lau break; 344878958fcaSMartin KaFai Lau case BPF_BTF_GET_FD_BY_ID: 344978958fcaSMartin KaFai Lau err = bpf_btf_get_fd_by_id(&attr); 345078958fcaSMartin KaFai Lau break; 345141bdc4b4SYonghong Song case BPF_TASK_FD_QUERY: 345241bdc4b4SYonghong Song err = bpf_task_fd_query(&attr, uattr); 345341bdc4b4SYonghong Song break; 3454bd513cd0SMauricio Vasquez B case BPF_MAP_LOOKUP_AND_DELETE_ELEM: 3455bd513cd0SMauricio Vasquez B err = map_lookup_and_delete_elem(&attr); 3456bd513cd0SMauricio Vasquez B break; 3457cb4d03abSBrian Vazquez case BPF_MAP_LOOKUP_BATCH: 3458cb4d03abSBrian Vazquez err = bpf_map_do_batch(&attr, uattr, BPF_MAP_LOOKUP_BATCH); 3459cb4d03abSBrian Vazquez break; 346005799638SYonghong Song case BPF_MAP_LOOKUP_AND_DELETE_BATCH: 346105799638SYonghong Song err = bpf_map_do_batch(&attr, uattr, 346205799638SYonghong Song BPF_MAP_LOOKUP_AND_DELETE_BATCH); 346305799638SYonghong Song break; 3464aa2e93b8SBrian Vazquez case BPF_MAP_UPDATE_BATCH: 3465aa2e93b8SBrian Vazquez err = bpf_map_do_batch(&attr, uattr, BPF_MAP_UPDATE_BATCH); 3466aa2e93b8SBrian Vazquez break; 3467aa2e93b8SBrian Vazquez case BPF_MAP_DELETE_BATCH: 3468aa2e93b8SBrian Vazquez err = bpf_map_do_batch(&attr, uattr, BPF_MAP_DELETE_BATCH); 3469aa2e93b8SBrian Vazquez break; 347099c55f7dSAlexei Starovoitov default: 347199c55f7dSAlexei Starovoitov err = -EINVAL; 347299c55f7dSAlexei Starovoitov break; 347399c55f7dSAlexei Starovoitov } 347499c55f7dSAlexei Starovoitov 347599c55f7dSAlexei Starovoitov return err; 347699c55f7dSAlexei Starovoitov } 3477