15b497af4SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 299c55f7dSAlexei Starovoitov /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 399c55f7dSAlexei Starovoitov */ 499c55f7dSAlexei Starovoitov #include <linux/bpf.h> 5aef2fedaSJakub Kicinski #include <linux/bpf-cgroup.h> 6a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 7f4364dcfSSean Young #include <linux/bpf_lirc.h> 84a1e7c0cSToke Høiland-Jørgensen #include <linux/bpf_verifier.h> 961df10c7SKumar Kartikeya Dwivedi #include <linux/bsearch.h> 10f56a653cSMartin KaFai Lau #include <linux/btf.h> 1199c55f7dSAlexei Starovoitov #include <linux/syscalls.h> 1299c55f7dSAlexei Starovoitov #include <linux/slab.h> 133f07c014SIngo Molnar #include <linux/sched/signal.h> 14d407bd25SDaniel Borkmann #include <linux/vmalloc.h> 15d407bd25SDaniel Borkmann #include <linux/mmzone.h> 1699c55f7dSAlexei Starovoitov #include <linux/anon_inodes.h> 1741bdc4b4SYonghong Song #include <linux/fdtable.h> 18db20fd2bSAlexei Starovoitov #include <linux/file.h> 1941bdc4b4SYonghong Song #include <linux/fs.h> 2009756af4SAlexei Starovoitov #include <linux/license.h> 2109756af4SAlexei Starovoitov #include <linux/filter.h> 22535e7b4bSMickaël Salaün #include <linux/kernel.h> 23dc4bb0e2SMartin KaFai Lau #include <linux/idr.h> 24cb4d2b3fSMartin KaFai Lau #include <linux/cred.h> 25cb4d2b3fSMartin KaFai Lau #include <linux/timekeeping.h> 26cb4d2b3fSMartin KaFai Lau #include <linux/ctype.h> 279ef09e35SMark Rutland #include <linux/nospec.h> 28bae141f5SDaniel Borkmann #include <linux/audit.h> 29ccfe29ebSAlexei Starovoitov #include <uapi/linux/btf.h> 30ca5999fdSMike Rapoport #include <linux/pgtable.h> 319e4e01dfSKP Singh #include <linux/bpf_lsm.h> 32457f4436SAndrii Nakryiko #include <linux/poll.h> 334d7d7f69SKumar Kartikeya Dwivedi #include <linux/sort.h> 34a3fd7ceeSJakub Sitnicki #include <linux/bpf-netns.h> 351e6c62a8SAlexei Starovoitov #include <linux/rcupdate_trace.h> 3648edc1f7SRoman Gushchin #include <linux/memcontrol.h> 370dcac272SJiri Olsa #include <linux/trace_events.h> 3899c55f7dSAlexei Starovoitov 39da765a2fSDaniel Borkmann #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 4014dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 4114dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 42da765a2fSDaniel Borkmann #define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY) 4314dc6f04SMartin KaFai Lau #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 44da765a2fSDaniel Borkmann #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \ 45da765a2fSDaniel Borkmann IS_FD_HASH(map)) 4614dc6f04SMartin KaFai Lau 476e71b04aSChenbo Feng #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) 486e71b04aSChenbo Feng 49b121d1e7SAlexei Starovoitov DEFINE_PER_CPU(int, bpf_prog_active); 50dc4bb0e2SMartin KaFai Lau static DEFINE_IDR(prog_idr); 51dc4bb0e2SMartin KaFai Lau static DEFINE_SPINLOCK(prog_idr_lock); 52f3f1c054SMartin KaFai Lau static DEFINE_IDR(map_idr); 53f3f1c054SMartin KaFai Lau static DEFINE_SPINLOCK(map_idr_lock); 54a3b80e10SAndrii Nakryiko static DEFINE_IDR(link_idr); 55a3b80e10SAndrii Nakryiko static DEFINE_SPINLOCK(link_idr_lock); 56b121d1e7SAlexei Starovoitov 5708389d88SDaniel Borkmann int sysctl_unprivileged_bpf_disabled __read_mostly = 5808389d88SDaniel Borkmann IS_BUILTIN(CONFIG_BPF_UNPRIV_DEFAULT_OFF) ? 2 : 0; 591be7f75dSAlexei Starovoitov 6040077e0cSJohannes Berg static const struct bpf_map_ops * const bpf_map_types[] = { 6191cc1a99SAlexei Starovoitov #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) 6240077e0cSJohannes Berg #define BPF_MAP_TYPE(_id, _ops) \ 6340077e0cSJohannes Berg [_id] = &_ops, 64f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) 6540077e0cSJohannes Berg #include <linux/bpf_types.h> 6640077e0cSJohannes Berg #undef BPF_PROG_TYPE 6740077e0cSJohannes Berg #undef BPF_MAP_TYPE 68f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 6940077e0cSJohannes Berg }; 7099c55f7dSAlexei Starovoitov 71752ba56fSMickaël Salaün /* 72752ba56fSMickaël Salaün * If we're handed a bigger struct than we know of, ensure all the unknown bits 73752ba56fSMickaël Salaün * are 0 - i.e. new user-space does not rely on any kernel feature extensions 74752ba56fSMickaël Salaün * we don't know about yet. 75752ba56fSMickaël Salaün * 76752ba56fSMickaël Salaün * There is a ToCToU between this function call and the following 77752ba56fSMickaël Salaün * copy_from_user() call. However, this is not a concern since this function is 78752ba56fSMickaël Salaün * meant to be a future-proofing of bits. 79752ba56fSMickaël Salaün */ 80af2ac3e1SAlexei Starovoitov int bpf_check_uarg_tail_zero(bpfptr_t uaddr, 8158291a74SMickaël Salaün size_t expected_size, 8258291a74SMickaël Salaün size_t actual_size) 8358291a74SMickaël Salaün { 84b7e4b65fSAl Viro int res; 8558291a74SMickaël Salaün 86752ba56fSMickaël Salaün if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 87752ba56fSMickaël Salaün return -E2BIG; 88752ba56fSMickaël Salaün 8958291a74SMickaël Salaün if (actual_size <= expected_size) 9058291a74SMickaël Salaün return 0; 9158291a74SMickaël Salaün 92af2ac3e1SAlexei Starovoitov if (uaddr.is_kernel) 93af2ac3e1SAlexei Starovoitov res = memchr_inv(uaddr.kernel + expected_size, 0, 94af2ac3e1SAlexei Starovoitov actual_size - expected_size) == NULL; 95af2ac3e1SAlexei Starovoitov else 96af2ac3e1SAlexei Starovoitov res = check_zeroed_user(uaddr.user + expected_size, 97af2ac3e1SAlexei Starovoitov actual_size - expected_size); 98b7e4b65fSAl Viro if (res < 0) 99b7e4b65fSAl Viro return res; 100b7e4b65fSAl Viro return res ? 0 : -E2BIG; 10158291a74SMickaël Salaün } 10258291a74SMickaël Salaün 103a3884572SJakub Kicinski const struct bpf_map_ops bpf_map_offload_ops = { 104f4d05259SMartin KaFai Lau .map_meta_equal = bpf_map_meta_equal, 105a3884572SJakub Kicinski .map_alloc = bpf_map_offload_map_alloc, 106a3884572SJakub Kicinski .map_free = bpf_map_offload_map_free, 107e8d2bec0SDaniel Borkmann .map_check_btf = map_check_no_btf, 108a3884572SJakub Kicinski }; 109a3884572SJakub Kicinski 11099c55f7dSAlexei Starovoitov static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 11199c55f7dSAlexei Starovoitov { 1121110f3a9SJakub Kicinski const struct bpf_map_ops *ops; 1139ef09e35SMark Rutland u32 type = attr->map_type; 11499c55f7dSAlexei Starovoitov struct bpf_map *map; 1151110f3a9SJakub Kicinski int err; 11699c55f7dSAlexei Starovoitov 1179ef09e35SMark Rutland if (type >= ARRAY_SIZE(bpf_map_types)) 1181110f3a9SJakub Kicinski return ERR_PTR(-EINVAL); 1199ef09e35SMark Rutland type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types)); 1209ef09e35SMark Rutland ops = bpf_map_types[type]; 1211110f3a9SJakub Kicinski if (!ops) 12240077e0cSJohannes Berg return ERR_PTR(-EINVAL); 12340077e0cSJohannes Berg 1241110f3a9SJakub Kicinski if (ops->map_alloc_check) { 1251110f3a9SJakub Kicinski err = ops->map_alloc_check(attr); 1261110f3a9SJakub Kicinski if (err) 1271110f3a9SJakub Kicinski return ERR_PTR(err); 1281110f3a9SJakub Kicinski } 129a3884572SJakub Kicinski if (attr->map_ifindex) 130a3884572SJakub Kicinski ops = &bpf_map_offload_ops; 1311110f3a9SJakub Kicinski map = ops->map_alloc(attr); 13299c55f7dSAlexei Starovoitov if (IS_ERR(map)) 13399c55f7dSAlexei Starovoitov return map; 1341110f3a9SJakub Kicinski map->ops = ops; 1359ef09e35SMark Rutland map->map_type = type; 13699c55f7dSAlexei Starovoitov return map; 13799c55f7dSAlexei Starovoitov } 13899c55f7dSAlexei Starovoitov 139353050beSDaniel Borkmann static void bpf_map_write_active_inc(struct bpf_map *map) 140353050beSDaniel Borkmann { 141353050beSDaniel Borkmann atomic64_inc(&map->writecnt); 142353050beSDaniel Borkmann } 143353050beSDaniel Borkmann 144353050beSDaniel Borkmann static void bpf_map_write_active_dec(struct bpf_map *map) 145353050beSDaniel Borkmann { 146353050beSDaniel Borkmann atomic64_dec(&map->writecnt); 147353050beSDaniel Borkmann } 148353050beSDaniel Borkmann 149353050beSDaniel Borkmann bool bpf_map_write_active(const struct bpf_map *map) 150353050beSDaniel Borkmann { 151353050beSDaniel Borkmann return atomic64_read(&map->writecnt) != 0; 152353050beSDaniel Borkmann } 153353050beSDaniel Borkmann 15480ee81e0SRoman Gushchin static u32 bpf_map_value_size(const struct bpf_map *map) 15515c14a3dSBrian Vazquez { 15615c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 15715c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 15815c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 15915c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 16015c14a3dSBrian Vazquez return round_up(map->value_size, 8) * num_possible_cpus(); 16115c14a3dSBrian Vazquez else if (IS_FD_MAP(map)) 16215c14a3dSBrian Vazquez return sizeof(u32); 16315c14a3dSBrian Vazquez else 16415c14a3dSBrian Vazquez return map->value_size; 16515c14a3dSBrian Vazquez } 16615c14a3dSBrian Vazquez 16715c14a3dSBrian Vazquez static void maybe_wait_bpf_programs(struct bpf_map *map) 16815c14a3dSBrian Vazquez { 16915c14a3dSBrian Vazquez /* Wait for any running BPF programs to complete so that 17015c14a3dSBrian Vazquez * userspace, when we return to it, knows that all programs 17115c14a3dSBrian Vazquez * that could be running use the new map value. 17215c14a3dSBrian Vazquez */ 17315c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS || 17415c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 17515c14a3dSBrian Vazquez synchronize_rcu(); 17615c14a3dSBrian Vazquez } 17715c14a3dSBrian Vazquez 17815c14a3dSBrian Vazquez static int bpf_map_update_value(struct bpf_map *map, struct fd f, void *key, 17915c14a3dSBrian Vazquez void *value, __u64 flags) 18015c14a3dSBrian Vazquez { 18115c14a3dSBrian Vazquez int err; 18215c14a3dSBrian Vazquez 18315c14a3dSBrian Vazquez /* Need to create a kthread, thus must support schedule */ 18415c14a3dSBrian Vazquez if (bpf_map_is_dev_bound(map)) { 18515c14a3dSBrian Vazquez return bpf_map_offload_update_elem(map, key, value, flags); 18615c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_CPUMAP || 18715c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 18815c14a3dSBrian Vazquez return map->ops->map_update_elem(map, key, value, flags); 18913b79d3fSLorenz Bauer } else if (map->map_type == BPF_MAP_TYPE_SOCKHASH || 19013b79d3fSLorenz Bauer map->map_type == BPF_MAP_TYPE_SOCKMAP) { 19113b79d3fSLorenz Bauer return sock_map_update_elem_sys(map, key, value, flags); 19215c14a3dSBrian Vazquez } else if (IS_FD_PROG_ARRAY(map)) { 19315c14a3dSBrian Vazquez return bpf_fd_array_map_update_elem(map, f.file, key, value, 19415c14a3dSBrian Vazquez flags); 19515c14a3dSBrian Vazquez } 19615c14a3dSBrian Vazquez 197b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 19815c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 19915c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 20015c14a3dSBrian Vazquez err = bpf_percpu_hash_update(map, key, value, flags); 20115c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 20215c14a3dSBrian Vazquez err = bpf_percpu_array_update(map, key, value, flags); 20315c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 20415c14a3dSBrian Vazquez err = bpf_percpu_cgroup_storage_update(map, key, value, 20515c14a3dSBrian Vazquez flags); 20615c14a3dSBrian Vazquez } else if (IS_FD_ARRAY(map)) { 20715c14a3dSBrian Vazquez rcu_read_lock(); 20815c14a3dSBrian Vazquez err = bpf_fd_array_map_update_elem(map, f.file, key, value, 20915c14a3dSBrian Vazquez flags); 21015c14a3dSBrian Vazquez rcu_read_unlock(); 21115c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 21215c14a3dSBrian Vazquez rcu_read_lock(); 21315c14a3dSBrian Vazquez err = bpf_fd_htab_map_update_elem(map, f.file, key, value, 21415c14a3dSBrian Vazquez flags); 21515c14a3dSBrian Vazquez rcu_read_unlock(); 21615c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 21715c14a3dSBrian Vazquez /* rcu_read_lock() is not needed */ 21815c14a3dSBrian Vazquez err = bpf_fd_reuseport_array_update_elem(map, key, value, 21915c14a3dSBrian Vazquez flags); 22015c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 2219330986cSJoanne Koong map->map_type == BPF_MAP_TYPE_STACK || 2229330986cSJoanne Koong map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) { 22315c14a3dSBrian Vazquez err = map->ops->map_push_elem(map, value, flags); 22415c14a3dSBrian Vazquez } else { 22515c14a3dSBrian Vazquez rcu_read_lock(); 22615c14a3dSBrian Vazquez err = map->ops->map_update_elem(map, key, value, flags); 22715c14a3dSBrian Vazquez rcu_read_unlock(); 22815c14a3dSBrian Vazquez } 229b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 23015c14a3dSBrian Vazquez maybe_wait_bpf_programs(map); 23115c14a3dSBrian Vazquez 23215c14a3dSBrian Vazquez return err; 23315c14a3dSBrian Vazquez } 23415c14a3dSBrian Vazquez 23515c14a3dSBrian Vazquez static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value, 23615c14a3dSBrian Vazquez __u64 flags) 23715c14a3dSBrian Vazquez { 23815c14a3dSBrian Vazquez void *ptr; 23915c14a3dSBrian Vazquez int err; 24015c14a3dSBrian Vazquez 241cb4d03abSBrian Vazquez if (bpf_map_is_dev_bound(map)) 242cb4d03abSBrian Vazquez return bpf_map_offload_lookup_elem(map, key, value); 24315c14a3dSBrian Vazquez 244b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 24515c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 24615c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 24715c14a3dSBrian Vazquez err = bpf_percpu_hash_copy(map, key, value); 24815c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 24915c14a3dSBrian Vazquez err = bpf_percpu_array_copy(map, key, value); 25015c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 25115c14a3dSBrian Vazquez err = bpf_percpu_cgroup_storage_copy(map, key, value); 25215c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 25315c14a3dSBrian Vazquez err = bpf_stackmap_copy(map, key, value); 25415c14a3dSBrian Vazquez } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) { 25515c14a3dSBrian Vazquez err = bpf_fd_array_map_lookup_elem(map, key, value); 25615c14a3dSBrian Vazquez } else if (IS_FD_HASH(map)) { 25715c14a3dSBrian Vazquez err = bpf_fd_htab_map_lookup_elem(map, key, value); 25815c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 25915c14a3dSBrian Vazquez err = bpf_fd_reuseport_array_lookup_elem(map, key, value); 26015c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 2619330986cSJoanne Koong map->map_type == BPF_MAP_TYPE_STACK || 2629330986cSJoanne Koong map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) { 26315c14a3dSBrian Vazquez err = map->ops->map_peek_elem(map, value); 26415c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 26515c14a3dSBrian Vazquez /* struct_ops map requires directly updating "value" */ 26615c14a3dSBrian Vazquez err = bpf_struct_ops_map_sys_lookup_elem(map, key, value); 26715c14a3dSBrian Vazquez } else { 26815c14a3dSBrian Vazquez rcu_read_lock(); 26915c14a3dSBrian Vazquez if (map->ops->map_lookup_elem_sys_only) 27015c14a3dSBrian Vazquez ptr = map->ops->map_lookup_elem_sys_only(map, key); 27115c14a3dSBrian Vazquez else 27215c14a3dSBrian Vazquez ptr = map->ops->map_lookup_elem(map, key); 27315c14a3dSBrian Vazquez if (IS_ERR(ptr)) { 27415c14a3dSBrian Vazquez err = PTR_ERR(ptr); 27515c14a3dSBrian Vazquez } else if (!ptr) { 27615c14a3dSBrian Vazquez err = -ENOENT; 27715c14a3dSBrian Vazquez } else { 27815c14a3dSBrian Vazquez err = 0; 27915c14a3dSBrian Vazquez if (flags & BPF_F_LOCK) 28015c14a3dSBrian Vazquez /* lock 'ptr' and copy everything but lock */ 28115c14a3dSBrian Vazquez copy_map_value_locked(map, value, ptr, true); 28215c14a3dSBrian Vazquez else 28315c14a3dSBrian Vazquez copy_map_value(map, value, ptr); 28468134668SAlexei Starovoitov /* mask lock and timer, since value wasn't zero inited */ 28568134668SAlexei Starovoitov check_and_init_map_value(map, value); 28615c14a3dSBrian Vazquez } 28715c14a3dSBrian Vazquez rcu_read_unlock(); 28815c14a3dSBrian Vazquez } 28915c14a3dSBrian Vazquez 290b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 29115c14a3dSBrian Vazquez maybe_wait_bpf_programs(map); 29215c14a3dSBrian Vazquez 29315c14a3dSBrian Vazquez return err; 29415c14a3dSBrian Vazquez } 29515c14a3dSBrian Vazquez 296d5299b67SRoman Gushchin /* Please, do not use this function outside from the map creation path 297d5299b67SRoman Gushchin * (e.g. in map update path) without taking care of setting the active 298d5299b67SRoman Gushchin * memory cgroup (see at bpf_map_kmalloc_node() for example). 299d5299b67SRoman Gushchin */ 300196e8ca7SDaniel Borkmann static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable) 301d407bd25SDaniel Borkmann { 302f01a7dbeSMartynas Pumputis /* We really just want to fail instead of triggering OOM killer 303f01a7dbeSMartynas Pumputis * under memory pressure, therefore we set __GFP_NORETRY to kmalloc, 304f01a7dbeSMartynas Pumputis * which is used for lower order allocation requests. 305f01a7dbeSMartynas Pumputis * 306f01a7dbeSMartynas Pumputis * It has been observed that higher order allocation requests done by 307f01a7dbeSMartynas Pumputis * vmalloc with __GFP_NORETRY being set might fail due to not trying 308f01a7dbeSMartynas Pumputis * to reclaim memory from the page cache, thus we set 309f01a7dbeSMartynas Pumputis * __GFP_RETRY_MAYFAIL to avoid such situations. 310d407bd25SDaniel Borkmann */ 311f01a7dbeSMartynas Pumputis 312d5299b67SRoman Gushchin const gfp_t gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_ACCOUNT; 313041de93fSChristoph Hellwig unsigned int flags = 0; 314041de93fSChristoph Hellwig unsigned long align = 1; 315d407bd25SDaniel Borkmann void *area; 316d407bd25SDaniel Borkmann 317196e8ca7SDaniel Borkmann if (size >= SIZE_MAX) 318196e8ca7SDaniel Borkmann return NULL; 319196e8ca7SDaniel Borkmann 320fc970227SAndrii Nakryiko /* kmalloc()'ed memory can't be mmap()'ed */ 321041de93fSChristoph Hellwig if (mmapable) { 322041de93fSChristoph Hellwig BUG_ON(!PAGE_ALIGNED(size)); 323041de93fSChristoph Hellwig align = SHMLBA; 324041de93fSChristoph Hellwig flags = VM_USERMAP; 325041de93fSChristoph Hellwig } else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 326041de93fSChristoph Hellwig area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY, 327f01a7dbeSMartynas Pumputis numa_node); 328d407bd25SDaniel Borkmann if (area != NULL) 329d407bd25SDaniel Borkmann return area; 330d407bd25SDaniel Borkmann } 331041de93fSChristoph Hellwig 332041de93fSChristoph Hellwig return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, 333041de93fSChristoph Hellwig gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL, 334041de93fSChristoph Hellwig flags, numa_node, __builtin_return_address(0)); 335d407bd25SDaniel Borkmann } 336d407bd25SDaniel Borkmann 337196e8ca7SDaniel Borkmann void *bpf_map_area_alloc(u64 size, int numa_node) 338fc970227SAndrii Nakryiko { 339fc970227SAndrii Nakryiko return __bpf_map_area_alloc(size, numa_node, false); 340fc970227SAndrii Nakryiko } 341fc970227SAndrii Nakryiko 342196e8ca7SDaniel Borkmann void *bpf_map_area_mmapable_alloc(u64 size, int numa_node) 343fc970227SAndrii Nakryiko { 344fc970227SAndrii Nakryiko return __bpf_map_area_alloc(size, numa_node, true); 345fc970227SAndrii Nakryiko } 346fc970227SAndrii Nakryiko 347d407bd25SDaniel Borkmann void bpf_map_area_free(void *area) 348d407bd25SDaniel Borkmann { 349d407bd25SDaniel Borkmann kvfree(area); 350d407bd25SDaniel Borkmann } 351d407bd25SDaniel Borkmann 352be70bcd5SDaniel Borkmann static u32 bpf_map_flags_retain_permanent(u32 flags) 353be70bcd5SDaniel Borkmann { 354be70bcd5SDaniel Borkmann /* Some map creation flags are not tied to the map object but 355be70bcd5SDaniel Borkmann * rather to the map fd instead, so they have no meaning upon 356be70bcd5SDaniel Borkmann * map object inspection since multiple file descriptors with 357be70bcd5SDaniel Borkmann * different (access) properties can exist here. Thus, given 358be70bcd5SDaniel Borkmann * this has zero meaning for the map itself, lets clear these 359be70bcd5SDaniel Borkmann * from here. 360be70bcd5SDaniel Borkmann */ 361be70bcd5SDaniel Borkmann return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY); 362be70bcd5SDaniel Borkmann } 363be70bcd5SDaniel Borkmann 364bd475643SJakub Kicinski void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) 365bd475643SJakub Kicinski { 366bd475643SJakub Kicinski map->map_type = attr->map_type; 367bd475643SJakub Kicinski map->key_size = attr->key_size; 368bd475643SJakub Kicinski map->value_size = attr->value_size; 369bd475643SJakub Kicinski map->max_entries = attr->max_entries; 370be70bcd5SDaniel Borkmann map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags); 371bd475643SJakub Kicinski map->numa_node = bpf_map_attr_numa_node(attr); 3729330986cSJoanne Koong map->map_extra = attr->map_extra; 373bd475643SJakub Kicinski } 374bd475643SJakub Kicinski 375f3f1c054SMartin KaFai Lau static int bpf_map_alloc_id(struct bpf_map *map) 376f3f1c054SMartin KaFai Lau { 377f3f1c054SMartin KaFai Lau int id; 378f3f1c054SMartin KaFai Lau 379b76354cdSShaohua Li idr_preload(GFP_KERNEL); 380f3f1c054SMartin KaFai Lau spin_lock_bh(&map_idr_lock); 381f3f1c054SMartin KaFai Lau id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 382f3f1c054SMartin KaFai Lau if (id > 0) 383f3f1c054SMartin KaFai Lau map->id = id; 384f3f1c054SMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 385b76354cdSShaohua Li idr_preload_end(); 386f3f1c054SMartin KaFai Lau 387f3f1c054SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 388f3f1c054SMartin KaFai Lau return -ENOSPC; 389f3f1c054SMartin KaFai Lau 390f3f1c054SMartin KaFai Lau return id > 0 ? 0 : id; 391f3f1c054SMartin KaFai Lau } 392f3f1c054SMartin KaFai Lau 393a3884572SJakub Kicinski void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) 394f3f1c054SMartin KaFai Lau { 395930651a7SEric Dumazet unsigned long flags; 396930651a7SEric Dumazet 397a3884572SJakub Kicinski /* Offloaded maps are removed from the IDR store when their device 398a3884572SJakub Kicinski * disappears - even if someone holds an fd to them they are unusable, 399a3884572SJakub Kicinski * the memory is gone, all ops will fail; they are simply waiting for 400a3884572SJakub Kicinski * refcnt to drop to be freed. 401a3884572SJakub Kicinski */ 402a3884572SJakub Kicinski if (!map->id) 403a3884572SJakub Kicinski return; 404a3884572SJakub Kicinski 405bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 406930651a7SEric Dumazet spin_lock_irqsave(&map_idr_lock, flags); 407bd5f5f4eSMartin KaFai Lau else 408bd5f5f4eSMartin KaFai Lau __acquire(&map_idr_lock); 409bd5f5f4eSMartin KaFai Lau 410f3f1c054SMartin KaFai Lau idr_remove(&map_idr, map->id); 411a3884572SJakub Kicinski map->id = 0; 412bd5f5f4eSMartin KaFai Lau 413bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 414930651a7SEric Dumazet spin_unlock_irqrestore(&map_idr_lock, flags); 415bd5f5f4eSMartin KaFai Lau else 416bd5f5f4eSMartin KaFai Lau __release(&map_idr_lock); 417f3f1c054SMartin KaFai Lau } 418f3f1c054SMartin KaFai Lau 41948edc1f7SRoman Gushchin #ifdef CONFIG_MEMCG_KMEM 42048edc1f7SRoman Gushchin static void bpf_map_save_memcg(struct bpf_map *map) 42148edc1f7SRoman Gushchin { 42248edc1f7SRoman Gushchin map->memcg = get_mem_cgroup_from_mm(current->mm); 42348edc1f7SRoman Gushchin } 42448edc1f7SRoman Gushchin 42548edc1f7SRoman Gushchin static void bpf_map_release_memcg(struct bpf_map *map) 42648edc1f7SRoman Gushchin { 42748edc1f7SRoman Gushchin mem_cgroup_put(map->memcg); 42848edc1f7SRoman Gushchin } 42948edc1f7SRoman Gushchin 43048edc1f7SRoman Gushchin void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, 43148edc1f7SRoman Gushchin int node) 43248edc1f7SRoman Gushchin { 43348edc1f7SRoman Gushchin struct mem_cgroup *old_memcg; 43448edc1f7SRoman Gushchin void *ptr; 43548edc1f7SRoman Gushchin 43648edc1f7SRoman Gushchin old_memcg = set_active_memcg(map->memcg); 43748edc1f7SRoman Gushchin ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node); 43848edc1f7SRoman Gushchin set_active_memcg(old_memcg); 43948edc1f7SRoman Gushchin 44048edc1f7SRoman Gushchin return ptr; 44148edc1f7SRoman Gushchin } 44248edc1f7SRoman Gushchin 44348edc1f7SRoman Gushchin void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags) 44448edc1f7SRoman Gushchin { 44548edc1f7SRoman Gushchin struct mem_cgroup *old_memcg; 44648edc1f7SRoman Gushchin void *ptr; 44748edc1f7SRoman Gushchin 44848edc1f7SRoman Gushchin old_memcg = set_active_memcg(map->memcg); 44948edc1f7SRoman Gushchin ptr = kzalloc(size, flags | __GFP_ACCOUNT); 45048edc1f7SRoman Gushchin set_active_memcg(old_memcg); 45148edc1f7SRoman Gushchin 45248edc1f7SRoman Gushchin return ptr; 45348edc1f7SRoman Gushchin } 45448edc1f7SRoman Gushchin 45548edc1f7SRoman Gushchin void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, 45648edc1f7SRoman Gushchin size_t align, gfp_t flags) 45748edc1f7SRoman Gushchin { 45848edc1f7SRoman Gushchin struct mem_cgroup *old_memcg; 45948edc1f7SRoman Gushchin void __percpu *ptr; 46048edc1f7SRoman Gushchin 46148edc1f7SRoman Gushchin old_memcg = set_active_memcg(map->memcg); 46248edc1f7SRoman Gushchin ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT); 46348edc1f7SRoman Gushchin set_active_memcg(old_memcg); 46448edc1f7SRoman Gushchin 46548edc1f7SRoman Gushchin return ptr; 46648edc1f7SRoman Gushchin } 46748edc1f7SRoman Gushchin 46848edc1f7SRoman Gushchin #else 46948edc1f7SRoman Gushchin static void bpf_map_save_memcg(struct bpf_map *map) 47048edc1f7SRoman Gushchin { 47148edc1f7SRoman Gushchin } 47248edc1f7SRoman Gushchin 47348edc1f7SRoman Gushchin static void bpf_map_release_memcg(struct bpf_map *map) 47448edc1f7SRoman Gushchin { 47548edc1f7SRoman Gushchin } 47648edc1f7SRoman Gushchin #endif 47748edc1f7SRoman Gushchin 47861df10c7SKumar Kartikeya Dwivedi static int bpf_map_kptr_off_cmp(const void *a, const void *b) 47961df10c7SKumar Kartikeya Dwivedi { 48061df10c7SKumar Kartikeya Dwivedi const struct bpf_map_value_off_desc *off_desc1 = a, *off_desc2 = b; 48161df10c7SKumar Kartikeya Dwivedi 48261df10c7SKumar Kartikeya Dwivedi if (off_desc1->offset < off_desc2->offset) 48361df10c7SKumar Kartikeya Dwivedi return -1; 48461df10c7SKumar Kartikeya Dwivedi else if (off_desc1->offset > off_desc2->offset) 48561df10c7SKumar Kartikeya Dwivedi return 1; 48661df10c7SKumar Kartikeya Dwivedi return 0; 48761df10c7SKumar Kartikeya Dwivedi } 48861df10c7SKumar Kartikeya Dwivedi 48961df10c7SKumar Kartikeya Dwivedi struct bpf_map_value_off_desc *bpf_map_kptr_off_contains(struct bpf_map *map, u32 offset) 49061df10c7SKumar Kartikeya Dwivedi { 49161df10c7SKumar Kartikeya Dwivedi /* Since members are iterated in btf_find_field in increasing order, 49261df10c7SKumar Kartikeya Dwivedi * offsets appended to kptr_off_tab are in increasing order, so we can 49361df10c7SKumar Kartikeya Dwivedi * do bsearch to find exact match. 49461df10c7SKumar Kartikeya Dwivedi */ 49561df10c7SKumar Kartikeya Dwivedi struct bpf_map_value_off *tab; 49661df10c7SKumar Kartikeya Dwivedi 49761df10c7SKumar Kartikeya Dwivedi if (!map_value_has_kptrs(map)) 49861df10c7SKumar Kartikeya Dwivedi return NULL; 49961df10c7SKumar Kartikeya Dwivedi tab = map->kptr_off_tab; 50061df10c7SKumar Kartikeya Dwivedi return bsearch(&offset, tab->off, tab->nr_off, sizeof(tab->off[0]), bpf_map_kptr_off_cmp); 50161df10c7SKumar Kartikeya Dwivedi } 50261df10c7SKumar Kartikeya Dwivedi 50361df10c7SKumar Kartikeya Dwivedi void bpf_map_free_kptr_off_tab(struct bpf_map *map) 50461df10c7SKumar Kartikeya Dwivedi { 50561df10c7SKumar Kartikeya Dwivedi struct bpf_map_value_off *tab = map->kptr_off_tab; 50661df10c7SKumar Kartikeya Dwivedi int i; 50761df10c7SKumar Kartikeya Dwivedi 50861df10c7SKumar Kartikeya Dwivedi if (!map_value_has_kptrs(map)) 50961df10c7SKumar Kartikeya Dwivedi return; 51014a324f6SKumar Kartikeya Dwivedi for (i = 0; i < tab->nr_off; i++) { 51114a324f6SKumar Kartikeya Dwivedi if (tab->off[i].kptr.module) 51214a324f6SKumar Kartikeya Dwivedi module_put(tab->off[i].kptr.module); 51361df10c7SKumar Kartikeya Dwivedi btf_put(tab->off[i].kptr.btf); 51414a324f6SKumar Kartikeya Dwivedi } 51561df10c7SKumar Kartikeya Dwivedi kfree(tab); 51661df10c7SKumar Kartikeya Dwivedi map->kptr_off_tab = NULL; 51761df10c7SKumar Kartikeya Dwivedi } 51861df10c7SKumar Kartikeya Dwivedi 51961df10c7SKumar Kartikeya Dwivedi struct bpf_map_value_off *bpf_map_copy_kptr_off_tab(const struct bpf_map *map) 52061df10c7SKumar Kartikeya Dwivedi { 52161df10c7SKumar Kartikeya Dwivedi struct bpf_map_value_off *tab = map->kptr_off_tab, *new_tab; 52261df10c7SKumar Kartikeya Dwivedi int size, i; 52361df10c7SKumar Kartikeya Dwivedi 52461df10c7SKumar Kartikeya Dwivedi if (!map_value_has_kptrs(map)) 52561df10c7SKumar Kartikeya Dwivedi return ERR_PTR(-ENOENT); 52661df10c7SKumar Kartikeya Dwivedi size = offsetof(struct bpf_map_value_off, off[tab->nr_off]); 52761df10c7SKumar Kartikeya Dwivedi new_tab = kmemdup(tab, size, GFP_KERNEL | __GFP_NOWARN); 52861df10c7SKumar Kartikeya Dwivedi if (!new_tab) 52961df10c7SKumar Kartikeya Dwivedi return ERR_PTR(-ENOMEM); 53061df10c7SKumar Kartikeya Dwivedi /* Do a deep copy of the kptr_off_tab */ 53114a324f6SKumar Kartikeya Dwivedi for (i = 0; i < tab->nr_off; i++) { 53261df10c7SKumar Kartikeya Dwivedi btf_get(tab->off[i].kptr.btf); 53314a324f6SKumar Kartikeya Dwivedi if (tab->off[i].kptr.module && !try_module_get(tab->off[i].kptr.module)) { 53414a324f6SKumar Kartikeya Dwivedi while (i--) { 53514a324f6SKumar Kartikeya Dwivedi if (tab->off[i].kptr.module) 53614a324f6SKumar Kartikeya Dwivedi module_put(tab->off[i].kptr.module); 53714a324f6SKumar Kartikeya Dwivedi btf_put(tab->off[i].kptr.btf); 53814a324f6SKumar Kartikeya Dwivedi } 53914a324f6SKumar Kartikeya Dwivedi kfree(new_tab); 54014a324f6SKumar Kartikeya Dwivedi return ERR_PTR(-ENXIO); 54114a324f6SKumar Kartikeya Dwivedi } 54214a324f6SKumar Kartikeya Dwivedi } 54361df10c7SKumar Kartikeya Dwivedi return new_tab; 54461df10c7SKumar Kartikeya Dwivedi } 54561df10c7SKumar Kartikeya Dwivedi 54661df10c7SKumar Kartikeya Dwivedi bool bpf_map_equal_kptr_off_tab(const struct bpf_map *map_a, const struct bpf_map *map_b) 54761df10c7SKumar Kartikeya Dwivedi { 54861df10c7SKumar Kartikeya Dwivedi struct bpf_map_value_off *tab_a = map_a->kptr_off_tab, *tab_b = map_b->kptr_off_tab; 54961df10c7SKumar Kartikeya Dwivedi bool a_has_kptr = map_value_has_kptrs(map_a), b_has_kptr = map_value_has_kptrs(map_b); 55061df10c7SKumar Kartikeya Dwivedi int size; 55161df10c7SKumar Kartikeya Dwivedi 55261df10c7SKumar Kartikeya Dwivedi if (!a_has_kptr && !b_has_kptr) 55361df10c7SKumar Kartikeya Dwivedi return true; 55461df10c7SKumar Kartikeya Dwivedi if (a_has_kptr != b_has_kptr) 55561df10c7SKumar Kartikeya Dwivedi return false; 55661df10c7SKumar Kartikeya Dwivedi if (tab_a->nr_off != tab_b->nr_off) 55761df10c7SKumar Kartikeya Dwivedi return false; 55861df10c7SKumar Kartikeya Dwivedi size = offsetof(struct bpf_map_value_off, off[tab_a->nr_off]); 55961df10c7SKumar Kartikeya Dwivedi return !memcmp(tab_a, tab_b, size); 56061df10c7SKumar Kartikeya Dwivedi } 56161df10c7SKumar Kartikeya Dwivedi 56214a324f6SKumar Kartikeya Dwivedi /* Caller must ensure map_value_has_kptrs is true. Note that this function can 56314a324f6SKumar Kartikeya Dwivedi * be called on a map value while the map_value is visible to BPF programs, as 56414a324f6SKumar Kartikeya Dwivedi * it ensures the correct synchronization, and we already enforce the same using 56514a324f6SKumar Kartikeya Dwivedi * the bpf_kptr_xchg helper on the BPF program side for referenced kptrs. 56614a324f6SKumar Kartikeya Dwivedi */ 56714a324f6SKumar Kartikeya Dwivedi void bpf_map_free_kptrs(struct bpf_map *map, void *map_value) 56814a324f6SKumar Kartikeya Dwivedi { 56914a324f6SKumar Kartikeya Dwivedi struct bpf_map_value_off *tab = map->kptr_off_tab; 57014a324f6SKumar Kartikeya Dwivedi unsigned long *btf_id_ptr; 57114a324f6SKumar Kartikeya Dwivedi int i; 57214a324f6SKumar Kartikeya Dwivedi 57314a324f6SKumar Kartikeya Dwivedi for (i = 0; i < tab->nr_off; i++) { 57414a324f6SKumar Kartikeya Dwivedi struct bpf_map_value_off_desc *off_desc = &tab->off[i]; 57514a324f6SKumar Kartikeya Dwivedi unsigned long old_ptr; 57614a324f6SKumar Kartikeya Dwivedi 57714a324f6SKumar Kartikeya Dwivedi btf_id_ptr = map_value + off_desc->offset; 57814a324f6SKumar Kartikeya Dwivedi if (off_desc->type == BPF_KPTR_UNREF) { 57914a324f6SKumar Kartikeya Dwivedi u64 *p = (u64 *)btf_id_ptr; 58014a324f6SKumar Kartikeya Dwivedi 58114a324f6SKumar Kartikeya Dwivedi WRITE_ONCE(p, 0); 58214a324f6SKumar Kartikeya Dwivedi continue; 58314a324f6SKumar Kartikeya Dwivedi } 58414a324f6SKumar Kartikeya Dwivedi old_ptr = xchg(btf_id_ptr, 0); 58514a324f6SKumar Kartikeya Dwivedi off_desc->kptr.dtor((void *)old_ptr); 58614a324f6SKumar Kartikeya Dwivedi } 58714a324f6SKumar Kartikeya Dwivedi } 58814a324f6SKumar Kartikeya Dwivedi 58999c55f7dSAlexei Starovoitov /* called from workqueue */ 59099c55f7dSAlexei Starovoitov static void bpf_map_free_deferred(struct work_struct *work) 59199c55f7dSAlexei Starovoitov { 59299c55f7dSAlexei Starovoitov struct bpf_map *map = container_of(work, struct bpf_map, work); 59399c55f7dSAlexei Starovoitov 594afdb09c7SChenbo Feng security_bpf_map_free(map); 5954d7d7f69SKumar Kartikeya Dwivedi kfree(map->off_arr); 59648edc1f7SRoman Gushchin bpf_map_release_memcg(map); 59714a324f6SKumar Kartikeya Dwivedi /* implementation dependent freeing, map_free callback also does 59814a324f6SKumar Kartikeya Dwivedi * bpf_map_free_kptr_off_tab, if needed. 59914a324f6SKumar Kartikeya Dwivedi */ 60099c55f7dSAlexei Starovoitov map->ops->map_free(map); 60199c55f7dSAlexei Starovoitov } 60299c55f7dSAlexei Starovoitov 603c9da161cSDaniel Borkmann static void bpf_map_put_uref(struct bpf_map *map) 604c9da161cSDaniel Borkmann { 6051e0bd5a0SAndrii Nakryiko if (atomic64_dec_and_test(&map->usercnt)) { 606ba6b8de4SJohn Fastabend if (map->ops->map_release_uref) 607ba6b8de4SJohn Fastabend map->ops->map_release_uref(map); 608c9da161cSDaniel Borkmann } 609c9da161cSDaniel Borkmann } 610c9da161cSDaniel Borkmann 61199c55f7dSAlexei Starovoitov /* decrement map refcnt and schedule it for freeing via workqueue 61299c55f7dSAlexei Starovoitov * (unrelying map implementation ops->map_free() might sleep) 61399c55f7dSAlexei Starovoitov */ 614bd5f5f4eSMartin KaFai Lau static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) 61599c55f7dSAlexei Starovoitov { 6161e0bd5a0SAndrii Nakryiko if (atomic64_dec_and_test(&map->refcnt)) { 61734ad5580SMartin KaFai Lau /* bpf_map_free_id() must be called first */ 618bd5f5f4eSMartin KaFai Lau bpf_map_free_id(map, do_idr_lock); 61978958fcaSMartin KaFai Lau btf_put(map->btf); 62099c55f7dSAlexei Starovoitov INIT_WORK(&map->work, bpf_map_free_deferred); 62199c55f7dSAlexei Starovoitov schedule_work(&map->work); 62299c55f7dSAlexei Starovoitov } 62399c55f7dSAlexei Starovoitov } 62499c55f7dSAlexei Starovoitov 625bd5f5f4eSMartin KaFai Lau void bpf_map_put(struct bpf_map *map) 626bd5f5f4eSMartin KaFai Lau { 627bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, true); 628bd5f5f4eSMartin KaFai Lau } 629630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_put); 630bd5f5f4eSMartin KaFai Lau 631c9da161cSDaniel Borkmann void bpf_map_put_with_uref(struct bpf_map *map) 632c9da161cSDaniel Borkmann { 633c9da161cSDaniel Borkmann bpf_map_put_uref(map); 634c9da161cSDaniel Borkmann bpf_map_put(map); 635c9da161cSDaniel Borkmann } 636c9da161cSDaniel Borkmann 63799c55f7dSAlexei Starovoitov static int bpf_map_release(struct inode *inode, struct file *filp) 63899c55f7dSAlexei Starovoitov { 63961d1b6a4SDaniel Borkmann struct bpf_map *map = filp->private_data; 64061d1b6a4SDaniel Borkmann 64161d1b6a4SDaniel Borkmann if (map->ops->map_release) 64261d1b6a4SDaniel Borkmann map->ops->map_release(map, filp); 64361d1b6a4SDaniel Borkmann 64461d1b6a4SDaniel Borkmann bpf_map_put_with_uref(map); 64599c55f7dSAlexei Starovoitov return 0; 64699c55f7dSAlexei Starovoitov } 64799c55f7dSAlexei Starovoitov 64887df15deSDaniel Borkmann static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f) 64987df15deSDaniel Borkmann { 65087df15deSDaniel Borkmann fmode_t mode = f.file->f_mode; 65187df15deSDaniel Borkmann 65287df15deSDaniel Borkmann /* Our file permissions may have been overridden by global 65387df15deSDaniel Borkmann * map permissions facing syscall side. 65487df15deSDaniel Borkmann */ 65587df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) 65687df15deSDaniel Borkmann mode &= ~FMODE_CAN_WRITE; 65787df15deSDaniel Borkmann return mode; 65887df15deSDaniel Borkmann } 65987df15deSDaniel Borkmann 660f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 66180ee81e0SRoman Gushchin /* Provides an approximation of the map's memory footprint. 66280ee81e0SRoman Gushchin * Used only to provide a backward compatibility and display 66380ee81e0SRoman Gushchin * a reasonable "memlock" info. 66480ee81e0SRoman Gushchin */ 66580ee81e0SRoman Gushchin static unsigned long bpf_map_memory_footprint(const struct bpf_map *map) 66680ee81e0SRoman Gushchin { 66780ee81e0SRoman Gushchin unsigned long size; 66880ee81e0SRoman Gushchin 66980ee81e0SRoman Gushchin size = round_up(map->key_size + bpf_map_value_size(map), 8); 67080ee81e0SRoman Gushchin 67180ee81e0SRoman Gushchin return round_up(map->max_entries * size, PAGE_SIZE); 67280ee81e0SRoman Gushchin } 67380ee81e0SRoman Gushchin 674f99bf205SDaniel Borkmann static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 675f99bf205SDaniel Borkmann { 676f45d5b6cSToke Hoiland-Jorgensen struct bpf_map *map = filp->private_data; 6772beee5f5SDaniel Borkmann u32 type = 0, jited = 0; 67821116b70SDaniel Borkmann 679f45d5b6cSToke Hoiland-Jorgensen if (map_type_contains_progs(map)) { 680f45d5b6cSToke Hoiland-Jorgensen spin_lock(&map->owner.lock); 681f45d5b6cSToke Hoiland-Jorgensen type = map->owner.type; 682f45d5b6cSToke Hoiland-Jorgensen jited = map->owner.jited; 683f45d5b6cSToke Hoiland-Jorgensen spin_unlock(&map->owner.lock); 68421116b70SDaniel Borkmann } 685f99bf205SDaniel Borkmann 686f99bf205SDaniel Borkmann seq_printf(m, 687f99bf205SDaniel Borkmann "map_type:\t%u\n" 688f99bf205SDaniel Borkmann "key_size:\t%u\n" 689f99bf205SDaniel Borkmann "value_size:\t%u\n" 690322cea2fSDaniel Borkmann "max_entries:\t%u\n" 69121116b70SDaniel Borkmann "map_flags:\t%#x\n" 6929330986cSJoanne Koong "map_extra:\t%#llx\n" 69380ee81e0SRoman Gushchin "memlock:\t%lu\n" 69487df15deSDaniel Borkmann "map_id:\t%u\n" 69587df15deSDaniel Borkmann "frozen:\t%u\n", 696f99bf205SDaniel Borkmann map->map_type, 697f99bf205SDaniel Borkmann map->key_size, 698f99bf205SDaniel Borkmann map->value_size, 699322cea2fSDaniel Borkmann map->max_entries, 70021116b70SDaniel Borkmann map->map_flags, 7019330986cSJoanne Koong (unsigned long long)map->map_extra, 70280ee81e0SRoman Gushchin bpf_map_memory_footprint(map), 70387df15deSDaniel Borkmann map->id, 70487df15deSDaniel Borkmann READ_ONCE(map->frozen)); 7052beee5f5SDaniel Borkmann if (type) { 7062beee5f5SDaniel Borkmann seq_printf(m, "owner_prog_type:\t%u\n", type); 7072beee5f5SDaniel Borkmann seq_printf(m, "owner_jited:\t%u\n", jited); 7089780c0abSDaniel Borkmann } 709f99bf205SDaniel Borkmann } 710f99bf205SDaniel Borkmann #endif 711f99bf205SDaniel Borkmann 7126e71b04aSChenbo Feng static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, 7136e71b04aSChenbo Feng loff_t *ppos) 7146e71b04aSChenbo Feng { 7156e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 7166e71b04aSChenbo Feng * f_mode with FMODE_CAN_READ. 7176e71b04aSChenbo Feng */ 7186e71b04aSChenbo Feng return -EINVAL; 7196e71b04aSChenbo Feng } 7206e71b04aSChenbo Feng 7216e71b04aSChenbo Feng static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, 7226e71b04aSChenbo Feng size_t siz, loff_t *ppos) 7236e71b04aSChenbo Feng { 7246e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 7256e71b04aSChenbo Feng * f_mode with FMODE_CAN_WRITE. 7266e71b04aSChenbo Feng */ 7276e71b04aSChenbo Feng return -EINVAL; 7286e71b04aSChenbo Feng } 7296e71b04aSChenbo Feng 730fc970227SAndrii Nakryiko /* called for any extra memory-mapped regions (except initial) */ 731fc970227SAndrii Nakryiko static void bpf_map_mmap_open(struct vm_area_struct *vma) 732fc970227SAndrii Nakryiko { 733fc970227SAndrii Nakryiko struct bpf_map *map = vma->vm_file->private_data; 734fc970227SAndrii Nakryiko 735353050beSDaniel Borkmann if (vma->vm_flags & VM_MAYWRITE) 736353050beSDaniel Borkmann bpf_map_write_active_inc(map); 737fc970227SAndrii Nakryiko } 738fc970227SAndrii Nakryiko 739fc970227SAndrii Nakryiko /* called for all unmapped memory region (including initial) */ 740fc970227SAndrii Nakryiko static void bpf_map_mmap_close(struct vm_area_struct *vma) 741fc970227SAndrii Nakryiko { 742fc970227SAndrii Nakryiko struct bpf_map *map = vma->vm_file->private_data; 743fc970227SAndrii Nakryiko 744353050beSDaniel Borkmann if (vma->vm_flags & VM_MAYWRITE) 745353050beSDaniel Borkmann bpf_map_write_active_dec(map); 746fc970227SAndrii Nakryiko } 747fc970227SAndrii Nakryiko 748fc970227SAndrii Nakryiko static const struct vm_operations_struct bpf_map_default_vmops = { 749fc970227SAndrii Nakryiko .open = bpf_map_mmap_open, 750fc970227SAndrii Nakryiko .close = bpf_map_mmap_close, 751fc970227SAndrii Nakryiko }; 752fc970227SAndrii Nakryiko 753fc970227SAndrii Nakryiko static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma) 754fc970227SAndrii Nakryiko { 755fc970227SAndrii Nakryiko struct bpf_map *map = filp->private_data; 756fc970227SAndrii Nakryiko int err; 757fc970227SAndrii Nakryiko 75868134668SAlexei Starovoitov if (!map->ops->map_mmap || map_value_has_spin_lock(map) || 75961df10c7SKumar Kartikeya Dwivedi map_value_has_timer(map) || map_value_has_kptrs(map)) 760fc970227SAndrii Nakryiko return -ENOTSUPP; 761fc970227SAndrii Nakryiko 762fc970227SAndrii Nakryiko if (!(vma->vm_flags & VM_SHARED)) 763fc970227SAndrii Nakryiko return -EINVAL; 764fc970227SAndrii Nakryiko 765fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 766fc970227SAndrii Nakryiko 767dfeb376dSAndrii Nakryiko if (vma->vm_flags & VM_WRITE) { 768dfeb376dSAndrii Nakryiko if (map->frozen) { 769fc970227SAndrii Nakryiko err = -EPERM; 770fc970227SAndrii Nakryiko goto out; 771fc970227SAndrii Nakryiko } 772dfeb376dSAndrii Nakryiko /* map is meant to be read-only, so do not allow mapping as 773dfeb376dSAndrii Nakryiko * writable, because it's possible to leak a writable page 774dfeb376dSAndrii Nakryiko * reference and allows user-space to still modify it after 775dfeb376dSAndrii Nakryiko * freezing, while verifier will assume contents do not change 776dfeb376dSAndrii Nakryiko */ 777dfeb376dSAndrii Nakryiko if (map->map_flags & BPF_F_RDONLY_PROG) { 778dfeb376dSAndrii Nakryiko err = -EACCES; 779dfeb376dSAndrii Nakryiko goto out; 780dfeb376dSAndrii Nakryiko } 781dfeb376dSAndrii Nakryiko } 782fc970227SAndrii Nakryiko 783fc970227SAndrii Nakryiko /* set default open/close callbacks */ 784fc970227SAndrii Nakryiko vma->vm_ops = &bpf_map_default_vmops; 785fc970227SAndrii Nakryiko vma->vm_private_data = map; 7861f6cb19bSAndrii Nakryiko vma->vm_flags &= ~VM_MAYEXEC; 7871f6cb19bSAndrii Nakryiko if (!(vma->vm_flags & VM_WRITE)) 7881f6cb19bSAndrii Nakryiko /* disallow re-mapping with PROT_WRITE */ 7891f6cb19bSAndrii Nakryiko vma->vm_flags &= ~VM_MAYWRITE; 790fc970227SAndrii Nakryiko 791fc970227SAndrii Nakryiko err = map->ops->map_mmap(map, vma); 792fc970227SAndrii Nakryiko if (err) 793fc970227SAndrii Nakryiko goto out; 794fc970227SAndrii Nakryiko 7951f6cb19bSAndrii Nakryiko if (vma->vm_flags & VM_MAYWRITE) 796353050beSDaniel Borkmann bpf_map_write_active_inc(map); 797fc970227SAndrii Nakryiko out: 798fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 799fc970227SAndrii Nakryiko return err; 800fc970227SAndrii Nakryiko } 801fc970227SAndrii Nakryiko 802457f4436SAndrii Nakryiko static __poll_t bpf_map_poll(struct file *filp, struct poll_table_struct *pts) 803457f4436SAndrii Nakryiko { 804457f4436SAndrii Nakryiko struct bpf_map *map = filp->private_data; 805457f4436SAndrii Nakryiko 806457f4436SAndrii Nakryiko if (map->ops->map_poll) 807457f4436SAndrii Nakryiko return map->ops->map_poll(map, filp, pts); 808457f4436SAndrii Nakryiko 809457f4436SAndrii Nakryiko return EPOLLERR; 810457f4436SAndrii Nakryiko } 811457f4436SAndrii Nakryiko 812f66e448cSChenbo Feng const struct file_operations bpf_map_fops = { 813f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 814f99bf205SDaniel Borkmann .show_fdinfo = bpf_map_show_fdinfo, 815f99bf205SDaniel Borkmann #endif 81699c55f7dSAlexei Starovoitov .release = bpf_map_release, 8176e71b04aSChenbo Feng .read = bpf_dummy_read, 8186e71b04aSChenbo Feng .write = bpf_dummy_write, 819fc970227SAndrii Nakryiko .mmap = bpf_map_mmap, 820457f4436SAndrii Nakryiko .poll = bpf_map_poll, 82199c55f7dSAlexei Starovoitov }; 82299c55f7dSAlexei Starovoitov 8236e71b04aSChenbo Feng int bpf_map_new_fd(struct bpf_map *map, int flags) 824aa79781bSDaniel Borkmann { 825afdb09c7SChenbo Feng int ret; 826afdb09c7SChenbo Feng 827afdb09c7SChenbo Feng ret = security_bpf_map(map, OPEN_FMODE(flags)); 828afdb09c7SChenbo Feng if (ret < 0) 829afdb09c7SChenbo Feng return ret; 830afdb09c7SChenbo Feng 831aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 8326e71b04aSChenbo Feng flags | O_CLOEXEC); 8336e71b04aSChenbo Feng } 8346e71b04aSChenbo Feng 8356e71b04aSChenbo Feng int bpf_get_file_flag(int flags) 8366e71b04aSChenbo Feng { 8376e71b04aSChenbo Feng if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) 8386e71b04aSChenbo Feng return -EINVAL; 8396e71b04aSChenbo Feng if (flags & BPF_F_RDONLY) 8406e71b04aSChenbo Feng return O_RDONLY; 8416e71b04aSChenbo Feng if (flags & BPF_F_WRONLY) 8426e71b04aSChenbo Feng return O_WRONLY; 8436e71b04aSChenbo Feng return O_RDWR; 844aa79781bSDaniel Borkmann } 845aa79781bSDaniel Borkmann 84699c55f7dSAlexei Starovoitov /* helper macro to check that unused fields 'union bpf_attr' are zero */ 84799c55f7dSAlexei Starovoitov #define CHECK_ATTR(CMD) \ 84899c55f7dSAlexei Starovoitov memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 84999c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD), 0, \ 85099c55f7dSAlexei Starovoitov sizeof(*attr) - \ 85199c55f7dSAlexei Starovoitov offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 85299c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD)) != NULL 85399c55f7dSAlexei Starovoitov 8548e7ae251SMartin KaFai Lau /* dst and src must have at least "size" number of bytes. 8558e7ae251SMartin KaFai Lau * Return strlen on success and < 0 on error. 856cb4d2b3fSMartin KaFai Lau */ 8578e7ae251SMartin KaFai Lau int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size) 858cb4d2b3fSMartin KaFai Lau { 8598e7ae251SMartin KaFai Lau const char *end = src + size; 8608e7ae251SMartin KaFai Lau const char *orig_src = src; 861cb4d2b3fSMartin KaFai Lau 8628e7ae251SMartin KaFai Lau memset(dst, 0, size); 8633e0ddc4fSDaniel Borkmann /* Copy all isalnum(), '_' and '.' chars. */ 864cb4d2b3fSMartin KaFai Lau while (src < end && *src) { 8653e0ddc4fSDaniel Borkmann if (!isalnum(*src) && 8663e0ddc4fSDaniel Borkmann *src != '_' && *src != '.') 867cb4d2b3fSMartin KaFai Lau return -EINVAL; 868cb4d2b3fSMartin KaFai Lau *dst++ = *src++; 869cb4d2b3fSMartin KaFai Lau } 870cb4d2b3fSMartin KaFai Lau 8718e7ae251SMartin KaFai Lau /* No '\0' found in "size" number of bytes */ 872cb4d2b3fSMartin KaFai Lau if (src == end) 873cb4d2b3fSMartin KaFai Lau return -EINVAL; 874cb4d2b3fSMartin KaFai Lau 8758e7ae251SMartin KaFai Lau return src - orig_src; 876cb4d2b3fSMartin KaFai Lau } 877cb4d2b3fSMartin KaFai Lau 878e8d2bec0SDaniel Borkmann int map_check_no_btf(const struct bpf_map *map, 8791b2b234bSRoman Gushchin const struct btf *btf, 880e8d2bec0SDaniel Borkmann const struct btf_type *key_type, 881e8d2bec0SDaniel Borkmann const struct btf_type *value_type) 882e8d2bec0SDaniel Borkmann { 883e8d2bec0SDaniel Borkmann return -ENOTSUPP; 884e8d2bec0SDaniel Borkmann } 885e8d2bec0SDaniel Borkmann 8864d7d7f69SKumar Kartikeya Dwivedi static int map_off_arr_cmp(const void *_a, const void *_b, const void *priv) 8874d7d7f69SKumar Kartikeya Dwivedi { 8884d7d7f69SKumar Kartikeya Dwivedi const u32 a = *(const u32 *)_a; 8894d7d7f69SKumar Kartikeya Dwivedi const u32 b = *(const u32 *)_b; 8904d7d7f69SKumar Kartikeya Dwivedi 8914d7d7f69SKumar Kartikeya Dwivedi if (a < b) 8924d7d7f69SKumar Kartikeya Dwivedi return -1; 8934d7d7f69SKumar Kartikeya Dwivedi else if (a > b) 8944d7d7f69SKumar Kartikeya Dwivedi return 1; 8954d7d7f69SKumar Kartikeya Dwivedi return 0; 8964d7d7f69SKumar Kartikeya Dwivedi } 8974d7d7f69SKumar Kartikeya Dwivedi 8984d7d7f69SKumar Kartikeya Dwivedi static void map_off_arr_swap(void *_a, void *_b, int size, const void *priv) 8994d7d7f69SKumar Kartikeya Dwivedi { 9004d7d7f69SKumar Kartikeya Dwivedi struct bpf_map *map = (struct bpf_map *)priv; 9014d7d7f69SKumar Kartikeya Dwivedi u32 *off_base = map->off_arr->field_off; 9024d7d7f69SKumar Kartikeya Dwivedi u32 *a = _a, *b = _b; 9034d7d7f69SKumar Kartikeya Dwivedi u8 *sz_a, *sz_b; 9044d7d7f69SKumar Kartikeya Dwivedi 9054d7d7f69SKumar Kartikeya Dwivedi sz_a = map->off_arr->field_sz + (a - off_base); 9064d7d7f69SKumar Kartikeya Dwivedi sz_b = map->off_arr->field_sz + (b - off_base); 9074d7d7f69SKumar Kartikeya Dwivedi 9084d7d7f69SKumar Kartikeya Dwivedi swap(*a, *b); 9094d7d7f69SKumar Kartikeya Dwivedi swap(*sz_a, *sz_b); 9104d7d7f69SKumar Kartikeya Dwivedi } 9114d7d7f69SKumar Kartikeya Dwivedi 9124d7d7f69SKumar Kartikeya Dwivedi static int bpf_map_alloc_off_arr(struct bpf_map *map) 9134d7d7f69SKumar Kartikeya Dwivedi { 9144d7d7f69SKumar Kartikeya Dwivedi bool has_spin_lock = map_value_has_spin_lock(map); 9154d7d7f69SKumar Kartikeya Dwivedi bool has_timer = map_value_has_timer(map); 9164d7d7f69SKumar Kartikeya Dwivedi bool has_kptrs = map_value_has_kptrs(map); 9174d7d7f69SKumar Kartikeya Dwivedi struct bpf_map_off_arr *off_arr; 9184d7d7f69SKumar Kartikeya Dwivedi u32 i; 9194d7d7f69SKumar Kartikeya Dwivedi 9204d7d7f69SKumar Kartikeya Dwivedi if (!has_spin_lock && !has_timer && !has_kptrs) { 9214d7d7f69SKumar Kartikeya Dwivedi map->off_arr = NULL; 9224d7d7f69SKumar Kartikeya Dwivedi return 0; 9234d7d7f69SKumar Kartikeya Dwivedi } 9244d7d7f69SKumar Kartikeya Dwivedi 9254d7d7f69SKumar Kartikeya Dwivedi off_arr = kmalloc(sizeof(*map->off_arr), GFP_KERNEL | __GFP_NOWARN); 9264d7d7f69SKumar Kartikeya Dwivedi if (!off_arr) 9274d7d7f69SKumar Kartikeya Dwivedi return -ENOMEM; 9284d7d7f69SKumar Kartikeya Dwivedi map->off_arr = off_arr; 9294d7d7f69SKumar Kartikeya Dwivedi 9304d7d7f69SKumar Kartikeya Dwivedi off_arr->cnt = 0; 9314d7d7f69SKumar Kartikeya Dwivedi if (has_spin_lock) { 9324d7d7f69SKumar Kartikeya Dwivedi i = off_arr->cnt; 9334d7d7f69SKumar Kartikeya Dwivedi 9344d7d7f69SKumar Kartikeya Dwivedi off_arr->field_off[i] = map->spin_lock_off; 9354d7d7f69SKumar Kartikeya Dwivedi off_arr->field_sz[i] = sizeof(struct bpf_spin_lock); 9364d7d7f69SKumar Kartikeya Dwivedi off_arr->cnt++; 9374d7d7f69SKumar Kartikeya Dwivedi } 9384d7d7f69SKumar Kartikeya Dwivedi if (has_timer) { 9394d7d7f69SKumar Kartikeya Dwivedi i = off_arr->cnt; 9404d7d7f69SKumar Kartikeya Dwivedi 9414d7d7f69SKumar Kartikeya Dwivedi off_arr->field_off[i] = map->timer_off; 9424d7d7f69SKumar Kartikeya Dwivedi off_arr->field_sz[i] = sizeof(struct bpf_timer); 9434d7d7f69SKumar Kartikeya Dwivedi off_arr->cnt++; 9444d7d7f69SKumar Kartikeya Dwivedi } 9454d7d7f69SKumar Kartikeya Dwivedi if (has_kptrs) { 9464d7d7f69SKumar Kartikeya Dwivedi struct bpf_map_value_off *tab = map->kptr_off_tab; 9474d7d7f69SKumar Kartikeya Dwivedi u32 *off = &off_arr->field_off[off_arr->cnt]; 9484d7d7f69SKumar Kartikeya Dwivedi u8 *sz = &off_arr->field_sz[off_arr->cnt]; 9494d7d7f69SKumar Kartikeya Dwivedi 9504d7d7f69SKumar Kartikeya Dwivedi for (i = 0; i < tab->nr_off; i++) { 9514d7d7f69SKumar Kartikeya Dwivedi *off++ = tab->off[i].offset; 9524d7d7f69SKumar Kartikeya Dwivedi *sz++ = sizeof(u64); 9534d7d7f69SKumar Kartikeya Dwivedi } 9544d7d7f69SKumar Kartikeya Dwivedi off_arr->cnt += tab->nr_off; 9554d7d7f69SKumar Kartikeya Dwivedi } 9564d7d7f69SKumar Kartikeya Dwivedi 9574d7d7f69SKumar Kartikeya Dwivedi if (off_arr->cnt == 1) 9584d7d7f69SKumar Kartikeya Dwivedi return 0; 9594d7d7f69SKumar Kartikeya Dwivedi sort_r(off_arr->field_off, off_arr->cnt, sizeof(off_arr->field_off[0]), 9604d7d7f69SKumar Kartikeya Dwivedi map_off_arr_cmp, map_off_arr_swap, map); 9614d7d7f69SKumar Kartikeya Dwivedi return 0; 9624d7d7f69SKumar Kartikeya Dwivedi } 9634d7d7f69SKumar Kartikeya Dwivedi 964d83525caSAlexei Starovoitov static int map_check_btf(struct bpf_map *map, const struct btf *btf, 965e8d2bec0SDaniel Borkmann u32 btf_key_id, u32 btf_value_id) 966e8d2bec0SDaniel Borkmann { 967e8d2bec0SDaniel Borkmann const struct btf_type *key_type, *value_type; 968e8d2bec0SDaniel Borkmann u32 key_size, value_size; 969e8d2bec0SDaniel Borkmann int ret = 0; 970e8d2bec0SDaniel Borkmann 9712824ecb7SDaniel Borkmann /* Some maps allow key to be unspecified. */ 9722824ecb7SDaniel Borkmann if (btf_key_id) { 973e8d2bec0SDaniel Borkmann key_type = btf_type_id_size(btf, &btf_key_id, &key_size); 974e8d2bec0SDaniel Borkmann if (!key_type || key_size != map->key_size) 975e8d2bec0SDaniel Borkmann return -EINVAL; 9762824ecb7SDaniel Borkmann } else { 9772824ecb7SDaniel Borkmann key_type = btf_type_by_id(btf, 0); 9782824ecb7SDaniel Borkmann if (!map->ops->map_check_btf) 9792824ecb7SDaniel Borkmann return -EINVAL; 9802824ecb7SDaniel Borkmann } 981e8d2bec0SDaniel Borkmann 982e8d2bec0SDaniel Borkmann value_type = btf_type_id_size(btf, &btf_value_id, &value_size); 983e8d2bec0SDaniel Borkmann if (!value_type || value_size != map->value_size) 984e8d2bec0SDaniel Borkmann return -EINVAL; 985e8d2bec0SDaniel Borkmann 986d83525caSAlexei Starovoitov map->spin_lock_off = btf_find_spin_lock(btf, value_type); 987d83525caSAlexei Starovoitov 988d83525caSAlexei Starovoitov if (map_value_has_spin_lock(map)) { 989591fe988SDaniel Borkmann if (map->map_flags & BPF_F_RDONLY_PROG) 990591fe988SDaniel Borkmann return -EACCES; 991d83525caSAlexei Starovoitov if (map->map_type != BPF_MAP_TYPE_HASH && 992e16d2f1aSAlexei Starovoitov map->map_type != BPF_MAP_TYPE_ARRAY && 9936ac99e8fSMartin KaFai Lau map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 9948ea63684SKP Singh map->map_type != BPF_MAP_TYPE_SK_STORAGE && 9954cf1bc1fSKP Singh map->map_type != BPF_MAP_TYPE_INODE_STORAGE && 9964cf1bc1fSKP Singh map->map_type != BPF_MAP_TYPE_TASK_STORAGE) 997d83525caSAlexei Starovoitov return -ENOTSUPP; 998d83525caSAlexei Starovoitov if (map->spin_lock_off + sizeof(struct bpf_spin_lock) > 999d83525caSAlexei Starovoitov map->value_size) { 1000d83525caSAlexei Starovoitov WARN_ONCE(1, 1001d83525caSAlexei Starovoitov "verifier bug spin_lock_off %d value_size %d\n", 1002d83525caSAlexei Starovoitov map->spin_lock_off, map->value_size); 1003d83525caSAlexei Starovoitov return -EFAULT; 1004d83525caSAlexei Starovoitov } 1005d83525caSAlexei Starovoitov } 1006d83525caSAlexei Starovoitov 100768134668SAlexei Starovoitov map->timer_off = btf_find_timer(btf, value_type); 100868134668SAlexei Starovoitov if (map_value_has_timer(map)) { 100968134668SAlexei Starovoitov if (map->map_flags & BPF_F_RDONLY_PROG) 101068134668SAlexei Starovoitov return -EACCES; 101168134668SAlexei Starovoitov if (map->map_type != BPF_MAP_TYPE_HASH && 101268134668SAlexei Starovoitov map->map_type != BPF_MAP_TYPE_LRU_HASH && 101368134668SAlexei Starovoitov map->map_type != BPF_MAP_TYPE_ARRAY) 101468134668SAlexei Starovoitov return -EOPNOTSUPP; 101568134668SAlexei Starovoitov } 101668134668SAlexei Starovoitov 101761df10c7SKumar Kartikeya Dwivedi map->kptr_off_tab = btf_parse_kptrs(btf, value_type); 101861df10c7SKumar Kartikeya Dwivedi if (map_value_has_kptrs(map)) { 101961df10c7SKumar Kartikeya Dwivedi if (!bpf_capable()) { 102061df10c7SKumar Kartikeya Dwivedi ret = -EPERM; 102161df10c7SKumar Kartikeya Dwivedi goto free_map_tab; 102261df10c7SKumar Kartikeya Dwivedi } 102361df10c7SKumar Kartikeya Dwivedi if (map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) { 102461df10c7SKumar Kartikeya Dwivedi ret = -EACCES; 102561df10c7SKumar Kartikeya Dwivedi goto free_map_tab; 102661df10c7SKumar Kartikeya Dwivedi } 102761df10c7SKumar Kartikeya Dwivedi if (map->map_type != BPF_MAP_TYPE_HASH && 102861df10c7SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_LRU_HASH && 102961df10c7SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_ARRAY) { 103061df10c7SKumar Kartikeya Dwivedi ret = -EOPNOTSUPP; 103161df10c7SKumar Kartikeya Dwivedi goto free_map_tab; 103261df10c7SKumar Kartikeya Dwivedi } 103361df10c7SKumar Kartikeya Dwivedi } 1034e8d2bec0SDaniel Borkmann 103561df10c7SKumar Kartikeya Dwivedi if (map->ops->map_check_btf) { 103661df10c7SKumar Kartikeya Dwivedi ret = map->ops->map_check_btf(map, btf, key_type, value_type); 103761df10c7SKumar Kartikeya Dwivedi if (ret < 0) 103861df10c7SKumar Kartikeya Dwivedi goto free_map_tab; 103961df10c7SKumar Kartikeya Dwivedi } 104061df10c7SKumar Kartikeya Dwivedi 104161df10c7SKumar Kartikeya Dwivedi return ret; 104261df10c7SKumar Kartikeya Dwivedi free_map_tab: 104361df10c7SKumar Kartikeya Dwivedi bpf_map_free_kptr_off_tab(map); 1044e8d2bec0SDaniel Borkmann return ret; 1045e8d2bec0SDaniel Borkmann } 1046e8d2bec0SDaniel Borkmann 10479330986cSJoanne Koong #define BPF_MAP_CREATE_LAST_FIELD map_extra 104899c55f7dSAlexei Starovoitov /* called via syscall */ 104999c55f7dSAlexei Starovoitov static int map_create(union bpf_attr *attr) 105099c55f7dSAlexei Starovoitov { 105196eabe7aSMartin KaFai Lau int numa_node = bpf_map_attr_numa_node(attr); 105299c55f7dSAlexei Starovoitov struct bpf_map *map; 10536e71b04aSChenbo Feng int f_flags; 105499c55f7dSAlexei Starovoitov int err; 105599c55f7dSAlexei Starovoitov 105699c55f7dSAlexei Starovoitov err = CHECK_ATTR(BPF_MAP_CREATE); 105799c55f7dSAlexei Starovoitov if (err) 105899c55f7dSAlexei Starovoitov return -EINVAL; 105999c55f7dSAlexei Starovoitov 106085d33df3SMartin KaFai Lau if (attr->btf_vmlinux_value_type_id) { 106185d33df3SMartin KaFai Lau if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS || 106285d33df3SMartin KaFai Lau attr->btf_key_type_id || attr->btf_value_type_id) 106385d33df3SMartin KaFai Lau return -EINVAL; 106485d33df3SMartin KaFai Lau } else if (attr->btf_key_type_id && !attr->btf_value_type_id) { 106585d33df3SMartin KaFai Lau return -EINVAL; 106685d33df3SMartin KaFai Lau } 106785d33df3SMartin KaFai Lau 10689330986cSJoanne Koong if (attr->map_type != BPF_MAP_TYPE_BLOOM_FILTER && 10699330986cSJoanne Koong attr->map_extra != 0) 10709330986cSJoanne Koong return -EINVAL; 10719330986cSJoanne Koong 10726e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->map_flags); 10736e71b04aSChenbo Feng if (f_flags < 0) 10746e71b04aSChenbo Feng return f_flags; 10756e71b04aSChenbo Feng 107696eabe7aSMartin KaFai Lau if (numa_node != NUMA_NO_NODE && 107796e5ae4eSEric Dumazet ((unsigned int)numa_node >= nr_node_ids || 107896e5ae4eSEric Dumazet !node_online(numa_node))) 107996eabe7aSMartin KaFai Lau return -EINVAL; 108096eabe7aSMartin KaFai Lau 108199c55f7dSAlexei Starovoitov /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 108299c55f7dSAlexei Starovoitov map = find_and_alloc_map(attr); 108399c55f7dSAlexei Starovoitov if (IS_ERR(map)) 108499c55f7dSAlexei Starovoitov return PTR_ERR(map); 108599c55f7dSAlexei Starovoitov 10868e7ae251SMartin KaFai Lau err = bpf_obj_name_cpy(map->name, attr->map_name, 10878e7ae251SMartin KaFai Lau sizeof(attr->map_name)); 10888e7ae251SMartin KaFai Lau if (err < 0) 1089b936ca64SRoman Gushchin goto free_map; 1090ad5b177bSMartin KaFai Lau 10911e0bd5a0SAndrii Nakryiko atomic64_set(&map->refcnt, 1); 10921e0bd5a0SAndrii Nakryiko atomic64_set(&map->usercnt, 1); 1093fc970227SAndrii Nakryiko mutex_init(&map->freeze_mutex); 1094f45d5b6cSToke Hoiland-Jorgensen spin_lock_init(&map->owner.lock); 109599c55f7dSAlexei Starovoitov 109685d33df3SMartin KaFai Lau map->spin_lock_off = -EINVAL; 109768134668SAlexei Starovoitov map->timer_off = -EINVAL; 109885d33df3SMartin KaFai Lau if (attr->btf_key_type_id || attr->btf_value_type_id || 109985d33df3SMartin KaFai Lau /* Even the map's value is a kernel's struct, 110085d33df3SMartin KaFai Lau * the bpf_prog.o must have BTF to begin with 110185d33df3SMartin KaFai Lau * to figure out the corresponding kernel's 110285d33df3SMartin KaFai Lau * counter part. Thus, attr->btf_fd has 110385d33df3SMartin KaFai Lau * to be valid also. 110485d33df3SMartin KaFai Lau */ 110585d33df3SMartin KaFai Lau attr->btf_vmlinux_value_type_id) { 1106a26ca7c9SMartin KaFai Lau struct btf *btf; 1107a26ca7c9SMartin KaFai Lau 1108a26ca7c9SMartin KaFai Lau btf = btf_get_by_fd(attr->btf_fd); 1109a26ca7c9SMartin KaFai Lau if (IS_ERR(btf)) { 1110a26ca7c9SMartin KaFai Lau err = PTR_ERR(btf); 1111b936ca64SRoman Gushchin goto free_map; 1112a26ca7c9SMartin KaFai Lau } 1113350a5c4dSAlexei Starovoitov if (btf_is_kernel(btf)) { 1114350a5c4dSAlexei Starovoitov btf_put(btf); 1115350a5c4dSAlexei Starovoitov err = -EACCES; 1116350a5c4dSAlexei Starovoitov goto free_map; 1117350a5c4dSAlexei Starovoitov } 111885d33df3SMartin KaFai Lau map->btf = btf; 1119a26ca7c9SMartin KaFai Lau 112085d33df3SMartin KaFai Lau if (attr->btf_value_type_id) { 1121e8d2bec0SDaniel Borkmann err = map_check_btf(map, btf, attr->btf_key_type_id, 11229b2cf328SMartin KaFai Lau attr->btf_value_type_id); 112385d33df3SMartin KaFai Lau if (err) 1124b936ca64SRoman Gushchin goto free_map; 1125a26ca7c9SMartin KaFai Lau } 1126a26ca7c9SMartin KaFai Lau 11279b2cf328SMartin KaFai Lau map->btf_key_type_id = attr->btf_key_type_id; 11289b2cf328SMartin KaFai Lau map->btf_value_type_id = attr->btf_value_type_id; 112985d33df3SMartin KaFai Lau map->btf_vmlinux_value_type_id = 113085d33df3SMartin KaFai Lau attr->btf_vmlinux_value_type_id; 1131a26ca7c9SMartin KaFai Lau } 1132a26ca7c9SMartin KaFai Lau 11334d7d7f69SKumar Kartikeya Dwivedi err = bpf_map_alloc_off_arr(map); 1134aaac3ba9SAlexei Starovoitov if (err) 1135b936ca64SRoman Gushchin goto free_map; 1136afdb09c7SChenbo Feng 11374d7d7f69SKumar Kartikeya Dwivedi err = security_bpf_map_alloc(map); 11384d7d7f69SKumar Kartikeya Dwivedi if (err) 11394d7d7f69SKumar Kartikeya Dwivedi goto free_map_off_arr; 11404d7d7f69SKumar Kartikeya Dwivedi 1141f3f1c054SMartin KaFai Lau err = bpf_map_alloc_id(map); 1142f3f1c054SMartin KaFai Lau if (err) 1143b936ca64SRoman Gushchin goto free_map_sec; 1144f3f1c054SMartin KaFai Lau 114548edc1f7SRoman Gushchin bpf_map_save_memcg(map); 114648edc1f7SRoman Gushchin 11476e71b04aSChenbo Feng err = bpf_map_new_fd(map, f_flags); 1148bd5f5f4eSMartin KaFai Lau if (err < 0) { 1149bd5f5f4eSMartin KaFai Lau /* failed to allocate fd. 1150352d20d6SPeng Sun * bpf_map_put_with_uref() is needed because the above 1151bd5f5f4eSMartin KaFai Lau * bpf_map_alloc_id() has published the map 1152bd5f5f4eSMartin KaFai Lau * to the userspace and the userspace may 1153bd5f5f4eSMartin KaFai Lau * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 1154bd5f5f4eSMartin KaFai Lau */ 1155352d20d6SPeng Sun bpf_map_put_with_uref(map); 1156bd5f5f4eSMartin KaFai Lau return err; 1157bd5f5f4eSMartin KaFai Lau } 115899c55f7dSAlexei Starovoitov 115999c55f7dSAlexei Starovoitov return err; 116099c55f7dSAlexei Starovoitov 1161afdb09c7SChenbo Feng free_map_sec: 1162afdb09c7SChenbo Feng security_bpf_map_free(map); 11634d7d7f69SKumar Kartikeya Dwivedi free_map_off_arr: 11644d7d7f69SKumar Kartikeya Dwivedi kfree(map->off_arr); 1165b936ca64SRoman Gushchin free_map: 1166a26ca7c9SMartin KaFai Lau btf_put(map->btf); 116799c55f7dSAlexei Starovoitov map->ops->map_free(map); 116899c55f7dSAlexei Starovoitov return err; 116999c55f7dSAlexei Starovoitov } 117099c55f7dSAlexei Starovoitov 1171db20fd2bSAlexei Starovoitov /* if error is returned, fd is released. 1172db20fd2bSAlexei Starovoitov * On success caller should complete fd access with matching fdput() 1173db20fd2bSAlexei Starovoitov */ 1174c2101297SDaniel Borkmann struct bpf_map *__bpf_map_get(struct fd f) 1175db20fd2bSAlexei Starovoitov { 1176db20fd2bSAlexei Starovoitov if (!f.file) 1177db20fd2bSAlexei Starovoitov return ERR_PTR(-EBADF); 1178db20fd2bSAlexei Starovoitov if (f.file->f_op != &bpf_map_fops) { 1179db20fd2bSAlexei Starovoitov fdput(f); 1180db20fd2bSAlexei Starovoitov return ERR_PTR(-EINVAL); 1181db20fd2bSAlexei Starovoitov } 1182db20fd2bSAlexei Starovoitov 1183c2101297SDaniel Borkmann return f.file->private_data; 1184c2101297SDaniel Borkmann } 1185c2101297SDaniel Borkmann 11861e0bd5a0SAndrii Nakryiko void bpf_map_inc(struct bpf_map *map) 1187c9da161cSDaniel Borkmann { 11881e0bd5a0SAndrii Nakryiko atomic64_inc(&map->refcnt); 1189c9da161cSDaniel Borkmann } 1190630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_inc); 1191c9da161cSDaniel Borkmann 11921e0bd5a0SAndrii Nakryiko void bpf_map_inc_with_uref(struct bpf_map *map) 11931e0bd5a0SAndrii Nakryiko { 11941e0bd5a0SAndrii Nakryiko atomic64_inc(&map->refcnt); 11951e0bd5a0SAndrii Nakryiko atomic64_inc(&map->usercnt); 11961e0bd5a0SAndrii Nakryiko } 11971e0bd5a0SAndrii Nakryiko EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref); 11981e0bd5a0SAndrii Nakryiko 11991ed4d924SMartin KaFai Lau struct bpf_map *bpf_map_get(u32 ufd) 12001ed4d924SMartin KaFai Lau { 12011ed4d924SMartin KaFai Lau struct fd f = fdget(ufd); 12021ed4d924SMartin KaFai Lau struct bpf_map *map; 12031ed4d924SMartin KaFai Lau 12041ed4d924SMartin KaFai Lau map = __bpf_map_get(f); 12051ed4d924SMartin KaFai Lau if (IS_ERR(map)) 12061ed4d924SMartin KaFai Lau return map; 12071ed4d924SMartin KaFai Lau 12081ed4d924SMartin KaFai Lau bpf_map_inc(map); 12091ed4d924SMartin KaFai Lau fdput(f); 12101ed4d924SMartin KaFai Lau 12111ed4d924SMartin KaFai Lau return map; 12121ed4d924SMartin KaFai Lau } 1213b1d18a75SAlexei Starovoitov EXPORT_SYMBOL(bpf_map_get); 12141ed4d924SMartin KaFai Lau 1215c9da161cSDaniel Borkmann struct bpf_map *bpf_map_get_with_uref(u32 ufd) 1216c2101297SDaniel Borkmann { 1217c2101297SDaniel Borkmann struct fd f = fdget(ufd); 1218c2101297SDaniel Borkmann struct bpf_map *map; 1219c2101297SDaniel Borkmann 1220c2101297SDaniel Borkmann map = __bpf_map_get(f); 1221c2101297SDaniel Borkmann if (IS_ERR(map)) 1222c2101297SDaniel Borkmann return map; 1223c2101297SDaniel Borkmann 12241e0bd5a0SAndrii Nakryiko bpf_map_inc_with_uref(map); 1225c2101297SDaniel Borkmann fdput(f); 1226db20fd2bSAlexei Starovoitov 1227db20fd2bSAlexei Starovoitov return map; 1228db20fd2bSAlexei Starovoitov } 1229db20fd2bSAlexei Starovoitov 1230bd5f5f4eSMartin KaFai Lau /* map_idr_lock should have been held */ 12311e0bd5a0SAndrii Nakryiko static struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref) 1232bd5f5f4eSMartin KaFai Lau { 1233bd5f5f4eSMartin KaFai Lau int refold; 1234bd5f5f4eSMartin KaFai Lau 12351e0bd5a0SAndrii Nakryiko refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0); 1236bd5f5f4eSMartin KaFai Lau if (!refold) 1237bd5f5f4eSMartin KaFai Lau return ERR_PTR(-ENOENT); 1238bd5f5f4eSMartin KaFai Lau if (uref) 12391e0bd5a0SAndrii Nakryiko atomic64_inc(&map->usercnt); 1240bd5f5f4eSMartin KaFai Lau 1241bd5f5f4eSMartin KaFai Lau return map; 1242bd5f5f4eSMartin KaFai Lau } 1243bd5f5f4eSMartin KaFai Lau 12441e0bd5a0SAndrii Nakryiko struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map) 1245b0e4701cSStanislav Fomichev { 1246b0e4701cSStanislav Fomichev spin_lock_bh(&map_idr_lock); 12471e0bd5a0SAndrii Nakryiko map = __bpf_map_inc_not_zero(map, false); 1248b0e4701cSStanislav Fomichev spin_unlock_bh(&map_idr_lock); 1249b0e4701cSStanislav Fomichev 1250b0e4701cSStanislav Fomichev return map; 1251b0e4701cSStanislav Fomichev } 1252b0e4701cSStanislav Fomichev EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero); 1253b0e4701cSStanislav Fomichev 1254b8cdc051SAlexei Starovoitov int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 1255b8cdc051SAlexei Starovoitov { 1256b8cdc051SAlexei Starovoitov return -ENOTSUPP; 1257b8cdc051SAlexei Starovoitov } 1258b8cdc051SAlexei Starovoitov 1259c9d29f46SMauricio Vasquez B static void *__bpf_copy_key(void __user *ukey, u64 key_size) 1260c9d29f46SMauricio Vasquez B { 1261c9d29f46SMauricio Vasquez B if (key_size) 126244779a4bSStanislav Fomichev return vmemdup_user(ukey, key_size); 1263c9d29f46SMauricio Vasquez B 1264c9d29f46SMauricio Vasquez B if (ukey) 1265c9d29f46SMauricio Vasquez B return ERR_PTR(-EINVAL); 1266c9d29f46SMauricio Vasquez B 1267c9d29f46SMauricio Vasquez B return NULL; 1268c9d29f46SMauricio Vasquez B } 1269c9d29f46SMauricio Vasquez B 1270af2ac3e1SAlexei Starovoitov static void *___bpf_copy_key(bpfptr_t ukey, u64 key_size) 1271af2ac3e1SAlexei Starovoitov { 1272af2ac3e1SAlexei Starovoitov if (key_size) 127344779a4bSStanislav Fomichev return kvmemdup_bpfptr(ukey, key_size); 1274af2ac3e1SAlexei Starovoitov 1275af2ac3e1SAlexei Starovoitov if (!bpfptr_is_null(ukey)) 1276af2ac3e1SAlexei Starovoitov return ERR_PTR(-EINVAL); 1277af2ac3e1SAlexei Starovoitov 1278af2ac3e1SAlexei Starovoitov return NULL; 1279af2ac3e1SAlexei Starovoitov } 1280af2ac3e1SAlexei Starovoitov 1281db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 128296049f3aSAlexei Starovoitov #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags 1283db20fd2bSAlexei Starovoitov 1284db20fd2bSAlexei Starovoitov static int map_lookup_elem(union bpf_attr *attr) 1285db20fd2bSAlexei Starovoitov { 1286535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1287535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 1288db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1289db20fd2bSAlexei Starovoitov struct bpf_map *map; 129015c14a3dSBrian Vazquez void *key, *value; 129115a07b33SAlexei Starovoitov u32 value_size; 1292592867bfSDaniel Borkmann struct fd f; 1293db20fd2bSAlexei Starovoitov int err; 1294db20fd2bSAlexei Starovoitov 1295db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 1296db20fd2bSAlexei Starovoitov return -EINVAL; 1297db20fd2bSAlexei Starovoitov 129896049f3aSAlexei Starovoitov if (attr->flags & ~BPF_F_LOCK) 129996049f3aSAlexei Starovoitov return -EINVAL; 130096049f3aSAlexei Starovoitov 1301592867bfSDaniel Borkmann f = fdget(ufd); 1302c2101297SDaniel Borkmann map = __bpf_map_get(f); 1303db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1304db20fd2bSAlexei Starovoitov return PTR_ERR(map); 130587df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 13066e71b04aSChenbo Feng err = -EPERM; 13076e71b04aSChenbo Feng goto err_put; 13086e71b04aSChenbo Feng } 13096e71b04aSChenbo Feng 131096049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 131196049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 131296049f3aSAlexei Starovoitov err = -EINVAL; 131396049f3aSAlexei Starovoitov goto err_put; 131496049f3aSAlexei Starovoitov } 131596049f3aSAlexei Starovoitov 1316c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1317e4448ed8SAl Viro if (IS_ERR(key)) { 1318e4448ed8SAl Viro err = PTR_ERR(key); 1319db20fd2bSAlexei Starovoitov goto err_put; 1320e4448ed8SAl Viro } 1321db20fd2bSAlexei Starovoitov 132215c14a3dSBrian Vazquez value_size = bpf_map_value_size(map); 132315a07b33SAlexei Starovoitov 13248ebe667cSAlexei Starovoitov err = -ENOMEM; 1325f0dce1d9SStanislav Fomichev value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN); 1326db20fd2bSAlexei Starovoitov if (!value) 13278ebe667cSAlexei Starovoitov goto free_key; 13288ebe667cSAlexei Starovoitov 13299330986cSJoanne Koong if (map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) { 13309330986cSJoanne Koong if (copy_from_user(value, uvalue, value_size)) 13319330986cSJoanne Koong err = -EFAULT; 13329330986cSJoanne Koong else 13339330986cSJoanne Koong err = bpf_map_copy_value(map, key, value, attr->flags); 13349330986cSJoanne Koong goto free_value; 13359330986cSJoanne Koong } 13369330986cSJoanne Koong 133715c14a3dSBrian Vazquez err = bpf_map_copy_value(map, key, value, attr->flags); 133815a07b33SAlexei Starovoitov if (err) 13398ebe667cSAlexei Starovoitov goto free_value; 1340db20fd2bSAlexei Starovoitov 1341db20fd2bSAlexei Starovoitov err = -EFAULT; 134215a07b33SAlexei Starovoitov if (copy_to_user(uvalue, value, value_size) != 0) 13438ebe667cSAlexei Starovoitov goto free_value; 1344db20fd2bSAlexei Starovoitov 1345db20fd2bSAlexei Starovoitov err = 0; 1346db20fd2bSAlexei Starovoitov 13478ebe667cSAlexei Starovoitov free_value: 1348f0dce1d9SStanislav Fomichev kvfree(value); 1349db20fd2bSAlexei Starovoitov free_key: 135044779a4bSStanislav Fomichev kvfree(key); 1351db20fd2bSAlexei Starovoitov err_put: 1352db20fd2bSAlexei Starovoitov fdput(f); 1353db20fd2bSAlexei Starovoitov return err; 1354db20fd2bSAlexei Starovoitov } 1355db20fd2bSAlexei Starovoitov 13561ae80cf3SDaniel Colascione 13573274f520SAlexei Starovoitov #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 1358db20fd2bSAlexei Starovoitov 1359af2ac3e1SAlexei Starovoitov static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) 1360db20fd2bSAlexei Starovoitov { 1361af2ac3e1SAlexei Starovoitov bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel); 1362af2ac3e1SAlexei Starovoitov bpfptr_t uvalue = make_bpfptr(attr->value, uattr.is_kernel); 1363db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1364db20fd2bSAlexei Starovoitov struct bpf_map *map; 1365db20fd2bSAlexei Starovoitov void *key, *value; 136615a07b33SAlexei Starovoitov u32 value_size; 1367592867bfSDaniel Borkmann struct fd f; 1368db20fd2bSAlexei Starovoitov int err; 1369db20fd2bSAlexei Starovoitov 1370db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 1371db20fd2bSAlexei Starovoitov return -EINVAL; 1372db20fd2bSAlexei Starovoitov 1373592867bfSDaniel Borkmann f = fdget(ufd); 1374c2101297SDaniel Borkmann map = __bpf_map_get(f); 1375db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1376db20fd2bSAlexei Starovoitov return PTR_ERR(map); 1377353050beSDaniel Borkmann bpf_map_write_active_inc(map); 137887df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 13796e71b04aSChenbo Feng err = -EPERM; 13806e71b04aSChenbo Feng goto err_put; 13816e71b04aSChenbo Feng } 13826e71b04aSChenbo Feng 138396049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 138496049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 138596049f3aSAlexei Starovoitov err = -EINVAL; 138696049f3aSAlexei Starovoitov goto err_put; 138796049f3aSAlexei Starovoitov } 138896049f3aSAlexei Starovoitov 1389af2ac3e1SAlexei Starovoitov key = ___bpf_copy_key(ukey, map->key_size); 1390e4448ed8SAl Viro if (IS_ERR(key)) { 1391e4448ed8SAl Viro err = PTR_ERR(key); 1392db20fd2bSAlexei Starovoitov goto err_put; 1393e4448ed8SAl Viro } 1394db20fd2bSAlexei Starovoitov 1395f0dce1d9SStanislav Fomichev value_size = bpf_map_value_size(map); 139615a07b33SAlexei Starovoitov 1397db20fd2bSAlexei Starovoitov err = -ENOMEM; 1398f0dce1d9SStanislav Fomichev value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN); 1399db20fd2bSAlexei Starovoitov if (!value) 1400db20fd2bSAlexei Starovoitov goto free_key; 1401db20fd2bSAlexei Starovoitov 1402db20fd2bSAlexei Starovoitov err = -EFAULT; 1403af2ac3e1SAlexei Starovoitov if (copy_from_bpfptr(value, uvalue, value_size) != 0) 1404db20fd2bSAlexei Starovoitov goto free_value; 1405db20fd2bSAlexei Starovoitov 140615c14a3dSBrian Vazquez err = bpf_map_update_value(map, f, key, value, attr->flags); 14076710e112SJesper Dangaard Brouer 1408db20fd2bSAlexei Starovoitov free_value: 1409f0dce1d9SStanislav Fomichev kvfree(value); 1410db20fd2bSAlexei Starovoitov free_key: 141144779a4bSStanislav Fomichev kvfree(key); 1412db20fd2bSAlexei Starovoitov err_put: 1413353050beSDaniel Borkmann bpf_map_write_active_dec(map); 1414db20fd2bSAlexei Starovoitov fdput(f); 1415db20fd2bSAlexei Starovoitov return err; 1416db20fd2bSAlexei Starovoitov } 1417db20fd2bSAlexei Starovoitov 1418db20fd2bSAlexei Starovoitov #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 1419db20fd2bSAlexei Starovoitov 1420db20fd2bSAlexei Starovoitov static int map_delete_elem(union bpf_attr *attr) 1421db20fd2bSAlexei Starovoitov { 1422535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1423db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1424db20fd2bSAlexei Starovoitov struct bpf_map *map; 1425592867bfSDaniel Borkmann struct fd f; 1426db20fd2bSAlexei Starovoitov void *key; 1427db20fd2bSAlexei Starovoitov int err; 1428db20fd2bSAlexei Starovoitov 1429db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 1430db20fd2bSAlexei Starovoitov return -EINVAL; 1431db20fd2bSAlexei Starovoitov 1432592867bfSDaniel Borkmann f = fdget(ufd); 1433c2101297SDaniel Borkmann map = __bpf_map_get(f); 1434db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1435db20fd2bSAlexei Starovoitov return PTR_ERR(map); 1436353050beSDaniel Borkmann bpf_map_write_active_inc(map); 143787df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 14386e71b04aSChenbo Feng err = -EPERM; 14396e71b04aSChenbo Feng goto err_put; 14406e71b04aSChenbo Feng } 14416e71b04aSChenbo Feng 1442c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1443e4448ed8SAl Viro if (IS_ERR(key)) { 1444e4448ed8SAl Viro err = PTR_ERR(key); 1445db20fd2bSAlexei Starovoitov goto err_put; 1446e4448ed8SAl Viro } 1447db20fd2bSAlexei Starovoitov 1448a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1449a3884572SJakub Kicinski err = bpf_map_offload_delete_elem(map, key); 1450a3884572SJakub Kicinski goto out; 145185d33df3SMartin KaFai Lau } else if (IS_FD_PROG_ARRAY(map) || 145285d33df3SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 145385d33df3SMartin KaFai Lau /* These maps require sleepable context */ 1454da765a2fSDaniel Borkmann err = map->ops->map_delete_elem(map, key); 1455da765a2fSDaniel Borkmann goto out; 1456a3884572SJakub Kicinski } 1457a3884572SJakub Kicinski 1458b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 1459db20fd2bSAlexei Starovoitov rcu_read_lock(); 1460db20fd2bSAlexei Starovoitov err = map->ops->map_delete_elem(map, key); 1461db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1462b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 14631ae80cf3SDaniel Colascione maybe_wait_bpf_programs(map); 1464a3884572SJakub Kicinski out: 146544779a4bSStanislav Fomichev kvfree(key); 1466db20fd2bSAlexei Starovoitov err_put: 1467353050beSDaniel Borkmann bpf_map_write_active_dec(map); 1468db20fd2bSAlexei Starovoitov fdput(f); 1469db20fd2bSAlexei Starovoitov return err; 1470db20fd2bSAlexei Starovoitov } 1471db20fd2bSAlexei Starovoitov 1472db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 1473db20fd2bSAlexei Starovoitov #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 1474db20fd2bSAlexei Starovoitov 1475db20fd2bSAlexei Starovoitov static int map_get_next_key(union bpf_attr *attr) 1476db20fd2bSAlexei Starovoitov { 1477535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1478535e7b4bSMickaël Salaün void __user *unext_key = u64_to_user_ptr(attr->next_key); 1479db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1480db20fd2bSAlexei Starovoitov struct bpf_map *map; 1481db20fd2bSAlexei Starovoitov void *key, *next_key; 1482592867bfSDaniel Borkmann struct fd f; 1483db20fd2bSAlexei Starovoitov int err; 1484db20fd2bSAlexei Starovoitov 1485db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 1486db20fd2bSAlexei Starovoitov return -EINVAL; 1487db20fd2bSAlexei Starovoitov 1488592867bfSDaniel Borkmann f = fdget(ufd); 1489c2101297SDaniel Borkmann map = __bpf_map_get(f); 1490db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1491db20fd2bSAlexei Starovoitov return PTR_ERR(map); 149287df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 14936e71b04aSChenbo Feng err = -EPERM; 14946e71b04aSChenbo Feng goto err_put; 14956e71b04aSChenbo Feng } 14966e71b04aSChenbo Feng 14978fe45924STeng Qin if (ukey) { 1498c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1499e4448ed8SAl Viro if (IS_ERR(key)) { 1500e4448ed8SAl Viro err = PTR_ERR(key); 1501db20fd2bSAlexei Starovoitov goto err_put; 1502e4448ed8SAl Viro } 15038fe45924STeng Qin } else { 15048fe45924STeng Qin key = NULL; 15058fe45924STeng Qin } 1506db20fd2bSAlexei Starovoitov 1507db20fd2bSAlexei Starovoitov err = -ENOMEM; 150844779a4bSStanislav Fomichev next_key = kvmalloc(map->key_size, GFP_USER); 1509db20fd2bSAlexei Starovoitov if (!next_key) 1510db20fd2bSAlexei Starovoitov goto free_key; 1511db20fd2bSAlexei Starovoitov 1512a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1513a3884572SJakub Kicinski err = bpf_map_offload_get_next_key(map, key, next_key); 1514a3884572SJakub Kicinski goto out; 1515a3884572SJakub Kicinski } 1516a3884572SJakub Kicinski 1517db20fd2bSAlexei Starovoitov rcu_read_lock(); 1518db20fd2bSAlexei Starovoitov err = map->ops->map_get_next_key(map, key, next_key); 1519db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1520a3884572SJakub Kicinski out: 1521db20fd2bSAlexei Starovoitov if (err) 1522db20fd2bSAlexei Starovoitov goto free_next_key; 1523db20fd2bSAlexei Starovoitov 1524db20fd2bSAlexei Starovoitov err = -EFAULT; 1525db20fd2bSAlexei Starovoitov if (copy_to_user(unext_key, next_key, map->key_size) != 0) 1526db20fd2bSAlexei Starovoitov goto free_next_key; 1527db20fd2bSAlexei Starovoitov 1528db20fd2bSAlexei Starovoitov err = 0; 1529db20fd2bSAlexei Starovoitov 1530db20fd2bSAlexei Starovoitov free_next_key: 153144779a4bSStanislav Fomichev kvfree(next_key); 1532db20fd2bSAlexei Starovoitov free_key: 153344779a4bSStanislav Fomichev kvfree(key); 1534db20fd2bSAlexei Starovoitov err_put: 1535db20fd2bSAlexei Starovoitov fdput(f); 1536db20fd2bSAlexei Starovoitov return err; 1537db20fd2bSAlexei Starovoitov } 1538db20fd2bSAlexei Starovoitov 1539aa2e93b8SBrian Vazquez int generic_map_delete_batch(struct bpf_map *map, 1540aa2e93b8SBrian Vazquez const union bpf_attr *attr, 1541aa2e93b8SBrian Vazquez union bpf_attr __user *uattr) 1542aa2e93b8SBrian Vazquez { 1543aa2e93b8SBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1544aa2e93b8SBrian Vazquez u32 cp, max_count; 1545aa2e93b8SBrian Vazquez int err = 0; 1546aa2e93b8SBrian Vazquez void *key; 1547aa2e93b8SBrian Vazquez 1548aa2e93b8SBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1549aa2e93b8SBrian Vazquez return -EINVAL; 1550aa2e93b8SBrian Vazquez 1551aa2e93b8SBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1552aa2e93b8SBrian Vazquez !map_value_has_spin_lock(map)) { 1553aa2e93b8SBrian Vazquez return -EINVAL; 1554aa2e93b8SBrian Vazquez } 1555aa2e93b8SBrian Vazquez 1556aa2e93b8SBrian Vazquez max_count = attr->batch.count; 1557aa2e93b8SBrian Vazquez if (!max_count) 1558aa2e93b8SBrian Vazquez return 0; 1559aa2e93b8SBrian Vazquez 156044779a4bSStanislav Fomichev key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 15612e3a94aaSBrian Vazquez if (!key) 15622e3a94aaSBrian Vazquez return -ENOMEM; 15632e3a94aaSBrian Vazquez 1564aa2e93b8SBrian Vazquez for (cp = 0; cp < max_count; cp++) { 15652e3a94aaSBrian Vazquez err = -EFAULT; 15662e3a94aaSBrian Vazquez if (copy_from_user(key, keys + cp * map->key_size, 15672e3a94aaSBrian Vazquez map->key_size)) 1568aa2e93b8SBrian Vazquez break; 1569aa2e93b8SBrian Vazquez 1570aa2e93b8SBrian Vazquez if (bpf_map_is_dev_bound(map)) { 1571aa2e93b8SBrian Vazquez err = bpf_map_offload_delete_elem(map, key); 1572aa2e93b8SBrian Vazquez break; 1573aa2e93b8SBrian Vazquez } 1574aa2e93b8SBrian Vazquez 1575b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 1576aa2e93b8SBrian Vazquez rcu_read_lock(); 1577aa2e93b8SBrian Vazquez err = map->ops->map_delete_elem(map, key); 1578aa2e93b8SBrian Vazquez rcu_read_unlock(); 1579b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 1580aa2e93b8SBrian Vazquez if (err) 1581aa2e93b8SBrian Vazquez break; 158275134f16SEric Dumazet cond_resched(); 1583aa2e93b8SBrian Vazquez } 1584aa2e93b8SBrian Vazquez if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) 1585aa2e93b8SBrian Vazquez err = -EFAULT; 15862e3a94aaSBrian Vazquez 158744779a4bSStanislav Fomichev kvfree(key); 15889087c6ffSEric Dumazet 15899087c6ffSEric Dumazet maybe_wait_bpf_programs(map); 1590aa2e93b8SBrian Vazquez return err; 1591aa2e93b8SBrian Vazquez } 1592aa2e93b8SBrian Vazquez 1593aa2e93b8SBrian Vazquez int generic_map_update_batch(struct bpf_map *map, 1594aa2e93b8SBrian Vazquez const union bpf_attr *attr, 1595aa2e93b8SBrian Vazquez union bpf_attr __user *uattr) 1596aa2e93b8SBrian Vazquez { 1597aa2e93b8SBrian Vazquez void __user *values = u64_to_user_ptr(attr->batch.values); 1598aa2e93b8SBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1599aa2e93b8SBrian Vazquez u32 value_size, cp, max_count; 1600fda7a387SXu Kuohai int ufd = attr->batch.map_fd; 1601aa2e93b8SBrian Vazquez void *key, *value; 1602aa2e93b8SBrian Vazquez struct fd f; 1603aa2e93b8SBrian Vazquez int err = 0; 1604aa2e93b8SBrian Vazquez 1605aa2e93b8SBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1606aa2e93b8SBrian Vazquez return -EINVAL; 1607aa2e93b8SBrian Vazquez 1608aa2e93b8SBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1609aa2e93b8SBrian Vazquez !map_value_has_spin_lock(map)) { 1610aa2e93b8SBrian Vazquez return -EINVAL; 1611aa2e93b8SBrian Vazquez } 1612aa2e93b8SBrian Vazquez 1613aa2e93b8SBrian Vazquez value_size = bpf_map_value_size(map); 1614aa2e93b8SBrian Vazquez 1615aa2e93b8SBrian Vazquez max_count = attr->batch.count; 1616aa2e93b8SBrian Vazquez if (!max_count) 1617aa2e93b8SBrian Vazquez return 0; 1618aa2e93b8SBrian Vazquez 161944779a4bSStanislav Fomichev key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 16202e3a94aaSBrian Vazquez if (!key) 1621aa2e93b8SBrian Vazquez return -ENOMEM; 1622aa2e93b8SBrian Vazquez 1623f0dce1d9SStanislav Fomichev value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN); 16242e3a94aaSBrian Vazquez if (!value) { 162544779a4bSStanislav Fomichev kvfree(key); 16262e3a94aaSBrian Vazquez return -ENOMEM; 1627aa2e93b8SBrian Vazquez } 16282e3a94aaSBrian Vazquez 1629fda7a387SXu Kuohai f = fdget(ufd); /* bpf_map_do_batch() guarantees ufd is valid */ 16302e3a94aaSBrian Vazquez for (cp = 0; cp < max_count; cp++) { 1631aa2e93b8SBrian Vazquez err = -EFAULT; 16322e3a94aaSBrian Vazquez if (copy_from_user(key, keys + cp * map->key_size, 16332e3a94aaSBrian Vazquez map->key_size) || 16342e3a94aaSBrian Vazquez copy_from_user(value, values + cp * value_size, value_size)) 1635aa2e93b8SBrian Vazquez break; 1636aa2e93b8SBrian Vazquez 1637aa2e93b8SBrian Vazquez err = bpf_map_update_value(map, f, key, value, 1638aa2e93b8SBrian Vazquez attr->batch.elem_flags); 1639aa2e93b8SBrian Vazquez 1640aa2e93b8SBrian Vazquez if (err) 1641aa2e93b8SBrian Vazquez break; 164275134f16SEric Dumazet cond_resched(); 1643aa2e93b8SBrian Vazquez } 1644aa2e93b8SBrian Vazquez 1645aa2e93b8SBrian Vazquez if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) 1646aa2e93b8SBrian Vazquez err = -EFAULT; 1647aa2e93b8SBrian Vazquez 1648f0dce1d9SStanislav Fomichev kvfree(value); 164944779a4bSStanislav Fomichev kvfree(key); 1650fda7a387SXu Kuohai fdput(f); 1651aa2e93b8SBrian Vazquez return err; 1652aa2e93b8SBrian Vazquez } 1653aa2e93b8SBrian Vazquez 1654cb4d03abSBrian Vazquez #define MAP_LOOKUP_RETRIES 3 1655cb4d03abSBrian Vazquez 1656cb4d03abSBrian Vazquez int generic_map_lookup_batch(struct bpf_map *map, 1657cb4d03abSBrian Vazquez const union bpf_attr *attr, 1658cb4d03abSBrian Vazquez union bpf_attr __user *uattr) 1659cb4d03abSBrian Vazquez { 1660cb4d03abSBrian Vazquez void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch); 1661cb4d03abSBrian Vazquez void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch); 1662cb4d03abSBrian Vazquez void __user *values = u64_to_user_ptr(attr->batch.values); 1663cb4d03abSBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1664cb4d03abSBrian Vazquez void *buf, *buf_prevkey, *prev_key, *key, *value; 1665cb4d03abSBrian Vazquez int err, retry = MAP_LOOKUP_RETRIES; 1666cb4d03abSBrian Vazquez u32 value_size, cp, max_count; 1667cb4d03abSBrian Vazquez 1668cb4d03abSBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1669cb4d03abSBrian Vazquez return -EINVAL; 1670cb4d03abSBrian Vazquez 1671cb4d03abSBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1672cb4d03abSBrian Vazquez !map_value_has_spin_lock(map)) 1673cb4d03abSBrian Vazquez return -EINVAL; 1674cb4d03abSBrian Vazquez 1675cb4d03abSBrian Vazquez value_size = bpf_map_value_size(map); 1676cb4d03abSBrian Vazquez 1677cb4d03abSBrian Vazquez max_count = attr->batch.count; 1678cb4d03abSBrian Vazquez if (!max_count) 1679cb4d03abSBrian Vazquez return 0; 1680cb4d03abSBrian Vazquez 1681cb4d03abSBrian Vazquez if (put_user(0, &uattr->batch.count)) 1682cb4d03abSBrian Vazquez return -EFAULT; 1683cb4d03abSBrian Vazquez 168444779a4bSStanislav Fomichev buf_prevkey = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 1685cb4d03abSBrian Vazquez if (!buf_prevkey) 1686cb4d03abSBrian Vazquez return -ENOMEM; 1687cb4d03abSBrian Vazquez 1688f0dce1d9SStanislav Fomichev buf = kvmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN); 1689cb4d03abSBrian Vazquez if (!buf) { 169044779a4bSStanislav Fomichev kvfree(buf_prevkey); 1691cb4d03abSBrian Vazquez return -ENOMEM; 1692cb4d03abSBrian Vazquez } 1693cb4d03abSBrian Vazquez 1694cb4d03abSBrian Vazquez err = -EFAULT; 1695cb4d03abSBrian Vazquez prev_key = NULL; 1696cb4d03abSBrian Vazquez if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size)) 1697cb4d03abSBrian Vazquez goto free_buf; 1698cb4d03abSBrian Vazquez key = buf; 1699cb4d03abSBrian Vazquez value = key + map->key_size; 1700cb4d03abSBrian Vazquez if (ubatch) 1701cb4d03abSBrian Vazquez prev_key = buf_prevkey; 1702cb4d03abSBrian Vazquez 1703cb4d03abSBrian Vazquez for (cp = 0; cp < max_count;) { 1704cb4d03abSBrian Vazquez rcu_read_lock(); 1705cb4d03abSBrian Vazquez err = map->ops->map_get_next_key(map, prev_key, key); 1706cb4d03abSBrian Vazquez rcu_read_unlock(); 1707cb4d03abSBrian Vazquez if (err) 1708cb4d03abSBrian Vazquez break; 1709cb4d03abSBrian Vazquez err = bpf_map_copy_value(map, key, value, 1710cb4d03abSBrian Vazquez attr->batch.elem_flags); 1711cb4d03abSBrian Vazquez 1712cb4d03abSBrian Vazquez if (err == -ENOENT) { 1713cb4d03abSBrian Vazquez if (retry) { 1714cb4d03abSBrian Vazquez retry--; 1715cb4d03abSBrian Vazquez continue; 1716cb4d03abSBrian Vazquez } 1717cb4d03abSBrian Vazquez err = -EINTR; 1718cb4d03abSBrian Vazquez break; 1719cb4d03abSBrian Vazquez } 1720cb4d03abSBrian Vazquez 1721cb4d03abSBrian Vazquez if (err) 1722cb4d03abSBrian Vazquez goto free_buf; 1723cb4d03abSBrian Vazquez 1724cb4d03abSBrian Vazquez if (copy_to_user(keys + cp * map->key_size, key, 1725cb4d03abSBrian Vazquez map->key_size)) { 1726cb4d03abSBrian Vazquez err = -EFAULT; 1727cb4d03abSBrian Vazquez goto free_buf; 1728cb4d03abSBrian Vazquez } 1729cb4d03abSBrian Vazquez if (copy_to_user(values + cp * value_size, value, value_size)) { 1730cb4d03abSBrian Vazquez err = -EFAULT; 1731cb4d03abSBrian Vazquez goto free_buf; 1732cb4d03abSBrian Vazquez } 1733cb4d03abSBrian Vazquez 1734cb4d03abSBrian Vazquez if (!prev_key) 1735cb4d03abSBrian Vazquez prev_key = buf_prevkey; 1736cb4d03abSBrian Vazquez 1737cb4d03abSBrian Vazquez swap(prev_key, key); 1738cb4d03abSBrian Vazquez retry = MAP_LOOKUP_RETRIES; 1739cb4d03abSBrian Vazquez cp++; 174075134f16SEric Dumazet cond_resched(); 1741cb4d03abSBrian Vazquez } 1742cb4d03abSBrian Vazquez 1743cb4d03abSBrian Vazquez if (err == -EFAULT) 1744cb4d03abSBrian Vazquez goto free_buf; 1745cb4d03abSBrian Vazquez 1746cb4d03abSBrian Vazquez if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) || 1747cb4d03abSBrian Vazquez (cp && copy_to_user(uobatch, prev_key, map->key_size)))) 1748cb4d03abSBrian Vazquez err = -EFAULT; 1749cb4d03abSBrian Vazquez 1750cb4d03abSBrian Vazquez free_buf: 175144779a4bSStanislav Fomichev kvfree(buf_prevkey); 1752f0dce1d9SStanislav Fomichev kvfree(buf); 1753cb4d03abSBrian Vazquez return err; 1754cb4d03abSBrian Vazquez } 1755cb4d03abSBrian Vazquez 17563e87f192SDenis Salopek #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD flags 1757bd513cd0SMauricio Vasquez B 1758bd513cd0SMauricio Vasquez B static int map_lookup_and_delete_elem(union bpf_attr *attr) 1759bd513cd0SMauricio Vasquez B { 1760bd513cd0SMauricio Vasquez B void __user *ukey = u64_to_user_ptr(attr->key); 1761bd513cd0SMauricio Vasquez B void __user *uvalue = u64_to_user_ptr(attr->value); 1762bd513cd0SMauricio Vasquez B int ufd = attr->map_fd; 1763bd513cd0SMauricio Vasquez B struct bpf_map *map; 1764540fefc0SAlexei Starovoitov void *key, *value; 1765bd513cd0SMauricio Vasquez B u32 value_size; 1766bd513cd0SMauricio Vasquez B struct fd f; 1767bd513cd0SMauricio Vasquez B int err; 1768bd513cd0SMauricio Vasquez B 1769bd513cd0SMauricio Vasquez B if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) 1770bd513cd0SMauricio Vasquez B return -EINVAL; 1771bd513cd0SMauricio Vasquez B 17723e87f192SDenis Salopek if (attr->flags & ~BPF_F_LOCK) 17733e87f192SDenis Salopek return -EINVAL; 17743e87f192SDenis Salopek 1775bd513cd0SMauricio Vasquez B f = fdget(ufd); 1776bd513cd0SMauricio Vasquez B map = __bpf_map_get(f); 1777bd513cd0SMauricio Vasquez B if (IS_ERR(map)) 1778bd513cd0SMauricio Vasquez B return PTR_ERR(map); 1779353050beSDaniel Borkmann bpf_map_write_active_inc(map); 17801ea0f912SAnton Protopopov if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) || 17811ea0f912SAnton Protopopov !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 1782bd513cd0SMauricio Vasquez B err = -EPERM; 1783bd513cd0SMauricio Vasquez B goto err_put; 1784bd513cd0SMauricio Vasquez B } 1785bd513cd0SMauricio Vasquez B 17863e87f192SDenis Salopek if (attr->flags && 17873e87f192SDenis Salopek (map->map_type == BPF_MAP_TYPE_QUEUE || 17883e87f192SDenis Salopek map->map_type == BPF_MAP_TYPE_STACK)) { 17893e87f192SDenis Salopek err = -EINVAL; 17903e87f192SDenis Salopek goto err_put; 17913e87f192SDenis Salopek } 17923e87f192SDenis Salopek 17933e87f192SDenis Salopek if ((attr->flags & BPF_F_LOCK) && 17943e87f192SDenis Salopek !map_value_has_spin_lock(map)) { 17953e87f192SDenis Salopek err = -EINVAL; 17963e87f192SDenis Salopek goto err_put; 17973e87f192SDenis Salopek } 17983e87f192SDenis Salopek 1799bd513cd0SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1800bd513cd0SMauricio Vasquez B if (IS_ERR(key)) { 1801bd513cd0SMauricio Vasquez B err = PTR_ERR(key); 1802bd513cd0SMauricio Vasquez B goto err_put; 1803bd513cd0SMauricio Vasquez B } 1804bd513cd0SMauricio Vasquez B 18053e87f192SDenis Salopek value_size = bpf_map_value_size(map); 1806bd513cd0SMauricio Vasquez B 1807bd513cd0SMauricio Vasquez B err = -ENOMEM; 1808f0dce1d9SStanislav Fomichev value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN); 1809bd513cd0SMauricio Vasquez B if (!value) 1810bd513cd0SMauricio Vasquez B goto free_key; 1811bd513cd0SMauricio Vasquez B 18123e87f192SDenis Salopek err = -ENOTSUPP; 1813bd513cd0SMauricio Vasquez B if (map->map_type == BPF_MAP_TYPE_QUEUE || 1814bd513cd0SMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 1815bd513cd0SMauricio Vasquez B err = map->ops->map_pop_elem(map, value); 18163e87f192SDenis Salopek } else if (map->map_type == BPF_MAP_TYPE_HASH || 18173e87f192SDenis Salopek map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 18183e87f192SDenis Salopek map->map_type == BPF_MAP_TYPE_LRU_HASH || 18193e87f192SDenis Salopek map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 18203e87f192SDenis Salopek if (!bpf_map_is_dev_bound(map)) { 18213e87f192SDenis Salopek bpf_disable_instrumentation(); 18223e87f192SDenis Salopek rcu_read_lock(); 18233e87f192SDenis Salopek err = map->ops->map_lookup_and_delete_elem(map, key, value, attr->flags); 18243e87f192SDenis Salopek rcu_read_unlock(); 18253e87f192SDenis Salopek bpf_enable_instrumentation(); 18263e87f192SDenis Salopek } 1827bd513cd0SMauricio Vasquez B } 1828bd513cd0SMauricio Vasquez B 1829bd513cd0SMauricio Vasquez B if (err) 1830bd513cd0SMauricio Vasquez B goto free_value; 1831bd513cd0SMauricio Vasquez B 18327f645462SWei Yongjun if (copy_to_user(uvalue, value, value_size) != 0) { 18337f645462SWei Yongjun err = -EFAULT; 1834bd513cd0SMauricio Vasquez B goto free_value; 18357f645462SWei Yongjun } 1836bd513cd0SMauricio Vasquez B 1837bd513cd0SMauricio Vasquez B err = 0; 1838bd513cd0SMauricio Vasquez B 1839bd513cd0SMauricio Vasquez B free_value: 1840f0dce1d9SStanislav Fomichev kvfree(value); 1841bd513cd0SMauricio Vasquez B free_key: 184244779a4bSStanislav Fomichev kvfree(key); 1843bd513cd0SMauricio Vasquez B err_put: 1844353050beSDaniel Borkmann bpf_map_write_active_dec(map); 1845bd513cd0SMauricio Vasquez B fdput(f); 1846bd513cd0SMauricio Vasquez B return err; 1847bd513cd0SMauricio Vasquez B } 1848bd513cd0SMauricio Vasquez B 184987df15deSDaniel Borkmann #define BPF_MAP_FREEZE_LAST_FIELD map_fd 185087df15deSDaniel Borkmann 185187df15deSDaniel Borkmann static int map_freeze(const union bpf_attr *attr) 185287df15deSDaniel Borkmann { 185387df15deSDaniel Borkmann int err = 0, ufd = attr->map_fd; 185487df15deSDaniel Borkmann struct bpf_map *map; 185587df15deSDaniel Borkmann struct fd f; 185687df15deSDaniel Borkmann 185787df15deSDaniel Borkmann if (CHECK_ATTR(BPF_MAP_FREEZE)) 185887df15deSDaniel Borkmann return -EINVAL; 185987df15deSDaniel Borkmann 186087df15deSDaniel Borkmann f = fdget(ufd); 186187df15deSDaniel Borkmann map = __bpf_map_get(f); 186287df15deSDaniel Borkmann if (IS_ERR(map)) 186387df15deSDaniel Borkmann return PTR_ERR(map); 1864fc970227SAndrii Nakryiko 186568134668SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS || 186661df10c7SKumar Kartikeya Dwivedi map_value_has_timer(map) || map_value_has_kptrs(map)) { 1867849b4d94SMartin KaFai Lau fdput(f); 1868849b4d94SMartin KaFai Lau return -ENOTSUPP; 1869849b4d94SMartin KaFai Lau } 1870849b4d94SMartin KaFai Lau 1871fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 1872353050beSDaniel Borkmann if (bpf_map_write_active(map)) { 1873fc970227SAndrii Nakryiko err = -EBUSY; 1874fc970227SAndrii Nakryiko goto err_put; 1875fc970227SAndrii Nakryiko } 187687df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) { 187787df15deSDaniel Borkmann err = -EBUSY; 187887df15deSDaniel Borkmann goto err_put; 187987df15deSDaniel Borkmann } 18802c78ee89SAlexei Starovoitov if (!bpf_capable()) { 188187df15deSDaniel Borkmann err = -EPERM; 188287df15deSDaniel Borkmann goto err_put; 188387df15deSDaniel Borkmann } 188487df15deSDaniel Borkmann 188587df15deSDaniel Borkmann WRITE_ONCE(map->frozen, true); 188687df15deSDaniel Borkmann err_put: 1887fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 188887df15deSDaniel Borkmann fdput(f); 188987df15deSDaniel Borkmann return err; 189087df15deSDaniel Borkmann } 189187df15deSDaniel Borkmann 18927de16e3aSJakub Kicinski static const struct bpf_prog_ops * const bpf_prog_types[] = { 189391cc1a99SAlexei Starovoitov #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 18947de16e3aSJakub Kicinski [_id] = & _name ## _prog_ops, 18957de16e3aSJakub Kicinski #define BPF_MAP_TYPE(_id, _ops) 1896f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) 18977de16e3aSJakub Kicinski #include <linux/bpf_types.h> 18987de16e3aSJakub Kicinski #undef BPF_PROG_TYPE 18997de16e3aSJakub Kicinski #undef BPF_MAP_TYPE 1900f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 19017de16e3aSJakub Kicinski }; 19027de16e3aSJakub Kicinski 190309756af4SAlexei Starovoitov static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 190409756af4SAlexei Starovoitov { 1905d0f1a451SDaniel Borkmann const struct bpf_prog_ops *ops; 1906d0f1a451SDaniel Borkmann 1907d0f1a451SDaniel Borkmann if (type >= ARRAY_SIZE(bpf_prog_types)) 1908d0f1a451SDaniel Borkmann return -EINVAL; 1909d0f1a451SDaniel Borkmann type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types)); 1910d0f1a451SDaniel Borkmann ops = bpf_prog_types[type]; 1911d0f1a451SDaniel Borkmann if (!ops) 1912be9370a7SJohannes Berg return -EINVAL; 191309756af4SAlexei Starovoitov 1914ab3f0063SJakub Kicinski if (!bpf_prog_is_dev_bound(prog->aux)) 1915d0f1a451SDaniel Borkmann prog->aux->ops = ops; 1916ab3f0063SJakub Kicinski else 1917ab3f0063SJakub Kicinski prog->aux->ops = &bpf_offload_prog_ops; 191824701eceSDaniel Borkmann prog->type = type; 191909756af4SAlexei Starovoitov return 0; 192009756af4SAlexei Starovoitov } 192109756af4SAlexei Starovoitov 1922bae141f5SDaniel Borkmann enum bpf_audit { 1923bae141f5SDaniel Borkmann BPF_AUDIT_LOAD, 1924bae141f5SDaniel Borkmann BPF_AUDIT_UNLOAD, 1925bae141f5SDaniel Borkmann BPF_AUDIT_MAX, 1926bae141f5SDaniel Borkmann }; 1927bae141f5SDaniel Borkmann 1928bae141f5SDaniel Borkmann static const char * const bpf_audit_str[BPF_AUDIT_MAX] = { 1929bae141f5SDaniel Borkmann [BPF_AUDIT_LOAD] = "LOAD", 1930bae141f5SDaniel Borkmann [BPF_AUDIT_UNLOAD] = "UNLOAD", 1931bae141f5SDaniel Borkmann }; 1932bae141f5SDaniel Borkmann 1933bae141f5SDaniel Borkmann static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op) 1934bae141f5SDaniel Borkmann { 1935bae141f5SDaniel Borkmann struct audit_context *ctx = NULL; 1936bae141f5SDaniel Borkmann struct audit_buffer *ab; 1937bae141f5SDaniel Borkmann 1938bae141f5SDaniel Borkmann if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX)) 1939bae141f5SDaniel Borkmann return; 1940bae141f5SDaniel Borkmann if (audit_enabled == AUDIT_OFF) 1941bae141f5SDaniel Borkmann return; 1942bae141f5SDaniel Borkmann if (op == BPF_AUDIT_LOAD) 1943bae141f5SDaniel Borkmann ctx = audit_context(); 1944bae141f5SDaniel Borkmann ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF); 1945bae141f5SDaniel Borkmann if (unlikely(!ab)) 1946bae141f5SDaniel Borkmann return; 1947bae141f5SDaniel Borkmann audit_log_format(ab, "prog-id=%u op=%s", 1948bae141f5SDaniel Borkmann prog->aux->id, bpf_audit_str[op]); 1949bae141f5SDaniel Borkmann audit_log_end(ab); 1950bae141f5SDaniel Borkmann } 1951bae141f5SDaniel Borkmann 1952dc4bb0e2SMartin KaFai Lau static int bpf_prog_alloc_id(struct bpf_prog *prog) 1953dc4bb0e2SMartin KaFai Lau { 1954dc4bb0e2SMartin KaFai Lau int id; 1955dc4bb0e2SMartin KaFai Lau 1956b76354cdSShaohua Li idr_preload(GFP_KERNEL); 1957dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1958dc4bb0e2SMartin KaFai Lau id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 1959dc4bb0e2SMartin KaFai Lau if (id > 0) 1960dc4bb0e2SMartin KaFai Lau prog->aux->id = id; 1961dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1962b76354cdSShaohua Li idr_preload_end(); 1963dc4bb0e2SMartin KaFai Lau 1964dc4bb0e2SMartin KaFai Lau /* id is in [1, INT_MAX) */ 1965dc4bb0e2SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 1966dc4bb0e2SMartin KaFai Lau return -ENOSPC; 1967dc4bb0e2SMartin KaFai Lau 1968dc4bb0e2SMartin KaFai Lau return id > 0 ? 0 : id; 1969dc4bb0e2SMartin KaFai Lau } 1970dc4bb0e2SMartin KaFai Lau 1971ad8ad79fSJakub Kicinski void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) 1972dc4bb0e2SMartin KaFai Lau { 1973d809e134SAlexei Starovoitov unsigned long flags; 1974d809e134SAlexei Starovoitov 1975ad8ad79fSJakub Kicinski /* cBPF to eBPF migrations are currently not in the idr store. 1976ad8ad79fSJakub Kicinski * Offloaded programs are removed from the store when their device 1977ad8ad79fSJakub Kicinski * disappears - even if someone grabs an fd to them they are unusable, 1978ad8ad79fSJakub Kicinski * simply waiting for refcnt to drop to be freed. 1979ad8ad79fSJakub Kicinski */ 1980dc4bb0e2SMartin KaFai Lau if (!prog->aux->id) 1981dc4bb0e2SMartin KaFai Lau return; 1982dc4bb0e2SMartin KaFai Lau 1983b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1984d809e134SAlexei Starovoitov spin_lock_irqsave(&prog_idr_lock, flags); 1985b16d9aa4SMartin KaFai Lau else 1986b16d9aa4SMartin KaFai Lau __acquire(&prog_idr_lock); 1987b16d9aa4SMartin KaFai Lau 1988dc4bb0e2SMartin KaFai Lau idr_remove(&prog_idr, prog->aux->id); 1989ad8ad79fSJakub Kicinski prog->aux->id = 0; 1990b16d9aa4SMartin KaFai Lau 1991b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1992d809e134SAlexei Starovoitov spin_unlock_irqrestore(&prog_idr_lock, flags); 1993b16d9aa4SMartin KaFai Lau else 1994b16d9aa4SMartin KaFai Lau __release(&prog_idr_lock); 1995dc4bb0e2SMartin KaFai Lau } 1996dc4bb0e2SMartin KaFai Lau 19971aacde3dSDaniel Borkmann static void __bpf_prog_put_rcu(struct rcu_head *rcu) 1998abf2e7d6SAlexei Starovoitov { 1999abf2e7d6SAlexei Starovoitov struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 2000abf2e7d6SAlexei Starovoitov 20013b4d9eb2SDaniel Borkmann kvfree(aux->func_info); 20028c1b6e69SAlexei Starovoitov kfree(aux->func_info_aux); 20033ac1f01bSRoman Gushchin free_uid(aux->user); 2004afdb09c7SChenbo Feng security_bpf_prog_free(aux); 2005abf2e7d6SAlexei Starovoitov bpf_prog_free(aux->prog); 2006abf2e7d6SAlexei Starovoitov } 2007abf2e7d6SAlexei Starovoitov 2008cd7455f1SDaniel Borkmann static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) 2009cd7455f1SDaniel Borkmann { 2010cd7455f1SDaniel Borkmann bpf_prog_kallsyms_del_all(prog); 2011cd7455f1SDaniel Borkmann btf_put(prog->aux->btf); 2012e16301fbSMartin KaFai Lau kvfree(prog->aux->jited_linfo); 2013e16301fbSMartin KaFai Lau kvfree(prog->aux->linfo); 2014e6ac2450SMartin KaFai Lau kfree(prog->aux->kfunc_tab); 201522dc4a0fSAndrii Nakryiko if (prog->aux->attach_btf) 201622dc4a0fSAndrii Nakryiko btf_put(prog->aux->attach_btf); 2017cd7455f1SDaniel Borkmann 20181e6c62a8SAlexei Starovoitov if (deferred) { 20191e6c62a8SAlexei Starovoitov if (prog->aux->sleepable) 20201e6c62a8SAlexei Starovoitov call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); 2021cd7455f1SDaniel Borkmann else 20221e6c62a8SAlexei Starovoitov call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 20231e6c62a8SAlexei Starovoitov } else { 2024cd7455f1SDaniel Borkmann __bpf_prog_put_rcu(&prog->aux->rcu); 2025cd7455f1SDaniel Borkmann } 20261e6c62a8SAlexei Starovoitov } 2027cd7455f1SDaniel Borkmann 2028d809e134SAlexei Starovoitov static void bpf_prog_put_deferred(struct work_struct *work) 202909756af4SAlexei Starovoitov { 2030d809e134SAlexei Starovoitov struct bpf_prog_aux *aux; 2031d809e134SAlexei Starovoitov struct bpf_prog *prog; 2032d809e134SAlexei Starovoitov 2033d809e134SAlexei Starovoitov aux = container_of(work, struct bpf_prog_aux, work); 2034d809e134SAlexei Starovoitov prog = aux->prog; 20356ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0); 2036bae141f5SDaniel Borkmann bpf_audit_prog(prog, BPF_AUDIT_UNLOAD); 2037d809e134SAlexei Starovoitov __bpf_prog_put_noref(prog, true); 2038d809e134SAlexei Starovoitov } 2039d809e134SAlexei Starovoitov 2040d809e134SAlexei Starovoitov static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 2041d809e134SAlexei Starovoitov { 2042d809e134SAlexei Starovoitov struct bpf_prog_aux *aux = prog->aux; 2043d809e134SAlexei Starovoitov 2044d809e134SAlexei Starovoitov if (atomic64_dec_and_test(&aux->refcnt)) { 204534ad5580SMartin KaFai Lau /* bpf_prog_free_id() must be called first */ 2046b16d9aa4SMartin KaFai Lau bpf_prog_free_id(prog, do_idr_lock); 2047d809e134SAlexei Starovoitov 2048d809e134SAlexei Starovoitov if (in_irq() || irqs_disabled()) { 2049d809e134SAlexei Starovoitov INIT_WORK(&aux->work, bpf_prog_put_deferred); 2050d809e134SAlexei Starovoitov schedule_work(&aux->work); 2051d809e134SAlexei Starovoitov } else { 2052d809e134SAlexei Starovoitov bpf_prog_put_deferred(&aux->work); 2053d809e134SAlexei Starovoitov } 205409756af4SAlexei Starovoitov } 2055a67edbf4SDaniel Borkmann } 2056b16d9aa4SMartin KaFai Lau 2057b16d9aa4SMartin KaFai Lau void bpf_prog_put(struct bpf_prog *prog) 2058b16d9aa4SMartin KaFai Lau { 2059b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, true); 2060b16d9aa4SMartin KaFai Lau } 2061e2e9b654SDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_put); 206209756af4SAlexei Starovoitov 206309756af4SAlexei Starovoitov static int bpf_prog_release(struct inode *inode, struct file *filp) 206409756af4SAlexei Starovoitov { 206509756af4SAlexei Starovoitov struct bpf_prog *prog = filp->private_data; 206609756af4SAlexei Starovoitov 20671aacde3dSDaniel Borkmann bpf_prog_put(prog); 206809756af4SAlexei Starovoitov return 0; 206909756af4SAlexei Starovoitov } 207009756af4SAlexei Starovoitov 207161a0abaeSEric Dumazet struct bpf_prog_kstats { 207261a0abaeSEric Dumazet u64 nsecs; 207361a0abaeSEric Dumazet u64 cnt; 207461a0abaeSEric Dumazet u64 misses; 207561a0abaeSEric Dumazet }; 207661a0abaeSEric Dumazet 2077492ecee8SAlexei Starovoitov static void bpf_prog_get_stats(const struct bpf_prog *prog, 207861a0abaeSEric Dumazet struct bpf_prog_kstats *stats) 2079492ecee8SAlexei Starovoitov { 20809ed9e9baSAlexei Starovoitov u64 nsecs = 0, cnt = 0, misses = 0; 2081492ecee8SAlexei Starovoitov int cpu; 2082492ecee8SAlexei Starovoitov 2083492ecee8SAlexei Starovoitov for_each_possible_cpu(cpu) { 2084492ecee8SAlexei Starovoitov const struct bpf_prog_stats *st; 2085492ecee8SAlexei Starovoitov unsigned int start; 20869ed9e9baSAlexei Starovoitov u64 tnsecs, tcnt, tmisses; 2087492ecee8SAlexei Starovoitov 2088700d4796SAlexei Starovoitov st = per_cpu_ptr(prog->stats, cpu); 2089492ecee8SAlexei Starovoitov do { 2090492ecee8SAlexei Starovoitov start = u64_stats_fetch_begin_irq(&st->syncp); 209161a0abaeSEric Dumazet tnsecs = u64_stats_read(&st->nsecs); 209261a0abaeSEric Dumazet tcnt = u64_stats_read(&st->cnt); 209361a0abaeSEric Dumazet tmisses = u64_stats_read(&st->misses); 2094492ecee8SAlexei Starovoitov } while (u64_stats_fetch_retry_irq(&st->syncp, start)); 2095492ecee8SAlexei Starovoitov nsecs += tnsecs; 2096492ecee8SAlexei Starovoitov cnt += tcnt; 20979ed9e9baSAlexei Starovoitov misses += tmisses; 2098492ecee8SAlexei Starovoitov } 2099492ecee8SAlexei Starovoitov stats->nsecs = nsecs; 2100492ecee8SAlexei Starovoitov stats->cnt = cnt; 21019ed9e9baSAlexei Starovoitov stats->misses = misses; 2102492ecee8SAlexei Starovoitov } 2103492ecee8SAlexei Starovoitov 21047bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 21057bd509e3SDaniel Borkmann static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 21067bd509e3SDaniel Borkmann { 21077bd509e3SDaniel Borkmann const struct bpf_prog *prog = filp->private_data; 2108f1f7714eSDaniel Borkmann char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 210961a0abaeSEric Dumazet struct bpf_prog_kstats stats; 21107bd509e3SDaniel Borkmann 2111492ecee8SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 2112f1f7714eSDaniel Borkmann bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 21137bd509e3SDaniel Borkmann seq_printf(m, 21147bd509e3SDaniel Borkmann "prog_type:\t%u\n" 21157bd509e3SDaniel Borkmann "prog_jited:\t%u\n" 2116f1f7714eSDaniel Borkmann "prog_tag:\t%s\n" 21174316b409SDaniel Borkmann "memlock:\t%llu\n" 2118492ecee8SAlexei Starovoitov "prog_id:\t%u\n" 2119492ecee8SAlexei Starovoitov "run_time_ns:\t%llu\n" 21209ed9e9baSAlexei Starovoitov "run_cnt:\t%llu\n" 2121aba64c7dSDave Marchevsky "recursion_misses:\t%llu\n" 2122aba64c7dSDave Marchevsky "verified_insns:\t%u\n", 21237bd509e3SDaniel Borkmann prog->type, 21247bd509e3SDaniel Borkmann prog->jited, 2125f1f7714eSDaniel Borkmann prog_tag, 21264316b409SDaniel Borkmann prog->pages * 1ULL << PAGE_SHIFT, 2127492ecee8SAlexei Starovoitov prog->aux->id, 2128492ecee8SAlexei Starovoitov stats.nsecs, 21299ed9e9baSAlexei Starovoitov stats.cnt, 2130aba64c7dSDave Marchevsky stats.misses, 2131aba64c7dSDave Marchevsky prog->aux->verified_insns); 21327bd509e3SDaniel Borkmann } 21337bd509e3SDaniel Borkmann #endif 21347bd509e3SDaniel Borkmann 2135f66e448cSChenbo Feng const struct file_operations bpf_prog_fops = { 21367bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 21377bd509e3SDaniel Borkmann .show_fdinfo = bpf_prog_show_fdinfo, 21387bd509e3SDaniel Borkmann #endif 213909756af4SAlexei Starovoitov .release = bpf_prog_release, 21406e71b04aSChenbo Feng .read = bpf_dummy_read, 21416e71b04aSChenbo Feng .write = bpf_dummy_write, 214209756af4SAlexei Starovoitov }; 214309756af4SAlexei Starovoitov 2144b2197755SDaniel Borkmann int bpf_prog_new_fd(struct bpf_prog *prog) 2145aa79781bSDaniel Borkmann { 2146afdb09c7SChenbo Feng int ret; 2147afdb09c7SChenbo Feng 2148afdb09c7SChenbo Feng ret = security_bpf_prog(prog); 2149afdb09c7SChenbo Feng if (ret < 0) 2150afdb09c7SChenbo Feng return ret; 2151afdb09c7SChenbo Feng 2152aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 2153aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC); 2154aa79781bSDaniel Borkmann } 2155aa79781bSDaniel Borkmann 2156113214beSDaniel Borkmann static struct bpf_prog *____bpf_prog_get(struct fd f) 215709756af4SAlexei Starovoitov { 215809756af4SAlexei Starovoitov if (!f.file) 215909756af4SAlexei Starovoitov return ERR_PTR(-EBADF); 216009756af4SAlexei Starovoitov if (f.file->f_op != &bpf_prog_fops) { 216109756af4SAlexei Starovoitov fdput(f); 216209756af4SAlexei Starovoitov return ERR_PTR(-EINVAL); 216309756af4SAlexei Starovoitov } 216409756af4SAlexei Starovoitov 2165c2101297SDaniel Borkmann return f.file->private_data; 216609756af4SAlexei Starovoitov } 216709756af4SAlexei Starovoitov 216885192dbfSAndrii Nakryiko void bpf_prog_add(struct bpf_prog *prog, int i) 216992117d84SAlexei Starovoitov { 217085192dbfSAndrii Nakryiko atomic64_add(i, &prog->aux->refcnt); 217192117d84SAlexei Starovoitov } 217259d3656dSBrenden Blanco EXPORT_SYMBOL_GPL(bpf_prog_add); 217359d3656dSBrenden Blanco 2174c540594fSDaniel Borkmann void bpf_prog_sub(struct bpf_prog *prog, int i) 2175c540594fSDaniel Borkmann { 2176c540594fSDaniel Borkmann /* Only to be used for undoing previous bpf_prog_add() in some 2177c540594fSDaniel Borkmann * error path. We still know that another entity in our call 2178c540594fSDaniel Borkmann * path holds a reference to the program, thus atomic_sub() can 2179c540594fSDaniel Borkmann * be safely used in such cases! 2180c540594fSDaniel Borkmann */ 218185192dbfSAndrii Nakryiko WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0); 2182c540594fSDaniel Borkmann } 2183c540594fSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_sub); 2184c540594fSDaniel Borkmann 218585192dbfSAndrii Nakryiko void bpf_prog_inc(struct bpf_prog *prog) 218659d3656dSBrenden Blanco { 218785192dbfSAndrii Nakryiko atomic64_inc(&prog->aux->refcnt); 218859d3656dSBrenden Blanco } 218997bc402dSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_inc); 219092117d84SAlexei Starovoitov 2191b16d9aa4SMartin KaFai Lau /* prog_idr_lock should have been held */ 2192a6f6df69SJohn Fastabend struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 2193b16d9aa4SMartin KaFai Lau { 2194b16d9aa4SMartin KaFai Lau int refold; 2195b16d9aa4SMartin KaFai Lau 219685192dbfSAndrii Nakryiko refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0); 2197b16d9aa4SMartin KaFai Lau 2198b16d9aa4SMartin KaFai Lau if (!refold) 2199b16d9aa4SMartin KaFai Lau return ERR_PTR(-ENOENT); 2200b16d9aa4SMartin KaFai Lau 2201b16d9aa4SMartin KaFai Lau return prog; 2202b16d9aa4SMartin KaFai Lau } 2203a6f6df69SJohn Fastabend EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 2204b16d9aa4SMartin KaFai Lau 2205040ee692SAl Viro bool bpf_prog_get_ok(struct bpf_prog *prog, 2206288b3de5SJakub Kicinski enum bpf_prog_type *attach_type, bool attach_drv) 2207248f346fSJakub Kicinski { 2208288b3de5SJakub Kicinski /* not an attachment, just a refcount inc, always allow */ 2209288b3de5SJakub Kicinski if (!attach_type) 2210288b3de5SJakub Kicinski return true; 2211248f346fSJakub Kicinski 2212248f346fSJakub Kicinski if (prog->type != *attach_type) 2213248f346fSJakub Kicinski return false; 2214288b3de5SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv) 2215248f346fSJakub Kicinski return false; 2216248f346fSJakub Kicinski 2217248f346fSJakub Kicinski return true; 2218248f346fSJakub Kicinski } 2219248f346fSJakub Kicinski 2220248f346fSJakub Kicinski static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, 2221288b3de5SJakub Kicinski bool attach_drv) 222209756af4SAlexei Starovoitov { 222309756af4SAlexei Starovoitov struct fd f = fdget(ufd); 222409756af4SAlexei Starovoitov struct bpf_prog *prog; 222509756af4SAlexei Starovoitov 2226113214beSDaniel Borkmann prog = ____bpf_prog_get(f); 222709756af4SAlexei Starovoitov if (IS_ERR(prog)) 222809756af4SAlexei Starovoitov return prog; 2229288b3de5SJakub Kicinski if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) { 2230113214beSDaniel Borkmann prog = ERR_PTR(-EINVAL); 2231113214beSDaniel Borkmann goto out; 2232113214beSDaniel Borkmann } 223309756af4SAlexei Starovoitov 223485192dbfSAndrii Nakryiko bpf_prog_inc(prog); 2235113214beSDaniel Borkmann out: 223609756af4SAlexei Starovoitov fdput(f); 223709756af4SAlexei Starovoitov return prog; 223809756af4SAlexei Starovoitov } 2239113214beSDaniel Borkmann 2240113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get(u32 ufd) 2241113214beSDaniel Borkmann { 2242288b3de5SJakub Kicinski return __bpf_prog_get(ufd, NULL, false); 2243113214beSDaniel Borkmann } 2244113214beSDaniel Borkmann 2245248f346fSJakub Kicinski struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 2246288b3de5SJakub Kicinski bool attach_drv) 2247248f346fSJakub Kicinski { 22484d220ed0SAlexei Starovoitov return __bpf_prog_get(ufd, &type, attach_drv); 2249248f346fSJakub Kicinski } 22506c8dfe21SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev); 2251248f346fSJakub Kicinski 2252aac3fc32SAndrey Ignatov /* Initially all BPF programs could be loaded w/o specifying 2253aac3fc32SAndrey Ignatov * expected_attach_type. Later for some of them specifying expected_attach_type 2254aac3fc32SAndrey Ignatov * at load time became required so that program could be validated properly. 2255aac3fc32SAndrey Ignatov * Programs of types that are allowed to be loaded both w/ and w/o (for 2256aac3fc32SAndrey Ignatov * backward compatibility) expected_attach_type, should have the default attach 2257aac3fc32SAndrey Ignatov * type assigned to expected_attach_type for the latter case, so that it can be 2258aac3fc32SAndrey Ignatov * validated later at attach time. 2259aac3fc32SAndrey Ignatov * 2260aac3fc32SAndrey Ignatov * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if 2261aac3fc32SAndrey Ignatov * prog type requires it but has some attach types that have to be backward 2262aac3fc32SAndrey Ignatov * compatible. 2263aac3fc32SAndrey Ignatov */ 2264aac3fc32SAndrey Ignatov static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) 2265aac3fc32SAndrey Ignatov { 2266aac3fc32SAndrey Ignatov switch (attr->prog_type) { 2267aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 2268aac3fc32SAndrey Ignatov /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't 2269aac3fc32SAndrey Ignatov * exist so checking for non-zero is the way to go here. 2270aac3fc32SAndrey Ignatov */ 2271aac3fc32SAndrey Ignatov if (!attr->expected_attach_type) 2272aac3fc32SAndrey Ignatov attr->expected_attach_type = 2273aac3fc32SAndrey Ignatov BPF_CGROUP_INET_SOCK_CREATE; 2274aac3fc32SAndrey Ignatov break; 2275d5e4ddaeSKuniyuki Iwashima case BPF_PROG_TYPE_SK_REUSEPORT: 2276d5e4ddaeSKuniyuki Iwashima if (!attr->expected_attach_type) 2277d5e4ddaeSKuniyuki Iwashima attr->expected_attach_type = 2278d5e4ddaeSKuniyuki Iwashima BPF_SK_REUSEPORT_SELECT; 2279d5e4ddaeSKuniyuki Iwashima break; 2280aac3fc32SAndrey Ignatov } 2281aac3fc32SAndrey Ignatov } 2282aac3fc32SAndrey Ignatov 22835e43f899SAndrey Ignatov static int 2284ccfe29ebSAlexei Starovoitov bpf_prog_load_check_attach(enum bpf_prog_type prog_type, 2285ccfe29ebSAlexei Starovoitov enum bpf_attach_type expected_attach_type, 2286290248a5SAndrii Nakryiko struct btf *attach_btf, u32 btf_id, 2287290248a5SAndrii Nakryiko struct bpf_prog *dst_prog) 22885e43f899SAndrey Ignatov { 228927ae7997SMartin KaFai Lau if (btf_id) { 2290c108e3c1SAlexei Starovoitov if (btf_id > BTF_MAX_TYPE) 2291c108e3c1SAlexei Starovoitov return -EINVAL; 229227ae7997SMartin KaFai Lau 2293290248a5SAndrii Nakryiko if (!attach_btf && !dst_prog) 2294290248a5SAndrii Nakryiko return -EINVAL; 2295290248a5SAndrii Nakryiko 229627ae7997SMartin KaFai Lau switch (prog_type) { 229727ae7997SMartin KaFai Lau case BPF_PROG_TYPE_TRACING: 22989e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 229927ae7997SMartin KaFai Lau case BPF_PROG_TYPE_STRUCT_OPS: 2300be8704ffSAlexei Starovoitov case BPF_PROG_TYPE_EXT: 2301c108e3c1SAlexei Starovoitov break; 2302c108e3c1SAlexei Starovoitov default: 2303c108e3c1SAlexei Starovoitov return -EINVAL; 2304c108e3c1SAlexei Starovoitov } 230527ae7997SMartin KaFai Lau } 230627ae7997SMartin KaFai Lau 2307290248a5SAndrii Nakryiko if (attach_btf && (!btf_id || dst_prog)) 2308290248a5SAndrii Nakryiko return -EINVAL; 2309290248a5SAndrii Nakryiko 2310290248a5SAndrii Nakryiko if (dst_prog && prog_type != BPF_PROG_TYPE_TRACING && 2311be8704ffSAlexei Starovoitov prog_type != BPF_PROG_TYPE_EXT) 231227ae7997SMartin KaFai Lau return -EINVAL; 2313c108e3c1SAlexei Starovoitov 2314c108e3c1SAlexei Starovoitov switch (prog_type) { 2315aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 2316aac3fc32SAndrey Ignatov switch (expected_attach_type) { 2317aac3fc32SAndrey Ignatov case BPF_CGROUP_INET_SOCK_CREATE: 2318f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 2319aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 2320aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 2321aac3fc32SAndrey Ignatov return 0; 2322aac3fc32SAndrey Ignatov default: 2323aac3fc32SAndrey Ignatov return -EINVAL; 2324aac3fc32SAndrey Ignatov } 23254fbac77dSAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 23264fbac77dSAndrey Ignatov switch (expected_attach_type) { 23274fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 23284fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 2329d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 2330d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 23311b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 23321b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 23331b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 23341b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 23351cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 23361cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 2337983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG: 2338983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG: 23395e43f899SAndrey Ignatov return 0; 23404fbac77dSAndrey Ignatov default: 23414fbac77dSAndrey Ignatov return -EINVAL; 23424fbac77dSAndrey Ignatov } 23435cf1e914Sbrakmo case BPF_PROG_TYPE_CGROUP_SKB: 23445cf1e914Sbrakmo switch (expected_attach_type) { 23455cf1e914Sbrakmo case BPF_CGROUP_INET_INGRESS: 23465cf1e914Sbrakmo case BPF_CGROUP_INET_EGRESS: 23475cf1e914Sbrakmo return 0; 23485cf1e914Sbrakmo default: 23495cf1e914Sbrakmo return -EINVAL; 23505cf1e914Sbrakmo } 23510d01da6aSStanislav Fomichev case BPF_PROG_TYPE_CGROUP_SOCKOPT: 23520d01da6aSStanislav Fomichev switch (expected_attach_type) { 23530d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT: 23540d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT: 23550d01da6aSStanislav Fomichev return 0; 23560d01da6aSStanislav Fomichev default: 23570d01da6aSStanislav Fomichev return -EINVAL; 23580d01da6aSStanislav Fomichev } 2359e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP: 2360e9ddbb77SJakub Sitnicki if (expected_attach_type == BPF_SK_LOOKUP) 2361e9ddbb77SJakub Sitnicki return 0; 2362e9ddbb77SJakub Sitnicki return -EINVAL; 2363d5e4ddaeSKuniyuki Iwashima case BPF_PROG_TYPE_SK_REUSEPORT: 2364d5e4ddaeSKuniyuki Iwashima switch (expected_attach_type) { 2365d5e4ddaeSKuniyuki Iwashima case BPF_SK_REUSEPORT_SELECT: 2366d5e4ddaeSKuniyuki Iwashima case BPF_SK_REUSEPORT_SELECT_OR_MIGRATE: 2367d5e4ddaeSKuniyuki Iwashima return 0; 2368d5e4ddaeSKuniyuki Iwashima default: 2369d5e4ddaeSKuniyuki Iwashima return -EINVAL; 2370d5e4ddaeSKuniyuki Iwashima } 237179a7f8bdSAlexei Starovoitov case BPF_PROG_TYPE_SYSCALL: 2372be8704ffSAlexei Starovoitov case BPF_PROG_TYPE_EXT: 2373be8704ffSAlexei Starovoitov if (expected_attach_type) 2374be8704ffSAlexei Starovoitov return -EINVAL; 2375df561f66SGustavo A. R. Silva fallthrough; 23764fbac77dSAndrey Ignatov default: 23774fbac77dSAndrey Ignatov return 0; 23784fbac77dSAndrey Ignatov } 23795e43f899SAndrey Ignatov } 23805e43f899SAndrey Ignatov 23812c78ee89SAlexei Starovoitov static bool is_net_admin_prog_type(enum bpf_prog_type prog_type) 23822c78ee89SAlexei Starovoitov { 23832c78ee89SAlexei Starovoitov switch (prog_type) { 23842c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SCHED_CLS: 23852c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SCHED_ACT: 23862c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_XDP: 23872c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_IN: 23882c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_OUT: 23892c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_XMIT: 23902c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_SEG6LOCAL: 23912c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_SKB: 23922c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_MSG: 23932c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LIRC_MODE2: 23942c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_FLOW_DISSECTOR: 23952c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_DEVICE: 23962c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCK: 23972c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 23982c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCKOPT: 23992c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SYSCTL: 24002c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SOCK_OPS: 24012c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_EXT: /* extends any prog */ 24022c78ee89SAlexei Starovoitov return true; 24032c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SKB: 24042c78ee89SAlexei Starovoitov /* always unpriv */ 24052c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_REUSEPORT: 24062c78ee89SAlexei Starovoitov /* equivalent to SOCKET_FILTER. need CAP_BPF only */ 24072c78ee89SAlexei Starovoitov default: 24082c78ee89SAlexei Starovoitov return false; 24092c78ee89SAlexei Starovoitov } 24102c78ee89SAlexei Starovoitov } 24112c78ee89SAlexei Starovoitov 24122c78ee89SAlexei Starovoitov static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) 24132c78ee89SAlexei Starovoitov { 24142c78ee89SAlexei Starovoitov switch (prog_type) { 24152c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_KPROBE: 24162c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_TRACEPOINT: 24172c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_PERF_EVENT: 24182c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_RAW_TRACEPOINT: 24192c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 24202c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_TRACING: 24212c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LSM: 24222c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_STRUCT_OPS: /* has access to struct sock */ 24232c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_EXT: /* extends any prog */ 24242c78ee89SAlexei Starovoitov return true; 24252c78ee89SAlexei Starovoitov default: 24262c78ee89SAlexei Starovoitov return false; 24272c78ee89SAlexei Starovoitov } 24282c78ee89SAlexei Starovoitov } 24292c78ee89SAlexei Starovoitov 243009756af4SAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 2431fbd94c7aSAlexei Starovoitov #define BPF_PROG_LOAD_LAST_FIELD core_relo_rec_size 243209756af4SAlexei Starovoitov 2433af2ac3e1SAlexei Starovoitov static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr) 243409756af4SAlexei Starovoitov { 243509756af4SAlexei Starovoitov enum bpf_prog_type type = attr->prog_type; 2436290248a5SAndrii Nakryiko struct bpf_prog *prog, *dst_prog = NULL; 2437290248a5SAndrii Nakryiko struct btf *attach_btf = NULL; 243809756af4SAlexei Starovoitov int err; 243909756af4SAlexei Starovoitov char license[128]; 244009756af4SAlexei Starovoitov bool is_gpl; 244109756af4SAlexei Starovoitov 244209756af4SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_LOAD)) 244309756af4SAlexei Starovoitov return -EINVAL; 244409756af4SAlexei Starovoitov 2445c240eff6SJiong Wang if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | 2446c240eff6SJiong Wang BPF_F_ANY_ALIGNMENT | 244710d274e8SAlexei Starovoitov BPF_F_TEST_STATE_FREQ | 24481e6c62a8SAlexei Starovoitov BPF_F_SLEEPABLE | 2449c2f2cdbeSLorenzo Bianconi BPF_F_TEST_RND_HI32 | 2450c2f2cdbeSLorenzo Bianconi BPF_F_XDP_HAS_FRAGS)) 2451e07b98d9SDavid S. Miller return -EINVAL; 2452e07b98d9SDavid S. Miller 2453e9ee9efcSDavid Miller if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && 2454e9ee9efcSDavid Miller (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && 24552c78ee89SAlexei Starovoitov !bpf_capable()) 2456e9ee9efcSDavid Miller return -EPERM; 2457e9ee9efcSDavid Miller 245809756af4SAlexei Starovoitov /* copy eBPF program license from user space */ 2459af2ac3e1SAlexei Starovoitov if (strncpy_from_bpfptr(license, 2460af2ac3e1SAlexei Starovoitov make_bpfptr(attr->license, uattr.is_kernel), 246109756af4SAlexei Starovoitov sizeof(license) - 1) < 0) 246209756af4SAlexei Starovoitov return -EFAULT; 246309756af4SAlexei Starovoitov license[sizeof(license) - 1] = 0; 246409756af4SAlexei Starovoitov 246509756af4SAlexei Starovoitov /* eBPF programs must be GPL compatible to use GPL-ed functions */ 246609756af4SAlexei Starovoitov is_gpl = license_is_gpl_compatible(license); 246709756af4SAlexei Starovoitov 2468c04c0d2bSAlexei Starovoitov if (attr->insn_cnt == 0 || 24692c78ee89SAlexei Starovoitov attr->insn_cnt > (bpf_capable() ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) 2470ef0915caSDaniel Borkmann return -E2BIG; 247180b7d819SChenbo Feng if (type != BPF_PROG_TYPE_SOCKET_FILTER && 247280b7d819SChenbo Feng type != BPF_PROG_TYPE_CGROUP_SKB && 24732c78ee89SAlexei Starovoitov !bpf_capable()) 24742c78ee89SAlexei Starovoitov return -EPERM; 24752c78ee89SAlexei Starovoitov 2476b338cb92SMaciej Żenczykowski if (is_net_admin_prog_type(type) && !capable(CAP_NET_ADMIN) && !capable(CAP_SYS_ADMIN)) 24772c78ee89SAlexei Starovoitov return -EPERM; 24782c78ee89SAlexei Starovoitov if (is_perfmon_prog_type(type) && !perfmon_capable()) 24791be7f75dSAlexei Starovoitov return -EPERM; 24801be7f75dSAlexei Starovoitov 2481290248a5SAndrii Nakryiko /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog 2482290248a5SAndrii Nakryiko * or btf, we need to check which one it is 2483290248a5SAndrii Nakryiko */ 2484290248a5SAndrii Nakryiko if (attr->attach_prog_fd) { 2485290248a5SAndrii Nakryiko dst_prog = bpf_prog_get(attr->attach_prog_fd); 2486290248a5SAndrii Nakryiko if (IS_ERR(dst_prog)) { 2487290248a5SAndrii Nakryiko dst_prog = NULL; 2488290248a5SAndrii Nakryiko attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd); 2489290248a5SAndrii Nakryiko if (IS_ERR(attach_btf)) 2490290248a5SAndrii Nakryiko return -EINVAL; 2491290248a5SAndrii Nakryiko if (!btf_is_kernel(attach_btf)) { 24928bdd8e27SAndrii Nakryiko /* attaching through specifying bpf_prog's BTF 24938bdd8e27SAndrii Nakryiko * objects directly might be supported eventually 24948bdd8e27SAndrii Nakryiko */ 2495290248a5SAndrii Nakryiko btf_put(attach_btf); 24968bdd8e27SAndrii Nakryiko return -ENOTSUPP; 2497290248a5SAndrii Nakryiko } 2498290248a5SAndrii Nakryiko } 2499290248a5SAndrii Nakryiko } else if (attr->attach_btf_id) { 2500290248a5SAndrii Nakryiko /* fall back to vmlinux BTF, if BTF type ID is specified */ 2501290248a5SAndrii Nakryiko attach_btf = bpf_get_btf_vmlinux(); 2502290248a5SAndrii Nakryiko if (IS_ERR(attach_btf)) 2503290248a5SAndrii Nakryiko return PTR_ERR(attach_btf); 2504290248a5SAndrii Nakryiko if (!attach_btf) 2505290248a5SAndrii Nakryiko return -EINVAL; 2506290248a5SAndrii Nakryiko btf_get(attach_btf); 2507290248a5SAndrii Nakryiko } 2508290248a5SAndrii Nakryiko 2509aac3fc32SAndrey Ignatov bpf_prog_load_fixup_attach_type(attr); 2510ccfe29ebSAlexei Starovoitov if (bpf_prog_load_check_attach(type, attr->expected_attach_type, 2511290248a5SAndrii Nakryiko attach_btf, attr->attach_btf_id, 2512290248a5SAndrii Nakryiko dst_prog)) { 2513290248a5SAndrii Nakryiko if (dst_prog) 2514290248a5SAndrii Nakryiko bpf_prog_put(dst_prog); 2515290248a5SAndrii Nakryiko if (attach_btf) 2516290248a5SAndrii Nakryiko btf_put(attach_btf); 25175e43f899SAndrey Ignatov return -EINVAL; 2518290248a5SAndrii Nakryiko } 25195e43f899SAndrey Ignatov 252009756af4SAlexei Starovoitov /* plain bpf_prog allocation */ 252109756af4SAlexei Starovoitov prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 2522290248a5SAndrii Nakryiko if (!prog) { 2523290248a5SAndrii Nakryiko if (dst_prog) 2524290248a5SAndrii Nakryiko bpf_prog_put(dst_prog); 2525290248a5SAndrii Nakryiko if (attach_btf) 2526290248a5SAndrii Nakryiko btf_put(attach_btf); 252709756af4SAlexei Starovoitov return -ENOMEM; 2528290248a5SAndrii Nakryiko } 252909756af4SAlexei Starovoitov 25305e43f899SAndrey Ignatov prog->expected_attach_type = attr->expected_attach_type; 2531290248a5SAndrii Nakryiko prog->aux->attach_btf = attach_btf; 2532ccfe29ebSAlexei Starovoitov prog->aux->attach_btf_id = attr->attach_btf_id; 25333aac1eadSToke Høiland-Jørgensen prog->aux->dst_prog = dst_prog; 25349a18eedbSJakub Kicinski prog->aux->offload_requested = !!attr->prog_ifindex; 25351e6c62a8SAlexei Starovoitov prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE; 2536c2f2cdbeSLorenzo Bianconi prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; 25379a18eedbSJakub Kicinski 2538afdb09c7SChenbo Feng err = security_bpf_prog_alloc(prog->aux); 2539aaac3ba9SAlexei Starovoitov if (err) 25403ac1f01bSRoman Gushchin goto free_prog; 2541aaac3ba9SAlexei Starovoitov 25423ac1f01bSRoman Gushchin prog->aux->user = get_current_user(); 254309756af4SAlexei Starovoitov prog->len = attr->insn_cnt; 254409756af4SAlexei Starovoitov 254509756af4SAlexei Starovoitov err = -EFAULT; 2546af2ac3e1SAlexei Starovoitov if (copy_from_bpfptr(prog->insns, 2547af2ac3e1SAlexei Starovoitov make_bpfptr(attr->insns, uattr.is_kernel), 2548aafe6ae9SDaniel Borkmann bpf_prog_insn_size(prog)) != 0) 25493ac1f01bSRoman Gushchin goto free_prog_sec; 255009756af4SAlexei Starovoitov 255109756af4SAlexei Starovoitov prog->orig_prog = NULL; 2552a91263d5SDaniel Borkmann prog->jited = 0; 255309756af4SAlexei Starovoitov 255485192dbfSAndrii Nakryiko atomic64_set(&prog->aux->refcnt, 1); 2555a91263d5SDaniel Borkmann prog->gpl_compatible = is_gpl ? 1 : 0; 255609756af4SAlexei Starovoitov 25579a18eedbSJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 2558ab3f0063SJakub Kicinski err = bpf_prog_offload_init(prog, attr); 2559ab3f0063SJakub Kicinski if (err) 25603ac1f01bSRoman Gushchin goto free_prog_sec; 2561ab3f0063SJakub Kicinski } 2562ab3f0063SJakub Kicinski 256309756af4SAlexei Starovoitov /* find program type: socket_filter vs tracing_filter */ 256409756af4SAlexei Starovoitov err = find_prog_type(type, prog); 256509756af4SAlexei Starovoitov if (err < 0) 25663ac1f01bSRoman Gushchin goto free_prog_sec; 256709756af4SAlexei Starovoitov 25689285ec4cSJason A. Donenfeld prog->aux->load_time = ktime_get_boottime_ns(); 25698e7ae251SMartin KaFai Lau err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, 25708e7ae251SMartin KaFai Lau sizeof(attr->prog_name)); 25718e7ae251SMartin KaFai Lau if (err < 0) 25723ac1f01bSRoman Gushchin goto free_prog_sec; 2573cb4d2b3fSMartin KaFai Lau 257409756af4SAlexei Starovoitov /* run eBPF verifier */ 2575838e9690SYonghong Song err = bpf_check(&prog, attr, uattr); 257609756af4SAlexei Starovoitov if (err < 0) 257709756af4SAlexei Starovoitov goto free_used_maps; 257809756af4SAlexei Starovoitov 2579d1c55ab5SDaniel Borkmann prog = bpf_prog_select_runtime(prog, &err); 258004fd61abSAlexei Starovoitov if (err < 0) 258104fd61abSAlexei Starovoitov goto free_used_maps; 258209756af4SAlexei Starovoitov 2583dc4bb0e2SMartin KaFai Lau err = bpf_prog_alloc_id(prog); 2584dc4bb0e2SMartin KaFai Lau if (err) 2585dc4bb0e2SMartin KaFai Lau goto free_used_maps; 2586dc4bb0e2SMartin KaFai Lau 2587c751798aSDaniel Borkmann /* Upon success of bpf_prog_alloc_id(), the BPF prog is 2588c751798aSDaniel Borkmann * effectively publicly exposed. However, retrieving via 2589c751798aSDaniel Borkmann * bpf_prog_get_fd_by_id() will take another reference, 2590c751798aSDaniel Borkmann * therefore it cannot be gone underneath us. 2591c751798aSDaniel Borkmann * 2592c751798aSDaniel Borkmann * Only for the time /after/ successful bpf_prog_new_fd() 2593c751798aSDaniel Borkmann * and before returning to userspace, we might just hold 2594c751798aSDaniel Borkmann * one reference and any parallel close on that fd could 2595c751798aSDaniel Borkmann * rip everything out. Hence, below notifications must 2596c751798aSDaniel Borkmann * happen before bpf_prog_new_fd(). 2597c751798aSDaniel Borkmann * 2598c751798aSDaniel Borkmann * Also, any failure handling from this point onwards must 2599c751798aSDaniel Borkmann * be using bpf_prog_put() given the program is exposed. 2600b16d9aa4SMartin KaFai Lau */ 260174451e66SDaniel Borkmann bpf_prog_kallsyms_add(prog); 26026ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); 2603bae141f5SDaniel Borkmann bpf_audit_prog(prog, BPF_AUDIT_LOAD); 2604c751798aSDaniel Borkmann 2605c751798aSDaniel Borkmann err = bpf_prog_new_fd(prog); 2606c751798aSDaniel Borkmann if (err < 0) 2607c751798aSDaniel Borkmann bpf_prog_put(prog); 260809756af4SAlexei Starovoitov return err; 260909756af4SAlexei Starovoitov 261009756af4SAlexei Starovoitov free_used_maps: 2611cd7455f1SDaniel Borkmann /* In case we have subprogs, we need to wait for a grace 2612cd7455f1SDaniel Borkmann * period before we can tear down JIT memory since symbols 2613cd7455f1SDaniel Borkmann * are already exposed under kallsyms. 2614cd7455f1SDaniel Borkmann */ 2615cd7455f1SDaniel Borkmann __bpf_prog_put_noref(prog, prog->aux->func_cnt); 2616cd7455f1SDaniel Borkmann return err; 2617afdb09c7SChenbo Feng free_prog_sec: 26183ac1f01bSRoman Gushchin free_uid(prog->aux->user); 2619afdb09c7SChenbo Feng security_bpf_prog_free(prog->aux); 26203ac1f01bSRoman Gushchin free_prog: 262122dc4a0fSAndrii Nakryiko if (prog->aux->attach_btf) 262222dc4a0fSAndrii Nakryiko btf_put(prog->aux->attach_btf); 262309756af4SAlexei Starovoitov bpf_prog_free(prog); 262409756af4SAlexei Starovoitov return err; 262509756af4SAlexei Starovoitov } 262609756af4SAlexei Starovoitov 26276e71b04aSChenbo Feng #define BPF_OBJ_LAST_FIELD file_flags 2628b2197755SDaniel Borkmann 2629b2197755SDaniel Borkmann static int bpf_obj_pin(const union bpf_attr *attr) 2630b2197755SDaniel Borkmann { 26316e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0) 2632b2197755SDaniel Borkmann return -EINVAL; 2633b2197755SDaniel Borkmann 2634535e7b4bSMickaël Salaün return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 2635b2197755SDaniel Borkmann } 2636b2197755SDaniel Borkmann 2637b2197755SDaniel Borkmann static int bpf_obj_get(const union bpf_attr *attr) 2638b2197755SDaniel Borkmann { 26396e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || 26406e71b04aSChenbo Feng attr->file_flags & ~BPF_OBJ_FLAG_MASK) 2641b2197755SDaniel Borkmann return -EINVAL; 2642b2197755SDaniel Borkmann 26436e71b04aSChenbo Feng return bpf_obj_get_user(u64_to_user_ptr(attr->pathname), 26446e71b04aSChenbo Feng attr->file_flags); 2645b2197755SDaniel Borkmann } 2646b2197755SDaniel Borkmann 2647f2e10bffSAndrii Nakryiko void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 2648a3b80e10SAndrii Nakryiko const struct bpf_link_ops *ops, struct bpf_prog *prog) 264970ed506cSAndrii Nakryiko { 265070ed506cSAndrii Nakryiko atomic64_set(&link->refcnt, 1); 2651f2e10bffSAndrii Nakryiko link->type = type; 2652a3b80e10SAndrii Nakryiko link->id = 0; 265370ed506cSAndrii Nakryiko link->ops = ops; 265470ed506cSAndrii Nakryiko link->prog = prog; 265570ed506cSAndrii Nakryiko } 265670ed506cSAndrii Nakryiko 2657a3b80e10SAndrii Nakryiko static void bpf_link_free_id(int id) 2658a3b80e10SAndrii Nakryiko { 2659a3b80e10SAndrii Nakryiko if (!id) 2660a3b80e10SAndrii Nakryiko return; 2661a3b80e10SAndrii Nakryiko 2662a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 2663a3b80e10SAndrii Nakryiko idr_remove(&link_idr, id); 2664a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 2665a3b80e10SAndrii Nakryiko } 2666a3b80e10SAndrii Nakryiko 266798868668SAndrii Nakryiko /* Clean up bpf_link and corresponding anon_inode file and FD. After 266898868668SAndrii Nakryiko * anon_inode is created, bpf_link can't be just kfree()'d due to deferred 2669a3b80e10SAndrii Nakryiko * anon_inode's release() call. This helper marksbpf_link as 2670a3b80e10SAndrii Nakryiko * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt 2671a3b80e10SAndrii Nakryiko * is not decremented, it's the responsibility of a calling code that failed 2672a3b80e10SAndrii Nakryiko * to complete bpf_link initialization. 267398868668SAndrii Nakryiko */ 2674a3b80e10SAndrii Nakryiko void bpf_link_cleanup(struct bpf_link_primer *primer) 2675babf3164SAndrii Nakryiko { 2676a3b80e10SAndrii Nakryiko primer->link->prog = NULL; 2677a3b80e10SAndrii Nakryiko bpf_link_free_id(primer->id); 2678a3b80e10SAndrii Nakryiko fput(primer->file); 2679a3b80e10SAndrii Nakryiko put_unused_fd(primer->fd); 2680babf3164SAndrii Nakryiko } 2681babf3164SAndrii Nakryiko 268270ed506cSAndrii Nakryiko void bpf_link_inc(struct bpf_link *link) 268370ed506cSAndrii Nakryiko { 268470ed506cSAndrii Nakryiko atomic64_inc(&link->refcnt); 268570ed506cSAndrii Nakryiko } 268670ed506cSAndrii Nakryiko 268770ed506cSAndrii Nakryiko /* bpf_link_free is guaranteed to be called from process context */ 268870ed506cSAndrii Nakryiko static void bpf_link_free(struct bpf_link *link) 268970ed506cSAndrii Nakryiko { 2690a3b80e10SAndrii Nakryiko bpf_link_free_id(link->id); 2691babf3164SAndrii Nakryiko if (link->prog) { 2692babf3164SAndrii Nakryiko /* detach BPF program, clean up used resources */ 269370ed506cSAndrii Nakryiko link->ops->release(link); 2694babf3164SAndrii Nakryiko bpf_prog_put(link->prog); 2695babf3164SAndrii Nakryiko } 2696babf3164SAndrii Nakryiko /* free bpf_link and its containing memory */ 2697babf3164SAndrii Nakryiko link->ops->dealloc(link); 269870ed506cSAndrii Nakryiko } 269970ed506cSAndrii Nakryiko 270070ed506cSAndrii Nakryiko static void bpf_link_put_deferred(struct work_struct *work) 270170ed506cSAndrii Nakryiko { 270270ed506cSAndrii Nakryiko struct bpf_link *link = container_of(work, struct bpf_link, work); 270370ed506cSAndrii Nakryiko 270470ed506cSAndrii Nakryiko bpf_link_free(link); 270570ed506cSAndrii Nakryiko } 270670ed506cSAndrii Nakryiko 270770ed506cSAndrii Nakryiko /* bpf_link_put can be called from atomic context, but ensures that resources 270870ed506cSAndrii Nakryiko * are freed from process context 270970ed506cSAndrii Nakryiko */ 271070ed506cSAndrii Nakryiko void bpf_link_put(struct bpf_link *link) 271170ed506cSAndrii Nakryiko { 271270ed506cSAndrii Nakryiko if (!atomic64_dec_and_test(&link->refcnt)) 271370ed506cSAndrii Nakryiko return; 271470ed506cSAndrii Nakryiko 2715f00f2f7fSAlexei Starovoitov if (in_atomic()) { 271670ed506cSAndrii Nakryiko INIT_WORK(&link->work, bpf_link_put_deferred); 271770ed506cSAndrii Nakryiko schedule_work(&link->work); 2718f00f2f7fSAlexei Starovoitov } else { 2719f00f2f7fSAlexei Starovoitov bpf_link_free(link); 2720f00f2f7fSAlexei Starovoitov } 272170ed506cSAndrii Nakryiko } 2722cb80ddc6SAlexei Starovoitov EXPORT_SYMBOL(bpf_link_put); 272370ed506cSAndrii Nakryiko 272470ed506cSAndrii Nakryiko static int bpf_link_release(struct inode *inode, struct file *filp) 272570ed506cSAndrii Nakryiko { 272670ed506cSAndrii Nakryiko struct bpf_link *link = filp->private_data; 272770ed506cSAndrii Nakryiko 272870ed506cSAndrii Nakryiko bpf_link_put(link); 2729fec56f58SAlexei Starovoitov return 0; 2730fec56f58SAlexei Starovoitov } 2731fec56f58SAlexei Starovoitov 273270ed506cSAndrii Nakryiko #ifdef CONFIG_PROC_FS 2733f2e10bffSAndrii Nakryiko #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) 2734f2e10bffSAndrii Nakryiko #define BPF_MAP_TYPE(_id, _ops) 2735f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) [_id] = #_name, 2736f2e10bffSAndrii Nakryiko static const char *bpf_link_type_strs[] = { 2737f2e10bffSAndrii Nakryiko [BPF_LINK_TYPE_UNSPEC] = "<invalid>", 2738f2e10bffSAndrii Nakryiko #include <linux/bpf_types.h> 2739f2e10bffSAndrii Nakryiko }; 2740f2e10bffSAndrii Nakryiko #undef BPF_PROG_TYPE 2741f2e10bffSAndrii Nakryiko #undef BPF_MAP_TYPE 2742f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 274370ed506cSAndrii Nakryiko 274470ed506cSAndrii Nakryiko static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) 274570ed506cSAndrii Nakryiko { 274670ed506cSAndrii Nakryiko const struct bpf_link *link = filp->private_data; 274770ed506cSAndrii Nakryiko const struct bpf_prog *prog = link->prog; 274870ed506cSAndrii Nakryiko char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 274970ed506cSAndrii Nakryiko 275070ed506cSAndrii Nakryiko bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 275170ed506cSAndrii Nakryiko seq_printf(m, 275270ed506cSAndrii Nakryiko "link_type:\t%s\n" 2753a3b80e10SAndrii Nakryiko "link_id:\t%u\n" 275470ed506cSAndrii Nakryiko "prog_tag:\t%s\n" 275570ed506cSAndrii Nakryiko "prog_id:\t%u\n", 2756f2e10bffSAndrii Nakryiko bpf_link_type_strs[link->type], 2757a3b80e10SAndrii Nakryiko link->id, 275870ed506cSAndrii Nakryiko prog_tag, 275970ed506cSAndrii Nakryiko prog->aux->id); 2760f2e10bffSAndrii Nakryiko if (link->ops->show_fdinfo) 2761f2e10bffSAndrii Nakryiko link->ops->show_fdinfo(link, m); 276270ed506cSAndrii Nakryiko } 276370ed506cSAndrii Nakryiko #endif 276470ed506cSAndrii Nakryiko 27656f302bfbSZou Wei static const struct file_operations bpf_link_fops = { 276670ed506cSAndrii Nakryiko #ifdef CONFIG_PROC_FS 276770ed506cSAndrii Nakryiko .show_fdinfo = bpf_link_show_fdinfo, 276870ed506cSAndrii Nakryiko #endif 276970ed506cSAndrii Nakryiko .release = bpf_link_release, 2770fec56f58SAlexei Starovoitov .read = bpf_dummy_read, 2771fec56f58SAlexei Starovoitov .write = bpf_dummy_write, 2772fec56f58SAlexei Starovoitov }; 2773fec56f58SAlexei Starovoitov 2774a3b80e10SAndrii Nakryiko static int bpf_link_alloc_id(struct bpf_link *link) 277570ed506cSAndrii Nakryiko { 2776a3b80e10SAndrii Nakryiko int id; 2777a3b80e10SAndrii Nakryiko 2778a3b80e10SAndrii Nakryiko idr_preload(GFP_KERNEL); 2779a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 2780a3b80e10SAndrii Nakryiko id = idr_alloc_cyclic(&link_idr, link, 1, INT_MAX, GFP_ATOMIC); 2781a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 2782a3b80e10SAndrii Nakryiko idr_preload_end(); 2783a3b80e10SAndrii Nakryiko 2784a3b80e10SAndrii Nakryiko return id; 278570ed506cSAndrii Nakryiko } 278670ed506cSAndrii Nakryiko 2787a3b80e10SAndrii Nakryiko /* Prepare bpf_link to be exposed to user-space by allocating anon_inode file, 2788a3b80e10SAndrii Nakryiko * reserving unused FD and allocating ID from link_idr. This is to be paired 2789a3b80e10SAndrii Nakryiko * with bpf_link_settle() to install FD and ID and expose bpf_link to 2790a3b80e10SAndrii Nakryiko * user-space, if bpf_link is successfully attached. If not, bpf_link and 2791a3b80e10SAndrii Nakryiko * pre-allocated resources are to be freed with bpf_cleanup() call. All the 2792a3b80e10SAndrii Nakryiko * transient state is passed around in struct bpf_link_primer. 2793a3b80e10SAndrii Nakryiko * This is preferred way to create and initialize bpf_link, especially when 2794a3b80e10SAndrii Nakryiko * there are complicated and expensive operations in between creating bpf_link 2795a3b80e10SAndrii Nakryiko * itself and attaching it to BPF hook. By using bpf_link_prime() and 2796a3b80e10SAndrii Nakryiko * bpf_link_settle() kernel code using bpf_link doesn't have to perform 2797a3b80e10SAndrii Nakryiko * expensive (and potentially failing) roll back operations in a rare case 2798a3b80e10SAndrii Nakryiko * that file, FD, or ID can't be allocated. 2799babf3164SAndrii Nakryiko */ 2800a3b80e10SAndrii Nakryiko int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer) 2801babf3164SAndrii Nakryiko { 2802babf3164SAndrii Nakryiko struct file *file; 2803a3b80e10SAndrii Nakryiko int fd, id; 2804babf3164SAndrii Nakryiko 2805babf3164SAndrii Nakryiko fd = get_unused_fd_flags(O_CLOEXEC); 2806babf3164SAndrii Nakryiko if (fd < 0) 2807a3b80e10SAndrii Nakryiko return fd; 2808babf3164SAndrii Nakryiko 2809babf3164SAndrii Nakryiko 2810a3b80e10SAndrii Nakryiko id = bpf_link_alloc_id(link); 2811a3b80e10SAndrii Nakryiko if (id < 0) { 2812a3b80e10SAndrii Nakryiko put_unused_fd(fd); 2813a3b80e10SAndrii Nakryiko return id; 2814a3b80e10SAndrii Nakryiko } 2815babf3164SAndrii Nakryiko 2816babf3164SAndrii Nakryiko file = anon_inode_getfile("bpf_link", &bpf_link_fops, link, O_CLOEXEC); 2817babf3164SAndrii Nakryiko if (IS_ERR(file)) { 2818138c6767SAndrii Nakryiko bpf_link_free_id(id); 2819babf3164SAndrii Nakryiko put_unused_fd(fd); 2820138c6767SAndrii Nakryiko return PTR_ERR(file); 2821babf3164SAndrii Nakryiko } 2822babf3164SAndrii Nakryiko 2823a3b80e10SAndrii Nakryiko primer->link = link; 2824a3b80e10SAndrii Nakryiko primer->file = file; 2825a3b80e10SAndrii Nakryiko primer->fd = fd; 2826a3b80e10SAndrii Nakryiko primer->id = id; 2827a3b80e10SAndrii Nakryiko return 0; 2828a3b80e10SAndrii Nakryiko } 2829a3b80e10SAndrii Nakryiko 2830a3b80e10SAndrii Nakryiko int bpf_link_settle(struct bpf_link_primer *primer) 2831a3b80e10SAndrii Nakryiko { 2832a3b80e10SAndrii Nakryiko /* make bpf_link fetchable by ID */ 2833a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 2834a3b80e10SAndrii Nakryiko primer->link->id = primer->id; 2835a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 2836a3b80e10SAndrii Nakryiko /* make bpf_link fetchable by FD */ 2837a3b80e10SAndrii Nakryiko fd_install(primer->fd, primer->file); 2838a3b80e10SAndrii Nakryiko /* pass through installed FD */ 2839a3b80e10SAndrii Nakryiko return primer->fd; 2840a3b80e10SAndrii Nakryiko } 2841a3b80e10SAndrii Nakryiko 2842a3b80e10SAndrii Nakryiko int bpf_link_new_fd(struct bpf_link *link) 2843a3b80e10SAndrii Nakryiko { 2844a3b80e10SAndrii Nakryiko return anon_inode_getfd("bpf-link", &bpf_link_fops, link, O_CLOEXEC); 2845babf3164SAndrii Nakryiko } 2846babf3164SAndrii Nakryiko 284770ed506cSAndrii Nakryiko struct bpf_link *bpf_link_get_from_fd(u32 ufd) 284870ed506cSAndrii Nakryiko { 284970ed506cSAndrii Nakryiko struct fd f = fdget(ufd); 285070ed506cSAndrii Nakryiko struct bpf_link *link; 285170ed506cSAndrii Nakryiko 285270ed506cSAndrii Nakryiko if (!f.file) 285370ed506cSAndrii Nakryiko return ERR_PTR(-EBADF); 285470ed506cSAndrii Nakryiko if (f.file->f_op != &bpf_link_fops) { 285570ed506cSAndrii Nakryiko fdput(f); 285670ed506cSAndrii Nakryiko return ERR_PTR(-EINVAL); 285770ed506cSAndrii Nakryiko } 285870ed506cSAndrii Nakryiko 285970ed506cSAndrii Nakryiko link = f.file->private_data; 286070ed506cSAndrii Nakryiko bpf_link_inc(link); 286170ed506cSAndrii Nakryiko fdput(f); 286270ed506cSAndrii Nakryiko 286370ed506cSAndrii Nakryiko return link; 286470ed506cSAndrii Nakryiko } 2865cb80ddc6SAlexei Starovoitov EXPORT_SYMBOL(bpf_link_get_from_fd); 286670ed506cSAndrii Nakryiko 286770ed506cSAndrii Nakryiko struct bpf_tracing_link { 286870ed506cSAndrii Nakryiko struct bpf_link link; 2869f2e10bffSAndrii Nakryiko enum bpf_attach_type attach_type; 28703aac1eadSToke Høiland-Jørgensen struct bpf_trampoline *trampoline; 28713aac1eadSToke Høiland-Jørgensen struct bpf_prog *tgt_prog; 287270ed506cSAndrii Nakryiko }; 287370ed506cSAndrii Nakryiko 287470ed506cSAndrii Nakryiko static void bpf_tracing_link_release(struct bpf_link *link) 287570ed506cSAndrii Nakryiko { 28763aac1eadSToke Høiland-Jørgensen struct bpf_tracing_link *tr_link = 28773aac1eadSToke Høiland-Jørgensen container_of(link, struct bpf_tracing_link, link); 28783aac1eadSToke Høiland-Jørgensen 28793aac1eadSToke Høiland-Jørgensen WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog, 28803aac1eadSToke Høiland-Jørgensen tr_link->trampoline)); 28813aac1eadSToke Høiland-Jørgensen 28823aac1eadSToke Høiland-Jørgensen bpf_trampoline_put(tr_link->trampoline); 28833aac1eadSToke Høiland-Jørgensen 28843aac1eadSToke Høiland-Jørgensen /* tgt_prog is NULL if target is a kernel function */ 28853aac1eadSToke Høiland-Jørgensen if (tr_link->tgt_prog) 28863aac1eadSToke Høiland-Jørgensen bpf_prog_put(tr_link->tgt_prog); 2887babf3164SAndrii Nakryiko } 2888babf3164SAndrii Nakryiko 2889babf3164SAndrii Nakryiko static void bpf_tracing_link_dealloc(struct bpf_link *link) 2890babf3164SAndrii Nakryiko { 289170ed506cSAndrii Nakryiko struct bpf_tracing_link *tr_link = 289270ed506cSAndrii Nakryiko container_of(link, struct bpf_tracing_link, link); 289370ed506cSAndrii Nakryiko 289470ed506cSAndrii Nakryiko kfree(tr_link); 289570ed506cSAndrii Nakryiko } 289670ed506cSAndrii Nakryiko 2897f2e10bffSAndrii Nakryiko static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link, 2898f2e10bffSAndrii Nakryiko struct seq_file *seq) 2899f2e10bffSAndrii Nakryiko { 2900f2e10bffSAndrii Nakryiko struct bpf_tracing_link *tr_link = 2901f2e10bffSAndrii Nakryiko container_of(link, struct bpf_tracing_link, link); 2902f2e10bffSAndrii Nakryiko 2903f2e10bffSAndrii Nakryiko seq_printf(seq, 2904f2e10bffSAndrii Nakryiko "attach_type:\t%d\n", 2905f2e10bffSAndrii Nakryiko tr_link->attach_type); 2906f2e10bffSAndrii Nakryiko } 2907f2e10bffSAndrii Nakryiko 2908f2e10bffSAndrii Nakryiko static int bpf_tracing_link_fill_link_info(const struct bpf_link *link, 2909f2e10bffSAndrii Nakryiko struct bpf_link_info *info) 2910f2e10bffSAndrii Nakryiko { 2911f2e10bffSAndrii Nakryiko struct bpf_tracing_link *tr_link = 2912f2e10bffSAndrii Nakryiko container_of(link, struct bpf_tracing_link, link); 2913f2e10bffSAndrii Nakryiko 2914f2e10bffSAndrii Nakryiko info->tracing.attach_type = tr_link->attach_type; 2915441e8c66SToke Høiland-Jørgensen bpf_trampoline_unpack_key(tr_link->trampoline->key, 2916441e8c66SToke Høiland-Jørgensen &info->tracing.target_obj_id, 2917441e8c66SToke Høiland-Jørgensen &info->tracing.target_btf_id); 2918f2e10bffSAndrii Nakryiko 2919f2e10bffSAndrii Nakryiko return 0; 2920f2e10bffSAndrii Nakryiko } 2921f2e10bffSAndrii Nakryiko 292270ed506cSAndrii Nakryiko static const struct bpf_link_ops bpf_tracing_link_lops = { 292370ed506cSAndrii Nakryiko .release = bpf_tracing_link_release, 2924babf3164SAndrii Nakryiko .dealloc = bpf_tracing_link_dealloc, 2925f2e10bffSAndrii Nakryiko .show_fdinfo = bpf_tracing_link_show_fdinfo, 2926f2e10bffSAndrii Nakryiko .fill_link_info = bpf_tracing_link_fill_link_info, 292770ed506cSAndrii Nakryiko }; 292870ed506cSAndrii Nakryiko 29294a1e7c0cSToke Høiland-Jørgensen static int bpf_tracing_prog_attach(struct bpf_prog *prog, 29304a1e7c0cSToke Høiland-Jørgensen int tgt_prog_fd, 29314a1e7c0cSToke Høiland-Jørgensen u32 btf_id) 2932fec56f58SAlexei Starovoitov { 2933a3b80e10SAndrii Nakryiko struct bpf_link_primer link_primer; 29343aac1eadSToke Høiland-Jørgensen struct bpf_prog *tgt_prog = NULL; 29354a1e7c0cSToke Høiland-Jørgensen struct bpf_trampoline *tr = NULL; 293670ed506cSAndrii Nakryiko struct bpf_tracing_link *link; 29374a1e7c0cSToke Høiland-Jørgensen u64 key = 0; 2938a3b80e10SAndrii Nakryiko int err; 2939fec56f58SAlexei Starovoitov 29409e4e01dfSKP Singh switch (prog->type) { 29419e4e01dfSKP Singh case BPF_PROG_TYPE_TRACING: 2942fec56f58SAlexei Starovoitov if (prog->expected_attach_type != BPF_TRACE_FENTRY && 2943be8704ffSAlexei Starovoitov prog->expected_attach_type != BPF_TRACE_FEXIT && 29449e4e01dfSKP Singh prog->expected_attach_type != BPF_MODIFY_RETURN) { 29459e4e01dfSKP Singh err = -EINVAL; 29469e4e01dfSKP Singh goto out_put_prog; 29479e4e01dfSKP Singh } 29489e4e01dfSKP Singh break; 29499e4e01dfSKP Singh case BPF_PROG_TYPE_EXT: 29509e4e01dfSKP Singh if (prog->expected_attach_type != 0) { 29519e4e01dfSKP Singh err = -EINVAL; 29529e4e01dfSKP Singh goto out_put_prog; 29539e4e01dfSKP Singh } 29549e4e01dfSKP Singh break; 29559e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 29569e4e01dfSKP Singh if (prog->expected_attach_type != BPF_LSM_MAC) { 29579e4e01dfSKP Singh err = -EINVAL; 29589e4e01dfSKP Singh goto out_put_prog; 29599e4e01dfSKP Singh } 29609e4e01dfSKP Singh break; 29619e4e01dfSKP Singh default: 2962fec56f58SAlexei Starovoitov err = -EINVAL; 2963fec56f58SAlexei Starovoitov goto out_put_prog; 2964fec56f58SAlexei Starovoitov } 2965fec56f58SAlexei Starovoitov 29664a1e7c0cSToke Høiland-Jørgensen if (!!tgt_prog_fd != !!btf_id) { 29674a1e7c0cSToke Høiland-Jørgensen err = -EINVAL; 29684a1e7c0cSToke Høiland-Jørgensen goto out_put_prog; 29694a1e7c0cSToke Høiland-Jørgensen } 29704a1e7c0cSToke Høiland-Jørgensen 29714a1e7c0cSToke Høiland-Jørgensen if (tgt_prog_fd) { 29724a1e7c0cSToke Høiland-Jørgensen /* For now we only allow new targets for BPF_PROG_TYPE_EXT */ 29734a1e7c0cSToke Høiland-Jørgensen if (prog->type != BPF_PROG_TYPE_EXT) { 29744a1e7c0cSToke Høiland-Jørgensen err = -EINVAL; 29754a1e7c0cSToke Høiland-Jørgensen goto out_put_prog; 29764a1e7c0cSToke Høiland-Jørgensen } 29774a1e7c0cSToke Høiland-Jørgensen 29784a1e7c0cSToke Høiland-Jørgensen tgt_prog = bpf_prog_get(tgt_prog_fd); 29794a1e7c0cSToke Høiland-Jørgensen if (IS_ERR(tgt_prog)) { 29804a1e7c0cSToke Høiland-Jørgensen err = PTR_ERR(tgt_prog); 29814a1e7c0cSToke Høiland-Jørgensen tgt_prog = NULL; 29824a1e7c0cSToke Høiland-Jørgensen goto out_put_prog; 29834a1e7c0cSToke Høiland-Jørgensen } 29844a1e7c0cSToke Høiland-Jørgensen 298522dc4a0fSAndrii Nakryiko key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id); 29864a1e7c0cSToke Høiland-Jørgensen } 29874a1e7c0cSToke Høiland-Jørgensen 298870ed506cSAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER); 298970ed506cSAndrii Nakryiko if (!link) { 299070ed506cSAndrii Nakryiko err = -ENOMEM; 2991fec56f58SAlexei Starovoitov goto out_put_prog; 2992fec56f58SAlexei Starovoitov } 2993f2e10bffSAndrii Nakryiko bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING, 2994f2e10bffSAndrii Nakryiko &bpf_tracing_link_lops, prog); 2995f2e10bffSAndrii Nakryiko link->attach_type = prog->expected_attach_type; 2996fec56f58SAlexei Starovoitov 29973aac1eadSToke Høiland-Jørgensen mutex_lock(&prog->aux->dst_mutex); 2998babf3164SAndrii Nakryiko 29994a1e7c0cSToke Høiland-Jørgensen /* There are a few possible cases here: 30004a1e7c0cSToke Høiland-Jørgensen * 30014a1e7c0cSToke Høiland-Jørgensen * - if prog->aux->dst_trampoline is set, the program was just loaded 30024a1e7c0cSToke Høiland-Jørgensen * and not yet attached to anything, so we can use the values stored 30034a1e7c0cSToke Høiland-Jørgensen * in prog->aux 30044a1e7c0cSToke Høiland-Jørgensen * 30054a1e7c0cSToke Høiland-Jørgensen * - if prog->aux->dst_trampoline is NULL, the program has already been 30064a1e7c0cSToke Høiland-Jørgensen * attached to a target and its initial target was cleared (below) 30074a1e7c0cSToke Høiland-Jørgensen * 30084a1e7c0cSToke Høiland-Jørgensen * - if tgt_prog != NULL, the caller specified tgt_prog_fd + 30094a1e7c0cSToke Høiland-Jørgensen * target_btf_id using the link_create API. 30104a1e7c0cSToke Høiland-Jørgensen * 30114a1e7c0cSToke Høiland-Jørgensen * - if tgt_prog == NULL when this function was called using the old 30124a1e7c0cSToke Høiland-Jørgensen * raw_tracepoint_open API, and we need a target from prog->aux 30134a1e7c0cSToke Høiland-Jørgensen * 3014f3a95075SJiri Olsa * - if prog->aux->dst_trampoline and tgt_prog is NULL, the program 3015f3a95075SJiri Olsa * was detached and is going for re-attachment. 30164a1e7c0cSToke Høiland-Jørgensen */ 30174a1e7c0cSToke Høiland-Jørgensen if (!prog->aux->dst_trampoline && !tgt_prog) { 3018f3a95075SJiri Olsa /* 3019f3a95075SJiri Olsa * Allow re-attach for TRACING and LSM programs. If it's 3020f3a95075SJiri Olsa * currently linked, bpf_trampoline_link_prog will fail. 3021f3a95075SJiri Olsa * EXT programs need to specify tgt_prog_fd, so they 3022f3a95075SJiri Olsa * re-attach in separate code path. 3023f3a95075SJiri Olsa */ 3024f3a95075SJiri Olsa if (prog->type != BPF_PROG_TYPE_TRACING && 3025f3a95075SJiri Olsa prog->type != BPF_PROG_TYPE_LSM) { 3026f3a95075SJiri Olsa err = -EINVAL; 30273aac1eadSToke Høiland-Jørgensen goto out_unlock; 30283aac1eadSToke Høiland-Jørgensen } 3029f3a95075SJiri Olsa btf_id = prog->aux->attach_btf_id; 3030f3a95075SJiri Olsa key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id); 3031f3a95075SJiri Olsa } 30324a1e7c0cSToke Høiland-Jørgensen 30334a1e7c0cSToke Høiland-Jørgensen if (!prog->aux->dst_trampoline || 30344a1e7c0cSToke Høiland-Jørgensen (key && key != prog->aux->dst_trampoline->key)) { 30354a1e7c0cSToke Høiland-Jørgensen /* If there is no saved target, or the specified target is 30364a1e7c0cSToke Høiland-Jørgensen * different from the destination specified at load time, we 30374a1e7c0cSToke Høiland-Jørgensen * need a new trampoline and a check for compatibility 30384a1e7c0cSToke Høiland-Jørgensen */ 30394a1e7c0cSToke Høiland-Jørgensen struct bpf_attach_target_info tgt_info = {}; 30404a1e7c0cSToke Høiland-Jørgensen 30414a1e7c0cSToke Høiland-Jørgensen err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id, 30424a1e7c0cSToke Høiland-Jørgensen &tgt_info); 30434a1e7c0cSToke Høiland-Jørgensen if (err) 30444a1e7c0cSToke Høiland-Jørgensen goto out_unlock; 30454a1e7c0cSToke Høiland-Jørgensen 30464a1e7c0cSToke Høiland-Jørgensen tr = bpf_trampoline_get(key, &tgt_info); 30474a1e7c0cSToke Høiland-Jørgensen if (!tr) { 30484a1e7c0cSToke Høiland-Jørgensen err = -ENOMEM; 30494a1e7c0cSToke Høiland-Jørgensen goto out_unlock; 30504a1e7c0cSToke Høiland-Jørgensen } 30514a1e7c0cSToke Høiland-Jørgensen } else { 30524a1e7c0cSToke Høiland-Jørgensen /* The caller didn't specify a target, or the target was the 30534a1e7c0cSToke Høiland-Jørgensen * same as the destination supplied during program load. This 30544a1e7c0cSToke Høiland-Jørgensen * means we can reuse the trampoline and reference from program 30554a1e7c0cSToke Høiland-Jørgensen * load time, and there is no need to allocate a new one. This 30564a1e7c0cSToke Høiland-Jørgensen * can only happen once for any program, as the saved values in 30574a1e7c0cSToke Høiland-Jørgensen * prog->aux are cleared below. 30584a1e7c0cSToke Høiland-Jørgensen */ 30593aac1eadSToke Høiland-Jørgensen tr = prog->aux->dst_trampoline; 30603aac1eadSToke Høiland-Jørgensen tgt_prog = prog->aux->dst_prog; 30614a1e7c0cSToke Høiland-Jørgensen } 30623aac1eadSToke Høiland-Jørgensen 30633aac1eadSToke Høiland-Jørgensen err = bpf_link_prime(&link->link, &link_primer); 30643aac1eadSToke Høiland-Jørgensen if (err) 30653aac1eadSToke Høiland-Jørgensen goto out_unlock; 30663aac1eadSToke Høiland-Jørgensen 30673aac1eadSToke Høiland-Jørgensen err = bpf_trampoline_link_prog(prog, tr); 3068babf3164SAndrii Nakryiko if (err) { 3069a3b80e10SAndrii Nakryiko bpf_link_cleanup(&link_primer); 30703aac1eadSToke Høiland-Jørgensen link = NULL; 30713aac1eadSToke Høiland-Jørgensen goto out_unlock; 3072babf3164SAndrii Nakryiko } 3073babf3164SAndrii Nakryiko 30743aac1eadSToke Høiland-Jørgensen link->tgt_prog = tgt_prog; 30753aac1eadSToke Høiland-Jørgensen link->trampoline = tr; 30763aac1eadSToke Høiland-Jørgensen 30774a1e7c0cSToke Høiland-Jørgensen /* Always clear the trampoline and target prog from prog->aux to make 30784a1e7c0cSToke Høiland-Jørgensen * sure the original attach destination is not kept alive after a 30794a1e7c0cSToke Høiland-Jørgensen * program is (re-)attached to another target. 30804a1e7c0cSToke Høiland-Jørgensen */ 30814a1e7c0cSToke Høiland-Jørgensen if (prog->aux->dst_prog && 30824a1e7c0cSToke Høiland-Jørgensen (tgt_prog_fd || tr != prog->aux->dst_trampoline)) 30834a1e7c0cSToke Høiland-Jørgensen /* got extra prog ref from syscall, or attaching to different prog */ 30844a1e7c0cSToke Høiland-Jørgensen bpf_prog_put(prog->aux->dst_prog); 30854a1e7c0cSToke Høiland-Jørgensen if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline) 30864a1e7c0cSToke Høiland-Jørgensen /* we allocated a new trampoline, so free the old one */ 30874a1e7c0cSToke Høiland-Jørgensen bpf_trampoline_put(prog->aux->dst_trampoline); 30884a1e7c0cSToke Høiland-Jørgensen 30893aac1eadSToke Høiland-Jørgensen prog->aux->dst_prog = NULL; 30903aac1eadSToke Høiland-Jørgensen prog->aux->dst_trampoline = NULL; 30913aac1eadSToke Høiland-Jørgensen mutex_unlock(&prog->aux->dst_mutex); 30923aac1eadSToke Høiland-Jørgensen 3093a3b80e10SAndrii Nakryiko return bpf_link_settle(&link_primer); 30943aac1eadSToke Høiland-Jørgensen out_unlock: 30954a1e7c0cSToke Høiland-Jørgensen if (tr && tr != prog->aux->dst_trampoline) 30964a1e7c0cSToke Høiland-Jørgensen bpf_trampoline_put(tr); 30973aac1eadSToke Høiland-Jørgensen mutex_unlock(&prog->aux->dst_mutex); 30983aac1eadSToke Høiland-Jørgensen kfree(link); 3099fec56f58SAlexei Starovoitov out_put_prog: 31004a1e7c0cSToke Høiland-Jørgensen if (tgt_prog_fd && tgt_prog) 31014a1e7c0cSToke Høiland-Jørgensen bpf_prog_put(tgt_prog); 3102fec56f58SAlexei Starovoitov return err; 3103fec56f58SAlexei Starovoitov } 3104fec56f58SAlexei Starovoitov 310570ed506cSAndrii Nakryiko struct bpf_raw_tp_link { 310670ed506cSAndrii Nakryiko struct bpf_link link; 3107c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 3108c4f6699dSAlexei Starovoitov }; 3109c4f6699dSAlexei Starovoitov 311070ed506cSAndrii Nakryiko static void bpf_raw_tp_link_release(struct bpf_link *link) 3111c4f6699dSAlexei Starovoitov { 311270ed506cSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 311370ed506cSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 3114c4f6699dSAlexei Starovoitov 311570ed506cSAndrii Nakryiko bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog); 3116a38d1107SMatt Mullins bpf_put_raw_tracepoint(raw_tp->btp); 3117babf3164SAndrii Nakryiko } 3118babf3164SAndrii Nakryiko 3119babf3164SAndrii Nakryiko static void bpf_raw_tp_link_dealloc(struct bpf_link *link) 3120babf3164SAndrii Nakryiko { 3121babf3164SAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 3122babf3164SAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 3123babf3164SAndrii Nakryiko 3124c4f6699dSAlexei Starovoitov kfree(raw_tp); 3125c4f6699dSAlexei Starovoitov } 3126c4f6699dSAlexei Starovoitov 3127f2e10bffSAndrii Nakryiko static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link, 3128f2e10bffSAndrii Nakryiko struct seq_file *seq) 3129f2e10bffSAndrii Nakryiko { 3130f2e10bffSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp_link = 3131f2e10bffSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 3132f2e10bffSAndrii Nakryiko 3133f2e10bffSAndrii Nakryiko seq_printf(seq, 3134f2e10bffSAndrii Nakryiko "tp_name:\t%s\n", 3135f2e10bffSAndrii Nakryiko raw_tp_link->btp->tp->name); 3136f2e10bffSAndrii Nakryiko } 3137f2e10bffSAndrii Nakryiko 3138f2e10bffSAndrii Nakryiko static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link, 3139f2e10bffSAndrii Nakryiko struct bpf_link_info *info) 3140f2e10bffSAndrii Nakryiko { 3141f2e10bffSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp_link = 3142f2e10bffSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 3143f2e10bffSAndrii Nakryiko char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name); 3144f2e10bffSAndrii Nakryiko const char *tp_name = raw_tp_link->btp->tp->name; 3145f2e10bffSAndrii Nakryiko u32 ulen = info->raw_tracepoint.tp_name_len; 3146f2e10bffSAndrii Nakryiko size_t tp_len = strlen(tp_name); 3147f2e10bffSAndrii Nakryiko 3148b474959dSYonghong Song if (!ulen ^ !ubuf) 3149f2e10bffSAndrii Nakryiko return -EINVAL; 3150f2e10bffSAndrii Nakryiko 3151f2e10bffSAndrii Nakryiko info->raw_tracepoint.tp_name_len = tp_len + 1; 3152f2e10bffSAndrii Nakryiko 3153f2e10bffSAndrii Nakryiko if (!ubuf) 3154f2e10bffSAndrii Nakryiko return 0; 3155f2e10bffSAndrii Nakryiko 3156f2e10bffSAndrii Nakryiko if (ulen >= tp_len + 1) { 3157f2e10bffSAndrii Nakryiko if (copy_to_user(ubuf, tp_name, tp_len + 1)) 3158f2e10bffSAndrii Nakryiko return -EFAULT; 3159f2e10bffSAndrii Nakryiko } else { 3160f2e10bffSAndrii Nakryiko char zero = '\0'; 3161f2e10bffSAndrii Nakryiko 3162f2e10bffSAndrii Nakryiko if (copy_to_user(ubuf, tp_name, ulen - 1)) 3163f2e10bffSAndrii Nakryiko return -EFAULT; 3164f2e10bffSAndrii Nakryiko if (put_user(zero, ubuf + ulen - 1)) 3165f2e10bffSAndrii Nakryiko return -EFAULT; 3166f2e10bffSAndrii Nakryiko return -ENOSPC; 3167f2e10bffSAndrii Nakryiko } 3168f2e10bffSAndrii Nakryiko 3169f2e10bffSAndrii Nakryiko return 0; 3170f2e10bffSAndrii Nakryiko } 3171f2e10bffSAndrii Nakryiko 3172a3b80e10SAndrii Nakryiko static const struct bpf_link_ops bpf_raw_tp_link_lops = { 317370ed506cSAndrii Nakryiko .release = bpf_raw_tp_link_release, 3174babf3164SAndrii Nakryiko .dealloc = bpf_raw_tp_link_dealloc, 3175f2e10bffSAndrii Nakryiko .show_fdinfo = bpf_raw_tp_link_show_fdinfo, 3176f2e10bffSAndrii Nakryiko .fill_link_info = bpf_raw_tp_link_fill_link_info, 3177c4f6699dSAlexei Starovoitov }; 3178c4f6699dSAlexei Starovoitov 3179b89fbfbbSAndrii Nakryiko #ifdef CONFIG_PERF_EVENTS 3180b89fbfbbSAndrii Nakryiko struct bpf_perf_link { 3181b89fbfbbSAndrii Nakryiko struct bpf_link link; 3182b89fbfbbSAndrii Nakryiko struct file *perf_file; 3183b89fbfbbSAndrii Nakryiko }; 3184b89fbfbbSAndrii Nakryiko 3185b89fbfbbSAndrii Nakryiko static void bpf_perf_link_release(struct bpf_link *link) 3186b89fbfbbSAndrii Nakryiko { 3187b89fbfbbSAndrii Nakryiko struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link); 3188b89fbfbbSAndrii Nakryiko struct perf_event *event = perf_link->perf_file->private_data; 3189b89fbfbbSAndrii Nakryiko 3190b89fbfbbSAndrii Nakryiko perf_event_free_bpf_prog(event); 3191b89fbfbbSAndrii Nakryiko fput(perf_link->perf_file); 3192b89fbfbbSAndrii Nakryiko } 3193b89fbfbbSAndrii Nakryiko 3194b89fbfbbSAndrii Nakryiko static void bpf_perf_link_dealloc(struct bpf_link *link) 3195b89fbfbbSAndrii Nakryiko { 3196b89fbfbbSAndrii Nakryiko struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link); 3197b89fbfbbSAndrii Nakryiko 3198b89fbfbbSAndrii Nakryiko kfree(perf_link); 3199b89fbfbbSAndrii Nakryiko } 3200b89fbfbbSAndrii Nakryiko 3201b89fbfbbSAndrii Nakryiko static const struct bpf_link_ops bpf_perf_link_lops = { 3202b89fbfbbSAndrii Nakryiko .release = bpf_perf_link_release, 3203b89fbfbbSAndrii Nakryiko .dealloc = bpf_perf_link_dealloc, 3204b89fbfbbSAndrii Nakryiko }; 3205b89fbfbbSAndrii Nakryiko 3206b89fbfbbSAndrii Nakryiko static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 3207b89fbfbbSAndrii Nakryiko { 3208b89fbfbbSAndrii Nakryiko struct bpf_link_primer link_primer; 3209b89fbfbbSAndrii Nakryiko struct bpf_perf_link *link; 3210b89fbfbbSAndrii Nakryiko struct perf_event *event; 3211b89fbfbbSAndrii Nakryiko struct file *perf_file; 3212b89fbfbbSAndrii Nakryiko int err; 3213b89fbfbbSAndrii Nakryiko 3214b89fbfbbSAndrii Nakryiko if (attr->link_create.flags) 3215b89fbfbbSAndrii Nakryiko return -EINVAL; 3216b89fbfbbSAndrii Nakryiko 3217b89fbfbbSAndrii Nakryiko perf_file = perf_event_get(attr->link_create.target_fd); 3218b89fbfbbSAndrii Nakryiko if (IS_ERR(perf_file)) 3219b89fbfbbSAndrii Nakryiko return PTR_ERR(perf_file); 3220b89fbfbbSAndrii Nakryiko 3221b89fbfbbSAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER); 3222b89fbfbbSAndrii Nakryiko if (!link) { 3223b89fbfbbSAndrii Nakryiko err = -ENOMEM; 3224b89fbfbbSAndrii Nakryiko goto out_put_file; 3225b89fbfbbSAndrii Nakryiko } 3226b89fbfbbSAndrii Nakryiko bpf_link_init(&link->link, BPF_LINK_TYPE_PERF_EVENT, &bpf_perf_link_lops, prog); 3227b89fbfbbSAndrii Nakryiko link->perf_file = perf_file; 3228b89fbfbbSAndrii Nakryiko 3229b89fbfbbSAndrii Nakryiko err = bpf_link_prime(&link->link, &link_primer); 3230b89fbfbbSAndrii Nakryiko if (err) { 3231b89fbfbbSAndrii Nakryiko kfree(link); 3232b89fbfbbSAndrii Nakryiko goto out_put_file; 3233b89fbfbbSAndrii Nakryiko } 3234b89fbfbbSAndrii Nakryiko 3235b89fbfbbSAndrii Nakryiko event = perf_file->private_data; 323682e6b1eeSAndrii Nakryiko err = perf_event_set_bpf_prog(event, prog, attr->link_create.perf_event.bpf_cookie); 3237b89fbfbbSAndrii Nakryiko if (err) { 3238b89fbfbbSAndrii Nakryiko bpf_link_cleanup(&link_primer); 3239b89fbfbbSAndrii Nakryiko goto out_put_file; 3240b89fbfbbSAndrii Nakryiko } 3241b89fbfbbSAndrii Nakryiko /* perf_event_set_bpf_prog() doesn't take its own refcnt on prog */ 3242b89fbfbbSAndrii Nakryiko bpf_prog_inc(prog); 3243b89fbfbbSAndrii Nakryiko 3244b89fbfbbSAndrii Nakryiko return bpf_link_settle(&link_primer); 3245b89fbfbbSAndrii Nakryiko 3246b89fbfbbSAndrii Nakryiko out_put_file: 3247b89fbfbbSAndrii Nakryiko fput(perf_file); 3248b89fbfbbSAndrii Nakryiko return err; 3249b89fbfbbSAndrii Nakryiko } 32500dcac272SJiri Olsa #else 32510dcac272SJiri Olsa static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 32520dcac272SJiri Olsa { 32530dcac272SJiri Olsa return -EOPNOTSUPP; 32540dcac272SJiri Olsa } 3255b89fbfbbSAndrii Nakryiko #endif /* CONFIG_PERF_EVENTS */ 3256b89fbfbbSAndrii Nakryiko 3257df86ca0dSAndrii Nakryiko static int bpf_raw_tp_link_attach(struct bpf_prog *prog, 3258df86ca0dSAndrii Nakryiko const char __user *user_tp_name) 3259c4f6699dSAlexei Starovoitov { 3260a3b80e10SAndrii Nakryiko struct bpf_link_primer link_primer; 3261babf3164SAndrii Nakryiko struct bpf_raw_tp_link *link; 3262c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 3263ac4414b5SAlexei Starovoitov const char *tp_name; 3264ac4414b5SAlexei Starovoitov char buf[128]; 3265a3b80e10SAndrii Nakryiko int err; 3266c4f6699dSAlexei Starovoitov 32679e4e01dfSKP Singh switch (prog->type) { 32689e4e01dfSKP Singh case BPF_PROG_TYPE_TRACING: 32699e4e01dfSKP Singh case BPF_PROG_TYPE_EXT: 32709e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 3271df86ca0dSAndrii Nakryiko if (user_tp_name) 3272fec56f58SAlexei Starovoitov /* The attach point for this category of programs 3273fec56f58SAlexei Starovoitov * should be specified via btf_id during program load. 3274ac4414b5SAlexei Starovoitov */ 3275df86ca0dSAndrii Nakryiko return -EINVAL; 32769e4e01dfSKP Singh if (prog->type == BPF_PROG_TYPE_TRACING && 32779e4e01dfSKP Singh prog->expected_attach_type == BPF_TRACE_RAW_TP) { 327838207291SMartin KaFai Lau tp_name = prog->aux->attach_func_name; 32799e4e01dfSKP Singh break; 32809e4e01dfSKP Singh } 3281df86ca0dSAndrii Nakryiko return bpf_tracing_prog_attach(prog, 0, 0); 32829e4e01dfSKP Singh case BPF_PROG_TYPE_RAW_TRACEPOINT: 32839e4e01dfSKP Singh case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 3284df86ca0dSAndrii Nakryiko if (strncpy_from_user(buf, user_tp_name, sizeof(buf) - 1) < 0) 3285df86ca0dSAndrii Nakryiko return -EFAULT; 3286ac4414b5SAlexei Starovoitov buf[sizeof(buf) - 1] = 0; 3287ac4414b5SAlexei Starovoitov tp_name = buf; 32889e4e01dfSKP Singh break; 32899e4e01dfSKP Singh default: 3290df86ca0dSAndrii Nakryiko return -EINVAL; 3291ac4414b5SAlexei Starovoitov } 3292c4f6699dSAlexei Starovoitov 3293a38d1107SMatt Mullins btp = bpf_get_raw_tracepoint(tp_name); 3294df86ca0dSAndrii Nakryiko if (!btp) 3295df86ca0dSAndrii Nakryiko return -ENOENT; 3296c4f6699dSAlexei Starovoitov 3297babf3164SAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER); 3298babf3164SAndrii Nakryiko if (!link) { 3299a38d1107SMatt Mullins err = -ENOMEM; 3300a38d1107SMatt Mullins goto out_put_btp; 3301a38d1107SMatt Mullins } 3302f2e10bffSAndrii Nakryiko bpf_link_init(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT, 3303f2e10bffSAndrii Nakryiko &bpf_raw_tp_link_lops, prog); 3304babf3164SAndrii Nakryiko link->btp = btp; 3305c4f6699dSAlexei Starovoitov 3306a3b80e10SAndrii Nakryiko err = bpf_link_prime(&link->link, &link_primer); 3307a3b80e10SAndrii Nakryiko if (err) { 3308babf3164SAndrii Nakryiko kfree(link); 3309babf3164SAndrii Nakryiko goto out_put_btp; 3310c4f6699dSAlexei Starovoitov } 3311babf3164SAndrii Nakryiko 3312babf3164SAndrii Nakryiko err = bpf_probe_register(link->btp, prog); 3313babf3164SAndrii Nakryiko if (err) { 3314a3b80e10SAndrii Nakryiko bpf_link_cleanup(&link_primer); 3315babf3164SAndrii Nakryiko goto out_put_btp; 3316babf3164SAndrii Nakryiko } 3317babf3164SAndrii Nakryiko 3318a3b80e10SAndrii Nakryiko return bpf_link_settle(&link_primer); 3319c4f6699dSAlexei Starovoitov 3320a38d1107SMatt Mullins out_put_btp: 3321a38d1107SMatt Mullins bpf_put_raw_tracepoint(btp); 3322c4f6699dSAlexei Starovoitov return err; 3323c4f6699dSAlexei Starovoitov } 3324c4f6699dSAlexei Starovoitov 3325df86ca0dSAndrii Nakryiko #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd 3326df86ca0dSAndrii Nakryiko 3327df86ca0dSAndrii Nakryiko static int bpf_raw_tracepoint_open(const union bpf_attr *attr) 3328df86ca0dSAndrii Nakryiko { 3329df86ca0dSAndrii Nakryiko struct bpf_prog *prog; 3330df86ca0dSAndrii Nakryiko int fd; 3331df86ca0dSAndrii Nakryiko 3332df86ca0dSAndrii Nakryiko if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN)) 3333df86ca0dSAndrii Nakryiko return -EINVAL; 3334df86ca0dSAndrii Nakryiko 3335df86ca0dSAndrii Nakryiko prog = bpf_prog_get(attr->raw_tracepoint.prog_fd); 3336df86ca0dSAndrii Nakryiko if (IS_ERR(prog)) 3337df86ca0dSAndrii Nakryiko return PTR_ERR(prog); 3338df86ca0dSAndrii Nakryiko 3339df86ca0dSAndrii Nakryiko fd = bpf_raw_tp_link_attach(prog, u64_to_user_ptr(attr->raw_tracepoint.name)); 3340df86ca0dSAndrii Nakryiko if (fd < 0) 3341df86ca0dSAndrii Nakryiko bpf_prog_put(prog); 3342df86ca0dSAndrii Nakryiko return fd; 3343df86ca0dSAndrii Nakryiko } 3344df86ca0dSAndrii Nakryiko 334533491588SAnders Roxell static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, 334633491588SAnders Roxell enum bpf_attach_type attach_type) 334733491588SAnders Roxell { 334833491588SAnders Roxell switch (prog->type) { 334933491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK: 335033491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 33510d01da6aSStanislav Fomichev case BPF_PROG_TYPE_CGROUP_SOCKOPT: 3352e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP: 335333491588SAnders Roxell return attach_type == prog->expected_attach_type ? 0 : -EINVAL; 33545cf1e914Sbrakmo case BPF_PROG_TYPE_CGROUP_SKB: 33552c78ee89SAlexei Starovoitov if (!capable(CAP_NET_ADMIN)) 33562c78ee89SAlexei Starovoitov /* cg-skb progs can be loaded by unpriv user. 33572c78ee89SAlexei Starovoitov * check permissions at attach time. 33582c78ee89SAlexei Starovoitov */ 33592c78ee89SAlexei Starovoitov return -EPERM; 33605cf1e914Sbrakmo return prog->enforce_expected_attach_type && 33615cf1e914Sbrakmo prog->expected_attach_type != attach_type ? 33625cf1e914Sbrakmo -EINVAL : 0; 336333491588SAnders Roxell default: 336433491588SAnders Roxell return 0; 336533491588SAnders Roxell } 336633491588SAnders Roxell } 336733491588SAnders Roxell 3368e28784e3SAndrii Nakryiko static enum bpf_prog_type 3369e28784e3SAndrii Nakryiko attach_type_to_prog_type(enum bpf_attach_type attach_type) 3370e28784e3SAndrii Nakryiko { 3371e28784e3SAndrii Nakryiko switch (attach_type) { 3372e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_INGRESS: 3373e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_EGRESS: 3374e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SKB; 3375e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_SOCK_CREATE: 3376f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 3377e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_POST_BIND: 3378e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_POST_BIND: 3379e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCK; 3380e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_BIND: 3381e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_BIND: 3382e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_CONNECT: 3383e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_CONNECT: 33841b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 33851b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 33861b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 33871b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 3388e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP4_SENDMSG: 3389e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP6_SENDMSG: 3390e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP4_RECVMSG: 3391e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP6_RECVMSG: 3392e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 3393e28784e3SAndrii Nakryiko case BPF_CGROUP_SOCK_OPS: 3394e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SOCK_OPS; 3395e28784e3SAndrii Nakryiko case BPF_CGROUP_DEVICE: 3396e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_DEVICE; 3397e28784e3SAndrii Nakryiko case BPF_SK_MSG_VERDICT: 3398e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SK_MSG; 3399e28784e3SAndrii Nakryiko case BPF_SK_SKB_STREAM_PARSER: 3400e28784e3SAndrii Nakryiko case BPF_SK_SKB_STREAM_VERDICT: 3401a7ba4558SCong Wang case BPF_SK_SKB_VERDICT: 3402e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SK_SKB; 3403e28784e3SAndrii Nakryiko case BPF_LIRC_MODE2: 3404e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_LIRC_MODE2; 3405e28784e3SAndrii Nakryiko case BPF_FLOW_DISSECTOR: 3406e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_FLOW_DISSECTOR; 3407e28784e3SAndrii Nakryiko case BPF_CGROUP_SYSCTL: 3408e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SYSCTL; 3409e28784e3SAndrii Nakryiko case BPF_CGROUP_GETSOCKOPT: 3410e28784e3SAndrii Nakryiko case BPF_CGROUP_SETSOCKOPT: 3411e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCKOPT; 3412de4e05caSYonghong Song case BPF_TRACE_ITER: 3413df86ca0dSAndrii Nakryiko case BPF_TRACE_RAW_TP: 3414df86ca0dSAndrii Nakryiko case BPF_TRACE_FENTRY: 3415df86ca0dSAndrii Nakryiko case BPF_TRACE_FEXIT: 3416df86ca0dSAndrii Nakryiko case BPF_MODIFY_RETURN: 3417de4e05caSYonghong Song return BPF_PROG_TYPE_TRACING; 3418df86ca0dSAndrii Nakryiko case BPF_LSM_MAC: 3419df86ca0dSAndrii Nakryiko return BPF_PROG_TYPE_LSM; 3420e9ddbb77SJakub Sitnicki case BPF_SK_LOOKUP: 3421e9ddbb77SJakub Sitnicki return BPF_PROG_TYPE_SK_LOOKUP; 3422aa8d3a71SAndrii Nakryiko case BPF_XDP: 3423aa8d3a71SAndrii Nakryiko return BPF_PROG_TYPE_XDP; 3424e28784e3SAndrii Nakryiko default: 3425e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_UNSPEC; 3426e28784e3SAndrii Nakryiko } 3427e28784e3SAndrii Nakryiko } 3428e28784e3SAndrii Nakryiko 34297dd68b32SAndrey Ignatov #define BPF_PROG_ATTACH_LAST_FIELD replace_bpf_fd 3430174a79ffSJohn Fastabend 3431324bda9eSAlexei Starovoitov #define BPF_F_ATTACH_MASK \ 34327dd68b32SAndrey Ignatov (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI | BPF_F_REPLACE) 3433324bda9eSAlexei Starovoitov 3434f4324551SDaniel Mack static int bpf_prog_attach(const union bpf_attr *attr) 3435f4324551SDaniel Mack { 34367f677633SAlexei Starovoitov enum bpf_prog_type ptype; 3437f4324551SDaniel Mack struct bpf_prog *prog; 34387f677633SAlexei Starovoitov int ret; 3439f4324551SDaniel Mack 3440f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_ATTACH)) 3441f4324551SDaniel Mack return -EINVAL; 3442f4324551SDaniel Mack 3443324bda9eSAlexei Starovoitov if (attr->attach_flags & ~BPF_F_ATTACH_MASK) 34447f677633SAlexei Starovoitov return -EINVAL; 34457f677633SAlexei Starovoitov 3446e28784e3SAndrii Nakryiko ptype = attach_type_to_prog_type(attr->attach_type); 3447e28784e3SAndrii Nakryiko if (ptype == BPF_PROG_TYPE_UNSPEC) 3448b2cd1257SDavid Ahern return -EINVAL; 3449b2cd1257SDavid Ahern 3450b2cd1257SDavid Ahern prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 3451f4324551SDaniel Mack if (IS_ERR(prog)) 3452f4324551SDaniel Mack return PTR_ERR(prog); 3453f4324551SDaniel Mack 34545e43f899SAndrey Ignatov if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) { 34555e43f899SAndrey Ignatov bpf_prog_put(prog); 34565e43f899SAndrey Ignatov return -EINVAL; 34575e43f899SAndrey Ignatov } 34585e43f899SAndrey Ignatov 3459fdb5c453SSean Young switch (ptype) { 3460fdb5c453SSean Young case BPF_PROG_TYPE_SK_SKB: 3461fdb5c453SSean Young case BPF_PROG_TYPE_SK_MSG: 3462604326b4SDaniel Borkmann ret = sock_map_get_from_fd(attr, prog); 3463fdb5c453SSean Young break; 3464fdb5c453SSean Young case BPF_PROG_TYPE_LIRC_MODE2: 3465fdb5c453SSean Young ret = lirc_prog_attach(attr, prog); 3466fdb5c453SSean Young break; 3467d58e468bSPetar Penkov case BPF_PROG_TYPE_FLOW_DISSECTOR: 3468a3fd7ceeSJakub Sitnicki ret = netns_bpf_prog_attach(attr, prog); 3469d58e468bSPetar Penkov break; 3470e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 3471e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 3472e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 3473e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 3474e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 3475e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 3476e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 3477fdb5c453SSean Young ret = cgroup_bpf_prog_attach(attr, ptype, prog); 3478e28784e3SAndrii Nakryiko break; 3479e28784e3SAndrii Nakryiko default: 3480e28784e3SAndrii Nakryiko ret = -EINVAL; 3481f4324551SDaniel Mack } 3482f4324551SDaniel Mack 34837f677633SAlexei Starovoitov if (ret) 34847f677633SAlexei Starovoitov bpf_prog_put(prog); 34857f677633SAlexei Starovoitov return ret; 3486f4324551SDaniel Mack } 3487f4324551SDaniel Mack 3488f4324551SDaniel Mack #define BPF_PROG_DETACH_LAST_FIELD attach_type 3489f4324551SDaniel Mack 3490f4324551SDaniel Mack static int bpf_prog_detach(const union bpf_attr *attr) 3491f4324551SDaniel Mack { 3492324bda9eSAlexei Starovoitov enum bpf_prog_type ptype; 3493f4324551SDaniel Mack 3494f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_DETACH)) 3495f4324551SDaniel Mack return -EINVAL; 3496f4324551SDaniel Mack 3497e28784e3SAndrii Nakryiko ptype = attach_type_to_prog_type(attr->attach_type); 3498e28784e3SAndrii Nakryiko 3499e28784e3SAndrii Nakryiko switch (ptype) { 3500e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SK_MSG: 3501e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SK_SKB: 3502bb0de313SLorenz Bauer return sock_map_prog_detach(attr, ptype); 3503e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_LIRC_MODE2: 3504f4364dcfSSean Young return lirc_prog_detach(attr); 3505e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_FLOW_DISSECTOR: 35064ac2add6SLorenz Bauer return netns_bpf_prog_detach(attr, ptype); 3507e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 3508e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 3509e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 3510e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 3511e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 3512e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 3513e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 3514e28784e3SAndrii Nakryiko return cgroup_bpf_prog_detach(attr, ptype); 3515f4324551SDaniel Mack default: 3516f4324551SDaniel Mack return -EINVAL; 3517f4324551SDaniel Mack } 3518f4324551SDaniel Mack } 351940304b2aSLawrence Brakmo 3520468e2f64SAlexei Starovoitov #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt 3521468e2f64SAlexei Starovoitov 3522468e2f64SAlexei Starovoitov static int bpf_prog_query(const union bpf_attr *attr, 3523468e2f64SAlexei Starovoitov union bpf_attr __user *uattr) 3524468e2f64SAlexei Starovoitov { 3525468e2f64SAlexei Starovoitov if (!capable(CAP_NET_ADMIN)) 3526468e2f64SAlexei Starovoitov return -EPERM; 3527468e2f64SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_QUERY)) 3528468e2f64SAlexei Starovoitov return -EINVAL; 3529468e2f64SAlexei Starovoitov if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE) 3530468e2f64SAlexei Starovoitov return -EINVAL; 3531468e2f64SAlexei Starovoitov 3532468e2f64SAlexei Starovoitov switch (attr->query.attach_type) { 3533468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_INGRESS: 3534468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_EGRESS: 3535468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_SOCK_CREATE: 3536f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 35374fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 35384fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 3539aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 3540aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 3541d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 3542d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 35431b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 35441b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 35451b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 35461b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 35471cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 35481cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 3549983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG: 3550983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG: 3551468e2f64SAlexei Starovoitov case BPF_CGROUP_SOCK_OPS: 3552ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 35537b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 35540d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT: 35550d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT: 3556e28784e3SAndrii Nakryiko return cgroup_bpf_prog_query(attr, uattr); 3557f4364dcfSSean Young case BPF_LIRC_MODE2: 3558f4364dcfSSean Young return lirc_prog_query(attr, uattr); 3559118c8e9aSStanislav Fomichev case BPF_FLOW_DISSECTOR: 3560e9ddbb77SJakub Sitnicki case BPF_SK_LOOKUP: 3561a3fd7ceeSJakub Sitnicki return netns_bpf_prog_query(attr, uattr); 3562748cd572SDi Zhu case BPF_SK_SKB_STREAM_PARSER: 3563748cd572SDi Zhu case BPF_SK_SKB_STREAM_VERDICT: 3564748cd572SDi Zhu case BPF_SK_MSG_VERDICT: 3565748cd572SDi Zhu case BPF_SK_SKB_VERDICT: 3566748cd572SDi Zhu return sock_map_bpf_prog_query(attr, uattr); 3567468e2f64SAlexei Starovoitov default: 3568468e2f64SAlexei Starovoitov return -EINVAL; 3569468e2f64SAlexei Starovoitov } 3570468e2f64SAlexei Starovoitov } 3571f4324551SDaniel Mack 3572b530e9e1SToke Høiland-Jørgensen #define BPF_PROG_TEST_RUN_LAST_FIELD test.batch_size 35731cf1cae9SAlexei Starovoitov 35741cf1cae9SAlexei Starovoitov static int bpf_prog_test_run(const union bpf_attr *attr, 35751cf1cae9SAlexei Starovoitov union bpf_attr __user *uattr) 35761cf1cae9SAlexei Starovoitov { 35771cf1cae9SAlexei Starovoitov struct bpf_prog *prog; 35781cf1cae9SAlexei Starovoitov int ret = -ENOTSUPP; 35791cf1cae9SAlexei Starovoitov 35801cf1cae9SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 35811cf1cae9SAlexei Starovoitov return -EINVAL; 35821cf1cae9SAlexei Starovoitov 3583b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_in && !attr->test.ctx_in) || 3584b0b9395dSStanislav Fomichev (!attr->test.ctx_size_in && attr->test.ctx_in)) 3585b0b9395dSStanislav Fomichev return -EINVAL; 3586b0b9395dSStanislav Fomichev 3587b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_out && !attr->test.ctx_out) || 3588b0b9395dSStanislav Fomichev (!attr->test.ctx_size_out && attr->test.ctx_out)) 3589b0b9395dSStanislav Fomichev return -EINVAL; 3590b0b9395dSStanislav Fomichev 35911cf1cae9SAlexei Starovoitov prog = bpf_prog_get(attr->test.prog_fd); 35921cf1cae9SAlexei Starovoitov if (IS_ERR(prog)) 35931cf1cae9SAlexei Starovoitov return PTR_ERR(prog); 35941cf1cae9SAlexei Starovoitov 35951cf1cae9SAlexei Starovoitov if (prog->aux->ops->test_run) 35961cf1cae9SAlexei Starovoitov ret = prog->aux->ops->test_run(prog, attr, uattr); 35971cf1cae9SAlexei Starovoitov 35981cf1cae9SAlexei Starovoitov bpf_prog_put(prog); 35991cf1cae9SAlexei Starovoitov return ret; 36001cf1cae9SAlexei Starovoitov } 36011cf1cae9SAlexei Starovoitov 360234ad5580SMartin KaFai Lau #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 360334ad5580SMartin KaFai Lau 360434ad5580SMartin KaFai Lau static int bpf_obj_get_next_id(const union bpf_attr *attr, 360534ad5580SMartin KaFai Lau union bpf_attr __user *uattr, 360634ad5580SMartin KaFai Lau struct idr *idr, 360734ad5580SMartin KaFai Lau spinlock_t *lock) 360834ad5580SMartin KaFai Lau { 360934ad5580SMartin KaFai Lau u32 next_id = attr->start_id; 361034ad5580SMartin KaFai Lau int err = 0; 361134ad5580SMartin KaFai Lau 361234ad5580SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 361334ad5580SMartin KaFai Lau return -EINVAL; 361434ad5580SMartin KaFai Lau 361534ad5580SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 361634ad5580SMartin KaFai Lau return -EPERM; 361734ad5580SMartin KaFai Lau 361834ad5580SMartin KaFai Lau next_id++; 361934ad5580SMartin KaFai Lau spin_lock_bh(lock); 362034ad5580SMartin KaFai Lau if (!idr_get_next(idr, &next_id)) 362134ad5580SMartin KaFai Lau err = -ENOENT; 362234ad5580SMartin KaFai Lau spin_unlock_bh(lock); 362334ad5580SMartin KaFai Lau 362434ad5580SMartin KaFai Lau if (!err) 362534ad5580SMartin KaFai Lau err = put_user(next_id, &uattr->next_id); 362634ad5580SMartin KaFai Lau 362734ad5580SMartin KaFai Lau return err; 362834ad5580SMartin KaFai Lau } 362934ad5580SMartin KaFai Lau 36306086d29dSYonghong Song struct bpf_map *bpf_map_get_curr_or_next(u32 *id) 36316086d29dSYonghong Song { 36326086d29dSYonghong Song struct bpf_map *map; 36336086d29dSYonghong Song 36346086d29dSYonghong Song spin_lock_bh(&map_idr_lock); 36356086d29dSYonghong Song again: 36366086d29dSYonghong Song map = idr_get_next(&map_idr, id); 36376086d29dSYonghong Song if (map) { 36386086d29dSYonghong Song map = __bpf_map_inc_not_zero(map, false); 36396086d29dSYonghong Song if (IS_ERR(map)) { 36406086d29dSYonghong Song (*id)++; 36416086d29dSYonghong Song goto again; 36426086d29dSYonghong Song } 36436086d29dSYonghong Song } 36446086d29dSYonghong Song spin_unlock_bh(&map_idr_lock); 36456086d29dSYonghong Song 36466086d29dSYonghong Song return map; 36476086d29dSYonghong Song } 36486086d29dSYonghong Song 3649a228a64fSAlexei Starovoitov struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id) 3650a228a64fSAlexei Starovoitov { 3651a228a64fSAlexei Starovoitov struct bpf_prog *prog; 3652a228a64fSAlexei Starovoitov 3653a228a64fSAlexei Starovoitov spin_lock_bh(&prog_idr_lock); 3654a228a64fSAlexei Starovoitov again: 3655a228a64fSAlexei Starovoitov prog = idr_get_next(&prog_idr, id); 3656a228a64fSAlexei Starovoitov if (prog) { 3657a228a64fSAlexei Starovoitov prog = bpf_prog_inc_not_zero(prog); 3658a228a64fSAlexei Starovoitov if (IS_ERR(prog)) { 3659a228a64fSAlexei Starovoitov (*id)++; 3660a228a64fSAlexei Starovoitov goto again; 3661a228a64fSAlexei Starovoitov } 3662a228a64fSAlexei Starovoitov } 3663a228a64fSAlexei Starovoitov spin_unlock_bh(&prog_idr_lock); 3664a228a64fSAlexei Starovoitov 3665a228a64fSAlexei Starovoitov return prog; 3666a228a64fSAlexei Starovoitov } 3667a228a64fSAlexei Starovoitov 3668b16d9aa4SMartin KaFai Lau #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 3669b16d9aa4SMartin KaFai Lau 36707e6897f9SBjörn Töpel struct bpf_prog *bpf_prog_by_id(u32 id) 36717e6897f9SBjörn Töpel { 36727e6897f9SBjörn Töpel struct bpf_prog *prog; 36737e6897f9SBjörn Töpel 36747e6897f9SBjörn Töpel if (!id) 36757e6897f9SBjörn Töpel return ERR_PTR(-ENOENT); 36767e6897f9SBjörn Töpel 36777e6897f9SBjörn Töpel spin_lock_bh(&prog_idr_lock); 36787e6897f9SBjörn Töpel prog = idr_find(&prog_idr, id); 36797e6897f9SBjörn Töpel if (prog) 36807e6897f9SBjörn Töpel prog = bpf_prog_inc_not_zero(prog); 36817e6897f9SBjörn Töpel else 36827e6897f9SBjörn Töpel prog = ERR_PTR(-ENOENT); 36837e6897f9SBjörn Töpel spin_unlock_bh(&prog_idr_lock); 36847e6897f9SBjörn Töpel return prog; 36857e6897f9SBjörn Töpel } 36867e6897f9SBjörn Töpel 3687b16d9aa4SMartin KaFai Lau static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 3688b16d9aa4SMartin KaFai Lau { 3689b16d9aa4SMartin KaFai Lau struct bpf_prog *prog; 3690b16d9aa4SMartin KaFai Lau u32 id = attr->prog_id; 3691b16d9aa4SMartin KaFai Lau int fd; 3692b16d9aa4SMartin KaFai Lau 3693b16d9aa4SMartin KaFai Lau if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 3694b16d9aa4SMartin KaFai Lau return -EINVAL; 3695b16d9aa4SMartin KaFai Lau 3696b16d9aa4SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 3697b16d9aa4SMartin KaFai Lau return -EPERM; 3698b16d9aa4SMartin KaFai Lau 36997e6897f9SBjörn Töpel prog = bpf_prog_by_id(id); 3700b16d9aa4SMartin KaFai Lau if (IS_ERR(prog)) 3701b16d9aa4SMartin KaFai Lau return PTR_ERR(prog); 3702b16d9aa4SMartin KaFai Lau 3703b16d9aa4SMartin KaFai Lau fd = bpf_prog_new_fd(prog); 3704b16d9aa4SMartin KaFai Lau if (fd < 0) 3705b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 3706b16d9aa4SMartin KaFai Lau 3707b16d9aa4SMartin KaFai Lau return fd; 3708b16d9aa4SMartin KaFai Lau } 3709b16d9aa4SMartin KaFai Lau 37106e71b04aSChenbo Feng #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags 3711bd5f5f4eSMartin KaFai Lau 3712bd5f5f4eSMartin KaFai Lau static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 3713bd5f5f4eSMartin KaFai Lau { 3714bd5f5f4eSMartin KaFai Lau struct bpf_map *map; 3715bd5f5f4eSMartin KaFai Lau u32 id = attr->map_id; 37166e71b04aSChenbo Feng int f_flags; 3717bd5f5f4eSMartin KaFai Lau int fd; 3718bd5f5f4eSMartin KaFai Lau 37196e71b04aSChenbo Feng if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || 37206e71b04aSChenbo Feng attr->open_flags & ~BPF_OBJ_FLAG_MASK) 3721bd5f5f4eSMartin KaFai Lau return -EINVAL; 3722bd5f5f4eSMartin KaFai Lau 3723bd5f5f4eSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 3724bd5f5f4eSMartin KaFai Lau return -EPERM; 3725bd5f5f4eSMartin KaFai Lau 37266e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->open_flags); 37276e71b04aSChenbo Feng if (f_flags < 0) 37286e71b04aSChenbo Feng return f_flags; 37296e71b04aSChenbo Feng 3730bd5f5f4eSMartin KaFai Lau spin_lock_bh(&map_idr_lock); 3731bd5f5f4eSMartin KaFai Lau map = idr_find(&map_idr, id); 3732bd5f5f4eSMartin KaFai Lau if (map) 3733b0e4701cSStanislav Fomichev map = __bpf_map_inc_not_zero(map, true); 3734bd5f5f4eSMartin KaFai Lau else 3735bd5f5f4eSMartin KaFai Lau map = ERR_PTR(-ENOENT); 3736bd5f5f4eSMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 3737bd5f5f4eSMartin KaFai Lau 3738bd5f5f4eSMartin KaFai Lau if (IS_ERR(map)) 3739bd5f5f4eSMartin KaFai Lau return PTR_ERR(map); 3740bd5f5f4eSMartin KaFai Lau 37416e71b04aSChenbo Feng fd = bpf_map_new_fd(map, f_flags); 3742bd5f5f4eSMartin KaFai Lau if (fd < 0) 3743781e6282SPeng Sun bpf_map_put_with_uref(map); 3744bd5f5f4eSMartin KaFai Lau 3745bd5f5f4eSMartin KaFai Lau return fd; 3746bd5f5f4eSMartin KaFai Lau } 3747bd5f5f4eSMartin KaFai Lau 37487105e828SDaniel Borkmann static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, 3749d8eca5bbSDaniel Borkmann unsigned long addr, u32 *off, 3750d8eca5bbSDaniel Borkmann u32 *type) 37517105e828SDaniel Borkmann { 3752d8eca5bbSDaniel Borkmann const struct bpf_map *map; 37537105e828SDaniel Borkmann int i; 37547105e828SDaniel Borkmann 3755984fe94fSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 3756d8eca5bbSDaniel Borkmann for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { 3757d8eca5bbSDaniel Borkmann map = prog->aux->used_maps[i]; 3758d8eca5bbSDaniel Borkmann if (map == (void *)addr) { 3759d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_FD; 3760984fe94fSYiFei Zhu goto out; 3761d8eca5bbSDaniel Borkmann } 3762d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta) 3763d8eca5bbSDaniel Borkmann continue; 3764d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta(map, addr, off)) { 3765d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_VALUE; 3766984fe94fSYiFei Zhu goto out; 3767d8eca5bbSDaniel Borkmann } 3768d8eca5bbSDaniel Borkmann } 3769984fe94fSYiFei Zhu map = NULL; 3770d8eca5bbSDaniel Borkmann 3771984fe94fSYiFei Zhu out: 3772984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 3773984fe94fSYiFei Zhu return map; 37747105e828SDaniel Borkmann } 37757105e828SDaniel Borkmann 377663960260SKees Cook static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, 377763960260SKees Cook const struct cred *f_cred) 37787105e828SDaniel Borkmann { 37797105e828SDaniel Borkmann const struct bpf_map *map; 37807105e828SDaniel Borkmann struct bpf_insn *insns; 3781d8eca5bbSDaniel Borkmann u32 off, type; 37827105e828SDaniel Borkmann u64 imm; 378329fcb05bSAndrii Nakryiko u8 code; 37847105e828SDaniel Borkmann int i; 37857105e828SDaniel Borkmann 37867105e828SDaniel Borkmann insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), 37877105e828SDaniel Borkmann GFP_USER); 37887105e828SDaniel Borkmann if (!insns) 37897105e828SDaniel Borkmann return insns; 37907105e828SDaniel Borkmann 37917105e828SDaniel Borkmann for (i = 0; i < prog->len; i++) { 379229fcb05bSAndrii Nakryiko code = insns[i].code; 379329fcb05bSAndrii Nakryiko 379429fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_TAIL_CALL)) { 37957105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 37967105e828SDaniel Borkmann insns[i].imm = BPF_FUNC_tail_call; 37977105e828SDaniel Borkmann /* fall-through */ 37987105e828SDaniel Borkmann } 379929fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_CALL) || 380029fcb05bSAndrii Nakryiko code == (BPF_JMP | BPF_CALL_ARGS)) { 380129fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_CALL_ARGS)) 38027105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 380363960260SKees Cook if (!bpf_dump_raw_ok(f_cred)) 38047105e828SDaniel Borkmann insns[i].imm = 0; 38057105e828SDaniel Borkmann continue; 38067105e828SDaniel Borkmann } 380729fcb05bSAndrii Nakryiko if (BPF_CLASS(code) == BPF_LDX && BPF_MODE(code) == BPF_PROBE_MEM) { 380829fcb05bSAndrii Nakryiko insns[i].code = BPF_LDX | BPF_SIZE(code) | BPF_MEM; 380929fcb05bSAndrii Nakryiko continue; 381029fcb05bSAndrii Nakryiko } 38117105e828SDaniel Borkmann 381229fcb05bSAndrii Nakryiko if (code != (BPF_LD | BPF_IMM | BPF_DW)) 38137105e828SDaniel Borkmann continue; 38147105e828SDaniel Borkmann 38157105e828SDaniel Borkmann imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; 3816d8eca5bbSDaniel Borkmann map = bpf_map_from_imm(prog, imm, &off, &type); 38177105e828SDaniel Borkmann if (map) { 3818d8eca5bbSDaniel Borkmann insns[i].src_reg = type; 38197105e828SDaniel Borkmann insns[i].imm = map->id; 3820d8eca5bbSDaniel Borkmann insns[i + 1].imm = off; 38217105e828SDaniel Borkmann continue; 38227105e828SDaniel Borkmann } 38237105e828SDaniel Borkmann } 38247105e828SDaniel Borkmann 38257105e828SDaniel Borkmann return insns; 38267105e828SDaniel Borkmann } 38277105e828SDaniel Borkmann 3828c454a46bSMartin KaFai Lau static int set_info_rec_size(struct bpf_prog_info *info) 3829c454a46bSMartin KaFai Lau { 3830c454a46bSMartin KaFai Lau /* 3831c454a46bSMartin KaFai Lau * Ensure info.*_rec_size is the same as kernel expected size 3832c454a46bSMartin KaFai Lau * 3833c454a46bSMartin KaFai Lau * or 3834c454a46bSMartin KaFai Lau * 3835c454a46bSMartin KaFai Lau * Only allow zero *_rec_size if both _rec_size and _cnt are 3836c454a46bSMartin KaFai Lau * zero. In this case, the kernel will set the expected 3837c454a46bSMartin KaFai Lau * _rec_size back to the info. 3838c454a46bSMartin KaFai Lau */ 3839c454a46bSMartin KaFai Lau 384011d8b82dSYonghong Song if ((info->nr_func_info || info->func_info_rec_size) && 3841c454a46bSMartin KaFai Lau info->func_info_rec_size != sizeof(struct bpf_func_info)) 3842c454a46bSMartin KaFai Lau return -EINVAL; 3843c454a46bSMartin KaFai Lau 384411d8b82dSYonghong Song if ((info->nr_line_info || info->line_info_rec_size) && 3845c454a46bSMartin KaFai Lau info->line_info_rec_size != sizeof(struct bpf_line_info)) 3846c454a46bSMartin KaFai Lau return -EINVAL; 3847c454a46bSMartin KaFai Lau 384811d8b82dSYonghong Song if ((info->nr_jited_line_info || info->jited_line_info_rec_size) && 3849c454a46bSMartin KaFai Lau info->jited_line_info_rec_size != sizeof(__u64)) 3850c454a46bSMartin KaFai Lau return -EINVAL; 3851c454a46bSMartin KaFai Lau 3852c454a46bSMartin KaFai Lau info->func_info_rec_size = sizeof(struct bpf_func_info); 3853c454a46bSMartin KaFai Lau info->line_info_rec_size = sizeof(struct bpf_line_info); 3854c454a46bSMartin KaFai Lau info->jited_line_info_rec_size = sizeof(__u64); 3855c454a46bSMartin KaFai Lau 3856c454a46bSMartin KaFai Lau return 0; 3857c454a46bSMartin KaFai Lau } 3858c454a46bSMartin KaFai Lau 385963960260SKees Cook static int bpf_prog_get_info_by_fd(struct file *file, 386063960260SKees Cook struct bpf_prog *prog, 38611e270976SMartin KaFai Lau const union bpf_attr *attr, 38621e270976SMartin KaFai Lau union bpf_attr __user *uattr) 38631e270976SMartin KaFai Lau { 38641e270976SMartin KaFai Lau struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 38655c6f2588SGreg Kroah-Hartman struct bpf_prog_info info; 38661e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 386761a0abaeSEric Dumazet struct bpf_prog_kstats stats; 38681e270976SMartin KaFai Lau char __user *uinsns; 38691e270976SMartin KaFai Lau u32 ulen; 38701e270976SMartin KaFai Lau int err; 38711e270976SMartin KaFai Lau 3872af2ac3e1SAlexei Starovoitov err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len); 38731e270976SMartin KaFai Lau if (err) 38741e270976SMartin KaFai Lau return err; 38751e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 38761e270976SMartin KaFai Lau 38775c6f2588SGreg Kroah-Hartman memset(&info, 0, sizeof(info)); 38781e270976SMartin KaFai Lau if (copy_from_user(&info, uinfo, info_len)) 387989b09689SDaniel Borkmann return -EFAULT; 38801e270976SMartin KaFai Lau 38811e270976SMartin KaFai Lau info.type = prog->type; 38821e270976SMartin KaFai Lau info.id = prog->aux->id; 3883cb4d2b3fSMartin KaFai Lau info.load_time = prog->aux->load_time; 3884cb4d2b3fSMartin KaFai Lau info.created_by_uid = from_kuid_munged(current_user_ns(), 3885cb4d2b3fSMartin KaFai Lau prog->aux->user->uid); 3886b85fab0eSJiri Olsa info.gpl_compatible = prog->gpl_compatible; 38871e270976SMartin KaFai Lau 38881e270976SMartin KaFai Lau memcpy(info.tag, prog->tag, sizeof(prog->tag)); 3889cb4d2b3fSMartin KaFai Lau memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); 3890cb4d2b3fSMartin KaFai Lau 3891984fe94fSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 3892cb4d2b3fSMartin KaFai Lau ulen = info.nr_map_ids; 3893cb4d2b3fSMartin KaFai Lau info.nr_map_ids = prog->aux->used_map_cnt; 3894cb4d2b3fSMartin KaFai Lau ulen = min_t(u32, info.nr_map_ids, ulen); 3895cb4d2b3fSMartin KaFai Lau if (ulen) { 3896721e08daSMartin KaFai Lau u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); 3897cb4d2b3fSMartin KaFai Lau u32 i; 3898cb4d2b3fSMartin KaFai Lau 3899cb4d2b3fSMartin KaFai Lau for (i = 0; i < ulen; i++) 3900cb4d2b3fSMartin KaFai Lau if (put_user(prog->aux->used_maps[i]->id, 3901984fe94fSYiFei Zhu &user_map_ids[i])) { 3902984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 3903cb4d2b3fSMartin KaFai Lau return -EFAULT; 3904cb4d2b3fSMartin KaFai Lau } 3905984fe94fSYiFei Zhu } 3906984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 39071e270976SMartin KaFai Lau 3908c454a46bSMartin KaFai Lau err = set_info_rec_size(&info); 3909c454a46bSMartin KaFai Lau if (err) 3910c454a46bSMartin KaFai Lau return err; 39117337224fSMartin KaFai Lau 39125f8f8b93SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 39135f8f8b93SAlexei Starovoitov info.run_time_ns = stats.nsecs; 39145f8f8b93SAlexei Starovoitov info.run_cnt = stats.cnt; 39159ed9e9baSAlexei Starovoitov info.recursion_misses = stats.misses; 39165f8f8b93SAlexei Starovoitov 3917aba64c7dSDave Marchevsky info.verified_insns = prog->aux->verified_insns; 3918aba64c7dSDave Marchevsky 39192c78ee89SAlexei Starovoitov if (!bpf_capable()) { 39201e270976SMartin KaFai Lau info.jited_prog_len = 0; 39211e270976SMartin KaFai Lau info.xlated_prog_len = 0; 3922dbecd738SSandipan Das info.nr_jited_ksyms = 0; 392328c2fae7SDaniel Borkmann info.nr_jited_func_lens = 0; 392411d8b82dSYonghong Song info.nr_func_info = 0; 392511d8b82dSYonghong Song info.nr_line_info = 0; 392611d8b82dSYonghong Song info.nr_jited_line_info = 0; 39271e270976SMartin KaFai Lau goto done; 39281e270976SMartin KaFai Lau } 39291e270976SMartin KaFai Lau 39301e270976SMartin KaFai Lau ulen = info.xlated_prog_len; 39319975a54bSDaniel Borkmann info.xlated_prog_len = bpf_prog_insn_size(prog); 39321e270976SMartin KaFai Lau if (info.xlated_prog_len && ulen) { 39337105e828SDaniel Borkmann struct bpf_insn *insns_sanitized; 39347105e828SDaniel Borkmann bool fault; 39357105e828SDaniel Borkmann 393663960260SKees Cook if (prog->blinded && !bpf_dump_raw_ok(file->f_cred)) { 39377105e828SDaniel Borkmann info.xlated_prog_insns = 0; 39387105e828SDaniel Borkmann goto done; 39397105e828SDaniel Borkmann } 394063960260SKees Cook insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred); 39417105e828SDaniel Borkmann if (!insns_sanitized) 39427105e828SDaniel Borkmann return -ENOMEM; 39431e270976SMartin KaFai Lau uinsns = u64_to_user_ptr(info.xlated_prog_insns); 39441e270976SMartin KaFai Lau ulen = min_t(u32, info.xlated_prog_len, ulen); 39457105e828SDaniel Borkmann fault = copy_to_user(uinsns, insns_sanitized, ulen); 39467105e828SDaniel Borkmann kfree(insns_sanitized); 39477105e828SDaniel Borkmann if (fault) 39481e270976SMartin KaFai Lau return -EFAULT; 39491e270976SMartin KaFai Lau } 39501e270976SMartin KaFai Lau 3951675fc275SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 3952675fc275SJakub Kicinski err = bpf_prog_offload_info_fill(&info, prog); 3953675fc275SJakub Kicinski if (err) 3954675fc275SJakub Kicinski return err; 3955fcfb126dSJiong Wang goto done; 3956fcfb126dSJiong Wang } 3957fcfb126dSJiong Wang 3958fcfb126dSJiong Wang /* NOTE: the following code is supposed to be skipped for offload. 3959fcfb126dSJiong Wang * bpf_prog_offload_info_fill() is the place to fill similar fields 3960fcfb126dSJiong Wang * for offload. 3961fcfb126dSJiong Wang */ 3962fcfb126dSJiong Wang ulen = info.jited_prog_len; 39634d56a76eSSandipan Das if (prog->aux->func_cnt) { 39644d56a76eSSandipan Das u32 i; 39654d56a76eSSandipan Das 39664d56a76eSSandipan Das info.jited_prog_len = 0; 39674d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) 39684d56a76eSSandipan Das info.jited_prog_len += prog->aux->func[i]->jited_len; 39694d56a76eSSandipan Das } else { 3970fcfb126dSJiong Wang info.jited_prog_len = prog->jited_len; 39714d56a76eSSandipan Das } 39724d56a76eSSandipan Das 3973fcfb126dSJiong Wang if (info.jited_prog_len && ulen) { 397463960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 3975fcfb126dSJiong Wang uinsns = u64_to_user_ptr(info.jited_prog_insns); 3976fcfb126dSJiong Wang ulen = min_t(u32, info.jited_prog_len, ulen); 39774d56a76eSSandipan Das 39784d56a76eSSandipan Das /* for multi-function programs, copy the JITed 39794d56a76eSSandipan Das * instructions for all the functions 39804d56a76eSSandipan Das */ 39814d56a76eSSandipan Das if (prog->aux->func_cnt) { 39824d56a76eSSandipan Das u32 len, free, i; 39834d56a76eSSandipan Das u8 *img; 39844d56a76eSSandipan Das 39854d56a76eSSandipan Das free = ulen; 39864d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) { 39874d56a76eSSandipan Das len = prog->aux->func[i]->jited_len; 39884d56a76eSSandipan Das len = min_t(u32, len, free); 39894d56a76eSSandipan Das img = (u8 *) prog->aux->func[i]->bpf_func; 39904d56a76eSSandipan Das if (copy_to_user(uinsns, img, len)) 39914d56a76eSSandipan Das return -EFAULT; 39924d56a76eSSandipan Das uinsns += len; 39934d56a76eSSandipan Das free -= len; 39944d56a76eSSandipan Das if (!free) 39954d56a76eSSandipan Das break; 39964d56a76eSSandipan Das } 39974d56a76eSSandipan Das } else { 3998fcfb126dSJiong Wang if (copy_to_user(uinsns, prog->bpf_func, ulen)) 3999fcfb126dSJiong Wang return -EFAULT; 40004d56a76eSSandipan Das } 4001fcfb126dSJiong Wang } else { 4002fcfb126dSJiong Wang info.jited_prog_insns = 0; 4003fcfb126dSJiong Wang } 4004675fc275SJakub Kicinski } 4005675fc275SJakub Kicinski 4006dbecd738SSandipan Das ulen = info.nr_jited_ksyms; 4007ff1889fcSSong Liu info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; 40087a5725ddSSong Liu if (ulen) { 400963960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 4010ff1889fcSSong Liu unsigned long ksym_addr; 4011dbecd738SSandipan Das u64 __user *user_ksyms; 4012dbecd738SSandipan Das u32 i; 4013dbecd738SSandipan Das 4014dbecd738SSandipan Das /* copy the address of the kernel symbol 4015dbecd738SSandipan Das * corresponding to each function 4016dbecd738SSandipan Das */ 4017dbecd738SSandipan Das ulen = min_t(u32, info.nr_jited_ksyms, ulen); 4018dbecd738SSandipan Das user_ksyms = u64_to_user_ptr(info.jited_ksyms); 4019ff1889fcSSong Liu if (prog->aux->func_cnt) { 4020dbecd738SSandipan Das for (i = 0; i < ulen; i++) { 4021ff1889fcSSong Liu ksym_addr = (unsigned long) 4022ff1889fcSSong Liu prog->aux->func[i]->bpf_func; 4023ff1889fcSSong Liu if (put_user((u64) ksym_addr, 4024ff1889fcSSong Liu &user_ksyms[i])) 4025ff1889fcSSong Liu return -EFAULT; 4026ff1889fcSSong Liu } 4027ff1889fcSSong Liu } else { 4028ff1889fcSSong Liu ksym_addr = (unsigned long) prog->bpf_func; 4029ff1889fcSSong Liu if (put_user((u64) ksym_addr, &user_ksyms[0])) 4030dbecd738SSandipan Das return -EFAULT; 4031dbecd738SSandipan Das } 4032dbecd738SSandipan Das } else { 4033dbecd738SSandipan Das info.jited_ksyms = 0; 4034dbecd738SSandipan Das } 4035dbecd738SSandipan Das } 4036dbecd738SSandipan Das 4037815581c1SSandipan Das ulen = info.nr_jited_func_lens; 4038ff1889fcSSong Liu info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; 40397a5725ddSSong Liu if (ulen) { 404063960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 4041815581c1SSandipan Das u32 __user *user_lens; 4042815581c1SSandipan Das u32 func_len, i; 4043815581c1SSandipan Das 4044815581c1SSandipan Das /* copy the JITed image lengths for each function */ 4045815581c1SSandipan Das ulen = min_t(u32, info.nr_jited_func_lens, ulen); 4046815581c1SSandipan Das user_lens = u64_to_user_ptr(info.jited_func_lens); 4047ff1889fcSSong Liu if (prog->aux->func_cnt) { 4048815581c1SSandipan Das for (i = 0; i < ulen; i++) { 4049ff1889fcSSong Liu func_len = 4050ff1889fcSSong Liu prog->aux->func[i]->jited_len; 4051815581c1SSandipan Das if (put_user(func_len, &user_lens[i])) 4052815581c1SSandipan Das return -EFAULT; 4053815581c1SSandipan Das } 4054815581c1SSandipan Das } else { 4055ff1889fcSSong Liu func_len = prog->jited_len; 4056ff1889fcSSong Liu if (put_user(func_len, &user_lens[0])) 4057ff1889fcSSong Liu return -EFAULT; 4058ff1889fcSSong Liu } 4059ff1889fcSSong Liu } else { 4060815581c1SSandipan Das info.jited_func_lens = 0; 4061815581c1SSandipan Das } 4062815581c1SSandipan Das } 4063815581c1SSandipan Das 40647337224fSMartin KaFai Lau if (prog->aux->btf) 406522dc4a0fSAndrii Nakryiko info.btf_id = btf_obj_id(prog->aux->btf); 4066838e9690SYonghong Song 406711d8b82dSYonghong Song ulen = info.nr_func_info; 406811d8b82dSYonghong Song info.nr_func_info = prog->aux->func_info_cnt; 406911d8b82dSYonghong Song if (info.nr_func_info && ulen) { 4070838e9690SYonghong Song char __user *user_finfo; 4071838e9690SYonghong Song 4072838e9690SYonghong Song user_finfo = u64_to_user_ptr(info.func_info); 407311d8b82dSYonghong Song ulen = min_t(u32, info.nr_func_info, ulen); 4074ba64e7d8SYonghong Song if (copy_to_user(user_finfo, prog->aux->func_info, 40757337224fSMartin KaFai Lau info.func_info_rec_size * ulen)) 4076838e9690SYonghong Song return -EFAULT; 4077838e9690SYonghong Song } 4078838e9690SYonghong Song 407911d8b82dSYonghong Song ulen = info.nr_line_info; 408011d8b82dSYonghong Song info.nr_line_info = prog->aux->nr_linfo; 408111d8b82dSYonghong Song if (info.nr_line_info && ulen) { 4082c454a46bSMartin KaFai Lau __u8 __user *user_linfo; 4083c454a46bSMartin KaFai Lau 4084c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.line_info); 408511d8b82dSYonghong Song ulen = min_t(u32, info.nr_line_info, ulen); 4086c454a46bSMartin KaFai Lau if (copy_to_user(user_linfo, prog->aux->linfo, 4087c454a46bSMartin KaFai Lau info.line_info_rec_size * ulen)) 4088c454a46bSMartin KaFai Lau return -EFAULT; 4089c454a46bSMartin KaFai Lau } 4090c454a46bSMartin KaFai Lau 409111d8b82dSYonghong Song ulen = info.nr_jited_line_info; 4092c454a46bSMartin KaFai Lau if (prog->aux->jited_linfo) 409311d8b82dSYonghong Song info.nr_jited_line_info = prog->aux->nr_linfo; 4094c454a46bSMartin KaFai Lau else 409511d8b82dSYonghong Song info.nr_jited_line_info = 0; 409611d8b82dSYonghong Song if (info.nr_jited_line_info && ulen) { 409763960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 4098c454a46bSMartin KaFai Lau __u64 __user *user_linfo; 4099c454a46bSMartin KaFai Lau u32 i; 4100c454a46bSMartin KaFai Lau 4101c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.jited_line_info); 410211d8b82dSYonghong Song ulen = min_t(u32, info.nr_jited_line_info, ulen); 4103c454a46bSMartin KaFai Lau for (i = 0; i < ulen; i++) { 4104c454a46bSMartin KaFai Lau if (put_user((__u64)(long)prog->aux->jited_linfo[i], 4105c454a46bSMartin KaFai Lau &user_linfo[i])) 4106c454a46bSMartin KaFai Lau return -EFAULT; 4107c454a46bSMartin KaFai Lau } 4108c454a46bSMartin KaFai Lau } else { 4109c454a46bSMartin KaFai Lau info.jited_line_info = 0; 4110c454a46bSMartin KaFai Lau } 4111c454a46bSMartin KaFai Lau } 4112c454a46bSMartin KaFai Lau 4113c872bdb3SSong Liu ulen = info.nr_prog_tags; 4114c872bdb3SSong Liu info.nr_prog_tags = prog->aux->func_cnt ? : 1; 4115c872bdb3SSong Liu if (ulen) { 4116c872bdb3SSong Liu __u8 __user (*user_prog_tags)[BPF_TAG_SIZE]; 4117c872bdb3SSong Liu u32 i; 4118c872bdb3SSong Liu 4119c872bdb3SSong Liu user_prog_tags = u64_to_user_ptr(info.prog_tags); 4120c872bdb3SSong Liu ulen = min_t(u32, info.nr_prog_tags, ulen); 4121c872bdb3SSong Liu if (prog->aux->func_cnt) { 4122c872bdb3SSong Liu for (i = 0; i < ulen; i++) { 4123c872bdb3SSong Liu if (copy_to_user(user_prog_tags[i], 4124c872bdb3SSong Liu prog->aux->func[i]->tag, 4125c872bdb3SSong Liu BPF_TAG_SIZE)) 4126c872bdb3SSong Liu return -EFAULT; 4127c872bdb3SSong Liu } 4128c872bdb3SSong Liu } else { 4129c872bdb3SSong Liu if (copy_to_user(user_prog_tags[0], 4130c872bdb3SSong Liu prog->tag, BPF_TAG_SIZE)) 4131c872bdb3SSong Liu return -EFAULT; 4132c872bdb3SSong Liu } 4133c872bdb3SSong Liu } 4134c872bdb3SSong Liu 41351e270976SMartin KaFai Lau done: 41361e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 41371e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 41381e270976SMartin KaFai Lau return -EFAULT; 41391e270976SMartin KaFai Lau 41401e270976SMartin KaFai Lau return 0; 41411e270976SMartin KaFai Lau } 41421e270976SMartin KaFai Lau 414363960260SKees Cook static int bpf_map_get_info_by_fd(struct file *file, 414463960260SKees Cook struct bpf_map *map, 41451e270976SMartin KaFai Lau const union bpf_attr *attr, 41461e270976SMartin KaFai Lau union bpf_attr __user *uattr) 41471e270976SMartin KaFai Lau { 41481e270976SMartin KaFai Lau struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 41495c6f2588SGreg Kroah-Hartman struct bpf_map_info info; 41501e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 41511e270976SMartin KaFai Lau int err; 41521e270976SMartin KaFai Lau 4153af2ac3e1SAlexei Starovoitov err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len); 41541e270976SMartin KaFai Lau if (err) 41551e270976SMartin KaFai Lau return err; 41561e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 41571e270976SMartin KaFai Lau 41585c6f2588SGreg Kroah-Hartman memset(&info, 0, sizeof(info)); 41591e270976SMartin KaFai Lau info.type = map->map_type; 41601e270976SMartin KaFai Lau info.id = map->id; 41611e270976SMartin KaFai Lau info.key_size = map->key_size; 41621e270976SMartin KaFai Lau info.value_size = map->value_size; 41631e270976SMartin KaFai Lau info.max_entries = map->max_entries; 41641e270976SMartin KaFai Lau info.map_flags = map->map_flags; 41659330986cSJoanne Koong info.map_extra = map->map_extra; 4166ad5b177bSMartin KaFai Lau memcpy(info.name, map->name, sizeof(map->name)); 41671e270976SMartin KaFai Lau 416878958fcaSMartin KaFai Lau if (map->btf) { 416922dc4a0fSAndrii Nakryiko info.btf_id = btf_obj_id(map->btf); 41709b2cf328SMartin KaFai Lau info.btf_key_type_id = map->btf_key_type_id; 41719b2cf328SMartin KaFai Lau info.btf_value_type_id = map->btf_value_type_id; 417278958fcaSMartin KaFai Lau } 417385d33df3SMartin KaFai Lau info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; 417478958fcaSMartin KaFai Lau 417552775b33SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 417652775b33SJakub Kicinski err = bpf_map_offload_info_fill(&info, map); 417752775b33SJakub Kicinski if (err) 417852775b33SJakub Kicinski return err; 417952775b33SJakub Kicinski } 418052775b33SJakub Kicinski 41811e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 41821e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 41831e270976SMartin KaFai Lau return -EFAULT; 41841e270976SMartin KaFai Lau 41851e270976SMartin KaFai Lau return 0; 41861e270976SMartin KaFai Lau } 41871e270976SMartin KaFai Lau 418863960260SKees Cook static int bpf_btf_get_info_by_fd(struct file *file, 418963960260SKees Cook struct btf *btf, 419062dab84cSMartin KaFai Lau const union bpf_attr *attr, 419162dab84cSMartin KaFai Lau union bpf_attr __user *uattr) 419262dab84cSMartin KaFai Lau { 419362dab84cSMartin KaFai Lau struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info); 419462dab84cSMartin KaFai Lau u32 info_len = attr->info.info_len; 419562dab84cSMartin KaFai Lau int err; 419662dab84cSMartin KaFai Lau 4197af2ac3e1SAlexei Starovoitov err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(*uinfo), info_len); 419862dab84cSMartin KaFai Lau if (err) 419962dab84cSMartin KaFai Lau return err; 420062dab84cSMartin KaFai Lau 420162dab84cSMartin KaFai Lau return btf_get_info_by_fd(btf, attr, uattr); 420262dab84cSMartin KaFai Lau } 420362dab84cSMartin KaFai Lau 420463960260SKees Cook static int bpf_link_get_info_by_fd(struct file *file, 420563960260SKees Cook struct bpf_link *link, 4206f2e10bffSAndrii Nakryiko const union bpf_attr *attr, 4207f2e10bffSAndrii Nakryiko union bpf_attr __user *uattr) 4208f2e10bffSAndrii Nakryiko { 4209f2e10bffSAndrii Nakryiko struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info); 4210f2e10bffSAndrii Nakryiko struct bpf_link_info info; 4211f2e10bffSAndrii Nakryiko u32 info_len = attr->info.info_len; 4212f2e10bffSAndrii Nakryiko int err; 4213f2e10bffSAndrii Nakryiko 4214af2ac3e1SAlexei Starovoitov err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len); 4215f2e10bffSAndrii Nakryiko if (err) 4216f2e10bffSAndrii Nakryiko return err; 4217f2e10bffSAndrii Nakryiko info_len = min_t(u32, sizeof(info), info_len); 4218f2e10bffSAndrii Nakryiko 4219f2e10bffSAndrii Nakryiko memset(&info, 0, sizeof(info)); 4220f2e10bffSAndrii Nakryiko if (copy_from_user(&info, uinfo, info_len)) 4221f2e10bffSAndrii Nakryiko return -EFAULT; 4222f2e10bffSAndrii Nakryiko 4223f2e10bffSAndrii Nakryiko info.type = link->type; 4224f2e10bffSAndrii Nakryiko info.id = link->id; 4225f2e10bffSAndrii Nakryiko info.prog_id = link->prog->aux->id; 4226f2e10bffSAndrii Nakryiko 4227f2e10bffSAndrii Nakryiko if (link->ops->fill_link_info) { 4228f2e10bffSAndrii Nakryiko err = link->ops->fill_link_info(link, &info); 4229f2e10bffSAndrii Nakryiko if (err) 4230f2e10bffSAndrii Nakryiko return err; 4231f2e10bffSAndrii Nakryiko } 4232f2e10bffSAndrii Nakryiko 4233f2e10bffSAndrii Nakryiko if (copy_to_user(uinfo, &info, info_len) || 4234f2e10bffSAndrii Nakryiko put_user(info_len, &uattr->info.info_len)) 4235f2e10bffSAndrii Nakryiko return -EFAULT; 4236f2e10bffSAndrii Nakryiko 4237f2e10bffSAndrii Nakryiko return 0; 4238f2e10bffSAndrii Nakryiko } 4239f2e10bffSAndrii Nakryiko 4240f2e10bffSAndrii Nakryiko 42411e270976SMartin KaFai Lau #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 42421e270976SMartin KaFai Lau 42431e270976SMartin KaFai Lau static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 42441e270976SMartin KaFai Lau union bpf_attr __user *uattr) 42451e270976SMartin KaFai Lau { 42461e270976SMartin KaFai Lau int ufd = attr->info.bpf_fd; 42471e270976SMartin KaFai Lau struct fd f; 42481e270976SMartin KaFai Lau int err; 42491e270976SMartin KaFai Lau 42501e270976SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 42511e270976SMartin KaFai Lau return -EINVAL; 42521e270976SMartin KaFai Lau 42531e270976SMartin KaFai Lau f = fdget(ufd); 42541e270976SMartin KaFai Lau if (!f.file) 42551e270976SMartin KaFai Lau return -EBADFD; 42561e270976SMartin KaFai Lau 42571e270976SMartin KaFai Lau if (f.file->f_op == &bpf_prog_fops) 425863960260SKees Cook err = bpf_prog_get_info_by_fd(f.file, f.file->private_data, attr, 42591e270976SMartin KaFai Lau uattr); 42601e270976SMartin KaFai Lau else if (f.file->f_op == &bpf_map_fops) 426163960260SKees Cook err = bpf_map_get_info_by_fd(f.file, f.file->private_data, attr, 42621e270976SMartin KaFai Lau uattr); 426360197cfbSMartin KaFai Lau else if (f.file->f_op == &btf_fops) 426463960260SKees Cook err = bpf_btf_get_info_by_fd(f.file, f.file->private_data, attr, uattr); 4265f2e10bffSAndrii Nakryiko else if (f.file->f_op == &bpf_link_fops) 426663960260SKees Cook err = bpf_link_get_info_by_fd(f.file, f.file->private_data, 4267f2e10bffSAndrii Nakryiko attr, uattr); 42681e270976SMartin KaFai Lau else 42691e270976SMartin KaFai Lau err = -EINVAL; 42701e270976SMartin KaFai Lau 42711e270976SMartin KaFai Lau fdput(f); 42721e270976SMartin KaFai Lau return err; 42731e270976SMartin KaFai Lau } 42741e270976SMartin KaFai Lau 4275f56a653cSMartin KaFai Lau #define BPF_BTF_LOAD_LAST_FIELD btf_log_level 4276f56a653cSMartin KaFai Lau 4277c571bd75SAlexei Starovoitov static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr) 4278f56a653cSMartin KaFai Lau { 4279f56a653cSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_LOAD)) 4280f56a653cSMartin KaFai Lau return -EINVAL; 4281f56a653cSMartin KaFai Lau 42822c78ee89SAlexei Starovoitov if (!bpf_capable()) 4283f56a653cSMartin KaFai Lau return -EPERM; 4284f56a653cSMartin KaFai Lau 4285c571bd75SAlexei Starovoitov return btf_new_fd(attr, uattr); 4286f56a653cSMartin KaFai Lau } 4287f56a653cSMartin KaFai Lau 428878958fcaSMartin KaFai Lau #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id 428978958fcaSMartin KaFai Lau 429078958fcaSMartin KaFai Lau static int bpf_btf_get_fd_by_id(const union bpf_attr *attr) 429178958fcaSMartin KaFai Lau { 429278958fcaSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID)) 429378958fcaSMartin KaFai Lau return -EINVAL; 429478958fcaSMartin KaFai Lau 429578958fcaSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 429678958fcaSMartin KaFai Lau return -EPERM; 429778958fcaSMartin KaFai Lau 429878958fcaSMartin KaFai Lau return btf_get_fd_by_id(attr->btf_id); 429978958fcaSMartin KaFai Lau } 430078958fcaSMartin KaFai Lau 430141bdc4b4SYonghong Song static int bpf_task_fd_query_copy(const union bpf_attr *attr, 430241bdc4b4SYonghong Song union bpf_attr __user *uattr, 430341bdc4b4SYonghong Song u32 prog_id, u32 fd_type, 430441bdc4b4SYonghong Song const char *buf, u64 probe_offset, 430541bdc4b4SYonghong Song u64 probe_addr) 430641bdc4b4SYonghong Song { 430741bdc4b4SYonghong Song char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf); 430841bdc4b4SYonghong Song u32 len = buf ? strlen(buf) : 0, input_len; 430941bdc4b4SYonghong Song int err = 0; 431041bdc4b4SYonghong Song 431141bdc4b4SYonghong Song if (put_user(len, &uattr->task_fd_query.buf_len)) 431241bdc4b4SYonghong Song return -EFAULT; 431341bdc4b4SYonghong Song input_len = attr->task_fd_query.buf_len; 431441bdc4b4SYonghong Song if (input_len && ubuf) { 431541bdc4b4SYonghong Song if (!len) { 431641bdc4b4SYonghong Song /* nothing to copy, just make ubuf NULL terminated */ 431741bdc4b4SYonghong Song char zero = '\0'; 431841bdc4b4SYonghong Song 431941bdc4b4SYonghong Song if (put_user(zero, ubuf)) 432041bdc4b4SYonghong Song return -EFAULT; 432141bdc4b4SYonghong Song } else if (input_len >= len + 1) { 432241bdc4b4SYonghong Song /* ubuf can hold the string with NULL terminator */ 432341bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, len + 1)) 432441bdc4b4SYonghong Song return -EFAULT; 432541bdc4b4SYonghong Song } else { 432641bdc4b4SYonghong Song /* ubuf cannot hold the string with NULL terminator, 432741bdc4b4SYonghong Song * do a partial copy with NULL terminator. 432841bdc4b4SYonghong Song */ 432941bdc4b4SYonghong Song char zero = '\0'; 433041bdc4b4SYonghong Song 433141bdc4b4SYonghong Song err = -ENOSPC; 433241bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, input_len - 1)) 433341bdc4b4SYonghong Song return -EFAULT; 433441bdc4b4SYonghong Song if (put_user(zero, ubuf + input_len - 1)) 433541bdc4b4SYonghong Song return -EFAULT; 433641bdc4b4SYonghong Song } 433741bdc4b4SYonghong Song } 433841bdc4b4SYonghong Song 433941bdc4b4SYonghong Song if (put_user(prog_id, &uattr->task_fd_query.prog_id) || 434041bdc4b4SYonghong Song put_user(fd_type, &uattr->task_fd_query.fd_type) || 434141bdc4b4SYonghong Song put_user(probe_offset, &uattr->task_fd_query.probe_offset) || 434241bdc4b4SYonghong Song put_user(probe_addr, &uattr->task_fd_query.probe_addr)) 434341bdc4b4SYonghong Song return -EFAULT; 434441bdc4b4SYonghong Song 434541bdc4b4SYonghong Song return err; 434641bdc4b4SYonghong Song } 434741bdc4b4SYonghong Song 434841bdc4b4SYonghong Song #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr 434941bdc4b4SYonghong Song 435041bdc4b4SYonghong Song static int bpf_task_fd_query(const union bpf_attr *attr, 435141bdc4b4SYonghong Song union bpf_attr __user *uattr) 435241bdc4b4SYonghong Song { 435341bdc4b4SYonghong Song pid_t pid = attr->task_fd_query.pid; 435441bdc4b4SYonghong Song u32 fd = attr->task_fd_query.fd; 435541bdc4b4SYonghong Song const struct perf_event *event; 435641bdc4b4SYonghong Song struct task_struct *task; 435741bdc4b4SYonghong Song struct file *file; 435841bdc4b4SYonghong Song int err; 435941bdc4b4SYonghong Song 436041bdc4b4SYonghong Song if (CHECK_ATTR(BPF_TASK_FD_QUERY)) 436141bdc4b4SYonghong Song return -EINVAL; 436241bdc4b4SYonghong Song 436341bdc4b4SYonghong Song if (!capable(CAP_SYS_ADMIN)) 436441bdc4b4SYonghong Song return -EPERM; 436541bdc4b4SYonghong Song 436641bdc4b4SYonghong Song if (attr->task_fd_query.flags != 0) 436741bdc4b4SYonghong Song return -EINVAL; 436841bdc4b4SYonghong Song 436941bdc4b4SYonghong Song task = get_pid_task(find_vpid(pid), PIDTYPE_PID); 437041bdc4b4SYonghong Song if (!task) 437141bdc4b4SYonghong Song return -ENOENT; 437241bdc4b4SYonghong Song 437341bdc4b4SYonghong Song err = 0; 4374b48845afSEric W. Biederman file = fget_task(task, fd); 4375b48845afSEric W. Biederman put_task_struct(task); 437641bdc4b4SYonghong Song if (!file) 4377b48845afSEric W. Biederman return -EBADF; 437841bdc4b4SYonghong Song 437970ed506cSAndrii Nakryiko if (file->f_op == &bpf_link_fops) { 438070ed506cSAndrii Nakryiko struct bpf_link *link = file->private_data; 438170ed506cSAndrii Nakryiko 4382a3b80e10SAndrii Nakryiko if (link->ops == &bpf_raw_tp_link_lops) { 438370ed506cSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 438470ed506cSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 438541bdc4b4SYonghong Song struct bpf_raw_event_map *btp = raw_tp->btp; 438641bdc4b4SYonghong Song 438741bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, 438870ed506cSAndrii Nakryiko raw_tp->link.prog->aux->id, 438941bdc4b4SYonghong Song BPF_FD_TYPE_RAW_TRACEPOINT, 439041bdc4b4SYonghong Song btp->tp->name, 0, 0); 439141bdc4b4SYonghong Song goto put_file; 439241bdc4b4SYonghong Song } 439370ed506cSAndrii Nakryiko goto out_not_supp; 439470ed506cSAndrii Nakryiko } 439541bdc4b4SYonghong Song 439641bdc4b4SYonghong Song event = perf_get_event(file); 439741bdc4b4SYonghong Song if (!IS_ERR(event)) { 439841bdc4b4SYonghong Song u64 probe_offset, probe_addr; 439941bdc4b4SYonghong Song u32 prog_id, fd_type; 440041bdc4b4SYonghong Song const char *buf; 440141bdc4b4SYonghong Song 440241bdc4b4SYonghong Song err = bpf_get_perf_event_info(event, &prog_id, &fd_type, 440341bdc4b4SYonghong Song &buf, &probe_offset, 440441bdc4b4SYonghong Song &probe_addr); 440541bdc4b4SYonghong Song if (!err) 440641bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, prog_id, 440741bdc4b4SYonghong Song fd_type, buf, 440841bdc4b4SYonghong Song probe_offset, 440941bdc4b4SYonghong Song probe_addr); 441041bdc4b4SYonghong Song goto put_file; 441141bdc4b4SYonghong Song } 441241bdc4b4SYonghong Song 441370ed506cSAndrii Nakryiko out_not_supp: 441441bdc4b4SYonghong Song err = -ENOTSUPP; 441541bdc4b4SYonghong Song put_file: 441641bdc4b4SYonghong Song fput(file); 441741bdc4b4SYonghong Song return err; 441841bdc4b4SYonghong Song } 441941bdc4b4SYonghong Song 4420cb4d03abSBrian Vazquez #define BPF_MAP_BATCH_LAST_FIELD batch.flags 4421cb4d03abSBrian Vazquez 4422cb4d03abSBrian Vazquez #define BPF_DO_BATCH(fn) \ 4423cb4d03abSBrian Vazquez do { \ 4424cb4d03abSBrian Vazquez if (!fn) { \ 4425cb4d03abSBrian Vazquez err = -ENOTSUPP; \ 4426cb4d03abSBrian Vazquez goto err_put; \ 4427cb4d03abSBrian Vazquez } \ 4428cb4d03abSBrian Vazquez err = fn(map, attr, uattr); \ 4429cb4d03abSBrian Vazquez } while (0) 4430cb4d03abSBrian Vazquez 4431cb4d03abSBrian Vazquez static int bpf_map_do_batch(const union bpf_attr *attr, 4432cb4d03abSBrian Vazquez union bpf_attr __user *uattr, 4433cb4d03abSBrian Vazquez int cmd) 4434cb4d03abSBrian Vazquez { 4435353050beSDaniel Borkmann bool has_read = cmd == BPF_MAP_LOOKUP_BATCH || 4436353050beSDaniel Borkmann cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH; 4437353050beSDaniel Borkmann bool has_write = cmd != BPF_MAP_LOOKUP_BATCH; 4438cb4d03abSBrian Vazquez struct bpf_map *map; 4439cb4d03abSBrian Vazquez int err, ufd; 4440cb4d03abSBrian Vazquez struct fd f; 4441cb4d03abSBrian Vazquez 4442cb4d03abSBrian Vazquez if (CHECK_ATTR(BPF_MAP_BATCH)) 4443cb4d03abSBrian Vazquez return -EINVAL; 4444cb4d03abSBrian Vazquez 4445cb4d03abSBrian Vazquez ufd = attr->batch.map_fd; 4446cb4d03abSBrian Vazquez f = fdget(ufd); 4447cb4d03abSBrian Vazquez map = __bpf_map_get(f); 4448cb4d03abSBrian Vazquez if (IS_ERR(map)) 4449cb4d03abSBrian Vazquez return PTR_ERR(map); 4450353050beSDaniel Borkmann if (has_write) 4451353050beSDaniel Borkmann bpf_map_write_active_inc(map); 4452353050beSDaniel Borkmann if (has_read && !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 4453cb4d03abSBrian Vazquez err = -EPERM; 4454cb4d03abSBrian Vazquez goto err_put; 4455cb4d03abSBrian Vazquez } 4456353050beSDaniel Borkmann if (has_write && !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 4457cb4d03abSBrian Vazquez err = -EPERM; 4458cb4d03abSBrian Vazquez goto err_put; 4459cb4d03abSBrian Vazquez } 4460cb4d03abSBrian Vazquez 4461cb4d03abSBrian Vazquez if (cmd == BPF_MAP_LOOKUP_BATCH) 4462cb4d03abSBrian Vazquez BPF_DO_BATCH(map->ops->map_lookup_batch); 446305799638SYonghong Song else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) 446405799638SYonghong Song BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch); 4465aa2e93b8SBrian Vazquez else if (cmd == BPF_MAP_UPDATE_BATCH) 4466aa2e93b8SBrian Vazquez BPF_DO_BATCH(map->ops->map_update_batch); 4467aa2e93b8SBrian Vazquez else 4468aa2e93b8SBrian Vazquez BPF_DO_BATCH(map->ops->map_delete_batch); 4469cb4d03abSBrian Vazquez err_put: 4470353050beSDaniel Borkmann if (has_write) 4471353050beSDaniel Borkmann bpf_map_write_active_dec(map); 4472cb4d03abSBrian Vazquez fdput(f); 4473cb4d03abSBrian Vazquez return err; 4474cb4d03abSBrian Vazquez } 4475cb4d03abSBrian Vazquez 4476ca74823cSJiri Olsa #define BPF_LINK_CREATE_LAST_FIELD link_create.kprobe_multi.cookies 4477af2ac3e1SAlexei Starovoitov static int link_create(union bpf_attr *attr, bpfptr_t uattr) 4478af6eea57SAndrii Nakryiko { 4479af6eea57SAndrii Nakryiko enum bpf_prog_type ptype; 4480af6eea57SAndrii Nakryiko struct bpf_prog *prog; 4481af6eea57SAndrii Nakryiko int ret; 4482af6eea57SAndrii Nakryiko 4483af6eea57SAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_CREATE)) 4484af6eea57SAndrii Nakryiko return -EINVAL; 4485af6eea57SAndrii Nakryiko 44864a1e7c0cSToke Høiland-Jørgensen prog = bpf_prog_get(attr->link_create.prog_fd); 4487af6eea57SAndrii Nakryiko if (IS_ERR(prog)) 4488af6eea57SAndrii Nakryiko return PTR_ERR(prog); 4489af6eea57SAndrii Nakryiko 4490af6eea57SAndrii Nakryiko ret = bpf_prog_attach_check_attach_type(prog, 4491af6eea57SAndrii Nakryiko attr->link_create.attach_type); 4492af6eea57SAndrii Nakryiko if (ret) 44934a1e7c0cSToke Høiland-Jørgensen goto out; 44944a1e7c0cSToke Høiland-Jørgensen 4495b89fbfbbSAndrii Nakryiko switch (prog->type) { 4496b89fbfbbSAndrii Nakryiko case BPF_PROG_TYPE_EXT: 4497df86ca0dSAndrii Nakryiko break; 4498b89fbfbbSAndrii Nakryiko case BPF_PROG_TYPE_PERF_EVENT: 4499b89fbfbbSAndrii Nakryiko case BPF_PROG_TYPE_TRACEPOINT: 4500b89fbfbbSAndrii Nakryiko if (attr->link_create.attach_type != BPF_PERF_EVENT) { 4501b89fbfbbSAndrii Nakryiko ret = -EINVAL; 4502b89fbfbbSAndrii Nakryiko goto out; 45034a1e7c0cSToke Høiland-Jørgensen } 4504b89fbfbbSAndrii Nakryiko break; 45050dcac272SJiri Olsa case BPF_PROG_TYPE_KPROBE: 45060dcac272SJiri Olsa if (attr->link_create.attach_type != BPF_PERF_EVENT && 45070dcac272SJiri Olsa attr->link_create.attach_type != BPF_TRACE_KPROBE_MULTI) { 45080dcac272SJiri Olsa ret = -EINVAL; 45090dcac272SJiri Olsa goto out; 45100dcac272SJiri Olsa } 45110dcac272SJiri Olsa break; 4512b89fbfbbSAndrii Nakryiko default: 45134a1e7c0cSToke Høiland-Jørgensen ptype = attach_type_to_prog_type(attr->link_create.attach_type); 45144a1e7c0cSToke Høiland-Jørgensen if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type) { 45154a1e7c0cSToke Høiland-Jørgensen ret = -EINVAL; 45164a1e7c0cSToke Høiland-Jørgensen goto out; 45174a1e7c0cSToke Høiland-Jørgensen } 4518b89fbfbbSAndrii Nakryiko break; 4519b89fbfbbSAndrii Nakryiko } 4520af6eea57SAndrii Nakryiko 4521df86ca0dSAndrii Nakryiko switch (prog->type) { 4522af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 4523af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 4524af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 4525af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 4526af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 4527af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 4528af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 4529af6eea57SAndrii Nakryiko ret = cgroup_bpf_link_attach(attr, prog); 4530af6eea57SAndrii Nakryiko break; 4531df86ca0dSAndrii Nakryiko case BPF_PROG_TYPE_EXT: 4532df86ca0dSAndrii Nakryiko ret = bpf_tracing_prog_attach(prog, 4533df86ca0dSAndrii Nakryiko attr->link_create.target_fd, 4534df86ca0dSAndrii Nakryiko attr->link_create.target_btf_id); 4535df86ca0dSAndrii Nakryiko break; 4536df86ca0dSAndrii Nakryiko case BPF_PROG_TYPE_LSM: 4537de4e05caSYonghong Song case BPF_PROG_TYPE_TRACING: 4538df86ca0dSAndrii Nakryiko if (attr->link_create.attach_type != prog->expected_attach_type) { 4539df86ca0dSAndrii Nakryiko ret = -EINVAL; 4540df86ca0dSAndrii Nakryiko goto out; 4541df86ca0dSAndrii Nakryiko } 4542df86ca0dSAndrii Nakryiko if (prog->expected_attach_type == BPF_TRACE_RAW_TP) 4543df86ca0dSAndrii Nakryiko ret = bpf_raw_tp_link_attach(prog, NULL); 4544df86ca0dSAndrii Nakryiko else if (prog->expected_attach_type == BPF_TRACE_ITER) 4545df86ca0dSAndrii Nakryiko ret = bpf_iter_link_attach(attr, uattr, prog); 4546df86ca0dSAndrii Nakryiko else 4547df86ca0dSAndrii Nakryiko ret = bpf_tracing_prog_attach(prog, 4548df86ca0dSAndrii Nakryiko attr->link_create.target_fd, 4549df86ca0dSAndrii Nakryiko attr->link_create.target_btf_id); 4550de4e05caSYonghong Song break; 45517f045a49SJakub Sitnicki case BPF_PROG_TYPE_FLOW_DISSECTOR: 4552e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP: 45537f045a49SJakub Sitnicki ret = netns_bpf_link_create(attr, prog); 45547f045a49SJakub Sitnicki break; 4555310ad797SAndrii Nakryiko #ifdef CONFIG_NET 4556aa8d3a71SAndrii Nakryiko case BPF_PROG_TYPE_XDP: 4557aa8d3a71SAndrii Nakryiko ret = bpf_xdp_link_attach(attr, prog); 4558aa8d3a71SAndrii Nakryiko break; 4559310ad797SAndrii Nakryiko #endif 4560b89fbfbbSAndrii Nakryiko case BPF_PROG_TYPE_PERF_EVENT: 4561b89fbfbbSAndrii Nakryiko case BPF_PROG_TYPE_TRACEPOINT: 4562b89fbfbbSAndrii Nakryiko ret = bpf_perf_link_attach(attr, prog); 4563b89fbfbbSAndrii Nakryiko break; 45640dcac272SJiri Olsa case BPF_PROG_TYPE_KPROBE: 45650dcac272SJiri Olsa if (attr->link_create.attach_type == BPF_PERF_EVENT) 45660dcac272SJiri Olsa ret = bpf_perf_link_attach(attr, prog); 45670dcac272SJiri Olsa else 45680dcac272SJiri Olsa ret = bpf_kprobe_multi_link_attach(attr, prog); 45690dcac272SJiri Olsa break; 4570af6eea57SAndrii Nakryiko default: 4571af6eea57SAndrii Nakryiko ret = -EINVAL; 4572af6eea57SAndrii Nakryiko } 4573af6eea57SAndrii Nakryiko 45744a1e7c0cSToke Høiland-Jørgensen out: 4575af6eea57SAndrii Nakryiko if (ret < 0) 4576af6eea57SAndrii Nakryiko bpf_prog_put(prog); 4577af6eea57SAndrii Nakryiko return ret; 4578af6eea57SAndrii Nakryiko } 4579af6eea57SAndrii Nakryiko 45800c991ebcSAndrii Nakryiko #define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd 45810c991ebcSAndrii Nakryiko 45820c991ebcSAndrii Nakryiko static int link_update(union bpf_attr *attr) 45830c991ebcSAndrii Nakryiko { 45840c991ebcSAndrii Nakryiko struct bpf_prog *old_prog = NULL, *new_prog; 45850c991ebcSAndrii Nakryiko struct bpf_link *link; 45860c991ebcSAndrii Nakryiko u32 flags; 45870c991ebcSAndrii Nakryiko int ret; 45880c991ebcSAndrii Nakryiko 45890c991ebcSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_UPDATE)) 45900c991ebcSAndrii Nakryiko return -EINVAL; 45910c991ebcSAndrii Nakryiko 45920c991ebcSAndrii Nakryiko flags = attr->link_update.flags; 45930c991ebcSAndrii Nakryiko if (flags & ~BPF_F_REPLACE) 45940c991ebcSAndrii Nakryiko return -EINVAL; 45950c991ebcSAndrii Nakryiko 45960c991ebcSAndrii Nakryiko link = bpf_link_get_from_fd(attr->link_update.link_fd); 45970c991ebcSAndrii Nakryiko if (IS_ERR(link)) 45980c991ebcSAndrii Nakryiko return PTR_ERR(link); 45990c991ebcSAndrii Nakryiko 46000c991ebcSAndrii Nakryiko new_prog = bpf_prog_get(attr->link_update.new_prog_fd); 46014adb7a4aSAndrii Nakryiko if (IS_ERR(new_prog)) { 46024adb7a4aSAndrii Nakryiko ret = PTR_ERR(new_prog); 46034adb7a4aSAndrii Nakryiko goto out_put_link; 46044adb7a4aSAndrii Nakryiko } 46050c991ebcSAndrii Nakryiko 46060c991ebcSAndrii Nakryiko if (flags & BPF_F_REPLACE) { 46070c991ebcSAndrii Nakryiko old_prog = bpf_prog_get(attr->link_update.old_prog_fd); 46080c991ebcSAndrii Nakryiko if (IS_ERR(old_prog)) { 46090c991ebcSAndrii Nakryiko ret = PTR_ERR(old_prog); 46100c991ebcSAndrii Nakryiko old_prog = NULL; 46110c991ebcSAndrii Nakryiko goto out_put_progs; 46120c991ebcSAndrii Nakryiko } 46134adb7a4aSAndrii Nakryiko } else if (attr->link_update.old_prog_fd) { 46144adb7a4aSAndrii Nakryiko ret = -EINVAL; 46154adb7a4aSAndrii Nakryiko goto out_put_progs; 46160c991ebcSAndrii Nakryiko } 46170c991ebcSAndrii Nakryiko 4618f9d04127SAndrii Nakryiko if (link->ops->update_prog) 4619f9d04127SAndrii Nakryiko ret = link->ops->update_prog(link, new_prog, old_prog); 4620f9d04127SAndrii Nakryiko else 46210c991ebcSAndrii Nakryiko ret = -EINVAL; 46220c991ebcSAndrii Nakryiko 46230c991ebcSAndrii Nakryiko out_put_progs: 46240c991ebcSAndrii Nakryiko if (old_prog) 46250c991ebcSAndrii Nakryiko bpf_prog_put(old_prog); 46260c991ebcSAndrii Nakryiko if (ret) 46270c991ebcSAndrii Nakryiko bpf_prog_put(new_prog); 46284adb7a4aSAndrii Nakryiko out_put_link: 46294adb7a4aSAndrii Nakryiko bpf_link_put(link); 46300c991ebcSAndrii Nakryiko return ret; 46310c991ebcSAndrii Nakryiko } 46320c991ebcSAndrii Nakryiko 463373b11c2aSAndrii Nakryiko #define BPF_LINK_DETACH_LAST_FIELD link_detach.link_fd 463473b11c2aSAndrii Nakryiko 463573b11c2aSAndrii Nakryiko static int link_detach(union bpf_attr *attr) 463673b11c2aSAndrii Nakryiko { 463773b11c2aSAndrii Nakryiko struct bpf_link *link; 463873b11c2aSAndrii Nakryiko int ret; 463973b11c2aSAndrii Nakryiko 464073b11c2aSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_DETACH)) 464173b11c2aSAndrii Nakryiko return -EINVAL; 464273b11c2aSAndrii Nakryiko 464373b11c2aSAndrii Nakryiko link = bpf_link_get_from_fd(attr->link_detach.link_fd); 464473b11c2aSAndrii Nakryiko if (IS_ERR(link)) 464573b11c2aSAndrii Nakryiko return PTR_ERR(link); 464673b11c2aSAndrii Nakryiko 464773b11c2aSAndrii Nakryiko if (link->ops->detach) 464873b11c2aSAndrii Nakryiko ret = link->ops->detach(link); 464973b11c2aSAndrii Nakryiko else 465073b11c2aSAndrii Nakryiko ret = -EOPNOTSUPP; 465173b11c2aSAndrii Nakryiko 465273b11c2aSAndrii Nakryiko bpf_link_put(link); 465373b11c2aSAndrii Nakryiko return ret; 465473b11c2aSAndrii Nakryiko } 465573b11c2aSAndrii Nakryiko 4656005142b8SAlexei Starovoitov static struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link) 46572d602c8cSAndrii Nakryiko { 4658005142b8SAlexei Starovoitov return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT); 4659005142b8SAlexei Starovoitov } 4660005142b8SAlexei Starovoitov 4661005142b8SAlexei Starovoitov struct bpf_link *bpf_link_by_id(u32 id) 4662005142b8SAlexei Starovoitov { 4663005142b8SAlexei Starovoitov struct bpf_link *link; 4664005142b8SAlexei Starovoitov 4665005142b8SAlexei Starovoitov if (!id) 4666005142b8SAlexei Starovoitov return ERR_PTR(-ENOENT); 4667005142b8SAlexei Starovoitov 4668005142b8SAlexei Starovoitov spin_lock_bh(&link_idr_lock); 4669005142b8SAlexei Starovoitov /* before link is "settled", ID is 0, pretend it doesn't exist yet */ 4670005142b8SAlexei Starovoitov link = idr_find(&link_idr, id); 4671005142b8SAlexei Starovoitov if (link) { 4672005142b8SAlexei Starovoitov if (link->id) 4673005142b8SAlexei Starovoitov link = bpf_link_inc_not_zero(link); 4674005142b8SAlexei Starovoitov else 4675005142b8SAlexei Starovoitov link = ERR_PTR(-EAGAIN); 4676005142b8SAlexei Starovoitov } else { 4677005142b8SAlexei Starovoitov link = ERR_PTR(-ENOENT); 4678005142b8SAlexei Starovoitov } 4679005142b8SAlexei Starovoitov spin_unlock_bh(&link_idr_lock); 4680005142b8SAlexei Starovoitov return link; 46812d602c8cSAndrii Nakryiko } 46822d602c8cSAndrii Nakryiko 4683*9f883612SDmitrii Dolgov struct bpf_link *bpf_link_get_curr_or_next(u32 *id) 4684*9f883612SDmitrii Dolgov { 4685*9f883612SDmitrii Dolgov struct bpf_link *link; 4686*9f883612SDmitrii Dolgov 4687*9f883612SDmitrii Dolgov spin_lock_bh(&link_idr_lock); 4688*9f883612SDmitrii Dolgov again: 4689*9f883612SDmitrii Dolgov link = idr_get_next(&link_idr, id); 4690*9f883612SDmitrii Dolgov if (link) { 4691*9f883612SDmitrii Dolgov link = bpf_link_inc_not_zero(link); 4692*9f883612SDmitrii Dolgov if (IS_ERR(link)) { 4693*9f883612SDmitrii Dolgov (*id)++; 4694*9f883612SDmitrii Dolgov goto again; 4695*9f883612SDmitrii Dolgov } 4696*9f883612SDmitrii Dolgov } 4697*9f883612SDmitrii Dolgov spin_unlock_bh(&link_idr_lock); 4698*9f883612SDmitrii Dolgov 4699*9f883612SDmitrii Dolgov return link; 4700*9f883612SDmitrii Dolgov } 4701*9f883612SDmitrii Dolgov 47022d602c8cSAndrii Nakryiko #define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id 47032d602c8cSAndrii Nakryiko 47042d602c8cSAndrii Nakryiko static int bpf_link_get_fd_by_id(const union bpf_attr *attr) 47052d602c8cSAndrii Nakryiko { 47062d602c8cSAndrii Nakryiko struct bpf_link *link; 47072d602c8cSAndrii Nakryiko u32 id = attr->link_id; 4708005142b8SAlexei Starovoitov int fd; 47092d602c8cSAndrii Nakryiko 47102d602c8cSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID)) 47112d602c8cSAndrii Nakryiko return -EINVAL; 47122d602c8cSAndrii Nakryiko 47132d602c8cSAndrii Nakryiko if (!capable(CAP_SYS_ADMIN)) 47142d602c8cSAndrii Nakryiko return -EPERM; 47152d602c8cSAndrii Nakryiko 4716005142b8SAlexei Starovoitov link = bpf_link_by_id(id); 4717005142b8SAlexei Starovoitov if (IS_ERR(link)) 4718005142b8SAlexei Starovoitov return PTR_ERR(link); 47192d602c8cSAndrii Nakryiko 47202d602c8cSAndrii Nakryiko fd = bpf_link_new_fd(link); 47212d602c8cSAndrii Nakryiko if (fd < 0) 47222d602c8cSAndrii Nakryiko bpf_link_put(link); 47232d602c8cSAndrii Nakryiko 47242d602c8cSAndrii Nakryiko return fd; 47252d602c8cSAndrii Nakryiko } 47262d602c8cSAndrii Nakryiko 4727d46edd67SSong Liu DEFINE_MUTEX(bpf_stats_enabled_mutex); 4728d46edd67SSong Liu 4729d46edd67SSong Liu static int bpf_stats_release(struct inode *inode, struct file *file) 4730d46edd67SSong Liu { 4731d46edd67SSong Liu mutex_lock(&bpf_stats_enabled_mutex); 4732d46edd67SSong Liu static_key_slow_dec(&bpf_stats_enabled_key.key); 4733d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 4734d46edd67SSong Liu return 0; 4735d46edd67SSong Liu } 4736d46edd67SSong Liu 4737d46edd67SSong Liu static const struct file_operations bpf_stats_fops = { 4738d46edd67SSong Liu .release = bpf_stats_release, 4739d46edd67SSong Liu }; 4740d46edd67SSong Liu 4741d46edd67SSong Liu static int bpf_enable_runtime_stats(void) 4742d46edd67SSong Liu { 4743d46edd67SSong Liu int fd; 4744d46edd67SSong Liu 4745d46edd67SSong Liu mutex_lock(&bpf_stats_enabled_mutex); 4746d46edd67SSong Liu 4747d46edd67SSong Liu /* Set a very high limit to avoid overflow */ 4748d46edd67SSong Liu if (static_key_count(&bpf_stats_enabled_key.key) > INT_MAX / 2) { 4749d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 4750d46edd67SSong Liu return -EBUSY; 4751d46edd67SSong Liu } 4752d46edd67SSong Liu 4753d46edd67SSong Liu fd = anon_inode_getfd("bpf-stats", &bpf_stats_fops, NULL, O_CLOEXEC); 4754d46edd67SSong Liu if (fd >= 0) 4755d46edd67SSong Liu static_key_slow_inc(&bpf_stats_enabled_key.key); 4756d46edd67SSong Liu 4757d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 4758d46edd67SSong Liu return fd; 4759d46edd67SSong Liu } 4760d46edd67SSong Liu 4761d46edd67SSong Liu #define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type 4762d46edd67SSong Liu 4763d46edd67SSong Liu static int bpf_enable_stats(union bpf_attr *attr) 4764d46edd67SSong Liu { 4765d46edd67SSong Liu 4766d46edd67SSong Liu if (CHECK_ATTR(BPF_ENABLE_STATS)) 4767d46edd67SSong Liu return -EINVAL; 4768d46edd67SSong Liu 4769d46edd67SSong Liu if (!capable(CAP_SYS_ADMIN)) 4770d46edd67SSong Liu return -EPERM; 4771d46edd67SSong Liu 4772d46edd67SSong Liu switch (attr->enable_stats.type) { 4773d46edd67SSong Liu case BPF_STATS_RUN_TIME: 4774d46edd67SSong Liu return bpf_enable_runtime_stats(); 4775d46edd67SSong Liu default: 4776d46edd67SSong Liu break; 4777d46edd67SSong Liu } 4778d46edd67SSong Liu return -EINVAL; 4779d46edd67SSong Liu } 4780d46edd67SSong Liu 4781ac51d99bSYonghong Song #define BPF_ITER_CREATE_LAST_FIELD iter_create.flags 4782ac51d99bSYonghong Song 4783ac51d99bSYonghong Song static int bpf_iter_create(union bpf_attr *attr) 4784ac51d99bSYonghong Song { 4785ac51d99bSYonghong Song struct bpf_link *link; 4786ac51d99bSYonghong Song int err; 4787ac51d99bSYonghong Song 4788ac51d99bSYonghong Song if (CHECK_ATTR(BPF_ITER_CREATE)) 4789ac51d99bSYonghong Song return -EINVAL; 4790ac51d99bSYonghong Song 4791ac51d99bSYonghong Song if (attr->iter_create.flags) 4792ac51d99bSYonghong Song return -EINVAL; 4793ac51d99bSYonghong Song 4794ac51d99bSYonghong Song link = bpf_link_get_from_fd(attr->iter_create.link_fd); 4795ac51d99bSYonghong Song if (IS_ERR(link)) 4796ac51d99bSYonghong Song return PTR_ERR(link); 4797ac51d99bSYonghong Song 4798ac51d99bSYonghong Song err = bpf_iter_new_fd(link); 4799ac51d99bSYonghong Song bpf_link_put(link); 4800ac51d99bSYonghong Song 4801ac51d99bSYonghong Song return err; 4802ac51d99bSYonghong Song } 4803ac51d99bSYonghong Song 4804ef15314aSYiFei Zhu #define BPF_PROG_BIND_MAP_LAST_FIELD prog_bind_map.flags 4805ef15314aSYiFei Zhu 4806ef15314aSYiFei Zhu static int bpf_prog_bind_map(union bpf_attr *attr) 4807ef15314aSYiFei Zhu { 4808ef15314aSYiFei Zhu struct bpf_prog *prog; 4809ef15314aSYiFei Zhu struct bpf_map *map; 4810ef15314aSYiFei Zhu struct bpf_map **used_maps_old, **used_maps_new; 4811ef15314aSYiFei Zhu int i, ret = 0; 4812ef15314aSYiFei Zhu 4813ef15314aSYiFei Zhu if (CHECK_ATTR(BPF_PROG_BIND_MAP)) 4814ef15314aSYiFei Zhu return -EINVAL; 4815ef15314aSYiFei Zhu 4816ef15314aSYiFei Zhu if (attr->prog_bind_map.flags) 4817ef15314aSYiFei Zhu return -EINVAL; 4818ef15314aSYiFei Zhu 4819ef15314aSYiFei Zhu prog = bpf_prog_get(attr->prog_bind_map.prog_fd); 4820ef15314aSYiFei Zhu if (IS_ERR(prog)) 4821ef15314aSYiFei Zhu return PTR_ERR(prog); 4822ef15314aSYiFei Zhu 4823ef15314aSYiFei Zhu map = bpf_map_get(attr->prog_bind_map.map_fd); 4824ef15314aSYiFei Zhu if (IS_ERR(map)) { 4825ef15314aSYiFei Zhu ret = PTR_ERR(map); 4826ef15314aSYiFei Zhu goto out_prog_put; 4827ef15314aSYiFei Zhu } 4828ef15314aSYiFei Zhu 4829ef15314aSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 4830ef15314aSYiFei Zhu 4831ef15314aSYiFei Zhu used_maps_old = prog->aux->used_maps; 4832ef15314aSYiFei Zhu 4833ef15314aSYiFei Zhu for (i = 0; i < prog->aux->used_map_cnt; i++) 48341028ae40SStanislav Fomichev if (used_maps_old[i] == map) { 48351028ae40SStanislav Fomichev bpf_map_put(map); 4836ef15314aSYiFei Zhu goto out_unlock; 48371028ae40SStanislav Fomichev } 4838ef15314aSYiFei Zhu 4839ef15314aSYiFei Zhu used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1, 4840ef15314aSYiFei Zhu sizeof(used_maps_new[0]), 4841ef15314aSYiFei Zhu GFP_KERNEL); 4842ef15314aSYiFei Zhu if (!used_maps_new) { 4843ef15314aSYiFei Zhu ret = -ENOMEM; 4844ef15314aSYiFei Zhu goto out_unlock; 4845ef15314aSYiFei Zhu } 4846ef15314aSYiFei Zhu 4847ef15314aSYiFei Zhu memcpy(used_maps_new, used_maps_old, 4848ef15314aSYiFei Zhu sizeof(used_maps_old[0]) * prog->aux->used_map_cnt); 4849ef15314aSYiFei Zhu used_maps_new[prog->aux->used_map_cnt] = map; 4850ef15314aSYiFei Zhu 4851ef15314aSYiFei Zhu prog->aux->used_map_cnt++; 4852ef15314aSYiFei Zhu prog->aux->used_maps = used_maps_new; 4853ef15314aSYiFei Zhu 4854ef15314aSYiFei Zhu kfree(used_maps_old); 4855ef15314aSYiFei Zhu 4856ef15314aSYiFei Zhu out_unlock: 4857ef15314aSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 4858ef15314aSYiFei Zhu 4859ef15314aSYiFei Zhu if (ret) 4860ef15314aSYiFei Zhu bpf_map_put(map); 4861ef15314aSYiFei Zhu out_prog_put: 4862ef15314aSYiFei Zhu bpf_prog_put(prog); 4863ef15314aSYiFei Zhu return ret; 4864ef15314aSYiFei Zhu } 4865ef15314aSYiFei Zhu 4866af2ac3e1SAlexei Starovoitov static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) 486799c55f7dSAlexei Starovoitov { 48688096f229SGreg Kroah-Hartman union bpf_attr attr; 486999c55f7dSAlexei Starovoitov int err; 487099c55f7dSAlexei Starovoitov 48712c78ee89SAlexei Starovoitov if (sysctl_unprivileged_bpf_disabled && !bpf_capable()) 487299c55f7dSAlexei Starovoitov return -EPERM; 487399c55f7dSAlexei Starovoitov 4874dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size); 487599c55f7dSAlexei Starovoitov if (err) 487699c55f7dSAlexei Starovoitov return err; 48771e270976SMartin KaFai Lau size = min_t(u32, size, sizeof(attr)); 487899c55f7dSAlexei Starovoitov 487999c55f7dSAlexei Starovoitov /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 48808096f229SGreg Kroah-Hartman memset(&attr, 0, sizeof(attr)); 4881af2ac3e1SAlexei Starovoitov if (copy_from_bpfptr(&attr, uattr, size) != 0) 488299c55f7dSAlexei Starovoitov return -EFAULT; 488399c55f7dSAlexei Starovoitov 4884afdb09c7SChenbo Feng err = security_bpf(cmd, &attr, size); 4885afdb09c7SChenbo Feng if (err < 0) 4886afdb09c7SChenbo Feng return err; 4887afdb09c7SChenbo Feng 488899c55f7dSAlexei Starovoitov switch (cmd) { 488999c55f7dSAlexei Starovoitov case BPF_MAP_CREATE: 489099c55f7dSAlexei Starovoitov err = map_create(&attr); 489199c55f7dSAlexei Starovoitov break; 4892db20fd2bSAlexei Starovoitov case BPF_MAP_LOOKUP_ELEM: 4893db20fd2bSAlexei Starovoitov err = map_lookup_elem(&attr); 4894db20fd2bSAlexei Starovoitov break; 4895db20fd2bSAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 4896af2ac3e1SAlexei Starovoitov err = map_update_elem(&attr, uattr); 4897db20fd2bSAlexei Starovoitov break; 4898db20fd2bSAlexei Starovoitov case BPF_MAP_DELETE_ELEM: 4899db20fd2bSAlexei Starovoitov err = map_delete_elem(&attr); 4900db20fd2bSAlexei Starovoitov break; 4901db20fd2bSAlexei Starovoitov case BPF_MAP_GET_NEXT_KEY: 4902db20fd2bSAlexei Starovoitov err = map_get_next_key(&attr); 4903db20fd2bSAlexei Starovoitov break; 490487df15deSDaniel Borkmann case BPF_MAP_FREEZE: 490587df15deSDaniel Borkmann err = map_freeze(&attr); 490687df15deSDaniel Borkmann break; 490709756af4SAlexei Starovoitov case BPF_PROG_LOAD: 4908838e9690SYonghong Song err = bpf_prog_load(&attr, uattr); 490909756af4SAlexei Starovoitov break; 4910b2197755SDaniel Borkmann case BPF_OBJ_PIN: 4911b2197755SDaniel Borkmann err = bpf_obj_pin(&attr); 4912b2197755SDaniel Borkmann break; 4913b2197755SDaniel Borkmann case BPF_OBJ_GET: 4914b2197755SDaniel Borkmann err = bpf_obj_get(&attr); 4915b2197755SDaniel Borkmann break; 4916f4324551SDaniel Mack case BPF_PROG_ATTACH: 4917f4324551SDaniel Mack err = bpf_prog_attach(&attr); 4918f4324551SDaniel Mack break; 4919f4324551SDaniel Mack case BPF_PROG_DETACH: 4920f4324551SDaniel Mack err = bpf_prog_detach(&attr); 4921f4324551SDaniel Mack break; 4922468e2f64SAlexei Starovoitov case BPF_PROG_QUERY: 4923af2ac3e1SAlexei Starovoitov err = bpf_prog_query(&attr, uattr.user); 4924468e2f64SAlexei Starovoitov break; 49251cf1cae9SAlexei Starovoitov case BPF_PROG_TEST_RUN: 4926af2ac3e1SAlexei Starovoitov err = bpf_prog_test_run(&attr, uattr.user); 49271cf1cae9SAlexei Starovoitov break; 492834ad5580SMartin KaFai Lau case BPF_PROG_GET_NEXT_ID: 4929af2ac3e1SAlexei Starovoitov err = bpf_obj_get_next_id(&attr, uattr.user, 493034ad5580SMartin KaFai Lau &prog_idr, &prog_idr_lock); 493134ad5580SMartin KaFai Lau break; 493234ad5580SMartin KaFai Lau case BPF_MAP_GET_NEXT_ID: 4933af2ac3e1SAlexei Starovoitov err = bpf_obj_get_next_id(&attr, uattr.user, 493434ad5580SMartin KaFai Lau &map_idr, &map_idr_lock); 493534ad5580SMartin KaFai Lau break; 49361b9ed84eSQuentin Monnet case BPF_BTF_GET_NEXT_ID: 4937af2ac3e1SAlexei Starovoitov err = bpf_obj_get_next_id(&attr, uattr.user, 49381b9ed84eSQuentin Monnet &btf_idr, &btf_idr_lock); 49391b9ed84eSQuentin Monnet break; 4940b16d9aa4SMartin KaFai Lau case BPF_PROG_GET_FD_BY_ID: 4941b16d9aa4SMartin KaFai Lau err = bpf_prog_get_fd_by_id(&attr); 4942b16d9aa4SMartin KaFai Lau break; 4943bd5f5f4eSMartin KaFai Lau case BPF_MAP_GET_FD_BY_ID: 4944bd5f5f4eSMartin KaFai Lau err = bpf_map_get_fd_by_id(&attr); 4945bd5f5f4eSMartin KaFai Lau break; 49461e270976SMartin KaFai Lau case BPF_OBJ_GET_INFO_BY_FD: 4947af2ac3e1SAlexei Starovoitov err = bpf_obj_get_info_by_fd(&attr, uattr.user); 49481e270976SMartin KaFai Lau break; 4949c4f6699dSAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN: 4950c4f6699dSAlexei Starovoitov err = bpf_raw_tracepoint_open(&attr); 4951c4f6699dSAlexei Starovoitov break; 4952f56a653cSMartin KaFai Lau case BPF_BTF_LOAD: 4953c571bd75SAlexei Starovoitov err = bpf_btf_load(&attr, uattr); 4954f56a653cSMartin KaFai Lau break; 495578958fcaSMartin KaFai Lau case BPF_BTF_GET_FD_BY_ID: 495678958fcaSMartin KaFai Lau err = bpf_btf_get_fd_by_id(&attr); 495778958fcaSMartin KaFai Lau break; 495841bdc4b4SYonghong Song case BPF_TASK_FD_QUERY: 4959af2ac3e1SAlexei Starovoitov err = bpf_task_fd_query(&attr, uattr.user); 496041bdc4b4SYonghong Song break; 4961bd513cd0SMauricio Vasquez B case BPF_MAP_LOOKUP_AND_DELETE_ELEM: 4962bd513cd0SMauricio Vasquez B err = map_lookup_and_delete_elem(&attr); 4963bd513cd0SMauricio Vasquez B break; 4964cb4d03abSBrian Vazquez case BPF_MAP_LOOKUP_BATCH: 4965af2ac3e1SAlexei Starovoitov err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_LOOKUP_BATCH); 4966cb4d03abSBrian Vazquez break; 496705799638SYonghong Song case BPF_MAP_LOOKUP_AND_DELETE_BATCH: 4968af2ac3e1SAlexei Starovoitov err = bpf_map_do_batch(&attr, uattr.user, 496905799638SYonghong Song BPF_MAP_LOOKUP_AND_DELETE_BATCH); 497005799638SYonghong Song break; 4971aa2e93b8SBrian Vazquez case BPF_MAP_UPDATE_BATCH: 4972af2ac3e1SAlexei Starovoitov err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_UPDATE_BATCH); 4973aa2e93b8SBrian Vazquez break; 4974aa2e93b8SBrian Vazquez case BPF_MAP_DELETE_BATCH: 4975af2ac3e1SAlexei Starovoitov err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_DELETE_BATCH); 4976aa2e93b8SBrian Vazquez break; 4977af6eea57SAndrii Nakryiko case BPF_LINK_CREATE: 4978af2ac3e1SAlexei Starovoitov err = link_create(&attr, uattr); 4979af6eea57SAndrii Nakryiko break; 49800c991ebcSAndrii Nakryiko case BPF_LINK_UPDATE: 49810c991ebcSAndrii Nakryiko err = link_update(&attr); 49820c991ebcSAndrii Nakryiko break; 49832d602c8cSAndrii Nakryiko case BPF_LINK_GET_FD_BY_ID: 49842d602c8cSAndrii Nakryiko err = bpf_link_get_fd_by_id(&attr); 49852d602c8cSAndrii Nakryiko break; 49862d602c8cSAndrii Nakryiko case BPF_LINK_GET_NEXT_ID: 4987af2ac3e1SAlexei Starovoitov err = bpf_obj_get_next_id(&attr, uattr.user, 49882d602c8cSAndrii Nakryiko &link_idr, &link_idr_lock); 49892d602c8cSAndrii Nakryiko break; 4990d46edd67SSong Liu case BPF_ENABLE_STATS: 4991d46edd67SSong Liu err = bpf_enable_stats(&attr); 4992d46edd67SSong Liu break; 4993ac51d99bSYonghong Song case BPF_ITER_CREATE: 4994ac51d99bSYonghong Song err = bpf_iter_create(&attr); 4995ac51d99bSYonghong Song break; 499673b11c2aSAndrii Nakryiko case BPF_LINK_DETACH: 499773b11c2aSAndrii Nakryiko err = link_detach(&attr); 499873b11c2aSAndrii Nakryiko break; 4999ef15314aSYiFei Zhu case BPF_PROG_BIND_MAP: 5000ef15314aSYiFei Zhu err = bpf_prog_bind_map(&attr); 5001ef15314aSYiFei Zhu break; 500299c55f7dSAlexei Starovoitov default: 500399c55f7dSAlexei Starovoitov err = -EINVAL; 500499c55f7dSAlexei Starovoitov break; 500599c55f7dSAlexei Starovoitov } 500699c55f7dSAlexei Starovoitov 500799c55f7dSAlexei Starovoitov return err; 500899c55f7dSAlexei Starovoitov } 500979a7f8bdSAlexei Starovoitov 5010af2ac3e1SAlexei Starovoitov SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 5011af2ac3e1SAlexei Starovoitov { 5012af2ac3e1SAlexei Starovoitov return __sys_bpf(cmd, USER_BPFPTR(uattr), size); 5013af2ac3e1SAlexei Starovoitov } 5014af2ac3e1SAlexei Starovoitov 501579a7f8bdSAlexei Starovoitov static bool syscall_prog_is_valid_access(int off, int size, 501679a7f8bdSAlexei Starovoitov enum bpf_access_type type, 501779a7f8bdSAlexei Starovoitov const struct bpf_prog *prog, 501879a7f8bdSAlexei Starovoitov struct bpf_insn_access_aux *info) 501979a7f8bdSAlexei Starovoitov { 502079a7f8bdSAlexei Starovoitov if (off < 0 || off >= U16_MAX) 502179a7f8bdSAlexei Starovoitov return false; 502279a7f8bdSAlexei Starovoitov if (off % size != 0) 502379a7f8bdSAlexei Starovoitov return false; 502479a7f8bdSAlexei Starovoitov return true; 502579a7f8bdSAlexei Starovoitov } 502679a7f8bdSAlexei Starovoitov 5027b1d18a75SAlexei Starovoitov BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size) 502879a7f8bdSAlexei Starovoitov { 5029b1d18a75SAlexei Starovoitov struct bpf_prog * __maybe_unused prog; 5030b1d18a75SAlexei Starovoitov 5031af2ac3e1SAlexei Starovoitov switch (cmd) { 5032af2ac3e1SAlexei Starovoitov case BPF_MAP_CREATE: 5033af2ac3e1SAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 5034af2ac3e1SAlexei Starovoitov case BPF_MAP_FREEZE: 5035af2ac3e1SAlexei Starovoitov case BPF_PROG_LOAD: 5036c571bd75SAlexei Starovoitov case BPF_BTF_LOAD: 5037b1d18a75SAlexei Starovoitov case BPF_LINK_CREATE: 5038b1d18a75SAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN: 5039af2ac3e1SAlexei Starovoitov break; 5040b1d18a75SAlexei Starovoitov #ifdef CONFIG_BPF_JIT /* __bpf_prog_enter_sleepable used by trampoline and JIT */ 5041b1d18a75SAlexei Starovoitov case BPF_PROG_TEST_RUN: 5042b1d18a75SAlexei Starovoitov if (attr->test.data_in || attr->test.data_out || 5043b1d18a75SAlexei Starovoitov attr->test.ctx_out || attr->test.duration || 5044b1d18a75SAlexei Starovoitov attr->test.repeat || attr->test.flags) 5045b1d18a75SAlexei Starovoitov return -EINVAL; 5046b1d18a75SAlexei Starovoitov 5047b1d18a75SAlexei Starovoitov prog = bpf_prog_get_type(attr->test.prog_fd, BPF_PROG_TYPE_SYSCALL); 5048b1d18a75SAlexei Starovoitov if (IS_ERR(prog)) 5049b1d18a75SAlexei Starovoitov return PTR_ERR(prog); 5050b1d18a75SAlexei Starovoitov 5051b1d18a75SAlexei Starovoitov if (attr->test.ctx_size_in < prog->aux->max_ctx_offset || 5052b1d18a75SAlexei Starovoitov attr->test.ctx_size_in > U16_MAX) { 5053b1d18a75SAlexei Starovoitov bpf_prog_put(prog); 5054b1d18a75SAlexei Starovoitov return -EINVAL; 5055b1d18a75SAlexei Starovoitov } 5056b1d18a75SAlexei Starovoitov 5057b1d18a75SAlexei Starovoitov if (!__bpf_prog_enter_sleepable(prog)) { 5058b1d18a75SAlexei Starovoitov /* recursion detected */ 5059b1d18a75SAlexei Starovoitov bpf_prog_put(prog); 5060b1d18a75SAlexei Starovoitov return -EBUSY; 5061b1d18a75SAlexei Starovoitov } 5062b1d18a75SAlexei Starovoitov attr->test.retval = bpf_prog_run(prog, (void *) (long) attr->test.ctx_in); 5063b1d18a75SAlexei Starovoitov __bpf_prog_exit_sleepable(prog, 0 /* bpf_prog_run does runtime stats */); 5064b1d18a75SAlexei Starovoitov bpf_prog_put(prog); 5065b1d18a75SAlexei Starovoitov return 0; 5066b1d18a75SAlexei Starovoitov #endif 5067af2ac3e1SAlexei Starovoitov default: 506879a7f8bdSAlexei Starovoitov return -EINVAL; 506979a7f8bdSAlexei Starovoitov } 5070af2ac3e1SAlexei Starovoitov return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size); 5071af2ac3e1SAlexei Starovoitov } 5072b1d18a75SAlexei Starovoitov EXPORT_SYMBOL(bpf_sys_bpf); 507379a7f8bdSAlexei Starovoitov 50743a2daa72SPu Lehui static const struct bpf_func_proto bpf_sys_bpf_proto = { 507579a7f8bdSAlexei Starovoitov .func = bpf_sys_bpf, 507679a7f8bdSAlexei Starovoitov .gpl_only = false, 507779a7f8bdSAlexei Starovoitov .ret_type = RET_INTEGER, 507879a7f8bdSAlexei Starovoitov .arg1_type = ARG_ANYTHING, 5079216e3cd2SHao Luo .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, 508079a7f8bdSAlexei Starovoitov .arg3_type = ARG_CONST_SIZE, 508179a7f8bdSAlexei Starovoitov }; 508279a7f8bdSAlexei Starovoitov 508379a7f8bdSAlexei Starovoitov const struct bpf_func_proto * __weak 508479a7f8bdSAlexei Starovoitov tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 508579a7f8bdSAlexei Starovoitov { 508679a7f8bdSAlexei Starovoitov return bpf_base_func_proto(func_id); 508779a7f8bdSAlexei Starovoitov } 508879a7f8bdSAlexei Starovoitov 50893abea089SAlexei Starovoitov BPF_CALL_1(bpf_sys_close, u32, fd) 50903abea089SAlexei Starovoitov { 50913abea089SAlexei Starovoitov /* When bpf program calls this helper there should not be 50923abea089SAlexei Starovoitov * an fdget() without matching completed fdput(). 50933abea089SAlexei Starovoitov * This helper is allowed in the following callchain only: 50943abea089SAlexei Starovoitov * sys_bpf->prog_test_run->bpf_prog->bpf_sys_close 50953abea089SAlexei Starovoitov */ 50963abea089SAlexei Starovoitov return close_fd(fd); 50973abea089SAlexei Starovoitov } 50983abea089SAlexei Starovoitov 50993a2daa72SPu Lehui static const struct bpf_func_proto bpf_sys_close_proto = { 51003abea089SAlexei Starovoitov .func = bpf_sys_close, 51013abea089SAlexei Starovoitov .gpl_only = false, 51023abea089SAlexei Starovoitov .ret_type = RET_INTEGER, 51033abea089SAlexei Starovoitov .arg1_type = ARG_ANYTHING, 51043abea089SAlexei Starovoitov }; 51053abea089SAlexei Starovoitov 5106d6aef08aSKumar Kartikeya Dwivedi BPF_CALL_4(bpf_kallsyms_lookup_name, const char *, name, int, name_sz, int, flags, u64 *, res) 5107d6aef08aSKumar Kartikeya Dwivedi { 5108d6aef08aSKumar Kartikeya Dwivedi if (flags) 5109d6aef08aSKumar Kartikeya Dwivedi return -EINVAL; 5110d6aef08aSKumar Kartikeya Dwivedi 5111d6aef08aSKumar Kartikeya Dwivedi if (name_sz <= 1 || name[name_sz - 1]) 5112d6aef08aSKumar Kartikeya Dwivedi return -EINVAL; 5113d6aef08aSKumar Kartikeya Dwivedi 5114d6aef08aSKumar Kartikeya Dwivedi if (!bpf_dump_raw_ok(current_cred())) 5115d6aef08aSKumar Kartikeya Dwivedi return -EPERM; 5116d6aef08aSKumar Kartikeya Dwivedi 5117d6aef08aSKumar Kartikeya Dwivedi *res = kallsyms_lookup_name(name); 5118d6aef08aSKumar Kartikeya Dwivedi return *res ? 0 : -ENOENT; 5119d6aef08aSKumar Kartikeya Dwivedi } 5120d6aef08aSKumar Kartikeya Dwivedi 5121d6aef08aSKumar Kartikeya Dwivedi const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = { 5122d6aef08aSKumar Kartikeya Dwivedi .func = bpf_kallsyms_lookup_name, 5123d6aef08aSKumar Kartikeya Dwivedi .gpl_only = false, 5124d6aef08aSKumar Kartikeya Dwivedi .ret_type = RET_INTEGER, 5125d6aef08aSKumar Kartikeya Dwivedi .arg1_type = ARG_PTR_TO_MEM, 5126d4efb170SKumar Kartikeya Dwivedi .arg2_type = ARG_CONST_SIZE_OR_ZERO, 5127d6aef08aSKumar Kartikeya Dwivedi .arg3_type = ARG_ANYTHING, 5128d6aef08aSKumar Kartikeya Dwivedi .arg4_type = ARG_PTR_TO_LONG, 5129d6aef08aSKumar Kartikeya Dwivedi }; 5130d6aef08aSKumar Kartikeya Dwivedi 513179a7f8bdSAlexei Starovoitov static const struct bpf_func_proto * 513279a7f8bdSAlexei Starovoitov syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 513379a7f8bdSAlexei Starovoitov { 513479a7f8bdSAlexei Starovoitov switch (func_id) { 513579a7f8bdSAlexei Starovoitov case BPF_FUNC_sys_bpf: 513679a7f8bdSAlexei Starovoitov return &bpf_sys_bpf_proto; 51373d78417bSAlexei Starovoitov case BPF_FUNC_btf_find_by_name_kind: 51383d78417bSAlexei Starovoitov return &bpf_btf_find_by_name_kind_proto; 51393abea089SAlexei Starovoitov case BPF_FUNC_sys_close: 51403abea089SAlexei Starovoitov return &bpf_sys_close_proto; 5141d6aef08aSKumar Kartikeya Dwivedi case BPF_FUNC_kallsyms_lookup_name: 5142d6aef08aSKumar Kartikeya Dwivedi return &bpf_kallsyms_lookup_name_proto; 514379a7f8bdSAlexei Starovoitov default: 514479a7f8bdSAlexei Starovoitov return tracing_prog_func_proto(func_id, prog); 514579a7f8bdSAlexei Starovoitov } 514679a7f8bdSAlexei Starovoitov } 514779a7f8bdSAlexei Starovoitov 514879a7f8bdSAlexei Starovoitov const struct bpf_verifier_ops bpf_syscall_verifier_ops = { 514979a7f8bdSAlexei Starovoitov .get_func_proto = syscall_prog_func_proto, 515079a7f8bdSAlexei Starovoitov .is_valid_access = syscall_prog_is_valid_access, 515179a7f8bdSAlexei Starovoitov }; 515279a7f8bdSAlexei Starovoitov 515379a7f8bdSAlexei Starovoitov const struct bpf_prog_ops bpf_syscall_prog_ops = { 515479a7f8bdSAlexei Starovoitov .test_run = bpf_prog_test_run_syscall, 515579a7f8bdSAlexei Starovoitov }; 51562900005eSYan Zhu 51572900005eSYan Zhu #ifdef CONFIG_SYSCTL 51582900005eSYan Zhu static int bpf_stats_handler(struct ctl_table *table, int write, 51592900005eSYan Zhu void *buffer, size_t *lenp, loff_t *ppos) 51602900005eSYan Zhu { 51612900005eSYan Zhu struct static_key *key = (struct static_key *)table->data; 51622900005eSYan Zhu static int saved_val; 51632900005eSYan Zhu int val, ret; 51642900005eSYan Zhu struct ctl_table tmp = { 51652900005eSYan Zhu .data = &val, 51662900005eSYan Zhu .maxlen = sizeof(val), 51672900005eSYan Zhu .mode = table->mode, 51682900005eSYan Zhu .extra1 = SYSCTL_ZERO, 51692900005eSYan Zhu .extra2 = SYSCTL_ONE, 51702900005eSYan Zhu }; 51712900005eSYan Zhu 51722900005eSYan Zhu if (write && !capable(CAP_SYS_ADMIN)) 51732900005eSYan Zhu return -EPERM; 51742900005eSYan Zhu 51752900005eSYan Zhu mutex_lock(&bpf_stats_enabled_mutex); 51762900005eSYan Zhu val = saved_val; 51772900005eSYan Zhu ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos); 51782900005eSYan Zhu if (write && !ret && val != saved_val) { 51792900005eSYan Zhu if (val) 51802900005eSYan Zhu static_key_slow_inc(key); 51812900005eSYan Zhu else 51822900005eSYan Zhu static_key_slow_dec(key); 51832900005eSYan Zhu saved_val = val; 51842900005eSYan Zhu } 51852900005eSYan Zhu mutex_unlock(&bpf_stats_enabled_mutex); 51862900005eSYan Zhu return ret; 51872900005eSYan Zhu } 51882900005eSYan Zhu 51892900005eSYan Zhu void __weak unpriv_ebpf_notify(int new_state) 51902900005eSYan Zhu { 51912900005eSYan Zhu } 51922900005eSYan Zhu 51932900005eSYan Zhu static int bpf_unpriv_handler(struct ctl_table *table, int write, 51942900005eSYan Zhu void *buffer, size_t *lenp, loff_t *ppos) 51952900005eSYan Zhu { 51962900005eSYan Zhu int ret, unpriv_enable = *(int *)table->data; 51972900005eSYan Zhu bool locked_state = unpriv_enable == 1; 51982900005eSYan Zhu struct ctl_table tmp = *table; 51992900005eSYan Zhu 52002900005eSYan Zhu if (write && !capable(CAP_SYS_ADMIN)) 52012900005eSYan Zhu return -EPERM; 52022900005eSYan Zhu 52032900005eSYan Zhu tmp.data = &unpriv_enable; 52042900005eSYan Zhu ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos); 52052900005eSYan Zhu if (write && !ret) { 52062900005eSYan Zhu if (locked_state && unpriv_enable != 1) 52072900005eSYan Zhu return -EPERM; 52082900005eSYan Zhu *(int *)table->data = unpriv_enable; 52092900005eSYan Zhu } 52102900005eSYan Zhu 52112900005eSYan Zhu unpriv_ebpf_notify(unpriv_enable); 52122900005eSYan Zhu 52132900005eSYan Zhu return ret; 52142900005eSYan Zhu } 52152900005eSYan Zhu 52162900005eSYan Zhu static struct ctl_table bpf_syscall_table[] = { 52172900005eSYan Zhu { 52182900005eSYan Zhu .procname = "unprivileged_bpf_disabled", 52192900005eSYan Zhu .data = &sysctl_unprivileged_bpf_disabled, 52202900005eSYan Zhu .maxlen = sizeof(sysctl_unprivileged_bpf_disabled), 52212900005eSYan Zhu .mode = 0644, 52222900005eSYan Zhu .proc_handler = bpf_unpriv_handler, 52232900005eSYan Zhu .extra1 = SYSCTL_ZERO, 52242900005eSYan Zhu .extra2 = SYSCTL_TWO, 52252900005eSYan Zhu }, 52262900005eSYan Zhu { 52272900005eSYan Zhu .procname = "bpf_stats_enabled", 52282900005eSYan Zhu .data = &bpf_stats_enabled_key.key, 52292900005eSYan Zhu .maxlen = sizeof(bpf_stats_enabled_key), 52302900005eSYan Zhu .mode = 0644, 52312900005eSYan Zhu .proc_handler = bpf_stats_handler, 52322900005eSYan Zhu }, 52332900005eSYan Zhu { } 52342900005eSYan Zhu }; 52352900005eSYan Zhu 52362900005eSYan Zhu static int __init bpf_syscall_sysctl_init(void) 52372900005eSYan Zhu { 52382900005eSYan Zhu register_sysctl_init("kernel", bpf_syscall_table); 52392900005eSYan Zhu return 0; 52402900005eSYan Zhu } 52412900005eSYan Zhu late_initcall(bpf_syscall_sysctl_init); 52422900005eSYan Zhu #endif /* CONFIG_SYSCTL */ 5243