15b497af4SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 299c55f7dSAlexei Starovoitov /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 399c55f7dSAlexei Starovoitov */ 499c55f7dSAlexei Starovoitov #include <linux/bpf.h> 5aef2fedaSJakub Kicinski #include <linux/bpf-cgroup.h> 6a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 7f4364dcfSSean Young #include <linux/bpf_lirc.h> 84a1e7c0cSToke Høiland-Jørgensen #include <linux/bpf_verifier.h> 961df10c7SKumar Kartikeya Dwivedi #include <linux/bsearch.h> 10f56a653cSMartin KaFai Lau #include <linux/btf.h> 1199c55f7dSAlexei Starovoitov #include <linux/syscalls.h> 1299c55f7dSAlexei Starovoitov #include <linux/slab.h> 133f07c014SIngo Molnar #include <linux/sched/signal.h> 14d407bd25SDaniel Borkmann #include <linux/vmalloc.h> 15d407bd25SDaniel Borkmann #include <linux/mmzone.h> 1699c55f7dSAlexei Starovoitov #include <linux/anon_inodes.h> 1741bdc4b4SYonghong Song #include <linux/fdtable.h> 18db20fd2bSAlexei Starovoitov #include <linux/file.h> 1941bdc4b4SYonghong Song #include <linux/fs.h> 2009756af4SAlexei Starovoitov #include <linux/license.h> 2109756af4SAlexei Starovoitov #include <linux/filter.h> 22535e7b4bSMickaël Salaün #include <linux/kernel.h> 23dc4bb0e2SMartin KaFai Lau #include <linux/idr.h> 24cb4d2b3fSMartin KaFai Lau #include <linux/cred.h> 25cb4d2b3fSMartin KaFai Lau #include <linux/timekeeping.h> 26cb4d2b3fSMartin KaFai Lau #include <linux/ctype.h> 279ef09e35SMark Rutland #include <linux/nospec.h> 28bae141f5SDaniel Borkmann #include <linux/audit.h> 29ccfe29ebSAlexei Starovoitov #include <uapi/linux/btf.h> 30ca5999fdSMike Rapoport #include <linux/pgtable.h> 319e4e01dfSKP Singh #include <linux/bpf_lsm.h> 32457f4436SAndrii Nakryiko #include <linux/poll.h> 334d7d7f69SKumar Kartikeya Dwivedi #include <linux/sort.h> 34a3fd7ceeSJakub Sitnicki #include <linux/bpf-netns.h> 351e6c62a8SAlexei Starovoitov #include <linux/rcupdate_trace.h> 3648edc1f7SRoman Gushchin #include <linux/memcontrol.h> 370dcac272SJiri Olsa #include <linux/trace_events.h> 3884601d6eSFlorian Westphal #include <net/netfilter/nf_bpf_link.h> 3999c55f7dSAlexei Starovoitov 40e420bed0SDaniel Borkmann #include <net/tcx.h> 41e420bed0SDaniel Borkmann 42da765a2fSDaniel Borkmann #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 4314dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 4414dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 45da765a2fSDaniel Borkmann #define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY) 4614dc6f04SMartin KaFai Lau #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 47da765a2fSDaniel Borkmann #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \ 48da765a2fSDaniel Borkmann IS_FD_HASH(map)) 4914dc6f04SMartin KaFai Lau 506e71b04aSChenbo Feng #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) 516e71b04aSChenbo Feng 52b121d1e7SAlexei Starovoitov DEFINE_PER_CPU(int, bpf_prog_active); 53dc4bb0e2SMartin KaFai Lau static DEFINE_IDR(prog_idr); 54dc4bb0e2SMartin KaFai Lau static DEFINE_SPINLOCK(prog_idr_lock); 55f3f1c054SMartin KaFai Lau static DEFINE_IDR(map_idr); 56f3f1c054SMartin KaFai Lau static DEFINE_SPINLOCK(map_idr_lock); 57a3b80e10SAndrii Nakryiko static DEFINE_IDR(link_idr); 58a3b80e10SAndrii Nakryiko static DEFINE_SPINLOCK(link_idr_lock); 59b121d1e7SAlexei Starovoitov 6008389d88SDaniel Borkmann int sysctl_unprivileged_bpf_disabled __read_mostly = 6108389d88SDaniel Borkmann IS_BUILTIN(CONFIG_BPF_UNPRIV_DEFAULT_OFF) ? 2 : 0; 621be7f75dSAlexei Starovoitov 6340077e0cSJohannes Berg static const struct bpf_map_ops * const bpf_map_types[] = { 6491cc1a99SAlexei Starovoitov #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) 6540077e0cSJohannes Berg #define BPF_MAP_TYPE(_id, _ops) \ 6640077e0cSJohannes Berg [_id] = &_ops, 67f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) 6840077e0cSJohannes Berg #include <linux/bpf_types.h> 6940077e0cSJohannes Berg #undef BPF_PROG_TYPE 7040077e0cSJohannes Berg #undef BPF_MAP_TYPE 71f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 7240077e0cSJohannes Berg }; 7399c55f7dSAlexei Starovoitov 74752ba56fSMickaël Salaün /* 75752ba56fSMickaël Salaün * If we're handed a bigger struct than we know of, ensure all the unknown bits 76752ba56fSMickaël Salaün * are 0 - i.e. new user-space does not rely on any kernel feature extensions 77752ba56fSMickaël Salaün * we don't know about yet. 78752ba56fSMickaël Salaün * 79752ba56fSMickaël Salaün * There is a ToCToU between this function call and the following 80752ba56fSMickaël Salaün * copy_from_user() call. However, this is not a concern since this function is 81752ba56fSMickaël Salaün * meant to be a future-proofing of bits. 82752ba56fSMickaël Salaün */ 83af2ac3e1SAlexei Starovoitov int bpf_check_uarg_tail_zero(bpfptr_t uaddr, 8458291a74SMickaël Salaün size_t expected_size, 8558291a74SMickaël Salaün size_t actual_size) 8658291a74SMickaël Salaün { 87b7e4b65fSAl Viro int res; 8858291a74SMickaël Salaün 89752ba56fSMickaël Salaün if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 90752ba56fSMickaël Salaün return -E2BIG; 91752ba56fSMickaël Salaün 9258291a74SMickaël Salaün if (actual_size <= expected_size) 9358291a74SMickaël Salaün return 0; 9458291a74SMickaël Salaün 95af2ac3e1SAlexei Starovoitov if (uaddr.is_kernel) 96af2ac3e1SAlexei Starovoitov res = memchr_inv(uaddr.kernel + expected_size, 0, 97af2ac3e1SAlexei Starovoitov actual_size - expected_size) == NULL; 98af2ac3e1SAlexei Starovoitov else 99af2ac3e1SAlexei Starovoitov res = check_zeroed_user(uaddr.user + expected_size, 100af2ac3e1SAlexei Starovoitov actual_size - expected_size); 101b7e4b65fSAl Viro if (res < 0) 102b7e4b65fSAl Viro return res; 103b7e4b65fSAl Viro return res ? 0 : -E2BIG; 10458291a74SMickaël Salaün } 10558291a74SMickaël Salaün 106a3884572SJakub Kicinski const struct bpf_map_ops bpf_map_offload_ops = { 107f4d05259SMartin KaFai Lau .map_meta_equal = bpf_map_meta_equal, 108a3884572SJakub Kicinski .map_alloc = bpf_map_offload_map_alloc, 109a3884572SJakub Kicinski .map_free = bpf_map_offload_map_free, 110e8d2bec0SDaniel Borkmann .map_check_btf = map_check_no_btf, 1119629363cSYafang Shao .map_mem_usage = bpf_map_offload_map_mem_usage, 112a3884572SJakub Kicinski }; 113a3884572SJakub Kicinski 114353050beSDaniel Borkmann static void bpf_map_write_active_inc(struct bpf_map *map) 115353050beSDaniel Borkmann { 116353050beSDaniel Borkmann atomic64_inc(&map->writecnt); 117353050beSDaniel Borkmann } 118353050beSDaniel Borkmann 119353050beSDaniel Borkmann static void bpf_map_write_active_dec(struct bpf_map *map) 120353050beSDaniel Borkmann { 121353050beSDaniel Borkmann atomic64_dec(&map->writecnt); 122353050beSDaniel Borkmann } 123353050beSDaniel Borkmann 124353050beSDaniel Borkmann bool bpf_map_write_active(const struct bpf_map *map) 125353050beSDaniel Borkmann { 126353050beSDaniel Borkmann return atomic64_read(&map->writecnt) != 0; 127353050beSDaniel Borkmann } 128353050beSDaniel Borkmann 12980ee81e0SRoman Gushchin static u32 bpf_map_value_size(const struct bpf_map *map) 13015c14a3dSBrian Vazquez { 13115c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 13215c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 13315c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 13415c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 13515c14a3dSBrian Vazquez return round_up(map->value_size, 8) * num_possible_cpus(); 13615c14a3dSBrian Vazquez else if (IS_FD_MAP(map)) 13715c14a3dSBrian Vazquez return sizeof(u32); 13815c14a3dSBrian Vazquez else 13915c14a3dSBrian Vazquez return map->value_size; 14015c14a3dSBrian Vazquez } 14115c14a3dSBrian Vazquez 14215c14a3dSBrian Vazquez static void maybe_wait_bpf_programs(struct bpf_map *map) 14315c14a3dSBrian Vazquez { 14415c14a3dSBrian Vazquez /* Wait for any running BPF programs to complete so that 14515c14a3dSBrian Vazquez * userspace, when we return to it, knows that all programs 14615c14a3dSBrian Vazquez * that could be running use the new map value. 14715c14a3dSBrian Vazquez */ 14815c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS || 14915c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 15015c14a3dSBrian Vazquez synchronize_rcu(); 15115c14a3dSBrian Vazquez } 15215c14a3dSBrian Vazquez 1533af43ba4SHou Tao static int bpf_map_update_value(struct bpf_map *map, struct file *map_file, 1543af43ba4SHou Tao void *key, void *value, __u64 flags) 15515c14a3dSBrian Vazquez { 15615c14a3dSBrian Vazquez int err; 15715c14a3dSBrian Vazquez 15815c14a3dSBrian Vazquez /* Need to create a kthread, thus must support schedule */ 1599d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) { 16015c14a3dSBrian Vazquez return bpf_map_offload_update_elem(map, key, value, flags); 16115c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_CPUMAP || 16215c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 16315c14a3dSBrian Vazquez return map->ops->map_update_elem(map, key, value, flags); 16413b79d3fSLorenz Bauer } else if (map->map_type == BPF_MAP_TYPE_SOCKHASH || 16513b79d3fSLorenz Bauer map->map_type == BPF_MAP_TYPE_SOCKMAP) { 16613b79d3fSLorenz Bauer return sock_map_update_elem_sys(map, key, value, flags); 16715c14a3dSBrian Vazquez } else if (IS_FD_PROG_ARRAY(map)) { 1683af43ba4SHou Tao return bpf_fd_array_map_update_elem(map, map_file, key, value, 16915c14a3dSBrian Vazquez flags); 17015c14a3dSBrian Vazquez } 17115c14a3dSBrian Vazquez 172b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 17315c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 17415c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 17515c14a3dSBrian Vazquez err = bpf_percpu_hash_update(map, key, value, flags); 17615c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 17715c14a3dSBrian Vazquez err = bpf_percpu_array_update(map, key, value, flags); 17815c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 17915c14a3dSBrian Vazquez err = bpf_percpu_cgroup_storage_update(map, key, value, 18015c14a3dSBrian Vazquez flags); 18115c14a3dSBrian Vazquez } else if (IS_FD_ARRAY(map)) { 18215c14a3dSBrian Vazquez rcu_read_lock(); 1833af43ba4SHou Tao err = bpf_fd_array_map_update_elem(map, map_file, key, value, 18415c14a3dSBrian Vazquez flags); 18515c14a3dSBrian Vazquez rcu_read_unlock(); 18615c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 18715c14a3dSBrian Vazquez rcu_read_lock(); 1883af43ba4SHou Tao err = bpf_fd_htab_map_update_elem(map, map_file, key, value, 18915c14a3dSBrian Vazquez flags); 19015c14a3dSBrian Vazquez rcu_read_unlock(); 19115c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 19215c14a3dSBrian Vazquez /* rcu_read_lock() is not needed */ 19315c14a3dSBrian Vazquez err = bpf_fd_reuseport_array_update_elem(map, key, value, 19415c14a3dSBrian Vazquez flags); 19515c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 1969330986cSJoanne Koong map->map_type == BPF_MAP_TYPE_STACK || 1979330986cSJoanne Koong map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) { 19815c14a3dSBrian Vazquez err = map->ops->map_push_elem(map, value, flags); 19915c14a3dSBrian Vazquez } else { 20015c14a3dSBrian Vazquez rcu_read_lock(); 20115c14a3dSBrian Vazquez err = map->ops->map_update_elem(map, key, value, flags); 20215c14a3dSBrian Vazquez rcu_read_unlock(); 20315c14a3dSBrian Vazquez } 204b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 20515c14a3dSBrian Vazquez maybe_wait_bpf_programs(map); 20615c14a3dSBrian Vazquez 20715c14a3dSBrian Vazquez return err; 20815c14a3dSBrian Vazquez } 20915c14a3dSBrian Vazquez 21015c14a3dSBrian Vazquez static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value, 21115c14a3dSBrian Vazquez __u64 flags) 21215c14a3dSBrian Vazquez { 21315c14a3dSBrian Vazquez void *ptr; 21415c14a3dSBrian Vazquez int err; 21515c14a3dSBrian Vazquez 2169d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) 217cb4d03abSBrian Vazquez return bpf_map_offload_lookup_elem(map, key, value); 21815c14a3dSBrian Vazquez 219b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 22015c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 22115c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 22215c14a3dSBrian Vazquez err = bpf_percpu_hash_copy(map, key, value); 22315c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 22415c14a3dSBrian Vazquez err = bpf_percpu_array_copy(map, key, value); 22515c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 22615c14a3dSBrian Vazquez err = bpf_percpu_cgroup_storage_copy(map, key, value); 22715c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 22815c14a3dSBrian Vazquez err = bpf_stackmap_copy(map, key, value); 22915c14a3dSBrian Vazquez } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) { 23015c14a3dSBrian Vazquez err = bpf_fd_array_map_lookup_elem(map, key, value); 23115c14a3dSBrian Vazquez } else if (IS_FD_HASH(map)) { 23215c14a3dSBrian Vazquez err = bpf_fd_htab_map_lookup_elem(map, key, value); 23315c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 23415c14a3dSBrian Vazquez err = bpf_fd_reuseport_array_lookup_elem(map, key, value); 23515c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 2369330986cSJoanne Koong map->map_type == BPF_MAP_TYPE_STACK || 2379330986cSJoanne Koong map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) { 23815c14a3dSBrian Vazquez err = map->ops->map_peek_elem(map, value); 23915c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 24015c14a3dSBrian Vazquez /* struct_ops map requires directly updating "value" */ 24115c14a3dSBrian Vazquez err = bpf_struct_ops_map_sys_lookup_elem(map, key, value); 24215c14a3dSBrian Vazquez } else { 24315c14a3dSBrian Vazquez rcu_read_lock(); 24415c14a3dSBrian Vazquez if (map->ops->map_lookup_elem_sys_only) 24515c14a3dSBrian Vazquez ptr = map->ops->map_lookup_elem_sys_only(map, key); 24615c14a3dSBrian Vazquez else 24715c14a3dSBrian Vazquez ptr = map->ops->map_lookup_elem(map, key); 24815c14a3dSBrian Vazquez if (IS_ERR(ptr)) { 24915c14a3dSBrian Vazquez err = PTR_ERR(ptr); 25015c14a3dSBrian Vazquez } else if (!ptr) { 25115c14a3dSBrian Vazquez err = -ENOENT; 25215c14a3dSBrian Vazquez } else { 25315c14a3dSBrian Vazquez err = 0; 25415c14a3dSBrian Vazquez if (flags & BPF_F_LOCK) 25515c14a3dSBrian Vazquez /* lock 'ptr' and copy everything but lock */ 25615c14a3dSBrian Vazquez copy_map_value_locked(map, value, ptr, true); 25715c14a3dSBrian Vazquez else 25815c14a3dSBrian Vazquez copy_map_value(map, value, ptr); 25968134668SAlexei Starovoitov /* mask lock and timer, since value wasn't zero inited */ 26068134668SAlexei Starovoitov check_and_init_map_value(map, value); 26115c14a3dSBrian Vazquez } 26215c14a3dSBrian Vazquez rcu_read_unlock(); 26315c14a3dSBrian Vazquez } 26415c14a3dSBrian Vazquez 265b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 26615c14a3dSBrian Vazquez maybe_wait_bpf_programs(map); 26715c14a3dSBrian Vazquez 26815c14a3dSBrian Vazquez return err; 26915c14a3dSBrian Vazquez } 27015c14a3dSBrian Vazquez 271d5299b67SRoman Gushchin /* Please, do not use this function outside from the map creation path 272d5299b67SRoman Gushchin * (e.g. in map update path) without taking care of setting the active 273d5299b67SRoman Gushchin * memory cgroup (see at bpf_map_kmalloc_node() for example). 274d5299b67SRoman Gushchin */ 275196e8ca7SDaniel Borkmann static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable) 276d407bd25SDaniel Borkmann { 277f01a7dbeSMartynas Pumputis /* We really just want to fail instead of triggering OOM killer 278f01a7dbeSMartynas Pumputis * under memory pressure, therefore we set __GFP_NORETRY to kmalloc, 279f01a7dbeSMartynas Pumputis * which is used for lower order allocation requests. 280f01a7dbeSMartynas Pumputis * 281f01a7dbeSMartynas Pumputis * It has been observed that higher order allocation requests done by 282f01a7dbeSMartynas Pumputis * vmalloc with __GFP_NORETRY being set might fail due to not trying 283f01a7dbeSMartynas Pumputis * to reclaim memory from the page cache, thus we set 284f01a7dbeSMartynas Pumputis * __GFP_RETRY_MAYFAIL to avoid such situations. 285d407bd25SDaniel Borkmann */ 286f01a7dbeSMartynas Pumputis 287ee53cbfbSYafang Shao gfp_t gfp = bpf_memcg_flags(__GFP_NOWARN | __GFP_ZERO); 288041de93fSChristoph Hellwig unsigned int flags = 0; 289041de93fSChristoph Hellwig unsigned long align = 1; 290d407bd25SDaniel Borkmann void *area; 291d407bd25SDaniel Borkmann 292196e8ca7SDaniel Borkmann if (size >= SIZE_MAX) 293196e8ca7SDaniel Borkmann return NULL; 294196e8ca7SDaniel Borkmann 295fc970227SAndrii Nakryiko /* kmalloc()'ed memory can't be mmap()'ed */ 296041de93fSChristoph Hellwig if (mmapable) { 297041de93fSChristoph Hellwig BUG_ON(!PAGE_ALIGNED(size)); 298041de93fSChristoph Hellwig align = SHMLBA; 299041de93fSChristoph Hellwig flags = VM_USERMAP; 300041de93fSChristoph Hellwig } else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 301041de93fSChristoph Hellwig area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY, 302f01a7dbeSMartynas Pumputis numa_node); 303d407bd25SDaniel Borkmann if (area != NULL) 304d407bd25SDaniel Borkmann return area; 305d407bd25SDaniel Borkmann } 306041de93fSChristoph Hellwig 307041de93fSChristoph Hellwig return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, 308041de93fSChristoph Hellwig gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL, 309041de93fSChristoph Hellwig flags, numa_node, __builtin_return_address(0)); 310d407bd25SDaniel Borkmann } 311d407bd25SDaniel Borkmann 312196e8ca7SDaniel Borkmann void *bpf_map_area_alloc(u64 size, int numa_node) 313fc970227SAndrii Nakryiko { 314fc970227SAndrii Nakryiko return __bpf_map_area_alloc(size, numa_node, false); 315fc970227SAndrii Nakryiko } 316fc970227SAndrii Nakryiko 317196e8ca7SDaniel Borkmann void *bpf_map_area_mmapable_alloc(u64 size, int numa_node) 318fc970227SAndrii Nakryiko { 319fc970227SAndrii Nakryiko return __bpf_map_area_alloc(size, numa_node, true); 320fc970227SAndrii Nakryiko } 321fc970227SAndrii Nakryiko 322d407bd25SDaniel Borkmann void bpf_map_area_free(void *area) 323d407bd25SDaniel Borkmann { 324d407bd25SDaniel Borkmann kvfree(area); 325d407bd25SDaniel Borkmann } 326d407bd25SDaniel Borkmann 327be70bcd5SDaniel Borkmann static u32 bpf_map_flags_retain_permanent(u32 flags) 328be70bcd5SDaniel Borkmann { 329be70bcd5SDaniel Borkmann /* Some map creation flags are not tied to the map object but 330be70bcd5SDaniel Borkmann * rather to the map fd instead, so they have no meaning upon 331be70bcd5SDaniel Borkmann * map object inspection since multiple file descriptors with 332be70bcd5SDaniel Borkmann * different (access) properties can exist here. Thus, given 333be70bcd5SDaniel Borkmann * this has zero meaning for the map itself, lets clear these 334be70bcd5SDaniel Borkmann * from here. 335be70bcd5SDaniel Borkmann */ 336be70bcd5SDaniel Borkmann return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY); 337be70bcd5SDaniel Borkmann } 338be70bcd5SDaniel Borkmann 339bd475643SJakub Kicinski void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) 340bd475643SJakub Kicinski { 341bd475643SJakub Kicinski map->map_type = attr->map_type; 342bd475643SJakub Kicinski map->key_size = attr->key_size; 343bd475643SJakub Kicinski map->value_size = attr->value_size; 344bd475643SJakub Kicinski map->max_entries = attr->max_entries; 345be70bcd5SDaniel Borkmann map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags); 346bd475643SJakub Kicinski map->numa_node = bpf_map_attr_numa_node(attr); 3479330986cSJoanne Koong map->map_extra = attr->map_extra; 348bd475643SJakub Kicinski } 349bd475643SJakub Kicinski 350f3f1c054SMartin KaFai Lau static int bpf_map_alloc_id(struct bpf_map *map) 351f3f1c054SMartin KaFai Lau { 352f3f1c054SMartin KaFai Lau int id; 353f3f1c054SMartin KaFai Lau 354b76354cdSShaohua Li idr_preload(GFP_KERNEL); 355f3f1c054SMartin KaFai Lau spin_lock_bh(&map_idr_lock); 356f3f1c054SMartin KaFai Lau id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 357f3f1c054SMartin KaFai Lau if (id > 0) 358f3f1c054SMartin KaFai Lau map->id = id; 359f3f1c054SMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 360b76354cdSShaohua Li idr_preload_end(); 361f3f1c054SMartin KaFai Lau 362f3f1c054SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 363f3f1c054SMartin KaFai Lau return -ENOSPC; 364f3f1c054SMartin KaFai Lau 365f3f1c054SMartin KaFai Lau return id > 0 ? 0 : id; 366f3f1c054SMartin KaFai Lau } 367f3f1c054SMartin KaFai Lau 368158e5e9eSTobias Klauser void bpf_map_free_id(struct bpf_map *map) 369f3f1c054SMartin KaFai Lau { 370930651a7SEric Dumazet unsigned long flags; 371930651a7SEric Dumazet 372a3884572SJakub Kicinski /* Offloaded maps are removed from the IDR store when their device 373a3884572SJakub Kicinski * disappears - even if someone holds an fd to them they are unusable, 374a3884572SJakub Kicinski * the memory is gone, all ops will fail; they are simply waiting for 375a3884572SJakub Kicinski * refcnt to drop to be freed. 376a3884572SJakub Kicinski */ 377a3884572SJakub Kicinski if (!map->id) 378a3884572SJakub Kicinski return; 379a3884572SJakub Kicinski 380930651a7SEric Dumazet spin_lock_irqsave(&map_idr_lock, flags); 381bd5f5f4eSMartin KaFai Lau 382f3f1c054SMartin KaFai Lau idr_remove(&map_idr, map->id); 383a3884572SJakub Kicinski map->id = 0; 384bd5f5f4eSMartin KaFai Lau 385930651a7SEric Dumazet spin_unlock_irqrestore(&map_idr_lock, flags); 386f3f1c054SMartin KaFai Lau } 387f3f1c054SMartin KaFai Lau 38848edc1f7SRoman Gushchin #ifdef CONFIG_MEMCG_KMEM 38948edc1f7SRoman Gushchin static void bpf_map_save_memcg(struct bpf_map *map) 39048edc1f7SRoman Gushchin { 3914201d9abSRoman Gushchin /* Currently if a map is created by a process belonging to the root 3924201d9abSRoman Gushchin * memory cgroup, get_obj_cgroup_from_current() will return NULL. 3934201d9abSRoman Gushchin * So we have to check map->objcg for being NULL each time it's 3944201d9abSRoman Gushchin * being used. 3954201d9abSRoman Gushchin */ 396ee53cbfbSYafang Shao if (memcg_bpf_enabled()) 3974201d9abSRoman Gushchin map->objcg = get_obj_cgroup_from_current(); 39848edc1f7SRoman Gushchin } 39948edc1f7SRoman Gushchin 40048edc1f7SRoman Gushchin static void bpf_map_release_memcg(struct bpf_map *map) 40148edc1f7SRoman Gushchin { 4024201d9abSRoman Gushchin if (map->objcg) 4034201d9abSRoman Gushchin obj_cgroup_put(map->objcg); 4044201d9abSRoman Gushchin } 4054201d9abSRoman Gushchin 4064201d9abSRoman Gushchin static struct mem_cgroup *bpf_map_get_memcg(const struct bpf_map *map) 4074201d9abSRoman Gushchin { 4084201d9abSRoman Gushchin if (map->objcg) 4094201d9abSRoman Gushchin return get_mem_cgroup_from_objcg(map->objcg); 4104201d9abSRoman Gushchin 4114201d9abSRoman Gushchin return root_mem_cgroup; 41248edc1f7SRoman Gushchin } 41348edc1f7SRoman Gushchin 41448edc1f7SRoman Gushchin void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, 41548edc1f7SRoman Gushchin int node) 41648edc1f7SRoman Gushchin { 4174201d9abSRoman Gushchin struct mem_cgroup *memcg, *old_memcg; 41848edc1f7SRoman Gushchin void *ptr; 41948edc1f7SRoman Gushchin 4204201d9abSRoman Gushchin memcg = bpf_map_get_memcg(map); 4214201d9abSRoman Gushchin old_memcg = set_active_memcg(memcg); 42248edc1f7SRoman Gushchin ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node); 42348edc1f7SRoman Gushchin set_active_memcg(old_memcg); 4244201d9abSRoman Gushchin mem_cgroup_put(memcg); 42548edc1f7SRoman Gushchin 42648edc1f7SRoman Gushchin return ptr; 42748edc1f7SRoman Gushchin } 42848edc1f7SRoman Gushchin 42948edc1f7SRoman Gushchin void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags) 43048edc1f7SRoman Gushchin { 4314201d9abSRoman Gushchin struct mem_cgroup *memcg, *old_memcg; 43248edc1f7SRoman Gushchin void *ptr; 43348edc1f7SRoman Gushchin 4344201d9abSRoman Gushchin memcg = bpf_map_get_memcg(map); 4354201d9abSRoman Gushchin old_memcg = set_active_memcg(memcg); 43648edc1f7SRoman Gushchin ptr = kzalloc(size, flags | __GFP_ACCOUNT); 43748edc1f7SRoman Gushchin set_active_memcg(old_memcg); 4384201d9abSRoman Gushchin mem_cgroup_put(memcg); 43948edc1f7SRoman Gushchin 44048edc1f7SRoman Gushchin return ptr; 44148edc1f7SRoman Gushchin } 44248edc1f7SRoman Gushchin 443ddef81b5SYafang Shao void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size, 444ddef81b5SYafang Shao gfp_t flags) 445ddef81b5SYafang Shao { 446ddef81b5SYafang Shao struct mem_cgroup *memcg, *old_memcg; 447ddef81b5SYafang Shao void *ptr; 448ddef81b5SYafang Shao 449ddef81b5SYafang Shao memcg = bpf_map_get_memcg(map); 450ddef81b5SYafang Shao old_memcg = set_active_memcg(memcg); 451ddef81b5SYafang Shao ptr = kvcalloc(n, size, flags | __GFP_ACCOUNT); 452ddef81b5SYafang Shao set_active_memcg(old_memcg); 453ddef81b5SYafang Shao mem_cgroup_put(memcg); 454ddef81b5SYafang Shao 455ddef81b5SYafang Shao return ptr; 456ddef81b5SYafang Shao } 457ddef81b5SYafang Shao 45848edc1f7SRoman Gushchin void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, 45948edc1f7SRoman Gushchin size_t align, gfp_t flags) 46048edc1f7SRoman Gushchin { 4614201d9abSRoman Gushchin struct mem_cgroup *memcg, *old_memcg; 46248edc1f7SRoman Gushchin void __percpu *ptr; 46348edc1f7SRoman Gushchin 4644201d9abSRoman Gushchin memcg = bpf_map_get_memcg(map); 4654201d9abSRoman Gushchin old_memcg = set_active_memcg(memcg); 46648edc1f7SRoman Gushchin ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT); 46748edc1f7SRoman Gushchin set_active_memcg(old_memcg); 4684201d9abSRoman Gushchin mem_cgroup_put(memcg); 46948edc1f7SRoman Gushchin 47048edc1f7SRoman Gushchin return ptr; 47148edc1f7SRoman Gushchin } 47248edc1f7SRoman Gushchin 47348edc1f7SRoman Gushchin #else 47448edc1f7SRoman Gushchin static void bpf_map_save_memcg(struct bpf_map *map) 47548edc1f7SRoman Gushchin { 47648edc1f7SRoman Gushchin } 47748edc1f7SRoman Gushchin 47848edc1f7SRoman Gushchin static void bpf_map_release_memcg(struct bpf_map *map) 47948edc1f7SRoman Gushchin { 48048edc1f7SRoman Gushchin } 48148edc1f7SRoman Gushchin #endif 48248edc1f7SRoman Gushchin 483aa3496acSKumar Kartikeya Dwivedi static int btf_field_cmp(const void *a, const void *b) 48461df10c7SKumar Kartikeya Dwivedi { 485aa3496acSKumar Kartikeya Dwivedi const struct btf_field *f1 = a, *f2 = b; 48661df10c7SKumar Kartikeya Dwivedi 487aa3496acSKumar Kartikeya Dwivedi if (f1->offset < f2->offset) 48861df10c7SKumar Kartikeya Dwivedi return -1; 489aa3496acSKumar Kartikeya Dwivedi else if (f1->offset > f2->offset) 49061df10c7SKumar Kartikeya Dwivedi return 1; 49161df10c7SKumar Kartikeya Dwivedi return 0; 49261df10c7SKumar Kartikeya Dwivedi } 49361df10c7SKumar Kartikeya Dwivedi 494aa3496acSKumar Kartikeya Dwivedi struct btf_field *btf_record_find(const struct btf_record *rec, u32 offset, 49574843b57SDave Marchevsky u32 field_mask) 49661df10c7SKumar Kartikeya Dwivedi { 497aa3496acSKumar Kartikeya Dwivedi struct btf_field *field; 49861df10c7SKumar Kartikeya Dwivedi 49974843b57SDave Marchevsky if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & field_mask)) 50061df10c7SKumar Kartikeya Dwivedi return NULL; 501aa3496acSKumar Kartikeya Dwivedi field = bsearch(&offset, rec->fields, rec->cnt, sizeof(rec->fields[0]), btf_field_cmp); 50274843b57SDave Marchevsky if (!field || !(field->type & field_mask)) 503aa3496acSKumar Kartikeya Dwivedi return NULL; 504aa3496acSKumar Kartikeya Dwivedi return field; 50561df10c7SKumar Kartikeya Dwivedi } 50661df10c7SKumar Kartikeya Dwivedi 507aa3496acSKumar Kartikeya Dwivedi void btf_record_free(struct btf_record *rec) 50861df10c7SKumar Kartikeya Dwivedi { 50961df10c7SKumar Kartikeya Dwivedi int i; 51061df10c7SKumar Kartikeya Dwivedi 511aa3496acSKumar Kartikeya Dwivedi if (IS_ERR_OR_NULL(rec)) 51261df10c7SKumar Kartikeya Dwivedi return; 513aa3496acSKumar Kartikeya Dwivedi for (i = 0; i < rec->cnt; i++) { 514aa3496acSKumar Kartikeya Dwivedi switch (rec->fields[i].type) { 515aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_UNREF: 516aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_REF: 517*55db92f4SYonghong Song case BPF_KPTR_PERCPU: 518aa3496acSKumar Kartikeya Dwivedi if (rec->fields[i].kptr.module) 519aa3496acSKumar Kartikeya Dwivedi module_put(rec->fields[i].kptr.module); 520aa3496acSKumar Kartikeya Dwivedi btf_put(rec->fields[i].kptr.btf); 521aa3496acSKumar Kartikeya Dwivedi break; 522f0c5941fSKumar Kartikeya Dwivedi case BPF_LIST_HEAD: 5238ffa5cc1SKumar Kartikeya Dwivedi case BPF_LIST_NODE: 5249c395c1bSDave Marchevsky case BPF_RB_ROOT: 5259c395c1bSDave Marchevsky case BPF_RB_NODE: 5269c395c1bSDave Marchevsky case BPF_SPIN_LOCK: 5279c395c1bSDave Marchevsky case BPF_TIMER: 528d54730b5SDave Marchevsky case BPF_REFCOUNT: 5299c395c1bSDave Marchevsky /* Nothing to release */ 530f0c5941fSKumar Kartikeya Dwivedi break; 531aa3496acSKumar Kartikeya Dwivedi default: 532aa3496acSKumar Kartikeya Dwivedi WARN_ON_ONCE(1); 53314a324f6SKumar Kartikeya Dwivedi continue; 53414a324f6SKumar Kartikeya Dwivedi } 535aa3496acSKumar Kartikeya Dwivedi } 536aa3496acSKumar Kartikeya Dwivedi kfree(rec); 537aa3496acSKumar Kartikeya Dwivedi } 538aa3496acSKumar Kartikeya Dwivedi 539aa3496acSKumar Kartikeya Dwivedi void bpf_map_free_record(struct bpf_map *map) 540aa3496acSKumar Kartikeya Dwivedi { 541aa3496acSKumar Kartikeya Dwivedi btf_record_free(map->record); 542aa3496acSKumar Kartikeya Dwivedi map->record = NULL; 543aa3496acSKumar Kartikeya Dwivedi } 544aa3496acSKumar Kartikeya Dwivedi 545aa3496acSKumar Kartikeya Dwivedi struct btf_record *btf_record_dup(const struct btf_record *rec) 546aa3496acSKumar Kartikeya Dwivedi { 547aa3496acSKumar Kartikeya Dwivedi const struct btf_field *fields; 548aa3496acSKumar Kartikeya Dwivedi struct btf_record *new_rec; 549aa3496acSKumar Kartikeya Dwivedi int ret, size, i; 550aa3496acSKumar Kartikeya Dwivedi 551aa3496acSKumar Kartikeya Dwivedi if (IS_ERR_OR_NULL(rec)) 552aa3496acSKumar Kartikeya Dwivedi return NULL; 553aa3496acSKumar Kartikeya Dwivedi size = offsetof(struct btf_record, fields[rec->cnt]); 554aa3496acSKumar Kartikeya Dwivedi new_rec = kmemdup(rec, size, GFP_KERNEL | __GFP_NOWARN); 555aa3496acSKumar Kartikeya Dwivedi if (!new_rec) 556aa3496acSKumar Kartikeya Dwivedi return ERR_PTR(-ENOMEM); 557aa3496acSKumar Kartikeya Dwivedi /* Do a deep copy of the btf_record */ 558aa3496acSKumar Kartikeya Dwivedi fields = rec->fields; 559aa3496acSKumar Kartikeya Dwivedi new_rec->cnt = 0; 560aa3496acSKumar Kartikeya Dwivedi for (i = 0; i < rec->cnt; i++) { 561aa3496acSKumar Kartikeya Dwivedi switch (fields[i].type) { 562aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_UNREF: 563aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_REF: 564*55db92f4SYonghong Song case BPF_KPTR_PERCPU: 565aa3496acSKumar Kartikeya Dwivedi btf_get(fields[i].kptr.btf); 566aa3496acSKumar Kartikeya Dwivedi if (fields[i].kptr.module && !try_module_get(fields[i].kptr.module)) { 567aa3496acSKumar Kartikeya Dwivedi ret = -ENXIO; 568aa3496acSKumar Kartikeya Dwivedi goto free; 569aa3496acSKumar Kartikeya Dwivedi } 570aa3496acSKumar Kartikeya Dwivedi break; 571f0c5941fSKumar Kartikeya Dwivedi case BPF_LIST_HEAD: 5728ffa5cc1SKumar Kartikeya Dwivedi case BPF_LIST_NODE: 5739c395c1bSDave Marchevsky case BPF_RB_ROOT: 5749c395c1bSDave Marchevsky case BPF_RB_NODE: 5759c395c1bSDave Marchevsky case BPF_SPIN_LOCK: 5769c395c1bSDave Marchevsky case BPF_TIMER: 577d54730b5SDave Marchevsky case BPF_REFCOUNT: 5789c395c1bSDave Marchevsky /* Nothing to acquire */ 579f0c5941fSKumar Kartikeya Dwivedi break; 580aa3496acSKumar Kartikeya Dwivedi default: 581aa3496acSKumar Kartikeya Dwivedi ret = -EFAULT; 582aa3496acSKumar Kartikeya Dwivedi WARN_ON_ONCE(1); 583aa3496acSKumar Kartikeya Dwivedi goto free; 584aa3496acSKumar Kartikeya Dwivedi } 585aa3496acSKumar Kartikeya Dwivedi new_rec->cnt++; 586aa3496acSKumar Kartikeya Dwivedi } 587aa3496acSKumar Kartikeya Dwivedi return new_rec; 588aa3496acSKumar Kartikeya Dwivedi free: 589aa3496acSKumar Kartikeya Dwivedi btf_record_free(new_rec); 590aa3496acSKumar Kartikeya Dwivedi return ERR_PTR(ret); 591aa3496acSKumar Kartikeya Dwivedi } 592aa3496acSKumar Kartikeya Dwivedi 593aa3496acSKumar Kartikeya Dwivedi bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b) 594aa3496acSKumar Kartikeya Dwivedi { 595aa3496acSKumar Kartikeya Dwivedi bool a_has_fields = !IS_ERR_OR_NULL(rec_a), b_has_fields = !IS_ERR_OR_NULL(rec_b); 596aa3496acSKumar Kartikeya Dwivedi int size; 597aa3496acSKumar Kartikeya Dwivedi 598aa3496acSKumar Kartikeya Dwivedi if (!a_has_fields && !b_has_fields) 599aa3496acSKumar Kartikeya Dwivedi return true; 600aa3496acSKumar Kartikeya Dwivedi if (a_has_fields != b_has_fields) 601aa3496acSKumar Kartikeya Dwivedi return false; 602aa3496acSKumar Kartikeya Dwivedi if (rec_a->cnt != rec_b->cnt) 603aa3496acSKumar Kartikeya Dwivedi return false; 604aa3496acSKumar Kartikeya Dwivedi size = offsetof(struct btf_record, fields[rec_a->cnt]); 605c22dfdd2SKumar Kartikeya Dwivedi /* btf_parse_fields uses kzalloc to allocate a btf_record, so unused 606c22dfdd2SKumar Kartikeya Dwivedi * members are zeroed out. So memcmp is safe to do without worrying 607c22dfdd2SKumar Kartikeya Dwivedi * about padding/unused fields. 608c22dfdd2SKumar Kartikeya Dwivedi * 609c22dfdd2SKumar Kartikeya Dwivedi * While spin_lock, timer, and kptr have no relation to map BTF, 610c22dfdd2SKumar Kartikeya Dwivedi * list_head metadata is specific to map BTF, the btf and value_rec 611c22dfdd2SKumar Kartikeya Dwivedi * members in particular. btf is the map BTF, while value_rec points to 612c22dfdd2SKumar Kartikeya Dwivedi * btf_record in that map BTF. 613c22dfdd2SKumar Kartikeya Dwivedi * 614c22dfdd2SKumar Kartikeya Dwivedi * So while by default, we don't rely on the map BTF (which the records 615c22dfdd2SKumar Kartikeya Dwivedi * were parsed from) matching for both records, which is not backwards 616c22dfdd2SKumar Kartikeya Dwivedi * compatible, in case list_head is part of it, we implicitly rely on 617c22dfdd2SKumar Kartikeya Dwivedi * that by way of depending on memcmp succeeding for it. 618c22dfdd2SKumar Kartikeya Dwivedi */ 619aa3496acSKumar Kartikeya Dwivedi return !memcmp(rec_a, rec_b, size); 620aa3496acSKumar Kartikeya Dwivedi } 621aa3496acSKumar Kartikeya Dwivedi 622db559117SKumar Kartikeya Dwivedi void bpf_obj_free_timer(const struct btf_record *rec, void *obj) 623db559117SKumar Kartikeya Dwivedi { 624db559117SKumar Kartikeya Dwivedi if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_TIMER))) 625db559117SKumar Kartikeya Dwivedi return; 626db559117SKumar Kartikeya Dwivedi bpf_timer_cancel_and_free(obj + rec->timer_off); 627db559117SKumar Kartikeya Dwivedi } 628db559117SKumar Kartikeya Dwivedi 6299e36a204SDave Marchevsky extern void __bpf_obj_drop_impl(void *p, const struct btf_record *rec); 6309e36a204SDave Marchevsky 631aa3496acSKumar Kartikeya Dwivedi void bpf_obj_free_fields(const struct btf_record *rec, void *obj) 632aa3496acSKumar Kartikeya Dwivedi { 633aa3496acSKumar Kartikeya Dwivedi const struct btf_field *fields; 634aa3496acSKumar Kartikeya Dwivedi int i; 635aa3496acSKumar Kartikeya Dwivedi 636aa3496acSKumar Kartikeya Dwivedi if (IS_ERR_OR_NULL(rec)) 637aa3496acSKumar Kartikeya Dwivedi return; 638aa3496acSKumar Kartikeya Dwivedi fields = rec->fields; 639aa3496acSKumar Kartikeya Dwivedi for (i = 0; i < rec->cnt; i++) { 640c8e18754SDave Marchevsky struct btf_struct_meta *pointee_struct_meta; 641aa3496acSKumar Kartikeya Dwivedi const struct btf_field *field = &fields[i]; 642aa3496acSKumar Kartikeya Dwivedi void *field_ptr = obj + field->offset; 643c8e18754SDave Marchevsky void *xchgd_field; 644aa3496acSKumar Kartikeya Dwivedi 645aa3496acSKumar Kartikeya Dwivedi switch (fields[i].type) { 646db559117SKumar Kartikeya Dwivedi case BPF_SPIN_LOCK: 647db559117SKumar Kartikeya Dwivedi break; 648db559117SKumar Kartikeya Dwivedi case BPF_TIMER: 649db559117SKumar Kartikeya Dwivedi bpf_timer_cancel_and_free(field_ptr); 650db559117SKumar Kartikeya Dwivedi break; 651aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_UNREF: 652aa3496acSKumar Kartikeya Dwivedi WRITE_ONCE(*(u64 *)field_ptr, 0); 653aa3496acSKumar Kartikeya Dwivedi break; 654aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_REF: 655*55db92f4SYonghong Song case BPF_KPTR_PERCPU: 656c8e18754SDave Marchevsky xchgd_field = (void *)xchg((unsigned long *)field_ptr, 0); 6571431d0b5SDavid Vernet if (!xchgd_field) 6581431d0b5SDavid Vernet break; 6591431d0b5SDavid Vernet 660c8e18754SDave Marchevsky if (!btf_is_kernel(field->kptr.btf)) { 661c8e18754SDave Marchevsky pointee_struct_meta = btf_find_struct_meta(field->kptr.btf, 662c8e18754SDave Marchevsky field->kptr.btf_id); 6639e36a204SDave Marchevsky migrate_disable(); 6649e36a204SDave Marchevsky __bpf_obj_drop_impl(xchgd_field, pointee_struct_meta ? 665c8e18754SDave Marchevsky pointee_struct_meta->record : 666c8e18754SDave Marchevsky NULL); 6679e36a204SDave Marchevsky migrate_enable(); 668c8e18754SDave Marchevsky } else { 669c8e18754SDave Marchevsky field->kptr.dtor(xchgd_field); 670c8e18754SDave Marchevsky } 671aa3496acSKumar Kartikeya Dwivedi break; 672f0c5941fSKumar Kartikeya Dwivedi case BPF_LIST_HEAD: 673f0c5941fSKumar Kartikeya Dwivedi if (WARN_ON_ONCE(rec->spin_lock_off < 0)) 674f0c5941fSKumar Kartikeya Dwivedi continue; 675f0c5941fSKumar Kartikeya Dwivedi bpf_list_head_free(field, field_ptr, obj + rec->spin_lock_off); 676f0c5941fSKumar Kartikeya Dwivedi break; 6779c395c1bSDave Marchevsky case BPF_RB_ROOT: 6789c395c1bSDave Marchevsky if (WARN_ON_ONCE(rec->spin_lock_off < 0)) 6799c395c1bSDave Marchevsky continue; 6809c395c1bSDave Marchevsky bpf_rb_root_free(field, field_ptr, obj + rec->spin_lock_off); 6819c395c1bSDave Marchevsky break; 6828ffa5cc1SKumar Kartikeya Dwivedi case BPF_LIST_NODE: 6839c395c1bSDave Marchevsky case BPF_RB_NODE: 684d54730b5SDave Marchevsky case BPF_REFCOUNT: 6858ffa5cc1SKumar Kartikeya Dwivedi break; 686aa3496acSKumar Kartikeya Dwivedi default: 687aa3496acSKumar Kartikeya Dwivedi WARN_ON_ONCE(1); 688aa3496acSKumar Kartikeya Dwivedi continue; 689aa3496acSKumar Kartikeya Dwivedi } 69014a324f6SKumar Kartikeya Dwivedi } 69114a324f6SKumar Kartikeya Dwivedi } 69214a324f6SKumar Kartikeya Dwivedi 69399c55f7dSAlexei Starovoitov /* called from workqueue */ 69499c55f7dSAlexei Starovoitov static void bpf_map_free_deferred(struct work_struct *work) 69599c55f7dSAlexei Starovoitov { 69699c55f7dSAlexei Starovoitov struct bpf_map *map = container_of(work, struct bpf_map, work); 697d7f5ef65SKumar Kartikeya Dwivedi struct btf_record *rec = map->record; 69899c55f7dSAlexei Starovoitov 699afdb09c7SChenbo Feng security_bpf_map_free(map); 70048edc1f7SRoman Gushchin bpf_map_release_memcg(map); 701d7f5ef65SKumar Kartikeya Dwivedi /* implementation dependent freeing */ 70299c55f7dSAlexei Starovoitov map->ops->map_free(map); 703cd2a8079SDave Marchevsky /* Delay freeing of btf_record for maps, as map_free 704d7f5ef65SKumar Kartikeya Dwivedi * callback usually needs access to them. It is better to do it here 705d7f5ef65SKumar Kartikeya Dwivedi * than require each callback to do the free itself manually. 706d7f5ef65SKumar Kartikeya Dwivedi * 707d7f5ef65SKumar Kartikeya Dwivedi * Note that the btf_record stashed in map->inner_map_meta->record was 708d7f5ef65SKumar Kartikeya Dwivedi * already freed using the map_free callback for map in map case which 709d7f5ef65SKumar Kartikeya Dwivedi * eventually calls bpf_map_free_meta, since inner_map_meta is only a 710d7f5ef65SKumar Kartikeya Dwivedi * template bpf_map struct used during verification. 711d7f5ef65SKumar Kartikeya Dwivedi */ 712d7f5ef65SKumar Kartikeya Dwivedi btf_record_free(rec); 71399c55f7dSAlexei Starovoitov } 71499c55f7dSAlexei Starovoitov 715c9da161cSDaniel Borkmann static void bpf_map_put_uref(struct bpf_map *map) 716c9da161cSDaniel Borkmann { 7171e0bd5a0SAndrii Nakryiko if (atomic64_dec_and_test(&map->usercnt)) { 718ba6b8de4SJohn Fastabend if (map->ops->map_release_uref) 719ba6b8de4SJohn Fastabend map->ops->map_release_uref(map); 720c9da161cSDaniel Borkmann } 721c9da161cSDaniel Borkmann } 722c9da161cSDaniel Borkmann 72399c55f7dSAlexei Starovoitov /* decrement map refcnt and schedule it for freeing via workqueue 724158e5e9eSTobias Klauser * (underlying map implementation ops->map_free() might sleep) 72599c55f7dSAlexei Starovoitov */ 726158e5e9eSTobias Klauser void bpf_map_put(struct bpf_map *map) 72799c55f7dSAlexei Starovoitov { 7281e0bd5a0SAndrii Nakryiko if (atomic64_dec_and_test(&map->refcnt)) { 72934ad5580SMartin KaFai Lau /* bpf_map_free_id() must be called first */ 730158e5e9eSTobias Klauser bpf_map_free_id(map); 73178958fcaSMartin KaFai Lau btf_put(map->btf); 73299c55f7dSAlexei Starovoitov INIT_WORK(&map->work, bpf_map_free_deferred); 7338d5a8011SAlexei Starovoitov /* Avoid spawning kworkers, since they all might contend 7348d5a8011SAlexei Starovoitov * for the same mutex like slab_mutex. 7358d5a8011SAlexei Starovoitov */ 7368d5a8011SAlexei Starovoitov queue_work(system_unbound_wq, &map->work); 73799c55f7dSAlexei Starovoitov } 73899c55f7dSAlexei Starovoitov } 739630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_put); 740bd5f5f4eSMartin KaFai Lau 741c9da161cSDaniel Borkmann void bpf_map_put_with_uref(struct bpf_map *map) 742c9da161cSDaniel Borkmann { 743c9da161cSDaniel Borkmann bpf_map_put_uref(map); 744c9da161cSDaniel Borkmann bpf_map_put(map); 745c9da161cSDaniel Borkmann } 746c9da161cSDaniel Borkmann 74799c55f7dSAlexei Starovoitov static int bpf_map_release(struct inode *inode, struct file *filp) 74899c55f7dSAlexei Starovoitov { 74961d1b6a4SDaniel Borkmann struct bpf_map *map = filp->private_data; 75061d1b6a4SDaniel Borkmann 75161d1b6a4SDaniel Borkmann if (map->ops->map_release) 75261d1b6a4SDaniel Borkmann map->ops->map_release(map, filp); 75361d1b6a4SDaniel Borkmann 75461d1b6a4SDaniel Borkmann bpf_map_put_with_uref(map); 75599c55f7dSAlexei Starovoitov return 0; 75699c55f7dSAlexei Starovoitov } 75799c55f7dSAlexei Starovoitov 75887df15deSDaniel Borkmann static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f) 75987df15deSDaniel Borkmann { 76087df15deSDaniel Borkmann fmode_t mode = f.file->f_mode; 76187df15deSDaniel Borkmann 76287df15deSDaniel Borkmann /* Our file permissions may have been overridden by global 76387df15deSDaniel Borkmann * map permissions facing syscall side. 76487df15deSDaniel Borkmann */ 76587df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) 76687df15deSDaniel Borkmann mode &= ~FMODE_CAN_WRITE; 76787df15deSDaniel Borkmann return mode; 76887df15deSDaniel Borkmann } 76987df15deSDaniel Borkmann 770f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 77190a5527dSYafang Shao /* Show the memory usage of a bpf map */ 77290a5527dSYafang Shao static u64 bpf_map_memory_usage(const struct bpf_map *map) 77380ee81e0SRoman Gushchin { 77490a5527dSYafang Shao return map->ops->map_mem_usage(map); 77580ee81e0SRoman Gushchin } 77680ee81e0SRoman Gushchin 777f99bf205SDaniel Borkmann static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 778f99bf205SDaniel Borkmann { 779f45d5b6cSToke Hoiland-Jorgensen struct bpf_map *map = filp->private_data; 7802beee5f5SDaniel Borkmann u32 type = 0, jited = 0; 78121116b70SDaniel Borkmann 782f45d5b6cSToke Hoiland-Jorgensen if (map_type_contains_progs(map)) { 783f45d5b6cSToke Hoiland-Jorgensen spin_lock(&map->owner.lock); 784f45d5b6cSToke Hoiland-Jorgensen type = map->owner.type; 785f45d5b6cSToke Hoiland-Jorgensen jited = map->owner.jited; 786f45d5b6cSToke Hoiland-Jorgensen spin_unlock(&map->owner.lock); 78721116b70SDaniel Borkmann } 788f99bf205SDaniel Borkmann 789f99bf205SDaniel Borkmann seq_printf(m, 790f99bf205SDaniel Borkmann "map_type:\t%u\n" 791f99bf205SDaniel Borkmann "key_size:\t%u\n" 792f99bf205SDaniel Borkmann "value_size:\t%u\n" 793322cea2fSDaniel Borkmann "max_entries:\t%u\n" 79421116b70SDaniel Borkmann "map_flags:\t%#x\n" 7959330986cSJoanne Koong "map_extra:\t%#llx\n" 79690a5527dSYafang Shao "memlock:\t%llu\n" 79787df15deSDaniel Borkmann "map_id:\t%u\n" 79887df15deSDaniel Borkmann "frozen:\t%u\n", 799f99bf205SDaniel Borkmann map->map_type, 800f99bf205SDaniel Borkmann map->key_size, 801f99bf205SDaniel Borkmann map->value_size, 802322cea2fSDaniel Borkmann map->max_entries, 80321116b70SDaniel Borkmann map->map_flags, 8049330986cSJoanne Koong (unsigned long long)map->map_extra, 80590a5527dSYafang Shao bpf_map_memory_usage(map), 80687df15deSDaniel Borkmann map->id, 80787df15deSDaniel Borkmann READ_ONCE(map->frozen)); 8082beee5f5SDaniel Borkmann if (type) { 8092beee5f5SDaniel Borkmann seq_printf(m, "owner_prog_type:\t%u\n", type); 8102beee5f5SDaniel Borkmann seq_printf(m, "owner_jited:\t%u\n", jited); 8119780c0abSDaniel Borkmann } 812f99bf205SDaniel Borkmann } 813f99bf205SDaniel Borkmann #endif 814f99bf205SDaniel Borkmann 8156e71b04aSChenbo Feng static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, 8166e71b04aSChenbo Feng loff_t *ppos) 8176e71b04aSChenbo Feng { 8186e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 8196e71b04aSChenbo Feng * f_mode with FMODE_CAN_READ. 8206e71b04aSChenbo Feng */ 8216e71b04aSChenbo Feng return -EINVAL; 8226e71b04aSChenbo Feng } 8236e71b04aSChenbo Feng 8246e71b04aSChenbo Feng static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, 8256e71b04aSChenbo Feng size_t siz, loff_t *ppos) 8266e71b04aSChenbo Feng { 8276e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 8286e71b04aSChenbo Feng * f_mode with FMODE_CAN_WRITE. 8296e71b04aSChenbo Feng */ 8306e71b04aSChenbo Feng return -EINVAL; 8316e71b04aSChenbo Feng } 8326e71b04aSChenbo Feng 833fc970227SAndrii Nakryiko /* called for any extra memory-mapped regions (except initial) */ 834fc970227SAndrii Nakryiko static void bpf_map_mmap_open(struct vm_area_struct *vma) 835fc970227SAndrii Nakryiko { 836fc970227SAndrii Nakryiko struct bpf_map *map = vma->vm_file->private_data; 837fc970227SAndrii Nakryiko 838353050beSDaniel Borkmann if (vma->vm_flags & VM_MAYWRITE) 839353050beSDaniel Borkmann bpf_map_write_active_inc(map); 840fc970227SAndrii Nakryiko } 841fc970227SAndrii Nakryiko 842fc970227SAndrii Nakryiko /* called for all unmapped memory region (including initial) */ 843fc970227SAndrii Nakryiko static void bpf_map_mmap_close(struct vm_area_struct *vma) 844fc970227SAndrii Nakryiko { 845fc970227SAndrii Nakryiko struct bpf_map *map = vma->vm_file->private_data; 846fc970227SAndrii Nakryiko 847353050beSDaniel Borkmann if (vma->vm_flags & VM_MAYWRITE) 848353050beSDaniel Borkmann bpf_map_write_active_dec(map); 849fc970227SAndrii Nakryiko } 850fc970227SAndrii Nakryiko 851fc970227SAndrii Nakryiko static const struct vm_operations_struct bpf_map_default_vmops = { 852fc970227SAndrii Nakryiko .open = bpf_map_mmap_open, 853fc970227SAndrii Nakryiko .close = bpf_map_mmap_close, 854fc970227SAndrii Nakryiko }; 855fc970227SAndrii Nakryiko 856fc970227SAndrii Nakryiko static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma) 857fc970227SAndrii Nakryiko { 858fc970227SAndrii Nakryiko struct bpf_map *map = filp->private_data; 859fc970227SAndrii Nakryiko int err; 860fc970227SAndrii Nakryiko 861db559117SKumar Kartikeya Dwivedi if (!map->ops->map_mmap || !IS_ERR_OR_NULL(map->record)) 862fc970227SAndrii Nakryiko return -ENOTSUPP; 863fc970227SAndrii Nakryiko 864fc970227SAndrii Nakryiko if (!(vma->vm_flags & VM_SHARED)) 865fc970227SAndrii Nakryiko return -EINVAL; 866fc970227SAndrii Nakryiko 867fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 868fc970227SAndrii Nakryiko 869dfeb376dSAndrii Nakryiko if (vma->vm_flags & VM_WRITE) { 870dfeb376dSAndrii Nakryiko if (map->frozen) { 871fc970227SAndrii Nakryiko err = -EPERM; 872fc970227SAndrii Nakryiko goto out; 873fc970227SAndrii Nakryiko } 874dfeb376dSAndrii Nakryiko /* map is meant to be read-only, so do not allow mapping as 875dfeb376dSAndrii Nakryiko * writable, because it's possible to leak a writable page 876dfeb376dSAndrii Nakryiko * reference and allows user-space to still modify it after 877dfeb376dSAndrii Nakryiko * freezing, while verifier will assume contents do not change 878dfeb376dSAndrii Nakryiko */ 879dfeb376dSAndrii Nakryiko if (map->map_flags & BPF_F_RDONLY_PROG) { 880dfeb376dSAndrii Nakryiko err = -EACCES; 881dfeb376dSAndrii Nakryiko goto out; 882dfeb376dSAndrii Nakryiko } 883dfeb376dSAndrii Nakryiko } 884fc970227SAndrii Nakryiko 885fc970227SAndrii Nakryiko /* set default open/close callbacks */ 886fc970227SAndrii Nakryiko vma->vm_ops = &bpf_map_default_vmops; 887fc970227SAndrii Nakryiko vma->vm_private_data = map; 8881c71222eSSuren Baghdasaryan vm_flags_clear(vma, VM_MAYEXEC); 8891f6cb19bSAndrii Nakryiko if (!(vma->vm_flags & VM_WRITE)) 8901f6cb19bSAndrii Nakryiko /* disallow re-mapping with PROT_WRITE */ 8911c71222eSSuren Baghdasaryan vm_flags_clear(vma, VM_MAYWRITE); 892fc970227SAndrii Nakryiko 893fc970227SAndrii Nakryiko err = map->ops->map_mmap(map, vma); 894fc970227SAndrii Nakryiko if (err) 895fc970227SAndrii Nakryiko goto out; 896fc970227SAndrii Nakryiko 8971f6cb19bSAndrii Nakryiko if (vma->vm_flags & VM_MAYWRITE) 898353050beSDaniel Borkmann bpf_map_write_active_inc(map); 899fc970227SAndrii Nakryiko out: 900fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 901fc970227SAndrii Nakryiko return err; 902fc970227SAndrii Nakryiko } 903fc970227SAndrii Nakryiko 904457f4436SAndrii Nakryiko static __poll_t bpf_map_poll(struct file *filp, struct poll_table_struct *pts) 905457f4436SAndrii Nakryiko { 906457f4436SAndrii Nakryiko struct bpf_map *map = filp->private_data; 907457f4436SAndrii Nakryiko 908457f4436SAndrii Nakryiko if (map->ops->map_poll) 909457f4436SAndrii Nakryiko return map->ops->map_poll(map, filp, pts); 910457f4436SAndrii Nakryiko 911457f4436SAndrii Nakryiko return EPOLLERR; 912457f4436SAndrii Nakryiko } 913457f4436SAndrii Nakryiko 914f66e448cSChenbo Feng const struct file_operations bpf_map_fops = { 915f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 916f99bf205SDaniel Borkmann .show_fdinfo = bpf_map_show_fdinfo, 917f99bf205SDaniel Borkmann #endif 91899c55f7dSAlexei Starovoitov .release = bpf_map_release, 9196e71b04aSChenbo Feng .read = bpf_dummy_read, 9206e71b04aSChenbo Feng .write = bpf_dummy_write, 921fc970227SAndrii Nakryiko .mmap = bpf_map_mmap, 922457f4436SAndrii Nakryiko .poll = bpf_map_poll, 92399c55f7dSAlexei Starovoitov }; 92499c55f7dSAlexei Starovoitov 9256e71b04aSChenbo Feng int bpf_map_new_fd(struct bpf_map *map, int flags) 926aa79781bSDaniel Borkmann { 927afdb09c7SChenbo Feng int ret; 928afdb09c7SChenbo Feng 929afdb09c7SChenbo Feng ret = security_bpf_map(map, OPEN_FMODE(flags)); 930afdb09c7SChenbo Feng if (ret < 0) 931afdb09c7SChenbo Feng return ret; 932afdb09c7SChenbo Feng 933aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 9346e71b04aSChenbo Feng flags | O_CLOEXEC); 9356e71b04aSChenbo Feng } 9366e71b04aSChenbo Feng 9376e71b04aSChenbo Feng int bpf_get_file_flag(int flags) 9386e71b04aSChenbo Feng { 9396e71b04aSChenbo Feng if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) 9406e71b04aSChenbo Feng return -EINVAL; 9416e71b04aSChenbo Feng if (flags & BPF_F_RDONLY) 9426e71b04aSChenbo Feng return O_RDONLY; 9436e71b04aSChenbo Feng if (flags & BPF_F_WRONLY) 9446e71b04aSChenbo Feng return O_WRONLY; 9456e71b04aSChenbo Feng return O_RDWR; 946aa79781bSDaniel Borkmann } 947aa79781bSDaniel Borkmann 94899c55f7dSAlexei Starovoitov /* helper macro to check that unused fields 'union bpf_attr' are zero */ 94999c55f7dSAlexei Starovoitov #define CHECK_ATTR(CMD) \ 95099c55f7dSAlexei Starovoitov memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 95199c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD), 0, \ 95299c55f7dSAlexei Starovoitov sizeof(*attr) - \ 95399c55f7dSAlexei Starovoitov offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 95499c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD)) != NULL 95599c55f7dSAlexei Starovoitov 9568e7ae251SMartin KaFai Lau /* dst and src must have at least "size" number of bytes. 9578e7ae251SMartin KaFai Lau * Return strlen on success and < 0 on error. 958cb4d2b3fSMartin KaFai Lau */ 9598e7ae251SMartin KaFai Lau int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size) 960cb4d2b3fSMartin KaFai Lau { 9618e7ae251SMartin KaFai Lau const char *end = src + size; 9628e7ae251SMartin KaFai Lau const char *orig_src = src; 963cb4d2b3fSMartin KaFai Lau 9648e7ae251SMartin KaFai Lau memset(dst, 0, size); 9653e0ddc4fSDaniel Borkmann /* Copy all isalnum(), '_' and '.' chars. */ 966cb4d2b3fSMartin KaFai Lau while (src < end && *src) { 9673e0ddc4fSDaniel Borkmann if (!isalnum(*src) && 9683e0ddc4fSDaniel Borkmann *src != '_' && *src != '.') 969cb4d2b3fSMartin KaFai Lau return -EINVAL; 970cb4d2b3fSMartin KaFai Lau *dst++ = *src++; 971cb4d2b3fSMartin KaFai Lau } 972cb4d2b3fSMartin KaFai Lau 9738e7ae251SMartin KaFai Lau /* No '\0' found in "size" number of bytes */ 974cb4d2b3fSMartin KaFai Lau if (src == end) 975cb4d2b3fSMartin KaFai Lau return -EINVAL; 976cb4d2b3fSMartin KaFai Lau 9778e7ae251SMartin KaFai Lau return src - orig_src; 978cb4d2b3fSMartin KaFai Lau } 979cb4d2b3fSMartin KaFai Lau 980e8d2bec0SDaniel Borkmann int map_check_no_btf(const struct bpf_map *map, 9811b2b234bSRoman Gushchin const struct btf *btf, 982e8d2bec0SDaniel Borkmann const struct btf_type *key_type, 983e8d2bec0SDaniel Borkmann const struct btf_type *value_type) 984e8d2bec0SDaniel Borkmann { 985e8d2bec0SDaniel Borkmann return -ENOTSUPP; 986e8d2bec0SDaniel Borkmann } 987e8d2bec0SDaniel Borkmann 988d83525caSAlexei Starovoitov static int map_check_btf(struct bpf_map *map, const struct btf *btf, 989e8d2bec0SDaniel Borkmann u32 btf_key_id, u32 btf_value_id) 990e8d2bec0SDaniel Borkmann { 991e8d2bec0SDaniel Borkmann const struct btf_type *key_type, *value_type; 992e8d2bec0SDaniel Borkmann u32 key_size, value_size; 993e8d2bec0SDaniel Borkmann int ret = 0; 994e8d2bec0SDaniel Borkmann 9952824ecb7SDaniel Borkmann /* Some maps allow key to be unspecified. */ 9962824ecb7SDaniel Borkmann if (btf_key_id) { 997e8d2bec0SDaniel Borkmann key_type = btf_type_id_size(btf, &btf_key_id, &key_size); 998e8d2bec0SDaniel Borkmann if (!key_type || key_size != map->key_size) 999e8d2bec0SDaniel Borkmann return -EINVAL; 10002824ecb7SDaniel Borkmann } else { 10012824ecb7SDaniel Borkmann key_type = btf_type_by_id(btf, 0); 10022824ecb7SDaniel Borkmann if (!map->ops->map_check_btf) 10032824ecb7SDaniel Borkmann return -EINVAL; 10042824ecb7SDaniel Borkmann } 1005e8d2bec0SDaniel Borkmann 1006e8d2bec0SDaniel Borkmann value_type = btf_type_id_size(btf, &btf_value_id, &value_size); 1007e8d2bec0SDaniel Borkmann if (!value_type || value_size != map->value_size) 1008e8d2bec0SDaniel Borkmann return -EINVAL; 1009e8d2bec0SDaniel Borkmann 1010f0c5941fSKumar Kartikeya Dwivedi map->record = btf_parse_fields(btf, value_type, 10119c395c1bSDave Marchevsky BPF_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD | 1012d54730b5SDave Marchevsky BPF_RB_ROOT | BPF_REFCOUNT, 1013db559117SKumar Kartikeya Dwivedi map->value_size); 1014aa3496acSKumar Kartikeya Dwivedi if (!IS_ERR_OR_NULL(map->record)) { 1015aa3496acSKumar Kartikeya Dwivedi int i; 1016aa3496acSKumar Kartikeya Dwivedi 101761df10c7SKumar Kartikeya Dwivedi if (!bpf_capable()) { 101861df10c7SKumar Kartikeya Dwivedi ret = -EPERM; 101961df10c7SKumar Kartikeya Dwivedi goto free_map_tab; 102061df10c7SKumar Kartikeya Dwivedi } 102161df10c7SKumar Kartikeya Dwivedi if (map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) { 102261df10c7SKumar Kartikeya Dwivedi ret = -EACCES; 102361df10c7SKumar Kartikeya Dwivedi goto free_map_tab; 102461df10c7SKumar Kartikeya Dwivedi } 1025aa3496acSKumar Kartikeya Dwivedi for (i = 0; i < sizeof(map->record->field_mask) * 8; i++) { 1026aa3496acSKumar Kartikeya Dwivedi switch (map->record->field_mask & (1 << i)) { 1027aa3496acSKumar Kartikeya Dwivedi case 0: 1028aa3496acSKumar Kartikeya Dwivedi continue; 1029db559117SKumar Kartikeya Dwivedi case BPF_SPIN_LOCK: 1030db559117SKumar Kartikeya Dwivedi if (map->map_type != BPF_MAP_TYPE_HASH && 1031db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_ARRAY && 1032db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 1033db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_SK_STORAGE && 1034db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_INODE_STORAGE && 1035db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_TASK_STORAGE && 1036db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) { 1037db559117SKumar Kartikeya Dwivedi ret = -EOPNOTSUPP; 1038db559117SKumar Kartikeya Dwivedi goto free_map_tab; 1039db559117SKumar Kartikeya Dwivedi } 1040db559117SKumar Kartikeya Dwivedi break; 1041db559117SKumar Kartikeya Dwivedi case BPF_TIMER: 1042db559117SKumar Kartikeya Dwivedi if (map->map_type != BPF_MAP_TYPE_HASH && 1043db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_LRU_HASH && 1044db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_ARRAY) { 1045c237bfa5SKumar Kartikeya Dwivedi ret = -EOPNOTSUPP; 1046db559117SKumar Kartikeya Dwivedi goto free_map_tab; 1047db559117SKumar Kartikeya Dwivedi } 1048db559117SKumar Kartikeya Dwivedi break; 1049aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_UNREF: 1050aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_REF: 1051*55db92f4SYonghong Song case BPF_KPTR_PERCPU: 1052d54730b5SDave Marchevsky case BPF_REFCOUNT: 105361df10c7SKumar Kartikeya Dwivedi if (map->map_type != BPF_MAP_TYPE_HASH && 105465334e64SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_PERCPU_HASH && 105561df10c7SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_LRU_HASH && 105665334e64SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH && 10576df4ea1fSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_ARRAY && 10589db44fddSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY && 10599db44fddSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_SK_STORAGE && 10609db44fddSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_INODE_STORAGE && 10619db44fddSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_TASK_STORAGE && 10629db44fddSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) { 106361df10c7SKumar Kartikeya Dwivedi ret = -EOPNOTSUPP; 106461df10c7SKumar Kartikeya Dwivedi goto free_map_tab; 106561df10c7SKumar Kartikeya Dwivedi } 1066aa3496acSKumar Kartikeya Dwivedi break; 1067f0c5941fSKumar Kartikeya Dwivedi case BPF_LIST_HEAD: 10689c395c1bSDave Marchevsky case BPF_RB_ROOT: 1069f0c5941fSKumar Kartikeya Dwivedi if (map->map_type != BPF_MAP_TYPE_HASH && 1070f0c5941fSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_LRU_HASH && 1071f0c5941fSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_ARRAY) { 1072f0c5941fSKumar Kartikeya Dwivedi ret = -EOPNOTSUPP; 1073f0c5941fSKumar Kartikeya Dwivedi goto free_map_tab; 1074f0c5941fSKumar Kartikeya Dwivedi } 1075f0c5941fSKumar Kartikeya Dwivedi break; 1076aa3496acSKumar Kartikeya Dwivedi default: 1077aa3496acSKumar Kartikeya Dwivedi /* Fail if map_type checks are missing for a field type */ 1078aa3496acSKumar Kartikeya Dwivedi ret = -EOPNOTSUPP; 1079aa3496acSKumar Kartikeya Dwivedi goto free_map_tab; 1080aa3496acSKumar Kartikeya Dwivedi } 1081aa3496acSKumar Kartikeya Dwivedi } 108261df10c7SKumar Kartikeya Dwivedi } 1083e8d2bec0SDaniel Borkmann 1084865ce09aSKumar Kartikeya Dwivedi ret = btf_check_and_fixup_fields(btf, map->record); 1085865ce09aSKumar Kartikeya Dwivedi if (ret < 0) 1086865ce09aSKumar Kartikeya Dwivedi goto free_map_tab; 1087865ce09aSKumar Kartikeya Dwivedi 108861df10c7SKumar Kartikeya Dwivedi if (map->ops->map_check_btf) { 108961df10c7SKumar Kartikeya Dwivedi ret = map->ops->map_check_btf(map, btf, key_type, value_type); 109061df10c7SKumar Kartikeya Dwivedi if (ret < 0) 109161df10c7SKumar Kartikeya Dwivedi goto free_map_tab; 109261df10c7SKumar Kartikeya Dwivedi } 109361df10c7SKumar Kartikeya Dwivedi 109461df10c7SKumar Kartikeya Dwivedi return ret; 109561df10c7SKumar Kartikeya Dwivedi free_map_tab: 1096aa3496acSKumar Kartikeya Dwivedi bpf_map_free_record(map); 1097e8d2bec0SDaniel Borkmann return ret; 1098e8d2bec0SDaniel Borkmann } 1099e8d2bec0SDaniel Borkmann 11009330986cSJoanne Koong #define BPF_MAP_CREATE_LAST_FIELD map_extra 110199c55f7dSAlexei Starovoitov /* called via syscall */ 110299c55f7dSAlexei Starovoitov static int map_create(union bpf_attr *attr) 110399c55f7dSAlexei Starovoitov { 110422db4122SAndrii Nakryiko const struct bpf_map_ops *ops; 110596eabe7aSMartin KaFai Lau int numa_node = bpf_map_attr_numa_node(attr); 110622db4122SAndrii Nakryiko u32 map_type = attr->map_type; 110799c55f7dSAlexei Starovoitov struct bpf_map *map; 11086e71b04aSChenbo Feng int f_flags; 110999c55f7dSAlexei Starovoitov int err; 111099c55f7dSAlexei Starovoitov 111199c55f7dSAlexei Starovoitov err = CHECK_ATTR(BPF_MAP_CREATE); 111299c55f7dSAlexei Starovoitov if (err) 111399c55f7dSAlexei Starovoitov return -EINVAL; 111499c55f7dSAlexei Starovoitov 111585d33df3SMartin KaFai Lau if (attr->btf_vmlinux_value_type_id) { 111685d33df3SMartin KaFai Lau if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS || 111785d33df3SMartin KaFai Lau attr->btf_key_type_id || attr->btf_value_type_id) 111885d33df3SMartin KaFai Lau return -EINVAL; 111985d33df3SMartin KaFai Lau } else if (attr->btf_key_type_id && !attr->btf_value_type_id) { 112085d33df3SMartin KaFai Lau return -EINVAL; 112185d33df3SMartin KaFai Lau } 112285d33df3SMartin KaFai Lau 11239330986cSJoanne Koong if (attr->map_type != BPF_MAP_TYPE_BLOOM_FILTER && 11249330986cSJoanne Koong attr->map_extra != 0) 11259330986cSJoanne Koong return -EINVAL; 11269330986cSJoanne Koong 11276e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->map_flags); 11286e71b04aSChenbo Feng if (f_flags < 0) 11296e71b04aSChenbo Feng return f_flags; 11306e71b04aSChenbo Feng 113196eabe7aSMartin KaFai Lau if (numa_node != NUMA_NO_NODE && 113296e5ae4eSEric Dumazet ((unsigned int)numa_node >= nr_node_ids || 113396e5ae4eSEric Dumazet !node_online(numa_node))) 113496eabe7aSMartin KaFai Lau return -EINVAL; 113596eabe7aSMartin KaFai Lau 113699c55f7dSAlexei Starovoitov /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 113722db4122SAndrii Nakryiko map_type = attr->map_type; 113822db4122SAndrii Nakryiko if (map_type >= ARRAY_SIZE(bpf_map_types)) 113922db4122SAndrii Nakryiko return -EINVAL; 114022db4122SAndrii Nakryiko map_type = array_index_nospec(map_type, ARRAY_SIZE(bpf_map_types)); 114122db4122SAndrii Nakryiko ops = bpf_map_types[map_type]; 114222db4122SAndrii Nakryiko if (!ops) 114322db4122SAndrii Nakryiko return -EINVAL; 114422db4122SAndrii Nakryiko 114522db4122SAndrii Nakryiko if (ops->map_alloc_check) { 114622db4122SAndrii Nakryiko err = ops->map_alloc_check(attr); 114722db4122SAndrii Nakryiko if (err) 114822db4122SAndrii Nakryiko return err; 114922db4122SAndrii Nakryiko } 115022db4122SAndrii Nakryiko if (attr->map_ifindex) 115122db4122SAndrii Nakryiko ops = &bpf_map_offload_ops; 115222db4122SAndrii Nakryiko if (!ops->map_mem_usage) 115322db4122SAndrii Nakryiko return -EINVAL; 115422db4122SAndrii Nakryiko 11551d28635aSAndrii Nakryiko /* Intent here is for unprivileged_bpf_disabled to block BPF map 11561d28635aSAndrii Nakryiko * creation for unprivileged users; other actions depend 11571d28635aSAndrii Nakryiko * on fd availability and access to bpffs, so are dependent on 11581d28635aSAndrii Nakryiko * object creation success. Even with unprivileged BPF disabled, 11591d28635aSAndrii Nakryiko * capability checks are still carried out. 11601d28635aSAndrii Nakryiko */ 11611d28635aSAndrii Nakryiko if (sysctl_unprivileged_bpf_disabled && !bpf_capable()) 11621d28635aSAndrii Nakryiko return -EPERM; 11631d28635aSAndrii Nakryiko 11646c3eba1cSAndrii Nakryiko /* check privileged map type permissions */ 11656c3eba1cSAndrii Nakryiko switch (map_type) { 11666c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_ARRAY: 11676c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_PERCPU_ARRAY: 11686c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_PROG_ARRAY: 11696c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 11706c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_CGROUP_ARRAY: 11716c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_ARRAY_OF_MAPS: 11726c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_HASH: 11736c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_PERCPU_HASH: 11746c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_HASH_OF_MAPS: 11756c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_RINGBUF: 11766c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_USER_RINGBUF: 11776c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_CGROUP_STORAGE: 11786c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 11796c3eba1cSAndrii Nakryiko /* unprivileged */ 11806c3eba1cSAndrii Nakryiko break; 11816c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_SK_STORAGE: 11826c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_INODE_STORAGE: 11836c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_TASK_STORAGE: 11846c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_CGRP_STORAGE: 11856c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_BLOOM_FILTER: 11866c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_LPM_TRIE: 11876c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 11886c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_STACK_TRACE: 11896c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_QUEUE: 11906c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_STACK: 11916c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_LRU_HASH: 11926c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_LRU_PERCPU_HASH: 11936c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_STRUCT_OPS: 11946c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_CPUMAP: 11956c3eba1cSAndrii Nakryiko if (!bpf_capable()) 11966c3eba1cSAndrii Nakryiko return -EPERM; 11976c3eba1cSAndrii Nakryiko break; 11986c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_SOCKMAP: 11996c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_SOCKHASH: 12006c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_DEVMAP: 12016c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_DEVMAP_HASH: 12026c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_XSKMAP: 12036c3eba1cSAndrii Nakryiko if (!capable(CAP_NET_ADMIN)) 12046c3eba1cSAndrii Nakryiko return -EPERM; 12056c3eba1cSAndrii Nakryiko break; 12066c3eba1cSAndrii Nakryiko default: 12076c3eba1cSAndrii Nakryiko WARN(1, "unsupported map type %d", map_type); 12086c3eba1cSAndrii Nakryiko return -EPERM; 12096c3eba1cSAndrii Nakryiko } 12106c3eba1cSAndrii Nakryiko 121122db4122SAndrii Nakryiko map = ops->map_alloc(attr); 121299c55f7dSAlexei Starovoitov if (IS_ERR(map)) 121399c55f7dSAlexei Starovoitov return PTR_ERR(map); 121422db4122SAndrii Nakryiko map->ops = ops; 121522db4122SAndrii Nakryiko map->map_type = map_type; 121699c55f7dSAlexei Starovoitov 12178e7ae251SMartin KaFai Lau err = bpf_obj_name_cpy(map->name, attr->map_name, 12188e7ae251SMartin KaFai Lau sizeof(attr->map_name)); 12198e7ae251SMartin KaFai Lau if (err < 0) 1220b936ca64SRoman Gushchin goto free_map; 1221ad5b177bSMartin KaFai Lau 12221e0bd5a0SAndrii Nakryiko atomic64_set(&map->refcnt, 1); 12231e0bd5a0SAndrii Nakryiko atomic64_set(&map->usercnt, 1); 1224fc970227SAndrii Nakryiko mutex_init(&map->freeze_mutex); 1225f45d5b6cSToke Hoiland-Jorgensen spin_lock_init(&map->owner.lock); 122699c55f7dSAlexei Starovoitov 122785d33df3SMartin KaFai Lau if (attr->btf_key_type_id || attr->btf_value_type_id || 122885d33df3SMartin KaFai Lau /* Even the map's value is a kernel's struct, 122985d33df3SMartin KaFai Lau * the bpf_prog.o must have BTF to begin with 123085d33df3SMartin KaFai Lau * to figure out the corresponding kernel's 123185d33df3SMartin KaFai Lau * counter part. Thus, attr->btf_fd has 123285d33df3SMartin KaFai Lau * to be valid also. 123385d33df3SMartin KaFai Lau */ 123485d33df3SMartin KaFai Lau attr->btf_vmlinux_value_type_id) { 1235a26ca7c9SMartin KaFai Lau struct btf *btf; 1236a26ca7c9SMartin KaFai Lau 1237a26ca7c9SMartin KaFai Lau btf = btf_get_by_fd(attr->btf_fd); 1238a26ca7c9SMartin KaFai Lau if (IS_ERR(btf)) { 1239a26ca7c9SMartin KaFai Lau err = PTR_ERR(btf); 1240b936ca64SRoman Gushchin goto free_map; 1241a26ca7c9SMartin KaFai Lau } 1242350a5c4dSAlexei Starovoitov if (btf_is_kernel(btf)) { 1243350a5c4dSAlexei Starovoitov btf_put(btf); 1244350a5c4dSAlexei Starovoitov err = -EACCES; 1245350a5c4dSAlexei Starovoitov goto free_map; 1246350a5c4dSAlexei Starovoitov } 124785d33df3SMartin KaFai Lau map->btf = btf; 1248a26ca7c9SMartin KaFai Lau 124985d33df3SMartin KaFai Lau if (attr->btf_value_type_id) { 1250e8d2bec0SDaniel Borkmann err = map_check_btf(map, btf, attr->btf_key_type_id, 12519b2cf328SMartin KaFai Lau attr->btf_value_type_id); 125285d33df3SMartin KaFai Lau if (err) 1253b936ca64SRoman Gushchin goto free_map; 1254a26ca7c9SMartin KaFai Lau } 1255a26ca7c9SMartin KaFai Lau 12569b2cf328SMartin KaFai Lau map->btf_key_type_id = attr->btf_key_type_id; 12579b2cf328SMartin KaFai Lau map->btf_value_type_id = attr->btf_value_type_id; 125885d33df3SMartin KaFai Lau map->btf_vmlinux_value_type_id = 125985d33df3SMartin KaFai Lau attr->btf_vmlinux_value_type_id; 1260a26ca7c9SMartin KaFai Lau } 1261a26ca7c9SMartin KaFai Lau 12624d7d7f69SKumar Kartikeya Dwivedi err = security_bpf_map_alloc(map); 12634d7d7f69SKumar Kartikeya Dwivedi if (err) 1264cd2a8079SDave Marchevsky goto free_map; 12654d7d7f69SKumar Kartikeya Dwivedi 1266f3f1c054SMartin KaFai Lau err = bpf_map_alloc_id(map); 1267f3f1c054SMartin KaFai Lau if (err) 1268b936ca64SRoman Gushchin goto free_map_sec; 1269f3f1c054SMartin KaFai Lau 127048edc1f7SRoman Gushchin bpf_map_save_memcg(map); 127148edc1f7SRoman Gushchin 12726e71b04aSChenbo Feng err = bpf_map_new_fd(map, f_flags); 1273bd5f5f4eSMartin KaFai Lau if (err < 0) { 1274bd5f5f4eSMartin KaFai Lau /* failed to allocate fd. 1275352d20d6SPeng Sun * bpf_map_put_with_uref() is needed because the above 1276bd5f5f4eSMartin KaFai Lau * bpf_map_alloc_id() has published the map 1277bd5f5f4eSMartin KaFai Lau * to the userspace and the userspace may 1278bd5f5f4eSMartin KaFai Lau * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 1279bd5f5f4eSMartin KaFai Lau */ 1280352d20d6SPeng Sun bpf_map_put_with_uref(map); 1281bd5f5f4eSMartin KaFai Lau return err; 1282bd5f5f4eSMartin KaFai Lau } 128399c55f7dSAlexei Starovoitov 128499c55f7dSAlexei Starovoitov return err; 128599c55f7dSAlexei Starovoitov 1286afdb09c7SChenbo Feng free_map_sec: 1287afdb09c7SChenbo Feng security_bpf_map_free(map); 1288b936ca64SRoman Gushchin free_map: 1289a26ca7c9SMartin KaFai Lau btf_put(map->btf); 129099c55f7dSAlexei Starovoitov map->ops->map_free(map); 129199c55f7dSAlexei Starovoitov return err; 129299c55f7dSAlexei Starovoitov } 129399c55f7dSAlexei Starovoitov 1294db20fd2bSAlexei Starovoitov /* if error is returned, fd is released. 1295db20fd2bSAlexei Starovoitov * On success caller should complete fd access with matching fdput() 1296db20fd2bSAlexei Starovoitov */ 1297c2101297SDaniel Borkmann struct bpf_map *__bpf_map_get(struct fd f) 1298db20fd2bSAlexei Starovoitov { 1299db20fd2bSAlexei Starovoitov if (!f.file) 1300db20fd2bSAlexei Starovoitov return ERR_PTR(-EBADF); 1301db20fd2bSAlexei Starovoitov if (f.file->f_op != &bpf_map_fops) { 1302db20fd2bSAlexei Starovoitov fdput(f); 1303db20fd2bSAlexei Starovoitov return ERR_PTR(-EINVAL); 1304db20fd2bSAlexei Starovoitov } 1305db20fd2bSAlexei Starovoitov 1306c2101297SDaniel Borkmann return f.file->private_data; 1307c2101297SDaniel Borkmann } 1308c2101297SDaniel Borkmann 13091e0bd5a0SAndrii Nakryiko void bpf_map_inc(struct bpf_map *map) 1310c9da161cSDaniel Borkmann { 13111e0bd5a0SAndrii Nakryiko atomic64_inc(&map->refcnt); 1312c9da161cSDaniel Borkmann } 1313630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_inc); 1314c9da161cSDaniel Borkmann 13151e0bd5a0SAndrii Nakryiko void bpf_map_inc_with_uref(struct bpf_map *map) 13161e0bd5a0SAndrii Nakryiko { 13171e0bd5a0SAndrii Nakryiko atomic64_inc(&map->refcnt); 13181e0bd5a0SAndrii Nakryiko atomic64_inc(&map->usercnt); 13191e0bd5a0SAndrii Nakryiko } 13201e0bd5a0SAndrii Nakryiko EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref); 13211e0bd5a0SAndrii Nakryiko 13221ed4d924SMartin KaFai Lau struct bpf_map *bpf_map_get(u32 ufd) 13231ed4d924SMartin KaFai Lau { 13241ed4d924SMartin KaFai Lau struct fd f = fdget(ufd); 13251ed4d924SMartin KaFai Lau struct bpf_map *map; 13261ed4d924SMartin KaFai Lau 13271ed4d924SMartin KaFai Lau map = __bpf_map_get(f); 13281ed4d924SMartin KaFai Lau if (IS_ERR(map)) 13291ed4d924SMartin KaFai Lau return map; 13301ed4d924SMartin KaFai Lau 13311ed4d924SMartin KaFai Lau bpf_map_inc(map); 13321ed4d924SMartin KaFai Lau fdput(f); 13331ed4d924SMartin KaFai Lau 13341ed4d924SMartin KaFai Lau return map; 13351ed4d924SMartin KaFai Lau } 1336b1d18a75SAlexei Starovoitov EXPORT_SYMBOL(bpf_map_get); 13371ed4d924SMartin KaFai Lau 1338c9da161cSDaniel Borkmann struct bpf_map *bpf_map_get_with_uref(u32 ufd) 1339c2101297SDaniel Borkmann { 1340c2101297SDaniel Borkmann struct fd f = fdget(ufd); 1341c2101297SDaniel Borkmann struct bpf_map *map; 1342c2101297SDaniel Borkmann 1343c2101297SDaniel Borkmann map = __bpf_map_get(f); 1344c2101297SDaniel Borkmann if (IS_ERR(map)) 1345c2101297SDaniel Borkmann return map; 1346c2101297SDaniel Borkmann 13471e0bd5a0SAndrii Nakryiko bpf_map_inc_with_uref(map); 1348c2101297SDaniel Borkmann fdput(f); 1349db20fd2bSAlexei Starovoitov 1350db20fd2bSAlexei Starovoitov return map; 1351db20fd2bSAlexei Starovoitov } 1352db20fd2bSAlexei Starovoitov 1353b671c206SKui-Feng Lee /* map_idr_lock should have been held or the map should have been 1354b671c206SKui-Feng Lee * protected by rcu read lock. 1355b671c206SKui-Feng Lee */ 1356b671c206SKui-Feng Lee struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref) 1357bd5f5f4eSMartin KaFai Lau { 1358bd5f5f4eSMartin KaFai Lau int refold; 1359bd5f5f4eSMartin KaFai Lau 13601e0bd5a0SAndrii Nakryiko refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0); 1361bd5f5f4eSMartin KaFai Lau if (!refold) 1362bd5f5f4eSMartin KaFai Lau return ERR_PTR(-ENOENT); 1363bd5f5f4eSMartin KaFai Lau if (uref) 13641e0bd5a0SAndrii Nakryiko atomic64_inc(&map->usercnt); 1365bd5f5f4eSMartin KaFai Lau 1366bd5f5f4eSMartin KaFai Lau return map; 1367bd5f5f4eSMartin KaFai Lau } 1368bd5f5f4eSMartin KaFai Lau 13691e0bd5a0SAndrii Nakryiko struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map) 1370b0e4701cSStanislav Fomichev { 1371b0e4701cSStanislav Fomichev spin_lock_bh(&map_idr_lock); 13721e0bd5a0SAndrii Nakryiko map = __bpf_map_inc_not_zero(map, false); 1373b0e4701cSStanislav Fomichev spin_unlock_bh(&map_idr_lock); 1374b0e4701cSStanislav Fomichev 1375b0e4701cSStanislav Fomichev return map; 1376b0e4701cSStanislav Fomichev } 1377b0e4701cSStanislav Fomichev EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero); 1378b0e4701cSStanislav Fomichev 1379b8cdc051SAlexei Starovoitov int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 1380b8cdc051SAlexei Starovoitov { 1381b8cdc051SAlexei Starovoitov return -ENOTSUPP; 1382b8cdc051SAlexei Starovoitov } 1383b8cdc051SAlexei Starovoitov 1384c9d29f46SMauricio Vasquez B static void *__bpf_copy_key(void __user *ukey, u64 key_size) 1385c9d29f46SMauricio Vasquez B { 1386c9d29f46SMauricio Vasquez B if (key_size) 138744779a4bSStanislav Fomichev return vmemdup_user(ukey, key_size); 1388c9d29f46SMauricio Vasquez B 1389c9d29f46SMauricio Vasquez B if (ukey) 1390c9d29f46SMauricio Vasquez B return ERR_PTR(-EINVAL); 1391c9d29f46SMauricio Vasquez B 1392c9d29f46SMauricio Vasquez B return NULL; 1393c9d29f46SMauricio Vasquez B } 1394c9d29f46SMauricio Vasquez B 1395af2ac3e1SAlexei Starovoitov static void *___bpf_copy_key(bpfptr_t ukey, u64 key_size) 1396af2ac3e1SAlexei Starovoitov { 1397af2ac3e1SAlexei Starovoitov if (key_size) 139844779a4bSStanislav Fomichev return kvmemdup_bpfptr(ukey, key_size); 1399af2ac3e1SAlexei Starovoitov 1400af2ac3e1SAlexei Starovoitov if (!bpfptr_is_null(ukey)) 1401af2ac3e1SAlexei Starovoitov return ERR_PTR(-EINVAL); 1402af2ac3e1SAlexei Starovoitov 1403af2ac3e1SAlexei Starovoitov return NULL; 1404af2ac3e1SAlexei Starovoitov } 1405af2ac3e1SAlexei Starovoitov 1406db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 140796049f3aSAlexei Starovoitov #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags 1408db20fd2bSAlexei Starovoitov 1409db20fd2bSAlexei Starovoitov static int map_lookup_elem(union bpf_attr *attr) 1410db20fd2bSAlexei Starovoitov { 1411535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1412535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 1413db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1414db20fd2bSAlexei Starovoitov struct bpf_map *map; 141515c14a3dSBrian Vazquez void *key, *value; 141615a07b33SAlexei Starovoitov u32 value_size; 1417592867bfSDaniel Borkmann struct fd f; 1418db20fd2bSAlexei Starovoitov int err; 1419db20fd2bSAlexei Starovoitov 1420db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 1421db20fd2bSAlexei Starovoitov return -EINVAL; 1422db20fd2bSAlexei Starovoitov 142396049f3aSAlexei Starovoitov if (attr->flags & ~BPF_F_LOCK) 142496049f3aSAlexei Starovoitov return -EINVAL; 142596049f3aSAlexei Starovoitov 1426592867bfSDaniel Borkmann f = fdget(ufd); 1427c2101297SDaniel Borkmann map = __bpf_map_get(f); 1428db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1429db20fd2bSAlexei Starovoitov return PTR_ERR(map); 143087df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 14316e71b04aSChenbo Feng err = -EPERM; 14326e71b04aSChenbo Feng goto err_put; 14336e71b04aSChenbo Feng } 14346e71b04aSChenbo Feng 143596049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 1436db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) { 143796049f3aSAlexei Starovoitov err = -EINVAL; 143896049f3aSAlexei Starovoitov goto err_put; 143996049f3aSAlexei Starovoitov } 144096049f3aSAlexei Starovoitov 1441c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1442e4448ed8SAl Viro if (IS_ERR(key)) { 1443e4448ed8SAl Viro err = PTR_ERR(key); 1444db20fd2bSAlexei Starovoitov goto err_put; 1445e4448ed8SAl Viro } 1446db20fd2bSAlexei Starovoitov 144715c14a3dSBrian Vazquez value_size = bpf_map_value_size(map); 144815a07b33SAlexei Starovoitov 14498ebe667cSAlexei Starovoitov err = -ENOMEM; 1450f0dce1d9SStanislav Fomichev value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN); 1451db20fd2bSAlexei Starovoitov if (!value) 14528ebe667cSAlexei Starovoitov goto free_key; 14538ebe667cSAlexei Starovoitov 14549330986cSJoanne Koong if (map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) { 14559330986cSJoanne Koong if (copy_from_user(value, uvalue, value_size)) 14569330986cSJoanne Koong err = -EFAULT; 14579330986cSJoanne Koong else 14589330986cSJoanne Koong err = bpf_map_copy_value(map, key, value, attr->flags); 14599330986cSJoanne Koong goto free_value; 14609330986cSJoanne Koong } 14619330986cSJoanne Koong 146215c14a3dSBrian Vazquez err = bpf_map_copy_value(map, key, value, attr->flags); 146315a07b33SAlexei Starovoitov if (err) 14648ebe667cSAlexei Starovoitov goto free_value; 1465db20fd2bSAlexei Starovoitov 1466db20fd2bSAlexei Starovoitov err = -EFAULT; 146715a07b33SAlexei Starovoitov if (copy_to_user(uvalue, value, value_size) != 0) 14688ebe667cSAlexei Starovoitov goto free_value; 1469db20fd2bSAlexei Starovoitov 1470db20fd2bSAlexei Starovoitov err = 0; 1471db20fd2bSAlexei Starovoitov 14728ebe667cSAlexei Starovoitov free_value: 1473f0dce1d9SStanislav Fomichev kvfree(value); 1474db20fd2bSAlexei Starovoitov free_key: 147544779a4bSStanislav Fomichev kvfree(key); 1476db20fd2bSAlexei Starovoitov err_put: 1477db20fd2bSAlexei Starovoitov fdput(f); 1478db20fd2bSAlexei Starovoitov return err; 1479db20fd2bSAlexei Starovoitov } 1480db20fd2bSAlexei Starovoitov 14811ae80cf3SDaniel Colascione 14823274f520SAlexei Starovoitov #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 1483db20fd2bSAlexei Starovoitov 1484af2ac3e1SAlexei Starovoitov static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) 1485db20fd2bSAlexei Starovoitov { 1486af2ac3e1SAlexei Starovoitov bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel); 1487af2ac3e1SAlexei Starovoitov bpfptr_t uvalue = make_bpfptr(attr->value, uattr.is_kernel); 1488db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1489db20fd2bSAlexei Starovoitov struct bpf_map *map; 1490db20fd2bSAlexei Starovoitov void *key, *value; 149115a07b33SAlexei Starovoitov u32 value_size; 1492592867bfSDaniel Borkmann struct fd f; 1493db20fd2bSAlexei Starovoitov int err; 1494db20fd2bSAlexei Starovoitov 1495db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 1496db20fd2bSAlexei Starovoitov return -EINVAL; 1497db20fd2bSAlexei Starovoitov 1498592867bfSDaniel Borkmann f = fdget(ufd); 1499c2101297SDaniel Borkmann map = __bpf_map_get(f); 1500db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1501db20fd2bSAlexei Starovoitov return PTR_ERR(map); 1502353050beSDaniel Borkmann bpf_map_write_active_inc(map); 150387df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 15046e71b04aSChenbo Feng err = -EPERM; 15056e71b04aSChenbo Feng goto err_put; 15066e71b04aSChenbo Feng } 15076e71b04aSChenbo Feng 150896049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 1509db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) { 151096049f3aSAlexei Starovoitov err = -EINVAL; 151196049f3aSAlexei Starovoitov goto err_put; 151296049f3aSAlexei Starovoitov } 151396049f3aSAlexei Starovoitov 1514af2ac3e1SAlexei Starovoitov key = ___bpf_copy_key(ukey, map->key_size); 1515e4448ed8SAl Viro if (IS_ERR(key)) { 1516e4448ed8SAl Viro err = PTR_ERR(key); 1517db20fd2bSAlexei Starovoitov goto err_put; 1518e4448ed8SAl Viro } 1519db20fd2bSAlexei Starovoitov 1520f0dce1d9SStanislav Fomichev value_size = bpf_map_value_size(map); 1521a02c118eSWang Yufen value = kvmemdup_bpfptr(uvalue, value_size); 1522a02c118eSWang Yufen if (IS_ERR(value)) { 1523a02c118eSWang Yufen err = PTR_ERR(value); 1524db20fd2bSAlexei Starovoitov goto free_key; 1525a02c118eSWang Yufen } 1526db20fd2bSAlexei Starovoitov 15273af43ba4SHou Tao err = bpf_map_update_value(map, f.file, key, value, attr->flags); 15286710e112SJesper Dangaard Brouer 1529f0dce1d9SStanislav Fomichev kvfree(value); 1530db20fd2bSAlexei Starovoitov free_key: 153144779a4bSStanislav Fomichev kvfree(key); 1532db20fd2bSAlexei Starovoitov err_put: 1533353050beSDaniel Borkmann bpf_map_write_active_dec(map); 1534db20fd2bSAlexei Starovoitov fdput(f); 1535db20fd2bSAlexei Starovoitov return err; 1536db20fd2bSAlexei Starovoitov } 1537db20fd2bSAlexei Starovoitov 1538db20fd2bSAlexei Starovoitov #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 1539db20fd2bSAlexei Starovoitov 1540b88df697SBenjamin Tissoires static int map_delete_elem(union bpf_attr *attr, bpfptr_t uattr) 1541db20fd2bSAlexei Starovoitov { 1542b88df697SBenjamin Tissoires bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel); 1543db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1544db20fd2bSAlexei Starovoitov struct bpf_map *map; 1545592867bfSDaniel Borkmann struct fd f; 1546db20fd2bSAlexei Starovoitov void *key; 1547db20fd2bSAlexei Starovoitov int err; 1548db20fd2bSAlexei Starovoitov 1549db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 1550db20fd2bSAlexei Starovoitov return -EINVAL; 1551db20fd2bSAlexei Starovoitov 1552592867bfSDaniel Borkmann f = fdget(ufd); 1553c2101297SDaniel Borkmann map = __bpf_map_get(f); 1554db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1555db20fd2bSAlexei Starovoitov return PTR_ERR(map); 1556353050beSDaniel Borkmann bpf_map_write_active_inc(map); 155787df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 15586e71b04aSChenbo Feng err = -EPERM; 15596e71b04aSChenbo Feng goto err_put; 15606e71b04aSChenbo Feng } 15616e71b04aSChenbo Feng 1562b88df697SBenjamin Tissoires key = ___bpf_copy_key(ukey, map->key_size); 1563e4448ed8SAl Viro if (IS_ERR(key)) { 1564e4448ed8SAl Viro err = PTR_ERR(key); 1565db20fd2bSAlexei Starovoitov goto err_put; 1566e4448ed8SAl Viro } 1567db20fd2bSAlexei Starovoitov 15689d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) { 1569a3884572SJakub Kicinski err = bpf_map_offload_delete_elem(map, key); 1570a3884572SJakub Kicinski goto out; 157185d33df3SMartin KaFai Lau } else if (IS_FD_PROG_ARRAY(map) || 157285d33df3SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 157385d33df3SMartin KaFai Lau /* These maps require sleepable context */ 1574da765a2fSDaniel Borkmann err = map->ops->map_delete_elem(map, key); 1575da765a2fSDaniel Borkmann goto out; 1576a3884572SJakub Kicinski } 1577a3884572SJakub Kicinski 1578b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 1579db20fd2bSAlexei Starovoitov rcu_read_lock(); 1580db20fd2bSAlexei Starovoitov err = map->ops->map_delete_elem(map, key); 1581db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1582b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 15831ae80cf3SDaniel Colascione maybe_wait_bpf_programs(map); 1584a3884572SJakub Kicinski out: 158544779a4bSStanislav Fomichev kvfree(key); 1586db20fd2bSAlexei Starovoitov err_put: 1587353050beSDaniel Borkmann bpf_map_write_active_dec(map); 1588db20fd2bSAlexei Starovoitov fdput(f); 1589db20fd2bSAlexei Starovoitov return err; 1590db20fd2bSAlexei Starovoitov } 1591db20fd2bSAlexei Starovoitov 1592db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 1593db20fd2bSAlexei Starovoitov #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 1594db20fd2bSAlexei Starovoitov 1595db20fd2bSAlexei Starovoitov static int map_get_next_key(union bpf_attr *attr) 1596db20fd2bSAlexei Starovoitov { 1597535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1598535e7b4bSMickaël Salaün void __user *unext_key = u64_to_user_ptr(attr->next_key); 1599db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1600db20fd2bSAlexei Starovoitov struct bpf_map *map; 1601db20fd2bSAlexei Starovoitov void *key, *next_key; 1602592867bfSDaniel Borkmann struct fd f; 1603db20fd2bSAlexei Starovoitov int err; 1604db20fd2bSAlexei Starovoitov 1605db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 1606db20fd2bSAlexei Starovoitov return -EINVAL; 1607db20fd2bSAlexei Starovoitov 1608592867bfSDaniel Borkmann f = fdget(ufd); 1609c2101297SDaniel Borkmann map = __bpf_map_get(f); 1610db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1611db20fd2bSAlexei Starovoitov return PTR_ERR(map); 161287df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 16136e71b04aSChenbo Feng err = -EPERM; 16146e71b04aSChenbo Feng goto err_put; 16156e71b04aSChenbo Feng } 16166e71b04aSChenbo Feng 16178fe45924STeng Qin if (ukey) { 1618c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1619e4448ed8SAl Viro if (IS_ERR(key)) { 1620e4448ed8SAl Viro err = PTR_ERR(key); 1621db20fd2bSAlexei Starovoitov goto err_put; 1622e4448ed8SAl Viro } 16238fe45924STeng Qin } else { 16248fe45924STeng Qin key = NULL; 16258fe45924STeng Qin } 1626db20fd2bSAlexei Starovoitov 1627db20fd2bSAlexei Starovoitov err = -ENOMEM; 162844779a4bSStanislav Fomichev next_key = kvmalloc(map->key_size, GFP_USER); 1629db20fd2bSAlexei Starovoitov if (!next_key) 1630db20fd2bSAlexei Starovoitov goto free_key; 1631db20fd2bSAlexei Starovoitov 16329d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) { 1633a3884572SJakub Kicinski err = bpf_map_offload_get_next_key(map, key, next_key); 1634a3884572SJakub Kicinski goto out; 1635a3884572SJakub Kicinski } 1636a3884572SJakub Kicinski 1637db20fd2bSAlexei Starovoitov rcu_read_lock(); 1638db20fd2bSAlexei Starovoitov err = map->ops->map_get_next_key(map, key, next_key); 1639db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1640a3884572SJakub Kicinski out: 1641db20fd2bSAlexei Starovoitov if (err) 1642db20fd2bSAlexei Starovoitov goto free_next_key; 1643db20fd2bSAlexei Starovoitov 1644db20fd2bSAlexei Starovoitov err = -EFAULT; 1645db20fd2bSAlexei Starovoitov if (copy_to_user(unext_key, next_key, map->key_size) != 0) 1646db20fd2bSAlexei Starovoitov goto free_next_key; 1647db20fd2bSAlexei Starovoitov 1648db20fd2bSAlexei Starovoitov err = 0; 1649db20fd2bSAlexei Starovoitov 1650db20fd2bSAlexei Starovoitov free_next_key: 165144779a4bSStanislav Fomichev kvfree(next_key); 1652db20fd2bSAlexei Starovoitov free_key: 165344779a4bSStanislav Fomichev kvfree(key); 1654db20fd2bSAlexei Starovoitov err_put: 1655db20fd2bSAlexei Starovoitov fdput(f); 1656db20fd2bSAlexei Starovoitov return err; 1657db20fd2bSAlexei Starovoitov } 1658db20fd2bSAlexei Starovoitov 1659aa2e93b8SBrian Vazquez int generic_map_delete_batch(struct bpf_map *map, 1660aa2e93b8SBrian Vazquez const union bpf_attr *attr, 1661aa2e93b8SBrian Vazquez union bpf_attr __user *uattr) 1662aa2e93b8SBrian Vazquez { 1663aa2e93b8SBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1664aa2e93b8SBrian Vazquez u32 cp, max_count; 1665aa2e93b8SBrian Vazquez int err = 0; 1666aa2e93b8SBrian Vazquez void *key; 1667aa2e93b8SBrian Vazquez 1668aa2e93b8SBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1669aa2e93b8SBrian Vazquez return -EINVAL; 1670aa2e93b8SBrian Vazquez 1671aa2e93b8SBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1672db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) { 1673aa2e93b8SBrian Vazquez return -EINVAL; 1674aa2e93b8SBrian Vazquez } 1675aa2e93b8SBrian Vazquez 1676aa2e93b8SBrian Vazquez max_count = attr->batch.count; 1677aa2e93b8SBrian Vazquez if (!max_count) 1678aa2e93b8SBrian Vazquez return 0; 1679aa2e93b8SBrian Vazquez 168044779a4bSStanislav Fomichev key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 16812e3a94aaSBrian Vazquez if (!key) 16822e3a94aaSBrian Vazquez return -ENOMEM; 16832e3a94aaSBrian Vazquez 1684aa2e93b8SBrian Vazquez for (cp = 0; cp < max_count; cp++) { 16852e3a94aaSBrian Vazquez err = -EFAULT; 16862e3a94aaSBrian Vazquez if (copy_from_user(key, keys + cp * map->key_size, 16872e3a94aaSBrian Vazquez map->key_size)) 1688aa2e93b8SBrian Vazquez break; 1689aa2e93b8SBrian Vazquez 16909d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) { 1691aa2e93b8SBrian Vazquez err = bpf_map_offload_delete_elem(map, key); 1692aa2e93b8SBrian Vazquez break; 1693aa2e93b8SBrian Vazquez } 1694aa2e93b8SBrian Vazquez 1695b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 1696aa2e93b8SBrian Vazquez rcu_read_lock(); 1697aa2e93b8SBrian Vazquez err = map->ops->map_delete_elem(map, key); 1698aa2e93b8SBrian Vazquez rcu_read_unlock(); 1699b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 1700aa2e93b8SBrian Vazquez if (err) 1701aa2e93b8SBrian Vazquez break; 170275134f16SEric Dumazet cond_resched(); 1703aa2e93b8SBrian Vazquez } 1704aa2e93b8SBrian Vazquez if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) 1705aa2e93b8SBrian Vazquez err = -EFAULT; 17062e3a94aaSBrian Vazquez 170744779a4bSStanislav Fomichev kvfree(key); 17089087c6ffSEric Dumazet 17099087c6ffSEric Dumazet maybe_wait_bpf_programs(map); 1710aa2e93b8SBrian Vazquez return err; 1711aa2e93b8SBrian Vazquez } 1712aa2e93b8SBrian Vazquez 17133af43ba4SHou Tao int generic_map_update_batch(struct bpf_map *map, struct file *map_file, 1714aa2e93b8SBrian Vazquez const union bpf_attr *attr, 1715aa2e93b8SBrian Vazquez union bpf_attr __user *uattr) 1716aa2e93b8SBrian Vazquez { 1717aa2e93b8SBrian Vazquez void __user *values = u64_to_user_ptr(attr->batch.values); 1718aa2e93b8SBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1719aa2e93b8SBrian Vazquez u32 value_size, cp, max_count; 1720aa2e93b8SBrian Vazquez void *key, *value; 1721aa2e93b8SBrian Vazquez int err = 0; 1722aa2e93b8SBrian Vazquez 1723aa2e93b8SBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1724aa2e93b8SBrian Vazquez return -EINVAL; 1725aa2e93b8SBrian Vazquez 1726aa2e93b8SBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1727db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) { 1728aa2e93b8SBrian Vazquez return -EINVAL; 1729aa2e93b8SBrian Vazquez } 1730aa2e93b8SBrian Vazquez 1731aa2e93b8SBrian Vazquez value_size = bpf_map_value_size(map); 1732aa2e93b8SBrian Vazquez 1733aa2e93b8SBrian Vazquez max_count = attr->batch.count; 1734aa2e93b8SBrian Vazquez if (!max_count) 1735aa2e93b8SBrian Vazquez return 0; 1736aa2e93b8SBrian Vazquez 173744779a4bSStanislav Fomichev key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 17382e3a94aaSBrian Vazquez if (!key) 1739aa2e93b8SBrian Vazquez return -ENOMEM; 1740aa2e93b8SBrian Vazquez 1741f0dce1d9SStanislav Fomichev value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN); 17422e3a94aaSBrian Vazquez if (!value) { 174344779a4bSStanislav Fomichev kvfree(key); 17442e3a94aaSBrian Vazquez return -ENOMEM; 1745aa2e93b8SBrian Vazquez } 17462e3a94aaSBrian Vazquez 17472e3a94aaSBrian Vazquez for (cp = 0; cp < max_count; cp++) { 1748aa2e93b8SBrian Vazquez err = -EFAULT; 17492e3a94aaSBrian Vazquez if (copy_from_user(key, keys + cp * map->key_size, 17502e3a94aaSBrian Vazquez map->key_size) || 17512e3a94aaSBrian Vazquez copy_from_user(value, values + cp * value_size, value_size)) 1752aa2e93b8SBrian Vazquez break; 1753aa2e93b8SBrian Vazquez 17543af43ba4SHou Tao err = bpf_map_update_value(map, map_file, key, value, 1755aa2e93b8SBrian Vazquez attr->batch.elem_flags); 1756aa2e93b8SBrian Vazquez 1757aa2e93b8SBrian Vazquez if (err) 1758aa2e93b8SBrian Vazquez break; 175975134f16SEric Dumazet cond_resched(); 1760aa2e93b8SBrian Vazquez } 1761aa2e93b8SBrian Vazquez 1762aa2e93b8SBrian Vazquez if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) 1763aa2e93b8SBrian Vazquez err = -EFAULT; 1764aa2e93b8SBrian Vazquez 1765f0dce1d9SStanislav Fomichev kvfree(value); 176644779a4bSStanislav Fomichev kvfree(key); 1767aa2e93b8SBrian Vazquez return err; 1768aa2e93b8SBrian Vazquez } 1769aa2e93b8SBrian Vazquez 1770cb4d03abSBrian Vazquez #define MAP_LOOKUP_RETRIES 3 1771cb4d03abSBrian Vazquez 1772cb4d03abSBrian Vazquez int generic_map_lookup_batch(struct bpf_map *map, 1773cb4d03abSBrian Vazquez const union bpf_attr *attr, 1774cb4d03abSBrian Vazquez union bpf_attr __user *uattr) 1775cb4d03abSBrian Vazquez { 1776cb4d03abSBrian Vazquez void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch); 1777cb4d03abSBrian Vazquez void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch); 1778cb4d03abSBrian Vazquez void __user *values = u64_to_user_ptr(attr->batch.values); 1779cb4d03abSBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1780cb4d03abSBrian Vazquez void *buf, *buf_prevkey, *prev_key, *key, *value; 1781cb4d03abSBrian Vazquez int err, retry = MAP_LOOKUP_RETRIES; 1782cb4d03abSBrian Vazquez u32 value_size, cp, max_count; 1783cb4d03abSBrian Vazquez 1784cb4d03abSBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1785cb4d03abSBrian Vazquez return -EINVAL; 1786cb4d03abSBrian Vazquez 1787cb4d03abSBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1788db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) 1789cb4d03abSBrian Vazquez return -EINVAL; 1790cb4d03abSBrian Vazquez 1791cb4d03abSBrian Vazquez value_size = bpf_map_value_size(map); 1792cb4d03abSBrian Vazquez 1793cb4d03abSBrian Vazquez max_count = attr->batch.count; 1794cb4d03abSBrian Vazquez if (!max_count) 1795cb4d03abSBrian Vazquez return 0; 1796cb4d03abSBrian Vazquez 1797cb4d03abSBrian Vazquez if (put_user(0, &uattr->batch.count)) 1798cb4d03abSBrian Vazquez return -EFAULT; 1799cb4d03abSBrian Vazquez 180044779a4bSStanislav Fomichev buf_prevkey = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 1801cb4d03abSBrian Vazquez if (!buf_prevkey) 1802cb4d03abSBrian Vazquez return -ENOMEM; 1803cb4d03abSBrian Vazquez 1804f0dce1d9SStanislav Fomichev buf = kvmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN); 1805cb4d03abSBrian Vazquez if (!buf) { 180644779a4bSStanislav Fomichev kvfree(buf_prevkey); 1807cb4d03abSBrian Vazquez return -ENOMEM; 1808cb4d03abSBrian Vazquez } 1809cb4d03abSBrian Vazquez 1810cb4d03abSBrian Vazquez err = -EFAULT; 1811cb4d03abSBrian Vazquez prev_key = NULL; 1812cb4d03abSBrian Vazquez if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size)) 1813cb4d03abSBrian Vazquez goto free_buf; 1814cb4d03abSBrian Vazquez key = buf; 1815cb4d03abSBrian Vazquez value = key + map->key_size; 1816cb4d03abSBrian Vazquez if (ubatch) 1817cb4d03abSBrian Vazquez prev_key = buf_prevkey; 1818cb4d03abSBrian Vazquez 1819cb4d03abSBrian Vazquez for (cp = 0; cp < max_count;) { 1820cb4d03abSBrian Vazquez rcu_read_lock(); 1821cb4d03abSBrian Vazquez err = map->ops->map_get_next_key(map, prev_key, key); 1822cb4d03abSBrian Vazquez rcu_read_unlock(); 1823cb4d03abSBrian Vazquez if (err) 1824cb4d03abSBrian Vazquez break; 1825cb4d03abSBrian Vazquez err = bpf_map_copy_value(map, key, value, 1826cb4d03abSBrian Vazquez attr->batch.elem_flags); 1827cb4d03abSBrian Vazquez 1828cb4d03abSBrian Vazquez if (err == -ENOENT) { 1829cb4d03abSBrian Vazquez if (retry) { 1830cb4d03abSBrian Vazquez retry--; 1831cb4d03abSBrian Vazquez continue; 1832cb4d03abSBrian Vazquez } 1833cb4d03abSBrian Vazquez err = -EINTR; 1834cb4d03abSBrian Vazquez break; 1835cb4d03abSBrian Vazquez } 1836cb4d03abSBrian Vazquez 1837cb4d03abSBrian Vazquez if (err) 1838cb4d03abSBrian Vazquez goto free_buf; 1839cb4d03abSBrian Vazquez 1840cb4d03abSBrian Vazquez if (copy_to_user(keys + cp * map->key_size, key, 1841cb4d03abSBrian Vazquez map->key_size)) { 1842cb4d03abSBrian Vazquez err = -EFAULT; 1843cb4d03abSBrian Vazquez goto free_buf; 1844cb4d03abSBrian Vazquez } 1845cb4d03abSBrian Vazquez if (copy_to_user(values + cp * value_size, value, value_size)) { 1846cb4d03abSBrian Vazquez err = -EFAULT; 1847cb4d03abSBrian Vazquez goto free_buf; 1848cb4d03abSBrian Vazquez } 1849cb4d03abSBrian Vazquez 1850cb4d03abSBrian Vazquez if (!prev_key) 1851cb4d03abSBrian Vazquez prev_key = buf_prevkey; 1852cb4d03abSBrian Vazquez 1853cb4d03abSBrian Vazquez swap(prev_key, key); 1854cb4d03abSBrian Vazquez retry = MAP_LOOKUP_RETRIES; 1855cb4d03abSBrian Vazquez cp++; 185675134f16SEric Dumazet cond_resched(); 1857cb4d03abSBrian Vazquez } 1858cb4d03abSBrian Vazquez 1859cb4d03abSBrian Vazquez if (err == -EFAULT) 1860cb4d03abSBrian Vazquez goto free_buf; 1861cb4d03abSBrian Vazquez 1862cb4d03abSBrian Vazquez if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) || 1863cb4d03abSBrian Vazquez (cp && copy_to_user(uobatch, prev_key, map->key_size)))) 1864cb4d03abSBrian Vazquez err = -EFAULT; 1865cb4d03abSBrian Vazquez 1866cb4d03abSBrian Vazquez free_buf: 186744779a4bSStanislav Fomichev kvfree(buf_prevkey); 1868f0dce1d9SStanislav Fomichev kvfree(buf); 1869cb4d03abSBrian Vazquez return err; 1870cb4d03abSBrian Vazquez } 1871cb4d03abSBrian Vazquez 18723e87f192SDenis Salopek #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD flags 1873bd513cd0SMauricio Vasquez B 1874bd513cd0SMauricio Vasquez B static int map_lookup_and_delete_elem(union bpf_attr *attr) 1875bd513cd0SMauricio Vasquez B { 1876bd513cd0SMauricio Vasquez B void __user *ukey = u64_to_user_ptr(attr->key); 1877bd513cd0SMauricio Vasquez B void __user *uvalue = u64_to_user_ptr(attr->value); 1878bd513cd0SMauricio Vasquez B int ufd = attr->map_fd; 1879bd513cd0SMauricio Vasquez B struct bpf_map *map; 1880540fefc0SAlexei Starovoitov void *key, *value; 1881bd513cd0SMauricio Vasquez B u32 value_size; 1882bd513cd0SMauricio Vasquez B struct fd f; 1883bd513cd0SMauricio Vasquez B int err; 1884bd513cd0SMauricio Vasquez B 1885bd513cd0SMauricio Vasquez B if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) 1886bd513cd0SMauricio Vasquez B return -EINVAL; 1887bd513cd0SMauricio Vasquez B 18883e87f192SDenis Salopek if (attr->flags & ~BPF_F_LOCK) 18893e87f192SDenis Salopek return -EINVAL; 18903e87f192SDenis Salopek 1891bd513cd0SMauricio Vasquez B f = fdget(ufd); 1892bd513cd0SMauricio Vasquez B map = __bpf_map_get(f); 1893bd513cd0SMauricio Vasquez B if (IS_ERR(map)) 1894bd513cd0SMauricio Vasquez B return PTR_ERR(map); 1895353050beSDaniel Borkmann bpf_map_write_active_inc(map); 18961ea0f912SAnton Protopopov if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) || 18971ea0f912SAnton Protopopov !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 1898bd513cd0SMauricio Vasquez B err = -EPERM; 1899bd513cd0SMauricio Vasquez B goto err_put; 1900bd513cd0SMauricio Vasquez B } 1901bd513cd0SMauricio Vasquez B 19023e87f192SDenis Salopek if (attr->flags && 19033e87f192SDenis Salopek (map->map_type == BPF_MAP_TYPE_QUEUE || 19043e87f192SDenis Salopek map->map_type == BPF_MAP_TYPE_STACK)) { 19053e87f192SDenis Salopek err = -EINVAL; 19063e87f192SDenis Salopek goto err_put; 19073e87f192SDenis Salopek } 19083e87f192SDenis Salopek 19093e87f192SDenis Salopek if ((attr->flags & BPF_F_LOCK) && 1910db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) { 19113e87f192SDenis Salopek err = -EINVAL; 19123e87f192SDenis Salopek goto err_put; 19133e87f192SDenis Salopek } 19143e87f192SDenis Salopek 1915bd513cd0SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1916bd513cd0SMauricio Vasquez B if (IS_ERR(key)) { 1917bd513cd0SMauricio Vasquez B err = PTR_ERR(key); 1918bd513cd0SMauricio Vasquez B goto err_put; 1919bd513cd0SMauricio Vasquez B } 1920bd513cd0SMauricio Vasquez B 19213e87f192SDenis Salopek value_size = bpf_map_value_size(map); 1922bd513cd0SMauricio Vasquez B 1923bd513cd0SMauricio Vasquez B err = -ENOMEM; 1924f0dce1d9SStanislav Fomichev value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN); 1925bd513cd0SMauricio Vasquez B if (!value) 1926bd513cd0SMauricio Vasquez B goto free_key; 1927bd513cd0SMauricio Vasquez B 19283e87f192SDenis Salopek err = -ENOTSUPP; 1929bd513cd0SMauricio Vasquez B if (map->map_type == BPF_MAP_TYPE_QUEUE || 1930bd513cd0SMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 1931bd513cd0SMauricio Vasquez B err = map->ops->map_pop_elem(map, value); 19323e87f192SDenis Salopek } else if (map->map_type == BPF_MAP_TYPE_HASH || 19333e87f192SDenis Salopek map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 19343e87f192SDenis Salopek map->map_type == BPF_MAP_TYPE_LRU_HASH || 19353e87f192SDenis Salopek map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 19369d03ebc7SStanislav Fomichev if (!bpf_map_is_offloaded(map)) { 19373e87f192SDenis Salopek bpf_disable_instrumentation(); 19383e87f192SDenis Salopek rcu_read_lock(); 19393e87f192SDenis Salopek err = map->ops->map_lookup_and_delete_elem(map, key, value, attr->flags); 19403e87f192SDenis Salopek rcu_read_unlock(); 19413e87f192SDenis Salopek bpf_enable_instrumentation(); 19423e87f192SDenis Salopek } 1943bd513cd0SMauricio Vasquez B } 1944bd513cd0SMauricio Vasquez B 1945bd513cd0SMauricio Vasquez B if (err) 1946bd513cd0SMauricio Vasquez B goto free_value; 1947bd513cd0SMauricio Vasquez B 19487f645462SWei Yongjun if (copy_to_user(uvalue, value, value_size) != 0) { 19497f645462SWei Yongjun err = -EFAULT; 1950bd513cd0SMauricio Vasquez B goto free_value; 19517f645462SWei Yongjun } 1952bd513cd0SMauricio Vasquez B 1953bd513cd0SMauricio Vasquez B err = 0; 1954bd513cd0SMauricio Vasquez B 1955bd513cd0SMauricio Vasquez B free_value: 1956f0dce1d9SStanislav Fomichev kvfree(value); 1957bd513cd0SMauricio Vasquez B free_key: 195844779a4bSStanislav Fomichev kvfree(key); 1959bd513cd0SMauricio Vasquez B err_put: 1960353050beSDaniel Borkmann bpf_map_write_active_dec(map); 1961bd513cd0SMauricio Vasquez B fdput(f); 1962bd513cd0SMauricio Vasquez B return err; 1963bd513cd0SMauricio Vasquez B } 1964bd513cd0SMauricio Vasquez B 196587df15deSDaniel Borkmann #define BPF_MAP_FREEZE_LAST_FIELD map_fd 196687df15deSDaniel Borkmann 196787df15deSDaniel Borkmann static int map_freeze(const union bpf_attr *attr) 196887df15deSDaniel Borkmann { 196987df15deSDaniel Borkmann int err = 0, ufd = attr->map_fd; 197087df15deSDaniel Borkmann struct bpf_map *map; 197187df15deSDaniel Borkmann struct fd f; 197287df15deSDaniel Borkmann 197387df15deSDaniel Borkmann if (CHECK_ATTR(BPF_MAP_FREEZE)) 197487df15deSDaniel Borkmann return -EINVAL; 197587df15deSDaniel Borkmann 197687df15deSDaniel Borkmann f = fdget(ufd); 197787df15deSDaniel Borkmann map = __bpf_map_get(f); 197887df15deSDaniel Borkmann if (IS_ERR(map)) 197987df15deSDaniel Borkmann return PTR_ERR(map); 1980fc970227SAndrii Nakryiko 1981db559117SKumar Kartikeya Dwivedi if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS || !IS_ERR_OR_NULL(map->record)) { 1982849b4d94SMartin KaFai Lau fdput(f); 1983849b4d94SMartin KaFai Lau return -ENOTSUPP; 1984849b4d94SMartin KaFai Lau } 1985849b4d94SMartin KaFai Lau 1986c4c84f6fSAndrii Nakryiko if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 19874266f41fSDaniel Borkmann fdput(f); 19884266f41fSDaniel Borkmann return -EPERM; 1989c4c84f6fSAndrii Nakryiko } 1990c4c84f6fSAndrii Nakryiko 1991fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 1992353050beSDaniel Borkmann if (bpf_map_write_active(map)) { 1993fc970227SAndrii Nakryiko err = -EBUSY; 1994fc970227SAndrii Nakryiko goto err_put; 1995fc970227SAndrii Nakryiko } 199687df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) { 199787df15deSDaniel Borkmann err = -EBUSY; 199887df15deSDaniel Borkmann goto err_put; 199987df15deSDaniel Borkmann } 200087df15deSDaniel Borkmann 200187df15deSDaniel Borkmann WRITE_ONCE(map->frozen, true); 200287df15deSDaniel Borkmann err_put: 2003fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 200487df15deSDaniel Borkmann fdput(f); 200587df15deSDaniel Borkmann return err; 200687df15deSDaniel Borkmann } 200787df15deSDaniel Borkmann 20087de16e3aSJakub Kicinski static const struct bpf_prog_ops * const bpf_prog_types[] = { 200991cc1a99SAlexei Starovoitov #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 20107de16e3aSJakub Kicinski [_id] = & _name ## _prog_ops, 20117de16e3aSJakub Kicinski #define BPF_MAP_TYPE(_id, _ops) 2012f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) 20137de16e3aSJakub Kicinski #include <linux/bpf_types.h> 20147de16e3aSJakub Kicinski #undef BPF_PROG_TYPE 20157de16e3aSJakub Kicinski #undef BPF_MAP_TYPE 2016f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 20177de16e3aSJakub Kicinski }; 20187de16e3aSJakub Kicinski 201909756af4SAlexei Starovoitov static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 202009756af4SAlexei Starovoitov { 2021d0f1a451SDaniel Borkmann const struct bpf_prog_ops *ops; 2022d0f1a451SDaniel Borkmann 2023d0f1a451SDaniel Borkmann if (type >= ARRAY_SIZE(bpf_prog_types)) 2024d0f1a451SDaniel Borkmann return -EINVAL; 2025d0f1a451SDaniel Borkmann type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types)); 2026d0f1a451SDaniel Borkmann ops = bpf_prog_types[type]; 2027d0f1a451SDaniel Borkmann if (!ops) 2028be9370a7SJohannes Berg return -EINVAL; 202909756af4SAlexei Starovoitov 20309d03ebc7SStanislav Fomichev if (!bpf_prog_is_offloaded(prog->aux)) 2031d0f1a451SDaniel Borkmann prog->aux->ops = ops; 2032ab3f0063SJakub Kicinski else 2033ab3f0063SJakub Kicinski prog->aux->ops = &bpf_offload_prog_ops; 203424701eceSDaniel Borkmann prog->type = type; 203509756af4SAlexei Starovoitov return 0; 203609756af4SAlexei Starovoitov } 203709756af4SAlexei Starovoitov 2038bae141f5SDaniel Borkmann enum bpf_audit { 2039bae141f5SDaniel Borkmann BPF_AUDIT_LOAD, 2040bae141f5SDaniel Borkmann BPF_AUDIT_UNLOAD, 2041bae141f5SDaniel Borkmann BPF_AUDIT_MAX, 2042bae141f5SDaniel Borkmann }; 2043bae141f5SDaniel Borkmann 2044bae141f5SDaniel Borkmann static const char * const bpf_audit_str[BPF_AUDIT_MAX] = { 2045bae141f5SDaniel Borkmann [BPF_AUDIT_LOAD] = "LOAD", 2046bae141f5SDaniel Borkmann [BPF_AUDIT_UNLOAD] = "UNLOAD", 2047bae141f5SDaniel Borkmann }; 2048bae141f5SDaniel Borkmann 2049bae141f5SDaniel Borkmann static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op) 2050bae141f5SDaniel Borkmann { 2051bae141f5SDaniel Borkmann struct audit_context *ctx = NULL; 2052bae141f5SDaniel Borkmann struct audit_buffer *ab; 2053bae141f5SDaniel Borkmann 2054bae141f5SDaniel Borkmann if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX)) 2055bae141f5SDaniel Borkmann return; 2056bae141f5SDaniel Borkmann if (audit_enabled == AUDIT_OFF) 2057bae141f5SDaniel Borkmann return; 2058ef01f4e2SPaul Moore if (!in_irq() && !irqs_disabled()) 2059bae141f5SDaniel Borkmann ctx = audit_context(); 2060bae141f5SDaniel Borkmann ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF); 2061bae141f5SDaniel Borkmann if (unlikely(!ab)) 2062bae141f5SDaniel Borkmann return; 2063bae141f5SDaniel Borkmann audit_log_format(ab, "prog-id=%u op=%s", 2064bae141f5SDaniel Borkmann prog->aux->id, bpf_audit_str[op]); 2065bae141f5SDaniel Borkmann audit_log_end(ab); 2066bae141f5SDaniel Borkmann } 2067bae141f5SDaniel Borkmann 2068dc4bb0e2SMartin KaFai Lau static int bpf_prog_alloc_id(struct bpf_prog *prog) 2069dc4bb0e2SMartin KaFai Lau { 2070dc4bb0e2SMartin KaFai Lau int id; 2071dc4bb0e2SMartin KaFai Lau 2072b76354cdSShaohua Li idr_preload(GFP_KERNEL); 2073dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 2074dc4bb0e2SMartin KaFai Lau id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 2075dc4bb0e2SMartin KaFai Lau if (id > 0) 2076dc4bb0e2SMartin KaFai Lau prog->aux->id = id; 2077dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 2078b76354cdSShaohua Li idr_preload_end(); 2079dc4bb0e2SMartin KaFai Lau 2080dc4bb0e2SMartin KaFai Lau /* id is in [1, INT_MAX) */ 2081dc4bb0e2SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 2082dc4bb0e2SMartin KaFai Lau return -ENOSPC; 2083dc4bb0e2SMartin KaFai Lau 2084dc4bb0e2SMartin KaFai Lau return id > 0 ? 0 : id; 2085dc4bb0e2SMartin KaFai Lau } 2086dc4bb0e2SMartin KaFai Lau 2087e7895f01SPaul Moore void bpf_prog_free_id(struct bpf_prog *prog) 2088dc4bb0e2SMartin KaFai Lau { 2089d809e134SAlexei Starovoitov unsigned long flags; 2090d809e134SAlexei Starovoitov 2091ad8ad79fSJakub Kicinski /* cBPF to eBPF migrations are currently not in the idr store. 2092ad8ad79fSJakub Kicinski * Offloaded programs are removed from the store when their device 2093ad8ad79fSJakub Kicinski * disappears - even if someone grabs an fd to them they are unusable, 2094ad8ad79fSJakub Kicinski * simply waiting for refcnt to drop to be freed. 2095ad8ad79fSJakub Kicinski */ 2096dc4bb0e2SMartin KaFai Lau if (!prog->aux->id) 2097dc4bb0e2SMartin KaFai Lau return; 2098dc4bb0e2SMartin KaFai Lau 2099d809e134SAlexei Starovoitov spin_lock_irqsave(&prog_idr_lock, flags); 2100dc4bb0e2SMartin KaFai Lau idr_remove(&prog_idr, prog->aux->id); 2101ad8ad79fSJakub Kicinski prog->aux->id = 0; 2102d809e134SAlexei Starovoitov spin_unlock_irqrestore(&prog_idr_lock, flags); 2103dc4bb0e2SMartin KaFai Lau } 2104dc4bb0e2SMartin KaFai Lau 21051aacde3dSDaniel Borkmann static void __bpf_prog_put_rcu(struct rcu_head *rcu) 2106abf2e7d6SAlexei Starovoitov { 2107abf2e7d6SAlexei Starovoitov struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 2108abf2e7d6SAlexei Starovoitov 21093b4d9eb2SDaniel Borkmann kvfree(aux->func_info); 21108c1b6e69SAlexei Starovoitov kfree(aux->func_info_aux); 21113ac1f01bSRoman Gushchin free_uid(aux->user); 2112afdb09c7SChenbo Feng security_bpf_prog_free(aux); 2113abf2e7d6SAlexei Starovoitov bpf_prog_free(aux->prog); 2114abf2e7d6SAlexei Starovoitov } 2115abf2e7d6SAlexei Starovoitov 2116cd7455f1SDaniel Borkmann static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) 2117cd7455f1SDaniel Borkmann { 2118cd7455f1SDaniel Borkmann bpf_prog_kallsyms_del_all(prog); 2119cd7455f1SDaniel Borkmann btf_put(prog->aux->btf); 212031bf1dbcSViktor Malik module_put(prog->aux->mod); 2121e16301fbSMartin KaFai Lau kvfree(prog->aux->jited_linfo); 2122e16301fbSMartin KaFai Lau kvfree(prog->aux->linfo); 2123e6ac2450SMartin KaFai Lau kfree(prog->aux->kfunc_tab); 212422dc4a0fSAndrii Nakryiko if (prog->aux->attach_btf) 212522dc4a0fSAndrii Nakryiko btf_put(prog->aux->attach_btf); 2126cd7455f1SDaniel Borkmann 21271e6c62a8SAlexei Starovoitov if (deferred) { 21281e6c62a8SAlexei Starovoitov if (prog->aux->sleepable) 21291e6c62a8SAlexei Starovoitov call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); 2130cd7455f1SDaniel Borkmann else 21311e6c62a8SAlexei Starovoitov call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 21321e6c62a8SAlexei Starovoitov } else { 2133cd7455f1SDaniel Borkmann __bpf_prog_put_rcu(&prog->aux->rcu); 2134cd7455f1SDaniel Borkmann } 21351e6c62a8SAlexei Starovoitov } 2136cd7455f1SDaniel Borkmann 2137d809e134SAlexei Starovoitov static void bpf_prog_put_deferred(struct work_struct *work) 213809756af4SAlexei Starovoitov { 2139d809e134SAlexei Starovoitov struct bpf_prog_aux *aux; 2140d809e134SAlexei Starovoitov struct bpf_prog *prog; 2141d809e134SAlexei Starovoitov 2142d809e134SAlexei Starovoitov aux = container_of(work, struct bpf_prog_aux, work); 2143d809e134SAlexei Starovoitov prog = aux->prog; 21446ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0); 2145bae141f5SDaniel Borkmann bpf_audit_prog(prog, BPF_AUDIT_UNLOAD); 2146e7895f01SPaul Moore bpf_prog_free_id(prog); 2147d809e134SAlexei Starovoitov __bpf_prog_put_noref(prog, true); 2148d809e134SAlexei Starovoitov } 2149d809e134SAlexei Starovoitov 2150e7895f01SPaul Moore static void __bpf_prog_put(struct bpf_prog *prog) 2151d809e134SAlexei Starovoitov { 2152d809e134SAlexei Starovoitov struct bpf_prog_aux *aux = prog->aux; 2153d809e134SAlexei Starovoitov 2154d809e134SAlexei Starovoitov if (atomic64_dec_and_test(&aux->refcnt)) { 2155d809e134SAlexei Starovoitov if (in_irq() || irqs_disabled()) { 2156d809e134SAlexei Starovoitov INIT_WORK(&aux->work, bpf_prog_put_deferred); 2157d809e134SAlexei Starovoitov schedule_work(&aux->work); 2158d809e134SAlexei Starovoitov } else { 2159d809e134SAlexei Starovoitov bpf_prog_put_deferred(&aux->work); 2160d809e134SAlexei Starovoitov } 216109756af4SAlexei Starovoitov } 2162a67edbf4SDaniel Borkmann } 2163b16d9aa4SMartin KaFai Lau 2164b16d9aa4SMartin KaFai Lau void bpf_prog_put(struct bpf_prog *prog) 2165b16d9aa4SMartin KaFai Lau { 2166e7895f01SPaul Moore __bpf_prog_put(prog); 2167b16d9aa4SMartin KaFai Lau } 2168e2e9b654SDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_put); 216909756af4SAlexei Starovoitov 217009756af4SAlexei Starovoitov static int bpf_prog_release(struct inode *inode, struct file *filp) 217109756af4SAlexei Starovoitov { 217209756af4SAlexei Starovoitov struct bpf_prog *prog = filp->private_data; 217309756af4SAlexei Starovoitov 21741aacde3dSDaniel Borkmann bpf_prog_put(prog); 217509756af4SAlexei Starovoitov return 0; 217609756af4SAlexei Starovoitov } 217709756af4SAlexei Starovoitov 217861a0abaeSEric Dumazet struct bpf_prog_kstats { 217961a0abaeSEric Dumazet u64 nsecs; 218061a0abaeSEric Dumazet u64 cnt; 218161a0abaeSEric Dumazet u64 misses; 218261a0abaeSEric Dumazet }; 218361a0abaeSEric Dumazet 218405b24ff9SJiri Olsa void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog) 218505b24ff9SJiri Olsa { 218605b24ff9SJiri Olsa struct bpf_prog_stats *stats; 218705b24ff9SJiri Olsa unsigned int flags; 218805b24ff9SJiri Olsa 218905b24ff9SJiri Olsa stats = this_cpu_ptr(prog->stats); 219005b24ff9SJiri Olsa flags = u64_stats_update_begin_irqsave(&stats->syncp); 219105b24ff9SJiri Olsa u64_stats_inc(&stats->misses); 219205b24ff9SJiri Olsa u64_stats_update_end_irqrestore(&stats->syncp, flags); 219305b24ff9SJiri Olsa } 219405b24ff9SJiri Olsa 2195492ecee8SAlexei Starovoitov static void bpf_prog_get_stats(const struct bpf_prog *prog, 219661a0abaeSEric Dumazet struct bpf_prog_kstats *stats) 2197492ecee8SAlexei Starovoitov { 21989ed9e9baSAlexei Starovoitov u64 nsecs = 0, cnt = 0, misses = 0; 2199492ecee8SAlexei Starovoitov int cpu; 2200492ecee8SAlexei Starovoitov 2201492ecee8SAlexei Starovoitov for_each_possible_cpu(cpu) { 2202492ecee8SAlexei Starovoitov const struct bpf_prog_stats *st; 2203492ecee8SAlexei Starovoitov unsigned int start; 22049ed9e9baSAlexei Starovoitov u64 tnsecs, tcnt, tmisses; 2205492ecee8SAlexei Starovoitov 2206700d4796SAlexei Starovoitov st = per_cpu_ptr(prog->stats, cpu); 2207492ecee8SAlexei Starovoitov do { 220897c4090bSThomas Gleixner start = u64_stats_fetch_begin(&st->syncp); 220961a0abaeSEric Dumazet tnsecs = u64_stats_read(&st->nsecs); 221061a0abaeSEric Dumazet tcnt = u64_stats_read(&st->cnt); 221161a0abaeSEric Dumazet tmisses = u64_stats_read(&st->misses); 221297c4090bSThomas Gleixner } while (u64_stats_fetch_retry(&st->syncp, start)); 2213492ecee8SAlexei Starovoitov nsecs += tnsecs; 2214492ecee8SAlexei Starovoitov cnt += tcnt; 22159ed9e9baSAlexei Starovoitov misses += tmisses; 2216492ecee8SAlexei Starovoitov } 2217492ecee8SAlexei Starovoitov stats->nsecs = nsecs; 2218492ecee8SAlexei Starovoitov stats->cnt = cnt; 22199ed9e9baSAlexei Starovoitov stats->misses = misses; 2220492ecee8SAlexei Starovoitov } 2221492ecee8SAlexei Starovoitov 22227bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 22237bd509e3SDaniel Borkmann static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 22247bd509e3SDaniel Borkmann { 22257bd509e3SDaniel Borkmann const struct bpf_prog *prog = filp->private_data; 2226f1f7714eSDaniel Borkmann char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 222761a0abaeSEric Dumazet struct bpf_prog_kstats stats; 22287bd509e3SDaniel Borkmann 2229492ecee8SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 2230f1f7714eSDaniel Borkmann bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 22317bd509e3SDaniel Borkmann seq_printf(m, 22327bd509e3SDaniel Borkmann "prog_type:\t%u\n" 22337bd509e3SDaniel Borkmann "prog_jited:\t%u\n" 2234f1f7714eSDaniel Borkmann "prog_tag:\t%s\n" 22354316b409SDaniel Borkmann "memlock:\t%llu\n" 2236492ecee8SAlexei Starovoitov "prog_id:\t%u\n" 2237492ecee8SAlexei Starovoitov "run_time_ns:\t%llu\n" 22389ed9e9baSAlexei Starovoitov "run_cnt:\t%llu\n" 2239aba64c7dSDave Marchevsky "recursion_misses:\t%llu\n" 2240aba64c7dSDave Marchevsky "verified_insns:\t%u\n", 22417bd509e3SDaniel Borkmann prog->type, 22427bd509e3SDaniel Borkmann prog->jited, 2243f1f7714eSDaniel Borkmann prog_tag, 22444316b409SDaniel Borkmann prog->pages * 1ULL << PAGE_SHIFT, 2245492ecee8SAlexei Starovoitov prog->aux->id, 2246492ecee8SAlexei Starovoitov stats.nsecs, 22479ed9e9baSAlexei Starovoitov stats.cnt, 2248aba64c7dSDave Marchevsky stats.misses, 2249aba64c7dSDave Marchevsky prog->aux->verified_insns); 22507bd509e3SDaniel Borkmann } 22517bd509e3SDaniel Borkmann #endif 22527bd509e3SDaniel Borkmann 2253f66e448cSChenbo Feng const struct file_operations bpf_prog_fops = { 22547bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 22557bd509e3SDaniel Borkmann .show_fdinfo = bpf_prog_show_fdinfo, 22567bd509e3SDaniel Borkmann #endif 225709756af4SAlexei Starovoitov .release = bpf_prog_release, 22586e71b04aSChenbo Feng .read = bpf_dummy_read, 22596e71b04aSChenbo Feng .write = bpf_dummy_write, 226009756af4SAlexei Starovoitov }; 226109756af4SAlexei Starovoitov 2262b2197755SDaniel Borkmann int bpf_prog_new_fd(struct bpf_prog *prog) 2263aa79781bSDaniel Borkmann { 2264afdb09c7SChenbo Feng int ret; 2265afdb09c7SChenbo Feng 2266afdb09c7SChenbo Feng ret = security_bpf_prog(prog); 2267afdb09c7SChenbo Feng if (ret < 0) 2268afdb09c7SChenbo Feng return ret; 2269afdb09c7SChenbo Feng 2270aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 2271aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC); 2272aa79781bSDaniel Borkmann } 2273aa79781bSDaniel Borkmann 2274113214beSDaniel Borkmann static struct bpf_prog *____bpf_prog_get(struct fd f) 227509756af4SAlexei Starovoitov { 227609756af4SAlexei Starovoitov if (!f.file) 227709756af4SAlexei Starovoitov return ERR_PTR(-EBADF); 227809756af4SAlexei Starovoitov if (f.file->f_op != &bpf_prog_fops) { 227909756af4SAlexei Starovoitov fdput(f); 228009756af4SAlexei Starovoitov return ERR_PTR(-EINVAL); 228109756af4SAlexei Starovoitov } 228209756af4SAlexei Starovoitov 2283c2101297SDaniel Borkmann return f.file->private_data; 228409756af4SAlexei Starovoitov } 228509756af4SAlexei Starovoitov 228685192dbfSAndrii Nakryiko void bpf_prog_add(struct bpf_prog *prog, int i) 228792117d84SAlexei Starovoitov { 228885192dbfSAndrii Nakryiko atomic64_add(i, &prog->aux->refcnt); 228992117d84SAlexei Starovoitov } 229059d3656dSBrenden Blanco EXPORT_SYMBOL_GPL(bpf_prog_add); 229159d3656dSBrenden Blanco 2292c540594fSDaniel Borkmann void bpf_prog_sub(struct bpf_prog *prog, int i) 2293c540594fSDaniel Borkmann { 2294c540594fSDaniel Borkmann /* Only to be used for undoing previous bpf_prog_add() in some 2295c540594fSDaniel Borkmann * error path. We still know that another entity in our call 2296c540594fSDaniel Borkmann * path holds a reference to the program, thus atomic_sub() can 2297c540594fSDaniel Borkmann * be safely used in such cases! 2298c540594fSDaniel Borkmann */ 229985192dbfSAndrii Nakryiko WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0); 2300c540594fSDaniel Borkmann } 2301c540594fSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_sub); 2302c540594fSDaniel Borkmann 230385192dbfSAndrii Nakryiko void bpf_prog_inc(struct bpf_prog *prog) 230459d3656dSBrenden Blanco { 230585192dbfSAndrii Nakryiko atomic64_inc(&prog->aux->refcnt); 230659d3656dSBrenden Blanco } 230797bc402dSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_inc); 230892117d84SAlexei Starovoitov 2309b16d9aa4SMartin KaFai Lau /* prog_idr_lock should have been held */ 2310a6f6df69SJohn Fastabend struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 2311b16d9aa4SMartin KaFai Lau { 2312b16d9aa4SMartin KaFai Lau int refold; 2313b16d9aa4SMartin KaFai Lau 231485192dbfSAndrii Nakryiko refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0); 2315b16d9aa4SMartin KaFai Lau 2316b16d9aa4SMartin KaFai Lau if (!refold) 2317b16d9aa4SMartin KaFai Lau return ERR_PTR(-ENOENT); 2318b16d9aa4SMartin KaFai Lau 2319b16d9aa4SMartin KaFai Lau return prog; 2320b16d9aa4SMartin KaFai Lau } 2321a6f6df69SJohn Fastabend EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 2322b16d9aa4SMartin KaFai Lau 2323040ee692SAl Viro bool bpf_prog_get_ok(struct bpf_prog *prog, 2324288b3de5SJakub Kicinski enum bpf_prog_type *attach_type, bool attach_drv) 2325248f346fSJakub Kicinski { 2326288b3de5SJakub Kicinski /* not an attachment, just a refcount inc, always allow */ 2327288b3de5SJakub Kicinski if (!attach_type) 2328288b3de5SJakub Kicinski return true; 2329248f346fSJakub Kicinski 2330248f346fSJakub Kicinski if (prog->type != *attach_type) 2331248f346fSJakub Kicinski return false; 23329d03ebc7SStanislav Fomichev if (bpf_prog_is_offloaded(prog->aux) && !attach_drv) 2333248f346fSJakub Kicinski return false; 2334248f346fSJakub Kicinski 2335248f346fSJakub Kicinski return true; 2336248f346fSJakub Kicinski } 2337248f346fSJakub Kicinski 2338248f346fSJakub Kicinski static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, 2339288b3de5SJakub Kicinski bool attach_drv) 234009756af4SAlexei Starovoitov { 234109756af4SAlexei Starovoitov struct fd f = fdget(ufd); 234209756af4SAlexei Starovoitov struct bpf_prog *prog; 234309756af4SAlexei Starovoitov 2344113214beSDaniel Borkmann prog = ____bpf_prog_get(f); 234509756af4SAlexei Starovoitov if (IS_ERR(prog)) 234609756af4SAlexei Starovoitov return prog; 2347288b3de5SJakub Kicinski if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) { 2348113214beSDaniel Borkmann prog = ERR_PTR(-EINVAL); 2349113214beSDaniel Borkmann goto out; 2350113214beSDaniel Borkmann } 235109756af4SAlexei Starovoitov 235285192dbfSAndrii Nakryiko bpf_prog_inc(prog); 2353113214beSDaniel Borkmann out: 235409756af4SAlexei Starovoitov fdput(f); 235509756af4SAlexei Starovoitov return prog; 235609756af4SAlexei Starovoitov } 2357113214beSDaniel Borkmann 2358113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get(u32 ufd) 2359113214beSDaniel Borkmann { 2360288b3de5SJakub Kicinski return __bpf_prog_get(ufd, NULL, false); 2361113214beSDaniel Borkmann } 2362113214beSDaniel Borkmann 2363248f346fSJakub Kicinski struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 2364288b3de5SJakub Kicinski bool attach_drv) 2365248f346fSJakub Kicinski { 23664d220ed0SAlexei Starovoitov return __bpf_prog_get(ufd, &type, attach_drv); 2367248f346fSJakub Kicinski } 23686c8dfe21SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev); 2369248f346fSJakub Kicinski 2370aac3fc32SAndrey Ignatov /* Initially all BPF programs could be loaded w/o specifying 2371aac3fc32SAndrey Ignatov * expected_attach_type. Later for some of them specifying expected_attach_type 2372aac3fc32SAndrey Ignatov * at load time became required so that program could be validated properly. 2373aac3fc32SAndrey Ignatov * Programs of types that are allowed to be loaded both w/ and w/o (for 2374aac3fc32SAndrey Ignatov * backward compatibility) expected_attach_type, should have the default attach 2375aac3fc32SAndrey Ignatov * type assigned to expected_attach_type for the latter case, so that it can be 2376aac3fc32SAndrey Ignatov * validated later at attach time. 2377aac3fc32SAndrey Ignatov * 2378aac3fc32SAndrey Ignatov * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if 2379aac3fc32SAndrey Ignatov * prog type requires it but has some attach types that have to be backward 2380aac3fc32SAndrey Ignatov * compatible. 2381aac3fc32SAndrey Ignatov */ 2382aac3fc32SAndrey Ignatov static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) 2383aac3fc32SAndrey Ignatov { 2384aac3fc32SAndrey Ignatov switch (attr->prog_type) { 2385aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 2386aac3fc32SAndrey Ignatov /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't 2387aac3fc32SAndrey Ignatov * exist so checking for non-zero is the way to go here. 2388aac3fc32SAndrey Ignatov */ 2389aac3fc32SAndrey Ignatov if (!attr->expected_attach_type) 2390aac3fc32SAndrey Ignatov attr->expected_attach_type = 2391aac3fc32SAndrey Ignatov BPF_CGROUP_INET_SOCK_CREATE; 2392aac3fc32SAndrey Ignatov break; 2393d5e4ddaeSKuniyuki Iwashima case BPF_PROG_TYPE_SK_REUSEPORT: 2394d5e4ddaeSKuniyuki Iwashima if (!attr->expected_attach_type) 2395d5e4ddaeSKuniyuki Iwashima attr->expected_attach_type = 2396d5e4ddaeSKuniyuki Iwashima BPF_SK_REUSEPORT_SELECT; 2397d5e4ddaeSKuniyuki Iwashima break; 2398aac3fc32SAndrey Ignatov } 2399aac3fc32SAndrey Ignatov } 2400aac3fc32SAndrey Ignatov 24015e43f899SAndrey Ignatov static int 2402ccfe29ebSAlexei Starovoitov bpf_prog_load_check_attach(enum bpf_prog_type prog_type, 2403ccfe29ebSAlexei Starovoitov enum bpf_attach_type expected_attach_type, 2404290248a5SAndrii Nakryiko struct btf *attach_btf, u32 btf_id, 2405290248a5SAndrii Nakryiko struct bpf_prog *dst_prog) 24065e43f899SAndrey Ignatov { 240727ae7997SMartin KaFai Lau if (btf_id) { 2408c108e3c1SAlexei Starovoitov if (btf_id > BTF_MAX_TYPE) 2409c108e3c1SAlexei Starovoitov return -EINVAL; 241027ae7997SMartin KaFai Lau 2411290248a5SAndrii Nakryiko if (!attach_btf && !dst_prog) 2412290248a5SAndrii Nakryiko return -EINVAL; 2413290248a5SAndrii Nakryiko 241427ae7997SMartin KaFai Lau switch (prog_type) { 241527ae7997SMartin KaFai Lau case BPF_PROG_TYPE_TRACING: 24169e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 241727ae7997SMartin KaFai Lau case BPF_PROG_TYPE_STRUCT_OPS: 2418be8704ffSAlexei Starovoitov case BPF_PROG_TYPE_EXT: 2419c108e3c1SAlexei Starovoitov break; 2420c108e3c1SAlexei Starovoitov default: 2421c108e3c1SAlexei Starovoitov return -EINVAL; 2422c108e3c1SAlexei Starovoitov } 242327ae7997SMartin KaFai Lau } 242427ae7997SMartin KaFai Lau 2425290248a5SAndrii Nakryiko if (attach_btf && (!btf_id || dst_prog)) 2426290248a5SAndrii Nakryiko return -EINVAL; 2427290248a5SAndrii Nakryiko 2428290248a5SAndrii Nakryiko if (dst_prog && prog_type != BPF_PROG_TYPE_TRACING && 2429be8704ffSAlexei Starovoitov prog_type != BPF_PROG_TYPE_EXT) 243027ae7997SMartin KaFai Lau return -EINVAL; 2431c108e3c1SAlexei Starovoitov 2432c108e3c1SAlexei Starovoitov switch (prog_type) { 2433aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 2434aac3fc32SAndrey Ignatov switch (expected_attach_type) { 2435aac3fc32SAndrey Ignatov case BPF_CGROUP_INET_SOCK_CREATE: 2436f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 2437aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 2438aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 2439aac3fc32SAndrey Ignatov return 0; 2440aac3fc32SAndrey Ignatov default: 2441aac3fc32SAndrey Ignatov return -EINVAL; 2442aac3fc32SAndrey Ignatov } 24434fbac77dSAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 24444fbac77dSAndrey Ignatov switch (expected_attach_type) { 24454fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 24464fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 2447d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 2448d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 24491b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 24501b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 24511b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 24521b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 24531cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 24541cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 2455983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG: 2456983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG: 24575e43f899SAndrey Ignatov return 0; 24584fbac77dSAndrey Ignatov default: 24594fbac77dSAndrey Ignatov return -EINVAL; 24604fbac77dSAndrey Ignatov } 24615cf1e914Sbrakmo case BPF_PROG_TYPE_CGROUP_SKB: 24625cf1e914Sbrakmo switch (expected_attach_type) { 24635cf1e914Sbrakmo case BPF_CGROUP_INET_INGRESS: 24645cf1e914Sbrakmo case BPF_CGROUP_INET_EGRESS: 24655cf1e914Sbrakmo return 0; 24665cf1e914Sbrakmo default: 24675cf1e914Sbrakmo return -EINVAL; 24685cf1e914Sbrakmo } 24690d01da6aSStanislav Fomichev case BPF_PROG_TYPE_CGROUP_SOCKOPT: 24700d01da6aSStanislav Fomichev switch (expected_attach_type) { 24710d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT: 24720d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT: 24730d01da6aSStanislav Fomichev return 0; 24740d01da6aSStanislav Fomichev default: 24750d01da6aSStanislav Fomichev return -EINVAL; 24760d01da6aSStanislav Fomichev } 2477e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP: 2478e9ddbb77SJakub Sitnicki if (expected_attach_type == BPF_SK_LOOKUP) 2479e9ddbb77SJakub Sitnicki return 0; 2480e9ddbb77SJakub Sitnicki return -EINVAL; 2481d5e4ddaeSKuniyuki Iwashima case BPF_PROG_TYPE_SK_REUSEPORT: 2482d5e4ddaeSKuniyuki Iwashima switch (expected_attach_type) { 2483d5e4ddaeSKuniyuki Iwashima case BPF_SK_REUSEPORT_SELECT: 2484d5e4ddaeSKuniyuki Iwashima case BPF_SK_REUSEPORT_SELECT_OR_MIGRATE: 2485d5e4ddaeSKuniyuki Iwashima return 0; 2486d5e4ddaeSKuniyuki Iwashima default: 2487d5e4ddaeSKuniyuki Iwashima return -EINVAL; 2488d5e4ddaeSKuniyuki Iwashima } 2489132328e8SFlorian Westphal case BPF_PROG_TYPE_NETFILTER: 2490132328e8SFlorian Westphal if (expected_attach_type == BPF_NETFILTER) 2491132328e8SFlorian Westphal return 0; 2492132328e8SFlorian Westphal return -EINVAL; 249379a7f8bdSAlexei Starovoitov case BPF_PROG_TYPE_SYSCALL: 2494be8704ffSAlexei Starovoitov case BPF_PROG_TYPE_EXT: 2495be8704ffSAlexei Starovoitov if (expected_attach_type) 2496be8704ffSAlexei Starovoitov return -EINVAL; 2497df561f66SGustavo A. R. Silva fallthrough; 24984fbac77dSAndrey Ignatov default: 24994fbac77dSAndrey Ignatov return 0; 25004fbac77dSAndrey Ignatov } 25015e43f899SAndrey Ignatov } 25025e43f899SAndrey Ignatov 25032c78ee89SAlexei Starovoitov static bool is_net_admin_prog_type(enum bpf_prog_type prog_type) 25042c78ee89SAlexei Starovoitov { 25052c78ee89SAlexei Starovoitov switch (prog_type) { 25062c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SCHED_CLS: 25072c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SCHED_ACT: 25082c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_XDP: 25092c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_IN: 25102c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_OUT: 25112c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_XMIT: 25122c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_SEG6LOCAL: 25132c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_SKB: 25142c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_MSG: 25152c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_FLOW_DISSECTOR: 25162c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_DEVICE: 25172c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCK: 25182c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 25192c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCKOPT: 25202c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SYSCTL: 25212c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SOCK_OPS: 25222c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_EXT: /* extends any prog */ 252384601d6eSFlorian Westphal case BPF_PROG_TYPE_NETFILTER: 25242c78ee89SAlexei Starovoitov return true; 25252c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SKB: 25262c78ee89SAlexei Starovoitov /* always unpriv */ 25272c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_REUSEPORT: 25282c78ee89SAlexei Starovoitov /* equivalent to SOCKET_FILTER. need CAP_BPF only */ 25292c78ee89SAlexei Starovoitov default: 25302c78ee89SAlexei Starovoitov return false; 25312c78ee89SAlexei Starovoitov } 25322c78ee89SAlexei Starovoitov } 25332c78ee89SAlexei Starovoitov 25342c78ee89SAlexei Starovoitov static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) 25352c78ee89SAlexei Starovoitov { 25362c78ee89SAlexei Starovoitov switch (prog_type) { 25372c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_KPROBE: 25382c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_TRACEPOINT: 25392c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_PERF_EVENT: 25402c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_RAW_TRACEPOINT: 25412c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 25422c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_TRACING: 25432c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LSM: 25442c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_STRUCT_OPS: /* has access to struct sock */ 25452c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_EXT: /* extends any prog */ 25462c78ee89SAlexei Starovoitov return true; 25472c78ee89SAlexei Starovoitov default: 25482c78ee89SAlexei Starovoitov return false; 25492c78ee89SAlexei Starovoitov } 25502c78ee89SAlexei Starovoitov } 25512c78ee89SAlexei Starovoitov 255209756af4SAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 255347a71c1fSAndrii Nakryiko #define BPF_PROG_LOAD_LAST_FIELD log_true_size 255409756af4SAlexei Starovoitov 255547a71c1fSAndrii Nakryiko static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) 255609756af4SAlexei Starovoitov { 255709756af4SAlexei Starovoitov enum bpf_prog_type type = attr->prog_type; 2558290248a5SAndrii Nakryiko struct bpf_prog *prog, *dst_prog = NULL; 2559290248a5SAndrii Nakryiko struct btf *attach_btf = NULL; 256009756af4SAlexei Starovoitov int err; 256109756af4SAlexei Starovoitov char license[128]; 256209756af4SAlexei Starovoitov 256309756af4SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_LOAD)) 256409756af4SAlexei Starovoitov return -EINVAL; 256509756af4SAlexei Starovoitov 2566c240eff6SJiong Wang if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | 2567c240eff6SJiong Wang BPF_F_ANY_ALIGNMENT | 256810d274e8SAlexei Starovoitov BPF_F_TEST_STATE_FREQ | 25691e6c62a8SAlexei Starovoitov BPF_F_SLEEPABLE | 2570c2f2cdbeSLorenzo Bianconi BPF_F_TEST_RND_HI32 | 25712b3486bcSStanislav Fomichev BPF_F_XDP_HAS_FRAGS | 25722b3486bcSStanislav Fomichev BPF_F_XDP_DEV_BOUND_ONLY)) 2573e07b98d9SDavid S. Miller return -EINVAL; 2574e07b98d9SDavid S. Miller 2575e9ee9efcSDavid Miller if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && 2576e9ee9efcSDavid Miller (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && 25772c78ee89SAlexei Starovoitov !bpf_capable()) 2578e9ee9efcSDavid Miller return -EPERM; 2579e9ee9efcSDavid Miller 25801d28635aSAndrii Nakryiko /* Intent here is for unprivileged_bpf_disabled to block BPF program 25811d28635aSAndrii Nakryiko * creation for unprivileged users; other actions depend 25821d28635aSAndrii Nakryiko * on fd availability and access to bpffs, so are dependent on 25831d28635aSAndrii Nakryiko * object creation success. Even with unprivileged BPF disabled, 25841d28635aSAndrii Nakryiko * capability checks are still carried out for these 25851d28635aSAndrii Nakryiko * and other operations. 25861d28635aSAndrii Nakryiko */ 25871d28635aSAndrii Nakryiko if (sysctl_unprivileged_bpf_disabled && !bpf_capable()) 25881d28635aSAndrii Nakryiko return -EPERM; 258909756af4SAlexei Starovoitov 2590c04c0d2bSAlexei Starovoitov if (attr->insn_cnt == 0 || 25912c78ee89SAlexei Starovoitov attr->insn_cnt > (bpf_capable() ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) 2592ef0915caSDaniel Borkmann return -E2BIG; 259380b7d819SChenbo Feng if (type != BPF_PROG_TYPE_SOCKET_FILTER && 259480b7d819SChenbo Feng type != BPF_PROG_TYPE_CGROUP_SKB && 25952c78ee89SAlexei Starovoitov !bpf_capable()) 25962c78ee89SAlexei Starovoitov return -EPERM; 25972c78ee89SAlexei Starovoitov 2598b338cb92SMaciej Żenczykowski if (is_net_admin_prog_type(type) && !capable(CAP_NET_ADMIN) && !capable(CAP_SYS_ADMIN)) 25992c78ee89SAlexei Starovoitov return -EPERM; 26002c78ee89SAlexei Starovoitov if (is_perfmon_prog_type(type) && !perfmon_capable()) 26011be7f75dSAlexei Starovoitov return -EPERM; 26021be7f75dSAlexei Starovoitov 2603290248a5SAndrii Nakryiko /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog 2604290248a5SAndrii Nakryiko * or btf, we need to check which one it is 2605290248a5SAndrii Nakryiko */ 2606290248a5SAndrii Nakryiko if (attr->attach_prog_fd) { 2607290248a5SAndrii Nakryiko dst_prog = bpf_prog_get(attr->attach_prog_fd); 2608290248a5SAndrii Nakryiko if (IS_ERR(dst_prog)) { 2609290248a5SAndrii Nakryiko dst_prog = NULL; 2610290248a5SAndrii Nakryiko attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd); 2611290248a5SAndrii Nakryiko if (IS_ERR(attach_btf)) 2612290248a5SAndrii Nakryiko return -EINVAL; 2613290248a5SAndrii Nakryiko if (!btf_is_kernel(attach_btf)) { 26148bdd8e27SAndrii Nakryiko /* attaching through specifying bpf_prog's BTF 26158bdd8e27SAndrii Nakryiko * objects directly might be supported eventually 26168bdd8e27SAndrii Nakryiko */ 2617290248a5SAndrii Nakryiko btf_put(attach_btf); 26188bdd8e27SAndrii Nakryiko return -ENOTSUPP; 2619290248a5SAndrii Nakryiko } 2620290248a5SAndrii Nakryiko } 2621290248a5SAndrii Nakryiko } else if (attr->attach_btf_id) { 2622290248a5SAndrii Nakryiko /* fall back to vmlinux BTF, if BTF type ID is specified */ 2623290248a5SAndrii Nakryiko attach_btf = bpf_get_btf_vmlinux(); 2624290248a5SAndrii Nakryiko if (IS_ERR(attach_btf)) 2625290248a5SAndrii Nakryiko return PTR_ERR(attach_btf); 2626290248a5SAndrii Nakryiko if (!attach_btf) 2627290248a5SAndrii Nakryiko return -EINVAL; 2628290248a5SAndrii Nakryiko btf_get(attach_btf); 2629290248a5SAndrii Nakryiko } 2630290248a5SAndrii Nakryiko 2631aac3fc32SAndrey Ignatov bpf_prog_load_fixup_attach_type(attr); 2632ccfe29ebSAlexei Starovoitov if (bpf_prog_load_check_attach(type, attr->expected_attach_type, 2633290248a5SAndrii Nakryiko attach_btf, attr->attach_btf_id, 2634290248a5SAndrii Nakryiko dst_prog)) { 2635290248a5SAndrii Nakryiko if (dst_prog) 2636290248a5SAndrii Nakryiko bpf_prog_put(dst_prog); 2637290248a5SAndrii Nakryiko if (attach_btf) 2638290248a5SAndrii Nakryiko btf_put(attach_btf); 26395e43f899SAndrey Ignatov return -EINVAL; 2640290248a5SAndrii Nakryiko } 26415e43f899SAndrey Ignatov 264209756af4SAlexei Starovoitov /* plain bpf_prog allocation */ 264309756af4SAlexei Starovoitov prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 2644290248a5SAndrii Nakryiko if (!prog) { 2645290248a5SAndrii Nakryiko if (dst_prog) 2646290248a5SAndrii Nakryiko bpf_prog_put(dst_prog); 2647290248a5SAndrii Nakryiko if (attach_btf) 2648290248a5SAndrii Nakryiko btf_put(attach_btf); 264909756af4SAlexei Starovoitov return -ENOMEM; 2650290248a5SAndrii Nakryiko } 265109756af4SAlexei Starovoitov 26525e43f899SAndrey Ignatov prog->expected_attach_type = attr->expected_attach_type; 2653290248a5SAndrii Nakryiko prog->aux->attach_btf = attach_btf; 2654ccfe29ebSAlexei Starovoitov prog->aux->attach_btf_id = attr->attach_btf_id; 26553aac1eadSToke Høiland-Jørgensen prog->aux->dst_prog = dst_prog; 26562b3486bcSStanislav Fomichev prog->aux->dev_bound = !!attr->prog_ifindex; 26571e6c62a8SAlexei Starovoitov prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE; 2658c2f2cdbeSLorenzo Bianconi prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; 26599a18eedbSJakub Kicinski 2660afdb09c7SChenbo Feng err = security_bpf_prog_alloc(prog->aux); 2661aaac3ba9SAlexei Starovoitov if (err) 26623ac1f01bSRoman Gushchin goto free_prog; 2663aaac3ba9SAlexei Starovoitov 26643ac1f01bSRoman Gushchin prog->aux->user = get_current_user(); 266509756af4SAlexei Starovoitov prog->len = attr->insn_cnt; 266609756af4SAlexei Starovoitov 266709756af4SAlexei Starovoitov err = -EFAULT; 2668af2ac3e1SAlexei Starovoitov if (copy_from_bpfptr(prog->insns, 2669af2ac3e1SAlexei Starovoitov make_bpfptr(attr->insns, uattr.is_kernel), 2670aafe6ae9SDaniel Borkmann bpf_prog_insn_size(prog)) != 0) 26713ac1f01bSRoman Gushchin goto free_prog_sec; 26727f6719f7SAndrii Nakryiko /* copy eBPF program license from user space */ 26737f6719f7SAndrii Nakryiko if (strncpy_from_bpfptr(license, 26747f6719f7SAndrii Nakryiko make_bpfptr(attr->license, uattr.is_kernel), 26757f6719f7SAndrii Nakryiko sizeof(license) - 1) < 0) 26767f6719f7SAndrii Nakryiko goto free_prog_sec; 26777f6719f7SAndrii Nakryiko license[sizeof(license) - 1] = 0; 26787f6719f7SAndrii Nakryiko 26797f6719f7SAndrii Nakryiko /* eBPF programs must be GPL compatible to use GPL-ed functions */ 26807f6719f7SAndrii Nakryiko prog->gpl_compatible = license_is_gpl_compatible(license) ? 1 : 0; 268109756af4SAlexei Starovoitov 268209756af4SAlexei Starovoitov prog->orig_prog = NULL; 2683a91263d5SDaniel Borkmann prog->jited = 0; 268409756af4SAlexei Starovoitov 268585192dbfSAndrii Nakryiko atomic64_set(&prog->aux->refcnt, 1); 268609756af4SAlexei Starovoitov 26879a18eedbSJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 26882b3486bcSStanislav Fomichev err = bpf_prog_dev_bound_init(prog, attr); 2689ab3f0063SJakub Kicinski if (err) 26903ac1f01bSRoman Gushchin goto free_prog_sec; 2691ab3f0063SJakub Kicinski } 2692ab3f0063SJakub Kicinski 2693fd7c211dSToke Høiland-Jørgensen if (type == BPF_PROG_TYPE_EXT && dst_prog && 2694fd7c211dSToke Høiland-Jørgensen bpf_prog_is_dev_bound(dst_prog->aux)) { 2695fd7c211dSToke Høiland-Jørgensen err = bpf_prog_dev_bound_inherit(prog, dst_prog); 2696cb4d2b3fSMartin KaFai Lau if (err) 2697cb4d2b3fSMartin KaFai Lau goto free_prog_sec; 2698cb4d2b3fSMartin KaFai Lau } 2699cb4d2b3fSMartin KaFai Lau 2700cb4d2b3fSMartin KaFai Lau /* find program type: socket_filter vs tracing_filter */ 270109756af4SAlexei Starovoitov err = find_prog_type(type, prog); 27029bac3d6dSAlexei Starovoitov if (err < 0) 27033ac1f01bSRoman Gushchin goto free_prog_sec; 270409756af4SAlexei Starovoitov 27059285ec4cSJason A. Donenfeld prog->aux->load_time = ktime_get_boottime_ns(); 27068e7ae251SMartin KaFai Lau err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, 27078e7ae251SMartin KaFai Lau sizeof(attr->prog_name)); 27088e7ae251SMartin KaFai Lau if (err < 0) 27093ac1f01bSRoman Gushchin goto free_prog_sec; 271009756af4SAlexei Starovoitov 271151580e79SAlexei Starovoitov /* run eBPF verifier */ 271247a71c1fSAndrii Nakryiko err = bpf_check(&prog, attr, uattr, uattr_size); 271309756af4SAlexei Starovoitov if (err < 0) 271409756af4SAlexei Starovoitov goto free_used_maps; 271509756af4SAlexei Starovoitov 2716d1c55ab5SDaniel Borkmann prog = bpf_prog_select_runtime(prog, &err); 271704fd61abSAlexei Starovoitov if (err < 0) 271804fd61abSAlexei Starovoitov goto free_used_maps; 271909756af4SAlexei Starovoitov 2720dc4bb0e2SMartin KaFai Lau err = bpf_prog_alloc_id(prog); 2721dc4bb0e2SMartin KaFai Lau if (err) 2722dc4bb0e2SMartin KaFai Lau goto free_used_maps; 2723dc4bb0e2SMartin KaFai Lau 2724c751798aSDaniel Borkmann /* Upon success of bpf_prog_alloc_id(), the BPF prog is 2725c751798aSDaniel Borkmann * effectively publicly exposed. However, retrieving via 2726c751798aSDaniel Borkmann * bpf_prog_get_fd_by_id() will take another reference, 2727c751798aSDaniel Borkmann * therefore it cannot be gone underneath us. 2728c751798aSDaniel Borkmann * 2729c751798aSDaniel Borkmann * Only for the time /after/ successful bpf_prog_new_fd() 2730c751798aSDaniel Borkmann * and before returning to userspace, we might just hold 2731c751798aSDaniel Borkmann * one reference and any parallel close on that fd could 2732c751798aSDaniel Borkmann * rip everything out. Hence, below notifications must 2733c751798aSDaniel Borkmann * happen before bpf_prog_new_fd(). 2734c751798aSDaniel Borkmann * 2735c751798aSDaniel Borkmann * Also, any failure handling from this point onwards must 2736c751798aSDaniel Borkmann * be using bpf_prog_put() given the program is exposed. 2737b16d9aa4SMartin KaFai Lau */ 273874451e66SDaniel Borkmann bpf_prog_kallsyms_add(prog); 27396ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); 2740bae141f5SDaniel Borkmann bpf_audit_prog(prog, BPF_AUDIT_LOAD); 2741c751798aSDaniel Borkmann 2742c751798aSDaniel Borkmann err = bpf_prog_new_fd(prog); 2743c751798aSDaniel Borkmann if (err < 0) 2744c751798aSDaniel Borkmann bpf_prog_put(prog); 274509756af4SAlexei Starovoitov return err; 274609756af4SAlexei Starovoitov 274709756af4SAlexei Starovoitov free_used_maps: 2748cd7455f1SDaniel Borkmann /* In case we have subprogs, we need to wait for a grace 2749cd7455f1SDaniel Borkmann * period before we can tear down JIT memory since symbols 2750cd7455f1SDaniel Borkmann * are already exposed under kallsyms. 2751cd7455f1SDaniel Borkmann */ 2752cd7455f1SDaniel Borkmann __bpf_prog_put_noref(prog, prog->aux->func_cnt); 2753cd7455f1SDaniel Borkmann return err; 2754afdb09c7SChenbo Feng free_prog_sec: 27553ac1f01bSRoman Gushchin free_uid(prog->aux->user); 2756afdb09c7SChenbo Feng security_bpf_prog_free(prog->aux); 27573ac1f01bSRoman Gushchin free_prog: 275822dc4a0fSAndrii Nakryiko if (prog->aux->attach_btf) 275922dc4a0fSAndrii Nakryiko btf_put(prog->aux->attach_btf); 276009756af4SAlexei Starovoitov bpf_prog_free(prog); 276109756af4SAlexei Starovoitov return err; 276209756af4SAlexei Starovoitov } 276309756af4SAlexei Starovoitov 2764cb8edce2SAndrii Nakryiko #define BPF_OBJ_LAST_FIELD path_fd 2765b2197755SDaniel Borkmann 2766b2197755SDaniel Borkmann static int bpf_obj_pin(const union bpf_attr *attr) 2767b2197755SDaniel Borkmann { 2768cb8edce2SAndrii Nakryiko int path_fd; 2769cb8edce2SAndrii Nakryiko 2770cb8edce2SAndrii Nakryiko if (CHECK_ATTR(BPF_OBJ) || attr->file_flags & ~BPF_F_PATH_FD) 2771b2197755SDaniel Borkmann return -EINVAL; 2772b2197755SDaniel Borkmann 2773cb8edce2SAndrii Nakryiko /* path_fd has to be accompanied by BPF_F_PATH_FD flag */ 2774cb8edce2SAndrii Nakryiko if (!(attr->file_flags & BPF_F_PATH_FD) && attr->path_fd) 2775cb8edce2SAndrii Nakryiko return -EINVAL; 2776cb8edce2SAndrii Nakryiko 2777cb8edce2SAndrii Nakryiko path_fd = attr->file_flags & BPF_F_PATH_FD ? attr->path_fd : AT_FDCWD; 2778cb8edce2SAndrii Nakryiko return bpf_obj_pin_user(attr->bpf_fd, path_fd, 2779cb8edce2SAndrii Nakryiko u64_to_user_ptr(attr->pathname)); 2780b2197755SDaniel Borkmann } 2781b2197755SDaniel Borkmann 2782b2197755SDaniel Borkmann static int bpf_obj_get(const union bpf_attr *attr) 2783b2197755SDaniel Borkmann { 2784cb8edce2SAndrii Nakryiko int path_fd; 2785cb8edce2SAndrii Nakryiko 27866e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || 2787cb8edce2SAndrii Nakryiko attr->file_flags & ~(BPF_OBJ_FLAG_MASK | BPF_F_PATH_FD)) 2788b2197755SDaniel Borkmann return -EINVAL; 2789b2197755SDaniel Borkmann 2790cb8edce2SAndrii Nakryiko /* path_fd has to be accompanied by BPF_F_PATH_FD flag */ 2791cb8edce2SAndrii Nakryiko if (!(attr->file_flags & BPF_F_PATH_FD) && attr->path_fd) 2792cb8edce2SAndrii Nakryiko return -EINVAL; 2793cb8edce2SAndrii Nakryiko 2794cb8edce2SAndrii Nakryiko path_fd = attr->file_flags & BPF_F_PATH_FD ? attr->path_fd : AT_FDCWD; 2795cb8edce2SAndrii Nakryiko return bpf_obj_get_user(path_fd, u64_to_user_ptr(attr->pathname), 27966e71b04aSChenbo Feng attr->file_flags); 2797b2197755SDaniel Borkmann } 2798b2197755SDaniel Borkmann 2799f2e10bffSAndrii Nakryiko void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 2800a3b80e10SAndrii Nakryiko const struct bpf_link_ops *ops, struct bpf_prog *prog) 280170ed506cSAndrii Nakryiko { 280270ed506cSAndrii Nakryiko atomic64_set(&link->refcnt, 1); 2803f2e10bffSAndrii Nakryiko link->type = type; 2804a3b80e10SAndrii Nakryiko link->id = 0; 280570ed506cSAndrii Nakryiko link->ops = ops; 280670ed506cSAndrii Nakryiko link->prog = prog; 280770ed506cSAndrii Nakryiko } 280870ed506cSAndrii Nakryiko 2809a3b80e10SAndrii Nakryiko static void bpf_link_free_id(int id) 2810a3b80e10SAndrii Nakryiko { 2811a3b80e10SAndrii Nakryiko if (!id) 2812a3b80e10SAndrii Nakryiko return; 2813a3b80e10SAndrii Nakryiko 2814a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 2815a3b80e10SAndrii Nakryiko idr_remove(&link_idr, id); 2816a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 2817a3b80e10SAndrii Nakryiko } 2818a3b80e10SAndrii Nakryiko 281998868668SAndrii Nakryiko /* Clean up bpf_link and corresponding anon_inode file and FD. After 282098868668SAndrii Nakryiko * anon_inode is created, bpf_link can't be just kfree()'d due to deferred 2821a3b80e10SAndrii Nakryiko * anon_inode's release() call. This helper marks bpf_link as 2822a3b80e10SAndrii Nakryiko * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt 2823a3b80e10SAndrii Nakryiko * is not decremented, it's the responsibility of a calling code that failed 2824a3b80e10SAndrii Nakryiko * to complete bpf_link initialization. 282589ae89f5SJiri Olsa * This helper eventually calls link's dealloc callback, but does not call 282689ae89f5SJiri Olsa * link's release callback. 282798868668SAndrii Nakryiko */ 2828a3b80e10SAndrii Nakryiko void bpf_link_cleanup(struct bpf_link_primer *primer) 2829babf3164SAndrii Nakryiko { 2830a3b80e10SAndrii Nakryiko primer->link->prog = NULL; 2831a3b80e10SAndrii Nakryiko bpf_link_free_id(primer->id); 2832a3b80e10SAndrii Nakryiko fput(primer->file); 2833a3b80e10SAndrii Nakryiko put_unused_fd(primer->fd); 2834babf3164SAndrii Nakryiko } 2835babf3164SAndrii Nakryiko 283670ed506cSAndrii Nakryiko void bpf_link_inc(struct bpf_link *link) 283770ed506cSAndrii Nakryiko { 283870ed506cSAndrii Nakryiko atomic64_inc(&link->refcnt); 283970ed506cSAndrii Nakryiko } 284070ed506cSAndrii Nakryiko 284170ed506cSAndrii Nakryiko /* bpf_link_free is guaranteed to be called from process context */ 284270ed506cSAndrii Nakryiko static void bpf_link_free(struct bpf_link *link) 284370ed506cSAndrii Nakryiko { 2844a3b80e10SAndrii Nakryiko bpf_link_free_id(link->id); 2845babf3164SAndrii Nakryiko if (link->prog) { 2846babf3164SAndrii Nakryiko /* detach BPF program, clean up used resources */ 284770ed506cSAndrii Nakryiko link->ops->release(link); 2848babf3164SAndrii Nakryiko bpf_prog_put(link->prog); 2849babf3164SAndrii Nakryiko } 2850babf3164SAndrii Nakryiko /* free bpf_link and its containing memory */ 2851babf3164SAndrii Nakryiko link->ops->dealloc(link); 285270ed506cSAndrii Nakryiko } 285370ed506cSAndrii Nakryiko 285470ed506cSAndrii Nakryiko static void bpf_link_put_deferred(struct work_struct *work) 285570ed506cSAndrii Nakryiko { 285670ed506cSAndrii Nakryiko struct bpf_link *link = container_of(work, struct bpf_link, work); 285770ed506cSAndrii Nakryiko 285870ed506cSAndrii Nakryiko bpf_link_free(link); 285970ed506cSAndrii Nakryiko } 286070ed506cSAndrii Nakryiko 2861ab5d47bdSSebastian Andrzej Siewior /* bpf_link_put might be called from atomic context. It needs to be called 2862ab5d47bdSSebastian Andrzej Siewior * from sleepable context in order to acquire sleeping locks during the process. 286370ed506cSAndrii Nakryiko */ 286470ed506cSAndrii Nakryiko void bpf_link_put(struct bpf_link *link) 286570ed506cSAndrii Nakryiko { 286670ed506cSAndrii Nakryiko if (!atomic64_dec_and_test(&link->refcnt)) 286770ed506cSAndrii Nakryiko return; 286870ed506cSAndrii Nakryiko 286970ed506cSAndrii Nakryiko INIT_WORK(&link->work, bpf_link_put_deferred); 287070ed506cSAndrii Nakryiko schedule_work(&link->work); 287170ed506cSAndrii Nakryiko } 2872cb80ddc6SAlexei Starovoitov EXPORT_SYMBOL(bpf_link_put); 287370ed506cSAndrii Nakryiko 2874ab5d47bdSSebastian Andrzej Siewior static void bpf_link_put_direct(struct bpf_link *link) 2875ab5d47bdSSebastian Andrzej Siewior { 2876ab5d47bdSSebastian Andrzej Siewior if (!atomic64_dec_and_test(&link->refcnt)) 2877ab5d47bdSSebastian Andrzej Siewior return; 2878ab5d47bdSSebastian Andrzej Siewior bpf_link_free(link); 2879ab5d47bdSSebastian Andrzej Siewior } 2880ab5d47bdSSebastian Andrzej Siewior 288170ed506cSAndrii Nakryiko static int bpf_link_release(struct inode *inode, struct file *filp) 288270ed506cSAndrii Nakryiko { 288370ed506cSAndrii Nakryiko struct bpf_link *link = filp->private_data; 288470ed506cSAndrii Nakryiko 2885ab5d47bdSSebastian Andrzej Siewior bpf_link_put_direct(link); 2886fec56f58SAlexei Starovoitov return 0; 2887fec56f58SAlexei Starovoitov } 2888fec56f58SAlexei Starovoitov 288970ed506cSAndrii Nakryiko #ifdef CONFIG_PROC_FS 2890f2e10bffSAndrii Nakryiko #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) 2891f2e10bffSAndrii Nakryiko #define BPF_MAP_TYPE(_id, _ops) 2892f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) [_id] = #_name, 2893f2e10bffSAndrii Nakryiko static const char *bpf_link_type_strs[] = { 2894f2e10bffSAndrii Nakryiko [BPF_LINK_TYPE_UNSPEC] = "<invalid>", 2895f2e10bffSAndrii Nakryiko #include <linux/bpf_types.h> 2896f2e10bffSAndrii Nakryiko }; 2897f2e10bffSAndrii Nakryiko #undef BPF_PROG_TYPE 2898f2e10bffSAndrii Nakryiko #undef BPF_MAP_TYPE 2899f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 290070ed506cSAndrii Nakryiko 290170ed506cSAndrii Nakryiko static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) 290270ed506cSAndrii Nakryiko { 290370ed506cSAndrii Nakryiko const struct bpf_link *link = filp->private_data; 290470ed506cSAndrii Nakryiko const struct bpf_prog *prog = link->prog; 290570ed506cSAndrii Nakryiko char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 290670ed506cSAndrii Nakryiko 290770ed506cSAndrii Nakryiko seq_printf(m, 290870ed506cSAndrii Nakryiko "link_type:\t%s\n" 290968b04864SKui-Feng Lee "link_id:\t%u\n", 291068b04864SKui-Feng Lee bpf_link_type_strs[link->type], 291168b04864SKui-Feng Lee link->id); 291268b04864SKui-Feng Lee if (prog) { 291368b04864SKui-Feng Lee bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 291468b04864SKui-Feng Lee seq_printf(m, 291570ed506cSAndrii Nakryiko "prog_tag:\t%s\n" 291670ed506cSAndrii Nakryiko "prog_id:\t%u\n", 291770ed506cSAndrii Nakryiko prog_tag, 291870ed506cSAndrii Nakryiko prog->aux->id); 291968b04864SKui-Feng Lee } 2920f2e10bffSAndrii Nakryiko if (link->ops->show_fdinfo) 2921f2e10bffSAndrii Nakryiko link->ops->show_fdinfo(link, m); 292270ed506cSAndrii Nakryiko } 292370ed506cSAndrii Nakryiko #endif 292470ed506cSAndrii Nakryiko 29256f302bfbSZou Wei static const struct file_operations bpf_link_fops = { 292670ed506cSAndrii Nakryiko #ifdef CONFIG_PROC_FS 292770ed506cSAndrii Nakryiko .show_fdinfo = bpf_link_show_fdinfo, 292870ed506cSAndrii Nakryiko #endif 292970ed506cSAndrii Nakryiko .release = bpf_link_release, 2930fec56f58SAlexei Starovoitov .read = bpf_dummy_read, 2931fec56f58SAlexei Starovoitov .write = bpf_dummy_write, 2932fec56f58SAlexei Starovoitov }; 2933fec56f58SAlexei Starovoitov 2934a3b80e10SAndrii Nakryiko static int bpf_link_alloc_id(struct bpf_link *link) 293570ed506cSAndrii Nakryiko { 2936a3b80e10SAndrii Nakryiko int id; 2937a3b80e10SAndrii Nakryiko 2938a3b80e10SAndrii Nakryiko idr_preload(GFP_KERNEL); 2939a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 2940a3b80e10SAndrii Nakryiko id = idr_alloc_cyclic(&link_idr, link, 1, INT_MAX, GFP_ATOMIC); 2941a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 2942a3b80e10SAndrii Nakryiko idr_preload_end(); 2943a3b80e10SAndrii Nakryiko 2944a3b80e10SAndrii Nakryiko return id; 294570ed506cSAndrii Nakryiko } 294670ed506cSAndrii Nakryiko 2947a3b80e10SAndrii Nakryiko /* Prepare bpf_link to be exposed to user-space by allocating anon_inode file, 2948a3b80e10SAndrii Nakryiko * reserving unused FD and allocating ID from link_idr. This is to be paired 2949a3b80e10SAndrii Nakryiko * with bpf_link_settle() to install FD and ID and expose bpf_link to 2950a3b80e10SAndrii Nakryiko * user-space, if bpf_link is successfully attached. If not, bpf_link and 2951a3b80e10SAndrii Nakryiko * pre-allocated resources are to be freed with bpf_cleanup() call. All the 2952a3b80e10SAndrii Nakryiko * transient state is passed around in struct bpf_link_primer. 2953a3b80e10SAndrii Nakryiko * This is preferred way to create and initialize bpf_link, especially when 2954a3b80e10SAndrii Nakryiko * there are complicated and expensive operations in between creating bpf_link 2955a3b80e10SAndrii Nakryiko * itself and attaching it to BPF hook. By using bpf_link_prime() and 2956a3b80e10SAndrii Nakryiko * bpf_link_settle() kernel code using bpf_link doesn't have to perform 2957a3b80e10SAndrii Nakryiko * expensive (and potentially failing) roll back operations in a rare case 2958a3b80e10SAndrii Nakryiko * that file, FD, or ID can't be allocated. 2959babf3164SAndrii Nakryiko */ 2960a3b80e10SAndrii Nakryiko int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer) 2961babf3164SAndrii Nakryiko { 2962babf3164SAndrii Nakryiko struct file *file; 2963a3b80e10SAndrii Nakryiko int fd, id; 2964babf3164SAndrii Nakryiko 2965babf3164SAndrii Nakryiko fd = get_unused_fd_flags(O_CLOEXEC); 2966babf3164SAndrii Nakryiko if (fd < 0) 2967a3b80e10SAndrii Nakryiko return fd; 2968babf3164SAndrii Nakryiko 2969babf3164SAndrii Nakryiko 2970a3b80e10SAndrii Nakryiko id = bpf_link_alloc_id(link); 2971a3b80e10SAndrii Nakryiko if (id < 0) { 2972a3b80e10SAndrii Nakryiko put_unused_fd(fd); 2973a3b80e10SAndrii Nakryiko return id; 2974a3b80e10SAndrii Nakryiko } 2975babf3164SAndrii Nakryiko 2976babf3164SAndrii Nakryiko file = anon_inode_getfile("bpf_link", &bpf_link_fops, link, O_CLOEXEC); 2977babf3164SAndrii Nakryiko if (IS_ERR(file)) { 2978138c6767SAndrii Nakryiko bpf_link_free_id(id); 2979babf3164SAndrii Nakryiko put_unused_fd(fd); 2980138c6767SAndrii Nakryiko return PTR_ERR(file); 2981babf3164SAndrii Nakryiko } 2982babf3164SAndrii Nakryiko 2983a3b80e10SAndrii Nakryiko primer->link = link; 2984a3b80e10SAndrii Nakryiko primer->file = file; 2985a3b80e10SAndrii Nakryiko primer->fd = fd; 2986a3b80e10SAndrii Nakryiko primer->id = id; 2987a3b80e10SAndrii Nakryiko return 0; 2988a3b80e10SAndrii Nakryiko } 2989a3b80e10SAndrii Nakryiko 2990a3b80e10SAndrii Nakryiko int bpf_link_settle(struct bpf_link_primer *primer) 2991a3b80e10SAndrii Nakryiko { 2992a3b80e10SAndrii Nakryiko /* make bpf_link fetchable by ID */ 2993a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 2994a3b80e10SAndrii Nakryiko primer->link->id = primer->id; 2995a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 2996a3b80e10SAndrii Nakryiko /* make bpf_link fetchable by FD */ 2997a3b80e10SAndrii Nakryiko fd_install(primer->fd, primer->file); 2998a3b80e10SAndrii Nakryiko /* pass through installed FD */ 2999a3b80e10SAndrii Nakryiko return primer->fd; 3000a3b80e10SAndrii Nakryiko } 3001a3b80e10SAndrii Nakryiko 3002a3b80e10SAndrii Nakryiko int bpf_link_new_fd(struct bpf_link *link) 3003a3b80e10SAndrii Nakryiko { 3004a3b80e10SAndrii Nakryiko return anon_inode_getfd("bpf-link", &bpf_link_fops, link, O_CLOEXEC); 3005babf3164SAndrii Nakryiko } 3006babf3164SAndrii Nakryiko 300770ed506cSAndrii Nakryiko struct bpf_link *bpf_link_get_from_fd(u32 ufd) 300870ed506cSAndrii Nakryiko { 300970ed506cSAndrii Nakryiko struct fd f = fdget(ufd); 301070ed506cSAndrii Nakryiko struct bpf_link *link; 301170ed506cSAndrii Nakryiko 301270ed506cSAndrii Nakryiko if (!f.file) 301370ed506cSAndrii Nakryiko return ERR_PTR(-EBADF); 301470ed506cSAndrii Nakryiko if (f.file->f_op != &bpf_link_fops) { 301570ed506cSAndrii Nakryiko fdput(f); 301670ed506cSAndrii Nakryiko return ERR_PTR(-EINVAL); 301770ed506cSAndrii Nakryiko } 301870ed506cSAndrii Nakryiko 301970ed506cSAndrii Nakryiko link = f.file->private_data; 302070ed506cSAndrii Nakryiko bpf_link_inc(link); 302170ed506cSAndrii Nakryiko fdput(f); 302270ed506cSAndrii Nakryiko 302370ed506cSAndrii Nakryiko return link; 302470ed506cSAndrii Nakryiko } 3025cb80ddc6SAlexei Starovoitov EXPORT_SYMBOL(bpf_link_get_from_fd); 302670ed506cSAndrii Nakryiko 302770ed506cSAndrii Nakryiko static void bpf_tracing_link_release(struct bpf_link *link) 302870ed506cSAndrii Nakryiko { 30293aac1eadSToke Høiland-Jørgensen struct bpf_tracing_link *tr_link = 3030f7e0beafSKui-Feng Lee container_of(link, struct bpf_tracing_link, link.link); 30313aac1eadSToke Høiland-Jørgensen 3032f7e0beafSKui-Feng Lee WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link, 30333aac1eadSToke Høiland-Jørgensen tr_link->trampoline)); 30343aac1eadSToke Høiland-Jørgensen 30353aac1eadSToke Høiland-Jørgensen bpf_trampoline_put(tr_link->trampoline); 30363aac1eadSToke Høiland-Jørgensen 30373aac1eadSToke Høiland-Jørgensen /* tgt_prog is NULL if target is a kernel function */ 30383aac1eadSToke Høiland-Jørgensen if (tr_link->tgt_prog) 30393aac1eadSToke Høiland-Jørgensen bpf_prog_put(tr_link->tgt_prog); 3040babf3164SAndrii Nakryiko } 3041babf3164SAndrii Nakryiko 3042babf3164SAndrii Nakryiko static void bpf_tracing_link_dealloc(struct bpf_link *link) 3043babf3164SAndrii Nakryiko { 304470ed506cSAndrii Nakryiko struct bpf_tracing_link *tr_link = 3045f7e0beafSKui-Feng Lee container_of(link, struct bpf_tracing_link, link.link); 304670ed506cSAndrii Nakryiko 304770ed506cSAndrii Nakryiko kfree(tr_link); 304870ed506cSAndrii Nakryiko } 304970ed506cSAndrii Nakryiko 3050f2e10bffSAndrii Nakryiko static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link, 3051f2e10bffSAndrii Nakryiko struct seq_file *seq) 3052f2e10bffSAndrii Nakryiko { 3053f2e10bffSAndrii Nakryiko struct bpf_tracing_link *tr_link = 3054f7e0beafSKui-Feng Lee container_of(link, struct bpf_tracing_link, link.link); 3055e859e429SYafang Shao u32 target_btf_id, target_obj_id; 3056f2e10bffSAndrii Nakryiko 3057e859e429SYafang Shao bpf_trampoline_unpack_key(tr_link->trampoline->key, 3058e859e429SYafang Shao &target_obj_id, &target_btf_id); 3059f2e10bffSAndrii Nakryiko seq_printf(seq, 3060e859e429SYafang Shao "attach_type:\t%d\n" 3061e859e429SYafang Shao "target_obj_id:\t%u\n" 3062e859e429SYafang Shao "target_btf_id:\t%u\n", 3063e859e429SYafang Shao tr_link->attach_type, 3064e859e429SYafang Shao target_obj_id, 3065e859e429SYafang Shao target_btf_id); 3066f2e10bffSAndrii Nakryiko } 3067f2e10bffSAndrii Nakryiko 3068f2e10bffSAndrii Nakryiko static int bpf_tracing_link_fill_link_info(const struct bpf_link *link, 3069f2e10bffSAndrii Nakryiko struct bpf_link_info *info) 3070f2e10bffSAndrii Nakryiko { 3071f2e10bffSAndrii Nakryiko struct bpf_tracing_link *tr_link = 3072f7e0beafSKui-Feng Lee container_of(link, struct bpf_tracing_link, link.link); 3073f2e10bffSAndrii Nakryiko 3074f2e10bffSAndrii Nakryiko info->tracing.attach_type = tr_link->attach_type; 3075441e8c66SToke Høiland-Jørgensen bpf_trampoline_unpack_key(tr_link->trampoline->key, 3076441e8c66SToke Høiland-Jørgensen &info->tracing.target_obj_id, 3077441e8c66SToke Høiland-Jørgensen &info->tracing.target_btf_id); 3078f2e10bffSAndrii Nakryiko 3079f2e10bffSAndrii Nakryiko return 0; 3080f2e10bffSAndrii Nakryiko } 3081f2e10bffSAndrii Nakryiko 308270ed506cSAndrii Nakryiko static const struct bpf_link_ops bpf_tracing_link_lops = { 308370ed506cSAndrii Nakryiko .release = bpf_tracing_link_release, 3084babf3164SAndrii Nakryiko .dealloc = bpf_tracing_link_dealloc, 3085f2e10bffSAndrii Nakryiko .show_fdinfo = bpf_tracing_link_show_fdinfo, 3086f2e10bffSAndrii Nakryiko .fill_link_info = bpf_tracing_link_fill_link_info, 308770ed506cSAndrii Nakryiko }; 308870ed506cSAndrii Nakryiko 30894a1e7c0cSToke Høiland-Jørgensen static int bpf_tracing_prog_attach(struct bpf_prog *prog, 30904a1e7c0cSToke Høiland-Jørgensen int tgt_prog_fd, 30912fcc8241SKui-Feng Lee u32 btf_id, 30922fcc8241SKui-Feng Lee u64 bpf_cookie) 3093fec56f58SAlexei Starovoitov { 3094a3b80e10SAndrii Nakryiko struct bpf_link_primer link_primer; 30953aac1eadSToke Høiland-Jørgensen struct bpf_prog *tgt_prog = NULL; 30964a1e7c0cSToke Høiland-Jørgensen struct bpf_trampoline *tr = NULL; 309770ed506cSAndrii Nakryiko struct bpf_tracing_link *link; 30984a1e7c0cSToke Høiland-Jørgensen u64 key = 0; 3099a3b80e10SAndrii Nakryiko int err; 3100fec56f58SAlexei Starovoitov 31019e4e01dfSKP Singh switch (prog->type) { 31029e4e01dfSKP Singh case BPF_PROG_TYPE_TRACING: 3103fec56f58SAlexei Starovoitov if (prog->expected_attach_type != BPF_TRACE_FENTRY && 3104be8704ffSAlexei Starovoitov prog->expected_attach_type != BPF_TRACE_FEXIT && 31059e4e01dfSKP Singh prog->expected_attach_type != BPF_MODIFY_RETURN) { 31069e4e01dfSKP Singh err = -EINVAL; 31079e4e01dfSKP Singh goto out_put_prog; 31089e4e01dfSKP Singh } 31099e4e01dfSKP Singh break; 31109e4e01dfSKP Singh case BPF_PROG_TYPE_EXT: 31119e4e01dfSKP Singh if (prog->expected_attach_type != 0) { 31129e4e01dfSKP Singh err = -EINVAL; 31139e4e01dfSKP Singh goto out_put_prog; 31149e4e01dfSKP Singh } 31159e4e01dfSKP Singh break; 31169e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 31179e4e01dfSKP Singh if (prog->expected_attach_type != BPF_LSM_MAC) { 31189e4e01dfSKP Singh err = -EINVAL; 31199e4e01dfSKP Singh goto out_put_prog; 31209e4e01dfSKP Singh } 31219e4e01dfSKP Singh break; 31229e4e01dfSKP Singh default: 3123fec56f58SAlexei Starovoitov err = -EINVAL; 3124fec56f58SAlexei Starovoitov goto out_put_prog; 3125fec56f58SAlexei Starovoitov } 3126fec56f58SAlexei Starovoitov 31274a1e7c0cSToke Høiland-Jørgensen if (!!tgt_prog_fd != !!btf_id) { 31284a1e7c0cSToke Høiland-Jørgensen err = -EINVAL; 31294a1e7c0cSToke Høiland-Jørgensen goto out_put_prog; 31304a1e7c0cSToke Høiland-Jørgensen } 31314a1e7c0cSToke Høiland-Jørgensen 31324a1e7c0cSToke Høiland-Jørgensen if (tgt_prog_fd) { 31334a1e7c0cSToke Høiland-Jørgensen /* For now we only allow new targets for BPF_PROG_TYPE_EXT */ 31344a1e7c0cSToke Høiland-Jørgensen if (prog->type != BPF_PROG_TYPE_EXT) { 31354a1e7c0cSToke Høiland-Jørgensen err = -EINVAL; 31364a1e7c0cSToke Høiland-Jørgensen goto out_put_prog; 31374a1e7c0cSToke Høiland-Jørgensen } 31384a1e7c0cSToke Høiland-Jørgensen 31394a1e7c0cSToke Høiland-Jørgensen tgt_prog = bpf_prog_get(tgt_prog_fd); 31404a1e7c0cSToke Høiland-Jørgensen if (IS_ERR(tgt_prog)) { 31414a1e7c0cSToke Høiland-Jørgensen err = PTR_ERR(tgt_prog); 31424a1e7c0cSToke Høiland-Jørgensen tgt_prog = NULL; 31434a1e7c0cSToke Høiland-Jørgensen goto out_put_prog; 31444a1e7c0cSToke Høiland-Jørgensen } 31454a1e7c0cSToke Høiland-Jørgensen 314622dc4a0fSAndrii Nakryiko key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id); 31474a1e7c0cSToke Høiland-Jørgensen } 31484a1e7c0cSToke Høiland-Jørgensen 314970ed506cSAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER); 315070ed506cSAndrii Nakryiko if (!link) { 315170ed506cSAndrii Nakryiko err = -ENOMEM; 3152fec56f58SAlexei Starovoitov goto out_put_prog; 3153fec56f58SAlexei Starovoitov } 3154f7e0beafSKui-Feng Lee bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING, 3155f2e10bffSAndrii Nakryiko &bpf_tracing_link_lops, prog); 3156f2e10bffSAndrii Nakryiko link->attach_type = prog->expected_attach_type; 31572fcc8241SKui-Feng Lee link->link.cookie = bpf_cookie; 3158fec56f58SAlexei Starovoitov 31593aac1eadSToke Høiland-Jørgensen mutex_lock(&prog->aux->dst_mutex); 3160babf3164SAndrii Nakryiko 31614a1e7c0cSToke Høiland-Jørgensen /* There are a few possible cases here: 31624a1e7c0cSToke Høiland-Jørgensen * 31634a1e7c0cSToke Høiland-Jørgensen * - if prog->aux->dst_trampoline is set, the program was just loaded 31644a1e7c0cSToke Høiland-Jørgensen * and not yet attached to anything, so we can use the values stored 31654a1e7c0cSToke Høiland-Jørgensen * in prog->aux 31664a1e7c0cSToke Høiland-Jørgensen * 31674a1e7c0cSToke Høiland-Jørgensen * - if prog->aux->dst_trampoline is NULL, the program has already been 31684a1e7c0cSToke Høiland-Jørgensen * attached to a target and its initial target was cleared (below) 31694a1e7c0cSToke Høiland-Jørgensen * 31704a1e7c0cSToke Høiland-Jørgensen * - if tgt_prog != NULL, the caller specified tgt_prog_fd + 31714a1e7c0cSToke Høiland-Jørgensen * target_btf_id using the link_create API. 31724a1e7c0cSToke Høiland-Jørgensen * 31734a1e7c0cSToke Høiland-Jørgensen * - if tgt_prog == NULL when this function was called using the old 31744a1e7c0cSToke Høiland-Jørgensen * raw_tracepoint_open API, and we need a target from prog->aux 31754a1e7c0cSToke Høiland-Jørgensen * 3176f3a95075SJiri Olsa * - if prog->aux->dst_trampoline and tgt_prog is NULL, the program 3177f3a95075SJiri Olsa * was detached and is going for re-attachment. 31784a1e7c0cSToke Høiland-Jørgensen */ 31794a1e7c0cSToke Høiland-Jørgensen if (!prog->aux->dst_trampoline && !tgt_prog) { 3180f3a95075SJiri Olsa /* 3181f3a95075SJiri Olsa * Allow re-attach for TRACING and LSM programs. If it's 3182f3a95075SJiri Olsa * currently linked, bpf_trampoline_link_prog will fail. 3183f3a95075SJiri Olsa * EXT programs need to specify tgt_prog_fd, so they 3184f3a95075SJiri Olsa * re-attach in separate code path. 3185f3a95075SJiri Olsa */ 3186f3a95075SJiri Olsa if (prog->type != BPF_PROG_TYPE_TRACING && 3187f3a95075SJiri Olsa prog->type != BPF_PROG_TYPE_LSM) { 3188f3a95075SJiri Olsa err = -EINVAL; 31893aac1eadSToke Høiland-Jørgensen goto out_unlock; 31903aac1eadSToke Høiland-Jørgensen } 3191f3a95075SJiri Olsa btf_id = prog->aux->attach_btf_id; 3192f3a95075SJiri Olsa key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id); 3193f3a95075SJiri Olsa } 31944a1e7c0cSToke Høiland-Jørgensen 31954a1e7c0cSToke Høiland-Jørgensen if (!prog->aux->dst_trampoline || 31964a1e7c0cSToke Høiland-Jørgensen (key && key != prog->aux->dst_trampoline->key)) { 31974a1e7c0cSToke Høiland-Jørgensen /* If there is no saved target, or the specified target is 31984a1e7c0cSToke Høiland-Jørgensen * different from the destination specified at load time, we 31994a1e7c0cSToke Høiland-Jørgensen * need a new trampoline and a check for compatibility 32004a1e7c0cSToke Høiland-Jørgensen */ 32014a1e7c0cSToke Høiland-Jørgensen struct bpf_attach_target_info tgt_info = {}; 32024a1e7c0cSToke Høiland-Jørgensen 32034a1e7c0cSToke Høiland-Jørgensen err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id, 32044a1e7c0cSToke Høiland-Jørgensen &tgt_info); 32054a1e7c0cSToke Høiland-Jørgensen if (err) 32064a1e7c0cSToke Høiland-Jørgensen goto out_unlock; 32074a1e7c0cSToke Høiland-Jørgensen 320831bf1dbcSViktor Malik if (tgt_info.tgt_mod) { 320931bf1dbcSViktor Malik module_put(prog->aux->mod); 321031bf1dbcSViktor Malik prog->aux->mod = tgt_info.tgt_mod; 321131bf1dbcSViktor Malik } 321231bf1dbcSViktor Malik 32134a1e7c0cSToke Høiland-Jørgensen tr = bpf_trampoline_get(key, &tgt_info); 32144a1e7c0cSToke Høiland-Jørgensen if (!tr) { 32154a1e7c0cSToke Høiland-Jørgensen err = -ENOMEM; 32164a1e7c0cSToke Høiland-Jørgensen goto out_unlock; 32174a1e7c0cSToke Høiland-Jørgensen } 32184a1e7c0cSToke Høiland-Jørgensen } else { 32194a1e7c0cSToke Høiland-Jørgensen /* The caller didn't specify a target, or the target was the 32204a1e7c0cSToke Høiland-Jørgensen * same as the destination supplied during program load. This 32214a1e7c0cSToke Høiland-Jørgensen * means we can reuse the trampoline and reference from program 32224a1e7c0cSToke Høiland-Jørgensen * load time, and there is no need to allocate a new one. This 32234a1e7c0cSToke Høiland-Jørgensen * can only happen once for any program, as the saved values in 32244a1e7c0cSToke Høiland-Jørgensen * prog->aux are cleared below. 32254a1e7c0cSToke Høiland-Jørgensen */ 32263aac1eadSToke Høiland-Jørgensen tr = prog->aux->dst_trampoline; 32273aac1eadSToke Høiland-Jørgensen tgt_prog = prog->aux->dst_prog; 32284a1e7c0cSToke Høiland-Jørgensen } 32293aac1eadSToke Høiland-Jørgensen 3230f7e0beafSKui-Feng Lee err = bpf_link_prime(&link->link.link, &link_primer); 32313aac1eadSToke Høiland-Jørgensen if (err) 32323aac1eadSToke Høiland-Jørgensen goto out_unlock; 32333aac1eadSToke Høiland-Jørgensen 3234f7e0beafSKui-Feng Lee err = bpf_trampoline_link_prog(&link->link, tr); 3235babf3164SAndrii Nakryiko if (err) { 3236a3b80e10SAndrii Nakryiko bpf_link_cleanup(&link_primer); 32373aac1eadSToke Høiland-Jørgensen link = NULL; 32383aac1eadSToke Høiland-Jørgensen goto out_unlock; 3239babf3164SAndrii Nakryiko } 3240babf3164SAndrii Nakryiko 32413aac1eadSToke Høiland-Jørgensen link->tgt_prog = tgt_prog; 32423aac1eadSToke Høiland-Jørgensen link->trampoline = tr; 32433aac1eadSToke Høiland-Jørgensen 32444a1e7c0cSToke Høiland-Jørgensen /* Always clear the trampoline and target prog from prog->aux to make 32454a1e7c0cSToke Høiland-Jørgensen * sure the original attach destination is not kept alive after a 32464a1e7c0cSToke Høiland-Jørgensen * program is (re-)attached to another target. 32474a1e7c0cSToke Høiland-Jørgensen */ 32484a1e7c0cSToke Høiland-Jørgensen if (prog->aux->dst_prog && 32494a1e7c0cSToke Høiland-Jørgensen (tgt_prog_fd || tr != prog->aux->dst_trampoline)) 32504a1e7c0cSToke Høiland-Jørgensen /* got extra prog ref from syscall, or attaching to different prog */ 32514a1e7c0cSToke Høiland-Jørgensen bpf_prog_put(prog->aux->dst_prog); 32524a1e7c0cSToke Høiland-Jørgensen if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline) 32534a1e7c0cSToke Høiland-Jørgensen /* we allocated a new trampoline, so free the old one */ 32544a1e7c0cSToke Høiland-Jørgensen bpf_trampoline_put(prog->aux->dst_trampoline); 32554a1e7c0cSToke Høiland-Jørgensen 32563aac1eadSToke Høiland-Jørgensen prog->aux->dst_prog = NULL; 32573aac1eadSToke Høiland-Jørgensen prog->aux->dst_trampoline = NULL; 32583aac1eadSToke Høiland-Jørgensen mutex_unlock(&prog->aux->dst_mutex); 32593aac1eadSToke Høiland-Jørgensen 3260a3b80e10SAndrii Nakryiko return bpf_link_settle(&link_primer); 32613aac1eadSToke Høiland-Jørgensen out_unlock: 32624a1e7c0cSToke Høiland-Jørgensen if (tr && tr != prog->aux->dst_trampoline) 32634a1e7c0cSToke Høiland-Jørgensen bpf_trampoline_put(tr); 32643aac1eadSToke Høiland-Jørgensen mutex_unlock(&prog->aux->dst_mutex); 32653aac1eadSToke Høiland-Jørgensen kfree(link); 3266fec56f58SAlexei Starovoitov out_put_prog: 32674a1e7c0cSToke Høiland-Jørgensen if (tgt_prog_fd && tgt_prog) 32684a1e7c0cSToke Høiland-Jørgensen bpf_prog_put(tgt_prog); 3269fec56f58SAlexei Starovoitov return err; 3270fec56f58SAlexei Starovoitov } 3271fec56f58SAlexei Starovoitov 327270ed506cSAndrii Nakryiko struct bpf_raw_tp_link { 327370ed506cSAndrii Nakryiko struct bpf_link link; 3274c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 3275c4f6699dSAlexei Starovoitov }; 3276c4f6699dSAlexei Starovoitov 327770ed506cSAndrii Nakryiko static void bpf_raw_tp_link_release(struct bpf_link *link) 3278c4f6699dSAlexei Starovoitov { 327970ed506cSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 328070ed506cSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 3281c4f6699dSAlexei Starovoitov 328270ed506cSAndrii Nakryiko bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog); 3283a38d1107SMatt Mullins bpf_put_raw_tracepoint(raw_tp->btp); 3284babf3164SAndrii Nakryiko } 3285babf3164SAndrii Nakryiko 3286babf3164SAndrii Nakryiko static void bpf_raw_tp_link_dealloc(struct bpf_link *link) 3287babf3164SAndrii Nakryiko { 3288babf3164SAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 3289babf3164SAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 3290babf3164SAndrii Nakryiko 3291c4f6699dSAlexei Starovoitov kfree(raw_tp); 3292c4f6699dSAlexei Starovoitov } 3293c4f6699dSAlexei Starovoitov 3294f2e10bffSAndrii Nakryiko static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link, 3295f2e10bffSAndrii Nakryiko struct seq_file *seq) 3296f2e10bffSAndrii Nakryiko { 3297f2e10bffSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp_link = 3298f2e10bffSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 3299f2e10bffSAndrii Nakryiko 3300f2e10bffSAndrii Nakryiko seq_printf(seq, 3301f2e10bffSAndrii Nakryiko "tp_name:\t%s\n", 3302f2e10bffSAndrii Nakryiko raw_tp_link->btp->tp->name); 3303f2e10bffSAndrii Nakryiko } 3304f2e10bffSAndrii Nakryiko 330557d48537SYafang Shao static int bpf_copy_to_user(char __user *ubuf, const char *buf, u32 ulen, 330657d48537SYafang Shao u32 len) 330757d48537SYafang Shao { 330857d48537SYafang Shao if (ulen >= len + 1) { 330957d48537SYafang Shao if (copy_to_user(ubuf, buf, len + 1)) 331057d48537SYafang Shao return -EFAULT; 331157d48537SYafang Shao } else { 331257d48537SYafang Shao char zero = '\0'; 331357d48537SYafang Shao 331457d48537SYafang Shao if (copy_to_user(ubuf, buf, ulen - 1)) 331557d48537SYafang Shao return -EFAULT; 331657d48537SYafang Shao if (put_user(zero, ubuf + ulen - 1)) 331757d48537SYafang Shao return -EFAULT; 331857d48537SYafang Shao return -ENOSPC; 331957d48537SYafang Shao } 332057d48537SYafang Shao 332157d48537SYafang Shao return 0; 332257d48537SYafang Shao } 332357d48537SYafang Shao 3324f2e10bffSAndrii Nakryiko static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link, 3325f2e10bffSAndrii Nakryiko struct bpf_link_info *info) 3326f2e10bffSAndrii Nakryiko { 3327f2e10bffSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp_link = 3328f2e10bffSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 3329f2e10bffSAndrii Nakryiko char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name); 3330f2e10bffSAndrii Nakryiko const char *tp_name = raw_tp_link->btp->tp->name; 3331f2e10bffSAndrii Nakryiko u32 ulen = info->raw_tracepoint.tp_name_len; 3332f2e10bffSAndrii Nakryiko size_t tp_len = strlen(tp_name); 3333f2e10bffSAndrii Nakryiko 3334b474959dSYonghong Song if (!ulen ^ !ubuf) 3335f2e10bffSAndrii Nakryiko return -EINVAL; 3336f2e10bffSAndrii Nakryiko 3337f2e10bffSAndrii Nakryiko info->raw_tracepoint.tp_name_len = tp_len + 1; 3338f2e10bffSAndrii Nakryiko 3339f2e10bffSAndrii Nakryiko if (!ubuf) 3340f2e10bffSAndrii Nakryiko return 0; 3341f2e10bffSAndrii Nakryiko 334257d48537SYafang Shao return bpf_copy_to_user(ubuf, tp_name, ulen, tp_len); 3343f2e10bffSAndrii Nakryiko } 3344f2e10bffSAndrii Nakryiko 3345a3b80e10SAndrii Nakryiko static const struct bpf_link_ops bpf_raw_tp_link_lops = { 334670ed506cSAndrii Nakryiko .release = bpf_raw_tp_link_release, 3347babf3164SAndrii Nakryiko .dealloc = bpf_raw_tp_link_dealloc, 3348f2e10bffSAndrii Nakryiko .show_fdinfo = bpf_raw_tp_link_show_fdinfo, 3349f2e10bffSAndrii Nakryiko .fill_link_info = bpf_raw_tp_link_fill_link_info, 3350c4f6699dSAlexei Starovoitov }; 3351c4f6699dSAlexei Starovoitov 3352b89fbfbbSAndrii Nakryiko #ifdef CONFIG_PERF_EVENTS 3353b89fbfbbSAndrii Nakryiko struct bpf_perf_link { 3354b89fbfbbSAndrii Nakryiko struct bpf_link link; 3355b89fbfbbSAndrii Nakryiko struct file *perf_file; 3356b89fbfbbSAndrii Nakryiko }; 3357b89fbfbbSAndrii Nakryiko 3358b89fbfbbSAndrii Nakryiko static void bpf_perf_link_release(struct bpf_link *link) 3359b89fbfbbSAndrii Nakryiko { 3360b89fbfbbSAndrii Nakryiko struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link); 3361b89fbfbbSAndrii Nakryiko struct perf_event *event = perf_link->perf_file->private_data; 3362b89fbfbbSAndrii Nakryiko 3363b89fbfbbSAndrii Nakryiko perf_event_free_bpf_prog(event); 3364b89fbfbbSAndrii Nakryiko fput(perf_link->perf_file); 3365b89fbfbbSAndrii Nakryiko } 3366b89fbfbbSAndrii Nakryiko 3367b89fbfbbSAndrii Nakryiko static void bpf_perf_link_dealloc(struct bpf_link *link) 3368b89fbfbbSAndrii Nakryiko { 3369b89fbfbbSAndrii Nakryiko struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link); 3370b89fbfbbSAndrii Nakryiko 3371b89fbfbbSAndrii Nakryiko kfree(perf_link); 3372b89fbfbbSAndrii Nakryiko } 3373b89fbfbbSAndrii Nakryiko 33741b715e1bSYafang Shao static int bpf_perf_link_fill_common(const struct perf_event *event, 33751b715e1bSYafang Shao char __user *uname, u32 ulen, 33761b715e1bSYafang Shao u64 *probe_offset, u64 *probe_addr, 33771b715e1bSYafang Shao u32 *fd_type) 33781b715e1bSYafang Shao { 33791b715e1bSYafang Shao const char *buf; 33801b715e1bSYafang Shao u32 prog_id; 33811b715e1bSYafang Shao size_t len; 33821b715e1bSYafang Shao int err; 33831b715e1bSYafang Shao 33841b715e1bSYafang Shao if (!ulen ^ !uname) 33851b715e1bSYafang Shao return -EINVAL; 33861b715e1bSYafang Shao 33871b715e1bSYafang Shao err = bpf_get_perf_event_info(event, &prog_id, fd_type, &buf, 33881b715e1bSYafang Shao probe_offset, probe_addr); 33891b715e1bSYafang Shao if (err) 33901b715e1bSYafang Shao return err; 33910aa35162SYafang Shao if (!uname) 33920aa35162SYafang Shao return 0; 33931b715e1bSYafang Shao if (buf) { 33941b715e1bSYafang Shao len = strlen(buf); 33951b715e1bSYafang Shao err = bpf_copy_to_user(uname, buf, ulen, len); 33961b715e1bSYafang Shao if (err) 33971b715e1bSYafang Shao return err; 33981b715e1bSYafang Shao } else { 33991b715e1bSYafang Shao char zero = '\0'; 34001b715e1bSYafang Shao 34011b715e1bSYafang Shao if (put_user(zero, uname)) 34021b715e1bSYafang Shao return -EFAULT; 34031b715e1bSYafang Shao } 34041b715e1bSYafang Shao return 0; 34051b715e1bSYafang Shao } 34061b715e1bSYafang Shao 34071b715e1bSYafang Shao #ifdef CONFIG_KPROBE_EVENTS 34081b715e1bSYafang Shao static int bpf_perf_link_fill_kprobe(const struct perf_event *event, 34091b715e1bSYafang Shao struct bpf_link_info *info) 34101b715e1bSYafang Shao { 34111b715e1bSYafang Shao char __user *uname; 34121b715e1bSYafang Shao u64 addr, offset; 34131b715e1bSYafang Shao u32 ulen, type; 34141b715e1bSYafang Shao int err; 34151b715e1bSYafang Shao 34161b715e1bSYafang Shao uname = u64_to_user_ptr(info->perf_event.kprobe.func_name); 34171b715e1bSYafang Shao ulen = info->perf_event.kprobe.name_len; 34181b715e1bSYafang Shao err = bpf_perf_link_fill_common(event, uname, ulen, &offset, &addr, 34191b715e1bSYafang Shao &type); 34201b715e1bSYafang Shao if (err) 34211b715e1bSYafang Shao return err; 34221b715e1bSYafang Shao if (type == BPF_FD_TYPE_KRETPROBE) 34231b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_KRETPROBE; 34241b715e1bSYafang Shao else 34251b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_KPROBE; 34261b715e1bSYafang Shao 34271b715e1bSYafang Shao info->perf_event.kprobe.offset = offset; 34281b715e1bSYafang Shao if (!kallsyms_show_value(current_cred())) 34291b715e1bSYafang Shao addr = 0; 34301b715e1bSYafang Shao info->perf_event.kprobe.addr = addr; 34311b715e1bSYafang Shao return 0; 34321b715e1bSYafang Shao } 34331b715e1bSYafang Shao #endif 34341b715e1bSYafang Shao 34351b715e1bSYafang Shao #ifdef CONFIG_UPROBE_EVENTS 34361b715e1bSYafang Shao static int bpf_perf_link_fill_uprobe(const struct perf_event *event, 34371b715e1bSYafang Shao struct bpf_link_info *info) 34381b715e1bSYafang Shao { 34391b715e1bSYafang Shao char __user *uname; 34401b715e1bSYafang Shao u64 addr, offset; 34411b715e1bSYafang Shao u32 ulen, type; 34421b715e1bSYafang Shao int err; 34431b715e1bSYafang Shao 34441b715e1bSYafang Shao uname = u64_to_user_ptr(info->perf_event.uprobe.file_name); 34451b715e1bSYafang Shao ulen = info->perf_event.uprobe.name_len; 34461b715e1bSYafang Shao err = bpf_perf_link_fill_common(event, uname, ulen, &offset, &addr, 34471b715e1bSYafang Shao &type); 34481b715e1bSYafang Shao if (err) 34491b715e1bSYafang Shao return err; 34501b715e1bSYafang Shao 34511b715e1bSYafang Shao if (type == BPF_FD_TYPE_URETPROBE) 34521b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_URETPROBE; 34531b715e1bSYafang Shao else 34541b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_UPROBE; 34551b715e1bSYafang Shao info->perf_event.uprobe.offset = offset; 34561b715e1bSYafang Shao return 0; 34571b715e1bSYafang Shao } 34581b715e1bSYafang Shao #endif 34591b715e1bSYafang Shao 34601b715e1bSYafang Shao static int bpf_perf_link_fill_probe(const struct perf_event *event, 34611b715e1bSYafang Shao struct bpf_link_info *info) 34621b715e1bSYafang Shao { 34631b715e1bSYafang Shao #ifdef CONFIG_KPROBE_EVENTS 34641b715e1bSYafang Shao if (event->tp_event->flags & TRACE_EVENT_FL_KPROBE) 34651b715e1bSYafang Shao return bpf_perf_link_fill_kprobe(event, info); 34661b715e1bSYafang Shao #endif 34671b715e1bSYafang Shao #ifdef CONFIG_UPROBE_EVENTS 34681b715e1bSYafang Shao if (event->tp_event->flags & TRACE_EVENT_FL_UPROBE) 34691b715e1bSYafang Shao return bpf_perf_link_fill_uprobe(event, info); 34701b715e1bSYafang Shao #endif 34711b715e1bSYafang Shao return -EOPNOTSUPP; 34721b715e1bSYafang Shao } 34731b715e1bSYafang Shao 34741b715e1bSYafang Shao static int bpf_perf_link_fill_tracepoint(const struct perf_event *event, 34751b715e1bSYafang Shao struct bpf_link_info *info) 34761b715e1bSYafang Shao { 34771b715e1bSYafang Shao char __user *uname; 34781b715e1bSYafang Shao u32 ulen; 34791b715e1bSYafang Shao 34801b715e1bSYafang Shao uname = u64_to_user_ptr(info->perf_event.tracepoint.tp_name); 34811b715e1bSYafang Shao ulen = info->perf_event.tracepoint.name_len; 34821b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_TRACEPOINT; 34831b715e1bSYafang Shao return bpf_perf_link_fill_common(event, uname, ulen, NULL, NULL, NULL); 34841b715e1bSYafang Shao } 34851b715e1bSYafang Shao 34861b715e1bSYafang Shao static int bpf_perf_link_fill_perf_event(const struct perf_event *event, 34871b715e1bSYafang Shao struct bpf_link_info *info) 34881b715e1bSYafang Shao { 34891b715e1bSYafang Shao info->perf_event.event.type = event->attr.type; 34901b715e1bSYafang Shao info->perf_event.event.config = event->attr.config; 34911b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_EVENT; 34921b715e1bSYafang Shao return 0; 34931b715e1bSYafang Shao } 34941b715e1bSYafang Shao 34951b715e1bSYafang Shao static int bpf_perf_link_fill_link_info(const struct bpf_link *link, 34961b715e1bSYafang Shao struct bpf_link_info *info) 34971b715e1bSYafang Shao { 34981b715e1bSYafang Shao struct bpf_perf_link *perf_link; 34991b715e1bSYafang Shao const struct perf_event *event; 35001b715e1bSYafang Shao 35011b715e1bSYafang Shao perf_link = container_of(link, struct bpf_perf_link, link); 35021b715e1bSYafang Shao event = perf_get_event(perf_link->perf_file); 35031b715e1bSYafang Shao if (IS_ERR(event)) 35041b715e1bSYafang Shao return PTR_ERR(event); 35051b715e1bSYafang Shao 35061b715e1bSYafang Shao switch (event->prog->type) { 35071b715e1bSYafang Shao case BPF_PROG_TYPE_PERF_EVENT: 35081b715e1bSYafang Shao return bpf_perf_link_fill_perf_event(event, info); 35091b715e1bSYafang Shao case BPF_PROG_TYPE_TRACEPOINT: 35101b715e1bSYafang Shao return bpf_perf_link_fill_tracepoint(event, info); 35111b715e1bSYafang Shao case BPF_PROG_TYPE_KPROBE: 35121b715e1bSYafang Shao return bpf_perf_link_fill_probe(event, info); 35131b715e1bSYafang Shao default: 35141b715e1bSYafang Shao return -EOPNOTSUPP; 35151b715e1bSYafang Shao } 35161b715e1bSYafang Shao } 35171b715e1bSYafang Shao 3518b89fbfbbSAndrii Nakryiko static const struct bpf_link_ops bpf_perf_link_lops = { 3519b89fbfbbSAndrii Nakryiko .release = bpf_perf_link_release, 3520b89fbfbbSAndrii Nakryiko .dealloc = bpf_perf_link_dealloc, 35211b715e1bSYafang Shao .fill_link_info = bpf_perf_link_fill_link_info, 3522b89fbfbbSAndrii Nakryiko }; 3523b89fbfbbSAndrii Nakryiko 3524b89fbfbbSAndrii Nakryiko static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 3525b89fbfbbSAndrii Nakryiko { 3526b89fbfbbSAndrii Nakryiko struct bpf_link_primer link_primer; 3527b89fbfbbSAndrii Nakryiko struct bpf_perf_link *link; 3528b89fbfbbSAndrii Nakryiko struct perf_event *event; 3529b89fbfbbSAndrii Nakryiko struct file *perf_file; 3530b89fbfbbSAndrii Nakryiko int err; 3531b89fbfbbSAndrii Nakryiko 3532b89fbfbbSAndrii Nakryiko if (attr->link_create.flags) 3533b89fbfbbSAndrii Nakryiko return -EINVAL; 3534b89fbfbbSAndrii Nakryiko 3535b89fbfbbSAndrii Nakryiko perf_file = perf_event_get(attr->link_create.target_fd); 3536b89fbfbbSAndrii Nakryiko if (IS_ERR(perf_file)) 3537b89fbfbbSAndrii Nakryiko return PTR_ERR(perf_file); 3538b89fbfbbSAndrii Nakryiko 3539b89fbfbbSAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER); 3540b89fbfbbSAndrii Nakryiko if (!link) { 3541b89fbfbbSAndrii Nakryiko err = -ENOMEM; 3542b89fbfbbSAndrii Nakryiko goto out_put_file; 3543b89fbfbbSAndrii Nakryiko } 3544b89fbfbbSAndrii Nakryiko bpf_link_init(&link->link, BPF_LINK_TYPE_PERF_EVENT, &bpf_perf_link_lops, prog); 3545b89fbfbbSAndrii Nakryiko link->perf_file = perf_file; 3546b89fbfbbSAndrii Nakryiko 3547b89fbfbbSAndrii Nakryiko err = bpf_link_prime(&link->link, &link_primer); 3548b89fbfbbSAndrii Nakryiko if (err) { 3549b89fbfbbSAndrii Nakryiko kfree(link); 3550b89fbfbbSAndrii Nakryiko goto out_put_file; 3551b89fbfbbSAndrii Nakryiko } 3552b89fbfbbSAndrii Nakryiko 3553b89fbfbbSAndrii Nakryiko event = perf_file->private_data; 355482e6b1eeSAndrii Nakryiko err = perf_event_set_bpf_prog(event, prog, attr->link_create.perf_event.bpf_cookie); 3555b89fbfbbSAndrii Nakryiko if (err) { 3556b89fbfbbSAndrii Nakryiko bpf_link_cleanup(&link_primer); 3557b89fbfbbSAndrii Nakryiko goto out_put_file; 3558b89fbfbbSAndrii Nakryiko } 3559b89fbfbbSAndrii Nakryiko /* perf_event_set_bpf_prog() doesn't take its own refcnt on prog */ 3560b89fbfbbSAndrii Nakryiko bpf_prog_inc(prog); 3561b89fbfbbSAndrii Nakryiko 3562b89fbfbbSAndrii Nakryiko return bpf_link_settle(&link_primer); 3563b89fbfbbSAndrii Nakryiko 3564b89fbfbbSAndrii Nakryiko out_put_file: 3565b89fbfbbSAndrii Nakryiko fput(perf_file); 3566b89fbfbbSAndrii Nakryiko return err; 3567b89fbfbbSAndrii Nakryiko } 35680dcac272SJiri Olsa #else 35690dcac272SJiri Olsa static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 35700dcac272SJiri Olsa { 35710dcac272SJiri Olsa return -EOPNOTSUPP; 35720dcac272SJiri Olsa } 3573b89fbfbbSAndrii Nakryiko #endif /* CONFIG_PERF_EVENTS */ 3574b89fbfbbSAndrii Nakryiko 3575df86ca0dSAndrii Nakryiko static int bpf_raw_tp_link_attach(struct bpf_prog *prog, 3576df86ca0dSAndrii Nakryiko const char __user *user_tp_name) 3577c4f6699dSAlexei Starovoitov { 3578a3b80e10SAndrii Nakryiko struct bpf_link_primer link_primer; 3579babf3164SAndrii Nakryiko struct bpf_raw_tp_link *link; 3580c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 3581ac4414b5SAlexei Starovoitov const char *tp_name; 3582ac4414b5SAlexei Starovoitov char buf[128]; 3583a3b80e10SAndrii Nakryiko int err; 3584c4f6699dSAlexei Starovoitov 35859e4e01dfSKP Singh switch (prog->type) { 35869e4e01dfSKP Singh case BPF_PROG_TYPE_TRACING: 35879e4e01dfSKP Singh case BPF_PROG_TYPE_EXT: 35889e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 3589df86ca0dSAndrii Nakryiko if (user_tp_name) 3590fec56f58SAlexei Starovoitov /* The attach point for this category of programs 3591fec56f58SAlexei Starovoitov * should be specified via btf_id during program load. 3592ac4414b5SAlexei Starovoitov */ 3593df86ca0dSAndrii Nakryiko return -EINVAL; 35949e4e01dfSKP Singh if (prog->type == BPF_PROG_TYPE_TRACING && 35959e4e01dfSKP Singh prog->expected_attach_type == BPF_TRACE_RAW_TP) { 359638207291SMartin KaFai Lau tp_name = prog->aux->attach_func_name; 35979e4e01dfSKP Singh break; 35989e4e01dfSKP Singh } 35992fcc8241SKui-Feng Lee return bpf_tracing_prog_attach(prog, 0, 0, 0); 36009e4e01dfSKP Singh case BPF_PROG_TYPE_RAW_TRACEPOINT: 36019e4e01dfSKP Singh case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 3602df86ca0dSAndrii Nakryiko if (strncpy_from_user(buf, user_tp_name, sizeof(buf) - 1) < 0) 3603df86ca0dSAndrii Nakryiko return -EFAULT; 3604ac4414b5SAlexei Starovoitov buf[sizeof(buf) - 1] = 0; 3605ac4414b5SAlexei Starovoitov tp_name = buf; 36069e4e01dfSKP Singh break; 36079e4e01dfSKP Singh default: 3608df86ca0dSAndrii Nakryiko return -EINVAL; 3609ac4414b5SAlexei Starovoitov } 3610c4f6699dSAlexei Starovoitov 3611a38d1107SMatt Mullins btp = bpf_get_raw_tracepoint(tp_name); 3612df86ca0dSAndrii Nakryiko if (!btp) 3613df86ca0dSAndrii Nakryiko return -ENOENT; 3614c4f6699dSAlexei Starovoitov 3615babf3164SAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER); 3616babf3164SAndrii Nakryiko if (!link) { 3617a38d1107SMatt Mullins err = -ENOMEM; 3618a38d1107SMatt Mullins goto out_put_btp; 3619a38d1107SMatt Mullins } 3620f2e10bffSAndrii Nakryiko bpf_link_init(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT, 3621f2e10bffSAndrii Nakryiko &bpf_raw_tp_link_lops, prog); 3622babf3164SAndrii Nakryiko link->btp = btp; 3623c4f6699dSAlexei Starovoitov 3624a3b80e10SAndrii Nakryiko err = bpf_link_prime(&link->link, &link_primer); 3625a3b80e10SAndrii Nakryiko if (err) { 3626babf3164SAndrii Nakryiko kfree(link); 3627babf3164SAndrii Nakryiko goto out_put_btp; 3628c4f6699dSAlexei Starovoitov } 3629babf3164SAndrii Nakryiko 3630babf3164SAndrii Nakryiko err = bpf_probe_register(link->btp, prog); 3631babf3164SAndrii Nakryiko if (err) { 3632a3b80e10SAndrii Nakryiko bpf_link_cleanup(&link_primer); 3633babf3164SAndrii Nakryiko goto out_put_btp; 3634babf3164SAndrii Nakryiko } 3635babf3164SAndrii Nakryiko 3636a3b80e10SAndrii Nakryiko return bpf_link_settle(&link_primer); 3637c4f6699dSAlexei Starovoitov 3638a38d1107SMatt Mullins out_put_btp: 3639a38d1107SMatt Mullins bpf_put_raw_tracepoint(btp); 3640c4f6699dSAlexei Starovoitov return err; 3641c4f6699dSAlexei Starovoitov } 3642c4f6699dSAlexei Starovoitov 3643df86ca0dSAndrii Nakryiko #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd 3644df86ca0dSAndrii Nakryiko 3645df86ca0dSAndrii Nakryiko static int bpf_raw_tracepoint_open(const union bpf_attr *attr) 3646df86ca0dSAndrii Nakryiko { 3647df86ca0dSAndrii Nakryiko struct bpf_prog *prog; 3648df86ca0dSAndrii Nakryiko int fd; 3649df86ca0dSAndrii Nakryiko 3650df86ca0dSAndrii Nakryiko if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN)) 3651df86ca0dSAndrii Nakryiko return -EINVAL; 3652df86ca0dSAndrii Nakryiko 3653df86ca0dSAndrii Nakryiko prog = bpf_prog_get(attr->raw_tracepoint.prog_fd); 3654df86ca0dSAndrii Nakryiko if (IS_ERR(prog)) 3655df86ca0dSAndrii Nakryiko return PTR_ERR(prog); 3656df86ca0dSAndrii Nakryiko 3657df86ca0dSAndrii Nakryiko fd = bpf_raw_tp_link_attach(prog, u64_to_user_ptr(attr->raw_tracepoint.name)); 3658df86ca0dSAndrii Nakryiko if (fd < 0) 3659df86ca0dSAndrii Nakryiko bpf_prog_put(prog); 3660df86ca0dSAndrii Nakryiko return fd; 3661df86ca0dSAndrii Nakryiko } 3662df86ca0dSAndrii Nakryiko 3663e28784e3SAndrii Nakryiko static enum bpf_prog_type 3664e28784e3SAndrii Nakryiko attach_type_to_prog_type(enum bpf_attach_type attach_type) 3665e28784e3SAndrii Nakryiko { 3666e28784e3SAndrii Nakryiko switch (attach_type) { 3667e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_INGRESS: 3668e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_EGRESS: 3669e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SKB; 3670e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_SOCK_CREATE: 3671f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 3672e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_POST_BIND: 3673e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_POST_BIND: 3674e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCK; 3675e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_BIND: 3676e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_BIND: 3677e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_CONNECT: 3678e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_CONNECT: 36791b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 36801b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 36811b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 36821b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 3683e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP4_SENDMSG: 3684e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP6_SENDMSG: 3685e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP4_RECVMSG: 3686e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP6_RECVMSG: 3687e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 3688e28784e3SAndrii Nakryiko case BPF_CGROUP_SOCK_OPS: 3689e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SOCK_OPS; 3690e28784e3SAndrii Nakryiko case BPF_CGROUP_DEVICE: 3691e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_DEVICE; 3692e28784e3SAndrii Nakryiko case BPF_SK_MSG_VERDICT: 3693e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SK_MSG; 3694e28784e3SAndrii Nakryiko case BPF_SK_SKB_STREAM_PARSER: 3695e28784e3SAndrii Nakryiko case BPF_SK_SKB_STREAM_VERDICT: 3696a7ba4558SCong Wang case BPF_SK_SKB_VERDICT: 3697e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SK_SKB; 3698e28784e3SAndrii Nakryiko case BPF_LIRC_MODE2: 3699e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_LIRC_MODE2; 3700e28784e3SAndrii Nakryiko case BPF_FLOW_DISSECTOR: 3701e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_FLOW_DISSECTOR; 3702e28784e3SAndrii Nakryiko case BPF_CGROUP_SYSCTL: 3703e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SYSCTL; 3704e28784e3SAndrii Nakryiko case BPF_CGROUP_GETSOCKOPT: 3705e28784e3SAndrii Nakryiko case BPF_CGROUP_SETSOCKOPT: 3706e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCKOPT; 3707de4e05caSYonghong Song case BPF_TRACE_ITER: 3708df86ca0dSAndrii Nakryiko case BPF_TRACE_RAW_TP: 3709df86ca0dSAndrii Nakryiko case BPF_TRACE_FENTRY: 3710df86ca0dSAndrii Nakryiko case BPF_TRACE_FEXIT: 3711df86ca0dSAndrii Nakryiko case BPF_MODIFY_RETURN: 3712de4e05caSYonghong Song return BPF_PROG_TYPE_TRACING; 3713df86ca0dSAndrii Nakryiko case BPF_LSM_MAC: 3714df86ca0dSAndrii Nakryiko return BPF_PROG_TYPE_LSM; 3715e9ddbb77SJakub Sitnicki case BPF_SK_LOOKUP: 3716e9ddbb77SJakub Sitnicki return BPF_PROG_TYPE_SK_LOOKUP; 3717aa8d3a71SAndrii Nakryiko case BPF_XDP: 3718aa8d3a71SAndrii Nakryiko return BPF_PROG_TYPE_XDP; 371969fd337aSStanislav Fomichev case BPF_LSM_CGROUP: 372069fd337aSStanislav Fomichev return BPF_PROG_TYPE_LSM; 3721e420bed0SDaniel Borkmann case BPF_TCX_INGRESS: 3722e420bed0SDaniel Borkmann case BPF_TCX_EGRESS: 3723e420bed0SDaniel Borkmann return BPF_PROG_TYPE_SCHED_CLS; 3724e28784e3SAndrii Nakryiko default: 3725e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_UNSPEC; 3726e28784e3SAndrii Nakryiko } 3727e28784e3SAndrii Nakryiko } 3728e28784e3SAndrii Nakryiko 37293505cb9fSJiri Olsa static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, 37303505cb9fSJiri Olsa enum bpf_attach_type attach_type) 37313505cb9fSJiri Olsa { 37323505cb9fSJiri Olsa enum bpf_prog_type ptype; 37333505cb9fSJiri Olsa 37343505cb9fSJiri Olsa switch (prog->type) { 37353505cb9fSJiri Olsa case BPF_PROG_TYPE_CGROUP_SOCK: 37363505cb9fSJiri Olsa case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 37373505cb9fSJiri Olsa case BPF_PROG_TYPE_CGROUP_SOCKOPT: 37383505cb9fSJiri Olsa case BPF_PROG_TYPE_SK_LOOKUP: 37393505cb9fSJiri Olsa return attach_type == prog->expected_attach_type ? 0 : -EINVAL; 37403505cb9fSJiri Olsa case BPF_PROG_TYPE_CGROUP_SKB: 37413505cb9fSJiri Olsa if (!capable(CAP_NET_ADMIN)) 37423505cb9fSJiri Olsa /* cg-skb progs can be loaded by unpriv user. 37433505cb9fSJiri Olsa * check permissions at attach time. 37443505cb9fSJiri Olsa */ 37453505cb9fSJiri Olsa return -EPERM; 37463505cb9fSJiri Olsa return prog->enforce_expected_attach_type && 37473505cb9fSJiri Olsa prog->expected_attach_type != attach_type ? 37483505cb9fSJiri Olsa -EINVAL : 0; 37493505cb9fSJiri Olsa case BPF_PROG_TYPE_EXT: 37503505cb9fSJiri Olsa return 0; 37513505cb9fSJiri Olsa case BPF_PROG_TYPE_NETFILTER: 37523505cb9fSJiri Olsa if (attach_type != BPF_NETFILTER) 37533505cb9fSJiri Olsa return -EINVAL; 37543505cb9fSJiri Olsa return 0; 37553505cb9fSJiri Olsa case BPF_PROG_TYPE_PERF_EVENT: 37563505cb9fSJiri Olsa case BPF_PROG_TYPE_TRACEPOINT: 37573505cb9fSJiri Olsa if (attach_type != BPF_PERF_EVENT) 37583505cb9fSJiri Olsa return -EINVAL; 37593505cb9fSJiri Olsa return 0; 37603505cb9fSJiri Olsa case BPF_PROG_TYPE_KPROBE: 37613505cb9fSJiri Olsa if (prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI && 37623505cb9fSJiri Olsa attach_type != BPF_TRACE_KPROBE_MULTI) 37633505cb9fSJiri Olsa return -EINVAL; 376489ae89f5SJiri Olsa if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI && 376589ae89f5SJiri Olsa attach_type != BPF_TRACE_UPROBE_MULTI) 376689ae89f5SJiri Olsa return -EINVAL; 37673505cb9fSJiri Olsa if (attach_type != BPF_PERF_EVENT && 376889ae89f5SJiri Olsa attach_type != BPF_TRACE_KPROBE_MULTI && 376989ae89f5SJiri Olsa attach_type != BPF_TRACE_UPROBE_MULTI) 37703505cb9fSJiri Olsa return -EINVAL; 37713505cb9fSJiri Olsa return 0; 37723505cb9fSJiri Olsa case BPF_PROG_TYPE_SCHED_CLS: 37733505cb9fSJiri Olsa if (attach_type != BPF_TCX_INGRESS && 37743505cb9fSJiri Olsa attach_type != BPF_TCX_EGRESS) 37753505cb9fSJiri Olsa return -EINVAL; 37763505cb9fSJiri Olsa return 0; 37773505cb9fSJiri Olsa default: 37783505cb9fSJiri Olsa ptype = attach_type_to_prog_type(attach_type); 37793505cb9fSJiri Olsa if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type) 37803505cb9fSJiri Olsa return -EINVAL; 37813505cb9fSJiri Olsa return 0; 37823505cb9fSJiri Olsa } 37833505cb9fSJiri Olsa } 37843505cb9fSJiri Olsa 3785e420bed0SDaniel Borkmann #define BPF_PROG_ATTACH_LAST_FIELD expected_revision 3786174a79ffSJohn Fastabend 3787e420bed0SDaniel Borkmann #define BPF_F_ATTACH_MASK_BASE \ 3788e420bed0SDaniel Borkmann (BPF_F_ALLOW_OVERRIDE | \ 3789e420bed0SDaniel Borkmann BPF_F_ALLOW_MULTI | \ 3790e420bed0SDaniel Borkmann BPF_F_REPLACE) 3791e420bed0SDaniel Borkmann 3792e420bed0SDaniel Borkmann #define BPF_F_ATTACH_MASK_MPROG \ 3793e420bed0SDaniel Borkmann (BPF_F_REPLACE | \ 3794e420bed0SDaniel Borkmann BPF_F_BEFORE | \ 3795e420bed0SDaniel Borkmann BPF_F_AFTER | \ 3796e420bed0SDaniel Borkmann BPF_F_ID | \ 3797e420bed0SDaniel Borkmann BPF_F_LINK) 3798324bda9eSAlexei Starovoitov 3799f4324551SDaniel Mack static int bpf_prog_attach(const union bpf_attr *attr) 3800f4324551SDaniel Mack { 38017f677633SAlexei Starovoitov enum bpf_prog_type ptype; 3802f4324551SDaniel Mack struct bpf_prog *prog; 3803e420bed0SDaniel Borkmann u32 mask; 38047f677633SAlexei Starovoitov int ret; 3805f4324551SDaniel Mack 3806f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_ATTACH)) 3807f4324551SDaniel Mack return -EINVAL; 3808f4324551SDaniel Mack 3809e28784e3SAndrii Nakryiko ptype = attach_type_to_prog_type(attr->attach_type); 3810e28784e3SAndrii Nakryiko if (ptype == BPF_PROG_TYPE_UNSPEC) 3811b2cd1257SDavid Ahern return -EINVAL; 3812e420bed0SDaniel Borkmann mask = bpf_mprog_supported(ptype) ? 3813e420bed0SDaniel Borkmann BPF_F_ATTACH_MASK_MPROG : BPF_F_ATTACH_MASK_BASE; 3814e420bed0SDaniel Borkmann if (attr->attach_flags & ~mask) 3815e420bed0SDaniel Borkmann return -EINVAL; 3816b2cd1257SDavid Ahern 3817b2cd1257SDavid Ahern prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 3818f4324551SDaniel Mack if (IS_ERR(prog)) 3819f4324551SDaniel Mack return PTR_ERR(prog); 3820f4324551SDaniel Mack 38215e43f899SAndrey Ignatov if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) { 38225e43f899SAndrey Ignatov bpf_prog_put(prog); 38235e43f899SAndrey Ignatov return -EINVAL; 38245e43f899SAndrey Ignatov } 38255e43f899SAndrey Ignatov 3826fdb5c453SSean Young switch (ptype) { 3827fdb5c453SSean Young case BPF_PROG_TYPE_SK_SKB: 3828fdb5c453SSean Young case BPF_PROG_TYPE_SK_MSG: 3829604326b4SDaniel Borkmann ret = sock_map_get_from_fd(attr, prog); 3830fdb5c453SSean Young break; 3831fdb5c453SSean Young case BPF_PROG_TYPE_LIRC_MODE2: 3832fdb5c453SSean Young ret = lirc_prog_attach(attr, prog); 3833fdb5c453SSean Young break; 3834d58e468bSPetar Penkov case BPF_PROG_TYPE_FLOW_DISSECTOR: 3835a3fd7ceeSJakub Sitnicki ret = netns_bpf_prog_attach(attr, prog); 3836d58e468bSPetar Penkov break; 3837e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 3838e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 3839e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 3840e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 3841e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 3842e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 3843e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 384469fd337aSStanislav Fomichev case BPF_PROG_TYPE_LSM: 384569fd337aSStanislav Fomichev if (ptype == BPF_PROG_TYPE_LSM && 384669fd337aSStanislav Fomichev prog->expected_attach_type != BPF_LSM_CGROUP) 3847e89f3edfSMilan Landaverde ret = -EINVAL; 3848e89f3edfSMilan Landaverde else 3849fdb5c453SSean Young ret = cgroup_bpf_prog_attach(attr, ptype, prog); 3850e28784e3SAndrii Nakryiko break; 3851e420bed0SDaniel Borkmann case BPF_PROG_TYPE_SCHED_CLS: 3852e420bed0SDaniel Borkmann ret = tcx_prog_attach(attr, prog); 3853e420bed0SDaniel Borkmann break; 3854e28784e3SAndrii Nakryiko default: 3855e28784e3SAndrii Nakryiko ret = -EINVAL; 3856f4324551SDaniel Mack } 3857f4324551SDaniel Mack 38587f677633SAlexei Starovoitov if (ret) 38597f677633SAlexei Starovoitov bpf_prog_put(prog); 38607f677633SAlexei Starovoitov return ret; 3861f4324551SDaniel Mack } 3862f4324551SDaniel Mack 3863e420bed0SDaniel Borkmann #define BPF_PROG_DETACH_LAST_FIELD expected_revision 3864f4324551SDaniel Mack 3865f4324551SDaniel Mack static int bpf_prog_detach(const union bpf_attr *attr) 3866f4324551SDaniel Mack { 3867e420bed0SDaniel Borkmann struct bpf_prog *prog = NULL; 3868324bda9eSAlexei Starovoitov enum bpf_prog_type ptype; 3869e420bed0SDaniel Borkmann int ret; 3870f4324551SDaniel Mack 3871f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_DETACH)) 3872f4324551SDaniel Mack return -EINVAL; 3873f4324551SDaniel Mack 3874e28784e3SAndrii Nakryiko ptype = attach_type_to_prog_type(attr->attach_type); 3875e420bed0SDaniel Borkmann if (bpf_mprog_supported(ptype)) { 3876e420bed0SDaniel Borkmann if (ptype == BPF_PROG_TYPE_UNSPEC) 3877e420bed0SDaniel Borkmann return -EINVAL; 3878e420bed0SDaniel Borkmann if (attr->attach_flags & ~BPF_F_ATTACH_MASK_MPROG) 3879e420bed0SDaniel Borkmann return -EINVAL; 3880e420bed0SDaniel Borkmann if (attr->attach_bpf_fd) { 3881e420bed0SDaniel Borkmann prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 3882e420bed0SDaniel Borkmann if (IS_ERR(prog)) 3883e420bed0SDaniel Borkmann return PTR_ERR(prog); 3884e420bed0SDaniel Borkmann } 3885e420bed0SDaniel Borkmann } 3886e28784e3SAndrii Nakryiko 3887e28784e3SAndrii Nakryiko switch (ptype) { 3888e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SK_MSG: 3889e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SK_SKB: 3890e420bed0SDaniel Borkmann ret = sock_map_prog_detach(attr, ptype); 3891e420bed0SDaniel Borkmann break; 3892e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_LIRC_MODE2: 3893e420bed0SDaniel Borkmann ret = lirc_prog_detach(attr); 3894e420bed0SDaniel Borkmann break; 3895e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_FLOW_DISSECTOR: 3896e420bed0SDaniel Borkmann ret = netns_bpf_prog_detach(attr, ptype); 3897e420bed0SDaniel Borkmann break; 3898e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 3899e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 3900e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 3901e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 3902e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 3903e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 3904e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 390569fd337aSStanislav Fomichev case BPF_PROG_TYPE_LSM: 3906e420bed0SDaniel Borkmann ret = cgroup_bpf_prog_detach(attr, ptype); 3907e420bed0SDaniel Borkmann break; 3908e420bed0SDaniel Borkmann case BPF_PROG_TYPE_SCHED_CLS: 3909e420bed0SDaniel Borkmann ret = tcx_prog_detach(attr, prog); 3910e420bed0SDaniel Borkmann break; 3911f4324551SDaniel Mack default: 3912e420bed0SDaniel Borkmann ret = -EINVAL; 3913f4324551SDaniel Mack } 391440304b2aSLawrence Brakmo 3915e420bed0SDaniel Borkmann if (prog) 3916e420bed0SDaniel Borkmann bpf_prog_put(prog); 3917e420bed0SDaniel Borkmann return ret; 3918e420bed0SDaniel Borkmann } 3919e420bed0SDaniel Borkmann 3920e420bed0SDaniel Borkmann #define BPF_PROG_QUERY_LAST_FIELD query.link_attach_flags 3921468e2f64SAlexei Starovoitov 3922468e2f64SAlexei Starovoitov static int bpf_prog_query(const union bpf_attr *attr, 3923468e2f64SAlexei Starovoitov union bpf_attr __user *uattr) 3924468e2f64SAlexei Starovoitov { 3925468e2f64SAlexei Starovoitov if (!capable(CAP_NET_ADMIN)) 3926468e2f64SAlexei Starovoitov return -EPERM; 3927468e2f64SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_QUERY)) 3928468e2f64SAlexei Starovoitov return -EINVAL; 3929468e2f64SAlexei Starovoitov if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE) 3930468e2f64SAlexei Starovoitov return -EINVAL; 3931468e2f64SAlexei Starovoitov 3932468e2f64SAlexei Starovoitov switch (attr->query.attach_type) { 3933468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_INGRESS: 3934468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_EGRESS: 3935468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_SOCK_CREATE: 3936f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 39374fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 39384fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 3939aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 3940aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 3941d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 3942d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 39431b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 39441b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 39451b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 39461b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 39471cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 39481cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 3949983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG: 3950983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG: 3951468e2f64SAlexei Starovoitov case BPF_CGROUP_SOCK_OPS: 3952ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 39537b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 39540d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT: 39550d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT: 3956b79c9fc9SStanislav Fomichev case BPF_LSM_CGROUP: 3957e28784e3SAndrii Nakryiko return cgroup_bpf_prog_query(attr, uattr); 3958f4364dcfSSean Young case BPF_LIRC_MODE2: 3959f4364dcfSSean Young return lirc_prog_query(attr, uattr); 3960118c8e9aSStanislav Fomichev case BPF_FLOW_DISSECTOR: 3961e9ddbb77SJakub Sitnicki case BPF_SK_LOOKUP: 3962a3fd7ceeSJakub Sitnicki return netns_bpf_prog_query(attr, uattr); 3963748cd572SDi Zhu case BPF_SK_SKB_STREAM_PARSER: 3964748cd572SDi Zhu case BPF_SK_SKB_STREAM_VERDICT: 3965748cd572SDi Zhu case BPF_SK_MSG_VERDICT: 3966748cd572SDi Zhu case BPF_SK_SKB_VERDICT: 3967748cd572SDi Zhu return sock_map_bpf_prog_query(attr, uattr); 3968e420bed0SDaniel Borkmann case BPF_TCX_INGRESS: 3969e420bed0SDaniel Borkmann case BPF_TCX_EGRESS: 3970e420bed0SDaniel Borkmann return tcx_prog_query(attr, uattr); 3971468e2f64SAlexei Starovoitov default: 3972468e2f64SAlexei Starovoitov return -EINVAL; 3973468e2f64SAlexei Starovoitov } 3974468e2f64SAlexei Starovoitov } 3975f4324551SDaniel Mack 3976b530e9e1SToke Høiland-Jørgensen #define BPF_PROG_TEST_RUN_LAST_FIELD test.batch_size 39771cf1cae9SAlexei Starovoitov 39781cf1cae9SAlexei Starovoitov static int bpf_prog_test_run(const union bpf_attr *attr, 39791cf1cae9SAlexei Starovoitov union bpf_attr __user *uattr) 39801cf1cae9SAlexei Starovoitov { 39811cf1cae9SAlexei Starovoitov struct bpf_prog *prog; 39821cf1cae9SAlexei Starovoitov int ret = -ENOTSUPP; 39831cf1cae9SAlexei Starovoitov 39841cf1cae9SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 39851cf1cae9SAlexei Starovoitov return -EINVAL; 39861cf1cae9SAlexei Starovoitov 3987b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_in && !attr->test.ctx_in) || 3988b0b9395dSStanislav Fomichev (!attr->test.ctx_size_in && attr->test.ctx_in)) 3989b0b9395dSStanislav Fomichev return -EINVAL; 3990b0b9395dSStanislav Fomichev 3991b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_out && !attr->test.ctx_out) || 3992b0b9395dSStanislav Fomichev (!attr->test.ctx_size_out && attr->test.ctx_out)) 3993b0b9395dSStanislav Fomichev return -EINVAL; 3994b0b9395dSStanislav Fomichev 39951cf1cae9SAlexei Starovoitov prog = bpf_prog_get(attr->test.prog_fd); 39961cf1cae9SAlexei Starovoitov if (IS_ERR(prog)) 39971cf1cae9SAlexei Starovoitov return PTR_ERR(prog); 39981cf1cae9SAlexei Starovoitov 39991cf1cae9SAlexei Starovoitov if (prog->aux->ops->test_run) 40001cf1cae9SAlexei Starovoitov ret = prog->aux->ops->test_run(prog, attr, uattr); 40011cf1cae9SAlexei Starovoitov 40021cf1cae9SAlexei Starovoitov bpf_prog_put(prog); 40031cf1cae9SAlexei Starovoitov return ret; 40041cf1cae9SAlexei Starovoitov } 40051cf1cae9SAlexei Starovoitov 400634ad5580SMartin KaFai Lau #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 400734ad5580SMartin KaFai Lau 400834ad5580SMartin KaFai Lau static int bpf_obj_get_next_id(const union bpf_attr *attr, 400934ad5580SMartin KaFai Lau union bpf_attr __user *uattr, 401034ad5580SMartin KaFai Lau struct idr *idr, 401134ad5580SMartin KaFai Lau spinlock_t *lock) 401234ad5580SMartin KaFai Lau { 401334ad5580SMartin KaFai Lau u32 next_id = attr->start_id; 401434ad5580SMartin KaFai Lau int err = 0; 401534ad5580SMartin KaFai Lau 401634ad5580SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 401734ad5580SMartin KaFai Lau return -EINVAL; 401834ad5580SMartin KaFai Lau 401934ad5580SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 402034ad5580SMartin KaFai Lau return -EPERM; 402134ad5580SMartin KaFai Lau 402234ad5580SMartin KaFai Lau next_id++; 402334ad5580SMartin KaFai Lau spin_lock_bh(lock); 402434ad5580SMartin KaFai Lau if (!idr_get_next(idr, &next_id)) 402534ad5580SMartin KaFai Lau err = -ENOENT; 402634ad5580SMartin KaFai Lau spin_unlock_bh(lock); 402734ad5580SMartin KaFai Lau 402834ad5580SMartin KaFai Lau if (!err) 402934ad5580SMartin KaFai Lau err = put_user(next_id, &uattr->next_id); 403034ad5580SMartin KaFai Lau 403134ad5580SMartin KaFai Lau return err; 403234ad5580SMartin KaFai Lau } 403334ad5580SMartin KaFai Lau 40346086d29dSYonghong Song struct bpf_map *bpf_map_get_curr_or_next(u32 *id) 40356086d29dSYonghong Song { 40366086d29dSYonghong Song struct bpf_map *map; 40376086d29dSYonghong Song 40386086d29dSYonghong Song spin_lock_bh(&map_idr_lock); 40396086d29dSYonghong Song again: 40406086d29dSYonghong Song map = idr_get_next(&map_idr, id); 40416086d29dSYonghong Song if (map) { 40426086d29dSYonghong Song map = __bpf_map_inc_not_zero(map, false); 40436086d29dSYonghong Song if (IS_ERR(map)) { 40446086d29dSYonghong Song (*id)++; 40456086d29dSYonghong Song goto again; 40466086d29dSYonghong Song } 40476086d29dSYonghong Song } 40486086d29dSYonghong Song spin_unlock_bh(&map_idr_lock); 40496086d29dSYonghong Song 40506086d29dSYonghong Song return map; 40516086d29dSYonghong Song } 40526086d29dSYonghong Song 4053a228a64fSAlexei Starovoitov struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id) 4054a228a64fSAlexei Starovoitov { 4055a228a64fSAlexei Starovoitov struct bpf_prog *prog; 4056a228a64fSAlexei Starovoitov 4057a228a64fSAlexei Starovoitov spin_lock_bh(&prog_idr_lock); 4058a228a64fSAlexei Starovoitov again: 4059a228a64fSAlexei Starovoitov prog = idr_get_next(&prog_idr, id); 4060a228a64fSAlexei Starovoitov if (prog) { 4061a228a64fSAlexei Starovoitov prog = bpf_prog_inc_not_zero(prog); 4062a228a64fSAlexei Starovoitov if (IS_ERR(prog)) { 4063a228a64fSAlexei Starovoitov (*id)++; 4064a228a64fSAlexei Starovoitov goto again; 4065a228a64fSAlexei Starovoitov } 4066a228a64fSAlexei Starovoitov } 4067a228a64fSAlexei Starovoitov spin_unlock_bh(&prog_idr_lock); 4068a228a64fSAlexei Starovoitov 4069a228a64fSAlexei Starovoitov return prog; 4070a228a64fSAlexei Starovoitov } 4071a228a64fSAlexei Starovoitov 4072b16d9aa4SMartin KaFai Lau #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 4073b16d9aa4SMartin KaFai Lau 40747e6897f9SBjörn Töpel struct bpf_prog *bpf_prog_by_id(u32 id) 40757e6897f9SBjörn Töpel { 40767e6897f9SBjörn Töpel struct bpf_prog *prog; 40777e6897f9SBjörn Töpel 40787e6897f9SBjörn Töpel if (!id) 40797e6897f9SBjörn Töpel return ERR_PTR(-ENOENT); 40807e6897f9SBjörn Töpel 40817e6897f9SBjörn Töpel spin_lock_bh(&prog_idr_lock); 40827e6897f9SBjörn Töpel prog = idr_find(&prog_idr, id); 40837e6897f9SBjörn Töpel if (prog) 40847e6897f9SBjörn Töpel prog = bpf_prog_inc_not_zero(prog); 40857e6897f9SBjörn Töpel else 40867e6897f9SBjörn Töpel prog = ERR_PTR(-ENOENT); 40877e6897f9SBjörn Töpel spin_unlock_bh(&prog_idr_lock); 40887e6897f9SBjörn Töpel return prog; 40897e6897f9SBjörn Töpel } 40907e6897f9SBjörn Töpel 4091b16d9aa4SMartin KaFai Lau static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 4092b16d9aa4SMartin KaFai Lau { 4093b16d9aa4SMartin KaFai Lau struct bpf_prog *prog; 4094b16d9aa4SMartin KaFai Lau u32 id = attr->prog_id; 4095b16d9aa4SMartin KaFai Lau int fd; 4096b16d9aa4SMartin KaFai Lau 4097b16d9aa4SMartin KaFai Lau if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 4098b16d9aa4SMartin KaFai Lau return -EINVAL; 4099b16d9aa4SMartin KaFai Lau 4100b16d9aa4SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 4101b16d9aa4SMartin KaFai Lau return -EPERM; 4102b16d9aa4SMartin KaFai Lau 41037e6897f9SBjörn Töpel prog = bpf_prog_by_id(id); 4104b16d9aa4SMartin KaFai Lau if (IS_ERR(prog)) 4105b16d9aa4SMartin KaFai Lau return PTR_ERR(prog); 4106b16d9aa4SMartin KaFai Lau 4107b16d9aa4SMartin KaFai Lau fd = bpf_prog_new_fd(prog); 4108b16d9aa4SMartin KaFai Lau if (fd < 0) 4109b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 4110b16d9aa4SMartin KaFai Lau 4111b16d9aa4SMartin KaFai Lau return fd; 4112b16d9aa4SMartin KaFai Lau } 4113b16d9aa4SMartin KaFai Lau 41146e71b04aSChenbo Feng #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags 4115bd5f5f4eSMartin KaFai Lau 4116bd5f5f4eSMartin KaFai Lau static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 4117bd5f5f4eSMartin KaFai Lau { 4118bd5f5f4eSMartin KaFai Lau struct bpf_map *map; 4119bd5f5f4eSMartin KaFai Lau u32 id = attr->map_id; 41206e71b04aSChenbo Feng int f_flags; 4121bd5f5f4eSMartin KaFai Lau int fd; 4122bd5f5f4eSMartin KaFai Lau 41236e71b04aSChenbo Feng if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || 41246e71b04aSChenbo Feng attr->open_flags & ~BPF_OBJ_FLAG_MASK) 4125bd5f5f4eSMartin KaFai Lau return -EINVAL; 4126bd5f5f4eSMartin KaFai Lau 4127bd5f5f4eSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 4128bd5f5f4eSMartin KaFai Lau return -EPERM; 4129bd5f5f4eSMartin KaFai Lau 41306e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->open_flags); 41316e71b04aSChenbo Feng if (f_flags < 0) 41326e71b04aSChenbo Feng return f_flags; 41336e71b04aSChenbo Feng 4134bd5f5f4eSMartin KaFai Lau spin_lock_bh(&map_idr_lock); 4135bd5f5f4eSMartin KaFai Lau map = idr_find(&map_idr, id); 4136bd5f5f4eSMartin KaFai Lau if (map) 4137b0e4701cSStanislav Fomichev map = __bpf_map_inc_not_zero(map, true); 4138bd5f5f4eSMartin KaFai Lau else 4139bd5f5f4eSMartin KaFai Lau map = ERR_PTR(-ENOENT); 4140bd5f5f4eSMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 4141bd5f5f4eSMartin KaFai Lau 4142bd5f5f4eSMartin KaFai Lau if (IS_ERR(map)) 4143bd5f5f4eSMartin KaFai Lau return PTR_ERR(map); 4144bd5f5f4eSMartin KaFai Lau 41456e71b04aSChenbo Feng fd = bpf_map_new_fd(map, f_flags); 4146bd5f5f4eSMartin KaFai Lau if (fd < 0) 4147781e6282SPeng Sun bpf_map_put_with_uref(map); 4148bd5f5f4eSMartin KaFai Lau 4149bd5f5f4eSMartin KaFai Lau return fd; 4150bd5f5f4eSMartin KaFai Lau } 4151bd5f5f4eSMartin KaFai Lau 41527105e828SDaniel Borkmann static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, 4153d8eca5bbSDaniel Borkmann unsigned long addr, u32 *off, 4154d8eca5bbSDaniel Borkmann u32 *type) 41557105e828SDaniel Borkmann { 4156d8eca5bbSDaniel Borkmann const struct bpf_map *map; 41577105e828SDaniel Borkmann int i; 41587105e828SDaniel Borkmann 4159984fe94fSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 4160d8eca5bbSDaniel Borkmann for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { 4161d8eca5bbSDaniel Borkmann map = prog->aux->used_maps[i]; 4162d8eca5bbSDaniel Borkmann if (map == (void *)addr) { 4163d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_FD; 4164984fe94fSYiFei Zhu goto out; 4165d8eca5bbSDaniel Borkmann } 4166d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta) 4167d8eca5bbSDaniel Borkmann continue; 4168d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta(map, addr, off)) { 4169d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_VALUE; 4170984fe94fSYiFei Zhu goto out; 4171d8eca5bbSDaniel Borkmann } 4172d8eca5bbSDaniel Borkmann } 4173984fe94fSYiFei Zhu map = NULL; 4174d8eca5bbSDaniel Borkmann 4175984fe94fSYiFei Zhu out: 4176984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 4177984fe94fSYiFei Zhu return map; 41787105e828SDaniel Borkmann } 41797105e828SDaniel Borkmann 418063960260SKees Cook static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, 418163960260SKees Cook const struct cred *f_cred) 41827105e828SDaniel Borkmann { 41837105e828SDaniel Borkmann const struct bpf_map *map; 41847105e828SDaniel Borkmann struct bpf_insn *insns; 4185d8eca5bbSDaniel Borkmann u32 off, type; 41867105e828SDaniel Borkmann u64 imm; 418729fcb05bSAndrii Nakryiko u8 code; 41887105e828SDaniel Borkmann int i; 41897105e828SDaniel Borkmann 41907105e828SDaniel Borkmann insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), 41917105e828SDaniel Borkmann GFP_USER); 41927105e828SDaniel Borkmann if (!insns) 41937105e828SDaniel Borkmann return insns; 41947105e828SDaniel Borkmann 41957105e828SDaniel Borkmann for (i = 0; i < prog->len; i++) { 419629fcb05bSAndrii Nakryiko code = insns[i].code; 419729fcb05bSAndrii Nakryiko 419829fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_TAIL_CALL)) { 41997105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 42007105e828SDaniel Borkmann insns[i].imm = BPF_FUNC_tail_call; 42017105e828SDaniel Borkmann /* fall-through */ 42027105e828SDaniel Borkmann } 420329fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_CALL) || 420429fcb05bSAndrii Nakryiko code == (BPF_JMP | BPF_CALL_ARGS)) { 420529fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_CALL_ARGS)) 42067105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 420763960260SKees Cook if (!bpf_dump_raw_ok(f_cred)) 42087105e828SDaniel Borkmann insns[i].imm = 0; 42097105e828SDaniel Borkmann continue; 42107105e828SDaniel Borkmann } 421129fcb05bSAndrii Nakryiko if (BPF_CLASS(code) == BPF_LDX && BPF_MODE(code) == BPF_PROBE_MEM) { 421229fcb05bSAndrii Nakryiko insns[i].code = BPF_LDX | BPF_SIZE(code) | BPF_MEM; 421329fcb05bSAndrii Nakryiko continue; 421429fcb05bSAndrii Nakryiko } 42157105e828SDaniel Borkmann 421629fcb05bSAndrii Nakryiko if (code != (BPF_LD | BPF_IMM | BPF_DW)) 42177105e828SDaniel Borkmann continue; 42187105e828SDaniel Borkmann 42197105e828SDaniel Borkmann imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; 4220d8eca5bbSDaniel Borkmann map = bpf_map_from_imm(prog, imm, &off, &type); 42217105e828SDaniel Borkmann if (map) { 4222d8eca5bbSDaniel Borkmann insns[i].src_reg = type; 42237105e828SDaniel Borkmann insns[i].imm = map->id; 4224d8eca5bbSDaniel Borkmann insns[i + 1].imm = off; 42257105e828SDaniel Borkmann continue; 42267105e828SDaniel Borkmann } 42277105e828SDaniel Borkmann } 42287105e828SDaniel Borkmann 42297105e828SDaniel Borkmann return insns; 42307105e828SDaniel Borkmann } 42317105e828SDaniel Borkmann 4232c454a46bSMartin KaFai Lau static int set_info_rec_size(struct bpf_prog_info *info) 4233c454a46bSMartin KaFai Lau { 4234c454a46bSMartin KaFai Lau /* 4235c454a46bSMartin KaFai Lau * Ensure info.*_rec_size is the same as kernel expected size 4236c454a46bSMartin KaFai Lau * 4237c454a46bSMartin KaFai Lau * or 4238c454a46bSMartin KaFai Lau * 4239c454a46bSMartin KaFai Lau * Only allow zero *_rec_size if both _rec_size and _cnt are 4240c454a46bSMartin KaFai Lau * zero. In this case, the kernel will set the expected 4241c454a46bSMartin KaFai Lau * _rec_size back to the info. 4242c454a46bSMartin KaFai Lau */ 4243c454a46bSMartin KaFai Lau 424411d8b82dSYonghong Song if ((info->nr_func_info || info->func_info_rec_size) && 4245c454a46bSMartin KaFai Lau info->func_info_rec_size != sizeof(struct bpf_func_info)) 4246c454a46bSMartin KaFai Lau return -EINVAL; 4247c454a46bSMartin KaFai Lau 424811d8b82dSYonghong Song if ((info->nr_line_info || info->line_info_rec_size) && 4249c454a46bSMartin KaFai Lau info->line_info_rec_size != sizeof(struct bpf_line_info)) 4250c454a46bSMartin KaFai Lau return -EINVAL; 4251c454a46bSMartin KaFai Lau 425211d8b82dSYonghong Song if ((info->nr_jited_line_info || info->jited_line_info_rec_size) && 4253c454a46bSMartin KaFai Lau info->jited_line_info_rec_size != sizeof(__u64)) 4254c454a46bSMartin KaFai Lau return -EINVAL; 4255c454a46bSMartin KaFai Lau 4256c454a46bSMartin KaFai Lau info->func_info_rec_size = sizeof(struct bpf_func_info); 4257c454a46bSMartin KaFai Lau info->line_info_rec_size = sizeof(struct bpf_line_info); 4258c454a46bSMartin KaFai Lau info->jited_line_info_rec_size = sizeof(__u64); 4259c454a46bSMartin KaFai Lau 4260c454a46bSMartin KaFai Lau return 0; 4261c454a46bSMartin KaFai Lau } 4262c454a46bSMartin KaFai Lau 426363960260SKees Cook static int bpf_prog_get_info_by_fd(struct file *file, 426463960260SKees Cook struct bpf_prog *prog, 42651e270976SMartin KaFai Lau const union bpf_attr *attr, 42661e270976SMartin KaFai Lau union bpf_attr __user *uattr) 42671e270976SMartin KaFai Lau { 42681e270976SMartin KaFai Lau struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 42696644aabbSStanislav Fomichev struct btf *attach_btf = bpf_prog_get_target_btf(prog); 42705c6f2588SGreg Kroah-Hartman struct bpf_prog_info info; 42711e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 427261a0abaeSEric Dumazet struct bpf_prog_kstats stats; 42731e270976SMartin KaFai Lau char __user *uinsns; 42741e270976SMartin KaFai Lau u32 ulen; 42751e270976SMartin KaFai Lau int err; 42761e270976SMartin KaFai Lau 4277af2ac3e1SAlexei Starovoitov err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len); 42781e270976SMartin KaFai Lau if (err) 42791e270976SMartin KaFai Lau return err; 42801e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 42811e270976SMartin KaFai Lau 42825c6f2588SGreg Kroah-Hartman memset(&info, 0, sizeof(info)); 42831e270976SMartin KaFai Lau if (copy_from_user(&info, uinfo, info_len)) 428489b09689SDaniel Borkmann return -EFAULT; 42851e270976SMartin KaFai Lau 42861e270976SMartin KaFai Lau info.type = prog->type; 42871e270976SMartin KaFai Lau info.id = prog->aux->id; 4288cb4d2b3fSMartin KaFai Lau info.load_time = prog->aux->load_time; 4289cb4d2b3fSMartin KaFai Lau info.created_by_uid = from_kuid_munged(current_user_ns(), 4290cb4d2b3fSMartin KaFai Lau prog->aux->user->uid); 4291b85fab0eSJiri Olsa info.gpl_compatible = prog->gpl_compatible; 42921e270976SMartin KaFai Lau 42931e270976SMartin KaFai Lau memcpy(info.tag, prog->tag, sizeof(prog->tag)); 4294cb4d2b3fSMartin KaFai Lau memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); 4295cb4d2b3fSMartin KaFai Lau 4296984fe94fSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 4297cb4d2b3fSMartin KaFai Lau ulen = info.nr_map_ids; 4298cb4d2b3fSMartin KaFai Lau info.nr_map_ids = prog->aux->used_map_cnt; 4299cb4d2b3fSMartin KaFai Lau ulen = min_t(u32, info.nr_map_ids, ulen); 4300cb4d2b3fSMartin KaFai Lau if (ulen) { 4301721e08daSMartin KaFai Lau u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); 4302cb4d2b3fSMartin KaFai Lau u32 i; 4303cb4d2b3fSMartin KaFai Lau 4304cb4d2b3fSMartin KaFai Lau for (i = 0; i < ulen; i++) 4305cb4d2b3fSMartin KaFai Lau if (put_user(prog->aux->used_maps[i]->id, 4306984fe94fSYiFei Zhu &user_map_ids[i])) { 4307984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 4308cb4d2b3fSMartin KaFai Lau return -EFAULT; 4309cb4d2b3fSMartin KaFai Lau } 4310984fe94fSYiFei Zhu } 4311984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 43121e270976SMartin KaFai Lau 4313c454a46bSMartin KaFai Lau err = set_info_rec_size(&info); 4314c454a46bSMartin KaFai Lau if (err) 4315c454a46bSMartin KaFai Lau return err; 43167337224fSMartin KaFai Lau 43175f8f8b93SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 43185f8f8b93SAlexei Starovoitov info.run_time_ns = stats.nsecs; 43195f8f8b93SAlexei Starovoitov info.run_cnt = stats.cnt; 43209ed9e9baSAlexei Starovoitov info.recursion_misses = stats.misses; 43215f8f8b93SAlexei Starovoitov 4322aba64c7dSDave Marchevsky info.verified_insns = prog->aux->verified_insns; 4323aba64c7dSDave Marchevsky 43242c78ee89SAlexei Starovoitov if (!bpf_capable()) { 43251e270976SMartin KaFai Lau info.jited_prog_len = 0; 43261e270976SMartin KaFai Lau info.xlated_prog_len = 0; 4327dbecd738SSandipan Das info.nr_jited_ksyms = 0; 432828c2fae7SDaniel Borkmann info.nr_jited_func_lens = 0; 432911d8b82dSYonghong Song info.nr_func_info = 0; 433011d8b82dSYonghong Song info.nr_line_info = 0; 433111d8b82dSYonghong Song info.nr_jited_line_info = 0; 43321e270976SMartin KaFai Lau goto done; 43331e270976SMartin KaFai Lau } 43341e270976SMartin KaFai Lau 43351e270976SMartin KaFai Lau ulen = info.xlated_prog_len; 43369975a54bSDaniel Borkmann info.xlated_prog_len = bpf_prog_insn_size(prog); 43371e270976SMartin KaFai Lau if (info.xlated_prog_len && ulen) { 43387105e828SDaniel Borkmann struct bpf_insn *insns_sanitized; 43397105e828SDaniel Borkmann bool fault; 43407105e828SDaniel Borkmann 434163960260SKees Cook if (prog->blinded && !bpf_dump_raw_ok(file->f_cred)) { 43427105e828SDaniel Borkmann info.xlated_prog_insns = 0; 43437105e828SDaniel Borkmann goto done; 43447105e828SDaniel Borkmann } 434563960260SKees Cook insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred); 43467105e828SDaniel Borkmann if (!insns_sanitized) 43477105e828SDaniel Borkmann return -ENOMEM; 43481e270976SMartin KaFai Lau uinsns = u64_to_user_ptr(info.xlated_prog_insns); 43491e270976SMartin KaFai Lau ulen = min_t(u32, info.xlated_prog_len, ulen); 43507105e828SDaniel Borkmann fault = copy_to_user(uinsns, insns_sanitized, ulen); 43517105e828SDaniel Borkmann kfree(insns_sanitized); 43527105e828SDaniel Borkmann if (fault) 43531e270976SMartin KaFai Lau return -EFAULT; 43541e270976SMartin KaFai Lau } 43551e270976SMartin KaFai Lau 43569d03ebc7SStanislav Fomichev if (bpf_prog_is_offloaded(prog->aux)) { 4357675fc275SJakub Kicinski err = bpf_prog_offload_info_fill(&info, prog); 4358675fc275SJakub Kicinski if (err) 4359675fc275SJakub Kicinski return err; 4360fcfb126dSJiong Wang goto done; 4361fcfb126dSJiong Wang } 4362fcfb126dSJiong Wang 4363fcfb126dSJiong Wang /* NOTE: the following code is supposed to be skipped for offload. 4364fcfb126dSJiong Wang * bpf_prog_offload_info_fill() is the place to fill similar fields 4365fcfb126dSJiong Wang * for offload. 4366fcfb126dSJiong Wang */ 4367fcfb126dSJiong Wang ulen = info.jited_prog_len; 43684d56a76eSSandipan Das if (prog->aux->func_cnt) { 43694d56a76eSSandipan Das u32 i; 43704d56a76eSSandipan Das 43714d56a76eSSandipan Das info.jited_prog_len = 0; 43724d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) 43734d56a76eSSandipan Das info.jited_prog_len += prog->aux->func[i]->jited_len; 43744d56a76eSSandipan Das } else { 4375fcfb126dSJiong Wang info.jited_prog_len = prog->jited_len; 43764d56a76eSSandipan Das } 43774d56a76eSSandipan Das 4378fcfb126dSJiong Wang if (info.jited_prog_len && ulen) { 437963960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 4380fcfb126dSJiong Wang uinsns = u64_to_user_ptr(info.jited_prog_insns); 4381fcfb126dSJiong Wang ulen = min_t(u32, info.jited_prog_len, ulen); 43824d56a76eSSandipan Das 43834d56a76eSSandipan Das /* for multi-function programs, copy the JITed 43844d56a76eSSandipan Das * instructions for all the functions 43854d56a76eSSandipan Das */ 43864d56a76eSSandipan Das if (prog->aux->func_cnt) { 43874d56a76eSSandipan Das u32 len, free, i; 43884d56a76eSSandipan Das u8 *img; 43894d56a76eSSandipan Das 43904d56a76eSSandipan Das free = ulen; 43914d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) { 43924d56a76eSSandipan Das len = prog->aux->func[i]->jited_len; 43934d56a76eSSandipan Das len = min_t(u32, len, free); 43944d56a76eSSandipan Das img = (u8 *) prog->aux->func[i]->bpf_func; 43954d56a76eSSandipan Das if (copy_to_user(uinsns, img, len)) 43964d56a76eSSandipan Das return -EFAULT; 43974d56a76eSSandipan Das uinsns += len; 43984d56a76eSSandipan Das free -= len; 43994d56a76eSSandipan Das if (!free) 44004d56a76eSSandipan Das break; 44014d56a76eSSandipan Das } 44024d56a76eSSandipan Das } else { 4403fcfb126dSJiong Wang if (copy_to_user(uinsns, prog->bpf_func, ulen)) 4404fcfb126dSJiong Wang return -EFAULT; 44054d56a76eSSandipan Das } 4406fcfb126dSJiong Wang } else { 4407fcfb126dSJiong Wang info.jited_prog_insns = 0; 4408fcfb126dSJiong Wang } 4409675fc275SJakub Kicinski } 4410675fc275SJakub Kicinski 4411dbecd738SSandipan Das ulen = info.nr_jited_ksyms; 4412ff1889fcSSong Liu info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; 44137a5725ddSSong Liu if (ulen) { 441463960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 4415ff1889fcSSong Liu unsigned long ksym_addr; 4416dbecd738SSandipan Das u64 __user *user_ksyms; 4417dbecd738SSandipan Das u32 i; 4418dbecd738SSandipan Das 4419dbecd738SSandipan Das /* copy the address of the kernel symbol 4420dbecd738SSandipan Das * corresponding to each function 4421dbecd738SSandipan Das */ 4422dbecd738SSandipan Das ulen = min_t(u32, info.nr_jited_ksyms, ulen); 4423dbecd738SSandipan Das user_ksyms = u64_to_user_ptr(info.jited_ksyms); 4424ff1889fcSSong Liu if (prog->aux->func_cnt) { 4425dbecd738SSandipan Das for (i = 0; i < ulen; i++) { 4426ff1889fcSSong Liu ksym_addr = (unsigned long) 4427ff1889fcSSong Liu prog->aux->func[i]->bpf_func; 4428ff1889fcSSong Liu if (put_user((u64) ksym_addr, 4429ff1889fcSSong Liu &user_ksyms[i])) 4430ff1889fcSSong Liu return -EFAULT; 4431ff1889fcSSong Liu } 4432ff1889fcSSong Liu } else { 4433ff1889fcSSong Liu ksym_addr = (unsigned long) prog->bpf_func; 4434ff1889fcSSong Liu if (put_user((u64) ksym_addr, &user_ksyms[0])) 4435dbecd738SSandipan Das return -EFAULT; 4436dbecd738SSandipan Das } 4437dbecd738SSandipan Das } else { 4438dbecd738SSandipan Das info.jited_ksyms = 0; 4439dbecd738SSandipan Das } 4440dbecd738SSandipan Das } 4441dbecd738SSandipan Das 4442815581c1SSandipan Das ulen = info.nr_jited_func_lens; 4443ff1889fcSSong Liu info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; 44447a5725ddSSong Liu if (ulen) { 444563960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 4446815581c1SSandipan Das u32 __user *user_lens; 4447815581c1SSandipan Das u32 func_len, i; 4448815581c1SSandipan Das 4449815581c1SSandipan Das /* copy the JITed image lengths for each function */ 4450815581c1SSandipan Das ulen = min_t(u32, info.nr_jited_func_lens, ulen); 4451815581c1SSandipan Das user_lens = u64_to_user_ptr(info.jited_func_lens); 4452ff1889fcSSong Liu if (prog->aux->func_cnt) { 4453815581c1SSandipan Das for (i = 0; i < ulen; i++) { 4454ff1889fcSSong Liu func_len = 4455ff1889fcSSong Liu prog->aux->func[i]->jited_len; 4456815581c1SSandipan Das if (put_user(func_len, &user_lens[i])) 4457815581c1SSandipan Das return -EFAULT; 4458815581c1SSandipan Das } 4459815581c1SSandipan Das } else { 4460ff1889fcSSong Liu func_len = prog->jited_len; 4461ff1889fcSSong Liu if (put_user(func_len, &user_lens[0])) 4462ff1889fcSSong Liu return -EFAULT; 4463ff1889fcSSong Liu } 4464ff1889fcSSong Liu } else { 4465815581c1SSandipan Das info.jited_func_lens = 0; 4466815581c1SSandipan Das } 4467815581c1SSandipan Das } 4468815581c1SSandipan Das 44697337224fSMartin KaFai Lau if (prog->aux->btf) 447022dc4a0fSAndrii Nakryiko info.btf_id = btf_obj_id(prog->aux->btf); 4471b79c9fc9SStanislav Fomichev info.attach_btf_id = prog->aux->attach_btf_id; 44726644aabbSStanislav Fomichev if (attach_btf) 44736644aabbSStanislav Fomichev info.attach_btf_obj_id = btf_obj_id(attach_btf); 4474838e9690SYonghong Song 447511d8b82dSYonghong Song ulen = info.nr_func_info; 447611d8b82dSYonghong Song info.nr_func_info = prog->aux->func_info_cnt; 447711d8b82dSYonghong Song if (info.nr_func_info && ulen) { 4478838e9690SYonghong Song char __user *user_finfo; 4479838e9690SYonghong Song 4480838e9690SYonghong Song user_finfo = u64_to_user_ptr(info.func_info); 448111d8b82dSYonghong Song ulen = min_t(u32, info.nr_func_info, ulen); 4482ba64e7d8SYonghong Song if (copy_to_user(user_finfo, prog->aux->func_info, 44837337224fSMartin KaFai Lau info.func_info_rec_size * ulen)) 4484838e9690SYonghong Song return -EFAULT; 4485838e9690SYonghong Song } 4486838e9690SYonghong Song 448711d8b82dSYonghong Song ulen = info.nr_line_info; 448811d8b82dSYonghong Song info.nr_line_info = prog->aux->nr_linfo; 448911d8b82dSYonghong Song if (info.nr_line_info && ulen) { 4490c454a46bSMartin KaFai Lau __u8 __user *user_linfo; 4491c454a46bSMartin KaFai Lau 4492c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.line_info); 449311d8b82dSYonghong Song ulen = min_t(u32, info.nr_line_info, ulen); 4494c454a46bSMartin KaFai Lau if (copy_to_user(user_linfo, prog->aux->linfo, 4495c454a46bSMartin KaFai Lau info.line_info_rec_size * ulen)) 4496c454a46bSMartin KaFai Lau return -EFAULT; 4497c454a46bSMartin KaFai Lau } 4498c454a46bSMartin KaFai Lau 449911d8b82dSYonghong Song ulen = info.nr_jited_line_info; 4500c454a46bSMartin KaFai Lau if (prog->aux->jited_linfo) 450111d8b82dSYonghong Song info.nr_jited_line_info = prog->aux->nr_linfo; 4502c454a46bSMartin KaFai Lau else 450311d8b82dSYonghong Song info.nr_jited_line_info = 0; 450411d8b82dSYonghong Song if (info.nr_jited_line_info && ulen) { 450563960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 45062cd00852SPu Lehui unsigned long line_addr; 4507c454a46bSMartin KaFai Lau __u64 __user *user_linfo; 4508c454a46bSMartin KaFai Lau u32 i; 4509c454a46bSMartin KaFai Lau 4510c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.jited_line_info); 451111d8b82dSYonghong Song ulen = min_t(u32, info.nr_jited_line_info, ulen); 4512c454a46bSMartin KaFai Lau for (i = 0; i < ulen; i++) { 45132cd00852SPu Lehui line_addr = (unsigned long)prog->aux->jited_linfo[i]; 45142cd00852SPu Lehui if (put_user((__u64)line_addr, &user_linfo[i])) 4515c454a46bSMartin KaFai Lau return -EFAULT; 4516c454a46bSMartin KaFai Lau } 4517c454a46bSMartin KaFai Lau } else { 4518c454a46bSMartin KaFai Lau info.jited_line_info = 0; 4519c454a46bSMartin KaFai Lau } 4520c454a46bSMartin KaFai Lau } 4521c454a46bSMartin KaFai Lau 4522c872bdb3SSong Liu ulen = info.nr_prog_tags; 4523c872bdb3SSong Liu info.nr_prog_tags = prog->aux->func_cnt ? : 1; 4524c872bdb3SSong Liu if (ulen) { 4525c872bdb3SSong Liu __u8 __user (*user_prog_tags)[BPF_TAG_SIZE]; 4526c872bdb3SSong Liu u32 i; 4527c872bdb3SSong Liu 4528c872bdb3SSong Liu user_prog_tags = u64_to_user_ptr(info.prog_tags); 4529c872bdb3SSong Liu ulen = min_t(u32, info.nr_prog_tags, ulen); 4530c872bdb3SSong Liu if (prog->aux->func_cnt) { 4531c872bdb3SSong Liu for (i = 0; i < ulen; i++) { 4532c872bdb3SSong Liu if (copy_to_user(user_prog_tags[i], 4533c872bdb3SSong Liu prog->aux->func[i]->tag, 4534c872bdb3SSong Liu BPF_TAG_SIZE)) 4535c872bdb3SSong Liu return -EFAULT; 4536c872bdb3SSong Liu } 4537c872bdb3SSong Liu } else { 4538c872bdb3SSong Liu if (copy_to_user(user_prog_tags[0], 4539c872bdb3SSong Liu prog->tag, BPF_TAG_SIZE)) 4540c872bdb3SSong Liu return -EFAULT; 4541c872bdb3SSong Liu } 4542c872bdb3SSong Liu } 4543c872bdb3SSong Liu 45441e270976SMartin KaFai Lau done: 45451e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 45461e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 45471e270976SMartin KaFai Lau return -EFAULT; 45481e270976SMartin KaFai Lau 45491e270976SMartin KaFai Lau return 0; 45501e270976SMartin KaFai Lau } 45511e270976SMartin KaFai Lau 455263960260SKees Cook static int bpf_map_get_info_by_fd(struct file *file, 455363960260SKees Cook struct bpf_map *map, 45541e270976SMartin KaFai Lau const union bpf_attr *attr, 45551e270976SMartin KaFai Lau union bpf_attr __user *uattr) 45561e270976SMartin KaFai Lau { 45571e270976SMartin KaFai Lau struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 45585c6f2588SGreg Kroah-Hartman struct bpf_map_info info; 45591e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 45601e270976SMartin KaFai Lau int err; 45611e270976SMartin KaFai Lau 4562af2ac3e1SAlexei Starovoitov err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len); 45631e270976SMartin KaFai Lau if (err) 45641e270976SMartin KaFai Lau return err; 45651e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 45661e270976SMartin KaFai Lau 45675c6f2588SGreg Kroah-Hartman memset(&info, 0, sizeof(info)); 45681e270976SMartin KaFai Lau info.type = map->map_type; 45691e270976SMartin KaFai Lau info.id = map->id; 45701e270976SMartin KaFai Lau info.key_size = map->key_size; 45711e270976SMartin KaFai Lau info.value_size = map->value_size; 45721e270976SMartin KaFai Lau info.max_entries = map->max_entries; 45731e270976SMartin KaFai Lau info.map_flags = map->map_flags; 45749330986cSJoanne Koong info.map_extra = map->map_extra; 4575ad5b177bSMartin KaFai Lau memcpy(info.name, map->name, sizeof(map->name)); 45761e270976SMartin KaFai Lau 457778958fcaSMartin KaFai Lau if (map->btf) { 457822dc4a0fSAndrii Nakryiko info.btf_id = btf_obj_id(map->btf); 45799b2cf328SMartin KaFai Lau info.btf_key_type_id = map->btf_key_type_id; 45809b2cf328SMartin KaFai Lau info.btf_value_type_id = map->btf_value_type_id; 458178958fcaSMartin KaFai Lau } 458285d33df3SMartin KaFai Lau info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; 458378958fcaSMartin KaFai Lau 45849d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) { 458552775b33SJakub Kicinski err = bpf_map_offload_info_fill(&info, map); 458652775b33SJakub Kicinski if (err) 458752775b33SJakub Kicinski return err; 458852775b33SJakub Kicinski } 458952775b33SJakub Kicinski 45901e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 45911e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 45921e270976SMartin KaFai Lau return -EFAULT; 45931e270976SMartin KaFai Lau 45941e270976SMartin KaFai Lau return 0; 45951e270976SMartin KaFai Lau } 45961e270976SMartin KaFai Lau 459763960260SKees Cook static int bpf_btf_get_info_by_fd(struct file *file, 459863960260SKees Cook struct btf *btf, 459962dab84cSMartin KaFai Lau const union bpf_attr *attr, 460062dab84cSMartin KaFai Lau union bpf_attr __user *uattr) 460162dab84cSMartin KaFai Lau { 460262dab84cSMartin KaFai Lau struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info); 460362dab84cSMartin KaFai Lau u32 info_len = attr->info.info_len; 460462dab84cSMartin KaFai Lau int err; 460562dab84cSMartin KaFai Lau 4606af2ac3e1SAlexei Starovoitov err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(*uinfo), info_len); 460762dab84cSMartin KaFai Lau if (err) 460862dab84cSMartin KaFai Lau return err; 460962dab84cSMartin KaFai Lau 461062dab84cSMartin KaFai Lau return btf_get_info_by_fd(btf, attr, uattr); 461162dab84cSMartin KaFai Lau } 461262dab84cSMartin KaFai Lau 461363960260SKees Cook static int bpf_link_get_info_by_fd(struct file *file, 461463960260SKees Cook struct bpf_link *link, 4615f2e10bffSAndrii Nakryiko const union bpf_attr *attr, 4616f2e10bffSAndrii Nakryiko union bpf_attr __user *uattr) 4617f2e10bffSAndrii Nakryiko { 4618f2e10bffSAndrii Nakryiko struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info); 4619f2e10bffSAndrii Nakryiko struct bpf_link_info info; 4620f2e10bffSAndrii Nakryiko u32 info_len = attr->info.info_len; 4621f2e10bffSAndrii Nakryiko int err; 4622f2e10bffSAndrii Nakryiko 4623af2ac3e1SAlexei Starovoitov err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len); 4624f2e10bffSAndrii Nakryiko if (err) 4625f2e10bffSAndrii Nakryiko return err; 4626f2e10bffSAndrii Nakryiko info_len = min_t(u32, sizeof(info), info_len); 4627f2e10bffSAndrii Nakryiko 4628f2e10bffSAndrii Nakryiko memset(&info, 0, sizeof(info)); 4629f2e10bffSAndrii Nakryiko if (copy_from_user(&info, uinfo, info_len)) 4630f2e10bffSAndrii Nakryiko return -EFAULT; 4631f2e10bffSAndrii Nakryiko 4632f2e10bffSAndrii Nakryiko info.type = link->type; 4633f2e10bffSAndrii Nakryiko info.id = link->id; 463468b04864SKui-Feng Lee if (link->prog) 4635f2e10bffSAndrii Nakryiko info.prog_id = link->prog->aux->id; 4636f2e10bffSAndrii Nakryiko 4637f2e10bffSAndrii Nakryiko if (link->ops->fill_link_info) { 4638f2e10bffSAndrii Nakryiko err = link->ops->fill_link_info(link, &info); 4639f2e10bffSAndrii Nakryiko if (err) 4640f2e10bffSAndrii Nakryiko return err; 4641f2e10bffSAndrii Nakryiko } 4642f2e10bffSAndrii Nakryiko 4643f2e10bffSAndrii Nakryiko if (copy_to_user(uinfo, &info, info_len) || 4644f2e10bffSAndrii Nakryiko put_user(info_len, &uattr->info.info_len)) 4645f2e10bffSAndrii Nakryiko return -EFAULT; 4646f2e10bffSAndrii Nakryiko 4647f2e10bffSAndrii Nakryiko return 0; 4648f2e10bffSAndrii Nakryiko } 4649f2e10bffSAndrii Nakryiko 4650f2e10bffSAndrii Nakryiko 46511e270976SMartin KaFai Lau #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 46521e270976SMartin KaFai Lau 46531e270976SMartin KaFai Lau static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 46541e270976SMartin KaFai Lau union bpf_attr __user *uattr) 46551e270976SMartin KaFai Lau { 46561e270976SMartin KaFai Lau int ufd = attr->info.bpf_fd; 46571e270976SMartin KaFai Lau struct fd f; 46581e270976SMartin KaFai Lau int err; 46591e270976SMartin KaFai Lau 46601e270976SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 46611e270976SMartin KaFai Lau return -EINVAL; 46621e270976SMartin KaFai Lau 46631e270976SMartin KaFai Lau f = fdget(ufd); 46641e270976SMartin KaFai Lau if (!f.file) 46651e270976SMartin KaFai Lau return -EBADFD; 46661e270976SMartin KaFai Lau 46671e270976SMartin KaFai Lau if (f.file->f_op == &bpf_prog_fops) 466863960260SKees Cook err = bpf_prog_get_info_by_fd(f.file, f.file->private_data, attr, 46691e270976SMartin KaFai Lau uattr); 46701e270976SMartin KaFai Lau else if (f.file->f_op == &bpf_map_fops) 467163960260SKees Cook err = bpf_map_get_info_by_fd(f.file, f.file->private_data, attr, 46721e270976SMartin KaFai Lau uattr); 467360197cfbSMartin KaFai Lau else if (f.file->f_op == &btf_fops) 467463960260SKees Cook err = bpf_btf_get_info_by_fd(f.file, f.file->private_data, attr, uattr); 4675f2e10bffSAndrii Nakryiko else if (f.file->f_op == &bpf_link_fops) 467663960260SKees Cook err = bpf_link_get_info_by_fd(f.file, f.file->private_data, 4677f2e10bffSAndrii Nakryiko attr, uattr); 46781e270976SMartin KaFai Lau else 46791e270976SMartin KaFai Lau err = -EINVAL; 46801e270976SMartin KaFai Lau 46811e270976SMartin KaFai Lau fdput(f); 46821e270976SMartin KaFai Lau return err; 46831e270976SMartin KaFai Lau } 46841e270976SMartin KaFai Lau 468547a71c1fSAndrii Nakryiko #define BPF_BTF_LOAD_LAST_FIELD btf_log_true_size 4686f56a653cSMartin KaFai Lau 468747a71c1fSAndrii Nakryiko static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size) 4688f56a653cSMartin KaFai Lau { 4689f56a653cSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_LOAD)) 4690f56a653cSMartin KaFai Lau return -EINVAL; 4691f56a653cSMartin KaFai Lau 46922c78ee89SAlexei Starovoitov if (!bpf_capable()) 4693f56a653cSMartin KaFai Lau return -EPERM; 4694f56a653cSMartin KaFai Lau 469547a71c1fSAndrii Nakryiko return btf_new_fd(attr, uattr, uattr_size); 4696f56a653cSMartin KaFai Lau } 4697f56a653cSMartin KaFai Lau 469878958fcaSMartin KaFai Lau #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id 469978958fcaSMartin KaFai Lau 470078958fcaSMartin KaFai Lau static int bpf_btf_get_fd_by_id(const union bpf_attr *attr) 470178958fcaSMartin KaFai Lau { 470278958fcaSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID)) 470378958fcaSMartin KaFai Lau return -EINVAL; 470478958fcaSMartin KaFai Lau 470578958fcaSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 470678958fcaSMartin KaFai Lau return -EPERM; 470778958fcaSMartin KaFai Lau 470878958fcaSMartin KaFai Lau return btf_get_fd_by_id(attr->btf_id); 470978958fcaSMartin KaFai Lau } 471078958fcaSMartin KaFai Lau 471141bdc4b4SYonghong Song static int bpf_task_fd_query_copy(const union bpf_attr *attr, 471241bdc4b4SYonghong Song union bpf_attr __user *uattr, 471341bdc4b4SYonghong Song u32 prog_id, u32 fd_type, 471441bdc4b4SYonghong Song const char *buf, u64 probe_offset, 471541bdc4b4SYonghong Song u64 probe_addr) 471641bdc4b4SYonghong Song { 471741bdc4b4SYonghong Song char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf); 471841bdc4b4SYonghong Song u32 len = buf ? strlen(buf) : 0, input_len; 471941bdc4b4SYonghong Song int err = 0; 472041bdc4b4SYonghong Song 472141bdc4b4SYonghong Song if (put_user(len, &uattr->task_fd_query.buf_len)) 472241bdc4b4SYonghong Song return -EFAULT; 472341bdc4b4SYonghong Song input_len = attr->task_fd_query.buf_len; 472441bdc4b4SYonghong Song if (input_len && ubuf) { 472541bdc4b4SYonghong Song if (!len) { 472641bdc4b4SYonghong Song /* nothing to copy, just make ubuf NULL terminated */ 472741bdc4b4SYonghong Song char zero = '\0'; 472841bdc4b4SYonghong Song 472941bdc4b4SYonghong Song if (put_user(zero, ubuf)) 473041bdc4b4SYonghong Song return -EFAULT; 473141bdc4b4SYonghong Song } else if (input_len >= len + 1) { 473241bdc4b4SYonghong Song /* ubuf can hold the string with NULL terminator */ 473341bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, len + 1)) 473441bdc4b4SYonghong Song return -EFAULT; 473541bdc4b4SYonghong Song } else { 473641bdc4b4SYonghong Song /* ubuf cannot hold the string with NULL terminator, 473741bdc4b4SYonghong Song * do a partial copy with NULL terminator. 473841bdc4b4SYonghong Song */ 473941bdc4b4SYonghong Song char zero = '\0'; 474041bdc4b4SYonghong Song 474141bdc4b4SYonghong Song err = -ENOSPC; 474241bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, input_len - 1)) 474341bdc4b4SYonghong Song return -EFAULT; 474441bdc4b4SYonghong Song if (put_user(zero, ubuf + input_len - 1)) 474541bdc4b4SYonghong Song return -EFAULT; 474641bdc4b4SYonghong Song } 474741bdc4b4SYonghong Song } 474841bdc4b4SYonghong Song 474941bdc4b4SYonghong Song if (put_user(prog_id, &uattr->task_fd_query.prog_id) || 475041bdc4b4SYonghong Song put_user(fd_type, &uattr->task_fd_query.fd_type) || 475141bdc4b4SYonghong Song put_user(probe_offset, &uattr->task_fd_query.probe_offset) || 475241bdc4b4SYonghong Song put_user(probe_addr, &uattr->task_fd_query.probe_addr)) 475341bdc4b4SYonghong Song return -EFAULT; 475441bdc4b4SYonghong Song 475541bdc4b4SYonghong Song return err; 475641bdc4b4SYonghong Song } 475741bdc4b4SYonghong Song 475841bdc4b4SYonghong Song #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr 475941bdc4b4SYonghong Song 476041bdc4b4SYonghong Song static int bpf_task_fd_query(const union bpf_attr *attr, 476141bdc4b4SYonghong Song union bpf_attr __user *uattr) 476241bdc4b4SYonghong Song { 476341bdc4b4SYonghong Song pid_t pid = attr->task_fd_query.pid; 476441bdc4b4SYonghong Song u32 fd = attr->task_fd_query.fd; 476541bdc4b4SYonghong Song const struct perf_event *event; 476641bdc4b4SYonghong Song struct task_struct *task; 476741bdc4b4SYonghong Song struct file *file; 476841bdc4b4SYonghong Song int err; 476941bdc4b4SYonghong Song 477041bdc4b4SYonghong Song if (CHECK_ATTR(BPF_TASK_FD_QUERY)) 477141bdc4b4SYonghong Song return -EINVAL; 477241bdc4b4SYonghong Song 477341bdc4b4SYonghong Song if (!capable(CAP_SYS_ADMIN)) 477441bdc4b4SYonghong Song return -EPERM; 477541bdc4b4SYonghong Song 477641bdc4b4SYonghong Song if (attr->task_fd_query.flags != 0) 477741bdc4b4SYonghong Song return -EINVAL; 477841bdc4b4SYonghong Song 477983c10cc3SLee Jones rcu_read_lock(); 478041bdc4b4SYonghong Song task = get_pid_task(find_vpid(pid), PIDTYPE_PID); 478183c10cc3SLee Jones rcu_read_unlock(); 478241bdc4b4SYonghong Song if (!task) 478341bdc4b4SYonghong Song return -ENOENT; 478441bdc4b4SYonghong Song 478541bdc4b4SYonghong Song err = 0; 4786b48845afSEric W. Biederman file = fget_task(task, fd); 4787b48845afSEric W. Biederman put_task_struct(task); 478841bdc4b4SYonghong Song if (!file) 4789b48845afSEric W. Biederman return -EBADF; 479041bdc4b4SYonghong Song 479170ed506cSAndrii Nakryiko if (file->f_op == &bpf_link_fops) { 479270ed506cSAndrii Nakryiko struct bpf_link *link = file->private_data; 479370ed506cSAndrii Nakryiko 4794a3b80e10SAndrii Nakryiko if (link->ops == &bpf_raw_tp_link_lops) { 479570ed506cSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 479670ed506cSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 479741bdc4b4SYonghong Song struct bpf_raw_event_map *btp = raw_tp->btp; 479841bdc4b4SYonghong Song 479941bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, 480070ed506cSAndrii Nakryiko raw_tp->link.prog->aux->id, 480141bdc4b4SYonghong Song BPF_FD_TYPE_RAW_TRACEPOINT, 480241bdc4b4SYonghong Song btp->tp->name, 0, 0); 480341bdc4b4SYonghong Song goto put_file; 480441bdc4b4SYonghong Song } 480570ed506cSAndrii Nakryiko goto out_not_supp; 480670ed506cSAndrii Nakryiko } 480741bdc4b4SYonghong Song 480841bdc4b4SYonghong Song event = perf_get_event(file); 480941bdc4b4SYonghong Song if (!IS_ERR(event)) { 481041bdc4b4SYonghong Song u64 probe_offset, probe_addr; 481141bdc4b4SYonghong Song u32 prog_id, fd_type; 481241bdc4b4SYonghong Song const char *buf; 481341bdc4b4SYonghong Song 481441bdc4b4SYonghong Song err = bpf_get_perf_event_info(event, &prog_id, &fd_type, 481541bdc4b4SYonghong Song &buf, &probe_offset, 481641bdc4b4SYonghong Song &probe_addr); 481741bdc4b4SYonghong Song if (!err) 481841bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, prog_id, 481941bdc4b4SYonghong Song fd_type, buf, 482041bdc4b4SYonghong Song probe_offset, 482141bdc4b4SYonghong Song probe_addr); 482241bdc4b4SYonghong Song goto put_file; 482341bdc4b4SYonghong Song } 482441bdc4b4SYonghong Song 482570ed506cSAndrii Nakryiko out_not_supp: 482641bdc4b4SYonghong Song err = -ENOTSUPP; 482741bdc4b4SYonghong Song put_file: 482841bdc4b4SYonghong Song fput(file); 482941bdc4b4SYonghong Song return err; 483041bdc4b4SYonghong Song } 483141bdc4b4SYonghong Song 4832cb4d03abSBrian Vazquez #define BPF_MAP_BATCH_LAST_FIELD batch.flags 4833cb4d03abSBrian Vazquez 48343af43ba4SHou Tao #define BPF_DO_BATCH(fn, ...) \ 4835cb4d03abSBrian Vazquez do { \ 4836cb4d03abSBrian Vazquez if (!fn) { \ 4837cb4d03abSBrian Vazquez err = -ENOTSUPP; \ 4838cb4d03abSBrian Vazquez goto err_put; \ 4839cb4d03abSBrian Vazquez } \ 48403af43ba4SHou Tao err = fn(__VA_ARGS__); \ 4841cb4d03abSBrian Vazquez } while (0) 4842cb4d03abSBrian Vazquez 4843cb4d03abSBrian Vazquez static int bpf_map_do_batch(const union bpf_attr *attr, 4844cb4d03abSBrian Vazquez union bpf_attr __user *uattr, 4845cb4d03abSBrian Vazquez int cmd) 4846cb4d03abSBrian Vazquez { 4847353050beSDaniel Borkmann bool has_read = cmd == BPF_MAP_LOOKUP_BATCH || 4848353050beSDaniel Borkmann cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH; 4849353050beSDaniel Borkmann bool has_write = cmd != BPF_MAP_LOOKUP_BATCH; 4850cb4d03abSBrian Vazquez struct bpf_map *map; 4851cb4d03abSBrian Vazquez int err, ufd; 4852cb4d03abSBrian Vazquez struct fd f; 4853cb4d03abSBrian Vazquez 4854cb4d03abSBrian Vazquez if (CHECK_ATTR(BPF_MAP_BATCH)) 4855cb4d03abSBrian Vazquez return -EINVAL; 4856cb4d03abSBrian Vazquez 4857cb4d03abSBrian Vazquez ufd = attr->batch.map_fd; 4858cb4d03abSBrian Vazquez f = fdget(ufd); 4859cb4d03abSBrian Vazquez map = __bpf_map_get(f); 4860cb4d03abSBrian Vazquez if (IS_ERR(map)) 4861cb4d03abSBrian Vazquez return PTR_ERR(map); 4862353050beSDaniel Borkmann if (has_write) 4863353050beSDaniel Borkmann bpf_map_write_active_inc(map); 4864353050beSDaniel Borkmann if (has_read && !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 4865cb4d03abSBrian Vazquez err = -EPERM; 4866cb4d03abSBrian Vazquez goto err_put; 4867cb4d03abSBrian Vazquez } 4868353050beSDaniel Borkmann if (has_write && !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 4869cb4d03abSBrian Vazquez err = -EPERM; 4870cb4d03abSBrian Vazquez goto err_put; 4871cb4d03abSBrian Vazquez } 4872cb4d03abSBrian Vazquez 4873cb4d03abSBrian Vazquez if (cmd == BPF_MAP_LOOKUP_BATCH) 48743af43ba4SHou Tao BPF_DO_BATCH(map->ops->map_lookup_batch, map, attr, uattr); 487505799638SYonghong Song else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) 48763af43ba4SHou Tao BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch, map, attr, uattr); 4877aa2e93b8SBrian Vazquez else if (cmd == BPF_MAP_UPDATE_BATCH) 48783af43ba4SHou Tao BPF_DO_BATCH(map->ops->map_update_batch, map, f.file, attr, uattr); 4879aa2e93b8SBrian Vazquez else 48803af43ba4SHou Tao BPF_DO_BATCH(map->ops->map_delete_batch, map, attr, uattr); 4881cb4d03abSBrian Vazquez err_put: 4882353050beSDaniel Borkmann if (has_write) 4883353050beSDaniel Borkmann bpf_map_write_active_dec(map); 4884cb4d03abSBrian Vazquez fdput(f); 4885cb4d03abSBrian Vazquez return err; 4886cb4d03abSBrian Vazquez } 4887cb4d03abSBrian Vazquez 4888b733eeadSJiri Olsa #define BPF_LINK_CREATE_LAST_FIELD link_create.uprobe_multi.pid 4889af2ac3e1SAlexei Starovoitov static int link_create(union bpf_attr *attr, bpfptr_t uattr) 4890af6eea57SAndrii Nakryiko { 4891af6eea57SAndrii Nakryiko struct bpf_prog *prog; 4892af6eea57SAndrii Nakryiko int ret; 4893af6eea57SAndrii Nakryiko 4894af6eea57SAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_CREATE)) 4895af6eea57SAndrii Nakryiko return -EINVAL; 4896af6eea57SAndrii Nakryiko 489768b04864SKui-Feng Lee if (attr->link_create.attach_type == BPF_STRUCT_OPS) 489868b04864SKui-Feng Lee return bpf_struct_ops_link_create(attr); 489968b04864SKui-Feng Lee 49004a1e7c0cSToke Høiland-Jørgensen prog = bpf_prog_get(attr->link_create.prog_fd); 4901af6eea57SAndrii Nakryiko if (IS_ERR(prog)) 4902af6eea57SAndrii Nakryiko return PTR_ERR(prog); 4903af6eea57SAndrii Nakryiko 4904af6eea57SAndrii Nakryiko ret = bpf_prog_attach_check_attach_type(prog, 4905af6eea57SAndrii Nakryiko attr->link_create.attach_type); 4906af6eea57SAndrii Nakryiko if (ret) 49074a1e7c0cSToke Høiland-Jørgensen goto out; 49084a1e7c0cSToke Høiland-Jørgensen 4909b89fbfbbSAndrii Nakryiko switch (prog->type) { 4910af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 4911af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 4912af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 4913af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 4914af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 4915af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 4916af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 4917af6eea57SAndrii Nakryiko ret = cgroup_bpf_link_attach(attr, prog); 4918af6eea57SAndrii Nakryiko break; 4919df86ca0dSAndrii Nakryiko case BPF_PROG_TYPE_EXT: 4920df86ca0dSAndrii Nakryiko ret = bpf_tracing_prog_attach(prog, 4921df86ca0dSAndrii Nakryiko attr->link_create.target_fd, 49222fcc8241SKui-Feng Lee attr->link_create.target_btf_id, 49232fcc8241SKui-Feng Lee attr->link_create.tracing.cookie); 4924df86ca0dSAndrii Nakryiko break; 4925df86ca0dSAndrii Nakryiko case BPF_PROG_TYPE_LSM: 4926de4e05caSYonghong Song case BPF_PROG_TYPE_TRACING: 4927df86ca0dSAndrii Nakryiko if (attr->link_create.attach_type != prog->expected_attach_type) { 4928df86ca0dSAndrii Nakryiko ret = -EINVAL; 4929df86ca0dSAndrii Nakryiko goto out; 4930df86ca0dSAndrii Nakryiko } 4931df86ca0dSAndrii Nakryiko if (prog->expected_attach_type == BPF_TRACE_RAW_TP) 4932df86ca0dSAndrii Nakryiko ret = bpf_raw_tp_link_attach(prog, NULL); 4933df86ca0dSAndrii Nakryiko else if (prog->expected_attach_type == BPF_TRACE_ITER) 4934df86ca0dSAndrii Nakryiko ret = bpf_iter_link_attach(attr, uattr, prog); 493569fd337aSStanislav Fomichev else if (prog->expected_attach_type == BPF_LSM_CGROUP) 493669fd337aSStanislav Fomichev ret = cgroup_bpf_link_attach(attr, prog); 4937df86ca0dSAndrii Nakryiko else 4938df86ca0dSAndrii Nakryiko ret = bpf_tracing_prog_attach(prog, 4939df86ca0dSAndrii Nakryiko attr->link_create.target_fd, 49402fcc8241SKui-Feng Lee attr->link_create.target_btf_id, 49412fcc8241SKui-Feng Lee attr->link_create.tracing.cookie); 4942de4e05caSYonghong Song break; 49437f045a49SJakub Sitnicki case BPF_PROG_TYPE_FLOW_DISSECTOR: 4944e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP: 49457f045a49SJakub Sitnicki ret = netns_bpf_link_create(attr, prog); 49467f045a49SJakub Sitnicki break; 4947310ad797SAndrii Nakryiko #ifdef CONFIG_NET 4948aa8d3a71SAndrii Nakryiko case BPF_PROG_TYPE_XDP: 4949aa8d3a71SAndrii Nakryiko ret = bpf_xdp_link_attach(attr, prog); 4950aa8d3a71SAndrii Nakryiko break; 4951e420bed0SDaniel Borkmann case BPF_PROG_TYPE_SCHED_CLS: 4952e420bed0SDaniel Borkmann ret = tcx_link_attach(attr, prog); 4953e420bed0SDaniel Borkmann break; 495484601d6eSFlorian Westphal case BPF_PROG_TYPE_NETFILTER: 495584601d6eSFlorian Westphal ret = bpf_nf_link_attach(attr, prog); 495684601d6eSFlorian Westphal break; 4957310ad797SAndrii Nakryiko #endif 4958b89fbfbbSAndrii Nakryiko case BPF_PROG_TYPE_PERF_EVENT: 4959b89fbfbbSAndrii Nakryiko case BPF_PROG_TYPE_TRACEPOINT: 4960b89fbfbbSAndrii Nakryiko ret = bpf_perf_link_attach(attr, prog); 4961b89fbfbbSAndrii Nakryiko break; 49620dcac272SJiri Olsa case BPF_PROG_TYPE_KPROBE: 49630dcac272SJiri Olsa if (attr->link_create.attach_type == BPF_PERF_EVENT) 49640dcac272SJiri Olsa ret = bpf_perf_link_attach(attr, prog); 496589ae89f5SJiri Olsa else if (attr->link_create.attach_type == BPF_TRACE_KPROBE_MULTI) 49660dcac272SJiri Olsa ret = bpf_kprobe_multi_link_attach(attr, prog); 496789ae89f5SJiri Olsa else if (attr->link_create.attach_type == BPF_TRACE_UPROBE_MULTI) 496889ae89f5SJiri Olsa ret = bpf_uprobe_multi_link_attach(attr, prog); 49690dcac272SJiri Olsa break; 4970af6eea57SAndrii Nakryiko default: 4971af6eea57SAndrii Nakryiko ret = -EINVAL; 4972af6eea57SAndrii Nakryiko } 4973af6eea57SAndrii Nakryiko 49744a1e7c0cSToke Høiland-Jørgensen out: 4975af6eea57SAndrii Nakryiko if (ret < 0) 4976af6eea57SAndrii Nakryiko bpf_prog_put(prog); 4977af6eea57SAndrii Nakryiko return ret; 4978af6eea57SAndrii Nakryiko } 4979af6eea57SAndrii Nakryiko 4980aef56f2eSKui-Feng Lee static int link_update_map(struct bpf_link *link, union bpf_attr *attr) 4981aef56f2eSKui-Feng Lee { 4982aef56f2eSKui-Feng Lee struct bpf_map *new_map, *old_map = NULL; 4983aef56f2eSKui-Feng Lee int ret; 4984aef56f2eSKui-Feng Lee 4985aef56f2eSKui-Feng Lee new_map = bpf_map_get(attr->link_update.new_map_fd); 4986aef56f2eSKui-Feng Lee if (IS_ERR(new_map)) 498755fbae05SMartin KaFai Lau return PTR_ERR(new_map); 4988aef56f2eSKui-Feng Lee 4989aef56f2eSKui-Feng Lee if (attr->link_update.flags & BPF_F_REPLACE) { 4990aef56f2eSKui-Feng Lee old_map = bpf_map_get(attr->link_update.old_map_fd); 4991aef56f2eSKui-Feng Lee if (IS_ERR(old_map)) { 499255fbae05SMartin KaFai Lau ret = PTR_ERR(old_map); 4993aef56f2eSKui-Feng Lee goto out_put; 4994aef56f2eSKui-Feng Lee } 4995aef56f2eSKui-Feng Lee } else if (attr->link_update.old_map_fd) { 4996aef56f2eSKui-Feng Lee ret = -EINVAL; 4997aef56f2eSKui-Feng Lee goto out_put; 4998aef56f2eSKui-Feng Lee } 4999aef56f2eSKui-Feng Lee 5000aef56f2eSKui-Feng Lee ret = link->ops->update_map(link, new_map, old_map); 5001aef56f2eSKui-Feng Lee 5002aef56f2eSKui-Feng Lee if (old_map) 5003aef56f2eSKui-Feng Lee bpf_map_put(old_map); 5004aef56f2eSKui-Feng Lee out_put: 5005aef56f2eSKui-Feng Lee bpf_map_put(new_map); 5006aef56f2eSKui-Feng Lee return ret; 5007aef56f2eSKui-Feng Lee } 5008aef56f2eSKui-Feng Lee 50090c991ebcSAndrii Nakryiko #define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd 50100c991ebcSAndrii Nakryiko 50110c991ebcSAndrii Nakryiko static int link_update(union bpf_attr *attr) 50120c991ebcSAndrii Nakryiko { 50130c991ebcSAndrii Nakryiko struct bpf_prog *old_prog = NULL, *new_prog; 50140c991ebcSAndrii Nakryiko struct bpf_link *link; 50150c991ebcSAndrii Nakryiko u32 flags; 50160c991ebcSAndrii Nakryiko int ret; 50170c991ebcSAndrii Nakryiko 50180c991ebcSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_UPDATE)) 50190c991ebcSAndrii Nakryiko return -EINVAL; 50200c991ebcSAndrii Nakryiko 50210c991ebcSAndrii Nakryiko flags = attr->link_update.flags; 50220c991ebcSAndrii Nakryiko if (flags & ~BPF_F_REPLACE) 50230c991ebcSAndrii Nakryiko return -EINVAL; 50240c991ebcSAndrii Nakryiko 50250c991ebcSAndrii Nakryiko link = bpf_link_get_from_fd(attr->link_update.link_fd); 50260c991ebcSAndrii Nakryiko if (IS_ERR(link)) 50270c991ebcSAndrii Nakryiko return PTR_ERR(link); 50280c991ebcSAndrii Nakryiko 5029aef56f2eSKui-Feng Lee if (link->ops->update_map) { 5030aef56f2eSKui-Feng Lee ret = link_update_map(link, attr); 5031aef56f2eSKui-Feng Lee goto out_put_link; 5032aef56f2eSKui-Feng Lee } 5033aef56f2eSKui-Feng Lee 50340c991ebcSAndrii Nakryiko new_prog = bpf_prog_get(attr->link_update.new_prog_fd); 50354adb7a4aSAndrii Nakryiko if (IS_ERR(new_prog)) { 50364adb7a4aSAndrii Nakryiko ret = PTR_ERR(new_prog); 50374adb7a4aSAndrii Nakryiko goto out_put_link; 50384adb7a4aSAndrii Nakryiko } 50390c991ebcSAndrii Nakryiko 50400c991ebcSAndrii Nakryiko if (flags & BPF_F_REPLACE) { 50410c991ebcSAndrii Nakryiko old_prog = bpf_prog_get(attr->link_update.old_prog_fd); 50420c991ebcSAndrii Nakryiko if (IS_ERR(old_prog)) { 50430c991ebcSAndrii Nakryiko ret = PTR_ERR(old_prog); 50440c991ebcSAndrii Nakryiko old_prog = NULL; 50450c991ebcSAndrii Nakryiko goto out_put_progs; 50460c991ebcSAndrii Nakryiko } 50474adb7a4aSAndrii Nakryiko } else if (attr->link_update.old_prog_fd) { 50484adb7a4aSAndrii Nakryiko ret = -EINVAL; 50494adb7a4aSAndrii Nakryiko goto out_put_progs; 50500c991ebcSAndrii Nakryiko } 50510c991ebcSAndrii Nakryiko 5052f9d04127SAndrii Nakryiko if (link->ops->update_prog) 5053f9d04127SAndrii Nakryiko ret = link->ops->update_prog(link, new_prog, old_prog); 5054f9d04127SAndrii Nakryiko else 50550c991ebcSAndrii Nakryiko ret = -EINVAL; 50560c991ebcSAndrii Nakryiko 50570c991ebcSAndrii Nakryiko out_put_progs: 50580c991ebcSAndrii Nakryiko if (old_prog) 50590c991ebcSAndrii Nakryiko bpf_prog_put(old_prog); 50600c991ebcSAndrii Nakryiko if (ret) 50610c991ebcSAndrii Nakryiko bpf_prog_put(new_prog); 50624adb7a4aSAndrii Nakryiko out_put_link: 5063ab5d47bdSSebastian Andrzej Siewior bpf_link_put_direct(link); 50640c991ebcSAndrii Nakryiko return ret; 50650c991ebcSAndrii Nakryiko } 50660c991ebcSAndrii Nakryiko 506773b11c2aSAndrii Nakryiko #define BPF_LINK_DETACH_LAST_FIELD link_detach.link_fd 506873b11c2aSAndrii Nakryiko 506973b11c2aSAndrii Nakryiko static int link_detach(union bpf_attr *attr) 507073b11c2aSAndrii Nakryiko { 507173b11c2aSAndrii Nakryiko struct bpf_link *link; 507273b11c2aSAndrii Nakryiko int ret; 507373b11c2aSAndrii Nakryiko 507473b11c2aSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_DETACH)) 507573b11c2aSAndrii Nakryiko return -EINVAL; 507673b11c2aSAndrii Nakryiko 507773b11c2aSAndrii Nakryiko link = bpf_link_get_from_fd(attr->link_detach.link_fd); 507873b11c2aSAndrii Nakryiko if (IS_ERR(link)) 507973b11c2aSAndrii Nakryiko return PTR_ERR(link); 508073b11c2aSAndrii Nakryiko 508173b11c2aSAndrii Nakryiko if (link->ops->detach) 508273b11c2aSAndrii Nakryiko ret = link->ops->detach(link); 508373b11c2aSAndrii Nakryiko else 508473b11c2aSAndrii Nakryiko ret = -EOPNOTSUPP; 508573b11c2aSAndrii Nakryiko 5086ab5d47bdSSebastian Andrzej Siewior bpf_link_put_direct(link); 508773b11c2aSAndrii Nakryiko return ret; 508873b11c2aSAndrii Nakryiko } 508973b11c2aSAndrii Nakryiko 5090005142b8SAlexei Starovoitov static struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link) 50912d602c8cSAndrii Nakryiko { 5092005142b8SAlexei Starovoitov return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT); 5093005142b8SAlexei Starovoitov } 5094005142b8SAlexei Starovoitov 5095005142b8SAlexei Starovoitov struct bpf_link *bpf_link_by_id(u32 id) 5096005142b8SAlexei Starovoitov { 5097005142b8SAlexei Starovoitov struct bpf_link *link; 5098005142b8SAlexei Starovoitov 5099005142b8SAlexei Starovoitov if (!id) 5100005142b8SAlexei Starovoitov return ERR_PTR(-ENOENT); 5101005142b8SAlexei Starovoitov 5102005142b8SAlexei Starovoitov spin_lock_bh(&link_idr_lock); 5103005142b8SAlexei Starovoitov /* before link is "settled", ID is 0, pretend it doesn't exist yet */ 5104005142b8SAlexei Starovoitov link = idr_find(&link_idr, id); 5105005142b8SAlexei Starovoitov if (link) { 5106005142b8SAlexei Starovoitov if (link->id) 5107005142b8SAlexei Starovoitov link = bpf_link_inc_not_zero(link); 5108005142b8SAlexei Starovoitov else 5109005142b8SAlexei Starovoitov link = ERR_PTR(-EAGAIN); 5110005142b8SAlexei Starovoitov } else { 5111005142b8SAlexei Starovoitov link = ERR_PTR(-ENOENT); 5112005142b8SAlexei Starovoitov } 5113005142b8SAlexei Starovoitov spin_unlock_bh(&link_idr_lock); 5114005142b8SAlexei Starovoitov return link; 51152d602c8cSAndrii Nakryiko } 51162d602c8cSAndrii Nakryiko 51179f883612SDmitrii Dolgov struct bpf_link *bpf_link_get_curr_or_next(u32 *id) 51189f883612SDmitrii Dolgov { 51199f883612SDmitrii Dolgov struct bpf_link *link; 51209f883612SDmitrii Dolgov 51219f883612SDmitrii Dolgov spin_lock_bh(&link_idr_lock); 51229f883612SDmitrii Dolgov again: 51239f883612SDmitrii Dolgov link = idr_get_next(&link_idr, id); 51249f883612SDmitrii Dolgov if (link) { 51259f883612SDmitrii Dolgov link = bpf_link_inc_not_zero(link); 51269f883612SDmitrii Dolgov if (IS_ERR(link)) { 51279f883612SDmitrii Dolgov (*id)++; 51289f883612SDmitrii Dolgov goto again; 51299f883612SDmitrii Dolgov } 51309f883612SDmitrii Dolgov } 51319f883612SDmitrii Dolgov spin_unlock_bh(&link_idr_lock); 51329f883612SDmitrii Dolgov 51339f883612SDmitrii Dolgov return link; 51349f883612SDmitrii Dolgov } 51359f883612SDmitrii Dolgov 51362d602c8cSAndrii Nakryiko #define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id 51372d602c8cSAndrii Nakryiko 51382d602c8cSAndrii Nakryiko static int bpf_link_get_fd_by_id(const union bpf_attr *attr) 51392d602c8cSAndrii Nakryiko { 51402d602c8cSAndrii Nakryiko struct bpf_link *link; 51412d602c8cSAndrii Nakryiko u32 id = attr->link_id; 5142005142b8SAlexei Starovoitov int fd; 51432d602c8cSAndrii Nakryiko 51442d602c8cSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID)) 51452d602c8cSAndrii Nakryiko return -EINVAL; 51462d602c8cSAndrii Nakryiko 51472d602c8cSAndrii Nakryiko if (!capable(CAP_SYS_ADMIN)) 51482d602c8cSAndrii Nakryiko return -EPERM; 51492d602c8cSAndrii Nakryiko 5150005142b8SAlexei Starovoitov link = bpf_link_by_id(id); 5151005142b8SAlexei Starovoitov if (IS_ERR(link)) 5152005142b8SAlexei Starovoitov return PTR_ERR(link); 51532d602c8cSAndrii Nakryiko 51542d602c8cSAndrii Nakryiko fd = bpf_link_new_fd(link); 51552d602c8cSAndrii Nakryiko if (fd < 0) 5156ab5d47bdSSebastian Andrzej Siewior bpf_link_put_direct(link); 51572d602c8cSAndrii Nakryiko 51582d602c8cSAndrii Nakryiko return fd; 51592d602c8cSAndrii Nakryiko } 51602d602c8cSAndrii Nakryiko 5161d46edd67SSong Liu DEFINE_MUTEX(bpf_stats_enabled_mutex); 5162d46edd67SSong Liu 5163d46edd67SSong Liu static int bpf_stats_release(struct inode *inode, struct file *file) 5164d46edd67SSong Liu { 5165d46edd67SSong Liu mutex_lock(&bpf_stats_enabled_mutex); 5166d46edd67SSong Liu static_key_slow_dec(&bpf_stats_enabled_key.key); 5167d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 5168d46edd67SSong Liu return 0; 5169d46edd67SSong Liu } 5170d46edd67SSong Liu 5171d46edd67SSong Liu static const struct file_operations bpf_stats_fops = { 5172d46edd67SSong Liu .release = bpf_stats_release, 5173d46edd67SSong Liu }; 5174d46edd67SSong Liu 5175d46edd67SSong Liu static int bpf_enable_runtime_stats(void) 5176d46edd67SSong Liu { 5177d46edd67SSong Liu int fd; 5178d46edd67SSong Liu 5179d46edd67SSong Liu mutex_lock(&bpf_stats_enabled_mutex); 5180d46edd67SSong Liu 5181d46edd67SSong Liu /* Set a very high limit to avoid overflow */ 5182d46edd67SSong Liu if (static_key_count(&bpf_stats_enabled_key.key) > INT_MAX / 2) { 5183d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 5184d46edd67SSong Liu return -EBUSY; 5185d46edd67SSong Liu } 5186d46edd67SSong Liu 5187d46edd67SSong Liu fd = anon_inode_getfd("bpf-stats", &bpf_stats_fops, NULL, O_CLOEXEC); 5188d46edd67SSong Liu if (fd >= 0) 5189d46edd67SSong Liu static_key_slow_inc(&bpf_stats_enabled_key.key); 5190d46edd67SSong Liu 5191d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 5192d46edd67SSong Liu return fd; 5193d46edd67SSong Liu } 5194d46edd67SSong Liu 5195d46edd67SSong Liu #define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type 5196d46edd67SSong Liu 5197d46edd67SSong Liu static int bpf_enable_stats(union bpf_attr *attr) 5198d46edd67SSong Liu { 5199d46edd67SSong Liu 5200d46edd67SSong Liu if (CHECK_ATTR(BPF_ENABLE_STATS)) 5201d46edd67SSong Liu return -EINVAL; 5202d46edd67SSong Liu 5203d46edd67SSong Liu if (!capable(CAP_SYS_ADMIN)) 5204d46edd67SSong Liu return -EPERM; 5205d46edd67SSong Liu 5206d46edd67SSong Liu switch (attr->enable_stats.type) { 5207d46edd67SSong Liu case BPF_STATS_RUN_TIME: 5208d46edd67SSong Liu return bpf_enable_runtime_stats(); 5209d46edd67SSong Liu default: 5210d46edd67SSong Liu break; 5211d46edd67SSong Liu } 5212d46edd67SSong Liu return -EINVAL; 5213d46edd67SSong Liu } 5214d46edd67SSong Liu 5215ac51d99bSYonghong Song #define BPF_ITER_CREATE_LAST_FIELD iter_create.flags 5216ac51d99bSYonghong Song 5217ac51d99bSYonghong Song static int bpf_iter_create(union bpf_attr *attr) 5218ac51d99bSYonghong Song { 5219ac51d99bSYonghong Song struct bpf_link *link; 5220ac51d99bSYonghong Song int err; 5221ac51d99bSYonghong Song 5222ac51d99bSYonghong Song if (CHECK_ATTR(BPF_ITER_CREATE)) 5223ac51d99bSYonghong Song return -EINVAL; 5224ac51d99bSYonghong Song 5225ac51d99bSYonghong Song if (attr->iter_create.flags) 5226ac51d99bSYonghong Song return -EINVAL; 5227ac51d99bSYonghong Song 5228ac51d99bSYonghong Song link = bpf_link_get_from_fd(attr->iter_create.link_fd); 5229ac51d99bSYonghong Song if (IS_ERR(link)) 5230ac51d99bSYonghong Song return PTR_ERR(link); 5231ac51d99bSYonghong Song 5232ac51d99bSYonghong Song err = bpf_iter_new_fd(link); 5233ab5d47bdSSebastian Andrzej Siewior bpf_link_put_direct(link); 5234ac51d99bSYonghong Song 5235ac51d99bSYonghong Song return err; 5236ac51d99bSYonghong Song } 5237ac51d99bSYonghong Song 5238ef15314aSYiFei Zhu #define BPF_PROG_BIND_MAP_LAST_FIELD prog_bind_map.flags 5239ef15314aSYiFei Zhu 5240ef15314aSYiFei Zhu static int bpf_prog_bind_map(union bpf_attr *attr) 5241ef15314aSYiFei Zhu { 5242ef15314aSYiFei Zhu struct bpf_prog *prog; 5243ef15314aSYiFei Zhu struct bpf_map *map; 5244ef15314aSYiFei Zhu struct bpf_map **used_maps_old, **used_maps_new; 5245ef15314aSYiFei Zhu int i, ret = 0; 5246ef15314aSYiFei Zhu 5247ef15314aSYiFei Zhu if (CHECK_ATTR(BPF_PROG_BIND_MAP)) 5248ef15314aSYiFei Zhu return -EINVAL; 5249ef15314aSYiFei Zhu 5250ef15314aSYiFei Zhu if (attr->prog_bind_map.flags) 5251ef15314aSYiFei Zhu return -EINVAL; 5252ef15314aSYiFei Zhu 5253ef15314aSYiFei Zhu prog = bpf_prog_get(attr->prog_bind_map.prog_fd); 5254ef15314aSYiFei Zhu if (IS_ERR(prog)) 5255ef15314aSYiFei Zhu return PTR_ERR(prog); 5256ef15314aSYiFei Zhu 5257ef15314aSYiFei Zhu map = bpf_map_get(attr->prog_bind_map.map_fd); 5258ef15314aSYiFei Zhu if (IS_ERR(map)) { 5259ef15314aSYiFei Zhu ret = PTR_ERR(map); 5260ef15314aSYiFei Zhu goto out_prog_put; 5261ef15314aSYiFei Zhu } 5262ef15314aSYiFei Zhu 5263ef15314aSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 5264ef15314aSYiFei Zhu 5265ef15314aSYiFei Zhu used_maps_old = prog->aux->used_maps; 5266ef15314aSYiFei Zhu 5267ef15314aSYiFei Zhu for (i = 0; i < prog->aux->used_map_cnt; i++) 52681028ae40SStanislav Fomichev if (used_maps_old[i] == map) { 52691028ae40SStanislav Fomichev bpf_map_put(map); 5270ef15314aSYiFei Zhu goto out_unlock; 52711028ae40SStanislav Fomichev } 5272ef15314aSYiFei Zhu 5273ef15314aSYiFei Zhu used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1, 5274ef15314aSYiFei Zhu sizeof(used_maps_new[0]), 5275ef15314aSYiFei Zhu GFP_KERNEL); 5276ef15314aSYiFei Zhu if (!used_maps_new) { 5277ef15314aSYiFei Zhu ret = -ENOMEM; 5278ef15314aSYiFei Zhu goto out_unlock; 5279ef15314aSYiFei Zhu } 5280ef15314aSYiFei Zhu 5281ef15314aSYiFei Zhu memcpy(used_maps_new, used_maps_old, 5282ef15314aSYiFei Zhu sizeof(used_maps_old[0]) * prog->aux->used_map_cnt); 5283ef15314aSYiFei Zhu used_maps_new[prog->aux->used_map_cnt] = map; 5284ef15314aSYiFei Zhu 5285ef15314aSYiFei Zhu prog->aux->used_map_cnt++; 5286ef15314aSYiFei Zhu prog->aux->used_maps = used_maps_new; 5287ef15314aSYiFei Zhu 5288ef15314aSYiFei Zhu kfree(used_maps_old); 5289ef15314aSYiFei Zhu 5290ef15314aSYiFei Zhu out_unlock: 5291ef15314aSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 5292ef15314aSYiFei Zhu 5293ef15314aSYiFei Zhu if (ret) 5294ef15314aSYiFei Zhu bpf_map_put(map); 5295ef15314aSYiFei Zhu out_prog_put: 5296ef15314aSYiFei Zhu bpf_prog_put(prog); 5297ef15314aSYiFei Zhu return ret; 5298ef15314aSYiFei Zhu } 5299ef15314aSYiFei Zhu 5300af2ac3e1SAlexei Starovoitov static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) 530199c55f7dSAlexei Starovoitov { 53028096f229SGreg Kroah-Hartman union bpf_attr attr; 530399c55f7dSAlexei Starovoitov int err; 530499c55f7dSAlexei Starovoitov 5305dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size); 530699c55f7dSAlexei Starovoitov if (err) 530799c55f7dSAlexei Starovoitov return err; 53081e270976SMartin KaFai Lau size = min_t(u32, size, sizeof(attr)); 530999c55f7dSAlexei Starovoitov 531099c55f7dSAlexei Starovoitov /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 53118096f229SGreg Kroah-Hartman memset(&attr, 0, sizeof(attr)); 5312af2ac3e1SAlexei Starovoitov if (copy_from_bpfptr(&attr, uattr, size) != 0) 531399c55f7dSAlexei Starovoitov return -EFAULT; 531499c55f7dSAlexei Starovoitov 5315afdb09c7SChenbo Feng err = security_bpf(cmd, &attr, size); 5316afdb09c7SChenbo Feng if (err < 0) 5317afdb09c7SChenbo Feng return err; 5318afdb09c7SChenbo Feng 531999c55f7dSAlexei Starovoitov switch (cmd) { 532099c55f7dSAlexei Starovoitov case BPF_MAP_CREATE: 532199c55f7dSAlexei Starovoitov err = map_create(&attr); 532299c55f7dSAlexei Starovoitov break; 5323db20fd2bSAlexei Starovoitov case BPF_MAP_LOOKUP_ELEM: 5324db20fd2bSAlexei Starovoitov err = map_lookup_elem(&attr); 5325db20fd2bSAlexei Starovoitov break; 5326db20fd2bSAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 5327af2ac3e1SAlexei Starovoitov err = map_update_elem(&attr, uattr); 5328db20fd2bSAlexei Starovoitov break; 5329db20fd2bSAlexei Starovoitov case BPF_MAP_DELETE_ELEM: 5330b88df697SBenjamin Tissoires err = map_delete_elem(&attr, uattr); 5331db20fd2bSAlexei Starovoitov break; 5332db20fd2bSAlexei Starovoitov case BPF_MAP_GET_NEXT_KEY: 5333db20fd2bSAlexei Starovoitov err = map_get_next_key(&attr); 5334db20fd2bSAlexei Starovoitov break; 533587df15deSDaniel Borkmann case BPF_MAP_FREEZE: 533687df15deSDaniel Borkmann err = map_freeze(&attr); 533787df15deSDaniel Borkmann break; 533809756af4SAlexei Starovoitov case BPF_PROG_LOAD: 533947a71c1fSAndrii Nakryiko err = bpf_prog_load(&attr, uattr, size); 534009756af4SAlexei Starovoitov break; 5341b2197755SDaniel Borkmann case BPF_OBJ_PIN: 5342b2197755SDaniel Borkmann err = bpf_obj_pin(&attr); 5343b2197755SDaniel Borkmann break; 5344b2197755SDaniel Borkmann case BPF_OBJ_GET: 5345b2197755SDaniel Borkmann err = bpf_obj_get(&attr); 5346b2197755SDaniel Borkmann break; 5347f4324551SDaniel Mack case BPF_PROG_ATTACH: 5348f4324551SDaniel Mack err = bpf_prog_attach(&attr); 5349f4324551SDaniel Mack break; 5350f4324551SDaniel Mack case BPF_PROG_DETACH: 5351f4324551SDaniel Mack err = bpf_prog_detach(&attr); 5352f4324551SDaniel Mack break; 5353468e2f64SAlexei Starovoitov case BPF_PROG_QUERY: 5354af2ac3e1SAlexei Starovoitov err = bpf_prog_query(&attr, uattr.user); 5355468e2f64SAlexei Starovoitov break; 53561cf1cae9SAlexei Starovoitov case BPF_PROG_TEST_RUN: 5357af2ac3e1SAlexei Starovoitov err = bpf_prog_test_run(&attr, uattr.user); 53581cf1cae9SAlexei Starovoitov break; 535934ad5580SMartin KaFai Lau case BPF_PROG_GET_NEXT_ID: 5360af2ac3e1SAlexei Starovoitov err = bpf_obj_get_next_id(&attr, uattr.user, 536134ad5580SMartin KaFai Lau &prog_idr, &prog_idr_lock); 536234ad5580SMartin KaFai Lau break; 536334ad5580SMartin KaFai Lau case BPF_MAP_GET_NEXT_ID: 5364af2ac3e1SAlexei Starovoitov err = bpf_obj_get_next_id(&attr, uattr.user, 536534ad5580SMartin KaFai Lau &map_idr, &map_idr_lock); 536634ad5580SMartin KaFai Lau break; 53671b9ed84eSQuentin Monnet case BPF_BTF_GET_NEXT_ID: 5368af2ac3e1SAlexei Starovoitov err = bpf_obj_get_next_id(&attr, uattr.user, 53691b9ed84eSQuentin Monnet &btf_idr, &btf_idr_lock); 53701b9ed84eSQuentin Monnet break; 5371b16d9aa4SMartin KaFai Lau case BPF_PROG_GET_FD_BY_ID: 5372b16d9aa4SMartin KaFai Lau err = bpf_prog_get_fd_by_id(&attr); 5373b16d9aa4SMartin KaFai Lau break; 5374bd5f5f4eSMartin KaFai Lau case BPF_MAP_GET_FD_BY_ID: 5375bd5f5f4eSMartin KaFai Lau err = bpf_map_get_fd_by_id(&attr); 5376bd5f5f4eSMartin KaFai Lau break; 53771e270976SMartin KaFai Lau case BPF_OBJ_GET_INFO_BY_FD: 5378af2ac3e1SAlexei Starovoitov err = bpf_obj_get_info_by_fd(&attr, uattr.user); 53791e270976SMartin KaFai Lau break; 5380c4f6699dSAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN: 5381c4f6699dSAlexei Starovoitov err = bpf_raw_tracepoint_open(&attr); 5382c4f6699dSAlexei Starovoitov break; 5383f56a653cSMartin KaFai Lau case BPF_BTF_LOAD: 538447a71c1fSAndrii Nakryiko err = bpf_btf_load(&attr, uattr, size); 5385f56a653cSMartin KaFai Lau break; 538678958fcaSMartin KaFai Lau case BPF_BTF_GET_FD_BY_ID: 538778958fcaSMartin KaFai Lau err = bpf_btf_get_fd_by_id(&attr); 538878958fcaSMartin KaFai Lau break; 538941bdc4b4SYonghong Song case BPF_TASK_FD_QUERY: 5390af2ac3e1SAlexei Starovoitov err = bpf_task_fd_query(&attr, uattr.user); 539141bdc4b4SYonghong Song break; 5392bd513cd0SMauricio Vasquez B case BPF_MAP_LOOKUP_AND_DELETE_ELEM: 5393bd513cd0SMauricio Vasquez B err = map_lookup_and_delete_elem(&attr); 5394bd513cd0SMauricio Vasquez B break; 5395cb4d03abSBrian Vazquez case BPF_MAP_LOOKUP_BATCH: 5396af2ac3e1SAlexei Starovoitov err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_LOOKUP_BATCH); 5397cb4d03abSBrian Vazquez break; 539805799638SYonghong Song case BPF_MAP_LOOKUP_AND_DELETE_BATCH: 5399af2ac3e1SAlexei Starovoitov err = bpf_map_do_batch(&attr, uattr.user, 540005799638SYonghong Song BPF_MAP_LOOKUP_AND_DELETE_BATCH); 540105799638SYonghong Song break; 5402aa2e93b8SBrian Vazquez case BPF_MAP_UPDATE_BATCH: 5403af2ac3e1SAlexei Starovoitov err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_UPDATE_BATCH); 5404aa2e93b8SBrian Vazquez break; 5405aa2e93b8SBrian Vazquez case BPF_MAP_DELETE_BATCH: 5406af2ac3e1SAlexei Starovoitov err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_DELETE_BATCH); 5407aa2e93b8SBrian Vazquez break; 5408af6eea57SAndrii Nakryiko case BPF_LINK_CREATE: 5409af2ac3e1SAlexei Starovoitov err = link_create(&attr, uattr); 5410af6eea57SAndrii Nakryiko break; 54110c991ebcSAndrii Nakryiko case BPF_LINK_UPDATE: 54120c991ebcSAndrii Nakryiko err = link_update(&attr); 54130c991ebcSAndrii Nakryiko break; 54142d602c8cSAndrii Nakryiko case BPF_LINK_GET_FD_BY_ID: 54152d602c8cSAndrii Nakryiko err = bpf_link_get_fd_by_id(&attr); 54162d602c8cSAndrii Nakryiko break; 54172d602c8cSAndrii Nakryiko case BPF_LINK_GET_NEXT_ID: 5418af2ac3e1SAlexei Starovoitov err = bpf_obj_get_next_id(&attr, uattr.user, 54192d602c8cSAndrii Nakryiko &link_idr, &link_idr_lock); 54202d602c8cSAndrii Nakryiko break; 5421d46edd67SSong Liu case BPF_ENABLE_STATS: 5422d46edd67SSong Liu err = bpf_enable_stats(&attr); 5423d46edd67SSong Liu break; 5424ac51d99bSYonghong Song case BPF_ITER_CREATE: 5425ac51d99bSYonghong Song err = bpf_iter_create(&attr); 5426ac51d99bSYonghong Song break; 542773b11c2aSAndrii Nakryiko case BPF_LINK_DETACH: 542873b11c2aSAndrii Nakryiko err = link_detach(&attr); 542973b11c2aSAndrii Nakryiko break; 5430ef15314aSYiFei Zhu case BPF_PROG_BIND_MAP: 5431ef15314aSYiFei Zhu err = bpf_prog_bind_map(&attr); 5432ef15314aSYiFei Zhu break; 543399c55f7dSAlexei Starovoitov default: 543499c55f7dSAlexei Starovoitov err = -EINVAL; 543599c55f7dSAlexei Starovoitov break; 543699c55f7dSAlexei Starovoitov } 543799c55f7dSAlexei Starovoitov 543899c55f7dSAlexei Starovoitov return err; 543999c55f7dSAlexei Starovoitov } 544079a7f8bdSAlexei Starovoitov 5441af2ac3e1SAlexei Starovoitov SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 5442af2ac3e1SAlexei Starovoitov { 5443af2ac3e1SAlexei Starovoitov return __sys_bpf(cmd, USER_BPFPTR(uattr), size); 5444af2ac3e1SAlexei Starovoitov } 5445af2ac3e1SAlexei Starovoitov 544679a7f8bdSAlexei Starovoitov static bool syscall_prog_is_valid_access(int off, int size, 544779a7f8bdSAlexei Starovoitov enum bpf_access_type type, 544879a7f8bdSAlexei Starovoitov const struct bpf_prog *prog, 544979a7f8bdSAlexei Starovoitov struct bpf_insn_access_aux *info) 545079a7f8bdSAlexei Starovoitov { 545179a7f8bdSAlexei Starovoitov if (off < 0 || off >= U16_MAX) 545279a7f8bdSAlexei Starovoitov return false; 545379a7f8bdSAlexei Starovoitov if (off % size != 0) 545479a7f8bdSAlexei Starovoitov return false; 545579a7f8bdSAlexei Starovoitov return true; 545679a7f8bdSAlexei Starovoitov } 545779a7f8bdSAlexei Starovoitov 5458b1d18a75SAlexei Starovoitov BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size) 545979a7f8bdSAlexei Starovoitov { 5460af2ac3e1SAlexei Starovoitov switch (cmd) { 5461af2ac3e1SAlexei Starovoitov case BPF_MAP_CREATE: 5462b88df697SBenjamin Tissoires case BPF_MAP_DELETE_ELEM: 5463af2ac3e1SAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 5464af2ac3e1SAlexei Starovoitov case BPF_MAP_FREEZE: 5465b88df697SBenjamin Tissoires case BPF_MAP_GET_FD_BY_ID: 5466af2ac3e1SAlexei Starovoitov case BPF_PROG_LOAD: 5467c571bd75SAlexei Starovoitov case BPF_BTF_LOAD: 5468b1d18a75SAlexei Starovoitov case BPF_LINK_CREATE: 5469b1d18a75SAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN: 5470af2ac3e1SAlexei Starovoitov break; 547186f44fceSAlexei Starovoitov default: 547286f44fceSAlexei Starovoitov return -EINVAL; 547386f44fceSAlexei Starovoitov } 547486f44fceSAlexei Starovoitov return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size); 547586f44fceSAlexei Starovoitov } 547686f44fceSAlexei Starovoitov 54774e4588f1SAlexei Starovoitov 54784e4588f1SAlexei Starovoitov /* To shut up -Wmissing-prototypes. 54794e4588f1SAlexei Starovoitov * This function is used by the kernel light skeleton 54804e4588f1SAlexei Starovoitov * to load bpf programs when modules are loaded or during kernel boot. 54814e4588f1SAlexei Starovoitov * See tools/lib/bpf/skel_internal.h 54824e4588f1SAlexei Starovoitov */ 54834e4588f1SAlexei Starovoitov int kern_sys_bpf(int cmd, union bpf_attr *attr, unsigned int size); 54844e4588f1SAlexei Starovoitov 548586f44fceSAlexei Starovoitov int kern_sys_bpf(int cmd, union bpf_attr *attr, unsigned int size) 548686f44fceSAlexei Starovoitov { 548786f44fceSAlexei Starovoitov struct bpf_prog * __maybe_unused prog; 548886f44fceSAlexei Starovoitov struct bpf_tramp_run_ctx __maybe_unused run_ctx; 548986f44fceSAlexei Starovoitov 549086f44fceSAlexei Starovoitov switch (cmd) { 5491b1d18a75SAlexei Starovoitov #ifdef CONFIG_BPF_JIT /* __bpf_prog_enter_sleepable used by trampoline and JIT */ 5492b1d18a75SAlexei Starovoitov case BPF_PROG_TEST_RUN: 5493b1d18a75SAlexei Starovoitov if (attr->test.data_in || attr->test.data_out || 5494b1d18a75SAlexei Starovoitov attr->test.ctx_out || attr->test.duration || 5495b1d18a75SAlexei Starovoitov attr->test.repeat || attr->test.flags) 5496b1d18a75SAlexei Starovoitov return -EINVAL; 5497b1d18a75SAlexei Starovoitov 5498b1d18a75SAlexei Starovoitov prog = bpf_prog_get_type(attr->test.prog_fd, BPF_PROG_TYPE_SYSCALL); 5499b1d18a75SAlexei Starovoitov if (IS_ERR(prog)) 5500b1d18a75SAlexei Starovoitov return PTR_ERR(prog); 5501b1d18a75SAlexei Starovoitov 5502b1d18a75SAlexei Starovoitov if (attr->test.ctx_size_in < prog->aux->max_ctx_offset || 5503b1d18a75SAlexei Starovoitov attr->test.ctx_size_in > U16_MAX) { 5504b1d18a75SAlexei Starovoitov bpf_prog_put(prog); 5505b1d18a75SAlexei Starovoitov return -EINVAL; 5506b1d18a75SAlexei Starovoitov } 5507b1d18a75SAlexei Starovoitov 5508e384c7b7SKui-Feng Lee run_ctx.bpf_cookie = 0; 5509271de525SMartin KaFai Lau if (!__bpf_prog_enter_sleepable_recur(prog, &run_ctx)) { 5510b1d18a75SAlexei Starovoitov /* recursion detected */ 55117645629fSSebastian Andrzej Siewior __bpf_prog_exit_sleepable_recur(prog, 0, &run_ctx); 5512b1d18a75SAlexei Starovoitov bpf_prog_put(prog); 5513b1d18a75SAlexei Starovoitov return -EBUSY; 5514b1d18a75SAlexei Starovoitov } 5515b1d18a75SAlexei Starovoitov attr->test.retval = bpf_prog_run(prog, (void *) (long) attr->test.ctx_in); 5516271de525SMartin KaFai Lau __bpf_prog_exit_sleepable_recur(prog, 0 /* bpf_prog_run does runtime stats */, 5517271de525SMartin KaFai Lau &run_ctx); 5518b1d18a75SAlexei Starovoitov bpf_prog_put(prog); 5519b1d18a75SAlexei Starovoitov return 0; 5520b1d18a75SAlexei Starovoitov #endif 5521af2ac3e1SAlexei Starovoitov default: 552286f44fceSAlexei Starovoitov return ____bpf_sys_bpf(cmd, attr, size); 552379a7f8bdSAlexei Starovoitov } 5524af2ac3e1SAlexei Starovoitov } 552586f44fceSAlexei Starovoitov EXPORT_SYMBOL(kern_sys_bpf); 552679a7f8bdSAlexei Starovoitov 55273a2daa72SPu Lehui static const struct bpf_func_proto bpf_sys_bpf_proto = { 552879a7f8bdSAlexei Starovoitov .func = bpf_sys_bpf, 552979a7f8bdSAlexei Starovoitov .gpl_only = false, 553079a7f8bdSAlexei Starovoitov .ret_type = RET_INTEGER, 553179a7f8bdSAlexei Starovoitov .arg1_type = ARG_ANYTHING, 5532216e3cd2SHao Luo .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, 553379a7f8bdSAlexei Starovoitov .arg3_type = ARG_CONST_SIZE, 553479a7f8bdSAlexei Starovoitov }; 553579a7f8bdSAlexei Starovoitov 553679a7f8bdSAlexei Starovoitov const struct bpf_func_proto * __weak 553779a7f8bdSAlexei Starovoitov tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 553879a7f8bdSAlexei Starovoitov { 553979a7f8bdSAlexei Starovoitov return bpf_base_func_proto(func_id); 554079a7f8bdSAlexei Starovoitov } 554179a7f8bdSAlexei Starovoitov 55423abea089SAlexei Starovoitov BPF_CALL_1(bpf_sys_close, u32, fd) 55433abea089SAlexei Starovoitov { 55443abea089SAlexei Starovoitov /* When bpf program calls this helper there should not be 55453abea089SAlexei Starovoitov * an fdget() without matching completed fdput(). 55463abea089SAlexei Starovoitov * This helper is allowed in the following callchain only: 55473abea089SAlexei Starovoitov * sys_bpf->prog_test_run->bpf_prog->bpf_sys_close 55483abea089SAlexei Starovoitov */ 55493abea089SAlexei Starovoitov return close_fd(fd); 55503abea089SAlexei Starovoitov } 55513abea089SAlexei Starovoitov 55523a2daa72SPu Lehui static const struct bpf_func_proto bpf_sys_close_proto = { 55533abea089SAlexei Starovoitov .func = bpf_sys_close, 55543abea089SAlexei Starovoitov .gpl_only = false, 55553abea089SAlexei Starovoitov .ret_type = RET_INTEGER, 55563abea089SAlexei Starovoitov .arg1_type = ARG_ANYTHING, 55573abea089SAlexei Starovoitov }; 55583abea089SAlexei Starovoitov 5559d6aef08aSKumar Kartikeya Dwivedi BPF_CALL_4(bpf_kallsyms_lookup_name, const char *, name, int, name_sz, int, flags, u64 *, res) 5560d6aef08aSKumar Kartikeya Dwivedi { 5561d6aef08aSKumar Kartikeya Dwivedi if (flags) 5562d6aef08aSKumar Kartikeya Dwivedi return -EINVAL; 5563d6aef08aSKumar Kartikeya Dwivedi 5564d6aef08aSKumar Kartikeya Dwivedi if (name_sz <= 1 || name[name_sz - 1]) 5565d6aef08aSKumar Kartikeya Dwivedi return -EINVAL; 5566d6aef08aSKumar Kartikeya Dwivedi 5567d6aef08aSKumar Kartikeya Dwivedi if (!bpf_dump_raw_ok(current_cred())) 5568d6aef08aSKumar Kartikeya Dwivedi return -EPERM; 5569d6aef08aSKumar Kartikeya Dwivedi 5570d6aef08aSKumar Kartikeya Dwivedi *res = kallsyms_lookup_name(name); 5571d6aef08aSKumar Kartikeya Dwivedi return *res ? 0 : -ENOENT; 5572d6aef08aSKumar Kartikeya Dwivedi } 5573d6aef08aSKumar Kartikeya Dwivedi 5574dc368e1cSJoanne Koong static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = { 5575d6aef08aSKumar Kartikeya Dwivedi .func = bpf_kallsyms_lookup_name, 5576d6aef08aSKumar Kartikeya Dwivedi .gpl_only = false, 5577d6aef08aSKumar Kartikeya Dwivedi .ret_type = RET_INTEGER, 5578d6aef08aSKumar Kartikeya Dwivedi .arg1_type = ARG_PTR_TO_MEM, 5579d4efb170SKumar Kartikeya Dwivedi .arg2_type = ARG_CONST_SIZE_OR_ZERO, 5580d6aef08aSKumar Kartikeya Dwivedi .arg3_type = ARG_ANYTHING, 5581d6aef08aSKumar Kartikeya Dwivedi .arg4_type = ARG_PTR_TO_LONG, 5582d6aef08aSKumar Kartikeya Dwivedi }; 5583d6aef08aSKumar Kartikeya Dwivedi 558479a7f8bdSAlexei Starovoitov static const struct bpf_func_proto * 558579a7f8bdSAlexei Starovoitov syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 558679a7f8bdSAlexei Starovoitov { 558779a7f8bdSAlexei Starovoitov switch (func_id) { 558879a7f8bdSAlexei Starovoitov case BPF_FUNC_sys_bpf: 558914b20b78SYiFei Zhu return !perfmon_capable() ? NULL : &bpf_sys_bpf_proto; 55903d78417bSAlexei Starovoitov case BPF_FUNC_btf_find_by_name_kind: 55913d78417bSAlexei Starovoitov return &bpf_btf_find_by_name_kind_proto; 55923abea089SAlexei Starovoitov case BPF_FUNC_sys_close: 55933abea089SAlexei Starovoitov return &bpf_sys_close_proto; 5594d6aef08aSKumar Kartikeya Dwivedi case BPF_FUNC_kallsyms_lookup_name: 5595d6aef08aSKumar Kartikeya Dwivedi return &bpf_kallsyms_lookup_name_proto; 559679a7f8bdSAlexei Starovoitov default: 559779a7f8bdSAlexei Starovoitov return tracing_prog_func_proto(func_id, prog); 559879a7f8bdSAlexei Starovoitov } 559979a7f8bdSAlexei Starovoitov } 560079a7f8bdSAlexei Starovoitov 560179a7f8bdSAlexei Starovoitov const struct bpf_verifier_ops bpf_syscall_verifier_ops = { 560279a7f8bdSAlexei Starovoitov .get_func_proto = syscall_prog_func_proto, 560379a7f8bdSAlexei Starovoitov .is_valid_access = syscall_prog_is_valid_access, 560479a7f8bdSAlexei Starovoitov }; 560579a7f8bdSAlexei Starovoitov 560679a7f8bdSAlexei Starovoitov const struct bpf_prog_ops bpf_syscall_prog_ops = { 560779a7f8bdSAlexei Starovoitov .test_run = bpf_prog_test_run_syscall, 560879a7f8bdSAlexei Starovoitov }; 56092900005eSYan Zhu 56102900005eSYan Zhu #ifdef CONFIG_SYSCTL 56112900005eSYan Zhu static int bpf_stats_handler(struct ctl_table *table, int write, 56122900005eSYan Zhu void *buffer, size_t *lenp, loff_t *ppos) 56132900005eSYan Zhu { 56142900005eSYan Zhu struct static_key *key = (struct static_key *)table->data; 56152900005eSYan Zhu static int saved_val; 56162900005eSYan Zhu int val, ret; 56172900005eSYan Zhu struct ctl_table tmp = { 56182900005eSYan Zhu .data = &val, 56192900005eSYan Zhu .maxlen = sizeof(val), 56202900005eSYan Zhu .mode = table->mode, 56212900005eSYan Zhu .extra1 = SYSCTL_ZERO, 56222900005eSYan Zhu .extra2 = SYSCTL_ONE, 56232900005eSYan Zhu }; 56242900005eSYan Zhu 56252900005eSYan Zhu if (write && !capable(CAP_SYS_ADMIN)) 56262900005eSYan Zhu return -EPERM; 56272900005eSYan Zhu 56282900005eSYan Zhu mutex_lock(&bpf_stats_enabled_mutex); 56292900005eSYan Zhu val = saved_val; 56302900005eSYan Zhu ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos); 56312900005eSYan Zhu if (write && !ret && val != saved_val) { 56322900005eSYan Zhu if (val) 56332900005eSYan Zhu static_key_slow_inc(key); 56342900005eSYan Zhu else 56352900005eSYan Zhu static_key_slow_dec(key); 56362900005eSYan Zhu saved_val = val; 56372900005eSYan Zhu } 56382900005eSYan Zhu mutex_unlock(&bpf_stats_enabled_mutex); 56392900005eSYan Zhu return ret; 56402900005eSYan Zhu } 56412900005eSYan Zhu 56422900005eSYan Zhu void __weak unpriv_ebpf_notify(int new_state) 56432900005eSYan Zhu { 56442900005eSYan Zhu } 56452900005eSYan Zhu 56462900005eSYan Zhu static int bpf_unpriv_handler(struct ctl_table *table, int write, 56472900005eSYan Zhu void *buffer, size_t *lenp, loff_t *ppos) 56482900005eSYan Zhu { 56492900005eSYan Zhu int ret, unpriv_enable = *(int *)table->data; 56502900005eSYan Zhu bool locked_state = unpriv_enable == 1; 56512900005eSYan Zhu struct ctl_table tmp = *table; 56522900005eSYan Zhu 56532900005eSYan Zhu if (write && !capable(CAP_SYS_ADMIN)) 56542900005eSYan Zhu return -EPERM; 56552900005eSYan Zhu 56562900005eSYan Zhu tmp.data = &unpriv_enable; 56572900005eSYan Zhu ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos); 56582900005eSYan Zhu if (write && !ret) { 56592900005eSYan Zhu if (locked_state && unpriv_enable != 1) 56602900005eSYan Zhu return -EPERM; 56612900005eSYan Zhu *(int *)table->data = unpriv_enable; 56622900005eSYan Zhu } 56632900005eSYan Zhu 5664fedf9920SKui-Feng Lee if (write) 56652900005eSYan Zhu unpriv_ebpf_notify(unpriv_enable); 56662900005eSYan Zhu 56672900005eSYan Zhu return ret; 56682900005eSYan Zhu } 56692900005eSYan Zhu 56702900005eSYan Zhu static struct ctl_table bpf_syscall_table[] = { 56712900005eSYan Zhu { 56722900005eSYan Zhu .procname = "unprivileged_bpf_disabled", 56732900005eSYan Zhu .data = &sysctl_unprivileged_bpf_disabled, 56742900005eSYan Zhu .maxlen = sizeof(sysctl_unprivileged_bpf_disabled), 56752900005eSYan Zhu .mode = 0644, 56762900005eSYan Zhu .proc_handler = bpf_unpriv_handler, 56772900005eSYan Zhu .extra1 = SYSCTL_ZERO, 56782900005eSYan Zhu .extra2 = SYSCTL_TWO, 56792900005eSYan Zhu }, 56802900005eSYan Zhu { 56812900005eSYan Zhu .procname = "bpf_stats_enabled", 56822900005eSYan Zhu .data = &bpf_stats_enabled_key.key, 56832900005eSYan Zhu .mode = 0644, 56842900005eSYan Zhu .proc_handler = bpf_stats_handler, 56852900005eSYan Zhu }, 56862900005eSYan Zhu { } 56872900005eSYan Zhu }; 56882900005eSYan Zhu 56892900005eSYan Zhu static int __init bpf_syscall_sysctl_init(void) 56902900005eSYan Zhu { 56912900005eSYan Zhu register_sysctl_init("kernel", bpf_syscall_table); 56922900005eSYan Zhu return 0; 56932900005eSYan Zhu } 56942900005eSYan Zhu late_initcall(bpf_syscall_sysctl_init); 56952900005eSYan Zhu #endif /* CONFIG_SYSCTL */ 5696