15b497af4SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 299c55f7dSAlexei Starovoitov /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 399c55f7dSAlexei Starovoitov */ 499c55f7dSAlexei Starovoitov #include <linux/bpf.h> 5aef2fedaSJakub Kicinski #include <linux/bpf-cgroup.h> 6a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 7f4364dcfSSean Young #include <linux/bpf_lirc.h> 84a1e7c0cSToke Høiland-Jørgensen #include <linux/bpf_verifier.h> 961df10c7SKumar Kartikeya Dwivedi #include <linux/bsearch.h> 10f56a653cSMartin KaFai Lau #include <linux/btf.h> 1199c55f7dSAlexei Starovoitov #include <linux/syscalls.h> 1299c55f7dSAlexei Starovoitov #include <linux/slab.h> 133f07c014SIngo Molnar #include <linux/sched/signal.h> 14d407bd25SDaniel Borkmann #include <linux/vmalloc.h> 15d407bd25SDaniel Borkmann #include <linux/mmzone.h> 1699c55f7dSAlexei Starovoitov #include <linux/anon_inodes.h> 1741bdc4b4SYonghong Song #include <linux/fdtable.h> 18db20fd2bSAlexei Starovoitov #include <linux/file.h> 1941bdc4b4SYonghong Song #include <linux/fs.h> 2009756af4SAlexei Starovoitov #include <linux/license.h> 2109756af4SAlexei Starovoitov #include <linux/filter.h> 22535e7b4bSMickaël Salaün #include <linux/kernel.h> 23dc4bb0e2SMartin KaFai Lau #include <linux/idr.h> 24cb4d2b3fSMartin KaFai Lau #include <linux/cred.h> 25cb4d2b3fSMartin KaFai Lau #include <linux/timekeeping.h> 26cb4d2b3fSMartin KaFai Lau #include <linux/ctype.h> 279ef09e35SMark Rutland #include <linux/nospec.h> 28bae141f5SDaniel Borkmann #include <linux/audit.h> 29ccfe29ebSAlexei Starovoitov #include <uapi/linux/btf.h> 30ca5999fdSMike Rapoport #include <linux/pgtable.h> 319e4e01dfSKP Singh #include <linux/bpf_lsm.h> 32457f4436SAndrii Nakryiko #include <linux/poll.h> 334d7d7f69SKumar Kartikeya Dwivedi #include <linux/sort.h> 34a3fd7ceeSJakub Sitnicki #include <linux/bpf-netns.h> 351e6c62a8SAlexei Starovoitov #include <linux/rcupdate_trace.h> 3648edc1f7SRoman Gushchin #include <linux/memcontrol.h> 370dcac272SJiri Olsa #include <linux/trace_events.h> 3899c55f7dSAlexei Starovoitov 3935dfaad7SDaniel Borkmann #include <net/netfilter/nf_bpf_link.h> 4035dfaad7SDaniel Borkmann #include <net/netkit.h> 41e420bed0SDaniel Borkmann #include <net/tcx.h> 42e420bed0SDaniel Borkmann 43da765a2fSDaniel Borkmann #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 4414dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 4514dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 46da765a2fSDaniel Borkmann #define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY) 4714dc6f04SMartin KaFai Lau #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 48da765a2fSDaniel Borkmann #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \ 49da765a2fSDaniel Borkmann IS_FD_HASH(map)) 5014dc6f04SMartin KaFai Lau 516e71b04aSChenbo Feng #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) 526e71b04aSChenbo Feng 53b121d1e7SAlexei Starovoitov DEFINE_PER_CPU(int, bpf_prog_active); 54dc4bb0e2SMartin KaFai Lau static DEFINE_IDR(prog_idr); 55dc4bb0e2SMartin KaFai Lau static DEFINE_SPINLOCK(prog_idr_lock); 56f3f1c054SMartin KaFai Lau static DEFINE_IDR(map_idr); 57f3f1c054SMartin KaFai Lau static DEFINE_SPINLOCK(map_idr_lock); 58a3b80e10SAndrii Nakryiko static DEFINE_IDR(link_idr); 59a3b80e10SAndrii Nakryiko static DEFINE_SPINLOCK(link_idr_lock); 60b121d1e7SAlexei Starovoitov 6108389d88SDaniel Borkmann int sysctl_unprivileged_bpf_disabled __read_mostly = 6208389d88SDaniel Borkmann IS_BUILTIN(CONFIG_BPF_UNPRIV_DEFAULT_OFF) ? 2 : 0; 631be7f75dSAlexei Starovoitov 6440077e0cSJohannes Berg static const struct bpf_map_ops * const bpf_map_types[] = { 6591cc1a99SAlexei Starovoitov #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) 6640077e0cSJohannes Berg #define BPF_MAP_TYPE(_id, _ops) \ 6740077e0cSJohannes Berg [_id] = &_ops, 68f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) 6940077e0cSJohannes Berg #include <linux/bpf_types.h> 7040077e0cSJohannes Berg #undef BPF_PROG_TYPE 7140077e0cSJohannes Berg #undef BPF_MAP_TYPE 72f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 7340077e0cSJohannes Berg }; 7499c55f7dSAlexei Starovoitov 75752ba56fSMickaël Salaün /* 76752ba56fSMickaël Salaün * If we're handed a bigger struct than we know of, ensure all the unknown bits 77752ba56fSMickaël Salaün * are 0 - i.e. new user-space does not rely on any kernel feature extensions 78752ba56fSMickaël Salaün * we don't know about yet. 79752ba56fSMickaël Salaün * 80752ba56fSMickaël Salaün * There is a ToCToU between this function call and the following 81752ba56fSMickaël Salaün * copy_from_user() call. However, this is not a concern since this function is 82752ba56fSMickaël Salaün * meant to be a future-proofing of bits. 83752ba56fSMickaël Salaün */ 84af2ac3e1SAlexei Starovoitov int bpf_check_uarg_tail_zero(bpfptr_t uaddr, 8558291a74SMickaël Salaün size_t expected_size, 8658291a74SMickaël Salaün size_t actual_size) 8758291a74SMickaël Salaün { 88b7e4b65fSAl Viro int res; 8958291a74SMickaël Salaün 90752ba56fSMickaël Salaün if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 91752ba56fSMickaël Salaün return -E2BIG; 92752ba56fSMickaël Salaün 9358291a74SMickaël Salaün if (actual_size <= expected_size) 9458291a74SMickaël Salaün return 0; 9558291a74SMickaël Salaün 96af2ac3e1SAlexei Starovoitov if (uaddr.is_kernel) 97af2ac3e1SAlexei Starovoitov res = memchr_inv(uaddr.kernel + expected_size, 0, 98af2ac3e1SAlexei Starovoitov actual_size - expected_size) == NULL; 99af2ac3e1SAlexei Starovoitov else 100af2ac3e1SAlexei Starovoitov res = check_zeroed_user(uaddr.user + expected_size, 101af2ac3e1SAlexei Starovoitov actual_size - expected_size); 102b7e4b65fSAl Viro if (res < 0) 103b7e4b65fSAl Viro return res; 104b7e4b65fSAl Viro return res ? 0 : -E2BIG; 10558291a74SMickaël Salaün } 10658291a74SMickaël Salaün 107a3884572SJakub Kicinski const struct bpf_map_ops bpf_map_offload_ops = { 108f4d05259SMartin KaFai Lau .map_meta_equal = bpf_map_meta_equal, 109a3884572SJakub Kicinski .map_alloc = bpf_map_offload_map_alloc, 110a3884572SJakub Kicinski .map_free = bpf_map_offload_map_free, 111e8d2bec0SDaniel Borkmann .map_check_btf = map_check_no_btf, 1129629363cSYafang Shao .map_mem_usage = bpf_map_offload_map_mem_usage, 113a3884572SJakub Kicinski }; 114a3884572SJakub Kicinski 115353050beSDaniel Borkmann static void bpf_map_write_active_inc(struct bpf_map *map) 116353050beSDaniel Borkmann { 117353050beSDaniel Borkmann atomic64_inc(&map->writecnt); 118353050beSDaniel Borkmann } 119353050beSDaniel Borkmann 120353050beSDaniel Borkmann static void bpf_map_write_active_dec(struct bpf_map *map) 121353050beSDaniel Borkmann { 122353050beSDaniel Borkmann atomic64_dec(&map->writecnt); 123353050beSDaniel Borkmann } 124353050beSDaniel Borkmann 125353050beSDaniel Borkmann bool bpf_map_write_active(const struct bpf_map *map) 126353050beSDaniel Borkmann { 127353050beSDaniel Borkmann return atomic64_read(&map->writecnt) != 0; 128353050beSDaniel Borkmann } 129353050beSDaniel Borkmann 13080ee81e0SRoman Gushchin static u32 bpf_map_value_size(const struct bpf_map *map) 13115c14a3dSBrian Vazquez { 13215c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 13315c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 13415c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 13515c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 13615c14a3dSBrian Vazquez return round_up(map->value_size, 8) * num_possible_cpus(); 13715c14a3dSBrian Vazquez else if (IS_FD_MAP(map)) 13815c14a3dSBrian Vazquez return sizeof(u32); 13915c14a3dSBrian Vazquez else 14015c14a3dSBrian Vazquez return map->value_size; 14115c14a3dSBrian Vazquez } 14215c14a3dSBrian Vazquez 14315c14a3dSBrian Vazquez static void maybe_wait_bpf_programs(struct bpf_map *map) 14415c14a3dSBrian Vazquez { 1452a0c6b41SHou Tao /* Wait for any running non-sleepable BPF programs to complete so that 1462a0c6b41SHou Tao * userspace, when we return to it, knows that all non-sleepable 1472a0c6b41SHou Tao * programs that could be running use the new map value. For sleepable 1482a0c6b41SHou Tao * BPF programs, synchronize_rcu_tasks_trace() should be used to wait 1492a0c6b41SHou Tao * for the completions of these programs, but considering the waiting 1502a0c6b41SHou Tao * time can be very long and userspace may think it will hang forever, 1512a0c6b41SHou Tao * so don't handle sleepable BPF programs now. 15215c14a3dSBrian Vazquez */ 15315c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS || 15415c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 15515c14a3dSBrian Vazquez synchronize_rcu(); 15615c14a3dSBrian Vazquez } 15715c14a3dSBrian Vazquez 1583af43ba4SHou Tao static int bpf_map_update_value(struct bpf_map *map, struct file *map_file, 1593af43ba4SHou Tao void *key, void *value, __u64 flags) 16015c14a3dSBrian Vazquez { 16115c14a3dSBrian Vazquez int err; 16215c14a3dSBrian Vazquez 16315c14a3dSBrian Vazquez /* Need to create a kthread, thus must support schedule */ 1649d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) { 16515c14a3dSBrian Vazquez return bpf_map_offload_update_elem(map, key, value, flags); 16615c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_CPUMAP || 16731746031SAlexei Starovoitov map->map_type == BPF_MAP_TYPE_ARENA || 16815c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 16915c14a3dSBrian Vazquez return map->ops->map_update_elem(map, key, value, flags); 17013b79d3fSLorenz Bauer } else if (map->map_type == BPF_MAP_TYPE_SOCKHASH || 17113b79d3fSLorenz Bauer map->map_type == BPF_MAP_TYPE_SOCKMAP) { 17213b79d3fSLorenz Bauer return sock_map_update_elem_sys(map, key, value, flags); 17315c14a3dSBrian Vazquez } else if (IS_FD_PROG_ARRAY(map)) { 1743af43ba4SHou Tao return bpf_fd_array_map_update_elem(map, map_file, key, value, 17515c14a3dSBrian Vazquez flags); 17615c14a3dSBrian Vazquez } 17715c14a3dSBrian Vazquez 178b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 17915c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 18015c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 18115c14a3dSBrian Vazquez err = bpf_percpu_hash_update(map, key, value, flags); 18215c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 18315c14a3dSBrian Vazquez err = bpf_percpu_array_update(map, key, value, flags); 18415c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 18515c14a3dSBrian Vazquez err = bpf_percpu_cgroup_storage_update(map, key, value, 18615c14a3dSBrian Vazquez flags); 18715c14a3dSBrian Vazquez } else if (IS_FD_ARRAY(map)) { 1883af43ba4SHou Tao err = bpf_fd_array_map_update_elem(map, map_file, key, value, 18915c14a3dSBrian Vazquez flags); 19015c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 1913af43ba4SHou Tao err = bpf_fd_htab_map_update_elem(map, map_file, key, value, 19215c14a3dSBrian Vazquez flags); 19315c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 19415c14a3dSBrian Vazquez /* rcu_read_lock() is not needed */ 19515c14a3dSBrian Vazquez err = bpf_fd_reuseport_array_update_elem(map, key, value, 19615c14a3dSBrian Vazquez flags); 19715c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 1989330986cSJoanne Koong map->map_type == BPF_MAP_TYPE_STACK || 1999330986cSJoanne Koong map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) { 20015c14a3dSBrian Vazquez err = map->ops->map_push_elem(map, value, flags); 20115c14a3dSBrian Vazquez } else { 20215c14a3dSBrian Vazquez rcu_read_lock(); 20315c14a3dSBrian Vazquez err = map->ops->map_update_elem(map, key, value, flags); 20415c14a3dSBrian Vazquez rcu_read_unlock(); 20515c14a3dSBrian Vazquez } 206b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 20715c14a3dSBrian Vazquez 20815c14a3dSBrian Vazquez return err; 20915c14a3dSBrian Vazquez } 21015c14a3dSBrian Vazquez 21115c14a3dSBrian Vazquez static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value, 21215c14a3dSBrian Vazquez __u64 flags) 21315c14a3dSBrian Vazquez { 21415c14a3dSBrian Vazquez void *ptr; 21515c14a3dSBrian Vazquez int err; 21615c14a3dSBrian Vazquez 2179d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) 218cb4d03abSBrian Vazquez return bpf_map_offload_lookup_elem(map, key, value); 21915c14a3dSBrian Vazquez 220b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 22115c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 22215c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 22315c14a3dSBrian Vazquez err = bpf_percpu_hash_copy(map, key, value); 22415c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 22515c14a3dSBrian Vazquez err = bpf_percpu_array_copy(map, key, value); 22615c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 22715c14a3dSBrian Vazquez err = bpf_percpu_cgroup_storage_copy(map, key, value); 22815c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 22915c14a3dSBrian Vazquez err = bpf_stackmap_copy(map, key, value); 23015c14a3dSBrian Vazquez } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) { 23115c14a3dSBrian Vazquez err = bpf_fd_array_map_lookup_elem(map, key, value); 23215c14a3dSBrian Vazquez } else if (IS_FD_HASH(map)) { 23315c14a3dSBrian Vazquez err = bpf_fd_htab_map_lookup_elem(map, key, value); 23415c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 23515c14a3dSBrian Vazquez err = bpf_fd_reuseport_array_lookup_elem(map, key, value); 23615c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 2379330986cSJoanne Koong map->map_type == BPF_MAP_TYPE_STACK || 2389330986cSJoanne Koong map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) { 23915c14a3dSBrian Vazquez err = map->ops->map_peek_elem(map, value); 24015c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 24115c14a3dSBrian Vazquez /* struct_ops map requires directly updating "value" */ 24215c14a3dSBrian Vazquez err = bpf_struct_ops_map_sys_lookup_elem(map, key, value); 24315c14a3dSBrian Vazquez } else { 24415c14a3dSBrian Vazquez rcu_read_lock(); 24515c14a3dSBrian Vazquez if (map->ops->map_lookup_elem_sys_only) 24615c14a3dSBrian Vazquez ptr = map->ops->map_lookup_elem_sys_only(map, key); 24715c14a3dSBrian Vazquez else 24815c14a3dSBrian Vazquez ptr = map->ops->map_lookup_elem(map, key); 24915c14a3dSBrian Vazquez if (IS_ERR(ptr)) { 25015c14a3dSBrian Vazquez err = PTR_ERR(ptr); 25115c14a3dSBrian Vazquez } else if (!ptr) { 25215c14a3dSBrian Vazquez err = -ENOENT; 25315c14a3dSBrian Vazquez } else { 25415c14a3dSBrian Vazquez err = 0; 25515c14a3dSBrian Vazquez if (flags & BPF_F_LOCK) 25615c14a3dSBrian Vazquez /* lock 'ptr' and copy everything but lock */ 25715c14a3dSBrian Vazquez copy_map_value_locked(map, value, ptr, true); 25815c14a3dSBrian Vazquez else 25915c14a3dSBrian Vazquez copy_map_value(map, value, ptr); 26068134668SAlexei Starovoitov /* mask lock and timer, since value wasn't zero inited */ 26168134668SAlexei Starovoitov check_and_init_map_value(map, value); 26215c14a3dSBrian Vazquez } 26315c14a3dSBrian Vazquez rcu_read_unlock(); 26415c14a3dSBrian Vazquez } 26515c14a3dSBrian Vazquez 266b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 26715c14a3dSBrian Vazquez 26815c14a3dSBrian Vazquez return err; 26915c14a3dSBrian Vazquez } 27015c14a3dSBrian Vazquez 271d5299b67SRoman Gushchin /* Please, do not use this function outside from the map creation path 272d5299b67SRoman Gushchin * (e.g. in map update path) without taking care of setting the active 273d5299b67SRoman Gushchin * memory cgroup (see at bpf_map_kmalloc_node() for example). 274d5299b67SRoman Gushchin */ 275196e8ca7SDaniel Borkmann static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable) 276d407bd25SDaniel Borkmann { 277f01a7dbeSMartynas Pumputis /* We really just want to fail instead of triggering OOM killer 278f01a7dbeSMartynas Pumputis * under memory pressure, therefore we set __GFP_NORETRY to kmalloc, 279f01a7dbeSMartynas Pumputis * which is used for lower order allocation requests. 280f01a7dbeSMartynas Pumputis * 281f01a7dbeSMartynas Pumputis * It has been observed that higher order allocation requests done by 282f01a7dbeSMartynas Pumputis * vmalloc with __GFP_NORETRY being set might fail due to not trying 283f01a7dbeSMartynas Pumputis * to reclaim memory from the page cache, thus we set 284f01a7dbeSMartynas Pumputis * __GFP_RETRY_MAYFAIL to avoid such situations. 285d407bd25SDaniel Borkmann */ 286f01a7dbeSMartynas Pumputis 287ee53cbfbSYafang Shao gfp_t gfp = bpf_memcg_flags(__GFP_NOWARN | __GFP_ZERO); 288041de93fSChristoph Hellwig unsigned int flags = 0; 289041de93fSChristoph Hellwig unsigned long align = 1; 290d407bd25SDaniel Borkmann void *area; 291d407bd25SDaniel Borkmann 292196e8ca7SDaniel Borkmann if (size >= SIZE_MAX) 293196e8ca7SDaniel Borkmann return NULL; 294196e8ca7SDaniel Borkmann 295fc970227SAndrii Nakryiko /* kmalloc()'ed memory can't be mmap()'ed */ 296041de93fSChristoph Hellwig if (mmapable) { 297041de93fSChristoph Hellwig BUG_ON(!PAGE_ALIGNED(size)); 298041de93fSChristoph Hellwig align = SHMLBA; 299041de93fSChristoph Hellwig flags = VM_USERMAP; 300041de93fSChristoph Hellwig } else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 301041de93fSChristoph Hellwig area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY, 302f01a7dbeSMartynas Pumputis numa_node); 303d407bd25SDaniel Borkmann if (area != NULL) 304d407bd25SDaniel Borkmann return area; 305d407bd25SDaniel Borkmann } 306041de93fSChristoph Hellwig 307041de93fSChristoph Hellwig return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, 308041de93fSChristoph Hellwig gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL, 309041de93fSChristoph Hellwig flags, numa_node, __builtin_return_address(0)); 310d407bd25SDaniel Borkmann } 311d407bd25SDaniel Borkmann 312196e8ca7SDaniel Borkmann void *bpf_map_area_alloc(u64 size, int numa_node) 313fc970227SAndrii Nakryiko { 314fc970227SAndrii Nakryiko return __bpf_map_area_alloc(size, numa_node, false); 315fc970227SAndrii Nakryiko } 316fc970227SAndrii Nakryiko 317196e8ca7SDaniel Borkmann void *bpf_map_area_mmapable_alloc(u64 size, int numa_node) 318fc970227SAndrii Nakryiko { 319fc970227SAndrii Nakryiko return __bpf_map_area_alloc(size, numa_node, true); 320fc970227SAndrii Nakryiko } 321fc970227SAndrii Nakryiko 322d407bd25SDaniel Borkmann void bpf_map_area_free(void *area) 323d407bd25SDaniel Borkmann { 324d407bd25SDaniel Borkmann kvfree(area); 325d407bd25SDaniel Borkmann } 326d407bd25SDaniel Borkmann 327be70bcd5SDaniel Borkmann static u32 bpf_map_flags_retain_permanent(u32 flags) 328be70bcd5SDaniel Borkmann { 329be70bcd5SDaniel Borkmann /* Some map creation flags are not tied to the map object but 330be70bcd5SDaniel Borkmann * rather to the map fd instead, so they have no meaning upon 331be70bcd5SDaniel Borkmann * map object inspection since multiple file descriptors with 332be70bcd5SDaniel Borkmann * different (access) properties can exist here. Thus, given 333be70bcd5SDaniel Borkmann * this has zero meaning for the map itself, lets clear these 334be70bcd5SDaniel Borkmann * from here. 335be70bcd5SDaniel Borkmann */ 336be70bcd5SDaniel Borkmann return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY); 337be70bcd5SDaniel Borkmann } 338be70bcd5SDaniel Borkmann 339bd475643SJakub Kicinski void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) 340bd475643SJakub Kicinski { 341bd475643SJakub Kicinski map->map_type = attr->map_type; 342bd475643SJakub Kicinski map->key_size = attr->key_size; 343bd475643SJakub Kicinski map->value_size = attr->value_size; 344bd475643SJakub Kicinski map->max_entries = attr->max_entries; 345be70bcd5SDaniel Borkmann map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags); 346bd475643SJakub Kicinski map->numa_node = bpf_map_attr_numa_node(attr); 3479330986cSJoanne Koong map->map_extra = attr->map_extra; 348bd475643SJakub Kicinski } 349bd475643SJakub Kicinski 350f3f1c054SMartin KaFai Lau static int bpf_map_alloc_id(struct bpf_map *map) 351f3f1c054SMartin KaFai Lau { 352f3f1c054SMartin KaFai Lau int id; 353f3f1c054SMartin KaFai Lau 354b76354cdSShaohua Li idr_preload(GFP_KERNEL); 355f3f1c054SMartin KaFai Lau spin_lock_bh(&map_idr_lock); 356f3f1c054SMartin KaFai Lau id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 357f3f1c054SMartin KaFai Lau if (id > 0) 358f3f1c054SMartin KaFai Lau map->id = id; 359f3f1c054SMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 360b76354cdSShaohua Li idr_preload_end(); 361f3f1c054SMartin KaFai Lau 362f3f1c054SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 363f3f1c054SMartin KaFai Lau return -ENOSPC; 364f3f1c054SMartin KaFai Lau 365f3f1c054SMartin KaFai Lau return id > 0 ? 0 : id; 366f3f1c054SMartin KaFai Lau } 367f3f1c054SMartin KaFai Lau 368158e5e9eSTobias Klauser void bpf_map_free_id(struct bpf_map *map) 369f3f1c054SMartin KaFai Lau { 370930651a7SEric Dumazet unsigned long flags; 371930651a7SEric Dumazet 372a3884572SJakub Kicinski /* Offloaded maps are removed from the IDR store when their device 373a3884572SJakub Kicinski * disappears - even if someone holds an fd to them they are unusable, 374a3884572SJakub Kicinski * the memory is gone, all ops will fail; they are simply waiting for 375a3884572SJakub Kicinski * refcnt to drop to be freed. 376a3884572SJakub Kicinski */ 377a3884572SJakub Kicinski if (!map->id) 378a3884572SJakub Kicinski return; 379a3884572SJakub Kicinski 380930651a7SEric Dumazet spin_lock_irqsave(&map_idr_lock, flags); 381bd5f5f4eSMartin KaFai Lau 382f3f1c054SMartin KaFai Lau idr_remove(&map_idr, map->id); 383a3884572SJakub Kicinski map->id = 0; 384bd5f5f4eSMartin KaFai Lau 385930651a7SEric Dumazet spin_unlock_irqrestore(&map_idr_lock, flags); 386f3f1c054SMartin KaFai Lau } 387f3f1c054SMartin KaFai Lau 38848edc1f7SRoman Gushchin #ifdef CONFIG_MEMCG_KMEM 38948edc1f7SRoman Gushchin static void bpf_map_save_memcg(struct bpf_map *map) 39048edc1f7SRoman Gushchin { 3914201d9abSRoman Gushchin /* Currently if a map is created by a process belonging to the root 3924201d9abSRoman Gushchin * memory cgroup, get_obj_cgroup_from_current() will return NULL. 3934201d9abSRoman Gushchin * So we have to check map->objcg for being NULL each time it's 3944201d9abSRoman Gushchin * being used. 3954201d9abSRoman Gushchin */ 396ee53cbfbSYafang Shao if (memcg_bpf_enabled()) 3974201d9abSRoman Gushchin map->objcg = get_obj_cgroup_from_current(); 39848edc1f7SRoman Gushchin } 39948edc1f7SRoman Gushchin 40048edc1f7SRoman Gushchin static void bpf_map_release_memcg(struct bpf_map *map) 40148edc1f7SRoman Gushchin { 4024201d9abSRoman Gushchin if (map->objcg) 4034201d9abSRoman Gushchin obj_cgroup_put(map->objcg); 4044201d9abSRoman Gushchin } 4054201d9abSRoman Gushchin 4064201d9abSRoman Gushchin static struct mem_cgroup *bpf_map_get_memcg(const struct bpf_map *map) 4074201d9abSRoman Gushchin { 4084201d9abSRoman Gushchin if (map->objcg) 4094201d9abSRoman Gushchin return get_mem_cgroup_from_objcg(map->objcg); 4104201d9abSRoman Gushchin 4114201d9abSRoman Gushchin return root_mem_cgroup; 41248edc1f7SRoman Gushchin } 41348edc1f7SRoman Gushchin 41448edc1f7SRoman Gushchin void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, 41548edc1f7SRoman Gushchin int node) 41648edc1f7SRoman Gushchin { 4174201d9abSRoman Gushchin struct mem_cgroup *memcg, *old_memcg; 41848edc1f7SRoman Gushchin void *ptr; 41948edc1f7SRoman Gushchin 4204201d9abSRoman Gushchin memcg = bpf_map_get_memcg(map); 4214201d9abSRoman Gushchin old_memcg = set_active_memcg(memcg); 42248edc1f7SRoman Gushchin ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node); 42348edc1f7SRoman Gushchin set_active_memcg(old_memcg); 4244201d9abSRoman Gushchin mem_cgroup_put(memcg); 42548edc1f7SRoman Gushchin 42648edc1f7SRoman Gushchin return ptr; 42748edc1f7SRoman Gushchin } 42848edc1f7SRoman Gushchin 42948edc1f7SRoman Gushchin void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags) 43048edc1f7SRoman Gushchin { 4314201d9abSRoman Gushchin struct mem_cgroup *memcg, *old_memcg; 43248edc1f7SRoman Gushchin void *ptr; 43348edc1f7SRoman Gushchin 4344201d9abSRoman Gushchin memcg = bpf_map_get_memcg(map); 4354201d9abSRoman Gushchin old_memcg = set_active_memcg(memcg); 43648edc1f7SRoman Gushchin ptr = kzalloc(size, flags | __GFP_ACCOUNT); 43748edc1f7SRoman Gushchin set_active_memcg(old_memcg); 4384201d9abSRoman Gushchin mem_cgroup_put(memcg); 43948edc1f7SRoman Gushchin 44048edc1f7SRoman Gushchin return ptr; 44148edc1f7SRoman Gushchin } 44248edc1f7SRoman Gushchin 443ddef81b5SYafang Shao void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size, 444ddef81b5SYafang Shao gfp_t flags) 445ddef81b5SYafang Shao { 446ddef81b5SYafang Shao struct mem_cgroup *memcg, *old_memcg; 447ddef81b5SYafang Shao void *ptr; 448ddef81b5SYafang Shao 449ddef81b5SYafang Shao memcg = bpf_map_get_memcg(map); 450ddef81b5SYafang Shao old_memcg = set_active_memcg(memcg); 451ddef81b5SYafang Shao ptr = kvcalloc(n, size, flags | __GFP_ACCOUNT); 452ddef81b5SYafang Shao set_active_memcg(old_memcg); 453ddef81b5SYafang Shao mem_cgroup_put(memcg); 454ddef81b5SYafang Shao 455ddef81b5SYafang Shao return ptr; 456ddef81b5SYafang Shao } 457ddef81b5SYafang Shao 45848edc1f7SRoman Gushchin void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, 45948edc1f7SRoman Gushchin size_t align, gfp_t flags) 46048edc1f7SRoman Gushchin { 4614201d9abSRoman Gushchin struct mem_cgroup *memcg, *old_memcg; 46248edc1f7SRoman Gushchin void __percpu *ptr; 46348edc1f7SRoman Gushchin 4644201d9abSRoman Gushchin memcg = bpf_map_get_memcg(map); 4654201d9abSRoman Gushchin old_memcg = set_active_memcg(memcg); 46648edc1f7SRoman Gushchin ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT); 46748edc1f7SRoman Gushchin set_active_memcg(old_memcg); 4684201d9abSRoman Gushchin mem_cgroup_put(memcg); 46948edc1f7SRoman Gushchin 47048edc1f7SRoman Gushchin return ptr; 47148edc1f7SRoman Gushchin } 47248edc1f7SRoman Gushchin 47348edc1f7SRoman Gushchin #else 47448edc1f7SRoman Gushchin static void bpf_map_save_memcg(struct bpf_map *map) 47548edc1f7SRoman Gushchin { 47648edc1f7SRoman Gushchin } 47748edc1f7SRoman Gushchin 47848edc1f7SRoman Gushchin static void bpf_map_release_memcg(struct bpf_map *map) 47948edc1f7SRoman Gushchin { 48048edc1f7SRoman Gushchin } 48148edc1f7SRoman Gushchin #endif 48248edc1f7SRoman Gushchin 48331746031SAlexei Starovoitov int bpf_map_alloc_pages(const struct bpf_map *map, gfp_t gfp, int nid, 48431746031SAlexei Starovoitov unsigned long nr_pages, struct page **pages) 48531746031SAlexei Starovoitov { 48631746031SAlexei Starovoitov unsigned long i, j; 48731746031SAlexei Starovoitov struct page *pg; 48831746031SAlexei Starovoitov int ret = 0; 48931746031SAlexei Starovoitov #ifdef CONFIG_MEMCG_KMEM 49031746031SAlexei Starovoitov struct mem_cgroup *memcg, *old_memcg; 49131746031SAlexei Starovoitov 49231746031SAlexei Starovoitov memcg = bpf_map_get_memcg(map); 49331746031SAlexei Starovoitov old_memcg = set_active_memcg(memcg); 49431746031SAlexei Starovoitov #endif 49531746031SAlexei Starovoitov for (i = 0; i < nr_pages; i++) { 49631746031SAlexei Starovoitov pg = alloc_pages_node(nid, gfp | __GFP_ACCOUNT, 0); 49731746031SAlexei Starovoitov 49831746031SAlexei Starovoitov if (pg) { 49931746031SAlexei Starovoitov pages[i] = pg; 50031746031SAlexei Starovoitov continue; 50131746031SAlexei Starovoitov } 50231746031SAlexei Starovoitov for (j = 0; j < i; j++) 50331746031SAlexei Starovoitov __free_page(pages[j]); 50431746031SAlexei Starovoitov ret = -ENOMEM; 50531746031SAlexei Starovoitov break; 50631746031SAlexei Starovoitov } 50731746031SAlexei Starovoitov 50831746031SAlexei Starovoitov #ifdef CONFIG_MEMCG_KMEM 50931746031SAlexei Starovoitov set_active_memcg(old_memcg); 51031746031SAlexei Starovoitov mem_cgroup_put(memcg); 51131746031SAlexei Starovoitov #endif 51231746031SAlexei Starovoitov return ret; 51331746031SAlexei Starovoitov } 51431746031SAlexei Starovoitov 51531746031SAlexei Starovoitov 516aa3496acSKumar Kartikeya Dwivedi static int btf_field_cmp(const void *a, const void *b) 51761df10c7SKumar Kartikeya Dwivedi { 518aa3496acSKumar Kartikeya Dwivedi const struct btf_field *f1 = a, *f2 = b; 51961df10c7SKumar Kartikeya Dwivedi 520aa3496acSKumar Kartikeya Dwivedi if (f1->offset < f2->offset) 52161df10c7SKumar Kartikeya Dwivedi return -1; 522aa3496acSKumar Kartikeya Dwivedi else if (f1->offset > f2->offset) 52361df10c7SKumar Kartikeya Dwivedi return 1; 52461df10c7SKumar Kartikeya Dwivedi return 0; 52561df10c7SKumar Kartikeya Dwivedi } 52661df10c7SKumar Kartikeya Dwivedi 527aa3496acSKumar Kartikeya Dwivedi struct btf_field *btf_record_find(const struct btf_record *rec, u32 offset, 52874843b57SDave Marchevsky u32 field_mask) 52961df10c7SKumar Kartikeya Dwivedi { 530aa3496acSKumar Kartikeya Dwivedi struct btf_field *field; 53161df10c7SKumar Kartikeya Dwivedi 53274843b57SDave Marchevsky if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & field_mask)) 53361df10c7SKumar Kartikeya Dwivedi return NULL; 534aa3496acSKumar Kartikeya Dwivedi field = bsearch(&offset, rec->fields, rec->cnt, sizeof(rec->fields[0]), btf_field_cmp); 53574843b57SDave Marchevsky if (!field || !(field->type & field_mask)) 536aa3496acSKumar Kartikeya Dwivedi return NULL; 537aa3496acSKumar Kartikeya Dwivedi return field; 53861df10c7SKumar Kartikeya Dwivedi } 53961df10c7SKumar Kartikeya Dwivedi 540aa3496acSKumar Kartikeya Dwivedi void btf_record_free(struct btf_record *rec) 54161df10c7SKumar Kartikeya Dwivedi { 54261df10c7SKumar Kartikeya Dwivedi int i; 54361df10c7SKumar Kartikeya Dwivedi 544aa3496acSKumar Kartikeya Dwivedi if (IS_ERR_OR_NULL(rec)) 54561df10c7SKumar Kartikeya Dwivedi return; 546aa3496acSKumar Kartikeya Dwivedi for (i = 0; i < rec->cnt; i++) { 547aa3496acSKumar Kartikeya Dwivedi switch (rec->fields[i].type) { 548aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_UNREF: 549aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_REF: 55055db92f4SYonghong Song case BPF_KPTR_PERCPU: 551aa3496acSKumar Kartikeya Dwivedi if (rec->fields[i].kptr.module) 552aa3496acSKumar Kartikeya Dwivedi module_put(rec->fields[i].kptr.module); 553aa3496acSKumar Kartikeya Dwivedi btf_put(rec->fields[i].kptr.btf); 554aa3496acSKumar Kartikeya Dwivedi break; 555f0c5941fSKumar Kartikeya Dwivedi case BPF_LIST_HEAD: 5568ffa5cc1SKumar Kartikeya Dwivedi case BPF_LIST_NODE: 5579c395c1bSDave Marchevsky case BPF_RB_ROOT: 5589c395c1bSDave Marchevsky case BPF_RB_NODE: 5599c395c1bSDave Marchevsky case BPF_SPIN_LOCK: 5609c395c1bSDave Marchevsky case BPF_TIMER: 561d54730b5SDave Marchevsky case BPF_REFCOUNT: 5629c395c1bSDave Marchevsky /* Nothing to release */ 563f0c5941fSKumar Kartikeya Dwivedi break; 564aa3496acSKumar Kartikeya Dwivedi default: 565aa3496acSKumar Kartikeya Dwivedi WARN_ON_ONCE(1); 56614a324f6SKumar Kartikeya Dwivedi continue; 56714a324f6SKumar Kartikeya Dwivedi } 568aa3496acSKumar Kartikeya Dwivedi } 569aa3496acSKumar Kartikeya Dwivedi kfree(rec); 570aa3496acSKumar Kartikeya Dwivedi } 571aa3496acSKumar Kartikeya Dwivedi 572aa3496acSKumar Kartikeya Dwivedi void bpf_map_free_record(struct bpf_map *map) 573aa3496acSKumar Kartikeya Dwivedi { 574aa3496acSKumar Kartikeya Dwivedi btf_record_free(map->record); 575aa3496acSKumar Kartikeya Dwivedi map->record = NULL; 576aa3496acSKumar Kartikeya Dwivedi } 577aa3496acSKumar Kartikeya Dwivedi 578aa3496acSKumar Kartikeya Dwivedi struct btf_record *btf_record_dup(const struct btf_record *rec) 579aa3496acSKumar Kartikeya Dwivedi { 580aa3496acSKumar Kartikeya Dwivedi const struct btf_field *fields; 581aa3496acSKumar Kartikeya Dwivedi struct btf_record *new_rec; 582aa3496acSKumar Kartikeya Dwivedi int ret, size, i; 583aa3496acSKumar Kartikeya Dwivedi 584aa3496acSKumar Kartikeya Dwivedi if (IS_ERR_OR_NULL(rec)) 585aa3496acSKumar Kartikeya Dwivedi return NULL; 586aa3496acSKumar Kartikeya Dwivedi size = offsetof(struct btf_record, fields[rec->cnt]); 587aa3496acSKumar Kartikeya Dwivedi new_rec = kmemdup(rec, size, GFP_KERNEL | __GFP_NOWARN); 588aa3496acSKumar Kartikeya Dwivedi if (!new_rec) 589aa3496acSKumar Kartikeya Dwivedi return ERR_PTR(-ENOMEM); 590aa3496acSKumar Kartikeya Dwivedi /* Do a deep copy of the btf_record */ 591aa3496acSKumar Kartikeya Dwivedi fields = rec->fields; 592aa3496acSKumar Kartikeya Dwivedi new_rec->cnt = 0; 593aa3496acSKumar Kartikeya Dwivedi for (i = 0; i < rec->cnt; i++) { 594aa3496acSKumar Kartikeya Dwivedi switch (fields[i].type) { 595aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_UNREF: 596aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_REF: 59755db92f4SYonghong Song case BPF_KPTR_PERCPU: 598aa3496acSKumar Kartikeya Dwivedi btf_get(fields[i].kptr.btf); 599aa3496acSKumar Kartikeya Dwivedi if (fields[i].kptr.module && !try_module_get(fields[i].kptr.module)) { 600aa3496acSKumar Kartikeya Dwivedi ret = -ENXIO; 601aa3496acSKumar Kartikeya Dwivedi goto free; 602aa3496acSKumar Kartikeya Dwivedi } 603aa3496acSKumar Kartikeya Dwivedi break; 604f0c5941fSKumar Kartikeya Dwivedi case BPF_LIST_HEAD: 6058ffa5cc1SKumar Kartikeya Dwivedi case BPF_LIST_NODE: 6069c395c1bSDave Marchevsky case BPF_RB_ROOT: 6079c395c1bSDave Marchevsky case BPF_RB_NODE: 6089c395c1bSDave Marchevsky case BPF_SPIN_LOCK: 6099c395c1bSDave Marchevsky case BPF_TIMER: 610d54730b5SDave Marchevsky case BPF_REFCOUNT: 6119c395c1bSDave Marchevsky /* Nothing to acquire */ 612f0c5941fSKumar Kartikeya Dwivedi break; 613aa3496acSKumar Kartikeya Dwivedi default: 614aa3496acSKumar Kartikeya Dwivedi ret = -EFAULT; 615aa3496acSKumar Kartikeya Dwivedi WARN_ON_ONCE(1); 616aa3496acSKumar Kartikeya Dwivedi goto free; 617aa3496acSKumar Kartikeya Dwivedi } 618aa3496acSKumar Kartikeya Dwivedi new_rec->cnt++; 619aa3496acSKumar Kartikeya Dwivedi } 620aa3496acSKumar Kartikeya Dwivedi return new_rec; 621aa3496acSKumar Kartikeya Dwivedi free: 622aa3496acSKumar Kartikeya Dwivedi btf_record_free(new_rec); 623aa3496acSKumar Kartikeya Dwivedi return ERR_PTR(ret); 624aa3496acSKumar Kartikeya Dwivedi } 625aa3496acSKumar Kartikeya Dwivedi 626aa3496acSKumar Kartikeya Dwivedi bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b) 627aa3496acSKumar Kartikeya Dwivedi { 628aa3496acSKumar Kartikeya Dwivedi bool a_has_fields = !IS_ERR_OR_NULL(rec_a), b_has_fields = !IS_ERR_OR_NULL(rec_b); 629aa3496acSKumar Kartikeya Dwivedi int size; 630aa3496acSKumar Kartikeya Dwivedi 631aa3496acSKumar Kartikeya Dwivedi if (!a_has_fields && !b_has_fields) 632aa3496acSKumar Kartikeya Dwivedi return true; 633aa3496acSKumar Kartikeya Dwivedi if (a_has_fields != b_has_fields) 634aa3496acSKumar Kartikeya Dwivedi return false; 635aa3496acSKumar Kartikeya Dwivedi if (rec_a->cnt != rec_b->cnt) 636aa3496acSKumar Kartikeya Dwivedi return false; 637aa3496acSKumar Kartikeya Dwivedi size = offsetof(struct btf_record, fields[rec_a->cnt]); 638c22dfdd2SKumar Kartikeya Dwivedi /* btf_parse_fields uses kzalloc to allocate a btf_record, so unused 639c22dfdd2SKumar Kartikeya Dwivedi * members are zeroed out. So memcmp is safe to do without worrying 640c22dfdd2SKumar Kartikeya Dwivedi * about padding/unused fields. 641c22dfdd2SKumar Kartikeya Dwivedi * 642c22dfdd2SKumar Kartikeya Dwivedi * While spin_lock, timer, and kptr have no relation to map BTF, 643c22dfdd2SKumar Kartikeya Dwivedi * list_head metadata is specific to map BTF, the btf and value_rec 644c22dfdd2SKumar Kartikeya Dwivedi * members in particular. btf is the map BTF, while value_rec points to 645c22dfdd2SKumar Kartikeya Dwivedi * btf_record in that map BTF. 646c22dfdd2SKumar Kartikeya Dwivedi * 647c22dfdd2SKumar Kartikeya Dwivedi * So while by default, we don't rely on the map BTF (which the records 648c22dfdd2SKumar Kartikeya Dwivedi * were parsed from) matching for both records, which is not backwards 649c22dfdd2SKumar Kartikeya Dwivedi * compatible, in case list_head is part of it, we implicitly rely on 650c22dfdd2SKumar Kartikeya Dwivedi * that by way of depending on memcmp succeeding for it. 651c22dfdd2SKumar Kartikeya Dwivedi */ 652aa3496acSKumar Kartikeya Dwivedi return !memcmp(rec_a, rec_b, size); 653aa3496acSKumar Kartikeya Dwivedi } 654aa3496acSKumar Kartikeya Dwivedi 655db559117SKumar Kartikeya Dwivedi void bpf_obj_free_timer(const struct btf_record *rec, void *obj) 656db559117SKumar Kartikeya Dwivedi { 657db559117SKumar Kartikeya Dwivedi if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_TIMER))) 658db559117SKumar Kartikeya Dwivedi return; 659db559117SKumar Kartikeya Dwivedi bpf_timer_cancel_and_free(obj + rec->timer_off); 660db559117SKumar Kartikeya Dwivedi } 661db559117SKumar Kartikeya Dwivedi 662aa3496acSKumar Kartikeya Dwivedi void bpf_obj_free_fields(const struct btf_record *rec, void *obj) 663aa3496acSKumar Kartikeya Dwivedi { 664aa3496acSKumar Kartikeya Dwivedi const struct btf_field *fields; 665aa3496acSKumar Kartikeya Dwivedi int i; 666aa3496acSKumar Kartikeya Dwivedi 667aa3496acSKumar Kartikeya Dwivedi if (IS_ERR_OR_NULL(rec)) 668aa3496acSKumar Kartikeya Dwivedi return; 669aa3496acSKumar Kartikeya Dwivedi fields = rec->fields; 670aa3496acSKumar Kartikeya Dwivedi for (i = 0; i < rec->cnt; i++) { 671c8e18754SDave Marchevsky struct btf_struct_meta *pointee_struct_meta; 672aa3496acSKumar Kartikeya Dwivedi const struct btf_field *field = &fields[i]; 673aa3496acSKumar Kartikeya Dwivedi void *field_ptr = obj + field->offset; 674c8e18754SDave Marchevsky void *xchgd_field; 675aa3496acSKumar Kartikeya Dwivedi 676aa3496acSKumar Kartikeya Dwivedi switch (fields[i].type) { 677db559117SKumar Kartikeya Dwivedi case BPF_SPIN_LOCK: 678db559117SKumar Kartikeya Dwivedi break; 679db559117SKumar Kartikeya Dwivedi case BPF_TIMER: 680db559117SKumar Kartikeya Dwivedi bpf_timer_cancel_and_free(field_ptr); 681db559117SKumar Kartikeya Dwivedi break; 682aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_UNREF: 683aa3496acSKumar Kartikeya Dwivedi WRITE_ONCE(*(u64 *)field_ptr, 0); 684aa3496acSKumar Kartikeya Dwivedi break; 685aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_REF: 68655db92f4SYonghong Song case BPF_KPTR_PERCPU: 687c8e18754SDave Marchevsky xchgd_field = (void *)xchg((unsigned long *)field_ptr, 0); 6881431d0b5SDavid Vernet if (!xchgd_field) 6891431d0b5SDavid Vernet break; 6901431d0b5SDavid Vernet 691c8e18754SDave Marchevsky if (!btf_is_kernel(field->kptr.btf)) { 692c8e18754SDave Marchevsky pointee_struct_meta = btf_find_struct_meta(field->kptr.btf, 693c8e18754SDave Marchevsky field->kptr.btf_id); 6949e36a204SDave Marchevsky migrate_disable(); 6959e36a204SDave Marchevsky __bpf_obj_drop_impl(xchgd_field, pointee_struct_meta ? 696e383a459SHou Tao pointee_struct_meta->record : NULL, 697e383a459SHou Tao fields[i].type == BPF_KPTR_PERCPU); 6989e36a204SDave Marchevsky migrate_enable(); 699c8e18754SDave Marchevsky } else { 700c8e18754SDave Marchevsky field->kptr.dtor(xchgd_field); 701c8e18754SDave Marchevsky } 702aa3496acSKumar Kartikeya Dwivedi break; 703f0c5941fSKumar Kartikeya Dwivedi case BPF_LIST_HEAD: 704f0c5941fSKumar Kartikeya Dwivedi if (WARN_ON_ONCE(rec->spin_lock_off < 0)) 705f0c5941fSKumar Kartikeya Dwivedi continue; 706f0c5941fSKumar Kartikeya Dwivedi bpf_list_head_free(field, field_ptr, obj + rec->spin_lock_off); 707f0c5941fSKumar Kartikeya Dwivedi break; 7089c395c1bSDave Marchevsky case BPF_RB_ROOT: 7099c395c1bSDave Marchevsky if (WARN_ON_ONCE(rec->spin_lock_off < 0)) 7109c395c1bSDave Marchevsky continue; 7119c395c1bSDave Marchevsky bpf_rb_root_free(field, field_ptr, obj + rec->spin_lock_off); 7129c395c1bSDave Marchevsky break; 7138ffa5cc1SKumar Kartikeya Dwivedi case BPF_LIST_NODE: 7149c395c1bSDave Marchevsky case BPF_RB_NODE: 715d54730b5SDave Marchevsky case BPF_REFCOUNT: 7168ffa5cc1SKumar Kartikeya Dwivedi break; 717aa3496acSKumar Kartikeya Dwivedi default: 718aa3496acSKumar Kartikeya Dwivedi WARN_ON_ONCE(1); 719aa3496acSKumar Kartikeya Dwivedi continue; 720aa3496acSKumar Kartikeya Dwivedi } 72114a324f6SKumar Kartikeya Dwivedi } 72214a324f6SKumar Kartikeya Dwivedi } 72314a324f6SKumar Kartikeya Dwivedi 72499c55f7dSAlexei Starovoitov /* called from workqueue */ 72599c55f7dSAlexei Starovoitov static void bpf_map_free_deferred(struct work_struct *work) 72699c55f7dSAlexei Starovoitov { 72799c55f7dSAlexei Starovoitov struct bpf_map *map = container_of(work, struct bpf_map, work); 728d7f5ef65SKumar Kartikeya Dwivedi struct btf_record *rec = map->record; 72959e5791fSYonghong Song struct btf *btf = map->btf; 73099c55f7dSAlexei Starovoitov 731afdb09c7SChenbo Feng security_bpf_map_free(map); 73248edc1f7SRoman Gushchin bpf_map_release_memcg(map); 733d7f5ef65SKumar Kartikeya Dwivedi /* implementation dependent freeing */ 73499c55f7dSAlexei Starovoitov map->ops->map_free(map); 735cd2a8079SDave Marchevsky /* Delay freeing of btf_record for maps, as map_free 736d7f5ef65SKumar Kartikeya Dwivedi * callback usually needs access to them. It is better to do it here 737d7f5ef65SKumar Kartikeya Dwivedi * than require each callback to do the free itself manually. 738d7f5ef65SKumar Kartikeya Dwivedi * 739d7f5ef65SKumar Kartikeya Dwivedi * Note that the btf_record stashed in map->inner_map_meta->record was 740d7f5ef65SKumar Kartikeya Dwivedi * already freed using the map_free callback for map in map case which 741d7f5ef65SKumar Kartikeya Dwivedi * eventually calls bpf_map_free_meta, since inner_map_meta is only a 742d7f5ef65SKumar Kartikeya Dwivedi * template bpf_map struct used during verification. 743d7f5ef65SKumar Kartikeya Dwivedi */ 744d7f5ef65SKumar Kartikeya Dwivedi btf_record_free(rec); 74559e5791fSYonghong Song /* Delay freeing of btf for maps, as map_free callback may need 74659e5791fSYonghong Song * struct_meta info which will be freed with btf_put(). 74759e5791fSYonghong Song */ 74859e5791fSYonghong Song btf_put(btf); 74999c55f7dSAlexei Starovoitov } 75099c55f7dSAlexei Starovoitov 751c9da161cSDaniel Borkmann static void bpf_map_put_uref(struct bpf_map *map) 752c9da161cSDaniel Borkmann { 7531e0bd5a0SAndrii Nakryiko if (atomic64_dec_and_test(&map->usercnt)) { 754ba6b8de4SJohn Fastabend if (map->ops->map_release_uref) 755ba6b8de4SJohn Fastabend map->ops->map_release_uref(map); 756c9da161cSDaniel Borkmann } 757c9da161cSDaniel Borkmann } 758c9da161cSDaniel Borkmann 75987667336SHou Tao static void bpf_map_free_in_work(struct bpf_map *map) 76087667336SHou Tao { 76187667336SHou Tao INIT_WORK(&map->work, bpf_map_free_deferred); 76287667336SHou Tao /* Avoid spawning kworkers, since they all might contend 76387667336SHou Tao * for the same mutex like slab_mutex. 76487667336SHou Tao */ 76587667336SHou Tao queue_work(system_unbound_wq, &map->work); 76687667336SHou Tao } 76787667336SHou Tao 76887667336SHou Tao static void bpf_map_free_rcu_gp(struct rcu_head *rcu) 76987667336SHou Tao { 77087667336SHou Tao bpf_map_free_in_work(container_of(rcu, struct bpf_map, rcu)); 77187667336SHou Tao } 77287667336SHou Tao 77387667336SHou Tao static void bpf_map_free_mult_rcu_gp(struct rcu_head *rcu) 77487667336SHou Tao { 77587667336SHou Tao if (rcu_trace_implies_rcu_gp()) 77687667336SHou Tao bpf_map_free_rcu_gp(rcu); 77787667336SHou Tao else 77887667336SHou Tao call_rcu(rcu, bpf_map_free_rcu_gp); 77987667336SHou Tao } 78087667336SHou Tao 78199c55f7dSAlexei Starovoitov /* decrement map refcnt and schedule it for freeing via workqueue 782158e5e9eSTobias Klauser * (underlying map implementation ops->map_free() might sleep) 78399c55f7dSAlexei Starovoitov */ 784158e5e9eSTobias Klauser void bpf_map_put(struct bpf_map *map) 78599c55f7dSAlexei Starovoitov { 7861e0bd5a0SAndrii Nakryiko if (atomic64_dec_and_test(&map->refcnt)) { 78734ad5580SMartin KaFai Lau /* bpf_map_free_id() must be called first */ 788158e5e9eSTobias Klauser bpf_map_free_id(map); 78987667336SHou Tao 790af66bfd3SHou Tao WARN_ON_ONCE(atomic64_read(&map->sleepable_refcnt)); 79187667336SHou Tao if (READ_ONCE(map->free_after_mult_rcu_gp)) 79287667336SHou Tao call_rcu_tasks_trace(&map->rcu, bpf_map_free_mult_rcu_gp); 793af66bfd3SHou Tao else if (READ_ONCE(map->free_after_rcu_gp)) 794af66bfd3SHou Tao call_rcu(&map->rcu, bpf_map_free_rcu_gp); 79587667336SHou Tao else 79687667336SHou Tao bpf_map_free_in_work(map); 79799c55f7dSAlexei Starovoitov } 79899c55f7dSAlexei Starovoitov } 799630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_put); 800bd5f5f4eSMartin KaFai Lau 801c9da161cSDaniel Borkmann void bpf_map_put_with_uref(struct bpf_map *map) 802c9da161cSDaniel Borkmann { 803c9da161cSDaniel Borkmann bpf_map_put_uref(map); 804c9da161cSDaniel Borkmann bpf_map_put(map); 805c9da161cSDaniel Borkmann } 806c9da161cSDaniel Borkmann 80799c55f7dSAlexei Starovoitov static int bpf_map_release(struct inode *inode, struct file *filp) 80899c55f7dSAlexei Starovoitov { 80961d1b6a4SDaniel Borkmann struct bpf_map *map = filp->private_data; 81061d1b6a4SDaniel Borkmann 81161d1b6a4SDaniel Borkmann if (map->ops->map_release) 81261d1b6a4SDaniel Borkmann map->ops->map_release(map, filp); 81361d1b6a4SDaniel Borkmann 81461d1b6a4SDaniel Borkmann bpf_map_put_with_uref(map); 81599c55f7dSAlexei Starovoitov return 0; 81699c55f7dSAlexei Starovoitov } 81799c55f7dSAlexei Starovoitov 81887df15deSDaniel Borkmann static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f) 81987df15deSDaniel Borkmann { 82087df15deSDaniel Borkmann fmode_t mode = f.file->f_mode; 82187df15deSDaniel Borkmann 82287df15deSDaniel Borkmann /* Our file permissions may have been overridden by global 82387df15deSDaniel Borkmann * map permissions facing syscall side. 82487df15deSDaniel Borkmann */ 82587df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) 82687df15deSDaniel Borkmann mode &= ~FMODE_CAN_WRITE; 82787df15deSDaniel Borkmann return mode; 82887df15deSDaniel Borkmann } 82987df15deSDaniel Borkmann 830f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 83190a5527dSYafang Shao /* Show the memory usage of a bpf map */ 83290a5527dSYafang Shao static u64 bpf_map_memory_usage(const struct bpf_map *map) 83380ee81e0SRoman Gushchin { 83490a5527dSYafang Shao return map->ops->map_mem_usage(map); 83580ee81e0SRoman Gushchin } 83680ee81e0SRoman Gushchin 837f99bf205SDaniel Borkmann static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 838f99bf205SDaniel Borkmann { 839f45d5b6cSToke Hoiland-Jorgensen struct bpf_map *map = filp->private_data; 8402beee5f5SDaniel Borkmann u32 type = 0, jited = 0; 84121116b70SDaniel Borkmann 842f45d5b6cSToke Hoiland-Jorgensen if (map_type_contains_progs(map)) { 843f45d5b6cSToke Hoiland-Jorgensen spin_lock(&map->owner.lock); 844f45d5b6cSToke Hoiland-Jorgensen type = map->owner.type; 845f45d5b6cSToke Hoiland-Jorgensen jited = map->owner.jited; 846f45d5b6cSToke Hoiland-Jorgensen spin_unlock(&map->owner.lock); 84721116b70SDaniel Borkmann } 848f99bf205SDaniel Borkmann 849f99bf205SDaniel Borkmann seq_printf(m, 850f99bf205SDaniel Borkmann "map_type:\t%u\n" 851f99bf205SDaniel Borkmann "key_size:\t%u\n" 852f99bf205SDaniel Borkmann "value_size:\t%u\n" 853322cea2fSDaniel Borkmann "max_entries:\t%u\n" 85421116b70SDaniel Borkmann "map_flags:\t%#x\n" 8559330986cSJoanne Koong "map_extra:\t%#llx\n" 85690a5527dSYafang Shao "memlock:\t%llu\n" 85787df15deSDaniel Borkmann "map_id:\t%u\n" 85887df15deSDaniel Borkmann "frozen:\t%u\n", 859f99bf205SDaniel Borkmann map->map_type, 860f99bf205SDaniel Borkmann map->key_size, 861f99bf205SDaniel Borkmann map->value_size, 862322cea2fSDaniel Borkmann map->max_entries, 86321116b70SDaniel Borkmann map->map_flags, 8649330986cSJoanne Koong (unsigned long long)map->map_extra, 86590a5527dSYafang Shao bpf_map_memory_usage(map), 86687df15deSDaniel Borkmann map->id, 86787df15deSDaniel Borkmann READ_ONCE(map->frozen)); 8682beee5f5SDaniel Borkmann if (type) { 8692beee5f5SDaniel Borkmann seq_printf(m, "owner_prog_type:\t%u\n", type); 8702beee5f5SDaniel Borkmann seq_printf(m, "owner_jited:\t%u\n", jited); 8719780c0abSDaniel Borkmann } 872f99bf205SDaniel Borkmann } 873f99bf205SDaniel Borkmann #endif 874f99bf205SDaniel Borkmann 8756e71b04aSChenbo Feng static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, 8766e71b04aSChenbo Feng loff_t *ppos) 8776e71b04aSChenbo Feng { 8786e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 8796e71b04aSChenbo Feng * f_mode with FMODE_CAN_READ. 8806e71b04aSChenbo Feng */ 8816e71b04aSChenbo Feng return -EINVAL; 8826e71b04aSChenbo Feng } 8836e71b04aSChenbo Feng 8846e71b04aSChenbo Feng static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, 8856e71b04aSChenbo Feng size_t siz, loff_t *ppos) 8866e71b04aSChenbo Feng { 8876e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 8886e71b04aSChenbo Feng * f_mode with FMODE_CAN_WRITE. 8896e71b04aSChenbo Feng */ 8906e71b04aSChenbo Feng return -EINVAL; 8916e71b04aSChenbo Feng } 8926e71b04aSChenbo Feng 893fc970227SAndrii Nakryiko /* called for any extra memory-mapped regions (except initial) */ 894fc970227SAndrii Nakryiko static void bpf_map_mmap_open(struct vm_area_struct *vma) 895fc970227SAndrii Nakryiko { 896fc970227SAndrii Nakryiko struct bpf_map *map = vma->vm_file->private_data; 897fc970227SAndrii Nakryiko 898353050beSDaniel Borkmann if (vma->vm_flags & VM_MAYWRITE) 899353050beSDaniel Borkmann bpf_map_write_active_inc(map); 900fc970227SAndrii Nakryiko } 901fc970227SAndrii Nakryiko 902fc970227SAndrii Nakryiko /* called for all unmapped memory region (including initial) */ 903fc970227SAndrii Nakryiko static void bpf_map_mmap_close(struct vm_area_struct *vma) 904fc970227SAndrii Nakryiko { 905fc970227SAndrii Nakryiko struct bpf_map *map = vma->vm_file->private_data; 906fc970227SAndrii Nakryiko 907353050beSDaniel Borkmann if (vma->vm_flags & VM_MAYWRITE) 908353050beSDaniel Borkmann bpf_map_write_active_dec(map); 909fc970227SAndrii Nakryiko } 910fc970227SAndrii Nakryiko 911fc970227SAndrii Nakryiko static const struct vm_operations_struct bpf_map_default_vmops = { 912fc970227SAndrii Nakryiko .open = bpf_map_mmap_open, 913fc970227SAndrii Nakryiko .close = bpf_map_mmap_close, 914fc970227SAndrii Nakryiko }; 915fc970227SAndrii Nakryiko 916fc970227SAndrii Nakryiko static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma) 917fc970227SAndrii Nakryiko { 918fc970227SAndrii Nakryiko struct bpf_map *map = filp->private_data; 919fc970227SAndrii Nakryiko int err; 920fc970227SAndrii Nakryiko 921db559117SKumar Kartikeya Dwivedi if (!map->ops->map_mmap || !IS_ERR_OR_NULL(map->record)) 922fc970227SAndrii Nakryiko return -ENOTSUPP; 923fc970227SAndrii Nakryiko 924fc970227SAndrii Nakryiko if (!(vma->vm_flags & VM_SHARED)) 925fc970227SAndrii Nakryiko return -EINVAL; 926fc970227SAndrii Nakryiko 927fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 928fc970227SAndrii Nakryiko 929dfeb376dSAndrii Nakryiko if (vma->vm_flags & VM_WRITE) { 930dfeb376dSAndrii Nakryiko if (map->frozen) { 931fc970227SAndrii Nakryiko err = -EPERM; 932fc970227SAndrii Nakryiko goto out; 933fc970227SAndrii Nakryiko } 934dfeb376dSAndrii Nakryiko /* map is meant to be read-only, so do not allow mapping as 935dfeb376dSAndrii Nakryiko * writable, because it's possible to leak a writable page 936dfeb376dSAndrii Nakryiko * reference and allows user-space to still modify it after 937dfeb376dSAndrii Nakryiko * freezing, while verifier will assume contents do not change 938dfeb376dSAndrii Nakryiko */ 939dfeb376dSAndrii Nakryiko if (map->map_flags & BPF_F_RDONLY_PROG) { 940dfeb376dSAndrii Nakryiko err = -EACCES; 941dfeb376dSAndrii Nakryiko goto out; 942dfeb376dSAndrii Nakryiko } 943dfeb376dSAndrii Nakryiko } 944fc970227SAndrii Nakryiko 945fc970227SAndrii Nakryiko /* set default open/close callbacks */ 946fc970227SAndrii Nakryiko vma->vm_ops = &bpf_map_default_vmops; 947fc970227SAndrii Nakryiko vma->vm_private_data = map; 9481c71222eSSuren Baghdasaryan vm_flags_clear(vma, VM_MAYEXEC); 9491f6cb19bSAndrii Nakryiko if (!(vma->vm_flags & VM_WRITE)) 9501f6cb19bSAndrii Nakryiko /* disallow re-mapping with PROT_WRITE */ 9511c71222eSSuren Baghdasaryan vm_flags_clear(vma, VM_MAYWRITE); 952fc970227SAndrii Nakryiko 953fc970227SAndrii Nakryiko err = map->ops->map_mmap(map, vma); 954fc970227SAndrii Nakryiko if (err) 955fc970227SAndrii Nakryiko goto out; 956fc970227SAndrii Nakryiko 9571f6cb19bSAndrii Nakryiko if (vma->vm_flags & VM_MAYWRITE) 958353050beSDaniel Borkmann bpf_map_write_active_inc(map); 959fc970227SAndrii Nakryiko out: 960fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 961fc970227SAndrii Nakryiko return err; 962fc970227SAndrii Nakryiko } 963fc970227SAndrii Nakryiko 964457f4436SAndrii Nakryiko static __poll_t bpf_map_poll(struct file *filp, struct poll_table_struct *pts) 965457f4436SAndrii Nakryiko { 966457f4436SAndrii Nakryiko struct bpf_map *map = filp->private_data; 967457f4436SAndrii Nakryiko 968457f4436SAndrii Nakryiko if (map->ops->map_poll) 969457f4436SAndrii Nakryiko return map->ops->map_poll(map, filp, pts); 970457f4436SAndrii Nakryiko 971457f4436SAndrii Nakryiko return EPOLLERR; 972457f4436SAndrii Nakryiko } 973457f4436SAndrii Nakryiko 974cf2c2e4aSAlexei Starovoitov static unsigned long bpf_get_unmapped_area(struct file *filp, unsigned long addr, 975cf2c2e4aSAlexei Starovoitov unsigned long len, unsigned long pgoff, 976cf2c2e4aSAlexei Starovoitov unsigned long flags) 977cf2c2e4aSAlexei Starovoitov { 978cf2c2e4aSAlexei Starovoitov struct bpf_map *map = filp->private_data; 979cf2c2e4aSAlexei Starovoitov 980cf2c2e4aSAlexei Starovoitov if (map->ops->map_get_unmapped_area) 981cf2c2e4aSAlexei Starovoitov return map->ops->map_get_unmapped_area(filp, addr, len, pgoff, flags); 982cf2c2e4aSAlexei Starovoitov #ifdef CONFIG_MMU 983cf2c2e4aSAlexei Starovoitov return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); 984cf2c2e4aSAlexei Starovoitov #else 985cf2c2e4aSAlexei Starovoitov return addr; 986cf2c2e4aSAlexei Starovoitov #endif 987cf2c2e4aSAlexei Starovoitov } 988cf2c2e4aSAlexei Starovoitov 989f66e448cSChenbo Feng const struct file_operations bpf_map_fops = { 990f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 991f99bf205SDaniel Borkmann .show_fdinfo = bpf_map_show_fdinfo, 992f99bf205SDaniel Borkmann #endif 99399c55f7dSAlexei Starovoitov .release = bpf_map_release, 9946e71b04aSChenbo Feng .read = bpf_dummy_read, 9956e71b04aSChenbo Feng .write = bpf_dummy_write, 996fc970227SAndrii Nakryiko .mmap = bpf_map_mmap, 997457f4436SAndrii Nakryiko .poll = bpf_map_poll, 998cf2c2e4aSAlexei Starovoitov .get_unmapped_area = bpf_get_unmapped_area, 99999c55f7dSAlexei Starovoitov }; 100099c55f7dSAlexei Starovoitov 10016e71b04aSChenbo Feng int bpf_map_new_fd(struct bpf_map *map, int flags) 1002aa79781bSDaniel Borkmann { 1003afdb09c7SChenbo Feng int ret; 1004afdb09c7SChenbo Feng 1005afdb09c7SChenbo Feng ret = security_bpf_map(map, OPEN_FMODE(flags)); 1006afdb09c7SChenbo Feng if (ret < 0) 1007afdb09c7SChenbo Feng return ret; 1008afdb09c7SChenbo Feng 1009aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 10106e71b04aSChenbo Feng flags | O_CLOEXEC); 10116e71b04aSChenbo Feng } 10126e71b04aSChenbo Feng 10136e71b04aSChenbo Feng int bpf_get_file_flag(int flags) 10146e71b04aSChenbo Feng { 10156e71b04aSChenbo Feng if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) 10166e71b04aSChenbo Feng return -EINVAL; 10176e71b04aSChenbo Feng if (flags & BPF_F_RDONLY) 10186e71b04aSChenbo Feng return O_RDONLY; 10196e71b04aSChenbo Feng if (flags & BPF_F_WRONLY) 10206e71b04aSChenbo Feng return O_WRONLY; 10216e71b04aSChenbo Feng return O_RDWR; 1022aa79781bSDaniel Borkmann } 1023aa79781bSDaniel Borkmann 102499c55f7dSAlexei Starovoitov /* helper macro to check that unused fields 'union bpf_attr' are zero */ 102599c55f7dSAlexei Starovoitov #define CHECK_ATTR(CMD) \ 102699c55f7dSAlexei Starovoitov memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 102799c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD), 0, \ 102899c55f7dSAlexei Starovoitov sizeof(*attr) - \ 102999c55f7dSAlexei Starovoitov offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 103099c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD)) != NULL 103199c55f7dSAlexei Starovoitov 10328e7ae251SMartin KaFai Lau /* dst and src must have at least "size" number of bytes. 10338e7ae251SMartin KaFai Lau * Return strlen on success and < 0 on error. 1034cb4d2b3fSMartin KaFai Lau */ 10358e7ae251SMartin KaFai Lau int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size) 1036cb4d2b3fSMartin KaFai Lau { 10378e7ae251SMartin KaFai Lau const char *end = src + size; 10388e7ae251SMartin KaFai Lau const char *orig_src = src; 1039cb4d2b3fSMartin KaFai Lau 10408e7ae251SMartin KaFai Lau memset(dst, 0, size); 10413e0ddc4fSDaniel Borkmann /* Copy all isalnum(), '_' and '.' chars. */ 1042cb4d2b3fSMartin KaFai Lau while (src < end && *src) { 10433e0ddc4fSDaniel Borkmann if (!isalnum(*src) && 10443e0ddc4fSDaniel Borkmann *src != '_' && *src != '.') 1045cb4d2b3fSMartin KaFai Lau return -EINVAL; 1046cb4d2b3fSMartin KaFai Lau *dst++ = *src++; 1047cb4d2b3fSMartin KaFai Lau } 1048cb4d2b3fSMartin KaFai Lau 10498e7ae251SMartin KaFai Lau /* No '\0' found in "size" number of bytes */ 1050cb4d2b3fSMartin KaFai Lau if (src == end) 1051cb4d2b3fSMartin KaFai Lau return -EINVAL; 1052cb4d2b3fSMartin KaFai Lau 10538e7ae251SMartin KaFai Lau return src - orig_src; 1054cb4d2b3fSMartin KaFai Lau } 1055cb4d2b3fSMartin KaFai Lau 1056e8d2bec0SDaniel Borkmann int map_check_no_btf(const struct bpf_map *map, 10571b2b234bSRoman Gushchin const struct btf *btf, 1058e8d2bec0SDaniel Borkmann const struct btf_type *key_type, 1059e8d2bec0SDaniel Borkmann const struct btf_type *value_type) 1060e8d2bec0SDaniel Borkmann { 1061e8d2bec0SDaniel Borkmann return -ENOTSUPP; 1062e8d2bec0SDaniel Borkmann } 1063e8d2bec0SDaniel Borkmann 1064a177fc2bSAndrii Nakryiko static int map_check_btf(struct bpf_map *map, struct bpf_token *token, 1065a177fc2bSAndrii Nakryiko const struct btf *btf, u32 btf_key_id, u32 btf_value_id) 1066e8d2bec0SDaniel Borkmann { 1067e8d2bec0SDaniel Borkmann const struct btf_type *key_type, *value_type; 1068e8d2bec0SDaniel Borkmann u32 key_size, value_size; 1069e8d2bec0SDaniel Borkmann int ret = 0; 1070e8d2bec0SDaniel Borkmann 10712824ecb7SDaniel Borkmann /* Some maps allow key to be unspecified. */ 10722824ecb7SDaniel Borkmann if (btf_key_id) { 1073e8d2bec0SDaniel Borkmann key_type = btf_type_id_size(btf, &btf_key_id, &key_size); 1074e8d2bec0SDaniel Borkmann if (!key_type || key_size != map->key_size) 1075e8d2bec0SDaniel Borkmann return -EINVAL; 10762824ecb7SDaniel Borkmann } else { 10772824ecb7SDaniel Borkmann key_type = btf_type_by_id(btf, 0); 10782824ecb7SDaniel Borkmann if (!map->ops->map_check_btf) 10792824ecb7SDaniel Borkmann return -EINVAL; 10802824ecb7SDaniel Borkmann } 1081e8d2bec0SDaniel Borkmann 1082e8d2bec0SDaniel Borkmann value_type = btf_type_id_size(btf, &btf_value_id, &value_size); 1083e8d2bec0SDaniel Borkmann if (!value_type || value_size != map->value_size) 1084e8d2bec0SDaniel Borkmann return -EINVAL; 1085e8d2bec0SDaniel Borkmann 1086f0c5941fSKumar Kartikeya Dwivedi map->record = btf_parse_fields(btf, value_type, 10879c395c1bSDave Marchevsky BPF_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD | 1088d54730b5SDave Marchevsky BPF_RB_ROOT | BPF_REFCOUNT, 1089db559117SKumar Kartikeya Dwivedi map->value_size); 1090aa3496acSKumar Kartikeya Dwivedi if (!IS_ERR_OR_NULL(map->record)) { 1091aa3496acSKumar Kartikeya Dwivedi int i; 1092aa3496acSKumar Kartikeya Dwivedi 1093a177fc2bSAndrii Nakryiko if (!bpf_token_capable(token, CAP_BPF)) { 109461df10c7SKumar Kartikeya Dwivedi ret = -EPERM; 109561df10c7SKumar Kartikeya Dwivedi goto free_map_tab; 109661df10c7SKumar Kartikeya Dwivedi } 109761df10c7SKumar Kartikeya Dwivedi if (map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) { 109861df10c7SKumar Kartikeya Dwivedi ret = -EACCES; 109961df10c7SKumar Kartikeya Dwivedi goto free_map_tab; 110061df10c7SKumar Kartikeya Dwivedi } 1101aa3496acSKumar Kartikeya Dwivedi for (i = 0; i < sizeof(map->record->field_mask) * 8; i++) { 1102aa3496acSKumar Kartikeya Dwivedi switch (map->record->field_mask & (1 << i)) { 1103aa3496acSKumar Kartikeya Dwivedi case 0: 1104aa3496acSKumar Kartikeya Dwivedi continue; 1105db559117SKumar Kartikeya Dwivedi case BPF_SPIN_LOCK: 1106db559117SKumar Kartikeya Dwivedi if (map->map_type != BPF_MAP_TYPE_HASH && 1107db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_ARRAY && 1108db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 1109db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_SK_STORAGE && 1110db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_INODE_STORAGE && 1111db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_TASK_STORAGE && 1112db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) { 1113db559117SKumar Kartikeya Dwivedi ret = -EOPNOTSUPP; 1114db559117SKumar Kartikeya Dwivedi goto free_map_tab; 1115db559117SKumar Kartikeya Dwivedi } 1116db559117SKumar Kartikeya Dwivedi break; 1117db559117SKumar Kartikeya Dwivedi case BPF_TIMER: 1118db559117SKumar Kartikeya Dwivedi if (map->map_type != BPF_MAP_TYPE_HASH && 1119db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_LRU_HASH && 1120db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_ARRAY) { 1121c237bfa5SKumar Kartikeya Dwivedi ret = -EOPNOTSUPP; 1122db559117SKumar Kartikeya Dwivedi goto free_map_tab; 1123db559117SKumar Kartikeya Dwivedi } 1124db559117SKumar Kartikeya Dwivedi break; 1125aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_UNREF: 1126aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_REF: 112755db92f4SYonghong Song case BPF_KPTR_PERCPU: 1128d54730b5SDave Marchevsky case BPF_REFCOUNT: 112961df10c7SKumar Kartikeya Dwivedi if (map->map_type != BPF_MAP_TYPE_HASH && 113065334e64SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_PERCPU_HASH && 113161df10c7SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_LRU_HASH && 113265334e64SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH && 11336df4ea1fSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_ARRAY && 11349db44fddSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY && 11359db44fddSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_SK_STORAGE && 11369db44fddSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_INODE_STORAGE && 11379db44fddSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_TASK_STORAGE && 11389db44fddSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) { 113961df10c7SKumar Kartikeya Dwivedi ret = -EOPNOTSUPP; 114061df10c7SKumar Kartikeya Dwivedi goto free_map_tab; 114161df10c7SKumar Kartikeya Dwivedi } 1142aa3496acSKumar Kartikeya Dwivedi break; 1143f0c5941fSKumar Kartikeya Dwivedi case BPF_LIST_HEAD: 11449c395c1bSDave Marchevsky case BPF_RB_ROOT: 1145f0c5941fSKumar Kartikeya Dwivedi if (map->map_type != BPF_MAP_TYPE_HASH && 1146f0c5941fSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_LRU_HASH && 1147f0c5941fSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_ARRAY) { 1148f0c5941fSKumar Kartikeya Dwivedi ret = -EOPNOTSUPP; 1149f0c5941fSKumar Kartikeya Dwivedi goto free_map_tab; 1150f0c5941fSKumar Kartikeya Dwivedi } 1151f0c5941fSKumar Kartikeya Dwivedi break; 1152aa3496acSKumar Kartikeya Dwivedi default: 1153aa3496acSKumar Kartikeya Dwivedi /* Fail if map_type checks are missing for a field type */ 1154aa3496acSKumar Kartikeya Dwivedi ret = -EOPNOTSUPP; 1155aa3496acSKumar Kartikeya Dwivedi goto free_map_tab; 1156aa3496acSKumar Kartikeya Dwivedi } 1157aa3496acSKumar Kartikeya Dwivedi } 115861df10c7SKumar Kartikeya Dwivedi } 1159e8d2bec0SDaniel Borkmann 1160865ce09aSKumar Kartikeya Dwivedi ret = btf_check_and_fixup_fields(btf, map->record); 1161865ce09aSKumar Kartikeya Dwivedi if (ret < 0) 1162865ce09aSKumar Kartikeya Dwivedi goto free_map_tab; 1163865ce09aSKumar Kartikeya Dwivedi 116461df10c7SKumar Kartikeya Dwivedi if (map->ops->map_check_btf) { 116561df10c7SKumar Kartikeya Dwivedi ret = map->ops->map_check_btf(map, btf, key_type, value_type); 116661df10c7SKumar Kartikeya Dwivedi if (ret < 0) 116761df10c7SKumar Kartikeya Dwivedi goto free_map_tab; 116861df10c7SKumar Kartikeya Dwivedi } 116961df10c7SKumar Kartikeya Dwivedi 117061df10c7SKumar Kartikeya Dwivedi return ret; 117161df10c7SKumar Kartikeya Dwivedi free_map_tab: 1172aa3496acSKumar Kartikeya Dwivedi bpf_map_free_record(map); 1173e8d2bec0SDaniel Borkmann return ret; 1174e8d2bec0SDaniel Borkmann } 1175e8d2bec0SDaniel Borkmann 1176ed1ad5a7SAndrii Nakryiko static bool bpf_net_capable(void) 1177ed1ad5a7SAndrii Nakryiko { 1178ed1ad5a7SAndrii Nakryiko return capable(CAP_NET_ADMIN) || capable(CAP_SYS_ADMIN); 1179ed1ad5a7SAndrii Nakryiko } 1180ed1ad5a7SAndrii Nakryiko 1181a177fc2bSAndrii Nakryiko #define BPF_MAP_CREATE_LAST_FIELD map_token_fd 118299c55f7dSAlexei Starovoitov /* called via syscall */ 118399c55f7dSAlexei Starovoitov static int map_create(union bpf_attr *attr) 118499c55f7dSAlexei Starovoitov { 118522db4122SAndrii Nakryiko const struct bpf_map_ops *ops; 1186a177fc2bSAndrii Nakryiko struct bpf_token *token = NULL; 118796eabe7aSMartin KaFai Lau int numa_node = bpf_map_attr_numa_node(attr); 118822db4122SAndrii Nakryiko u32 map_type = attr->map_type; 118999c55f7dSAlexei Starovoitov struct bpf_map *map; 1190a177fc2bSAndrii Nakryiko bool token_flag; 11916e71b04aSChenbo Feng int f_flags; 119299c55f7dSAlexei Starovoitov int err; 119399c55f7dSAlexei Starovoitov 119499c55f7dSAlexei Starovoitov err = CHECK_ATTR(BPF_MAP_CREATE); 119599c55f7dSAlexei Starovoitov if (err) 119699c55f7dSAlexei Starovoitov return -EINVAL; 119799c55f7dSAlexei Starovoitov 1198a177fc2bSAndrii Nakryiko /* check BPF_F_TOKEN_FD flag, remember if it's set, and then clear it 1199a177fc2bSAndrii Nakryiko * to avoid per-map type checks tripping on unknown flag 1200a177fc2bSAndrii Nakryiko */ 1201a177fc2bSAndrii Nakryiko token_flag = attr->map_flags & BPF_F_TOKEN_FD; 1202a177fc2bSAndrii Nakryiko attr->map_flags &= ~BPF_F_TOKEN_FD; 1203a177fc2bSAndrii Nakryiko 120485d33df3SMartin KaFai Lau if (attr->btf_vmlinux_value_type_id) { 120585d33df3SMartin KaFai Lau if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS || 120685d33df3SMartin KaFai Lau attr->btf_key_type_id || attr->btf_value_type_id) 120785d33df3SMartin KaFai Lau return -EINVAL; 120885d33df3SMartin KaFai Lau } else if (attr->btf_key_type_id && !attr->btf_value_type_id) { 120985d33df3SMartin KaFai Lau return -EINVAL; 121085d33df3SMartin KaFai Lau } 121185d33df3SMartin KaFai Lau 12129330986cSJoanne Koong if (attr->map_type != BPF_MAP_TYPE_BLOOM_FILTER && 121331746031SAlexei Starovoitov attr->map_type != BPF_MAP_TYPE_ARENA && 12149330986cSJoanne Koong attr->map_extra != 0) 12159330986cSJoanne Koong return -EINVAL; 12169330986cSJoanne Koong 12176e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->map_flags); 12186e71b04aSChenbo Feng if (f_flags < 0) 12196e71b04aSChenbo Feng return f_flags; 12206e71b04aSChenbo Feng 122196eabe7aSMartin KaFai Lau if (numa_node != NUMA_NO_NODE && 122296e5ae4eSEric Dumazet ((unsigned int)numa_node >= nr_node_ids || 122396e5ae4eSEric Dumazet !node_online(numa_node))) 122496eabe7aSMartin KaFai Lau return -EINVAL; 122596eabe7aSMartin KaFai Lau 122699c55f7dSAlexei Starovoitov /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 122722db4122SAndrii Nakryiko map_type = attr->map_type; 122822db4122SAndrii Nakryiko if (map_type >= ARRAY_SIZE(bpf_map_types)) 122922db4122SAndrii Nakryiko return -EINVAL; 123022db4122SAndrii Nakryiko map_type = array_index_nospec(map_type, ARRAY_SIZE(bpf_map_types)); 123122db4122SAndrii Nakryiko ops = bpf_map_types[map_type]; 123222db4122SAndrii Nakryiko if (!ops) 123322db4122SAndrii Nakryiko return -EINVAL; 123422db4122SAndrii Nakryiko 123522db4122SAndrii Nakryiko if (ops->map_alloc_check) { 123622db4122SAndrii Nakryiko err = ops->map_alloc_check(attr); 123722db4122SAndrii Nakryiko if (err) 123822db4122SAndrii Nakryiko return err; 123922db4122SAndrii Nakryiko } 124022db4122SAndrii Nakryiko if (attr->map_ifindex) 124122db4122SAndrii Nakryiko ops = &bpf_map_offload_ops; 124222db4122SAndrii Nakryiko if (!ops->map_mem_usage) 124322db4122SAndrii Nakryiko return -EINVAL; 124422db4122SAndrii Nakryiko 1245a177fc2bSAndrii Nakryiko if (token_flag) { 1246a177fc2bSAndrii Nakryiko token = bpf_token_get_from_fd(attr->map_token_fd); 1247a177fc2bSAndrii Nakryiko if (IS_ERR(token)) 1248a177fc2bSAndrii Nakryiko return PTR_ERR(token); 1249a177fc2bSAndrii Nakryiko 1250a177fc2bSAndrii Nakryiko /* if current token doesn't grant map creation permissions, 1251a177fc2bSAndrii Nakryiko * then we can't use this token, so ignore it and rely on 1252a177fc2bSAndrii Nakryiko * system-wide capabilities checks 1253a177fc2bSAndrii Nakryiko */ 1254a177fc2bSAndrii Nakryiko if (!bpf_token_allow_cmd(token, BPF_MAP_CREATE) || 1255a177fc2bSAndrii Nakryiko !bpf_token_allow_map_type(token, attr->map_type)) { 1256a177fc2bSAndrii Nakryiko bpf_token_put(token); 1257a177fc2bSAndrii Nakryiko token = NULL; 1258a177fc2bSAndrii Nakryiko } 1259a177fc2bSAndrii Nakryiko } 1260a177fc2bSAndrii Nakryiko 1261a177fc2bSAndrii Nakryiko err = -EPERM; 1262a177fc2bSAndrii Nakryiko 12631d28635aSAndrii Nakryiko /* Intent here is for unprivileged_bpf_disabled to block BPF map 12641d28635aSAndrii Nakryiko * creation for unprivileged users; other actions depend 12651d28635aSAndrii Nakryiko * on fd availability and access to bpffs, so are dependent on 12661d28635aSAndrii Nakryiko * object creation success. Even with unprivileged BPF disabled, 12671d28635aSAndrii Nakryiko * capability checks are still carried out. 12681d28635aSAndrii Nakryiko */ 1269a177fc2bSAndrii Nakryiko if (sysctl_unprivileged_bpf_disabled && !bpf_token_capable(token, CAP_BPF)) 1270a177fc2bSAndrii Nakryiko goto put_token; 12711d28635aSAndrii Nakryiko 12726c3eba1cSAndrii Nakryiko /* check privileged map type permissions */ 12736c3eba1cSAndrii Nakryiko switch (map_type) { 12746c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_ARRAY: 12756c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_PERCPU_ARRAY: 12766c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_PROG_ARRAY: 12776c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 12786c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_CGROUP_ARRAY: 12796c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_ARRAY_OF_MAPS: 12806c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_HASH: 12816c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_PERCPU_HASH: 12826c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_HASH_OF_MAPS: 12836c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_RINGBUF: 12846c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_USER_RINGBUF: 12856c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_CGROUP_STORAGE: 12866c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 12876c3eba1cSAndrii Nakryiko /* unprivileged */ 12886c3eba1cSAndrii Nakryiko break; 12896c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_SK_STORAGE: 12906c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_INODE_STORAGE: 12916c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_TASK_STORAGE: 12926c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_CGRP_STORAGE: 12936c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_BLOOM_FILTER: 12946c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_LPM_TRIE: 12956c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 12966c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_STACK_TRACE: 12976c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_QUEUE: 12986c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_STACK: 12996c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_LRU_HASH: 13006c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_LRU_PERCPU_HASH: 13016c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_STRUCT_OPS: 13026c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_CPUMAP: 130331746031SAlexei Starovoitov case BPF_MAP_TYPE_ARENA: 1304a177fc2bSAndrii Nakryiko if (!bpf_token_capable(token, CAP_BPF)) 1305a177fc2bSAndrii Nakryiko goto put_token; 13066c3eba1cSAndrii Nakryiko break; 13076c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_SOCKMAP: 13086c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_SOCKHASH: 13096c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_DEVMAP: 13106c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_DEVMAP_HASH: 13116c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_XSKMAP: 1312a177fc2bSAndrii Nakryiko if (!bpf_token_capable(token, CAP_NET_ADMIN)) 1313a177fc2bSAndrii Nakryiko goto put_token; 13146c3eba1cSAndrii Nakryiko break; 13156c3eba1cSAndrii Nakryiko default: 13166c3eba1cSAndrii Nakryiko WARN(1, "unsupported map type %d", map_type); 1317a177fc2bSAndrii Nakryiko goto put_token; 13186c3eba1cSAndrii Nakryiko } 13196c3eba1cSAndrii Nakryiko 132022db4122SAndrii Nakryiko map = ops->map_alloc(attr); 1321a177fc2bSAndrii Nakryiko if (IS_ERR(map)) { 1322a177fc2bSAndrii Nakryiko err = PTR_ERR(map); 1323a177fc2bSAndrii Nakryiko goto put_token; 1324a177fc2bSAndrii Nakryiko } 132522db4122SAndrii Nakryiko map->ops = ops; 132622db4122SAndrii Nakryiko map->map_type = map_type; 132799c55f7dSAlexei Starovoitov 13288e7ae251SMartin KaFai Lau err = bpf_obj_name_cpy(map->name, attr->map_name, 13298e7ae251SMartin KaFai Lau sizeof(attr->map_name)); 13308e7ae251SMartin KaFai Lau if (err < 0) 1331b936ca64SRoman Gushchin goto free_map; 1332ad5b177bSMartin KaFai Lau 13331e0bd5a0SAndrii Nakryiko atomic64_set(&map->refcnt, 1); 13341e0bd5a0SAndrii Nakryiko atomic64_set(&map->usercnt, 1); 1335fc970227SAndrii Nakryiko mutex_init(&map->freeze_mutex); 1336f45d5b6cSToke Hoiland-Jorgensen spin_lock_init(&map->owner.lock); 133799c55f7dSAlexei Starovoitov 133885d33df3SMartin KaFai Lau if (attr->btf_key_type_id || attr->btf_value_type_id || 133985d33df3SMartin KaFai Lau /* Even the map's value is a kernel's struct, 134085d33df3SMartin KaFai Lau * the bpf_prog.o must have BTF to begin with 134185d33df3SMartin KaFai Lau * to figure out the corresponding kernel's 134285d33df3SMartin KaFai Lau * counter part. Thus, attr->btf_fd has 134385d33df3SMartin KaFai Lau * to be valid also. 134485d33df3SMartin KaFai Lau */ 134585d33df3SMartin KaFai Lau attr->btf_vmlinux_value_type_id) { 1346a26ca7c9SMartin KaFai Lau struct btf *btf; 1347a26ca7c9SMartin KaFai Lau 1348a26ca7c9SMartin KaFai Lau btf = btf_get_by_fd(attr->btf_fd); 1349a26ca7c9SMartin KaFai Lau if (IS_ERR(btf)) { 1350a26ca7c9SMartin KaFai Lau err = PTR_ERR(btf); 1351b936ca64SRoman Gushchin goto free_map; 1352a26ca7c9SMartin KaFai Lau } 1353350a5c4dSAlexei Starovoitov if (btf_is_kernel(btf)) { 1354350a5c4dSAlexei Starovoitov btf_put(btf); 1355350a5c4dSAlexei Starovoitov err = -EACCES; 1356350a5c4dSAlexei Starovoitov goto free_map; 1357350a5c4dSAlexei Starovoitov } 135885d33df3SMartin KaFai Lau map->btf = btf; 1359a26ca7c9SMartin KaFai Lau 136085d33df3SMartin KaFai Lau if (attr->btf_value_type_id) { 1361a177fc2bSAndrii Nakryiko err = map_check_btf(map, token, btf, attr->btf_key_type_id, 13629b2cf328SMartin KaFai Lau attr->btf_value_type_id); 136385d33df3SMartin KaFai Lau if (err) 1364b936ca64SRoman Gushchin goto free_map; 1365a26ca7c9SMartin KaFai Lau } 1366a26ca7c9SMartin KaFai Lau 13679b2cf328SMartin KaFai Lau map->btf_key_type_id = attr->btf_key_type_id; 13689b2cf328SMartin KaFai Lau map->btf_value_type_id = attr->btf_value_type_id; 136985d33df3SMartin KaFai Lau map->btf_vmlinux_value_type_id = 137085d33df3SMartin KaFai Lau attr->btf_vmlinux_value_type_id; 1371a26ca7c9SMartin KaFai Lau } 1372a26ca7c9SMartin KaFai Lau 1373a2431c7eSAndrii Nakryiko err = security_bpf_map_create(map, attr, token); 13744d7d7f69SKumar Kartikeya Dwivedi if (err) 1375a2431c7eSAndrii Nakryiko goto free_map_sec; 13764d7d7f69SKumar Kartikeya Dwivedi 1377f3f1c054SMartin KaFai Lau err = bpf_map_alloc_id(map); 1378f3f1c054SMartin KaFai Lau if (err) 1379b936ca64SRoman Gushchin goto free_map_sec; 1380f3f1c054SMartin KaFai Lau 138148edc1f7SRoman Gushchin bpf_map_save_memcg(map); 1382a177fc2bSAndrii Nakryiko bpf_token_put(token); 138348edc1f7SRoman Gushchin 13846e71b04aSChenbo Feng err = bpf_map_new_fd(map, f_flags); 1385bd5f5f4eSMartin KaFai Lau if (err < 0) { 1386bd5f5f4eSMartin KaFai Lau /* failed to allocate fd. 1387352d20d6SPeng Sun * bpf_map_put_with_uref() is needed because the above 1388bd5f5f4eSMartin KaFai Lau * bpf_map_alloc_id() has published the map 1389bd5f5f4eSMartin KaFai Lau * to the userspace and the userspace may 1390bd5f5f4eSMartin KaFai Lau * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 1391bd5f5f4eSMartin KaFai Lau */ 1392352d20d6SPeng Sun bpf_map_put_with_uref(map); 1393bd5f5f4eSMartin KaFai Lau return err; 1394bd5f5f4eSMartin KaFai Lau } 139599c55f7dSAlexei Starovoitov 139699c55f7dSAlexei Starovoitov return err; 139799c55f7dSAlexei Starovoitov 1398afdb09c7SChenbo Feng free_map_sec: 1399afdb09c7SChenbo Feng security_bpf_map_free(map); 1400b936ca64SRoman Gushchin free_map: 1401a26ca7c9SMartin KaFai Lau btf_put(map->btf); 140299c55f7dSAlexei Starovoitov map->ops->map_free(map); 1403a177fc2bSAndrii Nakryiko put_token: 1404a177fc2bSAndrii Nakryiko bpf_token_put(token); 140599c55f7dSAlexei Starovoitov return err; 140699c55f7dSAlexei Starovoitov } 140799c55f7dSAlexei Starovoitov 1408db20fd2bSAlexei Starovoitov /* if error is returned, fd is released. 1409db20fd2bSAlexei Starovoitov * On success caller should complete fd access with matching fdput() 1410db20fd2bSAlexei Starovoitov */ 1411c2101297SDaniel Borkmann struct bpf_map *__bpf_map_get(struct fd f) 1412db20fd2bSAlexei Starovoitov { 1413db20fd2bSAlexei Starovoitov if (!f.file) 1414db20fd2bSAlexei Starovoitov return ERR_PTR(-EBADF); 1415db20fd2bSAlexei Starovoitov if (f.file->f_op != &bpf_map_fops) { 1416db20fd2bSAlexei Starovoitov fdput(f); 1417db20fd2bSAlexei Starovoitov return ERR_PTR(-EINVAL); 1418db20fd2bSAlexei Starovoitov } 1419db20fd2bSAlexei Starovoitov 1420c2101297SDaniel Borkmann return f.file->private_data; 1421c2101297SDaniel Borkmann } 1422c2101297SDaniel Borkmann 14231e0bd5a0SAndrii Nakryiko void bpf_map_inc(struct bpf_map *map) 1424c9da161cSDaniel Borkmann { 14251e0bd5a0SAndrii Nakryiko atomic64_inc(&map->refcnt); 1426c9da161cSDaniel Borkmann } 1427630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_inc); 1428c9da161cSDaniel Borkmann 14291e0bd5a0SAndrii Nakryiko void bpf_map_inc_with_uref(struct bpf_map *map) 14301e0bd5a0SAndrii Nakryiko { 14311e0bd5a0SAndrii Nakryiko atomic64_inc(&map->refcnt); 14321e0bd5a0SAndrii Nakryiko atomic64_inc(&map->usercnt); 14331e0bd5a0SAndrii Nakryiko } 14341e0bd5a0SAndrii Nakryiko EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref); 14351e0bd5a0SAndrii Nakryiko 14361ed4d924SMartin KaFai Lau struct bpf_map *bpf_map_get(u32 ufd) 14371ed4d924SMartin KaFai Lau { 14381ed4d924SMartin KaFai Lau struct fd f = fdget(ufd); 14391ed4d924SMartin KaFai Lau struct bpf_map *map; 14401ed4d924SMartin KaFai Lau 14411ed4d924SMartin KaFai Lau map = __bpf_map_get(f); 14421ed4d924SMartin KaFai Lau if (IS_ERR(map)) 14431ed4d924SMartin KaFai Lau return map; 14441ed4d924SMartin KaFai Lau 14451ed4d924SMartin KaFai Lau bpf_map_inc(map); 14461ed4d924SMartin KaFai Lau fdput(f); 14471ed4d924SMartin KaFai Lau 14481ed4d924SMartin KaFai Lau return map; 14491ed4d924SMartin KaFai Lau } 1450b1d18a75SAlexei Starovoitov EXPORT_SYMBOL(bpf_map_get); 14511ed4d924SMartin KaFai Lau 1452c9da161cSDaniel Borkmann struct bpf_map *bpf_map_get_with_uref(u32 ufd) 1453c2101297SDaniel Borkmann { 1454c2101297SDaniel Borkmann struct fd f = fdget(ufd); 1455c2101297SDaniel Borkmann struct bpf_map *map; 1456c2101297SDaniel Borkmann 1457c2101297SDaniel Borkmann map = __bpf_map_get(f); 1458c2101297SDaniel Borkmann if (IS_ERR(map)) 1459c2101297SDaniel Borkmann return map; 1460c2101297SDaniel Borkmann 14611e0bd5a0SAndrii Nakryiko bpf_map_inc_with_uref(map); 1462c2101297SDaniel Borkmann fdput(f); 1463db20fd2bSAlexei Starovoitov 1464db20fd2bSAlexei Starovoitov return map; 1465db20fd2bSAlexei Starovoitov } 1466db20fd2bSAlexei Starovoitov 1467b671c206SKui-Feng Lee /* map_idr_lock should have been held or the map should have been 1468b671c206SKui-Feng Lee * protected by rcu read lock. 1469b671c206SKui-Feng Lee */ 1470b671c206SKui-Feng Lee struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref) 1471bd5f5f4eSMartin KaFai Lau { 1472bd5f5f4eSMartin KaFai Lau int refold; 1473bd5f5f4eSMartin KaFai Lau 14741e0bd5a0SAndrii Nakryiko refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0); 1475bd5f5f4eSMartin KaFai Lau if (!refold) 1476bd5f5f4eSMartin KaFai Lau return ERR_PTR(-ENOENT); 1477bd5f5f4eSMartin KaFai Lau if (uref) 14781e0bd5a0SAndrii Nakryiko atomic64_inc(&map->usercnt); 1479bd5f5f4eSMartin KaFai Lau 1480bd5f5f4eSMartin KaFai Lau return map; 1481bd5f5f4eSMartin KaFai Lau } 1482bd5f5f4eSMartin KaFai Lau 14831e0bd5a0SAndrii Nakryiko struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map) 1484b0e4701cSStanislav Fomichev { 1485b0e4701cSStanislav Fomichev spin_lock_bh(&map_idr_lock); 14861e0bd5a0SAndrii Nakryiko map = __bpf_map_inc_not_zero(map, false); 1487b0e4701cSStanislav Fomichev spin_unlock_bh(&map_idr_lock); 1488b0e4701cSStanislav Fomichev 1489b0e4701cSStanislav Fomichev return map; 1490b0e4701cSStanislav Fomichev } 1491b0e4701cSStanislav Fomichev EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero); 1492b0e4701cSStanislav Fomichev 1493b8cdc051SAlexei Starovoitov int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 1494b8cdc051SAlexei Starovoitov { 1495b8cdc051SAlexei Starovoitov return -ENOTSUPP; 1496b8cdc051SAlexei Starovoitov } 1497b8cdc051SAlexei Starovoitov 1498c9d29f46SMauricio Vasquez B static void *__bpf_copy_key(void __user *ukey, u64 key_size) 1499c9d29f46SMauricio Vasquez B { 1500c9d29f46SMauricio Vasquez B if (key_size) 150144779a4bSStanislav Fomichev return vmemdup_user(ukey, key_size); 1502c9d29f46SMauricio Vasquez B 1503c9d29f46SMauricio Vasquez B if (ukey) 1504c9d29f46SMauricio Vasquez B return ERR_PTR(-EINVAL); 1505c9d29f46SMauricio Vasquez B 1506c9d29f46SMauricio Vasquez B return NULL; 1507c9d29f46SMauricio Vasquez B } 1508c9d29f46SMauricio Vasquez B 1509af2ac3e1SAlexei Starovoitov static void *___bpf_copy_key(bpfptr_t ukey, u64 key_size) 1510af2ac3e1SAlexei Starovoitov { 1511af2ac3e1SAlexei Starovoitov if (key_size) 151244779a4bSStanislav Fomichev return kvmemdup_bpfptr(ukey, key_size); 1513af2ac3e1SAlexei Starovoitov 1514af2ac3e1SAlexei Starovoitov if (!bpfptr_is_null(ukey)) 1515af2ac3e1SAlexei Starovoitov return ERR_PTR(-EINVAL); 1516af2ac3e1SAlexei Starovoitov 1517af2ac3e1SAlexei Starovoitov return NULL; 1518af2ac3e1SAlexei Starovoitov } 1519af2ac3e1SAlexei Starovoitov 1520db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 152196049f3aSAlexei Starovoitov #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags 1522db20fd2bSAlexei Starovoitov 1523db20fd2bSAlexei Starovoitov static int map_lookup_elem(union bpf_attr *attr) 1524db20fd2bSAlexei Starovoitov { 1525535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1526535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 1527db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1528db20fd2bSAlexei Starovoitov struct bpf_map *map; 152915c14a3dSBrian Vazquez void *key, *value; 153015a07b33SAlexei Starovoitov u32 value_size; 1531592867bfSDaniel Borkmann struct fd f; 1532db20fd2bSAlexei Starovoitov int err; 1533db20fd2bSAlexei Starovoitov 1534db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 1535db20fd2bSAlexei Starovoitov return -EINVAL; 1536db20fd2bSAlexei Starovoitov 153796049f3aSAlexei Starovoitov if (attr->flags & ~BPF_F_LOCK) 153896049f3aSAlexei Starovoitov return -EINVAL; 153996049f3aSAlexei Starovoitov 1540592867bfSDaniel Borkmann f = fdget(ufd); 1541c2101297SDaniel Borkmann map = __bpf_map_get(f); 1542db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1543db20fd2bSAlexei Starovoitov return PTR_ERR(map); 154487df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 15456e71b04aSChenbo Feng err = -EPERM; 15466e71b04aSChenbo Feng goto err_put; 15476e71b04aSChenbo Feng } 15486e71b04aSChenbo Feng 154996049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 1550db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) { 155196049f3aSAlexei Starovoitov err = -EINVAL; 155296049f3aSAlexei Starovoitov goto err_put; 155396049f3aSAlexei Starovoitov } 155496049f3aSAlexei Starovoitov 1555c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1556e4448ed8SAl Viro if (IS_ERR(key)) { 1557e4448ed8SAl Viro err = PTR_ERR(key); 1558db20fd2bSAlexei Starovoitov goto err_put; 1559e4448ed8SAl Viro } 1560db20fd2bSAlexei Starovoitov 156115c14a3dSBrian Vazquez value_size = bpf_map_value_size(map); 156215a07b33SAlexei Starovoitov 15638ebe667cSAlexei Starovoitov err = -ENOMEM; 1564f0dce1d9SStanislav Fomichev value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN); 1565db20fd2bSAlexei Starovoitov if (!value) 15668ebe667cSAlexei Starovoitov goto free_key; 15678ebe667cSAlexei Starovoitov 15689330986cSJoanne Koong if (map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) { 15699330986cSJoanne Koong if (copy_from_user(value, uvalue, value_size)) 15709330986cSJoanne Koong err = -EFAULT; 15719330986cSJoanne Koong else 15729330986cSJoanne Koong err = bpf_map_copy_value(map, key, value, attr->flags); 15739330986cSJoanne Koong goto free_value; 15749330986cSJoanne Koong } 15759330986cSJoanne Koong 157615c14a3dSBrian Vazquez err = bpf_map_copy_value(map, key, value, attr->flags); 157715a07b33SAlexei Starovoitov if (err) 15788ebe667cSAlexei Starovoitov goto free_value; 1579db20fd2bSAlexei Starovoitov 1580db20fd2bSAlexei Starovoitov err = -EFAULT; 158115a07b33SAlexei Starovoitov if (copy_to_user(uvalue, value, value_size) != 0) 15828ebe667cSAlexei Starovoitov goto free_value; 1583db20fd2bSAlexei Starovoitov 1584db20fd2bSAlexei Starovoitov err = 0; 1585db20fd2bSAlexei Starovoitov 15868ebe667cSAlexei Starovoitov free_value: 1587f0dce1d9SStanislav Fomichev kvfree(value); 1588db20fd2bSAlexei Starovoitov free_key: 158944779a4bSStanislav Fomichev kvfree(key); 1590db20fd2bSAlexei Starovoitov err_put: 1591db20fd2bSAlexei Starovoitov fdput(f); 1592db20fd2bSAlexei Starovoitov return err; 1593db20fd2bSAlexei Starovoitov } 1594db20fd2bSAlexei Starovoitov 15951ae80cf3SDaniel Colascione 15963274f520SAlexei Starovoitov #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 1597db20fd2bSAlexei Starovoitov 1598af2ac3e1SAlexei Starovoitov static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) 1599db20fd2bSAlexei Starovoitov { 1600af2ac3e1SAlexei Starovoitov bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel); 1601af2ac3e1SAlexei Starovoitov bpfptr_t uvalue = make_bpfptr(attr->value, uattr.is_kernel); 1602db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1603db20fd2bSAlexei Starovoitov struct bpf_map *map; 1604db20fd2bSAlexei Starovoitov void *key, *value; 160515a07b33SAlexei Starovoitov u32 value_size; 1606592867bfSDaniel Borkmann struct fd f; 1607db20fd2bSAlexei Starovoitov int err; 1608db20fd2bSAlexei Starovoitov 1609db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 1610db20fd2bSAlexei Starovoitov return -EINVAL; 1611db20fd2bSAlexei Starovoitov 1612592867bfSDaniel Borkmann f = fdget(ufd); 1613c2101297SDaniel Borkmann map = __bpf_map_get(f); 1614db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1615db20fd2bSAlexei Starovoitov return PTR_ERR(map); 1616353050beSDaniel Borkmann bpf_map_write_active_inc(map); 161787df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 16186e71b04aSChenbo Feng err = -EPERM; 16196e71b04aSChenbo Feng goto err_put; 16206e71b04aSChenbo Feng } 16216e71b04aSChenbo Feng 162296049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 1623db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) { 162496049f3aSAlexei Starovoitov err = -EINVAL; 162596049f3aSAlexei Starovoitov goto err_put; 162696049f3aSAlexei Starovoitov } 162796049f3aSAlexei Starovoitov 1628af2ac3e1SAlexei Starovoitov key = ___bpf_copy_key(ukey, map->key_size); 1629e4448ed8SAl Viro if (IS_ERR(key)) { 1630e4448ed8SAl Viro err = PTR_ERR(key); 1631db20fd2bSAlexei Starovoitov goto err_put; 1632e4448ed8SAl Viro } 1633db20fd2bSAlexei Starovoitov 1634f0dce1d9SStanislav Fomichev value_size = bpf_map_value_size(map); 1635a02c118eSWang Yufen value = kvmemdup_bpfptr(uvalue, value_size); 1636a02c118eSWang Yufen if (IS_ERR(value)) { 1637a02c118eSWang Yufen err = PTR_ERR(value); 1638db20fd2bSAlexei Starovoitov goto free_key; 1639a02c118eSWang Yufen } 1640db20fd2bSAlexei Starovoitov 16413af43ba4SHou Tao err = bpf_map_update_value(map, f.file, key, value, attr->flags); 164267ad2c73SHou Tao if (!err) 164337ba5b59SHou Tao maybe_wait_bpf_programs(map); 16446710e112SJesper Dangaard Brouer 1645f0dce1d9SStanislav Fomichev kvfree(value); 1646db20fd2bSAlexei Starovoitov free_key: 164744779a4bSStanislav Fomichev kvfree(key); 1648db20fd2bSAlexei Starovoitov err_put: 1649353050beSDaniel Borkmann bpf_map_write_active_dec(map); 1650db20fd2bSAlexei Starovoitov fdput(f); 1651db20fd2bSAlexei Starovoitov return err; 1652db20fd2bSAlexei Starovoitov } 1653db20fd2bSAlexei Starovoitov 1654db20fd2bSAlexei Starovoitov #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 1655db20fd2bSAlexei Starovoitov 1656b88df697SBenjamin Tissoires static int map_delete_elem(union bpf_attr *attr, bpfptr_t uattr) 1657db20fd2bSAlexei Starovoitov { 1658b88df697SBenjamin Tissoires bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel); 1659db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1660db20fd2bSAlexei Starovoitov struct bpf_map *map; 1661592867bfSDaniel Borkmann struct fd f; 1662db20fd2bSAlexei Starovoitov void *key; 1663db20fd2bSAlexei Starovoitov int err; 1664db20fd2bSAlexei Starovoitov 1665db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 1666db20fd2bSAlexei Starovoitov return -EINVAL; 1667db20fd2bSAlexei Starovoitov 1668592867bfSDaniel Borkmann f = fdget(ufd); 1669c2101297SDaniel Borkmann map = __bpf_map_get(f); 1670db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1671db20fd2bSAlexei Starovoitov return PTR_ERR(map); 1672353050beSDaniel Borkmann bpf_map_write_active_inc(map); 167387df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 16746e71b04aSChenbo Feng err = -EPERM; 16756e71b04aSChenbo Feng goto err_put; 16766e71b04aSChenbo Feng } 16776e71b04aSChenbo Feng 1678b88df697SBenjamin Tissoires key = ___bpf_copy_key(ukey, map->key_size); 1679e4448ed8SAl Viro if (IS_ERR(key)) { 1680e4448ed8SAl Viro err = PTR_ERR(key); 1681db20fd2bSAlexei Starovoitov goto err_put; 1682e4448ed8SAl Viro } 1683db20fd2bSAlexei Starovoitov 16849d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) { 1685a3884572SJakub Kicinski err = bpf_map_offload_delete_elem(map, key); 1686a3884572SJakub Kicinski goto out; 168785d33df3SMartin KaFai Lau } else if (IS_FD_PROG_ARRAY(map) || 168885d33df3SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 168985d33df3SMartin KaFai Lau /* These maps require sleepable context */ 1690da765a2fSDaniel Borkmann err = map->ops->map_delete_elem(map, key); 1691da765a2fSDaniel Borkmann goto out; 1692a3884572SJakub Kicinski } 1693a3884572SJakub Kicinski 1694b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 1695db20fd2bSAlexei Starovoitov rcu_read_lock(); 1696db20fd2bSAlexei Starovoitov err = map->ops->map_delete_elem(map, key); 1697db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1698b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 169967ad2c73SHou Tao if (!err) 17001ae80cf3SDaniel Colascione maybe_wait_bpf_programs(map); 1701a3884572SJakub Kicinski out: 170244779a4bSStanislav Fomichev kvfree(key); 1703db20fd2bSAlexei Starovoitov err_put: 1704353050beSDaniel Borkmann bpf_map_write_active_dec(map); 1705db20fd2bSAlexei Starovoitov fdput(f); 1706db20fd2bSAlexei Starovoitov return err; 1707db20fd2bSAlexei Starovoitov } 1708db20fd2bSAlexei Starovoitov 1709db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 1710db20fd2bSAlexei Starovoitov #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 1711db20fd2bSAlexei Starovoitov 1712db20fd2bSAlexei Starovoitov static int map_get_next_key(union bpf_attr *attr) 1713db20fd2bSAlexei Starovoitov { 1714535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1715535e7b4bSMickaël Salaün void __user *unext_key = u64_to_user_ptr(attr->next_key); 1716db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1717db20fd2bSAlexei Starovoitov struct bpf_map *map; 1718db20fd2bSAlexei Starovoitov void *key, *next_key; 1719592867bfSDaniel Borkmann struct fd f; 1720db20fd2bSAlexei Starovoitov int err; 1721db20fd2bSAlexei Starovoitov 1722db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 1723db20fd2bSAlexei Starovoitov return -EINVAL; 1724db20fd2bSAlexei Starovoitov 1725592867bfSDaniel Borkmann f = fdget(ufd); 1726c2101297SDaniel Borkmann map = __bpf_map_get(f); 1727db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1728db20fd2bSAlexei Starovoitov return PTR_ERR(map); 172987df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 17306e71b04aSChenbo Feng err = -EPERM; 17316e71b04aSChenbo Feng goto err_put; 17326e71b04aSChenbo Feng } 17336e71b04aSChenbo Feng 17348fe45924STeng Qin if (ukey) { 1735c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1736e4448ed8SAl Viro if (IS_ERR(key)) { 1737e4448ed8SAl Viro err = PTR_ERR(key); 1738db20fd2bSAlexei Starovoitov goto err_put; 1739e4448ed8SAl Viro } 17408fe45924STeng Qin } else { 17418fe45924STeng Qin key = NULL; 17428fe45924STeng Qin } 1743db20fd2bSAlexei Starovoitov 1744db20fd2bSAlexei Starovoitov err = -ENOMEM; 174544779a4bSStanislav Fomichev next_key = kvmalloc(map->key_size, GFP_USER); 1746db20fd2bSAlexei Starovoitov if (!next_key) 1747db20fd2bSAlexei Starovoitov goto free_key; 1748db20fd2bSAlexei Starovoitov 17499d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) { 1750a3884572SJakub Kicinski err = bpf_map_offload_get_next_key(map, key, next_key); 1751a3884572SJakub Kicinski goto out; 1752a3884572SJakub Kicinski } 1753a3884572SJakub Kicinski 1754db20fd2bSAlexei Starovoitov rcu_read_lock(); 1755db20fd2bSAlexei Starovoitov err = map->ops->map_get_next_key(map, key, next_key); 1756db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1757a3884572SJakub Kicinski out: 1758db20fd2bSAlexei Starovoitov if (err) 1759db20fd2bSAlexei Starovoitov goto free_next_key; 1760db20fd2bSAlexei Starovoitov 1761db20fd2bSAlexei Starovoitov err = -EFAULT; 1762db20fd2bSAlexei Starovoitov if (copy_to_user(unext_key, next_key, map->key_size) != 0) 1763db20fd2bSAlexei Starovoitov goto free_next_key; 1764db20fd2bSAlexei Starovoitov 1765db20fd2bSAlexei Starovoitov err = 0; 1766db20fd2bSAlexei Starovoitov 1767db20fd2bSAlexei Starovoitov free_next_key: 176844779a4bSStanislav Fomichev kvfree(next_key); 1769db20fd2bSAlexei Starovoitov free_key: 177044779a4bSStanislav Fomichev kvfree(key); 1771db20fd2bSAlexei Starovoitov err_put: 1772db20fd2bSAlexei Starovoitov fdput(f); 1773db20fd2bSAlexei Starovoitov return err; 1774db20fd2bSAlexei Starovoitov } 1775db20fd2bSAlexei Starovoitov 1776aa2e93b8SBrian Vazquez int generic_map_delete_batch(struct bpf_map *map, 1777aa2e93b8SBrian Vazquez const union bpf_attr *attr, 1778aa2e93b8SBrian Vazquez union bpf_attr __user *uattr) 1779aa2e93b8SBrian Vazquez { 1780aa2e93b8SBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1781aa2e93b8SBrian Vazquez u32 cp, max_count; 1782aa2e93b8SBrian Vazquez int err = 0; 1783aa2e93b8SBrian Vazquez void *key; 1784aa2e93b8SBrian Vazquez 1785aa2e93b8SBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1786aa2e93b8SBrian Vazquez return -EINVAL; 1787aa2e93b8SBrian Vazquez 1788aa2e93b8SBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1789db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) { 1790aa2e93b8SBrian Vazquez return -EINVAL; 1791aa2e93b8SBrian Vazquez } 1792aa2e93b8SBrian Vazquez 1793aa2e93b8SBrian Vazquez max_count = attr->batch.count; 1794aa2e93b8SBrian Vazquez if (!max_count) 1795aa2e93b8SBrian Vazquez return 0; 1796aa2e93b8SBrian Vazquez 179706e5c999SHou Tao if (put_user(0, &uattr->batch.count)) 179806e5c999SHou Tao return -EFAULT; 179906e5c999SHou Tao 180044779a4bSStanislav Fomichev key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 18012e3a94aaSBrian Vazquez if (!key) 18022e3a94aaSBrian Vazquez return -ENOMEM; 18032e3a94aaSBrian Vazquez 1804aa2e93b8SBrian Vazquez for (cp = 0; cp < max_count; cp++) { 18052e3a94aaSBrian Vazquez err = -EFAULT; 18062e3a94aaSBrian Vazquez if (copy_from_user(key, keys + cp * map->key_size, 18072e3a94aaSBrian Vazquez map->key_size)) 1808aa2e93b8SBrian Vazquez break; 1809aa2e93b8SBrian Vazquez 18109d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) { 1811aa2e93b8SBrian Vazquez err = bpf_map_offload_delete_elem(map, key); 1812aa2e93b8SBrian Vazquez break; 1813aa2e93b8SBrian Vazquez } 1814aa2e93b8SBrian Vazquez 1815b6e5dae1SThomas Gleixner bpf_disable_instrumentation(); 1816aa2e93b8SBrian Vazquez rcu_read_lock(); 1817aa2e93b8SBrian Vazquez err = map->ops->map_delete_elem(map, key); 1818aa2e93b8SBrian Vazquez rcu_read_unlock(); 1819b6e5dae1SThomas Gleixner bpf_enable_instrumentation(); 1820aa2e93b8SBrian Vazquez if (err) 1821aa2e93b8SBrian Vazquez break; 182275134f16SEric Dumazet cond_resched(); 1823aa2e93b8SBrian Vazquez } 1824aa2e93b8SBrian Vazquez if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) 1825aa2e93b8SBrian Vazquez err = -EFAULT; 18262e3a94aaSBrian Vazquez 182744779a4bSStanislav Fomichev kvfree(key); 18289087c6ffSEric Dumazet 1829aa2e93b8SBrian Vazquez return err; 1830aa2e93b8SBrian Vazquez } 1831aa2e93b8SBrian Vazquez 18323af43ba4SHou Tao int generic_map_update_batch(struct bpf_map *map, struct file *map_file, 1833aa2e93b8SBrian Vazquez const union bpf_attr *attr, 1834aa2e93b8SBrian Vazquez union bpf_attr __user *uattr) 1835aa2e93b8SBrian Vazquez { 1836aa2e93b8SBrian Vazquez void __user *values = u64_to_user_ptr(attr->batch.values); 1837aa2e93b8SBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1838aa2e93b8SBrian Vazquez u32 value_size, cp, max_count; 1839aa2e93b8SBrian Vazquez void *key, *value; 1840aa2e93b8SBrian Vazquez int err = 0; 1841aa2e93b8SBrian Vazquez 1842aa2e93b8SBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1843aa2e93b8SBrian Vazquez return -EINVAL; 1844aa2e93b8SBrian Vazquez 1845aa2e93b8SBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1846db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) { 1847aa2e93b8SBrian Vazquez return -EINVAL; 1848aa2e93b8SBrian Vazquez } 1849aa2e93b8SBrian Vazquez 1850aa2e93b8SBrian Vazquez value_size = bpf_map_value_size(map); 1851aa2e93b8SBrian Vazquez 1852aa2e93b8SBrian Vazquez max_count = attr->batch.count; 1853aa2e93b8SBrian Vazquez if (!max_count) 1854aa2e93b8SBrian Vazquez return 0; 1855aa2e93b8SBrian Vazquez 185606e5c999SHou Tao if (put_user(0, &uattr->batch.count)) 185706e5c999SHou Tao return -EFAULT; 185806e5c999SHou Tao 185944779a4bSStanislav Fomichev key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 18602e3a94aaSBrian Vazquez if (!key) 1861aa2e93b8SBrian Vazquez return -ENOMEM; 1862aa2e93b8SBrian Vazquez 1863f0dce1d9SStanislav Fomichev value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN); 18642e3a94aaSBrian Vazquez if (!value) { 186544779a4bSStanislav Fomichev kvfree(key); 18662e3a94aaSBrian Vazquez return -ENOMEM; 1867aa2e93b8SBrian Vazquez } 18682e3a94aaSBrian Vazquez 18692e3a94aaSBrian Vazquez for (cp = 0; cp < max_count; cp++) { 1870aa2e93b8SBrian Vazquez err = -EFAULT; 18712e3a94aaSBrian Vazquez if (copy_from_user(key, keys + cp * map->key_size, 18722e3a94aaSBrian Vazquez map->key_size) || 18732e3a94aaSBrian Vazquez copy_from_user(value, values + cp * value_size, value_size)) 1874aa2e93b8SBrian Vazquez break; 1875aa2e93b8SBrian Vazquez 18763af43ba4SHou Tao err = bpf_map_update_value(map, map_file, key, value, 1877aa2e93b8SBrian Vazquez attr->batch.elem_flags); 1878aa2e93b8SBrian Vazquez 1879aa2e93b8SBrian Vazquez if (err) 1880aa2e93b8SBrian Vazquez break; 188175134f16SEric Dumazet cond_resched(); 1882aa2e93b8SBrian Vazquez } 1883aa2e93b8SBrian Vazquez 1884aa2e93b8SBrian Vazquez if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) 1885aa2e93b8SBrian Vazquez err = -EFAULT; 1886aa2e93b8SBrian Vazquez 1887f0dce1d9SStanislav Fomichev kvfree(value); 188844779a4bSStanislav Fomichev kvfree(key); 188937ba5b59SHou Tao 1890aa2e93b8SBrian Vazquez return err; 1891aa2e93b8SBrian Vazquez } 1892aa2e93b8SBrian Vazquez 1893cb4d03abSBrian Vazquez #define MAP_LOOKUP_RETRIES 3 1894cb4d03abSBrian Vazquez 1895cb4d03abSBrian Vazquez int generic_map_lookup_batch(struct bpf_map *map, 1896cb4d03abSBrian Vazquez const union bpf_attr *attr, 1897cb4d03abSBrian Vazquez union bpf_attr __user *uattr) 1898cb4d03abSBrian Vazquez { 1899cb4d03abSBrian Vazquez void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch); 1900cb4d03abSBrian Vazquez void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch); 1901cb4d03abSBrian Vazquez void __user *values = u64_to_user_ptr(attr->batch.values); 1902cb4d03abSBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys); 1903cb4d03abSBrian Vazquez void *buf, *buf_prevkey, *prev_key, *key, *value; 1904cb4d03abSBrian Vazquez int err, retry = MAP_LOOKUP_RETRIES; 1905cb4d03abSBrian Vazquez u32 value_size, cp, max_count; 1906cb4d03abSBrian Vazquez 1907cb4d03abSBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK) 1908cb4d03abSBrian Vazquez return -EINVAL; 1909cb4d03abSBrian Vazquez 1910cb4d03abSBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) && 1911db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) 1912cb4d03abSBrian Vazquez return -EINVAL; 1913cb4d03abSBrian Vazquez 1914cb4d03abSBrian Vazquez value_size = bpf_map_value_size(map); 1915cb4d03abSBrian Vazquez 1916cb4d03abSBrian Vazquez max_count = attr->batch.count; 1917cb4d03abSBrian Vazquez if (!max_count) 1918cb4d03abSBrian Vazquez return 0; 1919cb4d03abSBrian Vazquez 1920cb4d03abSBrian Vazquez if (put_user(0, &uattr->batch.count)) 1921cb4d03abSBrian Vazquez return -EFAULT; 1922cb4d03abSBrian Vazquez 192344779a4bSStanislav Fomichev buf_prevkey = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN); 1924cb4d03abSBrian Vazquez if (!buf_prevkey) 1925cb4d03abSBrian Vazquez return -ENOMEM; 1926cb4d03abSBrian Vazquez 1927f0dce1d9SStanislav Fomichev buf = kvmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN); 1928cb4d03abSBrian Vazquez if (!buf) { 192944779a4bSStanislav Fomichev kvfree(buf_prevkey); 1930cb4d03abSBrian Vazquez return -ENOMEM; 1931cb4d03abSBrian Vazquez } 1932cb4d03abSBrian Vazquez 1933cb4d03abSBrian Vazquez err = -EFAULT; 1934cb4d03abSBrian Vazquez prev_key = NULL; 1935cb4d03abSBrian Vazquez if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size)) 1936cb4d03abSBrian Vazquez goto free_buf; 1937cb4d03abSBrian Vazquez key = buf; 1938cb4d03abSBrian Vazquez value = key + map->key_size; 1939cb4d03abSBrian Vazquez if (ubatch) 1940cb4d03abSBrian Vazquez prev_key = buf_prevkey; 1941cb4d03abSBrian Vazquez 1942cb4d03abSBrian Vazquez for (cp = 0; cp < max_count;) { 1943cb4d03abSBrian Vazquez rcu_read_lock(); 1944cb4d03abSBrian Vazquez err = map->ops->map_get_next_key(map, prev_key, key); 1945cb4d03abSBrian Vazquez rcu_read_unlock(); 1946cb4d03abSBrian Vazquez if (err) 1947cb4d03abSBrian Vazquez break; 1948cb4d03abSBrian Vazquez err = bpf_map_copy_value(map, key, value, 1949cb4d03abSBrian Vazquez attr->batch.elem_flags); 1950cb4d03abSBrian Vazquez 1951cb4d03abSBrian Vazquez if (err == -ENOENT) { 1952cb4d03abSBrian Vazquez if (retry) { 1953cb4d03abSBrian Vazquez retry--; 1954cb4d03abSBrian Vazquez continue; 1955cb4d03abSBrian Vazquez } 1956cb4d03abSBrian Vazquez err = -EINTR; 1957cb4d03abSBrian Vazquez break; 1958cb4d03abSBrian Vazquez } 1959cb4d03abSBrian Vazquez 1960cb4d03abSBrian Vazquez if (err) 1961cb4d03abSBrian Vazquez goto free_buf; 1962cb4d03abSBrian Vazquez 1963cb4d03abSBrian Vazquez if (copy_to_user(keys + cp * map->key_size, key, 1964cb4d03abSBrian Vazquez map->key_size)) { 1965cb4d03abSBrian Vazquez err = -EFAULT; 1966cb4d03abSBrian Vazquez goto free_buf; 1967cb4d03abSBrian Vazquez } 1968cb4d03abSBrian Vazquez if (copy_to_user(values + cp * value_size, value, value_size)) { 1969cb4d03abSBrian Vazquez err = -EFAULT; 1970cb4d03abSBrian Vazquez goto free_buf; 1971cb4d03abSBrian Vazquez } 1972cb4d03abSBrian Vazquez 1973cb4d03abSBrian Vazquez if (!prev_key) 1974cb4d03abSBrian Vazquez prev_key = buf_prevkey; 1975cb4d03abSBrian Vazquez 1976cb4d03abSBrian Vazquez swap(prev_key, key); 1977cb4d03abSBrian Vazquez retry = MAP_LOOKUP_RETRIES; 1978cb4d03abSBrian Vazquez cp++; 197975134f16SEric Dumazet cond_resched(); 1980cb4d03abSBrian Vazquez } 1981cb4d03abSBrian Vazquez 1982cb4d03abSBrian Vazquez if (err == -EFAULT) 1983cb4d03abSBrian Vazquez goto free_buf; 1984cb4d03abSBrian Vazquez 1985cb4d03abSBrian Vazquez if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) || 1986cb4d03abSBrian Vazquez (cp && copy_to_user(uobatch, prev_key, map->key_size)))) 1987cb4d03abSBrian Vazquez err = -EFAULT; 1988cb4d03abSBrian Vazquez 1989cb4d03abSBrian Vazquez free_buf: 199044779a4bSStanislav Fomichev kvfree(buf_prevkey); 1991f0dce1d9SStanislav Fomichev kvfree(buf); 1992cb4d03abSBrian Vazquez return err; 1993cb4d03abSBrian Vazquez } 1994cb4d03abSBrian Vazquez 19953e87f192SDenis Salopek #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD flags 1996bd513cd0SMauricio Vasquez B 1997bd513cd0SMauricio Vasquez B static int map_lookup_and_delete_elem(union bpf_attr *attr) 1998bd513cd0SMauricio Vasquez B { 1999bd513cd0SMauricio Vasquez B void __user *ukey = u64_to_user_ptr(attr->key); 2000bd513cd0SMauricio Vasquez B void __user *uvalue = u64_to_user_ptr(attr->value); 2001bd513cd0SMauricio Vasquez B int ufd = attr->map_fd; 2002bd513cd0SMauricio Vasquez B struct bpf_map *map; 2003540fefc0SAlexei Starovoitov void *key, *value; 2004bd513cd0SMauricio Vasquez B u32 value_size; 2005bd513cd0SMauricio Vasquez B struct fd f; 2006bd513cd0SMauricio Vasquez B int err; 2007bd513cd0SMauricio Vasquez B 2008bd513cd0SMauricio Vasquez B if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) 2009bd513cd0SMauricio Vasquez B return -EINVAL; 2010bd513cd0SMauricio Vasquez B 20113e87f192SDenis Salopek if (attr->flags & ~BPF_F_LOCK) 20123e87f192SDenis Salopek return -EINVAL; 20133e87f192SDenis Salopek 2014bd513cd0SMauricio Vasquez B f = fdget(ufd); 2015bd513cd0SMauricio Vasquez B map = __bpf_map_get(f); 2016bd513cd0SMauricio Vasquez B if (IS_ERR(map)) 2017bd513cd0SMauricio Vasquez B return PTR_ERR(map); 2018353050beSDaniel Borkmann bpf_map_write_active_inc(map); 20191ea0f912SAnton Protopopov if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) || 20201ea0f912SAnton Protopopov !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 2021bd513cd0SMauricio Vasquez B err = -EPERM; 2022bd513cd0SMauricio Vasquez B goto err_put; 2023bd513cd0SMauricio Vasquez B } 2024bd513cd0SMauricio Vasquez B 20253e87f192SDenis Salopek if (attr->flags && 20263e87f192SDenis Salopek (map->map_type == BPF_MAP_TYPE_QUEUE || 20273e87f192SDenis Salopek map->map_type == BPF_MAP_TYPE_STACK)) { 20283e87f192SDenis Salopek err = -EINVAL; 20293e87f192SDenis Salopek goto err_put; 20303e87f192SDenis Salopek } 20313e87f192SDenis Salopek 20323e87f192SDenis Salopek if ((attr->flags & BPF_F_LOCK) && 2033db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) { 20343e87f192SDenis Salopek err = -EINVAL; 20353e87f192SDenis Salopek goto err_put; 20363e87f192SDenis Salopek } 20373e87f192SDenis Salopek 2038bd513cd0SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 2039bd513cd0SMauricio Vasquez B if (IS_ERR(key)) { 2040bd513cd0SMauricio Vasquez B err = PTR_ERR(key); 2041bd513cd0SMauricio Vasquez B goto err_put; 2042bd513cd0SMauricio Vasquez B } 2043bd513cd0SMauricio Vasquez B 20443e87f192SDenis Salopek value_size = bpf_map_value_size(map); 2045bd513cd0SMauricio Vasquez B 2046bd513cd0SMauricio Vasquez B err = -ENOMEM; 2047f0dce1d9SStanislav Fomichev value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN); 2048bd513cd0SMauricio Vasquez B if (!value) 2049bd513cd0SMauricio Vasquez B goto free_key; 2050bd513cd0SMauricio Vasquez B 20513e87f192SDenis Salopek err = -ENOTSUPP; 2052bd513cd0SMauricio Vasquez B if (map->map_type == BPF_MAP_TYPE_QUEUE || 2053bd513cd0SMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 2054bd513cd0SMauricio Vasquez B err = map->ops->map_pop_elem(map, value); 20553e87f192SDenis Salopek } else if (map->map_type == BPF_MAP_TYPE_HASH || 20563e87f192SDenis Salopek map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 20573e87f192SDenis Salopek map->map_type == BPF_MAP_TYPE_LRU_HASH || 20583e87f192SDenis Salopek map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 20599d03ebc7SStanislav Fomichev if (!bpf_map_is_offloaded(map)) { 20603e87f192SDenis Salopek bpf_disable_instrumentation(); 20613e87f192SDenis Salopek rcu_read_lock(); 20623e87f192SDenis Salopek err = map->ops->map_lookup_and_delete_elem(map, key, value, attr->flags); 20633e87f192SDenis Salopek rcu_read_unlock(); 20643e87f192SDenis Salopek bpf_enable_instrumentation(); 20653e87f192SDenis Salopek } 2066bd513cd0SMauricio Vasquez B } 2067bd513cd0SMauricio Vasquez B 2068bd513cd0SMauricio Vasquez B if (err) 2069bd513cd0SMauricio Vasquez B goto free_value; 2070bd513cd0SMauricio Vasquez B 20717f645462SWei Yongjun if (copy_to_user(uvalue, value, value_size) != 0) { 20727f645462SWei Yongjun err = -EFAULT; 2073bd513cd0SMauricio Vasquez B goto free_value; 20747f645462SWei Yongjun } 2075bd513cd0SMauricio Vasquez B 2076bd513cd0SMauricio Vasquez B err = 0; 2077bd513cd0SMauricio Vasquez B 2078bd513cd0SMauricio Vasquez B free_value: 2079f0dce1d9SStanislav Fomichev kvfree(value); 2080bd513cd0SMauricio Vasquez B free_key: 208144779a4bSStanislav Fomichev kvfree(key); 2082bd513cd0SMauricio Vasquez B err_put: 2083353050beSDaniel Borkmann bpf_map_write_active_dec(map); 2084bd513cd0SMauricio Vasquez B fdput(f); 2085bd513cd0SMauricio Vasquez B return err; 2086bd513cd0SMauricio Vasquez B } 2087bd513cd0SMauricio Vasquez B 208887df15deSDaniel Borkmann #define BPF_MAP_FREEZE_LAST_FIELD map_fd 208987df15deSDaniel Borkmann 209087df15deSDaniel Borkmann static int map_freeze(const union bpf_attr *attr) 209187df15deSDaniel Borkmann { 209287df15deSDaniel Borkmann int err = 0, ufd = attr->map_fd; 209387df15deSDaniel Borkmann struct bpf_map *map; 209487df15deSDaniel Borkmann struct fd f; 209587df15deSDaniel Borkmann 209687df15deSDaniel Borkmann if (CHECK_ATTR(BPF_MAP_FREEZE)) 209787df15deSDaniel Borkmann return -EINVAL; 209887df15deSDaniel Borkmann 209987df15deSDaniel Borkmann f = fdget(ufd); 210087df15deSDaniel Borkmann map = __bpf_map_get(f); 210187df15deSDaniel Borkmann if (IS_ERR(map)) 210287df15deSDaniel Borkmann return PTR_ERR(map); 2103fc970227SAndrii Nakryiko 2104db559117SKumar Kartikeya Dwivedi if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS || !IS_ERR_OR_NULL(map->record)) { 2105849b4d94SMartin KaFai Lau fdput(f); 2106849b4d94SMartin KaFai Lau return -ENOTSUPP; 2107849b4d94SMartin KaFai Lau } 2108849b4d94SMartin KaFai Lau 2109c4c84f6fSAndrii Nakryiko if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 21104266f41fSDaniel Borkmann fdput(f); 21114266f41fSDaniel Borkmann return -EPERM; 2112c4c84f6fSAndrii Nakryiko } 2113c4c84f6fSAndrii Nakryiko 2114fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex); 2115353050beSDaniel Borkmann if (bpf_map_write_active(map)) { 2116fc970227SAndrii Nakryiko err = -EBUSY; 2117fc970227SAndrii Nakryiko goto err_put; 2118fc970227SAndrii Nakryiko } 211987df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) { 212087df15deSDaniel Borkmann err = -EBUSY; 212187df15deSDaniel Borkmann goto err_put; 212287df15deSDaniel Borkmann } 212387df15deSDaniel Borkmann 212487df15deSDaniel Borkmann WRITE_ONCE(map->frozen, true); 212587df15deSDaniel Borkmann err_put: 2126fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex); 212787df15deSDaniel Borkmann fdput(f); 212887df15deSDaniel Borkmann return err; 212987df15deSDaniel Borkmann } 213087df15deSDaniel Borkmann 21317de16e3aSJakub Kicinski static const struct bpf_prog_ops * const bpf_prog_types[] = { 213291cc1a99SAlexei Starovoitov #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 21337de16e3aSJakub Kicinski [_id] = & _name ## _prog_ops, 21347de16e3aSJakub Kicinski #define BPF_MAP_TYPE(_id, _ops) 2135f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) 21367de16e3aSJakub Kicinski #include <linux/bpf_types.h> 21377de16e3aSJakub Kicinski #undef BPF_PROG_TYPE 21387de16e3aSJakub Kicinski #undef BPF_MAP_TYPE 2139f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 21407de16e3aSJakub Kicinski }; 21417de16e3aSJakub Kicinski 214209756af4SAlexei Starovoitov static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 214309756af4SAlexei Starovoitov { 2144d0f1a451SDaniel Borkmann const struct bpf_prog_ops *ops; 2145d0f1a451SDaniel Borkmann 2146d0f1a451SDaniel Borkmann if (type >= ARRAY_SIZE(bpf_prog_types)) 2147d0f1a451SDaniel Borkmann return -EINVAL; 2148d0f1a451SDaniel Borkmann type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types)); 2149d0f1a451SDaniel Borkmann ops = bpf_prog_types[type]; 2150d0f1a451SDaniel Borkmann if (!ops) 2151be9370a7SJohannes Berg return -EINVAL; 215209756af4SAlexei Starovoitov 21539d03ebc7SStanislav Fomichev if (!bpf_prog_is_offloaded(prog->aux)) 2154d0f1a451SDaniel Borkmann prog->aux->ops = ops; 2155ab3f0063SJakub Kicinski else 2156ab3f0063SJakub Kicinski prog->aux->ops = &bpf_offload_prog_ops; 215724701eceSDaniel Borkmann prog->type = type; 215809756af4SAlexei Starovoitov return 0; 215909756af4SAlexei Starovoitov } 216009756af4SAlexei Starovoitov 2161bae141f5SDaniel Borkmann enum bpf_audit { 2162bae141f5SDaniel Borkmann BPF_AUDIT_LOAD, 2163bae141f5SDaniel Borkmann BPF_AUDIT_UNLOAD, 2164bae141f5SDaniel Borkmann BPF_AUDIT_MAX, 2165bae141f5SDaniel Borkmann }; 2166bae141f5SDaniel Borkmann 2167bae141f5SDaniel Borkmann static const char * const bpf_audit_str[BPF_AUDIT_MAX] = { 2168bae141f5SDaniel Borkmann [BPF_AUDIT_LOAD] = "LOAD", 2169bae141f5SDaniel Borkmann [BPF_AUDIT_UNLOAD] = "UNLOAD", 2170bae141f5SDaniel Borkmann }; 2171bae141f5SDaniel Borkmann 2172bae141f5SDaniel Borkmann static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op) 2173bae141f5SDaniel Borkmann { 2174bae141f5SDaniel Borkmann struct audit_context *ctx = NULL; 2175bae141f5SDaniel Borkmann struct audit_buffer *ab; 2176bae141f5SDaniel Borkmann 2177bae141f5SDaniel Borkmann if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX)) 2178bae141f5SDaniel Borkmann return; 2179bae141f5SDaniel Borkmann if (audit_enabled == AUDIT_OFF) 2180bae141f5SDaniel Borkmann return; 2181ef01f4e2SPaul Moore if (!in_irq() && !irqs_disabled()) 2182bae141f5SDaniel Borkmann ctx = audit_context(); 2183bae141f5SDaniel Borkmann ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF); 2184bae141f5SDaniel Borkmann if (unlikely(!ab)) 2185bae141f5SDaniel Borkmann return; 2186bae141f5SDaniel Borkmann audit_log_format(ab, "prog-id=%u op=%s", 2187bae141f5SDaniel Borkmann prog->aux->id, bpf_audit_str[op]); 2188bae141f5SDaniel Borkmann audit_log_end(ab); 2189bae141f5SDaniel Borkmann } 2190bae141f5SDaniel Borkmann 2191dc4bb0e2SMartin KaFai Lau static int bpf_prog_alloc_id(struct bpf_prog *prog) 2192dc4bb0e2SMartin KaFai Lau { 2193dc4bb0e2SMartin KaFai Lau int id; 2194dc4bb0e2SMartin KaFai Lau 2195b76354cdSShaohua Li idr_preload(GFP_KERNEL); 2196dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 2197dc4bb0e2SMartin KaFai Lau id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 2198dc4bb0e2SMartin KaFai Lau if (id > 0) 2199dc4bb0e2SMartin KaFai Lau prog->aux->id = id; 2200dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 2201b76354cdSShaohua Li idr_preload_end(); 2202dc4bb0e2SMartin KaFai Lau 2203dc4bb0e2SMartin KaFai Lau /* id is in [1, INT_MAX) */ 2204dc4bb0e2SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 2205dc4bb0e2SMartin KaFai Lau return -ENOSPC; 2206dc4bb0e2SMartin KaFai Lau 2207dc4bb0e2SMartin KaFai Lau return id > 0 ? 0 : id; 2208dc4bb0e2SMartin KaFai Lau } 2209dc4bb0e2SMartin KaFai Lau 2210e7895f01SPaul Moore void bpf_prog_free_id(struct bpf_prog *prog) 2211dc4bb0e2SMartin KaFai Lau { 2212d809e134SAlexei Starovoitov unsigned long flags; 2213d809e134SAlexei Starovoitov 2214ad8ad79fSJakub Kicinski /* cBPF to eBPF migrations are currently not in the idr store. 2215ad8ad79fSJakub Kicinski * Offloaded programs are removed from the store when their device 2216ad8ad79fSJakub Kicinski * disappears - even if someone grabs an fd to them they are unusable, 2217ad8ad79fSJakub Kicinski * simply waiting for refcnt to drop to be freed. 2218ad8ad79fSJakub Kicinski */ 2219dc4bb0e2SMartin KaFai Lau if (!prog->aux->id) 2220dc4bb0e2SMartin KaFai Lau return; 2221dc4bb0e2SMartin KaFai Lau 2222d809e134SAlexei Starovoitov spin_lock_irqsave(&prog_idr_lock, flags); 2223dc4bb0e2SMartin KaFai Lau idr_remove(&prog_idr, prog->aux->id); 2224ad8ad79fSJakub Kicinski prog->aux->id = 0; 2225d809e134SAlexei Starovoitov spin_unlock_irqrestore(&prog_idr_lock, flags); 2226dc4bb0e2SMartin KaFai Lau } 2227dc4bb0e2SMartin KaFai Lau 22281aacde3dSDaniel Borkmann static void __bpf_prog_put_rcu(struct rcu_head *rcu) 2229abf2e7d6SAlexei Starovoitov { 2230abf2e7d6SAlexei Starovoitov struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 2231abf2e7d6SAlexei Starovoitov 22323b4d9eb2SDaniel Borkmann kvfree(aux->func_info); 22338c1b6e69SAlexei Starovoitov kfree(aux->func_info_aux); 22343ac1f01bSRoman Gushchin free_uid(aux->user); 22351b67772eSAndrii Nakryiko security_bpf_prog_free(aux->prog); 2236abf2e7d6SAlexei Starovoitov bpf_prog_free(aux->prog); 2237abf2e7d6SAlexei Starovoitov } 2238abf2e7d6SAlexei Starovoitov 2239cd7455f1SDaniel Borkmann static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) 2240cd7455f1SDaniel Borkmann { 2241cd7455f1SDaniel Borkmann bpf_prog_kallsyms_del_all(prog); 2242cd7455f1SDaniel Borkmann btf_put(prog->aux->btf); 224331bf1dbcSViktor Malik module_put(prog->aux->mod); 2244e16301fbSMartin KaFai Lau kvfree(prog->aux->jited_linfo); 2245e16301fbSMartin KaFai Lau kvfree(prog->aux->linfo); 2246e6ac2450SMartin KaFai Lau kfree(prog->aux->kfunc_tab); 224722dc4a0fSAndrii Nakryiko if (prog->aux->attach_btf) 224822dc4a0fSAndrii Nakryiko btf_put(prog->aux->attach_btf); 2249cd7455f1SDaniel Borkmann 22501e6c62a8SAlexei Starovoitov if (deferred) { 225166c84731SAndrii Nakryiko if (prog->sleepable) 22521e6c62a8SAlexei Starovoitov call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); 2253cd7455f1SDaniel Borkmann else 22541e6c62a8SAlexei Starovoitov call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 22551e6c62a8SAlexei Starovoitov } else { 2256cd7455f1SDaniel Borkmann __bpf_prog_put_rcu(&prog->aux->rcu); 2257cd7455f1SDaniel Borkmann } 22581e6c62a8SAlexei Starovoitov } 2259cd7455f1SDaniel Borkmann 2260d809e134SAlexei Starovoitov static void bpf_prog_put_deferred(struct work_struct *work) 226109756af4SAlexei Starovoitov { 2262d809e134SAlexei Starovoitov struct bpf_prog_aux *aux; 2263d809e134SAlexei Starovoitov struct bpf_prog *prog; 2264d809e134SAlexei Starovoitov 2265d809e134SAlexei Starovoitov aux = container_of(work, struct bpf_prog_aux, work); 2266d809e134SAlexei Starovoitov prog = aux->prog; 22676ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0); 2268bae141f5SDaniel Borkmann bpf_audit_prog(prog, BPF_AUDIT_UNLOAD); 2269e7895f01SPaul Moore bpf_prog_free_id(prog); 2270d809e134SAlexei Starovoitov __bpf_prog_put_noref(prog, true); 2271d809e134SAlexei Starovoitov } 2272d809e134SAlexei Starovoitov 2273e7895f01SPaul Moore static void __bpf_prog_put(struct bpf_prog *prog) 2274d809e134SAlexei Starovoitov { 2275d809e134SAlexei Starovoitov struct bpf_prog_aux *aux = prog->aux; 2276d809e134SAlexei Starovoitov 2277d809e134SAlexei Starovoitov if (atomic64_dec_and_test(&aux->refcnt)) { 2278d809e134SAlexei Starovoitov if (in_irq() || irqs_disabled()) { 2279d809e134SAlexei Starovoitov INIT_WORK(&aux->work, bpf_prog_put_deferred); 2280d809e134SAlexei Starovoitov schedule_work(&aux->work); 2281d809e134SAlexei Starovoitov } else { 2282d809e134SAlexei Starovoitov bpf_prog_put_deferred(&aux->work); 2283d809e134SAlexei Starovoitov } 228409756af4SAlexei Starovoitov } 2285a67edbf4SDaniel Borkmann } 2286b16d9aa4SMartin KaFai Lau 2287b16d9aa4SMartin KaFai Lau void bpf_prog_put(struct bpf_prog *prog) 2288b16d9aa4SMartin KaFai Lau { 2289e7895f01SPaul Moore __bpf_prog_put(prog); 2290b16d9aa4SMartin KaFai Lau } 2291e2e9b654SDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_put); 229209756af4SAlexei Starovoitov 229309756af4SAlexei Starovoitov static int bpf_prog_release(struct inode *inode, struct file *filp) 229409756af4SAlexei Starovoitov { 229509756af4SAlexei Starovoitov struct bpf_prog *prog = filp->private_data; 229609756af4SAlexei Starovoitov 22971aacde3dSDaniel Borkmann bpf_prog_put(prog); 229809756af4SAlexei Starovoitov return 0; 229909756af4SAlexei Starovoitov } 230009756af4SAlexei Starovoitov 230161a0abaeSEric Dumazet struct bpf_prog_kstats { 230261a0abaeSEric Dumazet u64 nsecs; 230361a0abaeSEric Dumazet u64 cnt; 230461a0abaeSEric Dumazet u64 misses; 230561a0abaeSEric Dumazet }; 230661a0abaeSEric Dumazet 230705b24ff9SJiri Olsa void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog) 230805b24ff9SJiri Olsa { 230905b24ff9SJiri Olsa struct bpf_prog_stats *stats; 231005b24ff9SJiri Olsa unsigned int flags; 231105b24ff9SJiri Olsa 231205b24ff9SJiri Olsa stats = this_cpu_ptr(prog->stats); 231305b24ff9SJiri Olsa flags = u64_stats_update_begin_irqsave(&stats->syncp); 231405b24ff9SJiri Olsa u64_stats_inc(&stats->misses); 231505b24ff9SJiri Olsa u64_stats_update_end_irqrestore(&stats->syncp, flags); 231605b24ff9SJiri Olsa } 231705b24ff9SJiri Olsa 2318492ecee8SAlexei Starovoitov static void bpf_prog_get_stats(const struct bpf_prog *prog, 231961a0abaeSEric Dumazet struct bpf_prog_kstats *stats) 2320492ecee8SAlexei Starovoitov { 23219ed9e9baSAlexei Starovoitov u64 nsecs = 0, cnt = 0, misses = 0; 2322492ecee8SAlexei Starovoitov int cpu; 2323492ecee8SAlexei Starovoitov 2324492ecee8SAlexei Starovoitov for_each_possible_cpu(cpu) { 2325492ecee8SAlexei Starovoitov const struct bpf_prog_stats *st; 2326492ecee8SAlexei Starovoitov unsigned int start; 23279ed9e9baSAlexei Starovoitov u64 tnsecs, tcnt, tmisses; 2328492ecee8SAlexei Starovoitov 2329700d4796SAlexei Starovoitov st = per_cpu_ptr(prog->stats, cpu); 2330492ecee8SAlexei Starovoitov do { 233197c4090bSThomas Gleixner start = u64_stats_fetch_begin(&st->syncp); 233261a0abaeSEric Dumazet tnsecs = u64_stats_read(&st->nsecs); 233361a0abaeSEric Dumazet tcnt = u64_stats_read(&st->cnt); 233461a0abaeSEric Dumazet tmisses = u64_stats_read(&st->misses); 233597c4090bSThomas Gleixner } while (u64_stats_fetch_retry(&st->syncp, start)); 2336492ecee8SAlexei Starovoitov nsecs += tnsecs; 2337492ecee8SAlexei Starovoitov cnt += tcnt; 23389ed9e9baSAlexei Starovoitov misses += tmisses; 2339492ecee8SAlexei Starovoitov } 2340492ecee8SAlexei Starovoitov stats->nsecs = nsecs; 2341492ecee8SAlexei Starovoitov stats->cnt = cnt; 23429ed9e9baSAlexei Starovoitov stats->misses = misses; 2343492ecee8SAlexei Starovoitov } 2344492ecee8SAlexei Starovoitov 23457bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 23467bd509e3SDaniel Borkmann static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 23477bd509e3SDaniel Borkmann { 23487bd509e3SDaniel Borkmann const struct bpf_prog *prog = filp->private_data; 2349f1f7714eSDaniel Borkmann char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 235061a0abaeSEric Dumazet struct bpf_prog_kstats stats; 23517bd509e3SDaniel Borkmann 2352492ecee8SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 2353f1f7714eSDaniel Borkmann bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 23547bd509e3SDaniel Borkmann seq_printf(m, 23557bd509e3SDaniel Borkmann "prog_type:\t%u\n" 23567bd509e3SDaniel Borkmann "prog_jited:\t%u\n" 2357f1f7714eSDaniel Borkmann "prog_tag:\t%s\n" 23584316b409SDaniel Borkmann "memlock:\t%llu\n" 2359492ecee8SAlexei Starovoitov "prog_id:\t%u\n" 2360492ecee8SAlexei Starovoitov "run_time_ns:\t%llu\n" 23619ed9e9baSAlexei Starovoitov "run_cnt:\t%llu\n" 2362aba64c7dSDave Marchevsky "recursion_misses:\t%llu\n" 2363aba64c7dSDave Marchevsky "verified_insns:\t%u\n", 23647bd509e3SDaniel Borkmann prog->type, 23657bd509e3SDaniel Borkmann prog->jited, 2366f1f7714eSDaniel Borkmann prog_tag, 23674316b409SDaniel Borkmann prog->pages * 1ULL << PAGE_SHIFT, 2368492ecee8SAlexei Starovoitov prog->aux->id, 2369492ecee8SAlexei Starovoitov stats.nsecs, 23709ed9e9baSAlexei Starovoitov stats.cnt, 2371aba64c7dSDave Marchevsky stats.misses, 2372aba64c7dSDave Marchevsky prog->aux->verified_insns); 23737bd509e3SDaniel Borkmann } 23747bd509e3SDaniel Borkmann #endif 23757bd509e3SDaniel Borkmann 2376f66e448cSChenbo Feng const struct file_operations bpf_prog_fops = { 23777bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 23787bd509e3SDaniel Borkmann .show_fdinfo = bpf_prog_show_fdinfo, 23797bd509e3SDaniel Borkmann #endif 238009756af4SAlexei Starovoitov .release = bpf_prog_release, 23816e71b04aSChenbo Feng .read = bpf_dummy_read, 23826e71b04aSChenbo Feng .write = bpf_dummy_write, 238309756af4SAlexei Starovoitov }; 238409756af4SAlexei Starovoitov 2385b2197755SDaniel Borkmann int bpf_prog_new_fd(struct bpf_prog *prog) 2386aa79781bSDaniel Borkmann { 2387afdb09c7SChenbo Feng int ret; 2388afdb09c7SChenbo Feng 2389afdb09c7SChenbo Feng ret = security_bpf_prog(prog); 2390afdb09c7SChenbo Feng if (ret < 0) 2391afdb09c7SChenbo Feng return ret; 2392afdb09c7SChenbo Feng 2393aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 2394aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC); 2395aa79781bSDaniel Borkmann } 2396aa79781bSDaniel Borkmann 2397113214beSDaniel Borkmann static struct bpf_prog *____bpf_prog_get(struct fd f) 239809756af4SAlexei Starovoitov { 239909756af4SAlexei Starovoitov if (!f.file) 240009756af4SAlexei Starovoitov return ERR_PTR(-EBADF); 240109756af4SAlexei Starovoitov if (f.file->f_op != &bpf_prog_fops) { 240209756af4SAlexei Starovoitov fdput(f); 240309756af4SAlexei Starovoitov return ERR_PTR(-EINVAL); 240409756af4SAlexei Starovoitov } 240509756af4SAlexei Starovoitov 2406c2101297SDaniel Borkmann return f.file->private_data; 240709756af4SAlexei Starovoitov } 240809756af4SAlexei Starovoitov 240985192dbfSAndrii Nakryiko void bpf_prog_add(struct bpf_prog *prog, int i) 241092117d84SAlexei Starovoitov { 241185192dbfSAndrii Nakryiko atomic64_add(i, &prog->aux->refcnt); 241292117d84SAlexei Starovoitov } 241359d3656dSBrenden Blanco EXPORT_SYMBOL_GPL(bpf_prog_add); 241459d3656dSBrenden Blanco 2415c540594fSDaniel Borkmann void bpf_prog_sub(struct bpf_prog *prog, int i) 2416c540594fSDaniel Borkmann { 2417c540594fSDaniel Borkmann /* Only to be used for undoing previous bpf_prog_add() in some 2418c540594fSDaniel Borkmann * error path. We still know that another entity in our call 2419c540594fSDaniel Borkmann * path holds a reference to the program, thus atomic_sub() can 2420c540594fSDaniel Borkmann * be safely used in such cases! 2421c540594fSDaniel Borkmann */ 242285192dbfSAndrii Nakryiko WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0); 2423c540594fSDaniel Borkmann } 2424c540594fSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_sub); 2425c540594fSDaniel Borkmann 242685192dbfSAndrii Nakryiko void bpf_prog_inc(struct bpf_prog *prog) 242759d3656dSBrenden Blanco { 242885192dbfSAndrii Nakryiko atomic64_inc(&prog->aux->refcnt); 242959d3656dSBrenden Blanco } 243097bc402dSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_inc); 243192117d84SAlexei Starovoitov 2432b16d9aa4SMartin KaFai Lau /* prog_idr_lock should have been held */ 2433a6f6df69SJohn Fastabend struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 2434b16d9aa4SMartin KaFai Lau { 2435b16d9aa4SMartin KaFai Lau int refold; 2436b16d9aa4SMartin KaFai Lau 243785192dbfSAndrii Nakryiko refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0); 2438b16d9aa4SMartin KaFai Lau 2439b16d9aa4SMartin KaFai Lau if (!refold) 2440b16d9aa4SMartin KaFai Lau return ERR_PTR(-ENOENT); 2441b16d9aa4SMartin KaFai Lau 2442b16d9aa4SMartin KaFai Lau return prog; 2443b16d9aa4SMartin KaFai Lau } 2444a6f6df69SJohn Fastabend EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 2445b16d9aa4SMartin KaFai Lau 2446040ee692SAl Viro bool bpf_prog_get_ok(struct bpf_prog *prog, 2447288b3de5SJakub Kicinski enum bpf_prog_type *attach_type, bool attach_drv) 2448248f346fSJakub Kicinski { 2449288b3de5SJakub Kicinski /* not an attachment, just a refcount inc, always allow */ 2450288b3de5SJakub Kicinski if (!attach_type) 2451288b3de5SJakub Kicinski return true; 2452248f346fSJakub Kicinski 2453248f346fSJakub Kicinski if (prog->type != *attach_type) 2454248f346fSJakub Kicinski return false; 24559d03ebc7SStanislav Fomichev if (bpf_prog_is_offloaded(prog->aux) && !attach_drv) 2456248f346fSJakub Kicinski return false; 2457248f346fSJakub Kicinski 2458248f346fSJakub Kicinski return true; 2459248f346fSJakub Kicinski } 2460248f346fSJakub Kicinski 2461248f346fSJakub Kicinski static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, 2462288b3de5SJakub Kicinski bool attach_drv) 246309756af4SAlexei Starovoitov { 246409756af4SAlexei Starovoitov struct fd f = fdget(ufd); 246509756af4SAlexei Starovoitov struct bpf_prog *prog; 246609756af4SAlexei Starovoitov 2467113214beSDaniel Borkmann prog = ____bpf_prog_get(f); 246809756af4SAlexei Starovoitov if (IS_ERR(prog)) 246909756af4SAlexei Starovoitov return prog; 2470288b3de5SJakub Kicinski if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) { 2471113214beSDaniel Borkmann prog = ERR_PTR(-EINVAL); 2472113214beSDaniel Borkmann goto out; 2473113214beSDaniel Borkmann } 247409756af4SAlexei Starovoitov 247585192dbfSAndrii Nakryiko bpf_prog_inc(prog); 2476113214beSDaniel Borkmann out: 247709756af4SAlexei Starovoitov fdput(f); 247809756af4SAlexei Starovoitov return prog; 247909756af4SAlexei Starovoitov } 2480113214beSDaniel Borkmann 2481113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get(u32 ufd) 2482113214beSDaniel Borkmann { 2483288b3de5SJakub Kicinski return __bpf_prog_get(ufd, NULL, false); 2484113214beSDaniel Borkmann } 2485113214beSDaniel Borkmann 2486248f346fSJakub Kicinski struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 2487288b3de5SJakub Kicinski bool attach_drv) 2488248f346fSJakub Kicinski { 24894d220ed0SAlexei Starovoitov return __bpf_prog_get(ufd, &type, attach_drv); 2490248f346fSJakub Kicinski } 24916c8dfe21SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev); 2492248f346fSJakub Kicinski 2493aac3fc32SAndrey Ignatov /* Initially all BPF programs could be loaded w/o specifying 2494aac3fc32SAndrey Ignatov * expected_attach_type. Later for some of them specifying expected_attach_type 2495aac3fc32SAndrey Ignatov * at load time became required so that program could be validated properly. 2496aac3fc32SAndrey Ignatov * Programs of types that are allowed to be loaded both w/ and w/o (for 2497aac3fc32SAndrey Ignatov * backward compatibility) expected_attach_type, should have the default attach 2498aac3fc32SAndrey Ignatov * type assigned to expected_attach_type for the latter case, so that it can be 2499aac3fc32SAndrey Ignatov * validated later at attach time. 2500aac3fc32SAndrey Ignatov * 2501aac3fc32SAndrey Ignatov * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if 2502aac3fc32SAndrey Ignatov * prog type requires it but has some attach types that have to be backward 2503aac3fc32SAndrey Ignatov * compatible. 2504aac3fc32SAndrey Ignatov */ 2505aac3fc32SAndrey Ignatov static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) 2506aac3fc32SAndrey Ignatov { 2507aac3fc32SAndrey Ignatov switch (attr->prog_type) { 2508aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 2509aac3fc32SAndrey Ignatov /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't 2510aac3fc32SAndrey Ignatov * exist so checking for non-zero is the way to go here. 2511aac3fc32SAndrey Ignatov */ 2512aac3fc32SAndrey Ignatov if (!attr->expected_attach_type) 2513aac3fc32SAndrey Ignatov attr->expected_attach_type = 2514aac3fc32SAndrey Ignatov BPF_CGROUP_INET_SOCK_CREATE; 2515aac3fc32SAndrey Ignatov break; 2516d5e4ddaeSKuniyuki Iwashima case BPF_PROG_TYPE_SK_REUSEPORT: 2517d5e4ddaeSKuniyuki Iwashima if (!attr->expected_attach_type) 2518d5e4ddaeSKuniyuki Iwashima attr->expected_attach_type = 2519d5e4ddaeSKuniyuki Iwashima BPF_SK_REUSEPORT_SELECT; 2520d5e4ddaeSKuniyuki Iwashima break; 2521aac3fc32SAndrey Ignatov } 2522aac3fc32SAndrey Ignatov } 2523aac3fc32SAndrey Ignatov 25245e43f899SAndrey Ignatov static int 2525ccfe29ebSAlexei Starovoitov bpf_prog_load_check_attach(enum bpf_prog_type prog_type, 2526ccfe29ebSAlexei Starovoitov enum bpf_attach_type expected_attach_type, 2527290248a5SAndrii Nakryiko struct btf *attach_btf, u32 btf_id, 2528290248a5SAndrii Nakryiko struct bpf_prog *dst_prog) 25295e43f899SAndrey Ignatov { 253027ae7997SMartin KaFai Lau if (btf_id) { 2531c108e3c1SAlexei Starovoitov if (btf_id > BTF_MAX_TYPE) 2532c108e3c1SAlexei Starovoitov return -EINVAL; 253327ae7997SMartin KaFai Lau 2534290248a5SAndrii Nakryiko if (!attach_btf && !dst_prog) 2535290248a5SAndrii Nakryiko return -EINVAL; 2536290248a5SAndrii Nakryiko 253727ae7997SMartin KaFai Lau switch (prog_type) { 253827ae7997SMartin KaFai Lau case BPF_PROG_TYPE_TRACING: 25399e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 254027ae7997SMartin KaFai Lau case BPF_PROG_TYPE_STRUCT_OPS: 2541be8704ffSAlexei Starovoitov case BPF_PROG_TYPE_EXT: 2542c108e3c1SAlexei Starovoitov break; 2543c108e3c1SAlexei Starovoitov default: 2544c108e3c1SAlexei Starovoitov return -EINVAL; 2545c108e3c1SAlexei Starovoitov } 254627ae7997SMartin KaFai Lau } 254727ae7997SMartin KaFai Lau 2548290248a5SAndrii Nakryiko if (attach_btf && (!btf_id || dst_prog)) 2549290248a5SAndrii Nakryiko return -EINVAL; 2550290248a5SAndrii Nakryiko 2551290248a5SAndrii Nakryiko if (dst_prog && prog_type != BPF_PROG_TYPE_TRACING && 2552be8704ffSAlexei Starovoitov prog_type != BPF_PROG_TYPE_EXT) 255327ae7997SMartin KaFai Lau return -EINVAL; 2554c108e3c1SAlexei Starovoitov 2555c108e3c1SAlexei Starovoitov switch (prog_type) { 2556aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 2557aac3fc32SAndrey Ignatov switch (expected_attach_type) { 2558aac3fc32SAndrey Ignatov case BPF_CGROUP_INET_SOCK_CREATE: 2559f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 2560aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 2561aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 2562aac3fc32SAndrey Ignatov return 0; 2563aac3fc32SAndrey Ignatov default: 2564aac3fc32SAndrey Ignatov return -EINVAL; 2565aac3fc32SAndrey Ignatov } 25664fbac77dSAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 25674fbac77dSAndrey Ignatov switch (expected_attach_type) { 25684fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 25694fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 2570d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 2571d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 2572859051ddSDaan De Meyer case BPF_CGROUP_UNIX_CONNECT: 25731b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 25741b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 2575859051ddSDaan De Meyer case BPF_CGROUP_UNIX_GETPEERNAME: 25761b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 25771b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 2578859051ddSDaan De Meyer case BPF_CGROUP_UNIX_GETSOCKNAME: 25791cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 25801cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 2581859051ddSDaan De Meyer case BPF_CGROUP_UNIX_SENDMSG: 2582983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG: 2583983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG: 2584859051ddSDaan De Meyer case BPF_CGROUP_UNIX_RECVMSG: 25855e43f899SAndrey Ignatov return 0; 25864fbac77dSAndrey Ignatov default: 25874fbac77dSAndrey Ignatov return -EINVAL; 25884fbac77dSAndrey Ignatov } 25895cf1e914Sbrakmo case BPF_PROG_TYPE_CGROUP_SKB: 25905cf1e914Sbrakmo switch (expected_attach_type) { 25915cf1e914Sbrakmo case BPF_CGROUP_INET_INGRESS: 25925cf1e914Sbrakmo case BPF_CGROUP_INET_EGRESS: 25935cf1e914Sbrakmo return 0; 25945cf1e914Sbrakmo default: 25955cf1e914Sbrakmo return -EINVAL; 25965cf1e914Sbrakmo } 25970d01da6aSStanislav Fomichev case BPF_PROG_TYPE_CGROUP_SOCKOPT: 25980d01da6aSStanislav Fomichev switch (expected_attach_type) { 25990d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT: 26000d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT: 26010d01da6aSStanislav Fomichev return 0; 26020d01da6aSStanislav Fomichev default: 26030d01da6aSStanislav Fomichev return -EINVAL; 26040d01da6aSStanislav Fomichev } 2605e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP: 2606e9ddbb77SJakub Sitnicki if (expected_attach_type == BPF_SK_LOOKUP) 2607e9ddbb77SJakub Sitnicki return 0; 2608e9ddbb77SJakub Sitnicki return -EINVAL; 2609d5e4ddaeSKuniyuki Iwashima case BPF_PROG_TYPE_SK_REUSEPORT: 2610d5e4ddaeSKuniyuki Iwashima switch (expected_attach_type) { 2611d5e4ddaeSKuniyuki Iwashima case BPF_SK_REUSEPORT_SELECT: 2612d5e4ddaeSKuniyuki Iwashima case BPF_SK_REUSEPORT_SELECT_OR_MIGRATE: 2613d5e4ddaeSKuniyuki Iwashima return 0; 2614d5e4ddaeSKuniyuki Iwashima default: 2615d5e4ddaeSKuniyuki Iwashima return -EINVAL; 2616d5e4ddaeSKuniyuki Iwashima } 2617132328e8SFlorian Westphal case BPF_PROG_TYPE_NETFILTER: 2618132328e8SFlorian Westphal if (expected_attach_type == BPF_NETFILTER) 2619132328e8SFlorian Westphal return 0; 2620132328e8SFlorian Westphal return -EINVAL; 262179a7f8bdSAlexei Starovoitov case BPF_PROG_TYPE_SYSCALL: 2622be8704ffSAlexei Starovoitov case BPF_PROG_TYPE_EXT: 2623be8704ffSAlexei Starovoitov if (expected_attach_type) 2624be8704ffSAlexei Starovoitov return -EINVAL; 2625df561f66SGustavo A. R. Silva fallthrough; 26264fbac77dSAndrey Ignatov default: 26274fbac77dSAndrey Ignatov return 0; 26284fbac77dSAndrey Ignatov } 26295e43f899SAndrey Ignatov } 26305e43f899SAndrey Ignatov 26312c78ee89SAlexei Starovoitov static bool is_net_admin_prog_type(enum bpf_prog_type prog_type) 26322c78ee89SAlexei Starovoitov { 26332c78ee89SAlexei Starovoitov switch (prog_type) { 26342c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SCHED_CLS: 26352c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SCHED_ACT: 26362c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_XDP: 26372c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_IN: 26382c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_OUT: 26392c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_XMIT: 26402c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_SEG6LOCAL: 26412c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_SKB: 26422c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_MSG: 26432c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_FLOW_DISSECTOR: 26442c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_DEVICE: 26452c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCK: 26462c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 26472c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCKOPT: 26482c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SYSCTL: 26492c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SOCK_OPS: 26502c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_EXT: /* extends any prog */ 265184601d6eSFlorian Westphal case BPF_PROG_TYPE_NETFILTER: 26522c78ee89SAlexei Starovoitov return true; 26532c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SKB: 26542c78ee89SAlexei Starovoitov /* always unpriv */ 26552c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_REUSEPORT: 26562c78ee89SAlexei Starovoitov /* equivalent to SOCKET_FILTER. need CAP_BPF only */ 26572c78ee89SAlexei Starovoitov default: 26582c78ee89SAlexei Starovoitov return false; 26592c78ee89SAlexei Starovoitov } 26602c78ee89SAlexei Starovoitov } 26612c78ee89SAlexei Starovoitov 26622c78ee89SAlexei Starovoitov static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) 26632c78ee89SAlexei Starovoitov { 26642c78ee89SAlexei Starovoitov switch (prog_type) { 26652c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_KPROBE: 26662c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_TRACEPOINT: 26672c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_PERF_EVENT: 26682c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_RAW_TRACEPOINT: 26692c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 26702c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_TRACING: 26712c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LSM: 26722c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_STRUCT_OPS: /* has access to struct sock */ 26732c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_EXT: /* extends any prog */ 26742c78ee89SAlexei Starovoitov return true; 26752c78ee89SAlexei Starovoitov default: 26762c78ee89SAlexei Starovoitov return false; 26772c78ee89SAlexei Starovoitov } 26782c78ee89SAlexei Starovoitov } 26792c78ee89SAlexei Starovoitov 268009756af4SAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 2681caf8f28eSAndrii Nakryiko #define BPF_PROG_LOAD_LAST_FIELD prog_token_fd 268209756af4SAlexei Starovoitov 268347a71c1fSAndrii Nakryiko static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) 268409756af4SAlexei Starovoitov { 268509756af4SAlexei Starovoitov enum bpf_prog_type type = attr->prog_type; 2686290248a5SAndrii Nakryiko struct bpf_prog *prog, *dst_prog = NULL; 2687290248a5SAndrii Nakryiko struct btf *attach_btf = NULL; 2688caf8f28eSAndrii Nakryiko struct bpf_token *token = NULL; 2689caf8f28eSAndrii Nakryiko bool bpf_cap; 269009756af4SAlexei Starovoitov int err; 269109756af4SAlexei Starovoitov char license[128]; 269209756af4SAlexei Starovoitov 269309756af4SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_LOAD)) 269409756af4SAlexei Starovoitov return -EINVAL; 269509756af4SAlexei Starovoitov 2696c240eff6SJiong Wang if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | 2697c240eff6SJiong Wang BPF_F_ANY_ALIGNMENT | 269810d274e8SAlexei Starovoitov BPF_F_TEST_STATE_FREQ | 26991e6c62a8SAlexei Starovoitov BPF_F_SLEEPABLE | 2700c2f2cdbeSLorenzo Bianconi BPF_F_TEST_RND_HI32 | 27012b3486bcSStanislav Fomichev BPF_F_XDP_HAS_FRAGS | 27025f99f312SAndrii Nakryiko BPF_F_XDP_DEV_BOUND_ONLY | 2703caf8f28eSAndrii Nakryiko BPF_F_TEST_REG_INVARIANTS | 2704caf8f28eSAndrii Nakryiko BPF_F_TOKEN_FD)) 2705e07b98d9SDavid S. Miller return -EINVAL; 2706e07b98d9SDavid S. Miller 2707caf8f28eSAndrii Nakryiko bpf_prog_load_fixup_attach_type(attr); 2708caf8f28eSAndrii Nakryiko 2709caf8f28eSAndrii Nakryiko if (attr->prog_flags & BPF_F_TOKEN_FD) { 2710caf8f28eSAndrii Nakryiko token = bpf_token_get_from_fd(attr->prog_token_fd); 2711caf8f28eSAndrii Nakryiko if (IS_ERR(token)) 2712caf8f28eSAndrii Nakryiko return PTR_ERR(token); 2713caf8f28eSAndrii Nakryiko /* if current token doesn't grant prog loading permissions, 2714caf8f28eSAndrii Nakryiko * then we can't use this token, so ignore it and rely on 2715caf8f28eSAndrii Nakryiko * system-wide capabilities checks 2716caf8f28eSAndrii Nakryiko */ 2717caf8f28eSAndrii Nakryiko if (!bpf_token_allow_cmd(token, BPF_PROG_LOAD) || 2718caf8f28eSAndrii Nakryiko !bpf_token_allow_prog_type(token, attr->prog_type, 2719caf8f28eSAndrii Nakryiko attr->expected_attach_type)) { 2720caf8f28eSAndrii Nakryiko bpf_token_put(token); 2721caf8f28eSAndrii Nakryiko token = NULL; 2722caf8f28eSAndrii Nakryiko } 2723caf8f28eSAndrii Nakryiko } 2724caf8f28eSAndrii Nakryiko 2725caf8f28eSAndrii Nakryiko bpf_cap = bpf_token_capable(token, CAP_BPF); 2726caf8f28eSAndrii Nakryiko err = -EPERM; 2727caf8f28eSAndrii Nakryiko 2728e9ee9efcSDavid Miller if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && 2729e9ee9efcSDavid Miller (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && 2730caf8f28eSAndrii Nakryiko !bpf_cap) 2731caf8f28eSAndrii Nakryiko goto put_token; 2732e9ee9efcSDavid Miller 27331d28635aSAndrii Nakryiko /* Intent here is for unprivileged_bpf_disabled to block BPF program 27341d28635aSAndrii Nakryiko * creation for unprivileged users; other actions depend 27351d28635aSAndrii Nakryiko * on fd availability and access to bpffs, so are dependent on 27361d28635aSAndrii Nakryiko * object creation success. Even with unprivileged BPF disabled, 27371d28635aSAndrii Nakryiko * capability checks are still carried out for these 27381d28635aSAndrii Nakryiko * and other operations. 27391d28635aSAndrii Nakryiko */ 2740caf8f28eSAndrii Nakryiko if (sysctl_unprivileged_bpf_disabled && !bpf_cap) 2741caf8f28eSAndrii Nakryiko goto put_token; 274209756af4SAlexei Starovoitov 2743c04c0d2bSAlexei Starovoitov if (attr->insn_cnt == 0 || 2744caf8f28eSAndrii Nakryiko attr->insn_cnt > (bpf_cap ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) { 2745caf8f28eSAndrii Nakryiko err = -E2BIG; 2746caf8f28eSAndrii Nakryiko goto put_token; 2747caf8f28eSAndrii Nakryiko } 274880b7d819SChenbo Feng if (type != BPF_PROG_TYPE_SOCKET_FILTER && 274980b7d819SChenbo Feng type != BPF_PROG_TYPE_CGROUP_SKB && 2750caf8f28eSAndrii Nakryiko !bpf_cap) 2751caf8f28eSAndrii Nakryiko goto put_token; 27522c78ee89SAlexei Starovoitov 2753caf8f28eSAndrii Nakryiko if (is_net_admin_prog_type(type) && !bpf_token_capable(token, CAP_NET_ADMIN)) 2754caf8f28eSAndrii Nakryiko goto put_token; 2755caf8f28eSAndrii Nakryiko if (is_perfmon_prog_type(type) && !bpf_token_capable(token, CAP_PERFMON)) 2756caf8f28eSAndrii Nakryiko goto put_token; 27571be7f75dSAlexei Starovoitov 2758290248a5SAndrii Nakryiko /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog 2759290248a5SAndrii Nakryiko * or btf, we need to check which one it is 2760290248a5SAndrii Nakryiko */ 2761290248a5SAndrii Nakryiko if (attr->attach_prog_fd) { 2762290248a5SAndrii Nakryiko dst_prog = bpf_prog_get(attr->attach_prog_fd); 2763290248a5SAndrii Nakryiko if (IS_ERR(dst_prog)) { 2764290248a5SAndrii Nakryiko dst_prog = NULL; 2765290248a5SAndrii Nakryiko attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd); 2766caf8f28eSAndrii Nakryiko if (IS_ERR(attach_btf)) { 2767caf8f28eSAndrii Nakryiko err = -EINVAL; 2768caf8f28eSAndrii Nakryiko goto put_token; 2769caf8f28eSAndrii Nakryiko } 2770290248a5SAndrii Nakryiko if (!btf_is_kernel(attach_btf)) { 27718bdd8e27SAndrii Nakryiko /* attaching through specifying bpf_prog's BTF 27728bdd8e27SAndrii Nakryiko * objects directly might be supported eventually 27738bdd8e27SAndrii Nakryiko */ 2774290248a5SAndrii Nakryiko btf_put(attach_btf); 2775caf8f28eSAndrii Nakryiko err = -ENOTSUPP; 2776caf8f28eSAndrii Nakryiko goto put_token; 2777290248a5SAndrii Nakryiko } 2778290248a5SAndrii Nakryiko } 2779290248a5SAndrii Nakryiko } else if (attr->attach_btf_id) { 2780290248a5SAndrii Nakryiko /* fall back to vmlinux BTF, if BTF type ID is specified */ 2781290248a5SAndrii Nakryiko attach_btf = bpf_get_btf_vmlinux(); 2782caf8f28eSAndrii Nakryiko if (IS_ERR(attach_btf)) { 2783caf8f28eSAndrii Nakryiko err = PTR_ERR(attach_btf); 2784caf8f28eSAndrii Nakryiko goto put_token; 2785caf8f28eSAndrii Nakryiko } 2786caf8f28eSAndrii Nakryiko if (!attach_btf) { 2787caf8f28eSAndrii Nakryiko err = -EINVAL; 2788caf8f28eSAndrii Nakryiko goto put_token; 2789caf8f28eSAndrii Nakryiko } 2790290248a5SAndrii Nakryiko btf_get(attach_btf); 2791290248a5SAndrii Nakryiko } 2792290248a5SAndrii Nakryiko 2793ccfe29ebSAlexei Starovoitov if (bpf_prog_load_check_attach(type, attr->expected_attach_type, 2794290248a5SAndrii Nakryiko attach_btf, attr->attach_btf_id, 2795290248a5SAndrii Nakryiko dst_prog)) { 2796290248a5SAndrii Nakryiko if (dst_prog) 2797290248a5SAndrii Nakryiko bpf_prog_put(dst_prog); 2798290248a5SAndrii Nakryiko if (attach_btf) 2799290248a5SAndrii Nakryiko btf_put(attach_btf); 2800caf8f28eSAndrii Nakryiko err = -EINVAL; 2801caf8f28eSAndrii Nakryiko goto put_token; 2802290248a5SAndrii Nakryiko } 28035e43f899SAndrey Ignatov 280409756af4SAlexei Starovoitov /* plain bpf_prog allocation */ 280509756af4SAlexei Starovoitov prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 2806290248a5SAndrii Nakryiko if (!prog) { 2807290248a5SAndrii Nakryiko if (dst_prog) 2808290248a5SAndrii Nakryiko bpf_prog_put(dst_prog); 2809290248a5SAndrii Nakryiko if (attach_btf) 2810290248a5SAndrii Nakryiko btf_put(attach_btf); 2811caf8f28eSAndrii Nakryiko err = -EINVAL; 2812caf8f28eSAndrii Nakryiko goto put_token; 2813290248a5SAndrii Nakryiko } 281409756af4SAlexei Starovoitov 28155e43f899SAndrey Ignatov prog->expected_attach_type = attr->expected_attach_type; 281666c84731SAndrii Nakryiko prog->sleepable = !!(attr->prog_flags & BPF_F_SLEEPABLE); 2817290248a5SAndrii Nakryiko prog->aux->attach_btf = attach_btf; 2818ccfe29ebSAlexei Starovoitov prog->aux->attach_btf_id = attr->attach_btf_id; 28193aac1eadSToke Høiland-Jørgensen prog->aux->dst_prog = dst_prog; 28202b3486bcSStanislav Fomichev prog->aux->dev_bound = !!attr->prog_ifindex; 2821c2f2cdbeSLorenzo Bianconi prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; 28229a18eedbSJakub Kicinski 2823caf8f28eSAndrii Nakryiko /* move token into prog->aux, reuse taken refcnt */ 2824caf8f28eSAndrii Nakryiko prog->aux->token = token; 2825caf8f28eSAndrii Nakryiko token = NULL; 2826caf8f28eSAndrii Nakryiko 28273ac1f01bSRoman Gushchin prog->aux->user = get_current_user(); 282809756af4SAlexei Starovoitov prog->len = attr->insn_cnt; 282909756af4SAlexei Starovoitov 283009756af4SAlexei Starovoitov err = -EFAULT; 2831af2ac3e1SAlexei Starovoitov if (copy_from_bpfptr(prog->insns, 2832af2ac3e1SAlexei Starovoitov make_bpfptr(attr->insns, uattr.is_kernel), 2833aafe6ae9SDaniel Borkmann bpf_prog_insn_size(prog)) != 0) 28341b67772eSAndrii Nakryiko goto free_prog; 28357f6719f7SAndrii Nakryiko /* copy eBPF program license from user space */ 28367f6719f7SAndrii Nakryiko if (strncpy_from_bpfptr(license, 28377f6719f7SAndrii Nakryiko make_bpfptr(attr->license, uattr.is_kernel), 28387f6719f7SAndrii Nakryiko sizeof(license) - 1) < 0) 28391b67772eSAndrii Nakryiko goto free_prog; 28407f6719f7SAndrii Nakryiko license[sizeof(license) - 1] = 0; 28417f6719f7SAndrii Nakryiko 28427f6719f7SAndrii Nakryiko /* eBPF programs must be GPL compatible to use GPL-ed functions */ 28437f6719f7SAndrii Nakryiko prog->gpl_compatible = license_is_gpl_compatible(license) ? 1 : 0; 284409756af4SAlexei Starovoitov 284509756af4SAlexei Starovoitov prog->orig_prog = NULL; 2846a91263d5SDaniel Borkmann prog->jited = 0; 284709756af4SAlexei Starovoitov 284885192dbfSAndrii Nakryiko atomic64_set(&prog->aux->refcnt, 1); 284909756af4SAlexei Starovoitov 28509a18eedbSJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 28512b3486bcSStanislav Fomichev err = bpf_prog_dev_bound_init(prog, attr); 2852ab3f0063SJakub Kicinski if (err) 28531b67772eSAndrii Nakryiko goto free_prog; 2854ab3f0063SJakub Kicinski } 2855ab3f0063SJakub Kicinski 2856fd7c211dSToke Høiland-Jørgensen if (type == BPF_PROG_TYPE_EXT && dst_prog && 2857fd7c211dSToke Høiland-Jørgensen bpf_prog_is_dev_bound(dst_prog->aux)) { 2858fd7c211dSToke Høiland-Jørgensen err = bpf_prog_dev_bound_inherit(prog, dst_prog); 2859cb4d2b3fSMartin KaFai Lau if (err) 28601b67772eSAndrii Nakryiko goto free_prog; 2861cb4d2b3fSMartin KaFai Lau } 2862cb4d2b3fSMartin KaFai Lau 286319bfcdf9SDmitrii Dolgov /* 286419bfcdf9SDmitrii Dolgov * Bookkeeping for managing the program attachment chain. 286519bfcdf9SDmitrii Dolgov * 286619bfcdf9SDmitrii Dolgov * It might be tempting to set attach_tracing_prog flag at the attachment 286719bfcdf9SDmitrii Dolgov * time, but this will not prevent from loading bunch of tracing prog 286819bfcdf9SDmitrii Dolgov * first, then attach them one to another. 286919bfcdf9SDmitrii Dolgov * 287019bfcdf9SDmitrii Dolgov * The flag attach_tracing_prog is set for the whole program lifecycle, and 287119bfcdf9SDmitrii Dolgov * doesn't have to be cleared in bpf_tracing_link_release, since tracing 287219bfcdf9SDmitrii Dolgov * programs cannot change attachment target. 287319bfcdf9SDmitrii Dolgov */ 287419bfcdf9SDmitrii Dolgov if (type == BPF_PROG_TYPE_TRACING && dst_prog && 287519bfcdf9SDmitrii Dolgov dst_prog->type == BPF_PROG_TYPE_TRACING) { 287619bfcdf9SDmitrii Dolgov prog->aux->attach_tracing_prog = true; 287719bfcdf9SDmitrii Dolgov } 287819bfcdf9SDmitrii Dolgov 287909756af4SAlexei Starovoitov /* find program type: socket_filter vs tracing_filter */ 288009756af4SAlexei Starovoitov err = find_prog_type(type, prog); 288109756af4SAlexei Starovoitov if (err < 0) 28821b67772eSAndrii Nakryiko goto free_prog; 288309756af4SAlexei Starovoitov 28849285ec4cSJason A. Donenfeld prog->aux->load_time = ktime_get_boottime_ns(); 28858e7ae251SMartin KaFai Lau err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, 28868e7ae251SMartin KaFai Lau sizeof(attr->prog_name)); 28878e7ae251SMartin KaFai Lau if (err < 0) 28881b67772eSAndrii Nakryiko goto free_prog; 28891b67772eSAndrii Nakryiko 28901b67772eSAndrii Nakryiko err = security_bpf_prog_load(prog, attr, token); 28911b67772eSAndrii Nakryiko if (err) 28923ac1f01bSRoman Gushchin goto free_prog_sec; 2893cb4d2b3fSMartin KaFai Lau 289409756af4SAlexei Starovoitov /* run eBPF verifier */ 289547a71c1fSAndrii Nakryiko err = bpf_check(&prog, attr, uattr, uattr_size); 289609756af4SAlexei Starovoitov if (err < 0) 289709756af4SAlexei Starovoitov goto free_used_maps; 289809756af4SAlexei Starovoitov 2899d1c55ab5SDaniel Borkmann prog = bpf_prog_select_runtime(prog, &err); 290004fd61abSAlexei Starovoitov if (err < 0) 290104fd61abSAlexei Starovoitov goto free_used_maps; 290209756af4SAlexei Starovoitov 2903dc4bb0e2SMartin KaFai Lau err = bpf_prog_alloc_id(prog); 2904dc4bb0e2SMartin KaFai Lau if (err) 2905dc4bb0e2SMartin KaFai Lau goto free_used_maps; 2906dc4bb0e2SMartin KaFai Lau 2907c751798aSDaniel Borkmann /* Upon success of bpf_prog_alloc_id(), the BPF prog is 2908c751798aSDaniel Borkmann * effectively publicly exposed. However, retrieving via 2909c751798aSDaniel Borkmann * bpf_prog_get_fd_by_id() will take another reference, 2910c751798aSDaniel Borkmann * therefore it cannot be gone underneath us. 2911c751798aSDaniel Borkmann * 2912c751798aSDaniel Borkmann * Only for the time /after/ successful bpf_prog_new_fd() 2913c751798aSDaniel Borkmann * and before returning to userspace, we might just hold 2914c751798aSDaniel Borkmann * one reference and any parallel close on that fd could 2915c751798aSDaniel Borkmann * rip everything out. Hence, below notifications must 2916c751798aSDaniel Borkmann * happen before bpf_prog_new_fd(). 2917c751798aSDaniel Borkmann * 2918c751798aSDaniel Borkmann * Also, any failure handling from this point onwards must 2919c751798aSDaniel Borkmann * be using bpf_prog_put() given the program is exposed. 2920b16d9aa4SMartin KaFai Lau */ 292174451e66SDaniel Borkmann bpf_prog_kallsyms_add(prog); 29226ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); 2923bae141f5SDaniel Borkmann bpf_audit_prog(prog, BPF_AUDIT_LOAD); 2924c751798aSDaniel Borkmann 2925c751798aSDaniel Borkmann err = bpf_prog_new_fd(prog); 2926c751798aSDaniel Borkmann if (err < 0) 2927c751798aSDaniel Borkmann bpf_prog_put(prog); 292809756af4SAlexei Starovoitov return err; 292909756af4SAlexei Starovoitov 293009756af4SAlexei Starovoitov free_used_maps: 2931cd7455f1SDaniel Borkmann /* In case we have subprogs, we need to wait for a grace 2932cd7455f1SDaniel Borkmann * period before we can tear down JIT memory since symbols 2933cd7455f1SDaniel Borkmann * are already exposed under kallsyms. 2934cd7455f1SDaniel Borkmann */ 2935335d1c5bSKumar Kartikeya Dwivedi __bpf_prog_put_noref(prog, prog->aux->real_func_cnt); 2936cd7455f1SDaniel Borkmann return err; 29371b67772eSAndrii Nakryiko 2938afdb09c7SChenbo Feng free_prog_sec: 29391b67772eSAndrii Nakryiko security_bpf_prog_free(prog); 2940d17aff80SAndrii Nakryiko free_prog: 29411b67772eSAndrii Nakryiko free_uid(prog->aux->user); 294222dc4a0fSAndrii Nakryiko if (prog->aux->attach_btf) 294322dc4a0fSAndrii Nakryiko btf_put(prog->aux->attach_btf); 294409756af4SAlexei Starovoitov bpf_prog_free(prog); 2945caf8f28eSAndrii Nakryiko put_token: 2946caf8f28eSAndrii Nakryiko bpf_token_put(token); 294709756af4SAlexei Starovoitov return err; 294809756af4SAlexei Starovoitov } 294909756af4SAlexei Starovoitov 2950cb8edce2SAndrii Nakryiko #define BPF_OBJ_LAST_FIELD path_fd 2951b2197755SDaniel Borkmann 2952b2197755SDaniel Borkmann static int bpf_obj_pin(const union bpf_attr *attr) 2953b2197755SDaniel Borkmann { 2954cb8edce2SAndrii Nakryiko int path_fd; 2955cb8edce2SAndrii Nakryiko 2956cb8edce2SAndrii Nakryiko if (CHECK_ATTR(BPF_OBJ) || attr->file_flags & ~BPF_F_PATH_FD) 2957b2197755SDaniel Borkmann return -EINVAL; 2958b2197755SDaniel Borkmann 2959cb8edce2SAndrii Nakryiko /* path_fd has to be accompanied by BPF_F_PATH_FD flag */ 2960cb8edce2SAndrii Nakryiko if (!(attr->file_flags & BPF_F_PATH_FD) && attr->path_fd) 2961cb8edce2SAndrii Nakryiko return -EINVAL; 2962cb8edce2SAndrii Nakryiko 2963cb8edce2SAndrii Nakryiko path_fd = attr->file_flags & BPF_F_PATH_FD ? attr->path_fd : AT_FDCWD; 2964cb8edce2SAndrii Nakryiko return bpf_obj_pin_user(attr->bpf_fd, path_fd, 2965cb8edce2SAndrii Nakryiko u64_to_user_ptr(attr->pathname)); 2966b2197755SDaniel Borkmann } 2967b2197755SDaniel Borkmann 2968b2197755SDaniel Borkmann static int bpf_obj_get(const union bpf_attr *attr) 2969b2197755SDaniel Borkmann { 2970cb8edce2SAndrii Nakryiko int path_fd; 2971cb8edce2SAndrii Nakryiko 29726e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || 2973cb8edce2SAndrii Nakryiko attr->file_flags & ~(BPF_OBJ_FLAG_MASK | BPF_F_PATH_FD)) 2974b2197755SDaniel Borkmann return -EINVAL; 2975b2197755SDaniel Borkmann 2976cb8edce2SAndrii Nakryiko /* path_fd has to be accompanied by BPF_F_PATH_FD flag */ 2977cb8edce2SAndrii Nakryiko if (!(attr->file_flags & BPF_F_PATH_FD) && attr->path_fd) 2978cb8edce2SAndrii Nakryiko return -EINVAL; 2979cb8edce2SAndrii Nakryiko 2980cb8edce2SAndrii Nakryiko path_fd = attr->file_flags & BPF_F_PATH_FD ? attr->path_fd : AT_FDCWD; 2981cb8edce2SAndrii Nakryiko return bpf_obj_get_user(path_fd, u64_to_user_ptr(attr->pathname), 29826e71b04aSChenbo Feng attr->file_flags); 2983b2197755SDaniel Borkmann } 2984b2197755SDaniel Borkmann 2985f2e10bffSAndrii Nakryiko void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 2986a3b80e10SAndrii Nakryiko const struct bpf_link_ops *ops, struct bpf_prog *prog) 298770ed506cSAndrii Nakryiko { 298870ed506cSAndrii Nakryiko atomic64_set(&link->refcnt, 1); 2989f2e10bffSAndrii Nakryiko link->type = type; 2990a3b80e10SAndrii Nakryiko link->id = 0; 299170ed506cSAndrii Nakryiko link->ops = ops; 299270ed506cSAndrii Nakryiko link->prog = prog; 299370ed506cSAndrii Nakryiko } 299470ed506cSAndrii Nakryiko 2995a3b80e10SAndrii Nakryiko static void bpf_link_free_id(int id) 2996a3b80e10SAndrii Nakryiko { 2997a3b80e10SAndrii Nakryiko if (!id) 2998a3b80e10SAndrii Nakryiko return; 2999a3b80e10SAndrii Nakryiko 3000a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 3001a3b80e10SAndrii Nakryiko idr_remove(&link_idr, id); 3002a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 3003a3b80e10SAndrii Nakryiko } 3004a3b80e10SAndrii Nakryiko 300598868668SAndrii Nakryiko /* Clean up bpf_link and corresponding anon_inode file and FD. After 300698868668SAndrii Nakryiko * anon_inode is created, bpf_link can't be just kfree()'d due to deferred 3007a3b80e10SAndrii Nakryiko * anon_inode's release() call. This helper marks bpf_link as 3008a3b80e10SAndrii Nakryiko * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt 3009a3b80e10SAndrii Nakryiko * is not decremented, it's the responsibility of a calling code that failed 3010a3b80e10SAndrii Nakryiko * to complete bpf_link initialization. 301189ae89f5SJiri Olsa * This helper eventually calls link's dealloc callback, but does not call 301289ae89f5SJiri Olsa * link's release callback. 301398868668SAndrii Nakryiko */ 3014a3b80e10SAndrii Nakryiko void bpf_link_cleanup(struct bpf_link_primer *primer) 3015babf3164SAndrii Nakryiko { 3016a3b80e10SAndrii Nakryiko primer->link->prog = NULL; 3017a3b80e10SAndrii Nakryiko bpf_link_free_id(primer->id); 3018a3b80e10SAndrii Nakryiko fput(primer->file); 3019a3b80e10SAndrii Nakryiko put_unused_fd(primer->fd); 3020babf3164SAndrii Nakryiko } 3021babf3164SAndrii Nakryiko 302270ed506cSAndrii Nakryiko void bpf_link_inc(struct bpf_link *link) 302370ed506cSAndrii Nakryiko { 302470ed506cSAndrii Nakryiko atomic64_inc(&link->refcnt); 302570ed506cSAndrii Nakryiko } 302670ed506cSAndrii Nakryiko 3027*1a80dbcbSAndrii Nakryiko static void bpf_link_defer_dealloc_rcu_gp(struct rcu_head *rcu) 3028*1a80dbcbSAndrii Nakryiko { 3029*1a80dbcbSAndrii Nakryiko struct bpf_link *link = container_of(rcu, struct bpf_link, rcu); 3030*1a80dbcbSAndrii Nakryiko 3031*1a80dbcbSAndrii Nakryiko /* free bpf_link and its containing memory */ 3032*1a80dbcbSAndrii Nakryiko link->ops->dealloc_deferred(link); 3033*1a80dbcbSAndrii Nakryiko } 3034*1a80dbcbSAndrii Nakryiko 3035*1a80dbcbSAndrii Nakryiko static void bpf_link_defer_dealloc_mult_rcu_gp(struct rcu_head *rcu) 3036*1a80dbcbSAndrii Nakryiko { 3037*1a80dbcbSAndrii Nakryiko if (rcu_trace_implies_rcu_gp()) 3038*1a80dbcbSAndrii Nakryiko bpf_link_defer_dealloc_rcu_gp(rcu); 3039*1a80dbcbSAndrii Nakryiko else 3040*1a80dbcbSAndrii Nakryiko call_rcu(rcu, bpf_link_defer_dealloc_rcu_gp); 3041*1a80dbcbSAndrii Nakryiko } 3042*1a80dbcbSAndrii Nakryiko 304370ed506cSAndrii Nakryiko /* bpf_link_free is guaranteed to be called from process context */ 304470ed506cSAndrii Nakryiko static void bpf_link_free(struct bpf_link *link) 304570ed506cSAndrii Nakryiko { 3046*1a80dbcbSAndrii Nakryiko bool sleepable = false; 3047*1a80dbcbSAndrii Nakryiko 3048a3b80e10SAndrii Nakryiko bpf_link_free_id(link->id); 3049babf3164SAndrii Nakryiko if (link->prog) { 3050*1a80dbcbSAndrii Nakryiko sleepable = link->prog->sleepable; 3051babf3164SAndrii Nakryiko /* detach BPF program, clean up used resources */ 305270ed506cSAndrii Nakryiko link->ops->release(link); 3053babf3164SAndrii Nakryiko bpf_prog_put(link->prog); 3054babf3164SAndrii Nakryiko } 3055*1a80dbcbSAndrii Nakryiko if (link->ops->dealloc_deferred) { 3056*1a80dbcbSAndrii Nakryiko /* schedule BPF link deallocation; if underlying BPF program 3057*1a80dbcbSAndrii Nakryiko * is sleepable, we need to first wait for RCU tasks trace 3058*1a80dbcbSAndrii Nakryiko * sync, then go through "classic" RCU grace period 3059*1a80dbcbSAndrii Nakryiko */ 3060*1a80dbcbSAndrii Nakryiko if (sleepable) 3061*1a80dbcbSAndrii Nakryiko call_rcu_tasks_trace(&link->rcu, bpf_link_defer_dealloc_mult_rcu_gp); 3062*1a80dbcbSAndrii Nakryiko else 3063*1a80dbcbSAndrii Nakryiko call_rcu(&link->rcu, bpf_link_defer_dealloc_rcu_gp); 3064*1a80dbcbSAndrii Nakryiko } 3065*1a80dbcbSAndrii Nakryiko if (link->ops->dealloc) 3066babf3164SAndrii Nakryiko link->ops->dealloc(link); 306770ed506cSAndrii Nakryiko } 306870ed506cSAndrii Nakryiko 306970ed506cSAndrii Nakryiko static void bpf_link_put_deferred(struct work_struct *work) 307070ed506cSAndrii Nakryiko { 307170ed506cSAndrii Nakryiko struct bpf_link *link = container_of(work, struct bpf_link, work); 307270ed506cSAndrii Nakryiko 307370ed506cSAndrii Nakryiko bpf_link_free(link); 307470ed506cSAndrii Nakryiko } 307570ed506cSAndrii Nakryiko 3076ab5d47bdSSebastian Andrzej Siewior /* bpf_link_put might be called from atomic context. It needs to be called 3077ab5d47bdSSebastian Andrzej Siewior * from sleepable context in order to acquire sleeping locks during the process. 307870ed506cSAndrii Nakryiko */ 307970ed506cSAndrii Nakryiko void bpf_link_put(struct bpf_link *link) 308070ed506cSAndrii Nakryiko { 308170ed506cSAndrii Nakryiko if (!atomic64_dec_and_test(&link->refcnt)) 308270ed506cSAndrii Nakryiko return; 308370ed506cSAndrii Nakryiko 308470ed506cSAndrii Nakryiko INIT_WORK(&link->work, bpf_link_put_deferred); 308570ed506cSAndrii Nakryiko schedule_work(&link->work); 308670ed506cSAndrii Nakryiko } 3087cb80ddc6SAlexei Starovoitov EXPORT_SYMBOL(bpf_link_put); 308870ed506cSAndrii Nakryiko 3089ab5d47bdSSebastian Andrzej Siewior static void bpf_link_put_direct(struct bpf_link *link) 3090ab5d47bdSSebastian Andrzej Siewior { 3091ab5d47bdSSebastian Andrzej Siewior if (!atomic64_dec_and_test(&link->refcnt)) 3092ab5d47bdSSebastian Andrzej Siewior return; 3093ab5d47bdSSebastian Andrzej Siewior bpf_link_free(link); 3094ab5d47bdSSebastian Andrzej Siewior } 3095ab5d47bdSSebastian Andrzej Siewior 309670ed506cSAndrii Nakryiko static int bpf_link_release(struct inode *inode, struct file *filp) 309770ed506cSAndrii Nakryiko { 309870ed506cSAndrii Nakryiko struct bpf_link *link = filp->private_data; 309970ed506cSAndrii Nakryiko 3100ab5d47bdSSebastian Andrzej Siewior bpf_link_put_direct(link); 3101fec56f58SAlexei Starovoitov return 0; 3102fec56f58SAlexei Starovoitov } 3103fec56f58SAlexei Starovoitov 310470ed506cSAndrii Nakryiko #ifdef CONFIG_PROC_FS 3105f2e10bffSAndrii Nakryiko #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) 3106f2e10bffSAndrii Nakryiko #define BPF_MAP_TYPE(_id, _ops) 3107f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) [_id] = #_name, 3108f2e10bffSAndrii Nakryiko static const char *bpf_link_type_strs[] = { 3109f2e10bffSAndrii Nakryiko [BPF_LINK_TYPE_UNSPEC] = "<invalid>", 3110f2e10bffSAndrii Nakryiko #include <linux/bpf_types.h> 3111f2e10bffSAndrii Nakryiko }; 3112f2e10bffSAndrii Nakryiko #undef BPF_PROG_TYPE 3113f2e10bffSAndrii Nakryiko #undef BPF_MAP_TYPE 3114f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE 311570ed506cSAndrii Nakryiko 311670ed506cSAndrii Nakryiko static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) 311770ed506cSAndrii Nakryiko { 311870ed506cSAndrii Nakryiko const struct bpf_link *link = filp->private_data; 311970ed506cSAndrii Nakryiko const struct bpf_prog *prog = link->prog; 312070ed506cSAndrii Nakryiko char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 312170ed506cSAndrii Nakryiko 312270ed506cSAndrii Nakryiko seq_printf(m, 312370ed506cSAndrii Nakryiko "link_type:\t%s\n" 312468b04864SKui-Feng Lee "link_id:\t%u\n", 312568b04864SKui-Feng Lee bpf_link_type_strs[link->type], 312668b04864SKui-Feng Lee link->id); 312768b04864SKui-Feng Lee if (prog) { 312868b04864SKui-Feng Lee bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 312968b04864SKui-Feng Lee seq_printf(m, 313070ed506cSAndrii Nakryiko "prog_tag:\t%s\n" 313170ed506cSAndrii Nakryiko "prog_id:\t%u\n", 313270ed506cSAndrii Nakryiko prog_tag, 313370ed506cSAndrii Nakryiko prog->aux->id); 313468b04864SKui-Feng Lee } 3135f2e10bffSAndrii Nakryiko if (link->ops->show_fdinfo) 3136f2e10bffSAndrii Nakryiko link->ops->show_fdinfo(link, m); 313770ed506cSAndrii Nakryiko } 313870ed506cSAndrii Nakryiko #endif 313970ed506cSAndrii Nakryiko 31406f302bfbSZou Wei static const struct file_operations bpf_link_fops = { 314170ed506cSAndrii Nakryiko #ifdef CONFIG_PROC_FS 314270ed506cSAndrii Nakryiko .show_fdinfo = bpf_link_show_fdinfo, 314370ed506cSAndrii Nakryiko #endif 314470ed506cSAndrii Nakryiko .release = bpf_link_release, 3145fec56f58SAlexei Starovoitov .read = bpf_dummy_read, 3146fec56f58SAlexei Starovoitov .write = bpf_dummy_write, 3147fec56f58SAlexei Starovoitov }; 3148fec56f58SAlexei Starovoitov 3149a3b80e10SAndrii Nakryiko static int bpf_link_alloc_id(struct bpf_link *link) 315070ed506cSAndrii Nakryiko { 3151a3b80e10SAndrii Nakryiko int id; 3152a3b80e10SAndrii Nakryiko 3153a3b80e10SAndrii Nakryiko idr_preload(GFP_KERNEL); 3154a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 3155a3b80e10SAndrii Nakryiko id = idr_alloc_cyclic(&link_idr, link, 1, INT_MAX, GFP_ATOMIC); 3156a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 3157a3b80e10SAndrii Nakryiko idr_preload_end(); 3158a3b80e10SAndrii Nakryiko 3159a3b80e10SAndrii Nakryiko return id; 316070ed506cSAndrii Nakryiko } 316170ed506cSAndrii Nakryiko 3162a3b80e10SAndrii Nakryiko /* Prepare bpf_link to be exposed to user-space by allocating anon_inode file, 3163a3b80e10SAndrii Nakryiko * reserving unused FD and allocating ID from link_idr. This is to be paired 3164a3b80e10SAndrii Nakryiko * with bpf_link_settle() to install FD and ID and expose bpf_link to 3165a3b80e10SAndrii Nakryiko * user-space, if bpf_link is successfully attached. If not, bpf_link and 3166a3b80e10SAndrii Nakryiko * pre-allocated resources are to be freed with bpf_cleanup() call. All the 3167a3b80e10SAndrii Nakryiko * transient state is passed around in struct bpf_link_primer. 3168a3b80e10SAndrii Nakryiko * This is preferred way to create and initialize bpf_link, especially when 3169a3b80e10SAndrii Nakryiko * there are complicated and expensive operations in between creating bpf_link 3170a3b80e10SAndrii Nakryiko * itself and attaching it to BPF hook. By using bpf_link_prime() and 3171a3b80e10SAndrii Nakryiko * bpf_link_settle() kernel code using bpf_link doesn't have to perform 3172a3b80e10SAndrii Nakryiko * expensive (and potentially failing) roll back operations in a rare case 3173a3b80e10SAndrii Nakryiko * that file, FD, or ID can't be allocated. 3174babf3164SAndrii Nakryiko */ 3175a3b80e10SAndrii Nakryiko int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer) 3176babf3164SAndrii Nakryiko { 3177babf3164SAndrii Nakryiko struct file *file; 3178a3b80e10SAndrii Nakryiko int fd, id; 3179babf3164SAndrii Nakryiko 3180babf3164SAndrii Nakryiko fd = get_unused_fd_flags(O_CLOEXEC); 3181babf3164SAndrii Nakryiko if (fd < 0) 3182a3b80e10SAndrii Nakryiko return fd; 3183babf3164SAndrii Nakryiko 3184babf3164SAndrii Nakryiko 3185a3b80e10SAndrii Nakryiko id = bpf_link_alloc_id(link); 3186a3b80e10SAndrii Nakryiko if (id < 0) { 3187a3b80e10SAndrii Nakryiko put_unused_fd(fd); 3188a3b80e10SAndrii Nakryiko return id; 3189a3b80e10SAndrii Nakryiko } 3190babf3164SAndrii Nakryiko 3191babf3164SAndrii Nakryiko file = anon_inode_getfile("bpf_link", &bpf_link_fops, link, O_CLOEXEC); 3192babf3164SAndrii Nakryiko if (IS_ERR(file)) { 3193138c6767SAndrii Nakryiko bpf_link_free_id(id); 3194babf3164SAndrii Nakryiko put_unused_fd(fd); 3195138c6767SAndrii Nakryiko return PTR_ERR(file); 3196babf3164SAndrii Nakryiko } 3197babf3164SAndrii Nakryiko 3198a3b80e10SAndrii Nakryiko primer->link = link; 3199a3b80e10SAndrii Nakryiko primer->file = file; 3200a3b80e10SAndrii Nakryiko primer->fd = fd; 3201a3b80e10SAndrii Nakryiko primer->id = id; 3202a3b80e10SAndrii Nakryiko return 0; 3203a3b80e10SAndrii Nakryiko } 3204a3b80e10SAndrii Nakryiko 3205a3b80e10SAndrii Nakryiko int bpf_link_settle(struct bpf_link_primer *primer) 3206a3b80e10SAndrii Nakryiko { 3207a3b80e10SAndrii Nakryiko /* make bpf_link fetchable by ID */ 3208a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock); 3209a3b80e10SAndrii Nakryiko primer->link->id = primer->id; 3210a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock); 3211a3b80e10SAndrii Nakryiko /* make bpf_link fetchable by FD */ 3212a3b80e10SAndrii Nakryiko fd_install(primer->fd, primer->file); 3213a3b80e10SAndrii Nakryiko /* pass through installed FD */ 3214a3b80e10SAndrii Nakryiko return primer->fd; 3215a3b80e10SAndrii Nakryiko } 3216a3b80e10SAndrii Nakryiko 3217a3b80e10SAndrii Nakryiko int bpf_link_new_fd(struct bpf_link *link) 3218a3b80e10SAndrii Nakryiko { 3219a3b80e10SAndrii Nakryiko return anon_inode_getfd("bpf-link", &bpf_link_fops, link, O_CLOEXEC); 3220babf3164SAndrii Nakryiko } 3221babf3164SAndrii Nakryiko 322270ed506cSAndrii Nakryiko struct bpf_link *bpf_link_get_from_fd(u32 ufd) 322370ed506cSAndrii Nakryiko { 322470ed506cSAndrii Nakryiko struct fd f = fdget(ufd); 322570ed506cSAndrii Nakryiko struct bpf_link *link; 322670ed506cSAndrii Nakryiko 322770ed506cSAndrii Nakryiko if (!f.file) 322870ed506cSAndrii Nakryiko return ERR_PTR(-EBADF); 322970ed506cSAndrii Nakryiko if (f.file->f_op != &bpf_link_fops) { 323070ed506cSAndrii Nakryiko fdput(f); 323170ed506cSAndrii Nakryiko return ERR_PTR(-EINVAL); 323270ed506cSAndrii Nakryiko } 323370ed506cSAndrii Nakryiko 323470ed506cSAndrii Nakryiko link = f.file->private_data; 323570ed506cSAndrii Nakryiko bpf_link_inc(link); 323670ed506cSAndrii Nakryiko fdput(f); 323770ed506cSAndrii Nakryiko 323870ed506cSAndrii Nakryiko return link; 323970ed506cSAndrii Nakryiko } 3240cb80ddc6SAlexei Starovoitov EXPORT_SYMBOL(bpf_link_get_from_fd); 324170ed506cSAndrii Nakryiko 324270ed506cSAndrii Nakryiko static void bpf_tracing_link_release(struct bpf_link *link) 324370ed506cSAndrii Nakryiko { 32443aac1eadSToke Høiland-Jørgensen struct bpf_tracing_link *tr_link = 3245f7e0beafSKui-Feng Lee container_of(link, struct bpf_tracing_link, link.link); 32463aac1eadSToke Høiland-Jørgensen 3247f7e0beafSKui-Feng Lee WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link, 32483aac1eadSToke Høiland-Jørgensen tr_link->trampoline)); 32493aac1eadSToke Høiland-Jørgensen 32503aac1eadSToke Høiland-Jørgensen bpf_trampoline_put(tr_link->trampoline); 32513aac1eadSToke Høiland-Jørgensen 32523aac1eadSToke Høiland-Jørgensen /* tgt_prog is NULL if target is a kernel function */ 32533aac1eadSToke Høiland-Jørgensen if (tr_link->tgt_prog) 32543aac1eadSToke Høiland-Jørgensen bpf_prog_put(tr_link->tgt_prog); 3255babf3164SAndrii Nakryiko } 3256babf3164SAndrii Nakryiko 3257babf3164SAndrii Nakryiko static void bpf_tracing_link_dealloc(struct bpf_link *link) 3258babf3164SAndrii Nakryiko { 325970ed506cSAndrii Nakryiko struct bpf_tracing_link *tr_link = 3260f7e0beafSKui-Feng Lee container_of(link, struct bpf_tracing_link, link.link); 326170ed506cSAndrii Nakryiko 326270ed506cSAndrii Nakryiko kfree(tr_link); 326370ed506cSAndrii Nakryiko } 326470ed506cSAndrii Nakryiko 3265f2e10bffSAndrii Nakryiko static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link, 3266f2e10bffSAndrii Nakryiko struct seq_file *seq) 3267f2e10bffSAndrii Nakryiko { 3268f2e10bffSAndrii Nakryiko struct bpf_tracing_link *tr_link = 3269f7e0beafSKui-Feng Lee container_of(link, struct bpf_tracing_link, link.link); 3270e859e429SYafang Shao u32 target_btf_id, target_obj_id; 3271f2e10bffSAndrii Nakryiko 3272e859e429SYafang Shao bpf_trampoline_unpack_key(tr_link->trampoline->key, 3273e859e429SYafang Shao &target_obj_id, &target_btf_id); 3274f2e10bffSAndrii Nakryiko seq_printf(seq, 3275e859e429SYafang Shao "attach_type:\t%d\n" 3276e859e429SYafang Shao "target_obj_id:\t%u\n" 3277e859e429SYafang Shao "target_btf_id:\t%u\n", 3278e859e429SYafang Shao tr_link->attach_type, 3279e859e429SYafang Shao target_obj_id, 3280e859e429SYafang Shao target_btf_id); 3281f2e10bffSAndrii Nakryiko } 3282f2e10bffSAndrii Nakryiko 3283f2e10bffSAndrii Nakryiko static int bpf_tracing_link_fill_link_info(const struct bpf_link *link, 3284f2e10bffSAndrii Nakryiko struct bpf_link_info *info) 3285f2e10bffSAndrii Nakryiko { 3286f2e10bffSAndrii Nakryiko struct bpf_tracing_link *tr_link = 3287f7e0beafSKui-Feng Lee container_of(link, struct bpf_tracing_link, link.link); 3288f2e10bffSAndrii Nakryiko 3289f2e10bffSAndrii Nakryiko info->tracing.attach_type = tr_link->attach_type; 3290441e8c66SToke Høiland-Jørgensen bpf_trampoline_unpack_key(tr_link->trampoline->key, 3291441e8c66SToke Høiland-Jørgensen &info->tracing.target_obj_id, 3292441e8c66SToke Høiland-Jørgensen &info->tracing.target_btf_id); 3293f2e10bffSAndrii Nakryiko 3294f2e10bffSAndrii Nakryiko return 0; 3295f2e10bffSAndrii Nakryiko } 3296f2e10bffSAndrii Nakryiko 329770ed506cSAndrii Nakryiko static const struct bpf_link_ops bpf_tracing_link_lops = { 329870ed506cSAndrii Nakryiko .release = bpf_tracing_link_release, 3299babf3164SAndrii Nakryiko .dealloc = bpf_tracing_link_dealloc, 3300f2e10bffSAndrii Nakryiko .show_fdinfo = bpf_tracing_link_show_fdinfo, 3301f2e10bffSAndrii Nakryiko .fill_link_info = bpf_tracing_link_fill_link_info, 330270ed506cSAndrii Nakryiko }; 330370ed506cSAndrii Nakryiko 33044a1e7c0cSToke Høiland-Jørgensen static int bpf_tracing_prog_attach(struct bpf_prog *prog, 33054a1e7c0cSToke Høiland-Jørgensen int tgt_prog_fd, 33062fcc8241SKui-Feng Lee u32 btf_id, 33072fcc8241SKui-Feng Lee u64 bpf_cookie) 3308fec56f58SAlexei Starovoitov { 3309a3b80e10SAndrii Nakryiko struct bpf_link_primer link_primer; 33103aac1eadSToke Høiland-Jørgensen struct bpf_prog *tgt_prog = NULL; 33114a1e7c0cSToke Høiland-Jørgensen struct bpf_trampoline *tr = NULL; 331270ed506cSAndrii Nakryiko struct bpf_tracing_link *link; 33134a1e7c0cSToke Høiland-Jørgensen u64 key = 0; 3314a3b80e10SAndrii Nakryiko int err; 3315fec56f58SAlexei Starovoitov 33169e4e01dfSKP Singh switch (prog->type) { 33179e4e01dfSKP Singh case BPF_PROG_TYPE_TRACING: 3318fec56f58SAlexei Starovoitov if (prog->expected_attach_type != BPF_TRACE_FENTRY && 3319be8704ffSAlexei Starovoitov prog->expected_attach_type != BPF_TRACE_FEXIT && 33209e4e01dfSKP Singh prog->expected_attach_type != BPF_MODIFY_RETURN) { 33219e4e01dfSKP Singh err = -EINVAL; 33229e4e01dfSKP Singh goto out_put_prog; 33239e4e01dfSKP Singh } 33249e4e01dfSKP Singh break; 33259e4e01dfSKP Singh case BPF_PROG_TYPE_EXT: 33269e4e01dfSKP Singh if (prog->expected_attach_type != 0) { 33279e4e01dfSKP Singh err = -EINVAL; 33289e4e01dfSKP Singh goto out_put_prog; 33299e4e01dfSKP Singh } 33309e4e01dfSKP Singh break; 33319e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 33329e4e01dfSKP Singh if (prog->expected_attach_type != BPF_LSM_MAC) { 33339e4e01dfSKP Singh err = -EINVAL; 33349e4e01dfSKP Singh goto out_put_prog; 33359e4e01dfSKP Singh } 33369e4e01dfSKP Singh break; 33379e4e01dfSKP Singh default: 3338fec56f58SAlexei Starovoitov err = -EINVAL; 3339fec56f58SAlexei Starovoitov goto out_put_prog; 3340fec56f58SAlexei Starovoitov } 3341fec56f58SAlexei Starovoitov 33424a1e7c0cSToke Høiland-Jørgensen if (!!tgt_prog_fd != !!btf_id) { 33434a1e7c0cSToke Høiland-Jørgensen err = -EINVAL; 33444a1e7c0cSToke Høiland-Jørgensen goto out_put_prog; 33454a1e7c0cSToke Høiland-Jørgensen } 33464a1e7c0cSToke Høiland-Jørgensen 33474a1e7c0cSToke Høiland-Jørgensen if (tgt_prog_fd) { 334819bfcdf9SDmitrii Dolgov /* 334919bfcdf9SDmitrii Dolgov * For now we only allow new targets for BPF_PROG_TYPE_EXT. If this 335019bfcdf9SDmitrii Dolgov * part would be changed to implement the same for 335119bfcdf9SDmitrii Dolgov * BPF_PROG_TYPE_TRACING, do not forget to update the way how 335219bfcdf9SDmitrii Dolgov * attach_tracing_prog flag is set. 335319bfcdf9SDmitrii Dolgov */ 33544a1e7c0cSToke Høiland-Jørgensen if (prog->type != BPF_PROG_TYPE_EXT) { 33554a1e7c0cSToke Høiland-Jørgensen err = -EINVAL; 33564a1e7c0cSToke Høiland-Jørgensen goto out_put_prog; 33574a1e7c0cSToke Høiland-Jørgensen } 33584a1e7c0cSToke Høiland-Jørgensen 33594a1e7c0cSToke Høiland-Jørgensen tgt_prog = bpf_prog_get(tgt_prog_fd); 33604a1e7c0cSToke Høiland-Jørgensen if (IS_ERR(tgt_prog)) { 33614a1e7c0cSToke Høiland-Jørgensen err = PTR_ERR(tgt_prog); 33624a1e7c0cSToke Høiland-Jørgensen tgt_prog = NULL; 33634a1e7c0cSToke Høiland-Jørgensen goto out_put_prog; 33644a1e7c0cSToke Høiland-Jørgensen } 33654a1e7c0cSToke Høiland-Jørgensen 336622dc4a0fSAndrii Nakryiko key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id); 33674a1e7c0cSToke Høiland-Jørgensen } 33684a1e7c0cSToke Høiland-Jørgensen 336970ed506cSAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER); 337070ed506cSAndrii Nakryiko if (!link) { 337170ed506cSAndrii Nakryiko err = -ENOMEM; 3372fec56f58SAlexei Starovoitov goto out_put_prog; 3373fec56f58SAlexei Starovoitov } 3374f7e0beafSKui-Feng Lee bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING, 3375f2e10bffSAndrii Nakryiko &bpf_tracing_link_lops, prog); 3376f2e10bffSAndrii Nakryiko link->attach_type = prog->expected_attach_type; 33772fcc8241SKui-Feng Lee link->link.cookie = bpf_cookie; 3378fec56f58SAlexei Starovoitov 33793aac1eadSToke Høiland-Jørgensen mutex_lock(&prog->aux->dst_mutex); 3380babf3164SAndrii Nakryiko 33814a1e7c0cSToke Høiland-Jørgensen /* There are a few possible cases here: 33824a1e7c0cSToke Høiland-Jørgensen * 33834a1e7c0cSToke Høiland-Jørgensen * - if prog->aux->dst_trampoline is set, the program was just loaded 33844a1e7c0cSToke Høiland-Jørgensen * and not yet attached to anything, so we can use the values stored 33854a1e7c0cSToke Høiland-Jørgensen * in prog->aux 33864a1e7c0cSToke Høiland-Jørgensen * 33874a1e7c0cSToke Høiland-Jørgensen * - if prog->aux->dst_trampoline is NULL, the program has already been 33884a1e7c0cSToke Høiland-Jørgensen * attached to a target and its initial target was cleared (below) 33894a1e7c0cSToke Høiland-Jørgensen * 33904a1e7c0cSToke Høiland-Jørgensen * - if tgt_prog != NULL, the caller specified tgt_prog_fd + 33914a1e7c0cSToke Høiland-Jørgensen * target_btf_id using the link_create API. 33924a1e7c0cSToke Høiland-Jørgensen * 33934a1e7c0cSToke Høiland-Jørgensen * - if tgt_prog == NULL when this function was called using the old 33944a1e7c0cSToke Høiland-Jørgensen * raw_tracepoint_open API, and we need a target from prog->aux 33954a1e7c0cSToke Høiland-Jørgensen * 3396f3a95075SJiri Olsa * - if prog->aux->dst_trampoline and tgt_prog is NULL, the program 3397f3a95075SJiri Olsa * was detached and is going for re-attachment. 3398715d82baSJiri Olsa * 3399715d82baSJiri Olsa * - if prog->aux->dst_trampoline is NULL and tgt_prog and prog->aux->attach_btf 3400715d82baSJiri Olsa * are NULL, then program was already attached and user did not provide 3401715d82baSJiri Olsa * tgt_prog_fd so we have no way to find out or create trampoline 34024a1e7c0cSToke Høiland-Jørgensen */ 34034a1e7c0cSToke Høiland-Jørgensen if (!prog->aux->dst_trampoline && !tgt_prog) { 3404f3a95075SJiri Olsa /* 3405f3a95075SJiri Olsa * Allow re-attach for TRACING and LSM programs. If it's 3406f3a95075SJiri Olsa * currently linked, bpf_trampoline_link_prog will fail. 3407f3a95075SJiri Olsa * EXT programs need to specify tgt_prog_fd, so they 3408f3a95075SJiri Olsa * re-attach in separate code path. 3409f3a95075SJiri Olsa */ 3410f3a95075SJiri Olsa if (prog->type != BPF_PROG_TYPE_TRACING && 3411f3a95075SJiri Olsa prog->type != BPF_PROG_TYPE_LSM) { 3412f3a95075SJiri Olsa err = -EINVAL; 34133aac1eadSToke Høiland-Jørgensen goto out_unlock; 34143aac1eadSToke Høiland-Jørgensen } 3415715d82baSJiri Olsa /* We can allow re-attach only if we have valid attach_btf. */ 3416715d82baSJiri Olsa if (!prog->aux->attach_btf) { 3417715d82baSJiri Olsa err = -EINVAL; 3418715d82baSJiri Olsa goto out_unlock; 3419715d82baSJiri Olsa } 3420f3a95075SJiri Olsa btf_id = prog->aux->attach_btf_id; 3421f3a95075SJiri Olsa key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id); 3422f3a95075SJiri Olsa } 34234a1e7c0cSToke Høiland-Jørgensen 34244a1e7c0cSToke Høiland-Jørgensen if (!prog->aux->dst_trampoline || 34254a1e7c0cSToke Høiland-Jørgensen (key && key != prog->aux->dst_trampoline->key)) { 34264a1e7c0cSToke Høiland-Jørgensen /* If there is no saved target, or the specified target is 34274a1e7c0cSToke Høiland-Jørgensen * different from the destination specified at load time, we 34284a1e7c0cSToke Høiland-Jørgensen * need a new trampoline and a check for compatibility 34294a1e7c0cSToke Høiland-Jørgensen */ 34304a1e7c0cSToke Høiland-Jørgensen struct bpf_attach_target_info tgt_info = {}; 34314a1e7c0cSToke Høiland-Jørgensen 34324a1e7c0cSToke Høiland-Jørgensen err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id, 34334a1e7c0cSToke Høiland-Jørgensen &tgt_info); 34344a1e7c0cSToke Høiland-Jørgensen if (err) 34354a1e7c0cSToke Høiland-Jørgensen goto out_unlock; 34364a1e7c0cSToke Høiland-Jørgensen 343731bf1dbcSViktor Malik if (tgt_info.tgt_mod) { 343831bf1dbcSViktor Malik module_put(prog->aux->mod); 343931bf1dbcSViktor Malik prog->aux->mod = tgt_info.tgt_mod; 344031bf1dbcSViktor Malik } 344131bf1dbcSViktor Malik 34424a1e7c0cSToke Høiland-Jørgensen tr = bpf_trampoline_get(key, &tgt_info); 34434a1e7c0cSToke Høiland-Jørgensen if (!tr) { 34444a1e7c0cSToke Høiland-Jørgensen err = -ENOMEM; 34454a1e7c0cSToke Høiland-Jørgensen goto out_unlock; 34464a1e7c0cSToke Høiland-Jørgensen } 34474a1e7c0cSToke Høiland-Jørgensen } else { 34484a1e7c0cSToke Høiland-Jørgensen /* The caller didn't specify a target, or the target was the 34494a1e7c0cSToke Høiland-Jørgensen * same as the destination supplied during program load. This 34504a1e7c0cSToke Høiland-Jørgensen * means we can reuse the trampoline and reference from program 34514a1e7c0cSToke Høiland-Jørgensen * load time, and there is no need to allocate a new one. This 34524a1e7c0cSToke Høiland-Jørgensen * can only happen once for any program, as the saved values in 34534a1e7c0cSToke Høiland-Jørgensen * prog->aux are cleared below. 34544a1e7c0cSToke Høiland-Jørgensen */ 34553aac1eadSToke Høiland-Jørgensen tr = prog->aux->dst_trampoline; 34563aac1eadSToke Høiland-Jørgensen tgt_prog = prog->aux->dst_prog; 34574a1e7c0cSToke Høiland-Jørgensen } 34583aac1eadSToke Høiland-Jørgensen 3459f7e0beafSKui-Feng Lee err = bpf_link_prime(&link->link.link, &link_primer); 34603aac1eadSToke Høiland-Jørgensen if (err) 34613aac1eadSToke Høiland-Jørgensen goto out_unlock; 34623aac1eadSToke Høiland-Jørgensen 3463f7e0beafSKui-Feng Lee err = bpf_trampoline_link_prog(&link->link, tr); 3464babf3164SAndrii Nakryiko if (err) { 3465a3b80e10SAndrii Nakryiko bpf_link_cleanup(&link_primer); 34663aac1eadSToke Høiland-Jørgensen link = NULL; 34673aac1eadSToke Høiland-Jørgensen goto out_unlock; 3468babf3164SAndrii Nakryiko } 3469babf3164SAndrii Nakryiko 34703aac1eadSToke Høiland-Jørgensen link->tgt_prog = tgt_prog; 34713aac1eadSToke Høiland-Jørgensen link->trampoline = tr; 34723aac1eadSToke Høiland-Jørgensen 34734a1e7c0cSToke Høiland-Jørgensen /* Always clear the trampoline and target prog from prog->aux to make 34744a1e7c0cSToke Høiland-Jørgensen * sure the original attach destination is not kept alive after a 34754a1e7c0cSToke Høiland-Jørgensen * program is (re-)attached to another target. 34764a1e7c0cSToke Høiland-Jørgensen */ 34774a1e7c0cSToke Høiland-Jørgensen if (prog->aux->dst_prog && 34784a1e7c0cSToke Høiland-Jørgensen (tgt_prog_fd || tr != prog->aux->dst_trampoline)) 34794a1e7c0cSToke Høiland-Jørgensen /* got extra prog ref from syscall, or attaching to different prog */ 34804a1e7c0cSToke Høiland-Jørgensen bpf_prog_put(prog->aux->dst_prog); 34814a1e7c0cSToke Høiland-Jørgensen if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline) 34824a1e7c0cSToke Høiland-Jørgensen /* we allocated a new trampoline, so free the old one */ 34834a1e7c0cSToke Høiland-Jørgensen bpf_trampoline_put(prog->aux->dst_trampoline); 34844a1e7c0cSToke Høiland-Jørgensen 34853aac1eadSToke Høiland-Jørgensen prog->aux->dst_prog = NULL; 34863aac1eadSToke Høiland-Jørgensen prog->aux->dst_trampoline = NULL; 34873aac1eadSToke Høiland-Jørgensen mutex_unlock(&prog->aux->dst_mutex); 34883aac1eadSToke Høiland-Jørgensen 3489a3b80e10SAndrii Nakryiko return bpf_link_settle(&link_primer); 34903aac1eadSToke Høiland-Jørgensen out_unlock: 34914a1e7c0cSToke Høiland-Jørgensen if (tr && tr != prog->aux->dst_trampoline) 34924a1e7c0cSToke Høiland-Jørgensen bpf_trampoline_put(tr); 34933aac1eadSToke Høiland-Jørgensen mutex_unlock(&prog->aux->dst_mutex); 34943aac1eadSToke Høiland-Jørgensen kfree(link); 3495fec56f58SAlexei Starovoitov out_put_prog: 34964a1e7c0cSToke Høiland-Jørgensen if (tgt_prog_fd && tgt_prog) 34974a1e7c0cSToke Høiland-Jørgensen bpf_prog_put(tgt_prog); 3498fec56f58SAlexei Starovoitov return err; 3499fec56f58SAlexei Starovoitov } 3500fec56f58SAlexei Starovoitov 350170ed506cSAndrii Nakryiko struct bpf_raw_tp_link { 350270ed506cSAndrii Nakryiko struct bpf_link link; 3503c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 3504c4f6699dSAlexei Starovoitov }; 3505c4f6699dSAlexei Starovoitov 350670ed506cSAndrii Nakryiko static void bpf_raw_tp_link_release(struct bpf_link *link) 3507c4f6699dSAlexei Starovoitov { 350870ed506cSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 350970ed506cSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 3510c4f6699dSAlexei Starovoitov 351170ed506cSAndrii Nakryiko bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog); 3512a38d1107SMatt Mullins bpf_put_raw_tracepoint(raw_tp->btp); 3513babf3164SAndrii Nakryiko } 3514babf3164SAndrii Nakryiko 3515babf3164SAndrii Nakryiko static void bpf_raw_tp_link_dealloc(struct bpf_link *link) 3516babf3164SAndrii Nakryiko { 3517babf3164SAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 3518babf3164SAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 3519babf3164SAndrii Nakryiko 3520c4f6699dSAlexei Starovoitov kfree(raw_tp); 3521c4f6699dSAlexei Starovoitov } 3522c4f6699dSAlexei Starovoitov 3523f2e10bffSAndrii Nakryiko static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link, 3524f2e10bffSAndrii Nakryiko struct seq_file *seq) 3525f2e10bffSAndrii Nakryiko { 3526f2e10bffSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp_link = 3527f2e10bffSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 3528f2e10bffSAndrii Nakryiko 3529f2e10bffSAndrii Nakryiko seq_printf(seq, 3530f2e10bffSAndrii Nakryiko "tp_name:\t%s\n", 3531f2e10bffSAndrii Nakryiko raw_tp_link->btp->tp->name); 3532f2e10bffSAndrii Nakryiko } 3533f2e10bffSAndrii Nakryiko 353457d48537SYafang Shao static int bpf_copy_to_user(char __user *ubuf, const char *buf, u32 ulen, 353557d48537SYafang Shao u32 len) 353657d48537SYafang Shao { 353757d48537SYafang Shao if (ulen >= len + 1) { 353857d48537SYafang Shao if (copy_to_user(ubuf, buf, len + 1)) 353957d48537SYafang Shao return -EFAULT; 354057d48537SYafang Shao } else { 354157d48537SYafang Shao char zero = '\0'; 354257d48537SYafang Shao 354357d48537SYafang Shao if (copy_to_user(ubuf, buf, ulen - 1)) 354457d48537SYafang Shao return -EFAULT; 354557d48537SYafang Shao if (put_user(zero, ubuf + ulen - 1)) 354657d48537SYafang Shao return -EFAULT; 354757d48537SYafang Shao return -ENOSPC; 354857d48537SYafang Shao } 354957d48537SYafang Shao 355057d48537SYafang Shao return 0; 355157d48537SYafang Shao } 355257d48537SYafang Shao 3553f2e10bffSAndrii Nakryiko static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link, 3554f2e10bffSAndrii Nakryiko struct bpf_link_info *info) 3555f2e10bffSAndrii Nakryiko { 3556f2e10bffSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp_link = 3557f2e10bffSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 3558f2e10bffSAndrii Nakryiko char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name); 3559f2e10bffSAndrii Nakryiko const char *tp_name = raw_tp_link->btp->tp->name; 3560f2e10bffSAndrii Nakryiko u32 ulen = info->raw_tracepoint.tp_name_len; 3561f2e10bffSAndrii Nakryiko size_t tp_len = strlen(tp_name); 3562f2e10bffSAndrii Nakryiko 3563b474959dSYonghong Song if (!ulen ^ !ubuf) 3564f2e10bffSAndrii Nakryiko return -EINVAL; 3565f2e10bffSAndrii Nakryiko 3566f2e10bffSAndrii Nakryiko info->raw_tracepoint.tp_name_len = tp_len + 1; 3567f2e10bffSAndrii Nakryiko 3568f2e10bffSAndrii Nakryiko if (!ubuf) 3569f2e10bffSAndrii Nakryiko return 0; 3570f2e10bffSAndrii Nakryiko 357157d48537SYafang Shao return bpf_copy_to_user(ubuf, tp_name, ulen, tp_len); 3572f2e10bffSAndrii Nakryiko } 3573f2e10bffSAndrii Nakryiko 3574a3b80e10SAndrii Nakryiko static const struct bpf_link_ops bpf_raw_tp_link_lops = { 357570ed506cSAndrii Nakryiko .release = bpf_raw_tp_link_release, 3576*1a80dbcbSAndrii Nakryiko .dealloc_deferred = bpf_raw_tp_link_dealloc, 3577f2e10bffSAndrii Nakryiko .show_fdinfo = bpf_raw_tp_link_show_fdinfo, 3578f2e10bffSAndrii Nakryiko .fill_link_info = bpf_raw_tp_link_fill_link_info, 3579c4f6699dSAlexei Starovoitov }; 3580c4f6699dSAlexei Starovoitov 3581b89fbfbbSAndrii Nakryiko #ifdef CONFIG_PERF_EVENTS 3582b89fbfbbSAndrii Nakryiko struct bpf_perf_link { 3583b89fbfbbSAndrii Nakryiko struct bpf_link link; 3584b89fbfbbSAndrii Nakryiko struct file *perf_file; 3585b89fbfbbSAndrii Nakryiko }; 3586b89fbfbbSAndrii Nakryiko 3587b89fbfbbSAndrii Nakryiko static void bpf_perf_link_release(struct bpf_link *link) 3588b89fbfbbSAndrii Nakryiko { 3589b89fbfbbSAndrii Nakryiko struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link); 3590b89fbfbbSAndrii Nakryiko struct perf_event *event = perf_link->perf_file->private_data; 3591b89fbfbbSAndrii Nakryiko 3592b89fbfbbSAndrii Nakryiko perf_event_free_bpf_prog(event); 3593b89fbfbbSAndrii Nakryiko fput(perf_link->perf_file); 3594b89fbfbbSAndrii Nakryiko } 3595b89fbfbbSAndrii Nakryiko 3596b89fbfbbSAndrii Nakryiko static void bpf_perf_link_dealloc(struct bpf_link *link) 3597b89fbfbbSAndrii Nakryiko { 3598b89fbfbbSAndrii Nakryiko struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link); 3599b89fbfbbSAndrii Nakryiko 3600b89fbfbbSAndrii Nakryiko kfree(perf_link); 3601b89fbfbbSAndrii Nakryiko } 3602b89fbfbbSAndrii Nakryiko 36031b715e1bSYafang Shao static int bpf_perf_link_fill_common(const struct perf_event *event, 36041b715e1bSYafang Shao char __user *uname, u32 ulen, 36051b715e1bSYafang Shao u64 *probe_offset, u64 *probe_addr, 36063acf8aceSJiri Olsa u32 *fd_type, unsigned long *missed) 36071b715e1bSYafang Shao { 36081b715e1bSYafang Shao const char *buf; 36091b715e1bSYafang Shao u32 prog_id; 36101b715e1bSYafang Shao size_t len; 36111b715e1bSYafang Shao int err; 36121b715e1bSYafang Shao 36131b715e1bSYafang Shao if (!ulen ^ !uname) 36141b715e1bSYafang Shao return -EINVAL; 36151b715e1bSYafang Shao 36161b715e1bSYafang Shao err = bpf_get_perf_event_info(event, &prog_id, fd_type, &buf, 36173acf8aceSJiri Olsa probe_offset, probe_addr, missed); 36181b715e1bSYafang Shao if (err) 36191b715e1bSYafang Shao return err; 36200aa35162SYafang Shao if (!uname) 36210aa35162SYafang Shao return 0; 36221b715e1bSYafang Shao if (buf) { 36231b715e1bSYafang Shao len = strlen(buf); 36241b715e1bSYafang Shao err = bpf_copy_to_user(uname, buf, ulen, len); 36251b715e1bSYafang Shao if (err) 36261b715e1bSYafang Shao return err; 36271b715e1bSYafang Shao } else { 36281b715e1bSYafang Shao char zero = '\0'; 36291b715e1bSYafang Shao 36301b715e1bSYafang Shao if (put_user(zero, uname)) 36311b715e1bSYafang Shao return -EFAULT; 36321b715e1bSYafang Shao } 36331b715e1bSYafang Shao return 0; 36341b715e1bSYafang Shao } 36351b715e1bSYafang Shao 36361b715e1bSYafang Shao #ifdef CONFIG_KPROBE_EVENTS 36371b715e1bSYafang Shao static int bpf_perf_link_fill_kprobe(const struct perf_event *event, 36381b715e1bSYafang Shao struct bpf_link_info *info) 36391b715e1bSYafang Shao { 36403acf8aceSJiri Olsa unsigned long missed; 36411b715e1bSYafang Shao char __user *uname; 36421b715e1bSYafang Shao u64 addr, offset; 36431b715e1bSYafang Shao u32 ulen, type; 36441b715e1bSYafang Shao int err; 36451b715e1bSYafang Shao 36461b715e1bSYafang Shao uname = u64_to_user_ptr(info->perf_event.kprobe.func_name); 36471b715e1bSYafang Shao ulen = info->perf_event.kprobe.name_len; 36481b715e1bSYafang Shao err = bpf_perf_link_fill_common(event, uname, ulen, &offset, &addr, 36493acf8aceSJiri Olsa &type, &missed); 36501b715e1bSYafang Shao if (err) 36511b715e1bSYafang Shao return err; 36521b715e1bSYafang Shao if (type == BPF_FD_TYPE_KRETPROBE) 36531b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_KRETPROBE; 36541b715e1bSYafang Shao else 36551b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_KPROBE; 36561b715e1bSYafang Shao 36571b715e1bSYafang Shao info->perf_event.kprobe.offset = offset; 36583acf8aceSJiri Olsa info->perf_event.kprobe.missed = missed; 36591b715e1bSYafang Shao if (!kallsyms_show_value(current_cred())) 36601b715e1bSYafang Shao addr = 0; 36611b715e1bSYafang Shao info->perf_event.kprobe.addr = addr; 3662d5c16492SJiri Olsa info->perf_event.kprobe.cookie = event->bpf_cookie; 36631b715e1bSYafang Shao return 0; 36641b715e1bSYafang Shao } 36651b715e1bSYafang Shao #endif 36661b715e1bSYafang Shao 36671b715e1bSYafang Shao #ifdef CONFIG_UPROBE_EVENTS 36681b715e1bSYafang Shao static int bpf_perf_link_fill_uprobe(const struct perf_event *event, 36691b715e1bSYafang Shao struct bpf_link_info *info) 36701b715e1bSYafang Shao { 36711b715e1bSYafang Shao char __user *uname; 36721b715e1bSYafang Shao u64 addr, offset; 36731b715e1bSYafang Shao u32 ulen, type; 36741b715e1bSYafang Shao int err; 36751b715e1bSYafang Shao 36761b715e1bSYafang Shao uname = u64_to_user_ptr(info->perf_event.uprobe.file_name); 36771b715e1bSYafang Shao ulen = info->perf_event.uprobe.name_len; 36781b715e1bSYafang Shao err = bpf_perf_link_fill_common(event, uname, ulen, &offset, &addr, 36793acf8aceSJiri Olsa &type, NULL); 36801b715e1bSYafang Shao if (err) 36811b715e1bSYafang Shao return err; 36821b715e1bSYafang Shao 36831b715e1bSYafang Shao if (type == BPF_FD_TYPE_URETPROBE) 36841b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_URETPROBE; 36851b715e1bSYafang Shao else 36861b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_UPROBE; 36871b715e1bSYafang Shao info->perf_event.uprobe.offset = offset; 3688d5c16492SJiri Olsa info->perf_event.uprobe.cookie = event->bpf_cookie; 36891b715e1bSYafang Shao return 0; 36901b715e1bSYafang Shao } 36911b715e1bSYafang Shao #endif 36921b715e1bSYafang Shao 36931b715e1bSYafang Shao static int bpf_perf_link_fill_probe(const struct perf_event *event, 36941b715e1bSYafang Shao struct bpf_link_info *info) 36951b715e1bSYafang Shao { 36961b715e1bSYafang Shao #ifdef CONFIG_KPROBE_EVENTS 36971b715e1bSYafang Shao if (event->tp_event->flags & TRACE_EVENT_FL_KPROBE) 36981b715e1bSYafang Shao return bpf_perf_link_fill_kprobe(event, info); 36991b715e1bSYafang Shao #endif 37001b715e1bSYafang Shao #ifdef CONFIG_UPROBE_EVENTS 37011b715e1bSYafang Shao if (event->tp_event->flags & TRACE_EVENT_FL_UPROBE) 37021b715e1bSYafang Shao return bpf_perf_link_fill_uprobe(event, info); 37031b715e1bSYafang Shao #endif 37041b715e1bSYafang Shao return -EOPNOTSUPP; 37051b715e1bSYafang Shao } 37061b715e1bSYafang Shao 37071b715e1bSYafang Shao static int bpf_perf_link_fill_tracepoint(const struct perf_event *event, 37081b715e1bSYafang Shao struct bpf_link_info *info) 37091b715e1bSYafang Shao { 37101b715e1bSYafang Shao char __user *uname; 37111b715e1bSYafang Shao u32 ulen; 37121b715e1bSYafang Shao 37131b715e1bSYafang Shao uname = u64_to_user_ptr(info->perf_event.tracepoint.tp_name); 37141b715e1bSYafang Shao ulen = info->perf_event.tracepoint.name_len; 37151b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_TRACEPOINT; 3716d5c16492SJiri Olsa info->perf_event.tracepoint.cookie = event->bpf_cookie; 37173acf8aceSJiri Olsa return bpf_perf_link_fill_common(event, uname, ulen, NULL, NULL, NULL, NULL); 37181b715e1bSYafang Shao } 37191b715e1bSYafang Shao 37201b715e1bSYafang Shao static int bpf_perf_link_fill_perf_event(const struct perf_event *event, 37211b715e1bSYafang Shao struct bpf_link_info *info) 37221b715e1bSYafang Shao { 37231b715e1bSYafang Shao info->perf_event.event.type = event->attr.type; 37241b715e1bSYafang Shao info->perf_event.event.config = event->attr.config; 3725d5c16492SJiri Olsa info->perf_event.event.cookie = event->bpf_cookie; 37261b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_EVENT; 37271b715e1bSYafang Shao return 0; 37281b715e1bSYafang Shao } 37291b715e1bSYafang Shao 37301b715e1bSYafang Shao static int bpf_perf_link_fill_link_info(const struct bpf_link *link, 37311b715e1bSYafang Shao struct bpf_link_info *info) 37321b715e1bSYafang Shao { 37331b715e1bSYafang Shao struct bpf_perf_link *perf_link; 37341b715e1bSYafang Shao const struct perf_event *event; 37351b715e1bSYafang Shao 37361b715e1bSYafang Shao perf_link = container_of(link, struct bpf_perf_link, link); 37371b715e1bSYafang Shao event = perf_get_event(perf_link->perf_file); 37381b715e1bSYafang Shao if (IS_ERR(event)) 37391b715e1bSYafang Shao return PTR_ERR(event); 37401b715e1bSYafang Shao 37411b715e1bSYafang Shao switch (event->prog->type) { 37421b715e1bSYafang Shao case BPF_PROG_TYPE_PERF_EVENT: 37431b715e1bSYafang Shao return bpf_perf_link_fill_perf_event(event, info); 37441b715e1bSYafang Shao case BPF_PROG_TYPE_TRACEPOINT: 37451b715e1bSYafang Shao return bpf_perf_link_fill_tracepoint(event, info); 37461b715e1bSYafang Shao case BPF_PROG_TYPE_KPROBE: 37471b715e1bSYafang Shao return bpf_perf_link_fill_probe(event, info); 37481b715e1bSYafang Shao default: 37491b715e1bSYafang Shao return -EOPNOTSUPP; 37501b715e1bSYafang Shao } 37511b715e1bSYafang Shao } 37521b715e1bSYafang Shao 3753b89fbfbbSAndrii Nakryiko static const struct bpf_link_ops bpf_perf_link_lops = { 3754b89fbfbbSAndrii Nakryiko .release = bpf_perf_link_release, 3755b89fbfbbSAndrii Nakryiko .dealloc = bpf_perf_link_dealloc, 37561b715e1bSYafang Shao .fill_link_info = bpf_perf_link_fill_link_info, 3757b89fbfbbSAndrii Nakryiko }; 3758b89fbfbbSAndrii Nakryiko 3759b89fbfbbSAndrii Nakryiko static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 3760b89fbfbbSAndrii Nakryiko { 3761b89fbfbbSAndrii Nakryiko struct bpf_link_primer link_primer; 3762b89fbfbbSAndrii Nakryiko struct bpf_perf_link *link; 3763b89fbfbbSAndrii Nakryiko struct perf_event *event; 3764b89fbfbbSAndrii Nakryiko struct file *perf_file; 3765b89fbfbbSAndrii Nakryiko int err; 3766b89fbfbbSAndrii Nakryiko 3767b89fbfbbSAndrii Nakryiko if (attr->link_create.flags) 3768b89fbfbbSAndrii Nakryiko return -EINVAL; 3769b89fbfbbSAndrii Nakryiko 3770b89fbfbbSAndrii Nakryiko perf_file = perf_event_get(attr->link_create.target_fd); 3771b89fbfbbSAndrii Nakryiko if (IS_ERR(perf_file)) 3772b89fbfbbSAndrii Nakryiko return PTR_ERR(perf_file); 3773b89fbfbbSAndrii Nakryiko 3774b89fbfbbSAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER); 3775b89fbfbbSAndrii Nakryiko if (!link) { 3776b89fbfbbSAndrii Nakryiko err = -ENOMEM; 3777b89fbfbbSAndrii Nakryiko goto out_put_file; 3778b89fbfbbSAndrii Nakryiko } 3779b89fbfbbSAndrii Nakryiko bpf_link_init(&link->link, BPF_LINK_TYPE_PERF_EVENT, &bpf_perf_link_lops, prog); 3780b89fbfbbSAndrii Nakryiko link->perf_file = perf_file; 3781b89fbfbbSAndrii Nakryiko 3782b89fbfbbSAndrii Nakryiko err = bpf_link_prime(&link->link, &link_primer); 3783b89fbfbbSAndrii Nakryiko if (err) { 3784b89fbfbbSAndrii Nakryiko kfree(link); 3785b89fbfbbSAndrii Nakryiko goto out_put_file; 3786b89fbfbbSAndrii Nakryiko } 3787b89fbfbbSAndrii Nakryiko 3788b89fbfbbSAndrii Nakryiko event = perf_file->private_data; 378982e6b1eeSAndrii Nakryiko err = perf_event_set_bpf_prog(event, prog, attr->link_create.perf_event.bpf_cookie); 3790b89fbfbbSAndrii Nakryiko if (err) { 3791b89fbfbbSAndrii Nakryiko bpf_link_cleanup(&link_primer); 3792b89fbfbbSAndrii Nakryiko goto out_put_file; 3793b89fbfbbSAndrii Nakryiko } 3794b89fbfbbSAndrii Nakryiko /* perf_event_set_bpf_prog() doesn't take its own refcnt on prog */ 3795b89fbfbbSAndrii Nakryiko bpf_prog_inc(prog); 3796b89fbfbbSAndrii Nakryiko 3797b89fbfbbSAndrii Nakryiko return bpf_link_settle(&link_primer); 3798b89fbfbbSAndrii Nakryiko 3799b89fbfbbSAndrii Nakryiko out_put_file: 3800b89fbfbbSAndrii Nakryiko fput(perf_file); 3801b89fbfbbSAndrii Nakryiko return err; 3802b89fbfbbSAndrii Nakryiko } 38030dcac272SJiri Olsa #else 38040dcac272SJiri Olsa static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 38050dcac272SJiri Olsa { 38060dcac272SJiri Olsa return -EOPNOTSUPP; 38070dcac272SJiri Olsa } 3808b89fbfbbSAndrii Nakryiko #endif /* CONFIG_PERF_EVENTS */ 3809b89fbfbbSAndrii Nakryiko 3810df86ca0dSAndrii Nakryiko static int bpf_raw_tp_link_attach(struct bpf_prog *prog, 3811df86ca0dSAndrii Nakryiko const char __user *user_tp_name) 3812c4f6699dSAlexei Starovoitov { 3813a3b80e10SAndrii Nakryiko struct bpf_link_primer link_primer; 3814babf3164SAndrii Nakryiko struct bpf_raw_tp_link *link; 3815c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 3816ac4414b5SAlexei Starovoitov const char *tp_name; 3817ac4414b5SAlexei Starovoitov char buf[128]; 3818a3b80e10SAndrii Nakryiko int err; 3819c4f6699dSAlexei Starovoitov 38209e4e01dfSKP Singh switch (prog->type) { 38219e4e01dfSKP Singh case BPF_PROG_TYPE_TRACING: 38229e4e01dfSKP Singh case BPF_PROG_TYPE_EXT: 38239e4e01dfSKP Singh case BPF_PROG_TYPE_LSM: 3824df86ca0dSAndrii Nakryiko if (user_tp_name) 3825fec56f58SAlexei Starovoitov /* The attach point for this category of programs 3826fec56f58SAlexei Starovoitov * should be specified via btf_id during program load. 3827ac4414b5SAlexei Starovoitov */ 3828df86ca0dSAndrii Nakryiko return -EINVAL; 38299e4e01dfSKP Singh if (prog->type == BPF_PROG_TYPE_TRACING && 38309e4e01dfSKP Singh prog->expected_attach_type == BPF_TRACE_RAW_TP) { 383138207291SMartin KaFai Lau tp_name = prog->aux->attach_func_name; 38329e4e01dfSKP Singh break; 38339e4e01dfSKP Singh } 38342fcc8241SKui-Feng Lee return bpf_tracing_prog_attach(prog, 0, 0, 0); 38359e4e01dfSKP Singh case BPF_PROG_TYPE_RAW_TRACEPOINT: 38369e4e01dfSKP Singh case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 3837df86ca0dSAndrii Nakryiko if (strncpy_from_user(buf, user_tp_name, sizeof(buf) - 1) < 0) 3838df86ca0dSAndrii Nakryiko return -EFAULT; 3839ac4414b5SAlexei Starovoitov buf[sizeof(buf) - 1] = 0; 3840ac4414b5SAlexei Starovoitov tp_name = buf; 38419e4e01dfSKP Singh break; 38429e4e01dfSKP Singh default: 3843df86ca0dSAndrii Nakryiko return -EINVAL; 3844ac4414b5SAlexei Starovoitov } 3845c4f6699dSAlexei Starovoitov 3846a38d1107SMatt Mullins btp = bpf_get_raw_tracepoint(tp_name); 3847df86ca0dSAndrii Nakryiko if (!btp) 3848df86ca0dSAndrii Nakryiko return -ENOENT; 3849c4f6699dSAlexei Starovoitov 3850babf3164SAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER); 3851babf3164SAndrii Nakryiko if (!link) { 3852a38d1107SMatt Mullins err = -ENOMEM; 3853a38d1107SMatt Mullins goto out_put_btp; 3854a38d1107SMatt Mullins } 3855f2e10bffSAndrii Nakryiko bpf_link_init(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT, 3856f2e10bffSAndrii Nakryiko &bpf_raw_tp_link_lops, prog); 3857babf3164SAndrii Nakryiko link->btp = btp; 3858c4f6699dSAlexei Starovoitov 3859a3b80e10SAndrii Nakryiko err = bpf_link_prime(&link->link, &link_primer); 3860a3b80e10SAndrii Nakryiko if (err) { 3861babf3164SAndrii Nakryiko kfree(link); 3862babf3164SAndrii Nakryiko goto out_put_btp; 3863c4f6699dSAlexei Starovoitov } 3864babf3164SAndrii Nakryiko 3865babf3164SAndrii Nakryiko err = bpf_probe_register(link->btp, prog); 3866babf3164SAndrii Nakryiko if (err) { 3867a3b80e10SAndrii Nakryiko bpf_link_cleanup(&link_primer); 3868babf3164SAndrii Nakryiko goto out_put_btp; 3869babf3164SAndrii Nakryiko } 3870babf3164SAndrii Nakryiko 3871a3b80e10SAndrii Nakryiko return bpf_link_settle(&link_primer); 3872c4f6699dSAlexei Starovoitov 3873a38d1107SMatt Mullins out_put_btp: 3874a38d1107SMatt Mullins bpf_put_raw_tracepoint(btp); 3875c4f6699dSAlexei Starovoitov return err; 3876c4f6699dSAlexei Starovoitov } 3877c4f6699dSAlexei Starovoitov 3878df86ca0dSAndrii Nakryiko #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd 3879df86ca0dSAndrii Nakryiko 3880df86ca0dSAndrii Nakryiko static int bpf_raw_tracepoint_open(const union bpf_attr *attr) 3881df86ca0dSAndrii Nakryiko { 3882df86ca0dSAndrii Nakryiko struct bpf_prog *prog; 3883df86ca0dSAndrii Nakryiko int fd; 3884df86ca0dSAndrii Nakryiko 3885df86ca0dSAndrii Nakryiko if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN)) 3886df86ca0dSAndrii Nakryiko return -EINVAL; 3887df86ca0dSAndrii Nakryiko 3888df86ca0dSAndrii Nakryiko prog = bpf_prog_get(attr->raw_tracepoint.prog_fd); 3889df86ca0dSAndrii Nakryiko if (IS_ERR(prog)) 3890df86ca0dSAndrii Nakryiko return PTR_ERR(prog); 3891df86ca0dSAndrii Nakryiko 3892df86ca0dSAndrii Nakryiko fd = bpf_raw_tp_link_attach(prog, u64_to_user_ptr(attr->raw_tracepoint.name)); 3893df86ca0dSAndrii Nakryiko if (fd < 0) 3894df86ca0dSAndrii Nakryiko bpf_prog_put(prog); 3895df86ca0dSAndrii Nakryiko return fd; 3896df86ca0dSAndrii Nakryiko } 3897df86ca0dSAndrii Nakryiko 3898e28784e3SAndrii Nakryiko static enum bpf_prog_type 3899e28784e3SAndrii Nakryiko attach_type_to_prog_type(enum bpf_attach_type attach_type) 3900e28784e3SAndrii Nakryiko { 3901e28784e3SAndrii Nakryiko switch (attach_type) { 3902e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_INGRESS: 3903e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_EGRESS: 3904e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SKB; 3905e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_SOCK_CREATE: 3906f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 3907e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_POST_BIND: 3908e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_POST_BIND: 3909e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCK; 3910e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_BIND: 3911e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_BIND: 3912e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_CONNECT: 3913e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_CONNECT: 3914859051ddSDaan De Meyer case BPF_CGROUP_UNIX_CONNECT: 39151b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 39161b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 3917859051ddSDaan De Meyer case BPF_CGROUP_UNIX_GETPEERNAME: 39181b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 39191b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 3920859051ddSDaan De Meyer case BPF_CGROUP_UNIX_GETSOCKNAME: 3921e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP4_SENDMSG: 3922e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP6_SENDMSG: 3923859051ddSDaan De Meyer case BPF_CGROUP_UNIX_SENDMSG: 3924e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP4_RECVMSG: 3925e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP6_RECVMSG: 3926859051ddSDaan De Meyer case BPF_CGROUP_UNIX_RECVMSG: 3927e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 3928e28784e3SAndrii Nakryiko case BPF_CGROUP_SOCK_OPS: 3929e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SOCK_OPS; 3930e28784e3SAndrii Nakryiko case BPF_CGROUP_DEVICE: 3931e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_DEVICE; 3932e28784e3SAndrii Nakryiko case BPF_SK_MSG_VERDICT: 3933e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SK_MSG; 3934e28784e3SAndrii Nakryiko case BPF_SK_SKB_STREAM_PARSER: 3935e28784e3SAndrii Nakryiko case BPF_SK_SKB_STREAM_VERDICT: 3936a7ba4558SCong Wang case BPF_SK_SKB_VERDICT: 3937e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SK_SKB; 3938e28784e3SAndrii Nakryiko case BPF_LIRC_MODE2: 3939e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_LIRC_MODE2; 3940e28784e3SAndrii Nakryiko case BPF_FLOW_DISSECTOR: 3941e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_FLOW_DISSECTOR; 3942e28784e3SAndrii Nakryiko case BPF_CGROUP_SYSCTL: 3943e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SYSCTL; 3944e28784e3SAndrii Nakryiko case BPF_CGROUP_GETSOCKOPT: 3945e28784e3SAndrii Nakryiko case BPF_CGROUP_SETSOCKOPT: 3946e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCKOPT; 3947de4e05caSYonghong Song case BPF_TRACE_ITER: 3948df86ca0dSAndrii Nakryiko case BPF_TRACE_RAW_TP: 3949df86ca0dSAndrii Nakryiko case BPF_TRACE_FENTRY: 3950df86ca0dSAndrii Nakryiko case BPF_TRACE_FEXIT: 3951df86ca0dSAndrii Nakryiko case BPF_MODIFY_RETURN: 3952de4e05caSYonghong Song return BPF_PROG_TYPE_TRACING; 3953df86ca0dSAndrii Nakryiko case BPF_LSM_MAC: 3954df86ca0dSAndrii Nakryiko return BPF_PROG_TYPE_LSM; 3955e9ddbb77SJakub Sitnicki case BPF_SK_LOOKUP: 3956e9ddbb77SJakub Sitnicki return BPF_PROG_TYPE_SK_LOOKUP; 3957aa8d3a71SAndrii Nakryiko case BPF_XDP: 3958aa8d3a71SAndrii Nakryiko return BPF_PROG_TYPE_XDP; 395969fd337aSStanislav Fomichev case BPF_LSM_CGROUP: 396069fd337aSStanislav Fomichev return BPF_PROG_TYPE_LSM; 3961e420bed0SDaniel Borkmann case BPF_TCX_INGRESS: 3962e420bed0SDaniel Borkmann case BPF_TCX_EGRESS: 396335dfaad7SDaniel Borkmann case BPF_NETKIT_PRIMARY: 396435dfaad7SDaniel Borkmann case BPF_NETKIT_PEER: 3965e420bed0SDaniel Borkmann return BPF_PROG_TYPE_SCHED_CLS; 3966e28784e3SAndrii Nakryiko default: 3967e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_UNSPEC; 3968e28784e3SAndrii Nakryiko } 3969e28784e3SAndrii Nakryiko } 3970e28784e3SAndrii Nakryiko 39713505cb9fSJiri Olsa static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, 39723505cb9fSJiri Olsa enum bpf_attach_type attach_type) 39733505cb9fSJiri Olsa { 39743505cb9fSJiri Olsa enum bpf_prog_type ptype; 39753505cb9fSJiri Olsa 39763505cb9fSJiri Olsa switch (prog->type) { 39773505cb9fSJiri Olsa case BPF_PROG_TYPE_CGROUP_SOCK: 39783505cb9fSJiri Olsa case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 39793505cb9fSJiri Olsa case BPF_PROG_TYPE_CGROUP_SOCKOPT: 39803505cb9fSJiri Olsa case BPF_PROG_TYPE_SK_LOOKUP: 39813505cb9fSJiri Olsa return attach_type == prog->expected_attach_type ? 0 : -EINVAL; 39823505cb9fSJiri Olsa case BPF_PROG_TYPE_CGROUP_SKB: 3983caf8f28eSAndrii Nakryiko if (!bpf_token_capable(prog->aux->token, CAP_NET_ADMIN)) 39843505cb9fSJiri Olsa /* cg-skb progs can be loaded by unpriv user. 39853505cb9fSJiri Olsa * check permissions at attach time. 39863505cb9fSJiri Olsa */ 39873505cb9fSJiri Olsa return -EPERM; 39883505cb9fSJiri Olsa return prog->enforce_expected_attach_type && 39893505cb9fSJiri Olsa prog->expected_attach_type != attach_type ? 39903505cb9fSJiri Olsa -EINVAL : 0; 39913505cb9fSJiri Olsa case BPF_PROG_TYPE_EXT: 39923505cb9fSJiri Olsa return 0; 39933505cb9fSJiri Olsa case BPF_PROG_TYPE_NETFILTER: 39943505cb9fSJiri Olsa if (attach_type != BPF_NETFILTER) 39953505cb9fSJiri Olsa return -EINVAL; 39963505cb9fSJiri Olsa return 0; 39973505cb9fSJiri Olsa case BPF_PROG_TYPE_PERF_EVENT: 39983505cb9fSJiri Olsa case BPF_PROG_TYPE_TRACEPOINT: 39993505cb9fSJiri Olsa if (attach_type != BPF_PERF_EVENT) 40003505cb9fSJiri Olsa return -EINVAL; 40013505cb9fSJiri Olsa return 0; 40023505cb9fSJiri Olsa case BPF_PROG_TYPE_KPROBE: 40033505cb9fSJiri Olsa if (prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI && 40043505cb9fSJiri Olsa attach_type != BPF_TRACE_KPROBE_MULTI) 40053505cb9fSJiri Olsa return -EINVAL; 400689ae89f5SJiri Olsa if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI && 400789ae89f5SJiri Olsa attach_type != BPF_TRACE_UPROBE_MULTI) 400889ae89f5SJiri Olsa return -EINVAL; 40093505cb9fSJiri Olsa if (attach_type != BPF_PERF_EVENT && 401089ae89f5SJiri Olsa attach_type != BPF_TRACE_KPROBE_MULTI && 401189ae89f5SJiri Olsa attach_type != BPF_TRACE_UPROBE_MULTI) 40123505cb9fSJiri Olsa return -EINVAL; 40133505cb9fSJiri Olsa return 0; 40143505cb9fSJiri Olsa case BPF_PROG_TYPE_SCHED_CLS: 40153505cb9fSJiri Olsa if (attach_type != BPF_TCX_INGRESS && 401635dfaad7SDaniel Borkmann attach_type != BPF_TCX_EGRESS && 401735dfaad7SDaniel Borkmann attach_type != BPF_NETKIT_PRIMARY && 401835dfaad7SDaniel Borkmann attach_type != BPF_NETKIT_PEER) 40193505cb9fSJiri Olsa return -EINVAL; 40203505cb9fSJiri Olsa return 0; 40213505cb9fSJiri Olsa default: 40223505cb9fSJiri Olsa ptype = attach_type_to_prog_type(attach_type); 40233505cb9fSJiri Olsa if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type) 40243505cb9fSJiri Olsa return -EINVAL; 40253505cb9fSJiri Olsa return 0; 40263505cb9fSJiri Olsa } 40273505cb9fSJiri Olsa } 40283505cb9fSJiri Olsa 4029e420bed0SDaniel Borkmann #define BPF_PROG_ATTACH_LAST_FIELD expected_revision 4030174a79ffSJohn Fastabend 4031e420bed0SDaniel Borkmann #define BPF_F_ATTACH_MASK_BASE \ 4032e420bed0SDaniel Borkmann (BPF_F_ALLOW_OVERRIDE | \ 4033e420bed0SDaniel Borkmann BPF_F_ALLOW_MULTI | \ 4034e420bed0SDaniel Borkmann BPF_F_REPLACE) 4035e420bed0SDaniel Borkmann 4036e420bed0SDaniel Borkmann #define BPF_F_ATTACH_MASK_MPROG \ 4037e420bed0SDaniel Borkmann (BPF_F_REPLACE | \ 4038e420bed0SDaniel Borkmann BPF_F_BEFORE | \ 4039e420bed0SDaniel Borkmann BPF_F_AFTER | \ 4040e420bed0SDaniel Borkmann BPF_F_ID | \ 4041e420bed0SDaniel Borkmann BPF_F_LINK) 4042324bda9eSAlexei Starovoitov 4043f4324551SDaniel Mack static int bpf_prog_attach(const union bpf_attr *attr) 4044f4324551SDaniel Mack { 40457f677633SAlexei Starovoitov enum bpf_prog_type ptype; 4046f4324551SDaniel Mack struct bpf_prog *prog; 40477f677633SAlexei Starovoitov int ret; 4048f4324551SDaniel Mack 4049f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_ATTACH)) 4050f4324551SDaniel Mack return -EINVAL; 4051f4324551SDaniel Mack 4052e28784e3SAndrii Nakryiko ptype = attach_type_to_prog_type(attr->attach_type); 4053e28784e3SAndrii Nakryiko if (ptype == BPF_PROG_TYPE_UNSPEC) 4054b2cd1257SDavid Ahern return -EINVAL; 4055ba62d611SLorenz Bauer if (bpf_mprog_supported(ptype)) { 4056ba62d611SLorenz Bauer if (attr->attach_flags & ~BPF_F_ATTACH_MASK_MPROG) 4057e420bed0SDaniel Borkmann return -EINVAL; 4058ba62d611SLorenz Bauer } else { 4059ba62d611SLorenz Bauer if (attr->attach_flags & ~BPF_F_ATTACH_MASK_BASE) 4060ba62d611SLorenz Bauer return -EINVAL; 4061ba62d611SLorenz Bauer if (attr->relative_fd || 4062ba62d611SLorenz Bauer attr->expected_revision) 4063ba62d611SLorenz Bauer return -EINVAL; 4064ba62d611SLorenz Bauer } 4065b2cd1257SDavid Ahern 4066b2cd1257SDavid Ahern prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 4067f4324551SDaniel Mack if (IS_ERR(prog)) 4068f4324551SDaniel Mack return PTR_ERR(prog); 4069f4324551SDaniel Mack 40705e43f899SAndrey Ignatov if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) { 40715e43f899SAndrey Ignatov bpf_prog_put(prog); 40725e43f899SAndrey Ignatov return -EINVAL; 40735e43f899SAndrey Ignatov } 40745e43f899SAndrey Ignatov 4075fdb5c453SSean Young switch (ptype) { 4076fdb5c453SSean Young case BPF_PROG_TYPE_SK_SKB: 4077fdb5c453SSean Young case BPF_PROG_TYPE_SK_MSG: 4078604326b4SDaniel Borkmann ret = sock_map_get_from_fd(attr, prog); 4079fdb5c453SSean Young break; 4080fdb5c453SSean Young case BPF_PROG_TYPE_LIRC_MODE2: 4081fdb5c453SSean Young ret = lirc_prog_attach(attr, prog); 4082fdb5c453SSean Young break; 4083d58e468bSPetar Penkov case BPF_PROG_TYPE_FLOW_DISSECTOR: 4084a3fd7ceeSJakub Sitnicki ret = netns_bpf_prog_attach(attr, prog); 4085d58e468bSPetar Penkov break; 4086e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 4087e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 4088e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 4089e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 4090e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 4091e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 4092e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 409369fd337aSStanislav Fomichev case BPF_PROG_TYPE_LSM: 409469fd337aSStanislav Fomichev if (ptype == BPF_PROG_TYPE_LSM && 409569fd337aSStanislav Fomichev prog->expected_attach_type != BPF_LSM_CGROUP) 4096e89f3edfSMilan Landaverde ret = -EINVAL; 4097e89f3edfSMilan Landaverde else 4098fdb5c453SSean Young ret = cgroup_bpf_prog_attach(attr, ptype, prog); 4099e28784e3SAndrii Nakryiko break; 4100e420bed0SDaniel Borkmann case BPF_PROG_TYPE_SCHED_CLS: 410135dfaad7SDaniel Borkmann if (attr->attach_type == BPF_TCX_INGRESS || 410235dfaad7SDaniel Borkmann attr->attach_type == BPF_TCX_EGRESS) 4103e420bed0SDaniel Borkmann ret = tcx_prog_attach(attr, prog); 410435dfaad7SDaniel Borkmann else 410535dfaad7SDaniel Borkmann ret = netkit_prog_attach(attr, prog); 4106e420bed0SDaniel Borkmann break; 4107e28784e3SAndrii Nakryiko default: 4108e28784e3SAndrii Nakryiko ret = -EINVAL; 4109f4324551SDaniel Mack } 4110f4324551SDaniel Mack 41117f677633SAlexei Starovoitov if (ret) 41127f677633SAlexei Starovoitov bpf_prog_put(prog); 41137f677633SAlexei Starovoitov return ret; 4114f4324551SDaniel Mack } 4115f4324551SDaniel Mack 4116e420bed0SDaniel Borkmann #define BPF_PROG_DETACH_LAST_FIELD expected_revision 4117f4324551SDaniel Mack 4118f4324551SDaniel Mack static int bpf_prog_detach(const union bpf_attr *attr) 4119f4324551SDaniel Mack { 4120e420bed0SDaniel Borkmann struct bpf_prog *prog = NULL; 4121324bda9eSAlexei Starovoitov enum bpf_prog_type ptype; 4122e420bed0SDaniel Borkmann int ret; 4123f4324551SDaniel Mack 4124f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_DETACH)) 4125f4324551SDaniel Mack return -EINVAL; 4126f4324551SDaniel Mack 4127e28784e3SAndrii Nakryiko ptype = attach_type_to_prog_type(attr->attach_type); 4128e420bed0SDaniel Borkmann if (bpf_mprog_supported(ptype)) { 4129e420bed0SDaniel Borkmann if (ptype == BPF_PROG_TYPE_UNSPEC) 4130e420bed0SDaniel Borkmann return -EINVAL; 4131e420bed0SDaniel Borkmann if (attr->attach_flags & ~BPF_F_ATTACH_MASK_MPROG) 4132e420bed0SDaniel Borkmann return -EINVAL; 4133e420bed0SDaniel Borkmann if (attr->attach_bpf_fd) { 4134e420bed0SDaniel Borkmann prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 4135e420bed0SDaniel Borkmann if (IS_ERR(prog)) 4136e420bed0SDaniel Borkmann return PTR_ERR(prog); 4137e420bed0SDaniel Borkmann } 4138ba62d611SLorenz Bauer } else if (attr->attach_flags || 4139ba62d611SLorenz Bauer attr->relative_fd || 4140ba62d611SLorenz Bauer attr->expected_revision) { 4141ba62d611SLorenz Bauer return -EINVAL; 4142e420bed0SDaniel Borkmann } 4143e28784e3SAndrii Nakryiko 4144e28784e3SAndrii Nakryiko switch (ptype) { 4145e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SK_MSG: 4146e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SK_SKB: 4147e420bed0SDaniel Borkmann ret = sock_map_prog_detach(attr, ptype); 4148e420bed0SDaniel Borkmann break; 4149e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_LIRC_MODE2: 4150e420bed0SDaniel Borkmann ret = lirc_prog_detach(attr); 4151e420bed0SDaniel Borkmann break; 4152e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_FLOW_DISSECTOR: 4153e420bed0SDaniel Borkmann ret = netns_bpf_prog_detach(attr, ptype); 4154e420bed0SDaniel Borkmann break; 4155e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 4156e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 4157e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 4158e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 4159e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 4160e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 4161e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 416269fd337aSStanislav Fomichev case BPF_PROG_TYPE_LSM: 4163e420bed0SDaniel Borkmann ret = cgroup_bpf_prog_detach(attr, ptype); 4164e420bed0SDaniel Borkmann break; 4165e420bed0SDaniel Borkmann case BPF_PROG_TYPE_SCHED_CLS: 416635dfaad7SDaniel Borkmann if (attr->attach_type == BPF_TCX_INGRESS || 416735dfaad7SDaniel Borkmann attr->attach_type == BPF_TCX_EGRESS) 4168e420bed0SDaniel Borkmann ret = tcx_prog_detach(attr, prog); 416935dfaad7SDaniel Borkmann else 417035dfaad7SDaniel Borkmann ret = netkit_prog_detach(attr, prog); 4171e420bed0SDaniel Borkmann break; 4172f4324551SDaniel Mack default: 4173e420bed0SDaniel Borkmann ret = -EINVAL; 4174f4324551SDaniel Mack } 417540304b2aSLawrence Brakmo 4176e420bed0SDaniel Borkmann if (prog) 4177e420bed0SDaniel Borkmann bpf_prog_put(prog); 4178e420bed0SDaniel Borkmann return ret; 4179e420bed0SDaniel Borkmann } 4180e420bed0SDaniel Borkmann 4181a4fe7838SDaniel Borkmann #define BPF_PROG_QUERY_LAST_FIELD query.revision 4182468e2f64SAlexei Starovoitov 4183468e2f64SAlexei Starovoitov static int bpf_prog_query(const union bpf_attr *attr, 4184468e2f64SAlexei Starovoitov union bpf_attr __user *uattr) 4185468e2f64SAlexei Starovoitov { 4186ed1ad5a7SAndrii Nakryiko if (!bpf_net_capable()) 4187468e2f64SAlexei Starovoitov return -EPERM; 4188468e2f64SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_QUERY)) 4189468e2f64SAlexei Starovoitov return -EINVAL; 4190468e2f64SAlexei Starovoitov if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE) 4191468e2f64SAlexei Starovoitov return -EINVAL; 4192468e2f64SAlexei Starovoitov 4193468e2f64SAlexei Starovoitov switch (attr->query.attach_type) { 4194468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_INGRESS: 4195468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_EGRESS: 4196468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_SOCK_CREATE: 4197f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE: 41984fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 41994fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 4200aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 4201aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 4202d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 4203d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 4204859051ddSDaan De Meyer case BPF_CGROUP_UNIX_CONNECT: 42051b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME: 42061b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME: 4207859051ddSDaan De Meyer case BPF_CGROUP_UNIX_GETPEERNAME: 42081b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME: 42091b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME: 4210859051ddSDaan De Meyer case BPF_CGROUP_UNIX_GETSOCKNAME: 42111cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 42121cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 4213859051ddSDaan De Meyer case BPF_CGROUP_UNIX_SENDMSG: 4214983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG: 4215983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG: 4216859051ddSDaan De Meyer case BPF_CGROUP_UNIX_RECVMSG: 4217468e2f64SAlexei Starovoitov case BPF_CGROUP_SOCK_OPS: 4218ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 42197b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 42200d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT: 42210d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT: 4222b79c9fc9SStanislav Fomichev case BPF_LSM_CGROUP: 4223e28784e3SAndrii Nakryiko return cgroup_bpf_prog_query(attr, uattr); 4224f4364dcfSSean Young case BPF_LIRC_MODE2: 4225f4364dcfSSean Young return lirc_prog_query(attr, uattr); 4226118c8e9aSStanislav Fomichev case BPF_FLOW_DISSECTOR: 4227e9ddbb77SJakub Sitnicki case BPF_SK_LOOKUP: 4228a3fd7ceeSJakub Sitnicki return netns_bpf_prog_query(attr, uattr); 4229748cd572SDi Zhu case BPF_SK_SKB_STREAM_PARSER: 4230748cd572SDi Zhu case BPF_SK_SKB_STREAM_VERDICT: 4231748cd572SDi Zhu case BPF_SK_MSG_VERDICT: 4232748cd572SDi Zhu case BPF_SK_SKB_VERDICT: 4233748cd572SDi Zhu return sock_map_bpf_prog_query(attr, uattr); 4234e420bed0SDaniel Borkmann case BPF_TCX_INGRESS: 4235e420bed0SDaniel Borkmann case BPF_TCX_EGRESS: 4236e420bed0SDaniel Borkmann return tcx_prog_query(attr, uattr); 423735dfaad7SDaniel Borkmann case BPF_NETKIT_PRIMARY: 423835dfaad7SDaniel Borkmann case BPF_NETKIT_PEER: 423935dfaad7SDaniel Borkmann return netkit_prog_query(attr, uattr); 4240468e2f64SAlexei Starovoitov default: 4241468e2f64SAlexei Starovoitov return -EINVAL; 4242468e2f64SAlexei Starovoitov } 4243468e2f64SAlexei Starovoitov } 4244f4324551SDaniel Mack 4245b530e9e1SToke Høiland-Jørgensen #define BPF_PROG_TEST_RUN_LAST_FIELD test.batch_size 42461cf1cae9SAlexei Starovoitov 42471cf1cae9SAlexei Starovoitov static int bpf_prog_test_run(const union bpf_attr *attr, 42481cf1cae9SAlexei Starovoitov union bpf_attr __user *uattr) 42491cf1cae9SAlexei Starovoitov { 42501cf1cae9SAlexei Starovoitov struct bpf_prog *prog; 42511cf1cae9SAlexei Starovoitov int ret = -ENOTSUPP; 42521cf1cae9SAlexei Starovoitov 42531cf1cae9SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 42541cf1cae9SAlexei Starovoitov return -EINVAL; 42551cf1cae9SAlexei Starovoitov 4256b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_in && !attr->test.ctx_in) || 4257b0b9395dSStanislav Fomichev (!attr->test.ctx_size_in && attr->test.ctx_in)) 4258b0b9395dSStanislav Fomichev return -EINVAL; 4259b0b9395dSStanislav Fomichev 4260b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_out && !attr->test.ctx_out) || 4261b0b9395dSStanislav Fomichev (!attr->test.ctx_size_out && attr->test.ctx_out)) 4262b0b9395dSStanislav Fomichev return -EINVAL; 4263b0b9395dSStanislav Fomichev 42641cf1cae9SAlexei Starovoitov prog = bpf_prog_get(attr->test.prog_fd); 42651cf1cae9SAlexei Starovoitov if (IS_ERR(prog)) 42661cf1cae9SAlexei Starovoitov return PTR_ERR(prog); 42671cf1cae9SAlexei Starovoitov 42681cf1cae9SAlexei Starovoitov if (prog->aux->ops->test_run) 42691cf1cae9SAlexei Starovoitov ret = prog->aux->ops->test_run(prog, attr, uattr); 42701cf1cae9SAlexei Starovoitov 42711cf1cae9SAlexei Starovoitov bpf_prog_put(prog); 42721cf1cae9SAlexei Starovoitov return ret; 42731cf1cae9SAlexei Starovoitov } 42741cf1cae9SAlexei Starovoitov 427534ad5580SMartin KaFai Lau #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 427634ad5580SMartin KaFai Lau 427734ad5580SMartin KaFai Lau static int bpf_obj_get_next_id(const union bpf_attr *attr, 427834ad5580SMartin KaFai Lau union bpf_attr __user *uattr, 427934ad5580SMartin KaFai Lau struct idr *idr, 428034ad5580SMartin KaFai Lau spinlock_t *lock) 428134ad5580SMartin KaFai Lau { 428234ad5580SMartin KaFai Lau u32 next_id = attr->start_id; 428334ad5580SMartin KaFai Lau int err = 0; 428434ad5580SMartin KaFai Lau 428534ad5580SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 428634ad5580SMartin KaFai Lau return -EINVAL; 428734ad5580SMartin KaFai Lau 428834ad5580SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 428934ad5580SMartin KaFai Lau return -EPERM; 429034ad5580SMartin KaFai Lau 429134ad5580SMartin KaFai Lau next_id++; 429234ad5580SMartin KaFai Lau spin_lock_bh(lock); 429334ad5580SMartin KaFai Lau if (!idr_get_next(idr, &next_id)) 429434ad5580SMartin KaFai Lau err = -ENOENT; 429534ad5580SMartin KaFai Lau spin_unlock_bh(lock); 429634ad5580SMartin KaFai Lau 429734ad5580SMartin KaFai Lau if (!err) 429834ad5580SMartin KaFai Lau err = put_user(next_id, &uattr->next_id); 429934ad5580SMartin KaFai Lau 430034ad5580SMartin KaFai Lau return err; 430134ad5580SMartin KaFai Lau } 430234ad5580SMartin KaFai Lau 43036086d29dSYonghong Song struct bpf_map *bpf_map_get_curr_or_next(u32 *id) 43046086d29dSYonghong Song { 43056086d29dSYonghong Song struct bpf_map *map; 43066086d29dSYonghong Song 43076086d29dSYonghong Song spin_lock_bh(&map_idr_lock); 43086086d29dSYonghong Song again: 43096086d29dSYonghong Song map = idr_get_next(&map_idr, id); 43106086d29dSYonghong Song if (map) { 43116086d29dSYonghong Song map = __bpf_map_inc_not_zero(map, false); 43126086d29dSYonghong Song if (IS_ERR(map)) { 43136086d29dSYonghong Song (*id)++; 43146086d29dSYonghong Song goto again; 43156086d29dSYonghong Song } 43166086d29dSYonghong Song } 43176086d29dSYonghong Song spin_unlock_bh(&map_idr_lock); 43186086d29dSYonghong Song 43196086d29dSYonghong Song return map; 43206086d29dSYonghong Song } 43216086d29dSYonghong Song 4322a228a64fSAlexei Starovoitov struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id) 4323a228a64fSAlexei Starovoitov { 4324a228a64fSAlexei Starovoitov struct bpf_prog *prog; 4325a228a64fSAlexei Starovoitov 4326a228a64fSAlexei Starovoitov spin_lock_bh(&prog_idr_lock); 4327a228a64fSAlexei Starovoitov again: 4328a228a64fSAlexei Starovoitov prog = idr_get_next(&prog_idr, id); 4329a228a64fSAlexei Starovoitov if (prog) { 4330a228a64fSAlexei Starovoitov prog = bpf_prog_inc_not_zero(prog); 4331a228a64fSAlexei Starovoitov if (IS_ERR(prog)) { 4332a228a64fSAlexei Starovoitov (*id)++; 4333a228a64fSAlexei Starovoitov goto again; 4334a228a64fSAlexei Starovoitov } 4335a228a64fSAlexei Starovoitov } 4336a228a64fSAlexei Starovoitov spin_unlock_bh(&prog_idr_lock); 4337a228a64fSAlexei Starovoitov 4338a228a64fSAlexei Starovoitov return prog; 4339a228a64fSAlexei Starovoitov } 4340a228a64fSAlexei Starovoitov 4341b16d9aa4SMartin KaFai Lau #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 4342b16d9aa4SMartin KaFai Lau 43437e6897f9SBjörn Töpel struct bpf_prog *bpf_prog_by_id(u32 id) 43447e6897f9SBjörn Töpel { 43457e6897f9SBjörn Töpel struct bpf_prog *prog; 43467e6897f9SBjörn Töpel 43477e6897f9SBjörn Töpel if (!id) 43487e6897f9SBjörn Töpel return ERR_PTR(-ENOENT); 43497e6897f9SBjörn Töpel 43507e6897f9SBjörn Töpel spin_lock_bh(&prog_idr_lock); 43517e6897f9SBjörn Töpel prog = idr_find(&prog_idr, id); 43527e6897f9SBjörn Töpel if (prog) 43537e6897f9SBjörn Töpel prog = bpf_prog_inc_not_zero(prog); 43547e6897f9SBjörn Töpel else 43557e6897f9SBjörn Töpel prog = ERR_PTR(-ENOENT); 43567e6897f9SBjörn Töpel spin_unlock_bh(&prog_idr_lock); 43577e6897f9SBjörn Töpel return prog; 43587e6897f9SBjörn Töpel } 43597e6897f9SBjörn Töpel 4360b16d9aa4SMartin KaFai Lau static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 4361b16d9aa4SMartin KaFai Lau { 4362b16d9aa4SMartin KaFai Lau struct bpf_prog *prog; 4363b16d9aa4SMartin KaFai Lau u32 id = attr->prog_id; 4364b16d9aa4SMartin KaFai Lau int fd; 4365b16d9aa4SMartin KaFai Lau 4366b16d9aa4SMartin KaFai Lau if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 4367b16d9aa4SMartin KaFai Lau return -EINVAL; 4368b16d9aa4SMartin KaFai Lau 4369b16d9aa4SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 4370b16d9aa4SMartin KaFai Lau return -EPERM; 4371b16d9aa4SMartin KaFai Lau 43727e6897f9SBjörn Töpel prog = bpf_prog_by_id(id); 4373b16d9aa4SMartin KaFai Lau if (IS_ERR(prog)) 4374b16d9aa4SMartin KaFai Lau return PTR_ERR(prog); 4375b16d9aa4SMartin KaFai Lau 4376b16d9aa4SMartin KaFai Lau fd = bpf_prog_new_fd(prog); 4377b16d9aa4SMartin KaFai Lau if (fd < 0) 4378b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 4379b16d9aa4SMartin KaFai Lau 4380b16d9aa4SMartin KaFai Lau return fd; 4381b16d9aa4SMartin KaFai Lau } 4382b16d9aa4SMartin KaFai Lau 43836e71b04aSChenbo Feng #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags 4384bd5f5f4eSMartin KaFai Lau 4385bd5f5f4eSMartin KaFai Lau static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 4386bd5f5f4eSMartin KaFai Lau { 4387bd5f5f4eSMartin KaFai Lau struct bpf_map *map; 4388bd5f5f4eSMartin KaFai Lau u32 id = attr->map_id; 43896e71b04aSChenbo Feng int f_flags; 4390bd5f5f4eSMartin KaFai Lau int fd; 4391bd5f5f4eSMartin KaFai Lau 43926e71b04aSChenbo Feng if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || 43936e71b04aSChenbo Feng attr->open_flags & ~BPF_OBJ_FLAG_MASK) 4394bd5f5f4eSMartin KaFai Lau return -EINVAL; 4395bd5f5f4eSMartin KaFai Lau 4396bd5f5f4eSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 4397bd5f5f4eSMartin KaFai Lau return -EPERM; 4398bd5f5f4eSMartin KaFai Lau 43996e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->open_flags); 44006e71b04aSChenbo Feng if (f_flags < 0) 44016e71b04aSChenbo Feng return f_flags; 44026e71b04aSChenbo Feng 4403bd5f5f4eSMartin KaFai Lau spin_lock_bh(&map_idr_lock); 4404bd5f5f4eSMartin KaFai Lau map = idr_find(&map_idr, id); 4405bd5f5f4eSMartin KaFai Lau if (map) 4406b0e4701cSStanislav Fomichev map = __bpf_map_inc_not_zero(map, true); 4407bd5f5f4eSMartin KaFai Lau else 4408bd5f5f4eSMartin KaFai Lau map = ERR_PTR(-ENOENT); 4409bd5f5f4eSMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 4410bd5f5f4eSMartin KaFai Lau 4411bd5f5f4eSMartin KaFai Lau if (IS_ERR(map)) 4412bd5f5f4eSMartin KaFai Lau return PTR_ERR(map); 4413bd5f5f4eSMartin KaFai Lau 44146e71b04aSChenbo Feng fd = bpf_map_new_fd(map, f_flags); 4415bd5f5f4eSMartin KaFai Lau if (fd < 0) 4416781e6282SPeng Sun bpf_map_put_with_uref(map); 4417bd5f5f4eSMartin KaFai Lau 4418bd5f5f4eSMartin KaFai Lau return fd; 4419bd5f5f4eSMartin KaFai Lau } 4420bd5f5f4eSMartin KaFai Lau 44217105e828SDaniel Borkmann static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, 4422d8eca5bbSDaniel Borkmann unsigned long addr, u32 *off, 4423d8eca5bbSDaniel Borkmann u32 *type) 44247105e828SDaniel Borkmann { 4425d8eca5bbSDaniel Borkmann const struct bpf_map *map; 44267105e828SDaniel Borkmann int i; 44277105e828SDaniel Borkmann 4428984fe94fSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 4429d8eca5bbSDaniel Borkmann for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { 4430d8eca5bbSDaniel Borkmann map = prog->aux->used_maps[i]; 4431d8eca5bbSDaniel Borkmann if (map == (void *)addr) { 4432d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_FD; 4433984fe94fSYiFei Zhu goto out; 4434d8eca5bbSDaniel Borkmann } 4435d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta) 4436d8eca5bbSDaniel Borkmann continue; 4437d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta(map, addr, off)) { 4438d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_VALUE; 4439984fe94fSYiFei Zhu goto out; 4440d8eca5bbSDaniel Borkmann } 4441d8eca5bbSDaniel Borkmann } 4442984fe94fSYiFei Zhu map = NULL; 4443d8eca5bbSDaniel Borkmann 4444984fe94fSYiFei Zhu out: 4445984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 4446984fe94fSYiFei Zhu return map; 44477105e828SDaniel Borkmann } 44487105e828SDaniel Borkmann 444963960260SKees Cook static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, 445063960260SKees Cook const struct cred *f_cred) 44517105e828SDaniel Borkmann { 44527105e828SDaniel Borkmann const struct bpf_map *map; 44537105e828SDaniel Borkmann struct bpf_insn *insns; 4454d8eca5bbSDaniel Borkmann u32 off, type; 44557105e828SDaniel Borkmann u64 imm; 445629fcb05bSAndrii Nakryiko u8 code; 44577105e828SDaniel Borkmann int i; 44587105e828SDaniel Borkmann 44597105e828SDaniel Borkmann insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), 44607105e828SDaniel Borkmann GFP_USER); 44617105e828SDaniel Borkmann if (!insns) 44627105e828SDaniel Borkmann return insns; 44637105e828SDaniel Borkmann 44647105e828SDaniel Borkmann for (i = 0; i < prog->len; i++) { 446529fcb05bSAndrii Nakryiko code = insns[i].code; 446629fcb05bSAndrii Nakryiko 446729fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_TAIL_CALL)) { 44687105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 44697105e828SDaniel Borkmann insns[i].imm = BPF_FUNC_tail_call; 44707105e828SDaniel Borkmann /* fall-through */ 44717105e828SDaniel Borkmann } 447229fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_CALL) || 447329fcb05bSAndrii Nakryiko code == (BPF_JMP | BPF_CALL_ARGS)) { 447429fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_CALL_ARGS)) 44757105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 447663960260SKees Cook if (!bpf_dump_raw_ok(f_cred)) 44777105e828SDaniel Borkmann insns[i].imm = 0; 44787105e828SDaniel Borkmann continue; 44797105e828SDaniel Borkmann } 448029fcb05bSAndrii Nakryiko if (BPF_CLASS(code) == BPF_LDX && BPF_MODE(code) == BPF_PROBE_MEM) { 448129fcb05bSAndrii Nakryiko insns[i].code = BPF_LDX | BPF_SIZE(code) | BPF_MEM; 448229fcb05bSAndrii Nakryiko continue; 448329fcb05bSAndrii Nakryiko } 44847105e828SDaniel Borkmann 44856082b6c3SAlexei Starovoitov if ((BPF_CLASS(code) == BPF_LDX || BPF_CLASS(code) == BPF_STX || 44866082b6c3SAlexei Starovoitov BPF_CLASS(code) == BPF_ST) && BPF_MODE(code) == BPF_PROBE_MEM32) { 44876082b6c3SAlexei Starovoitov insns[i].code = BPF_CLASS(code) | BPF_SIZE(code) | BPF_MEM; 44886082b6c3SAlexei Starovoitov continue; 44896082b6c3SAlexei Starovoitov } 44906082b6c3SAlexei Starovoitov 449129fcb05bSAndrii Nakryiko if (code != (BPF_LD | BPF_IMM | BPF_DW)) 44927105e828SDaniel Borkmann continue; 44937105e828SDaniel Borkmann 44947105e828SDaniel Borkmann imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; 4495d8eca5bbSDaniel Borkmann map = bpf_map_from_imm(prog, imm, &off, &type); 44967105e828SDaniel Borkmann if (map) { 4497d8eca5bbSDaniel Borkmann insns[i].src_reg = type; 44987105e828SDaniel Borkmann insns[i].imm = map->id; 4499d8eca5bbSDaniel Borkmann insns[i + 1].imm = off; 45007105e828SDaniel Borkmann continue; 45017105e828SDaniel Borkmann } 45027105e828SDaniel Borkmann } 45037105e828SDaniel Borkmann 45047105e828SDaniel Borkmann return insns; 45057105e828SDaniel Borkmann } 45067105e828SDaniel Borkmann 4507c454a46bSMartin KaFai Lau static int set_info_rec_size(struct bpf_prog_info *info) 4508c454a46bSMartin KaFai Lau { 4509c454a46bSMartin KaFai Lau /* 4510c454a46bSMartin KaFai Lau * Ensure info.*_rec_size is the same as kernel expected size 4511c454a46bSMartin KaFai Lau * 4512c454a46bSMartin KaFai Lau * or 4513c454a46bSMartin KaFai Lau * 4514c454a46bSMartin KaFai Lau * Only allow zero *_rec_size if both _rec_size and _cnt are 4515c454a46bSMartin KaFai Lau * zero. In this case, the kernel will set the expected 4516c454a46bSMartin KaFai Lau * _rec_size back to the info. 4517c454a46bSMartin KaFai Lau */ 4518c454a46bSMartin KaFai Lau 451911d8b82dSYonghong Song if ((info->nr_func_info || info->func_info_rec_size) && 4520c454a46bSMartin KaFai Lau info->func_info_rec_size != sizeof(struct bpf_func_info)) 4521c454a46bSMartin KaFai Lau return -EINVAL; 4522c454a46bSMartin KaFai Lau 452311d8b82dSYonghong Song if ((info->nr_line_info || info->line_info_rec_size) && 4524c454a46bSMartin KaFai Lau info->line_info_rec_size != sizeof(struct bpf_line_info)) 4525c454a46bSMartin KaFai Lau return -EINVAL; 4526c454a46bSMartin KaFai Lau 452711d8b82dSYonghong Song if ((info->nr_jited_line_info || info->jited_line_info_rec_size) && 4528c454a46bSMartin KaFai Lau info->jited_line_info_rec_size != sizeof(__u64)) 4529c454a46bSMartin KaFai Lau return -EINVAL; 4530c454a46bSMartin KaFai Lau 4531c454a46bSMartin KaFai Lau info->func_info_rec_size = sizeof(struct bpf_func_info); 4532c454a46bSMartin KaFai Lau info->line_info_rec_size = sizeof(struct bpf_line_info); 4533c454a46bSMartin KaFai Lau info->jited_line_info_rec_size = sizeof(__u64); 4534c454a46bSMartin KaFai Lau 4535c454a46bSMartin KaFai Lau return 0; 4536c454a46bSMartin KaFai Lau } 4537c454a46bSMartin KaFai Lau 453863960260SKees Cook static int bpf_prog_get_info_by_fd(struct file *file, 453963960260SKees Cook struct bpf_prog *prog, 45401e270976SMartin KaFai Lau const union bpf_attr *attr, 45411e270976SMartin KaFai Lau union bpf_attr __user *uattr) 45421e270976SMartin KaFai Lau { 45431e270976SMartin KaFai Lau struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 45446644aabbSStanislav Fomichev struct btf *attach_btf = bpf_prog_get_target_btf(prog); 45455c6f2588SGreg Kroah-Hartman struct bpf_prog_info info; 45461e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 454761a0abaeSEric Dumazet struct bpf_prog_kstats stats; 45481e270976SMartin KaFai Lau char __user *uinsns; 45491e270976SMartin KaFai Lau u32 ulen; 45501e270976SMartin KaFai Lau int err; 45511e270976SMartin KaFai Lau 4552af2ac3e1SAlexei Starovoitov err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len); 45531e270976SMartin KaFai Lau if (err) 45541e270976SMartin KaFai Lau return err; 45551e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 45561e270976SMartin KaFai Lau 45575c6f2588SGreg Kroah-Hartman memset(&info, 0, sizeof(info)); 45581e270976SMartin KaFai Lau if (copy_from_user(&info, uinfo, info_len)) 455989b09689SDaniel Borkmann return -EFAULT; 45601e270976SMartin KaFai Lau 45611e270976SMartin KaFai Lau info.type = prog->type; 45621e270976SMartin KaFai Lau info.id = prog->aux->id; 4563cb4d2b3fSMartin KaFai Lau info.load_time = prog->aux->load_time; 4564cb4d2b3fSMartin KaFai Lau info.created_by_uid = from_kuid_munged(current_user_ns(), 4565cb4d2b3fSMartin KaFai Lau prog->aux->user->uid); 4566b85fab0eSJiri Olsa info.gpl_compatible = prog->gpl_compatible; 45671e270976SMartin KaFai Lau 45681e270976SMartin KaFai Lau memcpy(info.tag, prog->tag, sizeof(prog->tag)); 4569cb4d2b3fSMartin KaFai Lau memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); 4570cb4d2b3fSMartin KaFai Lau 4571984fe94fSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 4572cb4d2b3fSMartin KaFai Lau ulen = info.nr_map_ids; 4573cb4d2b3fSMartin KaFai Lau info.nr_map_ids = prog->aux->used_map_cnt; 4574cb4d2b3fSMartin KaFai Lau ulen = min_t(u32, info.nr_map_ids, ulen); 4575cb4d2b3fSMartin KaFai Lau if (ulen) { 4576721e08daSMartin KaFai Lau u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); 4577cb4d2b3fSMartin KaFai Lau u32 i; 4578cb4d2b3fSMartin KaFai Lau 4579cb4d2b3fSMartin KaFai Lau for (i = 0; i < ulen; i++) 4580cb4d2b3fSMartin KaFai Lau if (put_user(prog->aux->used_maps[i]->id, 4581984fe94fSYiFei Zhu &user_map_ids[i])) { 4582984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 4583cb4d2b3fSMartin KaFai Lau return -EFAULT; 4584cb4d2b3fSMartin KaFai Lau } 4585984fe94fSYiFei Zhu } 4586984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 45871e270976SMartin KaFai Lau 4588c454a46bSMartin KaFai Lau err = set_info_rec_size(&info); 4589c454a46bSMartin KaFai Lau if (err) 4590c454a46bSMartin KaFai Lau return err; 45917337224fSMartin KaFai Lau 45925f8f8b93SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 45935f8f8b93SAlexei Starovoitov info.run_time_ns = stats.nsecs; 45945f8f8b93SAlexei Starovoitov info.run_cnt = stats.cnt; 45959ed9e9baSAlexei Starovoitov info.recursion_misses = stats.misses; 45965f8f8b93SAlexei Starovoitov 4597aba64c7dSDave Marchevsky info.verified_insns = prog->aux->verified_insns; 4598aba64c7dSDave Marchevsky 45992c78ee89SAlexei Starovoitov if (!bpf_capable()) { 46001e270976SMartin KaFai Lau info.jited_prog_len = 0; 46011e270976SMartin KaFai Lau info.xlated_prog_len = 0; 4602dbecd738SSandipan Das info.nr_jited_ksyms = 0; 460328c2fae7SDaniel Borkmann info.nr_jited_func_lens = 0; 460411d8b82dSYonghong Song info.nr_func_info = 0; 460511d8b82dSYonghong Song info.nr_line_info = 0; 460611d8b82dSYonghong Song info.nr_jited_line_info = 0; 46071e270976SMartin KaFai Lau goto done; 46081e270976SMartin KaFai Lau } 46091e270976SMartin KaFai Lau 46101e270976SMartin KaFai Lau ulen = info.xlated_prog_len; 46119975a54bSDaniel Borkmann info.xlated_prog_len = bpf_prog_insn_size(prog); 46121e270976SMartin KaFai Lau if (info.xlated_prog_len && ulen) { 46137105e828SDaniel Borkmann struct bpf_insn *insns_sanitized; 46147105e828SDaniel Borkmann bool fault; 46157105e828SDaniel Borkmann 461663960260SKees Cook if (prog->blinded && !bpf_dump_raw_ok(file->f_cred)) { 46177105e828SDaniel Borkmann info.xlated_prog_insns = 0; 46187105e828SDaniel Borkmann goto done; 46197105e828SDaniel Borkmann } 462063960260SKees Cook insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred); 46217105e828SDaniel Borkmann if (!insns_sanitized) 46227105e828SDaniel Borkmann return -ENOMEM; 46231e270976SMartin KaFai Lau uinsns = u64_to_user_ptr(info.xlated_prog_insns); 46241e270976SMartin KaFai Lau ulen = min_t(u32, info.xlated_prog_len, ulen); 46257105e828SDaniel Borkmann fault = copy_to_user(uinsns, insns_sanitized, ulen); 46267105e828SDaniel Borkmann kfree(insns_sanitized); 46277105e828SDaniel Borkmann if (fault) 46281e270976SMartin KaFai Lau return -EFAULT; 46291e270976SMartin KaFai Lau } 46301e270976SMartin KaFai Lau 46319d03ebc7SStanislav Fomichev if (bpf_prog_is_offloaded(prog->aux)) { 4632675fc275SJakub Kicinski err = bpf_prog_offload_info_fill(&info, prog); 4633675fc275SJakub Kicinski if (err) 4634675fc275SJakub Kicinski return err; 4635fcfb126dSJiong Wang goto done; 4636fcfb126dSJiong Wang } 4637fcfb126dSJiong Wang 4638fcfb126dSJiong Wang /* NOTE: the following code is supposed to be skipped for offload. 4639fcfb126dSJiong Wang * bpf_prog_offload_info_fill() is the place to fill similar fields 4640fcfb126dSJiong Wang * for offload. 4641fcfb126dSJiong Wang */ 4642fcfb126dSJiong Wang ulen = info.jited_prog_len; 46434d56a76eSSandipan Das if (prog->aux->func_cnt) { 46444d56a76eSSandipan Das u32 i; 46454d56a76eSSandipan Das 46464d56a76eSSandipan Das info.jited_prog_len = 0; 46474d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) 46484d56a76eSSandipan Das info.jited_prog_len += prog->aux->func[i]->jited_len; 46494d56a76eSSandipan Das } else { 4650fcfb126dSJiong Wang info.jited_prog_len = prog->jited_len; 46514d56a76eSSandipan Das } 46524d56a76eSSandipan Das 4653fcfb126dSJiong Wang if (info.jited_prog_len && ulen) { 465463960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 4655fcfb126dSJiong Wang uinsns = u64_to_user_ptr(info.jited_prog_insns); 4656fcfb126dSJiong Wang ulen = min_t(u32, info.jited_prog_len, ulen); 46574d56a76eSSandipan Das 46584d56a76eSSandipan Das /* for multi-function programs, copy the JITed 46594d56a76eSSandipan Das * instructions for all the functions 46604d56a76eSSandipan Das */ 46614d56a76eSSandipan Das if (prog->aux->func_cnt) { 46624d56a76eSSandipan Das u32 len, free, i; 46634d56a76eSSandipan Das u8 *img; 46644d56a76eSSandipan Das 46654d56a76eSSandipan Das free = ulen; 46664d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) { 46674d56a76eSSandipan Das len = prog->aux->func[i]->jited_len; 46684d56a76eSSandipan Das len = min_t(u32, len, free); 46694d56a76eSSandipan Das img = (u8 *) prog->aux->func[i]->bpf_func; 46704d56a76eSSandipan Das if (copy_to_user(uinsns, img, len)) 46714d56a76eSSandipan Das return -EFAULT; 46724d56a76eSSandipan Das uinsns += len; 46734d56a76eSSandipan Das free -= len; 46744d56a76eSSandipan Das if (!free) 46754d56a76eSSandipan Das break; 46764d56a76eSSandipan Das } 46774d56a76eSSandipan Das } else { 4678fcfb126dSJiong Wang if (copy_to_user(uinsns, prog->bpf_func, ulen)) 4679fcfb126dSJiong Wang return -EFAULT; 46804d56a76eSSandipan Das } 4681fcfb126dSJiong Wang } else { 4682fcfb126dSJiong Wang info.jited_prog_insns = 0; 4683fcfb126dSJiong Wang } 4684675fc275SJakub Kicinski } 4685675fc275SJakub Kicinski 4686dbecd738SSandipan Das ulen = info.nr_jited_ksyms; 4687ff1889fcSSong Liu info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; 46887a5725ddSSong Liu if (ulen) { 468963960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 4690ff1889fcSSong Liu unsigned long ksym_addr; 4691dbecd738SSandipan Das u64 __user *user_ksyms; 4692dbecd738SSandipan Das u32 i; 4693dbecd738SSandipan Das 4694dbecd738SSandipan Das /* copy the address of the kernel symbol 4695dbecd738SSandipan Das * corresponding to each function 4696dbecd738SSandipan Das */ 4697dbecd738SSandipan Das ulen = min_t(u32, info.nr_jited_ksyms, ulen); 4698dbecd738SSandipan Das user_ksyms = u64_to_user_ptr(info.jited_ksyms); 4699ff1889fcSSong Liu if (prog->aux->func_cnt) { 4700dbecd738SSandipan Das for (i = 0; i < ulen; i++) { 4701ff1889fcSSong Liu ksym_addr = (unsigned long) 4702ff1889fcSSong Liu prog->aux->func[i]->bpf_func; 4703ff1889fcSSong Liu if (put_user((u64) ksym_addr, 4704ff1889fcSSong Liu &user_ksyms[i])) 4705ff1889fcSSong Liu return -EFAULT; 4706ff1889fcSSong Liu } 4707ff1889fcSSong Liu } else { 4708ff1889fcSSong Liu ksym_addr = (unsigned long) prog->bpf_func; 4709ff1889fcSSong Liu if (put_user((u64) ksym_addr, &user_ksyms[0])) 4710dbecd738SSandipan Das return -EFAULT; 4711dbecd738SSandipan Das } 4712dbecd738SSandipan Das } else { 4713dbecd738SSandipan Das info.jited_ksyms = 0; 4714dbecd738SSandipan Das } 4715dbecd738SSandipan Das } 4716dbecd738SSandipan Das 4717815581c1SSandipan Das ulen = info.nr_jited_func_lens; 4718ff1889fcSSong Liu info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; 47197a5725ddSSong Liu if (ulen) { 472063960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 4721815581c1SSandipan Das u32 __user *user_lens; 4722815581c1SSandipan Das u32 func_len, i; 4723815581c1SSandipan Das 4724815581c1SSandipan Das /* copy the JITed image lengths for each function */ 4725815581c1SSandipan Das ulen = min_t(u32, info.nr_jited_func_lens, ulen); 4726815581c1SSandipan Das user_lens = u64_to_user_ptr(info.jited_func_lens); 4727ff1889fcSSong Liu if (prog->aux->func_cnt) { 4728815581c1SSandipan Das for (i = 0; i < ulen; i++) { 4729ff1889fcSSong Liu func_len = 4730ff1889fcSSong Liu prog->aux->func[i]->jited_len; 4731815581c1SSandipan Das if (put_user(func_len, &user_lens[i])) 4732815581c1SSandipan Das return -EFAULT; 4733815581c1SSandipan Das } 4734815581c1SSandipan Das } else { 4735ff1889fcSSong Liu func_len = prog->jited_len; 4736ff1889fcSSong Liu if (put_user(func_len, &user_lens[0])) 4737ff1889fcSSong Liu return -EFAULT; 4738ff1889fcSSong Liu } 4739ff1889fcSSong Liu } else { 4740815581c1SSandipan Das info.jited_func_lens = 0; 4741815581c1SSandipan Das } 4742815581c1SSandipan Das } 4743815581c1SSandipan Das 47447337224fSMartin KaFai Lau if (prog->aux->btf) 474522dc4a0fSAndrii Nakryiko info.btf_id = btf_obj_id(prog->aux->btf); 4746b79c9fc9SStanislav Fomichev info.attach_btf_id = prog->aux->attach_btf_id; 47476644aabbSStanislav Fomichev if (attach_btf) 47486644aabbSStanislav Fomichev info.attach_btf_obj_id = btf_obj_id(attach_btf); 4749838e9690SYonghong Song 475011d8b82dSYonghong Song ulen = info.nr_func_info; 475111d8b82dSYonghong Song info.nr_func_info = prog->aux->func_info_cnt; 475211d8b82dSYonghong Song if (info.nr_func_info && ulen) { 4753838e9690SYonghong Song char __user *user_finfo; 4754838e9690SYonghong Song 4755838e9690SYonghong Song user_finfo = u64_to_user_ptr(info.func_info); 475611d8b82dSYonghong Song ulen = min_t(u32, info.nr_func_info, ulen); 4757ba64e7d8SYonghong Song if (copy_to_user(user_finfo, prog->aux->func_info, 47587337224fSMartin KaFai Lau info.func_info_rec_size * ulen)) 4759838e9690SYonghong Song return -EFAULT; 4760838e9690SYonghong Song } 4761838e9690SYonghong Song 476211d8b82dSYonghong Song ulen = info.nr_line_info; 476311d8b82dSYonghong Song info.nr_line_info = prog->aux->nr_linfo; 476411d8b82dSYonghong Song if (info.nr_line_info && ulen) { 4765c454a46bSMartin KaFai Lau __u8 __user *user_linfo; 4766c454a46bSMartin KaFai Lau 4767c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.line_info); 476811d8b82dSYonghong Song ulen = min_t(u32, info.nr_line_info, ulen); 4769c454a46bSMartin KaFai Lau if (copy_to_user(user_linfo, prog->aux->linfo, 4770c454a46bSMartin KaFai Lau info.line_info_rec_size * ulen)) 4771c454a46bSMartin KaFai Lau return -EFAULT; 4772c454a46bSMartin KaFai Lau } 4773c454a46bSMartin KaFai Lau 477411d8b82dSYonghong Song ulen = info.nr_jited_line_info; 4775c454a46bSMartin KaFai Lau if (prog->aux->jited_linfo) 477611d8b82dSYonghong Song info.nr_jited_line_info = prog->aux->nr_linfo; 4777c454a46bSMartin KaFai Lau else 477811d8b82dSYonghong Song info.nr_jited_line_info = 0; 477911d8b82dSYonghong Song if (info.nr_jited_line_info && ulen) { 478063960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) { 47812cd00852SPu Lehui unsigned long line_addr; 4782c454a46bSMartin KaFai Lau __u64 __user *user_linfo; 4783c454a46bSMartin KaFai Lau u32 i; 4784c454a46bSMartin KaFai Lau 4785c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.jited_line_info); 478611d8b82dSYonghong Song ulen = min_t(u32, info.nr_jited_line_info, ulen); 4787c454a46bSMartin KaFai Lau for (i = 0; i < ulen; i++) { 47882cd00852SPu Lehui line_addr = (unsigned long)prog->aux->jited_linfo[i]; 47892cd00852SPu Lehui if (put_user((__u64)line_addr, &user_linfo[i])) 4790c454a46bSMartin KaFai Lau return -EFAULT; 4791c454a46bSMartin KaFai Lau } 4792c454a46bSMartin KaFai Lau } else { 4793c454a46bSMartin KaFai Lau info.jited_line_info = 0; 4794c454a46bSMartin KaFai Lau } 4795c454a46bSMartin KaFai Lau } 4796c454a46bSMartin KaFai Lau 4797c872bdb3SSong Liu ulen = info.nr_prog_tags; 4798c872bdb3SSong Liu info.nr_prog_tags = prog->aux->func_cnt ? : 1; 4799c872bdb3SSong Liu if (ulen) { 4800c872bdb3SSong Liu __u8 __user (*user_prog_tags)[BPF_TAG_SIZE]; 4801c872bdb3SSong Liu u32 i; 4802c872bdb3SSong Liu 4803c872bdb3SSong Liu user_prog_tags = u64_to_user_ptr(info.prog_tags); 4804c872bdb3SSong Liu ulen = min_t(u32, info.nr_prog_tags, ulen); 4805c872bdb3SSong Liu if (prog->aux->func_cnt) { 4806c872bdb3SSong Liu for (i = 0; i < ulen; i++) { 4807c872bdb3SSong Liu if (copy_to_user(user_prog_tags[i], 4808c872bdb3SSong Liu prog->aux->func[i]->tag, 4809c872bdb3SSong Liu BPF_TAG_SIZE)) 4810c872bdb3SSong Liu return -EFAULT; 4811c872bdb3SSong Liu } 4812c872bdb3SSong Liu } else { 4813c872bdb3SSong Liu if (copy_to_user(user_prog_tags[0], 4814c872bdb3SSong Liu prog->tag, BPF_TAG_SIZE)) 4815c872bdb3SSong Liu return -EFAULT; 4816c872bdb3SSong Liu } 4817c872bdb3SSong Liu } 4818c872bdb3SSong Liu 48191e270976SMartin KaFai Lau done: 48201e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 48211e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 48221e270976SMartin KaFai Lau return -EFAULT; 48231e270976SMartin KaFai Lau 48241e270976SMartin KaFai Lau return 0; 48251e270976SMartin KaFai Lau } 48261e270976SMartin KaFai Lau 482763960260SKees Cook static int bpf_map_get_info_by_fd(struct file *file, 482863960260SKees Cook struct bpf_map *map, 48291e270976SMartin KaFai Lau const union bpf_attr *attr, 48301e270976SMartin KaFai Lau union bpf_attr __user *uattr) 48311e270976SMartin KaFai Lau { 48321e270976SMartin KaFai Lau struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 48335c6f2588SGreg Kroah-Hartman struct bpf_map_info info; 48341e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 48351e270976SMartin KaFai Lau int err; 48361e270976SMartin KaFai Lau 4837af2ac3e1SAlexei Starovoitov err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len); 48381e270976SMartin KaFai Lau if (err) 48391e270976SMartin KaFai Lau return err; 48401e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 48411e270976SMartin KaFai Lau 48425c6f2588SGreg Kroah-Hartman memset(&info, 0, sizeof(info)); 48431e270976SMartin KaFai Lau info.type = map->map_type; 48441e270976SMartin KaFai Lau info.id = map->id; 48451e270976SMartin KaFai Lau info.key_size = map->key_size; 48461e270976SMartin KaFai Lau info.value_size = map->value_size; 48471e270976SMartin KaFai Lau info.max_entries = map->max_entries; 48481e270976SMartin KaFai Lau info.map_flags = map->map_flags; 48499330986cSJoanne Koong info.map_extra = map->map_extra; 4850ad5b177bSMartin KaFai Lau memcpy(info.name, map->name, sizeof(map->name)); 48511e270976SMartin KaFai Lau 485278958fcaSMartin KaFai Lau if (map->btf) { 485322dc4a0fSAndrii Nakryiko info.btf_id = btf_obj_id(map->btf); 48549b2cf328SMartin KaFai Lau info.btf_key_type_id = map->btf_key_type_id; 48559b2cf328SMartin KaFai Lau info.btf_value_type_id = map->btf_value_type_id; 485678958fcaSMartin KaFai Lau } 485785d33df3SMartin KaFai Lau info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; 48581338b933SKui-Feng Lee if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) 48591338b933SKui-Feng Lee bpf_map_struct_ops_info_fill(&info, map); 486078958fcaSMartin KaFai Lau 48619d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) { 486252775b33SJakub Kicinski err = bpf_map_offload_info_fill(&info, map); 486352775b33SJakub Kicinski if (err) 486452775b33SJakub Kicinski return err; 486552775b33SJakub Kicinski } 486652775b33SJakub Kicinski 48671e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 48681e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 48691e270976SMartin KaFai Lau return -EFAULT; 48701e270976SMartin KaFai Lau 48711e270976SMartin KaFai Lau return 0; 48721e270976SMartin KaFai Lau } 48731e270976SMartin KaFai Lau 487463960260SKees Cook static int bpf_btf_get_info_by_fd(struct file *file, 487563960260SKees Cook struct btf *btf, 487662dab84cSMartin KaFai Lau const union bpf_attr *attr, 487762dab84cSMartin KaFai Lau union bpf_attr __user *uattr) 487862dab84cSMartin KaFai Lau { 487962dab84cSMartin KaFai Lau struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info); 488062dab84cSMartin KaFai Lau u32 info_len = attr->info.info_len; 488162dab84cSMartin KaFai Lau int err; 488262dab84cSMartin KaFai Lau 4883af2ac3e1SAlexei Starovoitov err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(*uinfo), info_len); 488462dab84cSMartin KaFai Lau if (err) 488562dab84cSMartin KaFai Lau return err; 488662dab84cSMartin KaFai Lau 488762dab84cSMartin KaFai Lau return btf_get_info_by_fd(btf, attr, uattr); 488862dab84cSMartin KaFai Lau } 488962dab84cSMartin KaFai Lau 489063960260SKees Cook static int bpf_link_get_info_by_fd(struct file *file, 489163960260SKees Cook struct bpf_link *link, 4892f2e10bffSAndrii Nakryiko const union bpf_attr *attr, 4893f2e10bffSAndrii Nakryiko union bpf_attr __user *uattr) 4894f2e10bffSAndrii Nakryiko { 4895f2e10bffSAndrii Nakryiko struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info); 4896f2e10bffSAndrii Nakryiko struct bpf_link_info info; 4897f2e10bffSAndrii Nakryiko u32 info_len = attr->info.info_len; 4898f2e10bffSAndrii Nakryiko int err; 4899f2e10bffSAndrii Nakryiko 4900af2ac3e1SAlexei Starovoitov err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len); 4901f2e10bffSAndrii Nakryiko if (err) 4902f2e10bffSAndrii Nakryiko return err; 4903f2e10bffSAndrii Nakryiko info_len = min_t(u32, sizeof(info), info_len); 4904f2e10bffSAndrii Nakryiko 4905f2e10bffSAndrii Nakryiko memset(&info, 0, sizeof(info)); 4906f2e10bffSAndrii Nakryiko if (copy_from_user(&info, uinfo, info_len)) 4907f2e10bffSAndrii Nakryiko return -EFAULT; 4908f2e10bffSAndrii Nakryiko 4909f2e10bffSAndrii Nakryiko info.type = link->type; 4910f2e10bffSAndrii Nakryiko info.id = link->id; 491168b04864SKui-Feng Lee if (link->prog) 4912f2e10bffSAndrii Nakryiko info.prog_id = link->prog->aux->id; 4913f2e10bffSAndrii Nakryiko 4914f2e10bffSAndrii Nakryiko if (link->ops->fill_link_info) { 4915f2e10bffSAndrii Nakryiko err = link->ops->fill_link_info(link, &info); 4916f2e10bffSAndrii Nakryiko if (err) 4917f2e10bffSAndrii Nakryiko return err; 4918f2e10bffSAndrii Nakryiko } 4919f2e10bffSAndrii Nakryiko 4920f2e10bffSAndrii Nakryiko if (copy_to_user(uinfo, &info, info_len) || 4921f2e10bffSAndrii Nakryiko put_user(info_len, &uattr->info.info_len)) 4922f2e10bffSAndrii Nakryiko return -EFAULT; 4923f2e10bffSAndrii Nakryiko 4924f2e10bffSAndrii Nakryiko return 0; 4925f2e10bffSAndrii Nakryiko } 4926f2e10bffSAndrii Nakryiko 4927f2e10bffSAndrii Nakryiko 49281e270976SMartin KaFai Lau #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 49291e270976SMartin KaFai Lau 49301e270976SMartin KaFai Lau static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 49311e270976SMartin KaFai Lau union bpf_attr __user *uattr) 49321e270976SMartin KaFai Lau { 49331e270976SMartin KaFai Lau int ufd = attr->info.bpf_fd; 49341e270976SMartin KaFai Lau struct fd f; 49351e270976SMartin KaFai Lau int err; 49361e270976SMartin KaFai Lau 49371e270976SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 49381e270976SMartin KaFai Lau return -EINVAL; 49391e270976SMartin KaFai Lau 49401e270976SMartin KaFai Lau f = fdget(ufd); 49411e270976SMartin KaFai Lau if (!f.file) 49421e270976SMartin KaFai Lau return -EBADFD; 49431e270976SMartin KaFai Lau 49441e270976SMartin KaFai Lau if (f.file->f_op == &bpf_prog_fops) 494563960260SKees Cook err = bpf_prog_get_info_by_fd(f.file, f.file->private_data, attr, 49461e270976SMartin KaFai Lau uattr); 49471e270976SMartin KaFai Lau else if (f.file->f_op == &bpf_map_fops) 494863960260SKees Cook err = bpf_map_get_info_by_fd(f.file, f.file->private_data, attr, 49491e270976SMartin KaFai Lau uattr); 495060197cfbSMartin KaFai Lau else if (f.file->f_op == &btf_fops) 495163960260SKees Cook err = bpf_btf_get_info_by_fd(f.file, f.file->private_data, attr, uattr); 4952f2e10bffSAndrii Nakryiko else if (f.file->f_op == &bpf_link_fops) 495363960260SKees Cook err = bpf_link_get_info_by_fd(f.file, f.file->private_data, 4954f2e10bffSAndrii Nakryiko attr, uattr); 49551e270976SMartin KaFai Lau else 49561e270976SMartin KaFai Lau err = -EINVAL; 49571e270976SMartin KaFai Lau 49581e270976SMartin KaFai Lau fdput(f); 49591e270976SMartin KaFai Lau return err; 49601e270976SMartin KaFai Lau } 49611e270976SMartin KaFai Lau 49629ea7c4bfSAndrii Nakryiko #define BPF_BTF_LOAD_LAST_FIELD btf_token_fd 4963f56a653cSMartin KaFai Lau 496447a71c1fSAndrii Nakryiko static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size) 4965f56a653cSMartin KaFai Lau { 49669ea7c4bfSAndrii Nakryiko struct bpf_token *token = NULL; 49679ea7c4bfSAndrii Nakryiko 4968f56a653cSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_LOAD)) 4969f56a653cSMartin KaFai Lau return -EINVAL; 4970f56a653cSMartin KaFai Lau 49719ea7c4bfSAndrii Nakryiko if (attr->btf_flags & ~BPF_F_TOKEN_FD) 49729ea7c4bfSAndrii Nakryiko return -EINVAL; 49739ea7c4bfSAndrii Nakryiko 49749ea7c4bfSAndrii Nakryiko if (attr->btf_flags & BPF_F_TOKEN_FD) { 49759ea7c4bfSAndrii Nakryiko token = bpf_token_get_from_fd(attr->btf_token_fd); 49769ea7c4bfSAndrii Nakryiko if (IS_ERR(token)) 49779ea7c4bfSAndrii Nakryiko return PTR_ERR(token); 49789ea7c4bfSAndrii Nakryiko if (!bpf_token_allow_cmd(token, BPF_BTF_LOAD)) { 49799ea7c4bfSAndrii Nakryiko bpf_token_put(token); 49809ea7c4bfSAndrii Nakryiko token = NULL; 49819ea7c4bfSAndrii Nakryiko } 49829ea7c4bfSAndrii Nakryiko } 49839ea7c4bfSAndrii Nakryiko 49849ea7c4bfSAndrii Nakryiko if (!bpf_token_capable(token, CAP_BPF)) { 49859ea7c4bfSAndrii Nakryiko bpf_token_put(token); 4986f56a653cSMartin KaFai Lau return -EPERM; 49879ea7c4bfSAndrii Nakryiko } 49889ea7c4bfSAndrii Nakryiko 49899ea7c4bfSAndrii Nakryiko bpf_token_put(token); 4990f56a653cSMartin KaFai Lau 499147a71c1fSAndrii Nakryiko return btf_new_fd(attr, uattr, uattr_size); 4992f56a653cSMartin KaFai Lau } 4993f56a653cSMartin KaFai Lau 499478958fcaSMartin KaFai Lau #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id 499578958fcaSMartin KaFai Lau 499678958fcaSMartin KaFai Lau static int bpf_btf_get_fd_by_id(const union bpf_attr *attr) 499778958fcaSMartin KaFai Lau { 499878958fcaSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID)) 499978958fcaSMartin KaFai Lau return -EINVAL; 500078958fcaSMartin KaFai Lau 500178958fcaSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 500278958fcaSMartin KaFai Lau return -EPERM; 500378958fcaSMartin KaFai Lau 500478958fcaSMartin KaFai Lau return btf_get_fd_by_id(attr->btf_id); 500578958fcaSMartin KaFai Lau } 500678958fcaSMartin KaFai Lau 500741bdc4b4SYonghong Song static int bpf_task_fd_query_copy(const union bpf_attr *attr, 500841bdc4b4SYonghong Song union bpf_attr __user *uattr, 500941bdc4b4SYonghong Song u32 prog_id, u32 fd_type, 501041bdc4b4SYonghong Song const char *buf, u64 probe_offset, 501141bdc4b4SYonghong Song u64 probe_addr) 501241bdc4b4SYonghong Song { 501341bdc4b4SYonghong Song char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf); 501441bdc4b4SYonghong Song u32 len = buf ? strlen(buf) : 0, input_len; 501541bdc4b4SYonghong Song int err = 0; 501641bdc4b4SYonghong Song 501741bdc4b4SYonghong Song if (put_user(len, &uattr->task_fd_query.buf_len)) 501841bdc4b4SYonghong Song return -EFAULT; 501941bdc4b4SYonghong Song input_len = attr->task_fd_query.buf_len; 502041bdc4b4SYonghong Song if (input_len && ubuf) { 502141bdc4b4SYonghong Song if (!len) { 502241bdc4b4SYonghong Song /* nothing to copy, just make ubuf NULL terminated */ 502341bdc4b4SYonghong Song char zero = '\0'; 502441bdc4b4SYonghong Song 502541bdc4b4SYonghong Song if (put_user(zero, ubuf)) 502641bdc4b4SYonghong Song return -EFAULT; 502741bdc4b4SYonghong Song } else if (input_len >= len + 1) { 502841bdc4b4SYonghong Song /* ubuf can hold the string with NULL terminator */ 502941bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, len + 1)) 503041bdc4b4SYonghong Song return -EFAULT; 503141bdc4b4SYonghong Song } else { 503241bdc4b4SYonghong Song /* ubuf cannot hold the string with NULL terminator, 503341bdc4b4SYonghong Song * do a partial copy with NULL terminator. 503441bdc4b4SYonghong Song */ 503541bdc4b4SYonghong Song char zero = '\0'; 503641bdc4b4SYonghong Song 503741bdc4b4SYonghong Song err = -ENOSPC; 503841bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, input_len - 1)) 503941bdc4b4SYonghong Song return -EFAULT; 504041bdc4b4SYonghong Song if (put_user(zero, ubuf + input_len - 1)) 504141bdc4b4SYonghong Song return -EFAULT; 504241bdc4b4SYonghong Song } 504341bdc4b4SYonghong Song } 504441bdc4b4SYonghong Song 504541bdc4b4SYonghong Song if (put_user(prog_id, &uattr->task_fd_query.prog_id) || 504641bdc4b4SYonghong Song put_user(fd_type, &uattr->task_fd_query.fd_type) || 504741bdc4b4SYonghong Song put_user(probe_offset, &uattr->task_fd_query.probe_offset) || 504841bdc4b4SYonghong Song put_user(probe_addr, &uattr->task_fd_query.probe_addr)) 504941bdc4b4SYonghong Song return -EFAULT; 505041bdc4b4SYonghong Song 505141bdc4b4SYonghong Song return err; 505241bdc4b4SYonghong Song } 505341bdc4b4SYonghong Song 505441bdc4b4SYonghong Song #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr 505541bdc4b4SYonghong Song 505641bdc4b4SYonghong Song static int bpf_task_fd_query(const union bpf_attr *attr, 505741bdc4b4SYonghong Song union bpf_attr __user *uattr) 505841bdc4b4SYonghong Song { 505941bdc4b4SYonghong Song pid_t pid = attr->task_fd_query.pid; 506041bdc4b4SYonghong Song u32 fd = attr->task_fd_query.fd; 506141bdc4b4SYonghong Song const struct perf_event *event; 506241bdc4b4SYonghong Song struct task_struct *task; 506341bdc4b4SYonghong Song struct file *file; 506441bdc4b4SYonghong Song int err; 506541bdc4b4SYonghong Song 506641bdc4b4SYonghong Song if (CHECK_ATTR(BPF_TASK_FD_QUERY)) 506741bdc4b4SYonghong Song return -EINVAL; 506841bdc4b4SYonghong Song 506941bdc4b4SYonghong Song if (!capable(CAP_SYS_ADMIN)) 507041bdc4b4SYonghong Song return -EPERM; 507141bdc4b4SYonghong Song 507241bdc4b4SYonghong Song if (attr->task_fd_query.flags != 0) 507341bdc4b4SYonghong Song return -EINVAL; 507441bdc4b4SYonghong Song 507583c10cc3SLee Jones rcu_read_lock(); 507641bdc4b4SYonghong Song task = get_pid_task(find_vpid(pid), PIDTYPE_PID); 507783c10cc3SLee Jones rcu_read_unlock(); 507841bdc4b4SYonghong Song if (!task) 507941bdc4b4SYonghong Song return -ENOENT; 508041bdc4b4SYonghong Song 508141bdc4b4SYonghong Song err = 0; 5082b48845afSEric W. Biederman file = fget_task(task, fd); 5083b48845afSEric W. Biederman put_task_struct(task); 508441bdc4b4SYonghong Song if (!file) 5085b48845afSEric W. Biederman return -EBADF; 508641bdc4b4SYonghong Song 508770ed506cSAndrii Nakryiko if (file->f_op == &bpf_link_fops) { 508870ed506cSAndrii Nakryiko struct bpf_link *link = file->private_data; 508970ed506cSAndrii Nakryiko 5090a3b80e10SAndrii Nakryiko if (link->ops == &bpf_raw_tp_link_lops) { 509170ed506cSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp = 509270ed506cSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link); 509341bdc4b4SYonghong Song struct bpf_raw_event_map *btp = raw_tp->btp; 509441bdc4b4SYonghong Song 509541bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, 509670ed506cSAndrii Nakryiko raw_tp->link.prog->aux->id, 509741bdc4b4SYonghong Song BPF_FD_TYPE_RAW_TRACEPOINT, 509841bdc4b4SYonghong Song btp->tp->name, 0, 0); 509941bdc4b4SYonghong Song goto put_file; 510041bdc4b4SYonghong Song } 510170ed506cSAndrii Nakryiko goto out_not_supp; 510270ed506cSAndrii Nakryiko } 510341bdc4b4SYonghong Song 510441bdc4b4SYonghong Song event = perf_get_event(file); 510541bdc4b4SYonghong Song if (!IS_ERR(event)) { 510641bdc4b4SYonghong Song u64 probe_offset, probe_addr; 510741bdc4b4SYonghong Song u32 prog_id, fd_type; 510841bdc4b4SYonghong Song const char *buf; 510941bdc4b4SYonghong Song 511041bdc4b4SYonghong Song err = bpf_get_perf_event_info(event, &prog_id, &fd_type, 511141bdc4b4SYonghong Song &buf, &probe_offset, 51123acf8aceSJiri Olsa &probe_addr, NULL); 511341bdc4b4SYonghong Song if (!err) 511441bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, prog_id, 511541bdc4b4SYonghong Song fd_type, buf, 511641bdc4b4SYonghong Song probe_offset, 511741bdc4b4SYonghong Song probe_addr); 511841bdc4b4SYonghong Song goto put_file; 511941bdc4b4SYonghong Song } 512041bdc4b4SYonghong Song 512170ed506cSAndrii Nakryiko out_not_supp: 512241bdc4b4SYonghong Song err = -ENOTSUPP; 512341bdc4b4SYonghong Song put_file: 512441bdc4b4SYonghong Song fput(file); 512541bdc4b4SYonghong Song return err; 512641bdc4b4SYonghong Song } 512741bdc4b4SYonghong Song 5128cb4d03abSBrian Vazquez #define BPF_MAP_BATCH_LAST_FIELD batch.flags 5129cb4d03abSBrian Vazquez 51303af43ba4SHou Tao #define BPF_DO_BATCH(fn, ...) \ 5131cb4d03abSBrian Vazquez do { \ 5132cb4d03abSBrian Vazquez if (!fn) { \ 5133cb4d03abSBrian Vazquez err = -ENOTSUPP; \ 5134cb4d03abSBrian Vazquez goto err_put; \ 5135cb4d03abSBrian Vazquez } \ 51363af43ba4SHou Tao err = fn(__VA_ARGS__); \ 5137cb4d03abSBrian Vazquez } while (0) 5138cb4d03abSBrian Vazquez 5139cb4d03abSBrian Vazquez static int bpf_map_do_batch(const union bpf_attr *attr, 5140cb4d03abSBrian Vazquez union bpf_attr __user *uattr, 5141cb4d03abSBrian Vazquez int cmd) 5142cb4d03abSBrian Vazquez { 5143353050beSDaniel Borkmann bool has_read = cmd == BPF_MAP_LOOKUP_BATCH || 5144353050beSDaniel Borkmann cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH; 5145353050beSDaniel Borkmann bool has_write = cmd != BPF_MAP_LOOKUP_BATCH; 5146cb4d03abSBrian Vazquez struct bpf_map *map; 5147cb4d03abSBrian Vazquez int err, ufd; 5148cb4d03abSBrian Vazquez struct fd f; 5149cb4d03abSBrian Vazquez 5150cb4d03abSBrian Vazquez if (CHECK_ATTR(BPF_MAP_BATCH)) 5151cb4d03abSBrian Vazquez return -EINVAL; 5152cb4d03abSBrian Vazquez 5153cb4d03abSBrian Vazquez ufd = attr->batch.map_fd; 5154cb4d03abSBrian Vazquez f = fdget(ufd); 5155cb4d03abSBrian Vazquez map = __bpf_map_get(f); 5156cb4d03abSBrian Vazquez if (IS_ERR(map)) 5157cb4d03abSBrian Vazquez return PTR_ERR(map); 5158353050beSDaniel Borkmann if (has_write) 5159353050beSDaniel Borkmann bpf_map_write_active_inc(map); 5160353050beSDaniel Borkmann if (has_read && !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 5161cb4d03abSBrian Vazquez err = -EPERM; 5162cb4d03abSBrian Vazquez goto err_put; 5163cb4d03abSBrian Vazquez } 5164353050beSDaniel Borkmann if (has_write && !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 5165cb4d03abSBrian Vazquez err = -EPERM; 5166cb4d03abSBrian Vazquez goto err_put; 5167cb4d03abSBrian Vazquez } 5168cb4d03abSBrian Vazquez 5169cb4d03abSBrian Vazquez if (cmd == BPF_MAP_LOOKUP_BATCH) 51703af43ba4SHou Tao BPF_DO_BATCH(map->ops->map_lookup_batch, map, attr, uattr); 517105799638SYonghong Song else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) 51723af43ba4SHou Tao BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch, map, attr, uattr); 5173aa2e93b8SBrian Vazquez else if (cmd == BPF_MAP_UPDATE_BATCH) 51743af43ba4SHou Tao BPF_DO_BATCH(map->ops->map_update_batch, map, f.file, attr, uattr); 5175aa2e93b8SBrian Vazquez else 51763af43ba4SHou Tao BPF_DO_BATCH(map->ops->map_delete_batch, map, attr, uattr); 5177cb4d03abSBrian Vazquez err_put: 517801277258SHou Tao if (has_write) { 517901277258SHou Tao maybe_wait_bpf_programs(map); 5180353050beSDaniel Borkmann bpf_map_write_active_dec(map); 518101277258SHou Tao } 5182cb4d03abSBrian Vazquez fdput(f); 5183cb4d03abSBrian Vazquez return err; 5184cb4d03abSBrian Vazquez } 5185cb4d03abSBrian Vazquez 5186b733eeadSJiri Olsa #define BPF_LINK_CREATE_LAST_FIELD link_create.uprobe_multi.pid 5187af2ac3e1SAlexei Starovoitov static int link_create(union bpf_attr *attr, bpfptr_t uattr) 5188af6eea57SAndrii Nakryiko { 5189af6eea57SAndrii Nakryiko struct bpf_prog *prog; 5190af6eea57SAndrii Nakryiko int ret; 5191af6eea57SAndrii Nakryiko 5192af6eea57SAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_CREATE)) 5193af6eea57SAndrii Nakryiko return -EINVAL; 5194af6eea57SAndrii Nakryiko 519568b04864SKui-Feng Lee if (attr->link_create.attach_type == BPF_STRUCT_OPS) 519668b04864SKui-Feng Lee return bpf_struct_ops_link_create(attr); 519768b04864SKui-Feng Lee 51984a1e7c0cSToke Høiland-Jørgensen prog = bpf_prog_get(attr->link_create.prog_fd); 5199af6eea57SAndrii Nakryiko if (IS_ERR(prog)) 5200af6eea57SAndrii Nakryiko return PTR_ERR(prog); 5201af6eea57SAndrii Nakryiko 5202af6eea57SAndrii Nakryiko ret = bpf_prog_attach_check_attach_type(prog, 5203af6eea57SAndrii Nakryiko attr->link_create.attach_type); 5204af6eea57SAndrii Nakryiko if (ret) 52054a1e7c0cSToke Høiland-Jørgensen goto out; 52064a1e7c0cSToke Høiland-Jørgensen 5207b89fbfbbSAndrii Nakryiko switch (prog->type) { 5208af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB: 5209af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK: 5210af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 5211af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS: 5212af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE: 5213af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL: 5214af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT: 5215af6eea57SAndrii Nakryiko ret = cgroup_bpf_link_attach(attr, prog); 5216af6eea57SAndrii Nakryiko break; 5217df86ca0dSAndrii Nakryiko case BPF_PROG_TYPE_EXT: 5218df86ca0dSAndrii Nakryiko ret = bpf_tracing_prog_attach(prog, 5219df86ca0dSAndrii Nakryiko attr->link_create.target_fd, 52202fcc8241SKui-Feng Lee attr->link_create.target_btf_id, 52212fcc8241SKui-Feng Lee attr->link_create.tracing.cookie); 5222df86ca0dSAndrii Nakryiko break; 5223df86ca0dSAndrii Nakryiko case BPF_PROG_TYPE_LSM: 5224de4e05caSYonghong Song case BPF_PROG_TYPE_TRACING: 5225df86ca0dSAndrii Nakryiko if (attr->link_create.attach_type != prog->expected_attach_type) { 5226df86ca0dSAndrii Nakryiko ret = -EINVAL; 5227df86ca0dSAndrii Nakryiko goto out; 5228df86ca0dSAndrii Nakryiko } 5229df86ca0dSAndrii Nakryiko if (prog->expected_attach_type == BPF_TRACE_RAW_TP) 5230df86ca0dSAndrii Nakryiko ret = bpf_raw_tp_link_attach(prog, NULL); 5231df86ca0dSAndrii Nakryiko else if (prog->expected_attach_type == BPF_TRACE_ITER) 5232df86ca0dSAndrii Nakryiko ret = bpf_iter_link_attach(attr, uattr, prog); 523369fd337aSStanislav Fomichev else if (prog->expected_attach_type == BPF_LSM_CGROUP) 523469fd337aSStanislav Fomichev ret = cgroup_bpf_link_attach(attr, prog); 5235df86ca0dSAndrii Nakryiko else 5236df86ca0dSAndrii Nakryiko ret = bpf_tracing_prog_attach(prog, 5237df86ca0dSAndrii Nakryiko attr->link_create.target_fd, 52382fcc8241SKui-Feng Lee attr->link_create.target_btf_id, 52392fcc8241SKui-Feng Lee attr->link_create.tracing.cookie); 5240de4e05caSYonghong Song break; 52417f045a49SJakub Sitnicki case BPF_PROG_TYPE_FLOW_DISSECTOR: 5242e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP: 52437f045a49SJakub Sitnicki ret = netns_bpf_link_create(attr, prog); 52447f045a49SJakub Sitnicki break; 5245310ad797SAndrii Nakryiko #ifdef CONFIG_NET 5246aa8d3a71SAndrii Nakryiko case BPF_PROG_TYPE_XDP: 5247aa8d3a71SAndrii Nakryiko ret = bpf_xdp_link_attach(attr, prog); 5248aa8d3a71SAndrii Nakryiko break; 5249e420bed0SDaniel Borkmann case BPF_PROG_TYPE_SCHED_CLS: 525035dfaad7SDaniel Borkmann if (attr->link_create.attach_type == BPF_TCX_INGRESS || 525135dfaad7SDaniel Borkmann attr->link_create.attach_type == BPF_TCX_EGRESS) 5252e420bed0SDaniel Borkmann ret = tcx_link_attach(attr, prog); 525335dfaad7SDaniel Borkmann else 525435dfaad7SDaniel Borkmann ret = netkit_link_attach(attr, prog); 5255e420bed0SDaniel Borkmann break; 525684601d6eSFlorian Westphal case BPF_PROG_TYPE_NETFILTER: 525784601d6eSFlorian Westphal ret = bpf_nf_link_attach(attr, prog); 525884601d6eSFlorian Westphal break; 5259310ad797SAndrii Nakryiko #endif 5260b89fbfbbSAndrii Nakryiko case BPF_PROG_TYPE_PERF_EVENT: 5261b89fbfbbSAndrii Nakryiko case BPF_PROG_TYPE_TRACEPOINT: 5262b89fbfbbSAndrii Nakryiko ret = bpf_perf_link_attach(attr, prog); 5263b89fbfbbSAndrii Nakryiko break; 52640dcac272SJiri Olsa case BPF_PROG_TYPE_KPROBE: 52650dcac272SJiri Olsa if (attr->link_create.attach_type == BPF_PERF_EVENT) 52660dcac272SJiri Olsa ret = bpf_perf_link_attach(attr, prog); 526789ae89f5SJiri Olsa else if (attr->link_create.attach_type == BPF_TRACE_KPROBE_MULTI) 52680dcac272SJiri Olsa ret = bpf_kprobe_multi_link_attach(attr, prog); 526989ae89f5SJiri Olsa else if (attr->link_create.attach_type == BPF_TRACE_UPROBE_MULTI) 527089ae89f5SJiri Olsa ret = bpf_uprobe_multi_link_attach(attr, prog); 52710dcac272SJiri Olsa break; 5272af6eea57SAndrii Nakryiko default: 5273af6eea57SAndrii Nakryiko ret = -EINVAL; 5274af6eea57SAndrii Nakryiko } 5275af6eea57SAndrii Nakryiko 52764a1e7c0cSToke Høiland-Jørgensen out: 5277af6eea57SAndrii Nakryiko if (ret < 0) 5278af6eea57SAndrii Nakryiko bpf_prog_put(prog); 5279af6eea57SAndrii Nakryiko return ret; 5280af6eea57SAndrii Nakryiko } 5281af6eea57SAndrii Nakryiko 5282aef56f2eSKui-Feng Lee static int link_update_map(struct bpf_link *link, union bpf_attr *attr) 5283aef56f2eSKui-Feng Lee { 5284aef56f2eSKui-Feng Lee struct bpf_map *new_map, *old_map = NULL; 5285aef56f2eSKui-Feng Lee int ret; 5286aef56f2eSKui-Feng Lee 5287aef56f2eSKui-Feng Lee new_map = bpf_map_get(attr->link_update.new_map_fd); 5288aef56f2eSKui-Feng Lee if (IS_ERR(new_map)) 528955fbae05SMartin KaFai Lau return PTR_ERR(new_map); 5290aef56f2eSKui-Feng Lee 5291aef56f2eSKui-Feng Lee if (attr->link_update.flags & BPF_F_REPLACE) { 5292aef56f2eSKui-Feng Lee old_map = bpf_map_get(attr->link_update.old_map_fd); 5293aef56f2eSKui-Feng Lee if (IS_ERR(old_map)) { 529455fbae05SMartin KaFai Lau ret = PTR_ERR(old_map); 5295aef56f2eSKui-Feng Lee goto out_put; 5296aef56f2eSKui-Feng Lee } 5297aef56f2eSKui-Feng Lee } else if (attr->link_update.old_map_fd) { 5298aef56f2eSKui-Feng Lee ret = -EINVAL; 5299aef56f2eSKui-Feng Lee goto out_put; 5300aef56f2eSKui-Feng Lee } 5301aef56f2eSKui-Feng Lee 5302aef56f2eSKui-Feng Lee ret = link->ops->update_map(link, new_map, old_map); 5303aef56f2eSKui-Feng Lee 5304aef56f2eSKui-Feng Lee if (old_map) 5305aef56f2eSKui-Feng Lee bpf_map_put(old_map); 5306aef56f2eSKui-Feng Lee out_put: 5307aef56f2eSKui-Feng Lee bpf_map_put(new_map); 5308aef56f2eSKui-Feng Lee return ret; 5309aef56f2eSKui-Feng Lee } 5310aef56f2eSKui-Feng Lee 53110c991ebcSAndrii Nakryiko #define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd 53120c991ebcSAndrii Nakryiko 53130c991ebcSAndrii Nakryiko static int link_update(union bpf_attr *attr) 53140c991ebcSAndrii Nakryiko { 53150c991ebcSAndrii Nakryiko struct bpf_prog *old_prog = NULL, *new_prog; 53160c991ebcSAndrii Nakryiko struct bpf_link *link; 53170c991ebcSAndrii Nakryiko u32 flags; 53180c991ebcSAndrii Nakryiko int ret; 53190c991ebcSAndrii Nakryiko 53200c991ebcSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_UPDATE)) 53210c991ebcSAndrii Nakryiko return -EINVAL; 53220c991ebcSAndrii Nakryiko 53230c991ebcSAndrii Nakryiko flags = attr->link_update.flags; 53240c991ebcSAndrii Nakryiko if (flags & ~BPF_F_REPLACE) 53250c991ebcSAndrii Nakryiko return -EINVAL; 53260c991ebcSAndrii Nakryiko 53270c991ebcSAndrii Nakryiko link = bpf_link_get_from_fd(attr->link_update.link_fd); 53280c991ebcSAndrii Nakryiko if (IS_ERR(link)) 53290c991ebcSAndrii Nakryiko return PTR_ERR(link); 53300c991ebcSAndrii Nakryiko 5331aef56f2eSKui-Feng Lee if (link->ops->update_map) { 5332aef56f2eSKui-Feng Lee ret = link_update_map(link, attr); 5333aef56f2eSKui-Feng Lee goto out_put_link; 5334aef56f2eSKui-Feng Lee } 5335aef56f2eSKui-Feng Lee 53360c991ebcSAndrii Nakryiko new_prog = bpf_prog_get(attr->link_update.new_prog_fd); 53374adb7a4aSAndrii Nakryiko if (IS_ERR(new_prog)) { 53384adb7a4aSAndrii Nakryiko ret = PTR_ERR(new_prog); 53394adb7a4aSAndrii Nakryiko goto out_put_link; 53404adb7a4aSAndrii Nakryiko } 53410c991ebcSAndrii Nakryiko 53420c991ebcSAndrii Nakryiko if (flags & BPF_F_REPLACE) { 53430c991ebcSAndrii Nakryiko old_prog = bpf_prog_get(attr->link_update.old_prog_fd); 53440c991ebcSAndrii Nakryiko if (IS_ERR(old_prog)) { 53450c991ebcSAndrii Nakryiko ret = PTR_ERR(old_prog); 53460c991ebcSAndrii Nakryiko old_prog = NULL; 53470c991ebcSAndrii Nakryiko goto out_put_progs; 53480c991ebcSAndrii Nakryiko } 53494adb7a4aSAndrii Nakryiko } else if (attr->link_update.old_prog_fd) { 53504adb7a4aSAndrii Nakryiko ret = -EINVAL; 53514adb7a4aSAndrii Nakryiko goto out_put_progs; 53520c991ebcSAndrii Nakryiko } 53530c991ebcSAndrii Nakryiko 5354f9d04127SAndrii Nakryiko if (link->ops->update_prog) 5355f9d04127SAndrii Nakryiko ret = link->ops->update_prog(link, new_prog, old_prog); 5356f9d04127SAndrii Nakryiko else 53570c991ebcSAndrii Nakryiko ret = -EINVAL; 53580c991ebcSAndrii Nakryiko 53590c991ebcSAndrii Nakryiko out_put_progs: 53600c991ebcSAndrii Nakryiko if (old_prog) 53610c991ebcSAndrii Nakryiko bpf_prog_put(old_prog); 53620c991ebcSAndrii Nakryiko if (ret) 53630c991ebcSAndrii Nakryiko bpf_prog_put(new_prog); 53644adb7a4aSAndrii Nakryiko out_put_link: 5365ab5d47bdSSebastian Andrzej Siewior bpf_link_put_direct(link); 53660c991ebcSAndrii Nakryiko return ret; 53670c991ebcSAndrii Nakryiko } 53680c991ebcSAndrii Nakryiko 536973b11c2aSAndrii Nakryiko #define BPF_LINK_DETACH_LAST_FIELD link_detach.link_fd 537073b11c2aSAndrii Nakryiko 537173b11c2aSAndrii Nakryiko static int link_detach(union bpf_attr *attr) 537273b11c2aSAndrii Nakryiko { 537373b11c2aSAndrii Nakryiko struct bpf_link *link; 537473b11c2aSAndrii Nakryiko int ret; 537573b11c2aSAndrii Nakryiko 537673b11c2aSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_DETACH)) 537773b11c2aSAndrii Nakryiko return -EINVAL; 537873b11c2aSAndrii Nakryiko 537973b11c2aSAndrii Nakryiko link = bpf_link_get_from_fd(attr->link_detach.link_fd); 538073b11c2aSAndrii Nakryiko if (IS_ERR(link)) 538173b11c2aSAndrii Nakryiko return PTR_ERR(link); 538273b11c2aSAndrii Nakryiko 538373b11c2aSAndrii Nakryiko if (link->ops->detach) 538473b11c2aSAndrii Nakryiko ret = link->ops->detach(link); 538573b11c2aSAndrii Nakryiko else 538673b11c2aSAndrii Nakryiko ret = -EOPNOTSUPP; 538773b11c2aSAndrii Nakryiko 5388ab5d47bdSSebastian Andrzej Siewior bpf_link_put_direct(link); 538973b11c2aSAndrii Nakryiko return ret; 539073b11c2aSAndrii Nakryiko } 539173b11c2aSAndrii Nakryiko 5392005142b8SAlexei Starovoitov static struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link) 53932d602c8cSAndrii Nakryiko { 5394005142b8SAlexei Starovoitov return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT); 5395005142b8SAlexei Starovoitov } 5396005142b8SAlexei Starovoitov 5397005142b8SAlexei Starovoitov struct bpf_link *bpf_link_by_id(u32 id) 5398005142b8SAlexei Starovoitov { 5399005142b8SAlexei Starovoitov struct bpf_link *link; 5400005142b8SAlexei Starovoitov 5401005142b8SAlexei Starovoitov if (!id) 5402005142b8SAlexei Starovoitov return ERR_PTR(-ENOENT); 5403005142b8SAlexei Starovoitov 5404005142b8SAlexei Starovoitov spin_lock_bh(&link_idr_lock); 5405005142b8SAlexei Starovoitov /* before link is "settled", ID is 0, pretend it doesn't exist yet */ 5406005142b8SAlexei Starovoitov link = idr_find(&link_idr, id); 5407005142b8SAlexei Starovoitov if (link) { 5408005142b8SAlexei Starovoitov if (link->id) 5409005142b8SAlexei Starovoitov link = bpf_link_inc_not_zero(link); 5410005142b8SAlexei Starovoitov else 5411005142b8SAlexei Starovoitov link = ERR_PTR(-EAGAIN); 5412005142b8SAlexei Starovoitov } else { 5413005142b8SAlexei Starovoitov link = ERR_PTR(-ENOENT); 5414005142b8SAlexei Starovoitov } 5415005142b8SAlexei Starovoitov spin_unlock_bh(&link_idr_lock); 5416005142b8SAlexei Starovoitov return link; 54172d602c8cSAndrii Nakryiko } 54182d602c8cSAndrii Nakryiko 54199f883612SDmitrii Dolgov struct bpf_link *bpf_link_get_curr_or_next(u32 *id) 54209f883612SDmitrii Dolgov { 54219f883612SDmitrii Dolgov struct bpf_link *link; 54229f883612SDmitrii Dolgov 54239f883612SDmitrii Dolgov spin_lock_bh(&link_idr_lock); 54249f883612SDmitrii Dolgov again: 54259f883612SDmitrii Dolgov link = idr_get_next(&link_idr, id); 54269f883612SDmitrii Dolgov if (link) { 54279f883612SDmitrii Dolgov link = bpf_link_inc_not_zero(link); 54289f883612SDmitrii Dolgov if (IS_ERR(link)) { 54299f883612SDmitrii Dolgov (*id)++; 54309f883612SDmitrii Dolgov goto again; 54319f883612SDmitrii Dolgov } 54329f883612SDmitrii Dolgov } 54339f883612SDmitrii Dolgov spin_unlock_bh(&link_idr_lock); 54349f883612SDmitrii Dolgov 54359f883612SDmitrii Dolgov return link; 54369f883612SDmitrii Dolgov } 54379f883612SDmitrii Dolgov 54382d602c8cSAndrii Nakryiko #define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id 54392d602c8cSAndrii Nakryiko 54402d602c8cSAndrii Nakryiko static int bpf_link_get_fd_by_id(const union bpf_attr *attr) 54412d602c8cSAndrii Nakryiko { 54422d602c8cSAndrii Nakryiko struct bpf_link *link; 54432d602c8cSAndrii Nakryiko u32 id = attr->link_id; 5444005142b8SAlexei Starovoitov int fd; 54452d602c8cSAndrii Nakryiko 54462d602c8cSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID)) 54472d602c8cSAndrii Nakryiko return -EINVAL; 54482d602c8cSAndrii Nakryiko 54492d602c8cSAndrii Nakryiko if (!capable(CAP_SYS_ADMIN)) 54502d602c8cSAndrii Nakryiko return -EPERM; 54512d602c8cSAndrii Nakryiko 5452005142b8SAlexei Starovoitov link = bpf_link_by_id(id); 5453005142b8SAlexei Starovoitov if (IS_ERR(link)) 5454005142b8SAlexei Starovoitov return PTR_ERR(link); 54552d602c8cSAndrii Nakryiko 54562d602c8cSAndrii Nakryiko fd = bpf_link_new_fd(link); 54572d602c8cSAndrii Nakryiko if (fd < 0) 5458ab5d47bdSSebastian Andrzej Siewior bpf_link_put_direct(link); 54592d602c8cSAndrii Nakryiko 54602d602c8cSAndrii Nakryiko return fd; 54612d602c8cSAndrii Nakryiko } 54622d602c8cSAndrii Nakryiko 5463d46edd67SSong Liu DEFINE_MUTEX(bpf_stats_enabled_mutex); 5464d46edd67SSong Liu 5465d46edd67SSong Liu static int bpf_stats_release(struct inode *inode, struct file *file) 5466d46edd67SSong Liu { 5467d46edd67SSong Liu mutex_lock(&bpf_stats_enabled_mutex); 5468d46edd67SSong Liu static_key_slow_dec(&bpf_stats_enabled_key.key); 5469d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 5470d46edd67SSong Liu return 0; 5471d46edd67SSong Liu } 5472d46edd67SSong Liu 5473d46edd67SSong Liu static const struct file_operations bpf_stats_fops = { 5474d46edd67SSong Liu .release = bpf_stats_release, 5475d46edd67SSong Liu }; 5476d46edd67SSong Liu 5477d46edd67SSong Liu static int bpf_enable_runtime_stats(void) 5478d46edd67SSong Liu { 5479d46edd67SSong Liu int fd; 5480d46edd67SSong Liu 5481d46edd67SSong Liu mutex_lock(&bpf_stats_enabled_mutex); 5482d46edd67SSong Liu 5483d46edd67SSong Liu /* Set a very high limit to avoid overflow */ 5484d46edd67SSong Liu if (static_key_count(&bpf_stats_enabled_key.key) > INT_MAX / 2) { 5485d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 5486d46edd67SSong Liu return -EBUSY; 5487d46edd67SSong Liu } 5488d46edd67SSong Liu 5489d46edd67SSong Liu fd = anon_inode_getfd("bpf-stats", &bpf_stats_fops, NULL, O_CLOEXEC); 5490d46edd67SSong Liu if (fd >= 0) 5491d46edd67SSong Liu static_key_slow_inc(&bpf_stats_enabled_key.key); 5492d46edd67SSong Liu 5493d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex); 5494d46edd67SSong Liu return fd; 5495d46edd67SSong Liu } 5496d46edd67SSong Liu 5497d46edd67SSong Liu #define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type 5498d46edd67SSong Liu 5499d46edd67SSong Liu static int bpf_enable_stats(union bpf_attr *attr) 5500d46edd67SSong Liu { 5501d46edd67SSong Liu 5502d46edd67SSong Liu if (CHECK_ATTR(BPF_ENABLE_STATS)) 5503d46edd67SSong Liu return -EINVAL; 5504d46edd67SSong Liu 5505d46edd67SSong Liu if (!capable(CAP_SYS_ADMIN)) 5506d46edd67SSong Liu return -EPERM; 5507d46edd67SSong Liu 5508d46edd67SSong Liu switch (attr->enable_stats.type) { 5509d46edd67SSong Liu case BPF_STATS_RUN_TIME: 5510d46edd67SSong Liu return bpf_enable_runtime_stats(); 5511d46edd67SSong Liu default: 5512d46edd67SSong Liu break; 5513d46edd67SSong Liu } 5514d46edd67SSong Liu return -EINVAL; 5515d46edd67SSong Liu } 5516d46edd67SSong Liu 5517ac51d99bSYonghong Song #define BPF_ITER_CREATE_LAST_FIELD iter_create.flags 5518ac51d99bSYonghong Song 5519ac51d99bSYonghong Song static int bpf_iter_create(union bpf_attr *attr) 5520ac51d99bSYonghong Song { 5521ac51d99bSYonghong Song struct bpf_link *link; 5522ac51d99bSYonghong Song int err; 5523ac51d99bSYonghong Song 5524ac51d99bSYonghong Song if (CHECK_ATTR(BPF_ITER_CREATE)) 5525ac51d99bSYonghong Song return -EINVAL; 5526ac51d99bSYonghong Song 5527ac51d99bSYonghong Song if (attr->iter_create.flags) 5528ac51d99bSYonghong Song return -EINVAL; 5529ac51d99bSYonghong Song 5530ac51d99bSYonghong Song link = bpf_link_get_from_fd(attr->iter_create.link_fd); 5531ac51d99bSYonghong Song if (IS_ERR(link)) 5532ac51d99bSYonghong Song return PTR_ERR(link); 5533ac51d99bSYonghong Song 5534ac51d99bSYonghong Song err = bpf_iter_new_fd(link); 5535ab5d47bdSSebastian Andrzej Siewior bpf_link_put_direct(link); 5536ac51d99bSYonghong Song 5537ac51d99bSYonghong Song return err; 5538ac51d99bSYonghong Song } 5539ac51d99bSYonghong Song 5540ef15314aSYiFei Zhu #define BPF_PROG_BIND_MAP_LAST_FIELD prog_bind_map.flags 5541ef15314aSYiFei Zhu 5542ef15314aSYiFei Zhu static int bpf_prog_bind_map(union bpf_attr *attr) 5543ef15314aSYiFei Zhu { 5544ef15314aSYiFei Zhu struct bpf_prog *prog; 5545ef15314aSYiFei Zhu struct bpf_map *map; 5546ef15314aSYiFei Zhu struct bpf_map **used_maps_old, **used_maps_new; 5547ef15314aSYiFei Zhu int i, ret = 0; 5548ef15314aSYiFei Zhu 5549ef15314aSYiFei Zhu if (CHECK_ATTR(BPF_PROG_BIND_MAP)) 5550ef15314aSYiFei Zhu return -EINVAL; 5551ef15314aSYiFei Zhu 5552ef15314aSYiFei Zhu if (attr->prog_bind_map.flags) 5553ef15314aSYiFei Zhu return -EINVAL; 5554ef15314aSYiFei Zhu 5555ef15314aSYiFei Zhu prog = bpf_prog_get(attr->prog_bind_map.prog_fd); 5556ef15314aSYiFei Zhu if (IS_ERR(prog)) 5557ef15314aSYiFei Zhu return PTR_ERR(prog); 5558ef15314aSYiFei Zhu 5559ef15314aSYiFei Zhu map = bpf_map_get(attr->prog_bind_map.map_fd); 5560ef15314aSYiFei Zhu if (IS_ERR(map)) { 5561ef15314aSYiFei Zhu ret = PTR_ERR(map); 5562ef15314aSYiFei Zhu goto out_prog_put; 5563ef15314aSYiFei Zhu } 5564ef15314aSYiFei Zhu 5565ef15314aSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex); 5566ef15314aSYiFei Zhu 5567ef15314aSYiFei Zhu used_maps_old = prog->aux->used_maps; 5568ef15314aSYiFei Zhu 5569ef15314aSYiFei Zhu for (i = 0; i < prog->aux->used_map_cnt; i++) 55701028ae40SStanislav Fomichev if (used_maps_old[i] == map) { 55711028ae40SStanislav Fomichev bpf_map_put(map); 5572ef15314aSYiFei Zhu goto out_unlock; 55731028ae40SStanislav Fomichev } 5574ef15314aSYiFei Zhu 5575ef15314aSYiFei Zhu used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1, 5576ef15314aSYiFei Zhu sizeof(used_maps_new[0]), 5577ef15314aSYiFei Zhu GFP_KERNEL); 5578ef15314aSYiFei Zhu if (!used_maps_new) { 5579ef15314aSYiFei Zhu ret = -ENOMEM; 5580ef15314aSYiFei Zhu goto out_unlock; 5581ef15314aSYiFei Zhu } 5582ef15314aSYiFei Zhu 5583af66bfd3SHou Tao /* The bpf program will not access the bpf map, but for the sake of 5584af66bfd3SHou Tao * simplicity, increase sleepable_refcnt for sleepable program as well. 5585af66bfd3SHou Tao */ 558666c84731SAndrii Nakryiko if (prog->sleepable) 5587af66bfd3SHou Tao atomic64_inc(&map->sleepable_refcnt); 5588ef15314aSYiFei Zhu memcpy(used_maps_new, used_maps_old, 5589ef15314aSYiFei Zhu sizeof(used_maps_old[0]) * prog->aux->used_map_cnt); 5590ef15314aSYiFei Zhu used_maps_new[prog->aux->used_map_cnt] = map; 5591ef15314aSYiFei Zhu 5592ef15314aSYiFei Zhu prog->aux->used_map_cnt++; 5593ef15314aSYiFei Zhu prog->aux->used_maps = used_maps_new; 5594ef15314aSYiFei Zhu 5595ef15314aSYiFei Zhu kfree(used_maps_old); 5596ef15314aSYiFei Zhu 5597ef15314aSYiFei Zhu out_unlock: 5598ef15314aSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex); 5599ef15314aSYiFei Zhu 5600ef15314aSYiFei Zhu if (ret) 5601ef15314aSYiFei Zhu bpf_map_put(map); 5602ef15314aSYiFei Zhu out_prog_put: 5603ef15314aSYiFei Zhu bpf_prog_put(prog); 5604ef15314aSYiFei Zhu return ret; 5605ef15314aSYiFei Zhu } 5606ef15314aSYiFei Zhu 560735f96de0SAndrii Nakryiko #define BPF_TOKEN_CREATE_LAST_FIELD token_create.bpffs_fd 560835f96de0SAndrii Nakryiko 560935f96de0SAndrii Nakryiko static int token_create(union bpf_attr *attr) 561035f96de0SAndrii Nakryiko { 561135f96de0SAndrii Nakryiko if (CHECK_ATTR(BPF_TOKEN_CREATE)) 561235f96de0SAndrii Nakryiko return -EINVAL; 561335f96de0SAndrii Nakryiko 561435f96de0SAndrii Nakryiko /* no flags are supported yet */ 561535f96de0SAndrii Nakryiko if (attr->token_create.flags) 561635f96de0SAndrii Nakryiko return -EINVAL; 561735f96de0SAndrii Nakryiko 561835f96de0SAndrii Nakryiko return bpf_token_create(attr); 561935f96de0SAndrii Nakryiko } 562035f96de0SAndrii Nakryiko 5621af2ac3e1SAlexei Starovoitov static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) 562299c55f7dSAlexei Starovoitov { 56238096f229SGreg Kroah-Hartman union bpf_attr attr; 562499c55f7dSAlexei Starovoitov int err; 562599c55f7dSAlexei Starovoitov 5626dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size); 562799c55f7dSAlexei Starovoitov if (err) 562899c55f7dSAlexei Starovoitov return err; 56291e270976SMartin KaFai Lau size = min_t(u32, size, sizeof(attr)); 563099c55f7dSAlexei Starovoitov 563199c55f7dSAlexei Starovoitov /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 56328096f229SGreg Kroah-Hartman memset(&attr, 0, sizeof(attr)); 5633af2ac3e1SAlexei Starovoitov if (copy_from_bpfptr(&attr, uattr, size) != 0) 563499c55f7dSAlexei Starovoitov return -EFAULT; 563599c55f7dSAlexei Starovoitov 5636afdb09c7SChenbo Feng err = security_bpf(cmd, &attr, size); 5637afdb09c7SChenbo Feng if (err < 0) 5638afdb09c7SChenbo Feng return err; 5639afdb09c7SChenbo Feng 564099c55f7dSAlexei Starovoitov switch (cmd) { 564199c55f7dSAlexei Starovoitov case BPF_MAP_CREATE: 564299c55f7dSAlexei Starovoitov err = map_create(&attr); 564399c55f7dSAlexei Starovoitov break; 5644db20fd2bSAlexei Starovoitov case BPF_MAP_LOOKUP_ELEM: 5645db20fd2bSAlexei Starovoitov err = map_lookup_elem(&attr); 5646db20fd2bSAlexei Starovoitov break; 5647db20fd2bSAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 5648af2ac3e1SAlexei Starovoitov err = map_update_elem(&attr, uattr); 5649db20fd2bSAlexei Starovoitov break; 5650db20fd2bSAlexei Starovoitov case BPF_MAP_DELETE_ELEM: 5651b88df697SBenjamin Tissoires err = map_delete_elem(&attr, uattr); 5652db20fd2bSAlexei Starovoitov break; 5653db20fd2bSAlexei Starovoitov case BPF_MAP_GET_NEXT_KEY: 5654db20fd2bSAlexei Starovoitov err = map_get_next_key(&attr); 5655db20fd2bSAlexei Starovoitov break; 565687df15deSDaniel Borkmann case BPF_MAP_FREEZE: 565787df15deSDaniel Borkmann err = map_freeze(&attr); 565887df15deSDaniel Borkmann break; 565909756af4SAlexei Starovoitov case BPF_PROG_LOAD: 566047a71c1fSAndrii Nakryiko err = bpf_prog_load(&attr, uattr, size); 566109756af4SAlexei Starovoitov break; 5662b2197755SDaniel Borkmann case BPF_OBJ_PIN: 5663b2197755SDaniel Borkmann err = bpf_obj_pin(&attr); 5664b2197755SDaniel Borkmann break; 5665b2197755SDaniel Borkmann case BPF_OBJ_GET: 5666b2197755SDaniel Borkmann err = bpf_obj_get(&attr); 5667b2197755SDaniel Borkmann break; 5668f4324551SDaniel Mack case BPF_PROG_ATTACH: 5669f4324551SDaniel Mack err = bpf_prog_attach(&attr); 5670f4324551SDaniel Mack break; 5671f4324551SDaniel Mack case BPF_PROG_DETACH: 5672f4324551SDaniel Mack err = bpf_prog_detach(&attr); 5673f4324551SDaniel Mack break; 5674468e2f64SAlexei Starovoitov case BPF_PROG_QUERY: 5675af2ac3e1SAlexei Starovoitov err = bpf_prog_query(&attr, uattr.user); 5676468e2f64SAlexei Starovoitov break; 56771cf1cae9SAlexei Starovoitov case BPF_PROG_TEST_RUN: 5678af2ac3e1SAlexei Starovoitov err = bpf_prog_test_run(&attr, uattr.user); 56791cf1cae9SAlexei Starovoitov break; 568034ad5580SMartin KaFai Lau case BPF_PROG_GET_NEXT_ID: 5681af2ac3e1SAlexei Starovoitov err = bpf_obj_get_next_id(&attr, uattr.user, 568234ad5580SMartin KaFai Lau &prog_idr, &prog_idr_lock); 568334ad5580SMartin KaFai Lau break; 568434ad5580SMartin KaFai Lau case BPF_MAP_GET_NEXT_ID: 5685af2ac3e1SAlexei Starovoitov err = bpf_obj_get_next_id(&attr, uattr.user, 568634ad5580SMartin KaFai Lau &map_idr, &map_idr_lock); 568734ad5580SMartin KaFai Lau break; 56881b9ed84eSQuentin Monnet case BPF_BTF_GET_NEXT_ID: 5689af2ac3e1SAlexei Starovoitov err = bpf_obj_get_next_id(&attr, uattr.user, 56901b9ed84eSQuentin Monnet &btf_idr, &btf_idr_lock); 56911b9ed84eSQuentin Monnet break; 5692b16d9aa4SMartin KaFai Lau case BPF_PROG_GET_FD_BY_ID: 5693b16d9aa4SMartin KaFai Lau err = bpf_prog_get_fd_by_id(&attr); 5694b16d9aa4SMartin KaFai Lau break; 5695bd5f5f4eSMartin KaFai Lau case BPF_MAP_GET_FD_BY_ID: 5696bd5f5f4eSMartin KaFai Lau err = bpf_map_get_fd_by_id(&attr); 5697bd5f5f4eSMartin KaFai Lau break; 56981e270976SMartin KaFai Lau case BPF_OBJ_GET_INFO_BY_FD: 5699af2ac3e1SAlexei Starovoitov err = bpf_obj_get_info_by_fd(&attr, uattr.user); 57001e270976SMartin KaFai Lau break; 5701c4f6699dSAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN: 5702c4f6699dSAlexei Starovoitov err = bpf_raw_tracepoint_open(&attr); 5703c4f6699dSAlexei Starovoitov break; 5704f56a653cSMartin KaFai Lau case BPF_BTF_LOAD: 570547a71c1fSAndrii Nakryiko err = bpf_btf_load(&attr, uattr, size); 5706f56a653cSMartin KaFai Lau break; 570778958fcaSMartin KaFai Lau case BPF_BTF_GET_FD_BY_ID: 570878958fcaSMartin KaFai Lau err = bpf_btf_get_fd_by_id(&attr); 570978958fcaSMartin KaFai Lau break; 571041bdc4b4SYonghong Song case BPF_TASK_FD_QUERY: 5711af2ac3e1SAlexei Starovoitov err = bpf_task_fd_query(&attr, uattr.user); 571241bdc4b4SYonghong Song break; 5713bd513cd0SMauricio Vasquez B case BPF_MAP_LOOKUP_AND_DELETE_ELEM: 5714bd513cd0SMauricio Vasquez B err = map_lookup_and_delete_elem(&attr); 5715bd513cd0SMauricio Vasquez B break; 5716cb4d03abSBrian Vazquez case BPF_MAP_LOOKUP_BATCH: 5717af2ac3e1SAlexei Starovoitov err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_LOOKUP_BATCH); 5718cb4d03abSBrian Vazquez break; 571905799638SYonghong Song case BPF_MAP_LOOKUP_AND_DELETE_BATCH: 5720af2ac3e1SAlexei Starovoitov err = bpf_map_do_batch(&attr, uattr.user, 572105799638SYonghong Song BPF_MAP_LOOKUP_AND_DELETE_BATCH); 572205799638SYonghong Song break; 5723aa2e93b8SBrian Vazquez case BPF_MAP_UPDATE_BATCH: 5724af2ac3e1SAlexei Starovoitov err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_UPDATE_BATCH); 5725aa2e93b8SBrian Vazquez break; 5726aa2e93b8SBrian Vazquez case BPF_MAP_DELETE_BATCH: 5727af2ac3e1SAlexei Starovoitov err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_DELETE_BATCH); 5728aa2e93b8SBrian Vazquez break; 5729af6eea57SAndrii Nakryiko case BPF_LINK_CREATE: 5730af2ac3e1SAlexei Starovoitov err = link_create(&attr, uattr); 5731af6eea57SAndrii Nakryiko break; 57320c991ebcSAndrii Nakryiko case BPF_LINK_UPDATE: 57330c991ebcSAndrii Nakryiko err = link_update(&attr); 57340c991ebcSAndrii Nakryiko break; 57352d602c8cSAndrii Nakryiko case BPF_LINK_GET_FD_BY_ID: 57362d602c8cSAndrii Nakryiko err = bpf_link_get_fd_by_id(&attr); 57372d602c8cSAndrii Nakryiko break; 57382d602c8cSAndrii Nakryiko case BPF_LINK_GET_NEXT_ID: 5739af2ac3e1SAlexei Starovoitov err = bpf_obj_get_next_id(&attr, uattr.user, 57402d602c8cSAndrii Nakryiko &link_idr, &link_idr_lock); 57412d602c8cSAndrii Nakryiko break; 5742d46edd67SSong Liu case BPF_ENABLE_STATS: 5743d46edd67SSong Liu err = bpf_enable_stats(&attr); 5744d46edd67SSong Liu break; 5745ac51d99bSYonghong Song case BPF_ITER_CREATE: 5746ac51d99bSYonghong Song err = bpf_iter_create(&attr); 5747ac51d99bSYonghong Song break; 574873b11c2aSAndrii Nakryiko case BPF_LINK_DETACH: 574973b11c2aSAndrii Nakryiko err = link_detach(&attr); 575073b11c2aSAndrii Nakryiko break; 5751ef15314aSYiFei Zhu case BPF_PROG_BIND_MAP: 5752ef15314aSYiFei Zhu err = bpf_prog_bind_map(&attr); 5753ef15314aSYiFei Zhu break; 575435f96de0SAndrii Nakryiko case BPF_TOKEN_CREATE: 575535f96de0SAndrii Nakryiko err = token_create(&attr); 575635f96de0SAndrii Nakryiko break; 575799c55f7dSAlexei Starovoitov default: 575899c55f7dSAlexei Starovoitov err = -EINVAL; 575999c55f7dSAlexei Starovoitov break; 576099c55f7dSAlexei Starovoitov } 576199c55f7dSAlexei Starovoitov 576299c55f7dSAlexei Starovoitov return err; 576399c55f7dSAlexei Starovoitov } 576479a7f8bdSAlexei Starovoitov 5765af2ac3e1SAlexei Starovoitov SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 5766af2ac3e1SAlexei Starovoitov { 5767af2ac3e1SAlexei Starovoitov return __sys_bpf(cmd, USER_BPFPTR(uattr), size); 5768af2ac3e1SAlexei Starovoitov } 5769af2ac3e1SAlexei Starovoitov 577079a7f8bdSAlexei Starovoitov static bool syscall_prog_is_valid_access(int off, int size, 577179a7f8bdSAlexei Starovoitov enum bpf_access_type type, 577279a7f8bdSAlexei Starovoitov const struct bpf_prog *prog, 577379a7f8bdSAlexei Starovoitov struct bpf_insn_access_aux *info) 577479a7f8bdSAlexei Starovoitov { 577579a7f8bdSAlexei Starovoitov if (off < 0 || off >= U16_MAX) 577679a7f8bdSAlexei Starovoitov return false; 577779a7f8bdSAlexei Starovoitov if (off % size != 0) 577879a7f8bdSAlexei Starovoitov return false; 577979a7f8bdSAlexei Starovoitov return true; 578079a7f8bdSAlexei Starovoitov } 578179a7f8bdSAlexei Starovoitov 5782b1d18a75SAlexei Starovoitov BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size) 578379a7f8bdSAlexei Starovoitov { 5784af2ac3e1SAlexei Starovoitov switch (cmd) { 5785af2ac3e1SAlexei Starovoitov case BPF_MAP_CREATE: 5786b88df697SBenjamin Tissoires case BPF_MAP_DELETE_ELEM: 5787af2ac3e1SAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 5788af2ac3e1SAlexei Starovoitov case BPF_MAP_FREEZE: 5789b88df697SBenjamin Tissoires case BPF_MAP_GET_FD_BY_ID: 5790af2ac3e1SAlexei Starovoitov case BPF_PROG_LOAD: 5791c571bd75SAlexei Starovoitov case BPF_BTF_LOAD: 5792b1d18a75SAlexei Starovoitov case BPF_LINK_CREATE: 5793b1d18a75SAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN: 5794af2ac3e1SAlexei Starovoitov break; 579586f44fceSAlexei Starovoitov default: 579686f44fceSAlexei Starovoitov return -EINVAL; 579786f44fceSAlexei Starovoitov } 579886f44fceSAlexei Starovoitov return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size); 579986f44fceSAlexei Starovoitov } 580086f44fceSAlexei Starovoitov 58014e4588f1SAlexei Starovoitov 58024e4588f1SAlexei Starovoitov /* To shut up -Wmissing-prototypes. 58034e4588f1SAlexei Starovoitov * This function is used by the kernel light skeleton 58044e4588f1SAlexei Starovoitov * to load bpf programs when modules are loaded or during kernel boot. 58054e4588f1SAlexei Starovoitov * See tools/lib/bpf/skel_internal.h 58064e4588f1SAlexei Starovoitov */ 58074e4588f1SAlexei Starovoitov int kern_sys_bpf(int cmd, union bpf_attr *attr, unsigned int size); 58084e4588f1SAlexei Starovoitov 580986f44fceSAlexei Starovoitov int kern_sys_bpf(int cmd, union bpf_attr *attr, unsigned int size) 581086f44fceSAlexei Starovoitov { 581186f44fceSAlexei Starovoitov struct bpf_prog * __maybe_unused prog; 581286f44fceSAlexei Starovoitov struct bpf_tramp_run_ctx __maybe_unused run_ctx; 581386f44fceSAlexei Starovoitov 581486f44fceSAlexei Starovoitov switch (cmd) { 5815b1d18a75SAlexei Starovoitov #ifdef CONFIG_BPF_JIT /* __bpf_prog_enter_sleepable used by trampoline and JIT */ 5816b1d18a75SAlexei Starovoitov case BPF_PROG_TEST_RUN: 5817b1d18a75SAlexei Starovoitov if (attr->test.data_in || attr->test.data_out || 5818b1d18a75SAlexei Starovoitov attr->test.ctx_out || attr->test.duration || 5819b1d18a75SAlexei Starovoitov attr->test.repeat || attr->test.flags) 5820b1d18a75SAlexei Starovoitov return -EINVAL; 5821b1d18a75SAlexei Starovoitov 5822b1d18a75SAlexei Starovoitov prog = bpf_prog_get_type(attr->test.prog_fd, BPF_PROG_TYPE_SYSCALL); 5823b1d18a75SAlexei Starovoitov if (IS_ERR(prog)) 5824b1d18a75SAlexei Starovoitov return PTR_ERR(prog); 5825b1d18a75SAlexei Starovoitov 5826b1d18a75SAlexei Starovoitov if (attr->test.ctx_size_in < prog->aux->max_ctx_offset || 5827b1d18a75SAlexei Starovoitov attr->test.ctx_size_in > U16_MAX) { 5828b1d18a75SAlexei Starovoitov bpf_prog_put(prog); 5829b1d18a75SAlexei Starovoitov return -EINVAL; 5830b1d18a75SAlexei Starovoitov } 5831b1d18a75SAlexei Starovoitov 5832e384c7b7SKui-Feng Lee run_ctx.bpf_cookie = 0; 5833271de525SMartin KaFai Lau if (!__bpf_prog_enter_sleepable_recur(prog, &run_ctx)) { 5834b1d18a75SAlexei Starovoitov /* recursion detected */ 58357645629fSSebastian Andrzej Siewior __bpf_prog_exit_sleepable_recur(prog, 0, &run_ctx); 5836b1d18a75SAlexei Starovoitov bpf_prog_put(prog); 5837b1d18a75SAlexei Starovoitov return -EBUSY; 5838b1d18a75SAlexei Starovoitov } 5839b1d18a75SAlexei Starovoitov attr->test.retval = bpf_prog_run(prog, (void *) (long) attr->test.ctx_in); 5840271de525SMartin KaFai Lau __bpf_prog_exit_sleepable_recur(prog, 0 /* bpf_prog_run does runtime stats */, 5841271de525SMartin KaFai Lau &run_ctx); 5842b1d18a75SAlexei Starovoitov bpf_prog_put(prog); 5843b1d18a75SAlexei Starovoitov return 0; 5844b1d18a75SAlexei Starovoitov #endif 5845af2ac3e1SAlexei Starovoitov default: 584686f44fceSAlexei Starovoitov return ____bpf_sys_bpf(cmd, attr, size); 584779a7f8bdSAlexei Starovoitov } 5848af2ac3e1SAlexei Starovoitov } 584986f44fceSAlexei Starovoitov EXPORT_SYMBOL(kern_sys_bpf); 585079a7f8bdSAlexei Starovoitov 58513a2daa72SPu Lehui static const struct bpf_func_proto bpf_sys_bpf_proto = { 585279a7f8bdSAlexei Starovoitov .func = bpf_sys_bpf, 585379a7f8bdSAlexei Starovoitov .gpl_only = false, 585479a7f8bdSAlexei Starovoitov .ret_type = RET_INTEGER, 585579a7f8bdSAlexei Starovoitov .arg1_type = ARG_ANYTHING, 5856216e3cd2SHao Luo .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, 585779a7f8bdSAlexei Starovoitov .arg3_type = ARG_CONST_SIZE, 585879a7f8bdSAlexei Starovoitov }; 585979a7f8bdSAlexei Starovoitov 586079a7f8bdSAlexei Starovoitov const struct bpf_func_proto * __weak 586179a7f8bdSAlexei Starovoitov tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 586279a7f8bdSAlexei Starovoitov { 5863bbc1d247SAndrii Nakryiko return bpf_base_func_proto(func_id, prog); 586479a7f8bdSAlexei Starovoitov } 586579a7f8bdSAlexei Starovoitov 58663abea089SAlexei Starovoitov BPF_CALL_1(bpf_sys_close, u32, fd) 58673abea089SAlexei Starovoitov { 58683abea089SAlexei Starovoitov /* When bpf program calls this helper there should not be 58693abea089SAlexei Starovoitov * an fdget() without matching completed fdput(). 58703abea089SAlexei Starovoitov * This helper is allowed in the following callchain only: 58713abea089SAlexei Starovoitov * sys_bpf->prog_test_run->bpf_prog->bpf_sys_close 58723abea089SAlexei Starovoitov */ 58733abea089SAlexei Starovoitov return close_fd(fd); 58743abea089SAlexei Starovoitov } 58753abea089SAlexei Starovoitov 58763a2daa72SPu Lehui static const struct bpf_func_proto bpf_sys_close_proto = { 58773abea089SAlexei Starovoitov .func = bpf_sys_close, 58783abea089SAlexei Starovoitov .gpl_only = false, 58793abea089SAlexei Starovoitov .ret_type = RET_INTEGER, 58803abea089SAlexei Starovoitov .arg1_type = ARG_ANYTHING, 58813abea089SAlexei Starovoitov }; 58823abea089SAlexei Starovoitov 5883d6aef08aSKumar Kartikeya Dwivedi BPF_CALL_4(bpf_kallsyms_lookup_name, const char *, name, int, name_sz, int, flags, u64 *, res) 5884d6aef08aSKumar Kartikeya Dwivedi { 5885d6aef08aSKumar Kartikeya Dwivedi if (flags) 5886d6aef08aSKumar Kartikeya Dwivedi return -EINVAL; 5887d6aef08aSKumar Kartikeya Dwivedi 5888d6aef08aSKumar Kartikeya Dwivedi if (name_sz <= 1 || name[name_sz - 1]) 5889d6aef08aSKumar Kartikeya Dwivedi return -EINVAL; 5890d6aef08aSKumar Kartikeya Dwivedi 5891d6aef08aSKumar Kartikeya Dwivedi if (!bpf_dump_raw_ok(current_cred())) 5892d6aef08aSKumar Kartikeya Dwivedi return -EPERM; 5893d6aef08aSKumar Kartikeya Dwivedi 5894d6aef08aSKumar Kartikeya Dwivedi *res = kallsyms_lookup_name(name); 5895d6aef08aSKumar Kartikeya Dwivedi return *res ? 0 : -ENOENT; 5896d6aef08aSKumar Kartikeya Dwivedi } 5897d6aef08aSKumar Kartikeya Dwivedi 5898dc368e1cSJoanne Koong static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = { 5899d6aef08aSKumar Kartikeya Dwivedi .func = bpf_kallsyms_lookup_name, 5900d6aef08aSKumar Kartikeya Dwivedi .gpl_only = false, 5901d6aef08aSKumar Kartikeya Dwivedi .ret_type = RET_INTEGER, 5902d6aef08aSKumar Kartikeya Dwivedi .arg1_type = ARG_PTR_TO_MEM, 5903d4efb170SKumar Kartikeya Dwivedi .arg2_type = ARG_CONST_SIZE_OR_ZERO, 5904d6aef08aSKumar Kartikeya Dwivedi .arg3_type = ARG_ANYTHING, 5905d6aef08aSKumar Kartikeya Dwivedi .arg4_type = ARG_PTR_TO_LONG, 5906d6aef08aSKumar Kartikeya Dwivedi }; 5907d6aef08aSKumar Kartikeya Dwivedi 590879a7f8bdSAlexei Starovoitov static const struct bpf_func_proto * 590979a7f8bdSAlexei Starovoitov syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 591079a7f8bdSAlexei Starovoitov { 591179a7f8bdSAlexei Starovoitov switch (func_id) { 591279a7f8bdSAlexei Starovoitov case BPF_FUNC_sys_bpf: 5913bbc1d247SAndrii Nakryiko return !bpf_token_capable(prog->aux->token, CAP_PERFMON) 5914bbc1d247SAndrii Nakryiko ? NULL : &bpf_sys_bpf_proto; 59153d78417bSAlexei Starovoitov case BPF_FUNC_btf_find_by_name_kind: 59163d78417bSAlexei Starovoitov return &bpf_btf_find_by_name_kind_proto; 59173abea089SAlexei Starovoitov case BPF_FUNC_sys_close: 59183abea089SAlexei Starovoitov return &bpf_sys_close_proto; 5919d6aef08aSKumar Kartikeya Dwivedi case BPF_FUNC_kallsyms_lookup_name: 5920d6aef08aSKumar Kartikeya Dwivedi return &bpf_kallsyms_lookup_name_proto; 592179a7f8bdSAlexei Starovoitov default: 592279a7f8bdSAlexei Starovoitov return tracing_prog_func_proto(func_id, prog); 592379a7f8bdSAlexei Starovoitov } 592479a7f8bdSAlexei Starovoitov } 592579a7f8bdSAlexei Starovoitov 592679a7f8bdSAlexei Starovoitov const struct bpf_verifier_ops bpf_syscall_verifier_ops = { 592779a7f8bdSAlexei Starovoitov .get_func_proto = syscall_prog_func_proto, 592879a7f8bdSAlexei Starovoitov .is_valid_access = syscall_prog_is_valid_access, 592979a7f8bdSAlexei Starovoitov }; 593079a7f8bdSAlexei Starovoitov 593179a7f8bdSAlexei Starovoitov const struct bpf_prog_ops bpf_syscall_prog_ops = { 593279a7f8bdSAlexei Starovoitov .test_run = bpf_prog_test_run_syscall, 593379a7f8bdSAlexei Starovoitov }; 59342900005eSYan Zhu 59352900005eSYan Zhu #ifdef CONFIG_SYSCTL 59362900005eSYan Zhu static int bpf_stats_handler(struct ctl_table *table, int write, 59372900005eSYan Zhu void *buffer, size_t *lenp, loff_t *ppos) 59382900005eSYan Zhu { 59392900005eSYan Zhu struct static_key *key = (struct static_key *)table->data; 59402900005eSYan Zhu static int saved_val; 59412900005eSYan Zhu int val, ret; 59422900005eSYan Zhu struct ctl_table tmp = { 59432900005eSYan Zhu .data = &val, 59442900005eSYan Zhu .maxlen = sizeof(val), 59452900005eSYan Zhu .mode = table->mode, 59462900005eSYan Zhu .extra1 = SYSCTL_ZERO, 59472900005eSYan Zhu .extra2 = SYSCTL_ONE, 59482900005eSYan Zhu }; 59492900005eSYan Zhu 59502900005eSYan Zhu if (write && !capable(CAP_SYS_ADMIN)) 59512900005eSYan Zhu return -EPERM; 59522900005eSYan Zhu 59532900005eSYan Zhu mutex_lock(&bpf_stats_enabled_mutex); 59542900005eSYan Zhu val = saved_val; 59552900005eSYan Zhu ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos); 59562900005eSYan Zhu if (write && !ret && val != saved_val) { 59572900005eSYan Zhu if (val) 59582900005eSYan Zhu static_key_slow_inc(key); 59592900005eSYan Zhu else 59602900005eSYan Zhu static_key_slow_dec(key); 59612900005eSYan Zhu saved_val = val; 59622900005eSYan Zhu } 59632900005eSYan Zhu mutex_unlock(&bpf_stats_enabled_mutex); 59642900005eSYan Zhu return ret; 59652900005eSYan Zhu } 59662900005eSYan Zhu 59672900005eSYan Zhu void __weak unpriv_ebpf_notify(int new_state) 59682900005eSYan Zhu { 59692900005eSYan Zhu } 59702900005eSYan Zhu 59712900005eSYan Zhu static int bpf_unpriv_handler(struct ctl_table *table, int write, 59722900005eSYan Zhu void *buffer, size_t *lenp, loff_t *ppos) 59732900005eSYan Zhu { 59742900005eSYan Zhu int ret, unpriv_enable = *(int *)table->data; 59752900005eSYan Zhu bool locked_state = unpriv_enable == 1; 59762900005eSYan Zhu struct ctl_table tmp = *table; 59772900005eSYan Zhu 59782900005eSYan Zhu if (write && !capable(CAP_SYS_ADMIN)) 59792900005eSYan Zhu return -EPERM; 59802900005eSYan Zhu 59812900005eSYan Zhu tmp.data = &unpriv_enable; 59822900005eSYan Zhu ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos); 59832900005eSYan Zhu if (write && !ret) { 59842900005eSYan Zhu if (locked_state && unpriv_enable != 1) 59852900005eSYan Zhu return -EPERM; 59862900005eSYan Zhu *(int *)table->data = unpriv_enable; 59872900005eSYan Zhu } 59882900005eSYan Zhu 5989fedf9920SKui-Feng Lee if (write) 59902900005eSYan Zhu unpriv_ebpf_notify(unpriv_enable); 59912900005eSYan Zhu 59922900005eSYan Zhu return ret; 59932900005eSYan Zhu } 59942900005eSYan Zhu 59952900005eSYan Zhu static struct ctl_table bpf_syscall_table[] = { 59962900005eSYan Zhu { 59972900005eSYan Zhu .procname = "unprivileged_bpf_disabled", 59982900005eSYan Zhu .data = &sysctl_unprivileged_bpf_disabled, 59992900005eSYan Zhu .maxlen = sizeof(sysctl_unprivileged_bpf_disabled), 60002900005eSYan Zhu .mode = 0644, 60012900005eSYan Zhu .proc_handler = bpf_unpriv_handler, 60022900005eSYan Zhu .extra1 = SYSCTL_ZERO, 60032900005eSYan Zhu .extra2 = SYSCTL_TWO, 60042900005eSYan Zhu }, 60052900005eSYan Zhu { 60062900005eSYan Zhu .procname = "bpf_stats_enabled", 60072900005eSYan Zhu .data = &bpf_stats_enabled_key.key, 60082900005eSYan Zhu .mode = 0644, 60092900005eSYan Zhu .proc_handler = bpf_stats_handler, 60102900005eSYan Zhu }, 60112900005eSYan Zhu { } 60122900005eSYan Zhu }; 60132900005eSYan Zhu 60142900005eSYan Zhu static int __init bpf_syscall_sysctl_init(void) 60152900005eSYan Zhu { 60162900005eSYan Zhu register_sysctl_init("kernel", bpf_syscall_table); 60172900005eSYan Zhu return 0; 60182900005eSYan Zhu } 60192900005eSYan Zhu late_initcall(bpf_syscall_sysctl_init); 60202900005eSYan Zhu #endif /* CONFIG_SYSCTL */ 6021