199c55f7dSAlexei Starovoitov /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 299c55f7dSAlexei Starovoitov * 399c55f7dSAlexei Starovoitov * This program is free software; you can redistribute it and/or 499c55f7dSAlexei Starovoitov * modify it under the terms of version 2 of the GNU General Public 599c55f7dSAlexei Starovoitov * License as published by the Free Software Foundation. 699c55f7dSAlexei Starovoitov * 799c55f7dSAlexei Starovoitov * This program is distributed in the hope that it will be useful, but 899c55f7dSAlexei Starovoitov * WITHOUT ANY WARRANTY; without even the implied warranty of 999c55f7dSAlexei Starovoitov * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 1099c55f7dSAlexei Starovoitov * General Public License for more details. 1199c55f7dSAlexei Starovoitov */ 1299c55f7dSAlexei Starovoitov #include <linux/bpf.h> 13a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 14f4364dcfSSean Young #include <linux/bpf_lirc.h> 15f56a653cSMartin KaFai Lau #include <linux/btf.h> 1699c55f7dSAlexei Starovoitov #include <linux/syscalls.h> 1799c55f7dSAlexei Starovoitov #include <linux/slab.h> 183f07c014SIngo Molnar #include <linux/sched/signal.h> 19d407bd25SDaniel Borkmann #include <linux/vmalloc.h> 20d407bd25SDaniel Borkmann #include <linux/mmzone.h> 2199c55f7dSAlexei Starovoitov #include <linux/anon_inodes.h> 2241bdc4b4SYonghong Song #include <linux/fdtable.h> 23db20fd2bSAlexei Starovoitov #include <linux/file.h> 2441bdc4b4SYonghong Song #include <linux/fs.h> 2509756af4SAlexei Starovoitov #include <linux/license.h> 2609756af4SAlexei Starovoitov #include <linux/filter.h> 272541517cSAlexei Starovoitov #include <linux/version.h> 28535e7b4bSMickaël Salaün #include <linux/kernel.h> 29dc4bb0e2SMartin KaFai Lau #include <linux/idr.h> 30cb4d2b3fSMartin KaFai Lau #include <linux/cred.h> 31cb4d2b3fSMartin KaFai Lau #include <linux/timekeeping.h> 32cb4d2b3fSMartin KaFai Lau #include <linux/ctype.h> 339ef09e35SMark Rutland #include <linux/nospec.h> 3499c55f7dSAlexei Starovoitov 3514dc6f04SMartin KaFai Lau #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ 3614dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 3714dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 3814dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 3914dc6f04SMartin KaFai Lau #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 4014dc6f04SMartin KaFai Lau #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map)) 4114dc6f04SMartin KaFai Lau 426e71b04aSChenbo Feng #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) 436e71b04aSChenbo Feng 44b121d1e7SAlexei Starovoitov DEFINE_PER_CPU(int, bpf_prog_active); 45dc4bb0e2SMartin KaFai Lau static DEFINE_IDR(prog_idr); 46dc4bb0e2SMartin KaFai Lau static DEFINE_SPINLOCK(prog_idr_lock); 47f3f1c054SMartin KaFai Lau static DEFINE_IDR(map_idr); 48f3f1c054SMartin KaFai Lau static DEFINE_SPINLOCK(map_idr_lock); 49b121d1e7SAlexei Starovoitov 501be7f75dSAlexei Starovoitov int sysctl_unprivileged_bpf_disabled __read_mostly; 511be7f75dSAlexei Starovoitov 5240077e0cSJohannes Berg static const struct bpf_map_ops * const bpf_map_types[] = { 5340077e0cSJohannes Berg #define BPF_PROG_TYPE(_id, _ops) 5440077e0cSJohannes Berg #define BPF_MAP_TYPE(_id, _ops) \ 5540077e0cSJohannes Berg [_id] = &_ops, 5640077e0cSJohannes Berg #include <linux/bpf_types.h> 5740077e0cSJohannes Berg #undef BPF_PROG_TYPE 5840077e0cSJohannes Berg #undef BPF_MAP_TYPE 5940077e0cSJohannes Berg }; 6099c55f7dSAlexei Starovoitov 61752ba56fSMickaël Salaün /* 62752ba56fSMickaël Salaün * If we're handed a bigger struct than we know of, ensure all the unknown bits 63752ba56fSMickaël Salaün * are 0 - i.e. new user-space does not rely on any kernel feature extensions 64752ba56fSMickaël Salaün * we don't know about yet. 65752ba56fSMickaël Salaün * 66752ba56fSMickaël Salaün * There is a ToCToU between this function call and the following 67752ba56fSMickaël Salaün * copy_from_user() call. However, this is not a concern since this function is 68752ba56fSMickaël Salaün * meant to be a future-proofing of bits. 69752ba56fSMickaël Salaün */ 70dcab51f1SMartin KaFai Lau int bpf_check_uarg_tail_zero(void __user *uaddr, 7158291a74SMickaël Salaün size_t expected_size, 7258291a74SMickaël Salaün size_t actual_size) 7358291a74SMickaël Salaün { 7458291a74SMickaël Salaün unsigned char __user *addr; 7558291a74SMickaël Salaün unsigned char __user *end; 7658291a74SMickaël Salaün unsigned char val; 7758291a74SMickaël Salaün int err; 7858291a74SMickaël Salaün 79752ba56fSMickaël Salaün if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 80752ba56fSMickaël Salaün return -E2BIG; 81752ba56fSMickaël Salaün 8296d4f267SLinus Torvalds if (unlikely(!access_ok(uaddr, actual_size))) 83752ba56fSMickaël Salaün return -EFAULT; 84752ba56fSMickaël Salaün 8558291a74SMickaël Salaün if (actual_size <= expected_size) 8658291a74SMickaël Salaün return 0; 8758291a74SMickaël Salaün 8858291a74SMickaël Salaün addr = uaddr + expected_size; 8958291a74SMickaël Salaün end = uaddr + actual_size; 9058291a74SMickaël Salaün 9158291a74SMickaël Salaün for (; addr < end; addr++) { 9258291a74SMickaël Salaün err = get_user(val, addr); 9358291a74SMickaël Salaün if (err) 9458291a74SMickaël Salaün return err; 9558291a74SMickaël Salaün if (val) 9658291a74SMickaël Salaün return -E2BIG; 9758291a74SMickaël Salaün } 9858291a74SMickaël Salaün 9958291a74SMickaël Salaün return 0; 10058291a74SMickaël Salaün } 10158291a74SMickaël Salaün 102a3884572SJakub Kicinski const struct bpf_map_ops bpf_map_offload_ops = { 103a3884572SJakub Kicinski .map_alloc = bpf_map_offload_map_alloc, 104a3884572SJakub Kicinski .map_free = bpf_map_offload_map_free, 105e8d2bec0SDaniel Borkmann .map_check_btf = map_check_no_btf, 106a3884572SJakub Kicinski }; 107a3884572SJakub Kicinski 10899c55f7dSAlexei Starovoitov static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 10999c55f7dSAlexei Starovoitov { 1101110f3a9SJakub Kicinski const struct bpf_map_ops *ops; 1119ef09e35SMark Rutland u32 type = attr->map_type; 11299c55f7dSAlexei Starovoitov struct bpf_map *map; 1131110f3a9SJakub Kicinski int err; 11499c55f7dSAlexei Starovoitov 1159ef09e35SMark Rutland if (type >= ARRAY_SIZE(bpf_map_types)) 1161110f3a9SJakub Kicinski return ERR_PTR(-EINVAL); 1179ef09e35SMark Rutland type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types)); 1189ef09e35SMark Rutland ops = bpf_map_types[type]; 1191110f3a9SJakub Kicinski if (!ops) 12040077e0cSJohannes Berg return ERR_PTR(-EINVAL); 12140077e0cSJohannes Berg 1221110f3a9SJakub Kicinski if (ops->map_alloc_check) { 1231110f3a9SJakub Kicinski err = ops->map_alloc_check(attr); 1241110f3a9SJakub Kicinski if (err) 1251110f3a9SJakub Kicinski return ERR_PTR(err); 1261110f3a9SJakub Kicinski } 127a3884572SJakub Kicinski if (attr->map_ifindex) 128a3884572SJakub Kicinski ops = &bpf_map_offload_ops; 1291110f3a9SJakub Kicinski map = ops->map_alloc(attr); 13099c55f7dSAlexei Starovoitov if (IS_ERR(map)) 13199c55f7dSAlexei Starovoitov return map; 1321110f3a9SJakub Kicinski map->ops = ops; 1339ef09e35SMark Rutland map->map_type = type; 13499c55f7dSAlexei Starovoitov return map; 13599c55f7dSAlexei Starovoitov } 13699c55f7dSAlexei Starovoitov 13796eabe7aSMartin KaFai Lau void *bpf_map_area_alloc(size_t size, int numa_node) 138d407bd25SDaniel Borkmann { 139f01a7dbeSMartynas Pumputis /* We really just want to fail instead of triggering OOM killer 140f01a7dbeSMartynas Pumputis * under memory pressure, therefore we set __GFP_NORETRY to kmalloc, 141f01a7dbeSMartynas Pumputis * which is used for lower order allocation requests. 142f01a7dbeSMartynas Pumputis * 143f01a7dbeSMartynas Pumputis * It has been observed that higher order allocation requests done by 144f01a7dbeSMartynas Pumputis * vmalloc with __GFP_NORETRY being set might fail due to not trying 145f01a7dbeSMartynas Pumputis * to reclaim memory from the page cache, thus we set 146f01a7dbeSMartynas Pumputis * __GFP_RETRY_MAYFAIL to avoid such situations. 147d407bd25SDaniel Borkmann */ 148f01a7dbeSMartynas Pumputis 149f01a7dbeSMartynas Pumputis const gfp_t flags = __GFP_NOWARN | __GFP_ZERO; 150d407bd25SDaniel Borkmann void *area; 151d407bd25SDaniel Borkmann 152d407bd25SDaniel Borkmann if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 153f01a7dbeSMartynas Pumputis area = kmalloc_node(size, GFP_USER | __GFP_NORETRY | flags, 154f01a7dbeSMartynas Pumputis numa_node); 155d407bd25SDaniel Borkmann if (area != NULL) 156d407bd25SDaniel Borkmann return area; 157d407bd25SDaniel Borkmann } 158d407bd25SDaniel Borkmann 159f01a7dbeSMartynas Pumputis return __vmalloc_node_flags_caller(size, numa_node, 160f01a7dbeSMartynas Pumputis GFP_KERNEL | __GFP_RETRY_MAYFAIL | 161f01a7dbeSMartynas Pumputis flags, __builtin_return_address(0)); 162d407bd25SDaniel Borkmann } 163d407bd25SDaniel Borkmann 164d407bd25SDaniel Borkmann void bpf_map_area_free(void *area) 165d407bd25SDaniel Borkmann { 166d407bd25SDaniel Borkmann kvfree(area); 167d407bd25SDaniel Borkmann } 168d407bd25SDaniel Borkmann 169be70bcd5SDaniel Borkmann static u32 bpf_map_flags_retain_permanent(u32 flags) 170be70bcd5SDaniel Borkmann { 171be70bcd5SDaniel Borkmann /* Some map creation flags are not tied to the map object but 172be70bcd5SDaniel Borkmann * rather to the map fd instead, so they have no meaning upon 173be70bcd5SDaniel Borkmann * map object inspection since multiple file descriptors with 174be70bcd5SDaniel Borkmann * different (access) properties can exist here. Thus, given 175be70bcd5SDaniel Borkmann * this has zero meaning for the map itself, lets clear these 176be70bcd5SDaniel Borkmann * from here. 177be70bcd5SDaniel Borkmann */ 178be70bcd5SDaniel Borkmann return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY); 179be70bcd5SDaniel Borkmann } 180be70bcd5SDaniel Borkmann 181bd475643SJakub Kicinski void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) 182bd475643SJakub Kicinski { 183bd475643SJakub Kicinski map->map_type = attr->map_type; 184bd475643SJakub Kicinski map->key_size = attr->key_size; 185bd475643SJakub Kicinski map->value_size = attr->value_size; 186bd475643SJakub Kicinski map->max_entries = attr->max_entries; 187be70bcd5SDaniel Borkmann map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags); 188bd475643SJakub Kicinski map->numa_node = bpf_map_attr_numa_node(attr); 189bd475643SJakub Kicinski } 190bd475643SJakub Kicinski 1916c905981SAlexei Starovoitov int bpf_map_precharge_memlock(u32 pages) 1926c905981SAlexei Starovoitov { 1936c905981SAlexei Starovoitov struct user_struct *user = get_current_user(); 1946c905981SAlexei Starovoitov unsigned long memlock_limit, cur; 1956c905981SAlexei Starovoitov 1966c905981SAlexei Starovoitov memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 1976c905981SAlexei Starovoitov cur = atomic_long_read(&user->locked_vm); 1986c905981SAlexei Starovoitov free_uid(user); 1996c905981SAlexei Starovoitov if (cur + pages > memlock_limit) 2006c905981SAlexei Starovoitov return -EPERM; 2016c905981SAlexei Starovoitov return 0; 2026c905981SAlexei Starovoitov } 2036c905981SAlexei Starovoitov 2040a4c58f5SRoman Gushchin static int bpf_charge_memlock(struct user_struct *user, u32 pages) 205aaac3ba9SAlexei Starovoitov { 2060a4c58f5SRoman Gushchin unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 207aaac3ba9SAlexei Starovoitov 2080a4c58f5SRoman Gushchin if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) { 2090a4c58f5SRoman Gushchin atomic_long_sub(pages, &user->locked_vm); 210aaac3ba9SAlexei Starovoitov return -EPERM; 211aaac3ba9SAlexei Starovoitov } 212aaac3ba9SAlexei Starovoitov return 0; 213aaac3ba9SAlexei Starovoitov } 214aaac3ba9SAlexei Starovoitov 2150a4c58f5SRoman Gushchin static void bpf_uncharge_memlock(struct user_struct *user, u32 pages) 2160a4c58f5SRoman Gushchin { 2170a4c58f5SRoman Gushchin atomic_long_sub(pages, &user->locked_vm); 2180a4c58f5SRoman Gushchin } 2190a4c58f5SRoman Gushchin 2200a4c58f5SRoman Gushchin static int bpf_map_init_memlock(struct bpf_map *map) 2210a4c58f5SRoman Gushchin { 2220a4c58f5SRoman Gushchin struct user_struct *user = get_current_user(); 2230a4c58f5SRoman Gushchin int ret; 2240a4c58f5SRoman Gushchin 2250a4c58f5SRoman Gushchin ret = bpf_charge_memlock(user, map->pages); 2260a4c58f5SRoman Gushchin if (ret) { 2270a4c58f5SRoman Gushchin free_uid(user); 2280a4c58f5SRoman Gushchin return ret; 2290a4c58f5SRoman Gushchin } 2300a4c58f5SRoman Gushchin map->user = user; 2310a4c58f5SRoman Gushchin return ret; 2320a4c58f5SRoman Gushchin } 2330a4c58f5SRoman Gushchin 2340a4c58f5SRoman Gushchin static void bpf_map_release_memlock(struct bpf_map *map) 235aaac3ba9SAlexei Starovoitov { 236aaac3ba9SAlexei Starovoitov struct user_struct *user = map->user; 2370a4c58f5SRoman Gushchin bpf_uncharge_memlock(user, map->pages); 238aaac3ba9SAlexei Starovoitov free_uid(user); 239aaac3ba9SAlexei Starovoitov } 240aaac3ba9SAlexei Starovoitov 2410a4c58f5SRoman Gushchin int bpf_map_charge_memlock(struct bpf_map *map, u32 pages) 2420a4c58f5SRoman Gushchin { 2430a4c58f5SRoman Gushchin int ret; 2440a4c58f5SRoman Gushchin 2450a4c58f5SRoman Gushchin ret = bpf_charge_memlock(map->user, pages); 2460a4c58f5SRoman Gushchin if (ret) 2470a4c58f5SRoman Gushchin return ret; 2480a4c58f5SRoman Gushchin map->pages += pages; 2490a4c58f5SRoman Gushchin return ret; 2500a4c58f5SRoman Gushchin } 2510a4c58f5SRoman Gushchin 2520a4c58f5SRoman Gushchin void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages) 2530a4c58f5SRoman Gushchin { 2540a4c58f5SRoman Gushchin bpf_uncharge_memlock(map->user, pages); 2550a4c58f5SRoman Gushchin map->pages -= pages; 2560a4c58f5SRoman Gushchin } 2570a4c58f5SRoman Gushchin 258f3f1c054SMartin KaFai Lau static int bpf_map_alloc_id(struct bpf_map *map) 259f3f1c054SMartin KaFai Lau { 260f3f1c054SMartin KaFai Lau int id; 261f3f1c054SMartin KaFai Lau 262b76354cdSShaohua Li idr_preload(GFP_KERNEL); 263f3f1c054SMartin KaFai Lau spin_lock_bh(&map_idr_lock); 264f3f1c054SMartin KaFai Lau id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 265f3f1c054SMartin KaFai Lau if (id > 0) 266f3f1c054SMartin KaFai Lau map->id = id; 267f3f1c054SMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 268b76354cdSShaohua Li idr_preload_end(); 269f3f1c054SMartin KaFai Lau 270f3f1c054SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 271f3f1c054SMartin KaFai Lau return -ENOSPC; 272f3f1c054SMartin KaFai Lau 273f3f1c054SMartin KaFai Lau return id > 0 ? 0 : id; 274f3f1c054SMartin KaFai Lau } 275f3f1c054SMartin KaFai Lau 276a3884572SJakub Kicinski void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) 277f3f1c054SMartin KaFai Lau { 278930651a7SEric Dumazet unsigned long flags; 279930651a7SEric Dumazet 280a3884572SJakub Kicinski /* Offloaded maps are removed from the IDR store when their device 281a3884572SJakub Kicinski * disappears - even if someone holds an fd to them they are unusable, 282a3884572SJakub Kicinski * the memory is gone, all ops will fail; they are simply waiting for 283a3884572SJakub Kicinski * refcnt to drop to be freed. 284a3884572SJakub Kicinski */ 285a3884572SJakub Kicinski if (!map->id) 286a3884572SJakub Kicinski return; 287a3884572SJakub Kicinski 288bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 289930651a7SEric Dumazet spin_lock_irqsave(&map_idr_lock, flags); 290bd5f5f4eSMartin KaFai Lau else 291bd5f5f4eSMartin KaFai Lau __acquire(&map_idr_lock); 292bd5f5f4eSMartin KaFai Lau 293f3f1c054SMartin KaFai Lau idr_remove(&map_idr, map->id); 294a3884572SJakub Kicinski map->id = 0; 295bd5f5f4eSMartin KaFai Lau 296bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 297930651a7SEric Dumazet spin_unlock_irqrestore(&map_idr_lock, flags); 298bd5f5f4eSMartin KaFai Lau else 299bd5f5f4eSMartin KaFai Lau __release(&map_idr_lock); 300f3f1c054SMartin KaFai Lau } 301f3f1c054SMartin KaFai Lau 30299c55f7dSAlexei Starovoitov /* called from workqueue */ 30399c55f7dSAlexei Starovoitov static void bpf_map_free_deferred(struct work_struct *work) 30499c55f7dSAlexei Starovoitov { 30599c55f7dSAlexei Starovoitov struct bpf_map *map = container_of(work, struct bpf_map, work); 30699c55f7dSAlexei Starovoitov 3070a4c58f5SRoman Gushchin bpf_map_release_memlock(map); 308afdb09c7SChenbo Feng security_bpf_map_free(map); 30999c55f7dSAlexei Starovoitov /* implementation dependent freeing */ 31099c55f7dSAlexei Starovoitov map->ops->map_free(map); 31199c55f7dSAlexei Starovoitov } 31299c55f7dSAlexei Starovoitov 313c9da161cSDaniel Borkmann static void bpf_map_put_uref(struct bpf_map *map) 314c9da161cSDaniel Borkmann { 315c9da161cSDaniel Borkmann if (atomic_dec_and_test(&map->usercnt)) { 316ba6b8de4SJohn Fastabend if (map->ops->map_release_uref) 317ba6b8de4SJohn Fastabend map->ops->map_release_uref(map); 318c9da161cSDaniel Borkmann } 319c9da161cSDaniel Borkmann } 320c9da161cSDaniel Borkmann 32199c55f7dSAlexei Starovoitov /* decrement map refcnt and schedule it for freeing via workqueue 32299c55f7dSAlexei Starovoitov * (unrelying map implementation ops->map_free() might sleep) 32399c55f7dSAlexei Starovoitov */ 324bd5f5f4eSMartin KaFai Lau static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) 32599c55f7dSAlexei Starovoitov { 32699c55f7dSAlexei Starovoitov if (atomic_dec_and_test(&map->refcnt)) { 32734ad5580SMartin KaFai Lau /* bpf_map_free_id() must be called first */ 328bd5f5f4eSMartin KaFai Lau bpf_map_free_id(map, do_idr_lock); 32978958fcaSMartin KaFai Lau btf_put(map->btf); 33099c55f7dSAlexei Starovoitov INIT_WORK(&map->work, bpf_map_free_deferred); 33199c55f7dSAlexei Starovoitov schedule_work(&map->work); 33299c55f7dSAlexei Starovoitov } 33399c55f7dSAlexei Starovoitov } 33499c55f7dSAlexei Starovoitov 335bd5f5f4eSMartin KaFai Lau void bpf_map_put(struct bpf_map *map) 336bd5f5f4eSMartin KaFai Lau { 337bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, true); 338bd5f5f4eSMartin KaFai Lau } 339630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_put); 340bd5f5f4eSMartin KaFai Lau 341c9da161cSDaniel Borkmann void bpf_map_put_with_uref(struct bpf_map *map) 342c9da161cSDaniel Borkmann { 343c9da161cSDaniel Borkmann bpf_map_put_uref(map); 344c9da161cSDaniel Borkmann bpf_map_put(map); 345c9da161cSDaniel Borkmann } 346c9da161cSDaniel Borkmann 34799c55f7dSAlexei Starovoitov static int bpf_map_release(struct inode *inode, struct file *filp) 34899c55f7dSAlexei Starovoitov { 34961d1b6a4SDaniel Borkmann struct bpf_map *map = filp->private_data; 35061d1b6a4SDaniel Borkmann 35161d1b6a4SDaniel Borkmann if (map->ops->map_release) 35261d1b6a4SDaniel Borkmann map->ops->map_release(map, filp); 35361d1b6a4SDaniel Borkmann 35461d1b6a4SDaniel Borkmann bpf_map_put_with_uref(map); 35599c55f7dSAlexei Starovoitov return 0; 35699c55f7dSAlexei Starovoitov } 35799c55f7dSAlexei Starovoitov 35887df15deSDaniel Borkmann static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f) 35987df15deSDaniel Borkmann { 36087df15deSDaniel Borkmann fmode_t mode = f.file->f_mode; 36187df15deSDaniel Borkmann 36287df15deSDaniel Borkmann /* Our file permissions may have been overridden by global 36387df15deSDaniel Borkmann * map permissions facing syscall side. 36487df15deSDaniel Borkmann */ 36587df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) 36687df15deSDaniel Borkmann mode &= ~FMODE_CAN_WRITE; 36787df15deSDaniel Borkmann return mode; 36887df15deSDaniel Borkmann } 36987df15deSDaniel Borkmann 370f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 371f99bf205SDaniel Borkmann static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 372f99bf205SDaniel Borkmann { 373f99bf205SDaniel Borkmann const struct bpf_map *map = filp->private_data; 37421116b70SDaniel Borkmann const struct bpf_array *array; 37521116b70SDaniel Borkmann u32 owner_prog_type = 0; 3769780c0abSDaniel Borkmann u32 owner_jited = 0; 37721116b70SDaniel Borkmann 37821116b70SDaniel Borkmann if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 37921116b70SDaniel Borkmann array = container_of(map, struct bpf_array, map); 38021116b70SDaniel Borkmann owner_prog_type = array->owner_prog_type; 3819780c0abSDaniel Borkmann owner_jited = array->owner_jited; 38221116b70SDaniel Borkmann } 383f99bf205SDaniel Borkmann 384f99bf205SDaniel Borkmann seq_printf(m, 385f99bf205SDaniel Borkmann "map_type:\t%u\n" 386f99bf205SDaniel Borkmann "key_size:\t%u\n" 387f99bf205SDaniel Borkmann "value_size:\t%u\n" 388322cea2fSDaniel Borkmann "max_entries:\t%u\n" 38921116b70SDaniel Borkmann "map_flags:\t%#x\n" 3904316b409SDaniel Borkmann "memlock:\t%llu\n" 39187df15deSDaniel Borkmann "map_id:\t%u\n" 39287df15deSDaniel Borkmann "frozen:\t%u\n", 393f99bf205SDaniel Borkmann map->map_type, 394f99bf205SDaniel Borkmann map->key_size, 395f99bf205SDaniel Borkmann map->value_size, 396322cea2fSDaniel Borkmann map->max_entries, 39721116b70SDaniel Borkmann map->map_flags, 3984316b409SDaniel Borkmann map->pages * 1ULL << PAGE_SHIFT, 39987df15deSDaniel Borkmann map->id, 40087df15deSDaniel Borkmann READ_ONCE(map->frozen)); 40121116b70SDaniel Borkmann 4029780c0abSDaniel Borkmann if (owner_prog_type) { 40321116b70SDaniel Borkmann seq_printf(m, "owner_prog_type:\t%u\n", 40421116b70SDaniel Borkmann owner_prog_type); 4059780c0abSDaniel Borkmann seq_printf(m, "owner_jited:\t%u\n", 4069780c0abSDaniel Borkmann owner_jited); 4079780c0abSDaniel Borkmann } 408f99bf205SDaniel Borkmann } 409f99bf205SDaniel Borkmann #endif 410f99bf205SDaniel Borkmann 4116e71b04aSChenbo Feng static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, 4126e71b04aSChenbo Feng loff_t *ppos) 4136e71b04aSChenbo Feng { 4146e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 4156e71b04aSChenbo Feng * f_mode with FMODE_CAN_READ. 4166e71b04aSChenbo Feng */ 4176e71b04aSChenbo Feng return -EINVAL; 4186e71b04aSChenbo Feng } 4196e71b04aSChenbo Feng 4206e71b04aSChenbo Feng static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, 4216e71b04aSChenbo Feng size_t siz, loff_t *ppos) 4226e71b04aSChenbo Feng { 4236e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 4246e71b04aSChenbo Feng * f_mode with FMODE_CAN_WRITE. 4256e71b04aSChenbo Feng */ 4266e71b04aSChenbo Feng return -EINVAL; 4276e71b04aSChenbo Feng } 4286e71b04aSChenbo Feng 429f66e448cSChenbo Feng const struct file_operations bpf_map_fops = { 430f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 431f99bf205SDaniel Borkmann .show_fdinfo = bpf_map_show_fdinfo, 432f99bf205SDaniel Borkmann #endif 43399c55f7dSAlexei Starovoitov .release = bpf_map_release, 4346e71b04aSChenbo Feng .read = bpf_dummy_read, 4356e71b04aSChenbo Feng .write = bpf_dummy_write, 43699c55f7dSAlexei Starovoitov }; 43799c55f7dSAlexei Starovoitov 4386e71b04aSChenbo Feng int bpf_map_new_fd(struct bpf_map *map, int flags) 439aa79781bSDaniel Borkmann { 440afdb09c7SChenbo Feng int ret; 441afdb09c7SChenbo Feng 442afdb09c7SChenbo Feng ret = security_bpf_map(map, OPEN_FMODE(flags)); 443afdb09c7SChenbo Feng if (ret < 0) 444afdb09c7SChenbo Feng return ret; 445afdb09c7SChenbo Feng 446aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 4476e71b04aSChenbo Feng flags | O_CLOEXEC); 4486e71b04aSChenbo Feng } 4496e71b04aSChenbo Feng 4506e71b04aSChenbo Feng int bpf_get_file_flag(int flags) 4516e71b04aSChenbo Feng { 4526e71b04aSChenbo Feng if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) 4536e71b04aSChenbo Feng return -EINVAL; 4546e71b04aSChenbo Feng if (flags & BPF_F_RDONLY) 4556e71b04aSChenbo Feng return O_RDONLY; 4566e71b04aSChenbo Feng if (flags & BPF_F_WRONLY) 4576e71b04aSChenbo Feng return O_WRONLY; 4586e71b04aSChenbo Feng return O_RDWR; 459aa79781bSDaniel Borkmann } 460aa79781bSDaniel Borkmann 46199c55f7dSAlexei Starovoitov /* helper macro to check that unused fields 'union bpf_attr' are zero */ 46299c55f7dSAlexei Starovoitov #define CHECK_ATTR(CMD) \ 46399c55f7dSAlexei Starovoitov memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 46499c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD), 0, \ 46599c55f7dSAlexei Starovoitov sizeof(*attr) - \ 46699c55f7dSAlexei Starovoitov offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 46799c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD)) != NULL 46899c55f7dSAlexei Starovoitov 469cb4d2b3fSMartin KaFai Lau /* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes. 470cb4d2b3fSMartin KaFai Lau * Return 0 on success and < 0 on error. 471cb4d2b3fSMartin KaFai Lau */ 472cb4d2b3fSMartin KaFai Lau static int bpf_obj_name_cpy(char *dst, const char *src) 473cb4d2b3fSMartin KaFai Lau { 474cb4d2b3fSMartin KaFai Lau const char *end = src + BPF_OBJ_NAME_LEN; 475cb4d2b3fSMartin KaFai Lau 476473d9734SMartin KaFai Lau memset(dst, 0, BPF_OBJ_NAME_LEN); 4773e0ddc4fSDaniel Borkmann /* Copy all isalnum(), '_' and '.' chars. */ 478cb4d2b3fSMartin KaFai Lau while (src < end && *src) { 4793e0ddc4fSDaniel Borkmann if (!isalnum(*src) && 4803e0ddc4fSDaniel Borkmann *src != '_' && *src != '.') 481cb4d2b3fSMartin KaFai Lau return -EINVAL; 482cb4d2b3fSMartin KaFai Lau *dst++ = *src++; 483cb4d2b3fSMartin KaFai Lau } 484cb4d2b3fSMartin KaFai Lau 485cb4d2b3fSMartin KaFai Lau /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */ 486cb4d2b3fSMartin KaFai Lau if (src == end) 487cb4d2b3fSMartin KaFai Lau return -EINVAL; 488cb4d2b3fSMartin KaFai Lau 489cb4d2b3fSMartin KaFai Lau return 0; 490cb4d2b3fSMartin KaFai Lau } 491cb4d2b3fSMartin KaFai Lau 492e8d2bec0SDaniel Borkmann int map_check_no_btf(const struct bpf_map *map, 4931b2b234bSRoman Gushchin const struct btf *btf, 494e8d2bec0SDaniel Borkmann const struct btf_type *key_type, 495e8d2bec0SDaniel Borkmann const struct btf_type *value_type) 496e8d2bec0SDaniel Borkmann { 497e8d2bec0SDaniel Borkmann return -ENOTSUPP; 498e8d2bec0SDaniel Borkmann } 499e8d2bec0SDaniel Borkmann 500d83525caSAlexei Starovoitov static int map_check_btf(struct bpf_map *map, const struct btf *btf, 501e8d2bec0SDaniel Borkmann u32 btf_key_id, u32 btf_value_id) 502e8d2bec0SDaniel Borkmann { 503e8d2bec0SDaniel Borkmann const struct btf_type *key_type, *value_type; 504e8d2bec0SDaniel Borkmann u32 key_size, value_size; 505e8d2bec0SDaniel Borkmann int ret = 0; 506e8d2bec0SDaniel Borkmann 5072824ecb7SDaniel Borkmann /* Some maps allow key to be unspecified. */ 5082824ecb7SDaniel Borkmann if (btf_key_id) { 509e8d2bec0SDaniel Borkmann key_type = btf_type_id_size(btf, &btf_key_id, &key_size); 510e8d2bec0SDaniel Borkmann if (!key_type || key_size != map->key_size) 511e8d2bec0SDaniel Borkmann return -EINVAL; 5122824ecb7SDaniel Borkmann } else { 5132824ecb7SDaniel Borkmann key_type = btf_type_by_id(btf, 0); 5142824ecb7SDaniel Borkmann if (!map->ops->map_check_btf) 5152824ecb7SDaniel Borkmann return -EINVAL; 5162824ecb7SDaniel Borkmann } 517e8d2bec0SDaniel Borkmann 518e8d2bec0SDaniel Borkmann value_type = btf_type_id_size(btf, &btf_value_id, &value_size); 519e8d2bec0SDaniel Borkmann if (!value_type || value_size != map->value_size) 520e8d2bec0SDaniel Borkmann return -EINVAL; 521e8d2bec0SDaniel Borkmann 522d83525caSAlexei Starovoitov map->spin_lock_off = btf_find_spin_lock(btf, value_type); 523d83525caSAlexei Starovoitov 524d83525caSAlexei Starovoitov if (map_value_has_spin_lock(map)) { 525591fe988SDaniel Borkmann if (map->map_flags & BPF_F_RDONLY_PROG) 526591fe988SDaniel Borkmann return -EACCES; 527d83525caSAlexei Starovoitov if (map->map_type != BPF_MAP_TYPE_HASH && 528e16d2f1aSAlexei Starovoitov map->map_type != BPF_MAP_TYPE_ARRAY && 529e16d2f1aSAlexei Starovoitov map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE) 530d83525caSAlexei Starovoitov return -ENOTSUPP; 531d83525caSAlexei Starovoitov if (map->spin_lock_off + sizeof(struct bpf_spin_lock) > 532d83525caSAlexei Starovoitov map->value_size) { 533d83525caSAlexei Starovoitov WARN_ONCE(1, 534d83525caSAlexei Starovoitov "verifier bug spin_lock_off %d value_size %d\n", 535d83525caSAlexei Starovoitov map->spin_lock_off, map->value_size); 536d83525caSAlexei Starovoitov return -EFAULT; 537d83525caSAlexei Starovoitov } 538d83525caSAlexei Starovoitov } 539d83525caSAlexei Starovoitov 540e8d2bec0SDaniel Borkmann if (map->ops->map_check_btf) 5411b2b234bSRoman Gushchin ret = map->ops->map_check_btf(map, btf, key_type, value_type); 542e8d2bec0SDaniel Borkmann 543e8d2bec0SDaniel Borkmann return ret; 544e8d2bec0SDaniel Borkmann } 545e8d2bec0SDaniel Borkmann 5469b2cf328SMartin KaFai Lau #define BPF_MAP_CREATE_LAST_FIELD btf_value_type_id 54799c55f7dSAlexei Starovoitov /* called via syscall */ 54899c55f7dSAlexei Starovoitov static int map_create(union bpf_attr *attr) 54999c55f7dSAlexei Starovoitov { 55096eabe7aSMartin KaFai Lau int numa_node = bpf_map_attr_numa_node(attr); 55199c55f7dSAlexei Starovoitov struct bpf_map *map; 5526e71b04aSChenbo Feng int f_flags; 55399c55f7dSAlexei Starovoitov int err; 55499c55f7dSAlexei Starovoitov 55599c55f7dSAlexei Starovoitov err = CHECK_ATTR(BPF_MAP_CREATE); 55699c55f7dSAlexei Starovoitov if (err) 55799c55f7dSAlexei Starovoitov return -EINVAL; 55899c55f7dSAlexei Starovoitov 5596e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->map_flags); 5606e71b04aSChenbo Feng if (f_flags < 0) 5616e71b04aSChenbo Feng return f_flags; 5626e71b04aSChenbo Feng 56396eabe7aSMartin KaFai Lau if (numa_node != NUMA_NO_NODE && 56496e5ae4eSEric Dumazet ((unsigned int)numa_node >= nr_node_ids || 56596e5ae4eSEric Dumazet !node_online(numa_node))) 56696eabe7aSMartin KaFai Lau return -EINVAL; 56796eabe7aSMartin KaFai Lau 56899c55f7dSAlexei Starovoitov /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 56999c55f7dSAlexei Starovoitov map = find_and_alloc_map(attr); 57099c55f7dSAlexei Starovoitov if (IS_ERR(map)) 57199c55f7dSAlexei Starovoitov return PTR_ERR(map); 57299c55f7dSAlexei Starovoitov 573ad5b177bSMartin KaFai Lau err = bpf_obj_name_cpy(map->name, attr->map_name); 574ad5b177bSMartin KaFai Lau if (err) 575ad5b177bSMartin KaFai Lau goto free_map_nouncharge; 576ad5b177bSMartin KaFai Lau 57799c55f7dSAlexei Starovoitov atomic_set(&map->refcnt, 1); 578c9da161cSDaniel Borkmann atomic_set(&map->usercnt, 1); 57999c55f7dSAlexei Starovoitov 580e8d2bec0SDaniel Borkmann if (attr->btf_key_type_id || attr->btf_value_type_id) { 581a26ca7c9SMartin KaFai Lau struct btf *btf; 582a26ca7c9SMartin KaFai Lau 5832824ecb7SDaniel Borkmann if (!attr->btf_value_type_id) { 584a26ca7c9SMartin KaFai Lau err = -EINVAL; 585a26ca7c9SMartin KaFai Lau goto free_map_nouncharge; 586a26ca7c9SMartin KaFai Lau } 587a26ca7c9SMartin KaFai Lau 588a26ca7c9SMartin KaFai Lau btf = btf_get_by_fd(attr->btf_fd); 589a26ca7c9SMartin KaFai Lau if (IS_ERR(btf)) { 590a26ca7c9SMartin KaFai Lau err = PTR_ERR(btf); 591a26ca7c9SMartin KaFai Lau goto free_map_nouncharge; 592a26ca7c9SMartin KaFai Lau } 593a26ca7c9SMartin KaFai Lau 594e8d2bec0SDaniel Borkmann err = map_check_btf(map, btf, attr->btf_key_type_id, 5959b2cf328SMartin KaFai Lau attr->btf_value_type_id); 596a26ca7c9SMartin KaFai Lau if (err) { 597a26ca7c9SMartin KaFai Lau btf_put(btf); 598a26ca7c9SMartin KaFai Lau goto free_map_nouncharge; 599a26ca7c9SMartin KaFai Lau } 600a26ca7c9SMartin KaFai Lau 601a26ca7c9SMartin KaFai Lau map->btf = btf; 6029b2cf328SMartin KaFai Lau map->btf_key_type_id = attr->btf_key_type_id; 6039b2cf328SMartin KaFai Lau map->btf_value_type_id = attr->btf_value_type_id; 604d83525caSAlexei Starovoitov } else { 605d83525caSAlexei Starovoitov map->spin_lock_off = -EINVAL; 606a26ca7c9SMartin KaFai Lau } 607a26ca7c9SMartin KaFai Lau 608afdb09c7SChenbo Feng err = security_bpf_map_alloc(map); 609aaac3ba9SAlexei Starovoitov if (err) 61020b2b24fSDaniel Borkmann goto free_map_nouncharge; 611aaac3ba9SAlexei Starovoitov 6120a4c58f5SRoman Gushchin err = bpf_map_init_memlock(map); 613afdb09c7SChenbo Feng if (err) 614afdb09c7SChenbo Feng goto free_map_sec; 615afdb09c7SChenbo Feng 616f3f1c054SMartin KaFai Lau err = bpf_map_alloc_id(map); 617f3f1c054SMartin KaFai Lau if (err) 618f3f1c054SMartin KaFai Lau goto free_map; 619f3f1c054SMartin KaFai Lau 6206e71b04aSChenbo Feng err = bpf_map_new_fd(map, f_flags); 621bd5f5f4eSMartin KaFai Lau if (err < 0) { 622bd5f5f4eSMartin KaFai Lau /* failed to allocate fd. 623352d20d6SPeng Sun * bpf_map_put_with_uref() is needed because the above 624bd5f5f4eSMartin KaFai Lau * bpf_map_alloc_id() has published the map 625bd5f5f4eSMartin KaFai Lau * to the userspace and the userspace may 626bd5f5f4eSMartin KaFai Lau * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 627bd5f5f4eSMartin KaFai Lau */ 628352d20d6SPeng Sun bpf_map_put_with_uref(map); 629bd5f5f4eSMartin KaFai Lau return err; 630bd5f5f4eSMartin KaFai Lau } 63199c55f7dSAlexei Starovoitov 63299c55f7dSAlexei Starovoitov return err; 63399c55f7dSAlexei Starovoitov 63499c55f7dSAlexei Starovoitov free_map: 6350a4c58f5SRoman Gushchin bpf_map_release_memlock(map); 636afdb09c7SChenbo Feng free_map_sec: 637afdb09c7SChenbo Feng security_bpf_map_free(map); 63820b2b24fSDaniel Borkmann free_map_nouncharge: 639a26ca7c9SMartin KaFai Lau btf_put(map->btf); 64099c55f7dSAlexei Starovoitov map->ops->map_free(map); 64199c55f7dSAlexei Starovoitov return err; 64299c55f7dSAlexei Starovoitov } 64399c55f7dSAlexei Starovoitov 644db20fd2bSAlexei Starovoitov /* if error is returned, fd is released. 645db20fd2bSAlexei Starovoitov * On success caller should complete fd access with matching fdput() 646db20fd2bSAlexei Starovoitov */ 647c2101297SDaniel Borkmann struct bpf_map *__bpf_map_get(struct fd f) 648db20fd2bSAlexei Starovoitov { 649db20fd2bSAlexei Starovoitov if (!f.file) 650db20fd2bSAlexei Starovoitov return ERR_PTR(-EBADF); 651db20fd2bSAlexei Starovoitov if (f.file->f_op != &bpf_map_fops) { 652db20fd2bSAlexei Starovoitov fdput(f); 653db20fd2bSAlexei Starovoitov return ERR_PTR(-EINVAL); 654db20fd2bSAlexei Starovoitov } 655db20fd2bSAlexei Starovoitov 656c2101297SDaniel Borkmann return f.file->private_data; 657c2101297SDaniel Borkmann } 658c2101297SDaniel Borkmann 65992117d84SAlexei Starovoitov /* prog's and map's refcnt limit */ 66092117d84SAlexei Starovoitov #define BPF_MAX_REFCNT 32768 66192117d84SAlexei Starovoitov 66292117d84SAlexei Starovoitov struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref) 663c9da161cSDaniel Borkmann { 66492117d84SAlexei Starovoitov if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) { 66592117d84SAlexei Starovoitov atomic_dec(&map->refcnt); 66692117d84SAlexei Starovoitov return ERR_PTR(-EBUSY); 66792117d84SAlexei Starovoitov } 668c9da161cSDaniel Borkmann if (uref) 669c9da161cSDaniel Borkmann atomic_inc(&map->usercnt); 67092117d84SAlexei Starovoitov return map; 671c9da161cSDaniel Borkmann } 672630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_inc); 673c9da161cSDaniel Borkmann 674c9da161cSDaniel Borkmann struct bpf_map *bpf_map_get_with_uref(u32 ufd) 675c2101297SDaniel Borkmann { 676c2101297SDaniel Borkmann struct fd f = fdget(ufd); 677c2101297SDaniel Borkmann struct bpf_map *map; 678c2101297SDaniel Borkmann 679c2101297SDaniel Borkmann map = __bpf_map_get(f); 680c2101297SDaniel Borkmann if (IS_ERR(map)) 681c2101297SDaniel Borkmann return map; 682c2101297SDaniel Borkmann 68392117d84SAlexei Starovoitov map = bpf_map_inc(map, true); 684c2101297SDaniel Borkmann fdput(f); 685db20fd2bSAlexei Starovoitov 686db20fd2bSAlexei Starovoitov return map; 687db20fd2bSAlexei Starovoitov } 688db20fd2bSAlexei Starovoitov 689bd5f5f4eSMartin KaFai Lau /* map_idr_lock should have been held */ 690bd5f5f4eSMartin KaFai Lau static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map, 691bd5f5f4eSMartin KaFai Lau bool uref) 692bd5f5f4eSMartin KaFai Lau { 693bd5f5f4eSMartin KaFai Lau int refold; 694bd5f5f4eSMartin KaFai Lau 695bfc18e38SMark Rutland refold = atomic_fetch_add_unless(&map->refcnt, 1, 0); 696bd5f5f4eSMartin KaFai Lau 697bd5f5f4eSMartin KaFai Lau if (refold >= BPF_MAX_REFCNT) { 698bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, false); 699bd5f5f4eSMartin KaFai Lau return ERR_PTR(-EBUSY); 700bd5f5f4eSMartin KaFai Lau } 701bd5f5f4eSMartin KaFai Lau 702bd5f5f4eSMartin KaFai Lau if (!refold) 703bd5f5f4eSMartin KaFai Lau return ERR_PTR(-ENOENT); 704bd5f5f4eSMartin KaFai Lau 705bd5f5f4eSMartin KaFai Lau if (uref) 706bd5f5f4eSMartin KaFai Lau atomic_inc(&map->usercnt); 707bd5f5f4eSMartin KaFai Lau 708bd5f5f4eSMartin KaFai Lau return map; 709bd5f5f4eSMartin KaFai Lau } 710bd5f5f4eSMartin KaFai Lau 711b8cdc051SAlexei Starovoitov int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 712b8cdc051SAlexei Starovoitov { 713b8cdc051SAlexei Starovoitov return -ENOTSUPP; 714b8cdc051SAlexei Starovoitov } 715b8cdc051SAlexei Starovoitov 716c9d29f46SMauricio Vasquez B static void *__bpf_copy_key(void __user *ukey, u64 key_size) 717c9d29f46SMauricio Vasquez B { 718c9d29f46SMauricio Vasquez B if (key_size) 719c9d29f46SMauricio Vasquez B return memdup_user(ukey, key_size); 720c9d29f46SMauricio Vasquez B 721c9d29f46SMauricio Vasquez B if (ukey) 722c9d29f46SMauricio Vasquez B return ERR_PTR(-EINVAL); 723c9d29f46SMauricio Vasquez B 724c9d29f46SMauricio Vasquez B return NULL; 725c9d29f46SMauricio Vasquez B } 726c9d29f46SMauricio Vasquez B 727db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 72896049f3aSAlexei Starovoitov #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags 729db20fd2bSAlexei Starovoitov 730db20fd2bSAlexei Starovoitov static int map_lookup_elem(union bpf_attr *attr) 731db20fd2bSAlexei Starovoitov { 732535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 733535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 734db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 735db20fd2bSAlexei Starovoitov struct bpf_map *map; 7368ebe667cSAlexei Starovoitov void *key, *value, *ptr; 73715a07b33SAlexei Starovoitov u32 value_size; 738592867bfSDaniel Borkmann struct fd f; 739db20fd2bSAlexei Starovoitov int err; 740db20fd2bSAlexei Starovoitov 741db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 742db20fd2bSAlexei Starovoitov return -EINVAL; 743db20fd2bSAlexei Starovoitov 74496049f3aSAlexei Starovoitov if (attr->flags & ~BPF_F_LOCK) 74596049f3aSAlexei Starovoitov return -EINVAL; 74696049f3aSAlexei Starovoitov 747592867bfSDaniel Borkmann f = fdget(ufd); 748c2101297SDaniel Borkmann map = __bpf_map_get(f); 749db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 750db20fd2bSAlexei Starovoitov return PTR_ERR(map); 75187df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 7526e71b04aSChenbo Feng err = -EPERM; 7536e71b04aSChenbo Feng goto err_put; 7546e71b04aSChenbo Feng } 7556e71b04aSChenbo Feng 75696049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 75796049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 75896049f3aSAlexei Starovoitov err = -EINVAL; 75996049f3aSAlexei Starovoitov goto err_put; 76096049f3aSAlexei Starovoitov } 76196049f3aSAlexei Starovoitov 762c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 763e4448ed8SAl Viro if (IS_ERR(key)) { 764e4448ed8SAl Viro err = PTR_ERR(key); 765db20fd2bSAlexei Starovoitov goto err_put; 766e4448ed8SAl Viro } 767db20fd2bSAlexei Starovoitov 76815a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 7698f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 770b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 771b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 77215a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 77314dc6f04SMartin KaFai Lau else if (IS_FD_MAP(map)) 77414dc6f04SMartin KaFai Lau value_size = sizeof(u32); 77515a07b33SAlexei Starovoitov else 77615a07b33SAlexei Starovoitov value_size = map->value_size; 77715a07b33SAlexei Starovoitov 7788ebe667cSAlexei Starovoitov err = -ENOMEM; 77915a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 780db20fd2bSAlexei Starovoitov if (!value) 7818ebe667cSAlexei Starovoitov goto free_key; 7828ebe667cSAlexei Starovoitov 783a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 784a3884572SJakub Kicinski err = bpf_map_offload_lookup_elem(map, key, value); 7857c4cd051SMartin KaFai Lau goto done; 7867c4cd051SMartin KaFai Lau } 7877c4cd051SMartin KaFai Lau 7887c4cd051SMartin KaFai Lau preempt_disable(); 7897c4cd051SMartin KaFai Lau this_cpu_inc(bpf_prog_active); 7907c4cd051SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 7918f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 79215a07b33SAlexei Starovoitov err = bpf_percpu_hash_copy(map, key, value); 79315a07b33SAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 79415a07b33SAlexei Starovoitov err = bpf_percpu_array_copy(map, key, value); 795b741f163SRoman Gushchin } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 796b741f163SRoman Gushchin err = bpf_percpu_cgroup_storage_copy(map, key, value); 797557c0c6eSAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 798557c0c6eSAlexei Starovoitov err = bpf_stackmap_copy(map, key, value); 79914dc6f04SMartin KaFai Lau } else if (IS_FD_ARRAY(map)) { 80014dc6f04SMartin KaFai Lau err = bpf_fd_array_map_lookup_elem(map, key, value); 80114dc6f04SMartin KaFai Lau } else if (IS_FD_HASH(map)) { 80214dc6f04SMartin KaFai Lau err = bpf_fd_htab_map_lookup_elem(map, key, value); 8035dc4c4b7SMartin KaFai Lau } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 8045dc4c4b7SMartin KaFai Lau err = bpf_fd_reuseport_array_lookup_elem(map, key, value); 805f1a2e44aSMauricio Vasquez B } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 806f1a2e44aSMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 807f1a2e44aSMauricio Vasquez B err = map->ops->map_peek_elem(map, value); 80815a07b33SAlexei Starovoitov } else { 8098ebe667cSAlexei Starovoitov rcu_read_lock(); 8108ebe667cSAlexei Starovoitov ptr = map->ops->map_lookup_elem(map, key); 811509db283SPrashant Bhole if (IS_ERR(ptr)) { 812509db283SPrashant Bhole err = PTR_ERR(ptr); 813509db283SPrashant Bhole } else if (!ptr) { 814509db283SPrashant Bhole err = -ENOENT; 815509db283SPrashant Bhole } else { 816509db283SPrashant Bhole err = 0; 81796049f3aSAlexei Starovoitov if (attr->flags & BPF_F_LOCK) 81896049f3aSAlexei Starovoitov /* lock 'ptr' and copy everything but lock */ 81996049f3aSAlexei Starovoitov copy_map_value_locked(map, value, ptr, true); 82096049f3aSAlexei Starovoitov else 821d83525caSAlexei Starovoitov copy_map_value(map, value, ptr); 82296049f3aSAlexei Starovoitov /* mask lock, since value wasn't zero inited */ 82396049f3aSAlexei Starovoitov check_and_init_map_lock(map, value); 824509db283SPrashant Bhole } 8258ebe667cSAlexei Starovoitov rcu_read_unlock(); 82615a07b33SAlexei Starovoitov } 8277c4cd051SMartin KaFai Lau this_cpu_dec(bpf_prog_active); 8287c4cd051SMartin KaFai Lau preempt_enable(); 8298ebe667cSAlexei Starovoitov 8307c4cd051SMartin KaFai Lau done: 83115a07b33SAlexei Starovoitov if (err) 8328ebe667cSAlexei Starovoitov goto free_value; 833db20fd2bSAlexei Starovoitov 834db20fd2bSAlexei Starovoitov err = -EFAULT; 83515a07b33SAlexei Starovoitov if (copy_to_user(uvalue, value, value_size) != 0) 8368ebe667cSAlexei Starovoitov goto free_value; 837db20fd2bSAlexei Starovoitov 838db20fd2bSAlexei Starovoitov err = 0; 839db20fd2bSAlexei Starovoitov 8408ebe667cSAlexei Starovoitov free_value: 8418ebe667cSAlexei Starovoitov kfree(value); 842db20fd2bSAlexei Starovoitov free_key: 843db20fd2bSAlexei Starovoitov kfree(key); 844db20fd2bSAlexei Starovoitov err_put: 845db20fd2bSAlexei Starovoitov fdput(f); 846db20fd2bSAlexei Starovoitov return err; 847db20fd2bSAlexei Starovoitov } 848db20fd2bSAlexei Starovoitov 8491ae80cf3SDaniel Colascione static void maybe_wait_bpf_programs(struct bpf_map *map) 8501ae80cf3SDaniel Colascione { 8511ae80cf3SDaniel Colascione /* Wait for any running BPF programs to complete so that 8521ae80cf3SDaniel Colascione * userspace, when we return to it, knows that all programs 8531ae80cf3SDaniel Colascione * that could be running use the new map value. 8541ae80cf3SDaniel Colascione */ 8551ae80cf3SDaniel Colascione if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS || 8561ae80cf3SDaniel Colascione map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 8571ae80cf3SDaniel Colascione synchronize_rcu(); 8581ae80cf3SDaniel Colascione } 8591ae80cf3SDaniel Colascione 8603274f520SAlexei Starovoitov #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 861db20fd2bSAlexei Starovoitov 862db20fd2bSAlexei Starovoitov static int map_update_elem(union bpf_attr *attr) 863db20fd2bSAlexei Starovoitov { 864535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 865535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 866db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 867db20fd2bSAlexei Starovoitov struct bpf_map *map; 868db20fd2bSAlexei Starovoitov void *key, *value; 86915a07b33SAlexei Starovoitov u32 value_size; 870592867bfSDaniel Borkmann struct fd f; 871db20fd2bSAlexei Starovoitov int err; 872db20fd2bSAlexei Starovoitov 873db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 874db20fd2bSAlexei Starovoitov return -EINVAL; 875db20fd2bSAlexei Starovoitov 876592867bfSDaniel Borkmann f = fdget(ufd); 877c2101297SDaniel Borkmann map = __bpf_map_get(f); 878db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 879db20fd2bSAlexei Starovoitov return PTR_ERR(map); 88087df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 8816e71b04aSChenbo Feng err = -EPERM; 8826e71b04aSChenbo Feng goto err_put; 8836e71b04aSChenbo Feng } 8846e71b04aSChenbo Feng 88596049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 88696049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 88796049f3aSAlexei Starovoitov err = -EINVAL; 88896049f3aSAlexei Starovoitov goto err_put; 88996049f3aSAlexei Starovoitov } 89096049f3aSAlexei Starovoitov 891c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 892e4448ed8SAl Viro if (IS_ERR(key)) { 893e4448ed8SAl Viro err = PTR_ERR(key); 894db20fd2bSAlexei Starovoitov goto err_put; 895e4448ed8SAl Viro } 896db20fd2bSAlexei Starovoitov 89715a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 8988f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 899b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 900b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 90115a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 90215a07b33SAlexei Starovoitov else 90315a07b33SAlexei Starovoitov value_size = map->value_size; 90415a07b33SAlexei Starovoitov 905db20fd2bSAlexei Starovoitov err = -ENOMEM; 90615a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 907db20fd2bSAlexei Starovoitov if (!value) 908db20fd2bSAlexei Starovoitov goto free_key; 909db20fd2bSAlexei Starovoitov 910db20fd2bSAlexei Starovoitov err = -EFAULT; 91115a07b33SAlexei Starovoitov if (copy_from_user(value, uvalue, value_size) != 0) 912db20fd2bSAlexei Starovoitov goto free_value; 913db20fd2bSAlexei Starovoitov 9146710e112SJesper Dangaard Brouer /* Need to create a kthread, thus must support schedule */ 915a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 916a3884572SJakub Kicinski err = bpf_map_offload_update_elem(map, key, value, attr->flags); 917a3884572SJakub Kicinski goto out; 91899ba2b5aSJohn Fastabend } else if (map->map_type == BPF_MAP_TYPE_CPUMAP || 91999ba2b5aSJohn Fastabend map->map_type == BPF_MAP_TYPE_SOCKHASH || 92099ba2b5aSJohn Fastabend map->map_type == BPF_MAP_TYPE_SOCKMAP) { 9216710e112SJesper Dangaard Brouer err = map->ops->map_update_elem(map, key, value, attr->flags); 9226710e112SJesper Dangaard Brouer goto out; 9236710e112SJesper Dangaard Brouer } 9246710e112SJesper Dangaard Brouer 925b121d1e7SAlexei Starovoitov /* must increment bpf_prog_active to avoid kprobe+bpf triggering from 926b121d1e7SAlexei Starovoitov * inside bpf map update or delete otherwise deadlocks are possible 927b121d1e7SAlexei Starovoitov */ 928b121d1e7SAlexei Starovoitov preempt_disable(); 929b121d1e7SAlexei Starovoitov __this_cpu_inc(bpf_prog_active); 9308f844938SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 9318f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 93215a07b33SAlexei Starovoitov err = bpf_percpu_hash_update(map, key, value, attr->flags); 93315a07b33SAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 93415a07b33SAlexei Starovoitov err = bpf_percpu_array_update(map, key, value, attr->flags); 935b741f163SRoman Gushchin } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 936b741f163SRoman Gushchin err = bpf_percpu_cgroup_storage_update(map, key, value, 937b741f163SRoman Gushchin attr->flags); 9389c147b56SMickaël Salaün } else if (IS_FD_ARRAY(map)) { 939d056a788SDaniel Borkmann rcu_read_lock(); 940d056a788SDaniel Borkmann err = bpf_fd_array_map_update_elem(map, f.file, key, value, 941d056a788SDaniel Borkmann attr->flags); 942d056a788SDaniel Borkmann rcu_read_unlock(); 943bcc6b1b7SMartin KaFai Lau } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 944bcc6b1b7SMartin KaFai Lau rcu_read_lock(); 945bcc6b1b7SMartin KaFai Lau err = bpf_fd_htab_map_update_elem(map, f.file, key, value, 946bcc6b1b7SMartin KaFai Lau attr->flags); 947bcc6b1b7SMartin KaFai Lau rcu_read_unlock(); 9485dc4c4b7SMartin KaFai Lau } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 9495dc4c4b7SMartin KaFai Lau /* rcu_read_lock() is not needed */ 9505dc4c4b7SMartin KaFai Lau err = bpf_fd_reuseport_array_update_elem(map, key, value, 9515dc4c4b7SMartin KaFai Lau attr->flags); 952f1a2e44aSMauricio Vasquez B } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 953f1a2e44aSMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 954f1a2e44aSMauricio Vasquez B err = map->ops->map_push_elem(map, value, attr->flags); 95515a07b33SAlexei Starovoitov } else { 956db20fd2bSAlexei Starovoitov rcu_read_lock(); 9573274f520SAlexei Starovoitov err = map->ops->map_update_elem(map, key, value, attr->flags); 958db20fd2bSAlexei Starovoitov rcu_read_unlock(); 95915a07b33SAlexei Starovoitov } 960b121d1e7SAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 961b121d1e7SAlexei Starovoitov preempt_enable(); 9621ae80cf3SDaniel Colascione maybe_wait_bpf_programs(map); 9636710e112SJesper Dangaard Brouer out: 964db20fd2bSAlexei Starovoitov free_value: 965db20fd2bSAlexei Starovoitov kfree(value); 966db20fd2bSAlexei Starovoitov free_key: 967db20fd2bSAlexei Starovoitov kfree(key); 968db20fd2bSAlexei Starovoitov err_put: 969db20fd2bSAlexei Starovoitov fdput(f); 970db20fd2bSAlexei Starovoitov return err; 971db20fd2bSAlexei Starovoitov } 972db20fd2bSAlexei Starovoitov 973db20fd2bSAlexei Starovoitov #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 974db20fd2bSAlexei Starovoitov 975db20fd2bSAlexei Starovoitov static int map_delete_elem(union bpf_attr *attr) 976db20fd2bSAlexei Starovoitov { 977535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 978db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 979db20fd2bSAlexei Starovoitov struct bpf_map *map; 980592867bfSDaniel Borkmann struct fd f; 981db20fd2bSAlexei Starovoitov void *key; 982db20fd2bSAlexei Starovoitov int err; 983db20fd2bSAlexei Starovoitov 984db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 985db20fd2bSAlexei Starovoitov return -EINVAL; 986db20fd2bSAlexei Starovoitov 987592867bfSDaniel Borkmann f = fdget(ufd); 988c2101297SDaniel Borkmann map = __bpf_map_get(f); 989db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 990db20fd2bSAlexei Starovoitov return PTR_ERR(map); 99187df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 9926e71b04aSChenbo Feng err = -EPERM; 9936e71b04aSChenbo Feng goto err_put; 9946e71b04aSChenbo Feng } 9956e71b04aSChenbo Feng 996c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 997e4448ed8SAl Viro if (IS_ERR(key)) { 998e4448ed8SAl Viro err = PTR_ERR(key); 999db20fd2bSAlexei Starovoitov goto err_put; 1000e4448ed8SAl Viro } 1001db20fd2bSAlexei Starovoitov 1002a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1003a3884572SJakub Kicinski err = bpf_map_offload_delete_elem(map, key); 1004a3884572SJakub Kicinski goto out; 1005a3884572SJakub Kicinski } 1006a3884572SJakub Kicinski 1007b121d1e7SAlexei Starovoitov preempt_disable(); 1008b121d1e7SAlexei Starovoitov __this_cpu_inc(bpf_prog_active); 1009db20fd2bSAlexei Starovoitov rcu_read_lock(); 1010db20fd2bSAlexei Starovoitov err = map->ops->map_delete_elem(map, key); 1011db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1012b121d1e7SAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 1013b121d1e7SAlexei Starovoitov preempt_enable(); 10141ae80cf3SDaniel Colascione maybe_wait_bpf_programs(map); 1015a3884572SJakub Kicinski out: 1016db20fd2bSAlexei Starovoitov kfree(key); 1017db20fd2bSAlexei Starovoitov err_put: 1018db20fd2bSAlexei Starovoitov fdput(f); 1019db20fd2bSAlexei Starovoitov return err; 1020db20fd2bSAlexei Starovoitov } 1021db20fd2bSAlexei Starovoitov 1022db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 1023db20fd2bSAlexei Starovoitov #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 1024db20fd2bSAlexei Starovoitov 1025db20fd2bSAlexei Starovoitov static int map_get_next_key(union bpf_attr *attr) 1026db20fd2bSAlexei Starovoitov { 1027535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1028535e7b4bSMickaël Salaün void __user *unext_key = u64_to_user_ptr(attr->next_key); 1029db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1030db20fd2bSAlexei Starovoitov struct bpf_map *map; 1031db20fd2bSAlexei Starovoitov void *key, *next_key; 1032592867bfSDaniel Borkmann struct fd f; 1033db20fd2bSAlexei Starovoitov int err; 1034db20fd2bSAlexei Starovoitov 1035db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 1036db20fd2bSAlexei Starovoitov return -EINVAL; 1037db20fd2bSAlexei Starovoitov 1038592867bfSDaniel Borkmann f = fdget(ufd); 1039c2101297SDaniel Borkmann map = __bpf_map_get(f); 1040db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1041db20fd2bSAlexei Starovoitov return PTR_ERR(map); 104287df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 10436e71b04aSChenbo Feng err = -EPERM; 10446e71b04aSChenbo Feng goto err_put; 10456e71b04aSChenbo Feng } 10466e71b04aSChenbo Feng 10478fe45924STeng Qin if (ukey) { 1048c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1049e4448ed8SAl Viro if (IS_ERR(key)) { 1050e4448ed8SAl Viro err = PTR_ERR(key); 1051db20fd2bSAlexei Starovoitov goto err_put; 1052e4448ed8SAl Viro } 10538fe45924STeng Qin } else { 10548fe45924STeng Qin key = NULL; 10558fe45924STeng Qin } 1056db20fd2bSAlexei Starovoitov 1057db20fd2bSAlexei Starovoitov err = -ENOMEM; 1058db20fd2bSAlexei Starovoitov next_key = kmalloc(map->key_size, GFP_USER); 1059db20fd2bSAlexei Starovoitov if (!next_key) 1060db20fd2bSAlexei Starovoitov goto free_key; 1061db20fd2bSAlexei Starovoitov 1062a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1063a3884572SJakub Kicinski err = bpf_map_offload_get_next_key(map, key, next_key); 1064a3884572SJakub Kicinski goto out; 1065a3884572SJakub Kicinski } 1066a3884572SJakub Kicinski 1067db20fd2bSAlexei Starovoitov rcu_read_lock(); 1068db20fd2bSAlexei Starovoitov err = map->ops->map_get_next_key(map, key, next_key); 1069db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1070a3884572SJakub Kicinski out: 1071db20fd2bSAlexei Starovoitov if (err) 1072db20fd2bSAlexei Starovoitov goto free_next_key; 1073db20fd2bSAlexei Starovoitov 1074db20fd2bSAlexei Starovoitov err = -EFAULT; 1075db20fd2bSAlexei Starovoitov if (copy_to_user(unext_key, next_key, map->key_size) != 0) 1076db20fd2bSAlexei Starovoitov goto free_next_key; 1077db20fd2bSAlexei Starovoitov 1078db20fd2bSAlexei Starovoitov err = 0; 1079db20fd2bSAlexei Starovoitov 1080db20fd2bSAlexei Starovoitov free_next_key: 1081db20fd2bSAlexei Starovoitov kfree(next_key); 1082db20fd2bSAlexei Starovoitov free_key: 1083db20fd2bSAlexei Starovoitov kfree(key); 1084db20fd2bSAlexei Starovoitov err_put: 1085db20fd2bSAlexei Starovoitov fdput(f); 1086db20fd2bSAlexei Starovoitov return err; 1087db20fd2bSAlexei Starovoitov } 1088db20fd2bSAlexei Starovoitov 1089bd513cd0SMauricio Vasquez B #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value 1090bd513cd0SMauricio Vasquez B 1091bd513cd0SMauricio Vasquez B static int map_lookup_and_delete_elem(union bpf_attr *attr) 1092bd513cd0SMauricio Vasquez B { 1093bd513cd0SMauricio Vasquez B void __user *ukey = u64_to_user_ptr(attr->key); 1094bd513cd0SMauricio Vasquez B void __user *uvalue = u64_to_user_ptr(attr->value); 1095bd513cd0SMauricio Vasquez B int ufd = attr->map_fd; 1096bd513cd0SMauricio Vasquez B struct bpf_map *map; 1097540fefc0SAlexei Starovoitov void *key, *value; 1098bd513cd0SMauricio Vasquez B u32 value_size; 1099bd513cd0SMauricio Vasquez B struct fd f; 1100bd513cd0SMauricio Vasquez B int err; 1101bd513cd0SMauricio Vasquez B 1102bd513cd0SMauricio Vasquez B if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) 1103bd513cd0SMauricio Vasquez B return -EINVAL; 1104bd513cd0SMauricio Vasquez B 1105bd513cd0SMauricio Vasquez B f = fdget(ufd); 1106bd513cd0SMauricio Vasquez B map = __bpf_map_get(f); 1107bd513cd0SMauricio Vasquez B if (IS_ERR(map)) 1108bd513cd0SMauricio Vasquez B return PTR_ERR(map); 110987df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 1110bd513cd0SMauricio Vasquez B err = -EPERM; 1111bd513cd0SMauricio Vasquez B goto err_put; 1112bd513cd0SMauricio Vasquez B } 1113bd513cd0SMauricio Vasquez B 1114bd513cd0SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1115bd513cd0SMauricio Vasquez B if (IS_ERR(key)) { 1116bd513cd0SMauricio Vasquez B err = PTR_ERR(key); 1117bd513cd0SMauricio Vasquez B goto err_put; 1118bd513cd0SMauricio Vasquez B } 1119bd513cd0SMauricio Vasquez B 1120bd513cd0SMauricio Vasquez B value_size = map->value_size; 1121bd513cd0SMauricio Vasquez B 1122bd513cd0SMauricio Vasquez B err = -ENOMEM; 1123bd513cd0SMauricio Vasquez B value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1124bd513cd0SMauricio Vasquez B if (!value) 1125bd513cd0SMauricio Vasquez B goto free_key; 1126bd513cd0SMauricio Vasquez B 1127bd513cd0SMauricio Vasquez B if (map->map_type == BPF_MAP_TYPE_QUEUE || 1128bd513cd0SMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 1129bd513cd0SMauricio Vasquez B err = map->ops->map_pop_elem(map, value); 1130bd513cd0SMauricio Vasquez B } else { 1131bd513cd0SMauricio Vasquez B err = -ENOTSUPP; 1132bd513cd0SMauricio Vasquez B } 1133bd513cd0SMauricio Vasquez B 1134bd513cd0SMauricio Vasquez B if (err) 1135bd513cd0SMauricio Vasquez B goto free_value; 1136bd513cd0SMauricio Vasquez B 1137bd513cd0SMauricio Vasquez B if (copy_to_user(uvalue, value, value_size) != 0) 1138bd513cd0SMauricio Vasquez B goto free_value; 1139bd513cd0SMauricio Vasquez B 1140bd513cd0SMauricio Vasquez B err = 0; 1141bd513cd0SMauricio Vasquez B 1142bd513cd0SMauricio Vasquez B free_value: 1143bd513cd0SMauricio Vasquez B kfree(value); 1144bd513cd0SMauricio Vasquez B free_key: 1145bd513cd0SMauricio Vasquez B kfree(key); 1146bd513cd0SMauricio Vasquez B err_put: 1147bd513cd0SMauricio Vasquez B fdput(f); 1148bd513cd0SMauricio Vasquez B return err; 1149bd513cd0SMauricio Vasquez B } 1150bd513cd0SMauricio Vasquez B 115187df15deSDaniel Borkmann #define BPF_MAP_FREEZE_LAST_FIELD map_fd 115287df15deSDaniel Borkmann 115387df15deSDaniel Borkmann static int map_freeze(const union bpf_attr *attr) 115487df15deSDaniel Borkmann { 115587df15deSDaniel Borkmann int err = 0, ufd = attr->map_fd; 115687df15deSDaniel Borkmann struct bpf_map *map; 115787df15deSDaniel Borkmann struct fd f; 115887df15deSDaniel Borkmann 115987df15deSDaniel Borkmann if (CHECK_ATTR(BPF_MAP_FREEZE)) 116087df15deSDaniel Borkmann return -EINVAL; 116187df15deSDaniel Borkmann 116287df15deSDaniel Borkmann f = fdget(ufd); 116387df15deSDaniel Borkmann map = __bpf_map_get(f); 116487df15deSDaniel Borkmann if (IS_ERR(map)) 116587df15deSDaniel Borkmann return PTR_ERR(map); 116687df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) { 116787df15deSDaniel Borkmann err = -EBUSY; 116887df15deSDaniel Borkmann goto err_put; 116987df15deSDaniel Borkmann } 117087df15deSDaniel Borkmann if (!capable(CAP_SYS_ADMIN)) { 117187df15deSDaniel Borkmann err = -EPERM; 117287df15deSDaniel Borkmann goto err_put; 117387df15deSDaniel Borkmann } 117487df15deSDaniel Borkmann 117587df15deSDaniel Borkmann WRITE_ONCE(map->frozen, true); 117687df15deSDaniel Borkmann err_put: 117787df15deSDaniel Borkmann fdput(f); 117887df15deSDaniel Borkmann return err; 117987df15deSDaniel Borkmann } 118087df15deSDaniel Borkmann 11817de16e3aSJakub Kicinski static const struct bpf_prog_ops * const bpf_prog_types[] = { 11827de16e3aSJakub Kicinski #define BPF_PROG_TYPE(_id, _name) \ 11837de16e3aSJakub Kicinski [_id] = & _name ## _prog_ops, 11847de16e3aSJakub Kicinski #define BPF_MAP_TYPE(_id, _ops) 11857de16e3aSJakub Kicinski #include <linux/bpf_types.h> 11867de16e3aSJakub Kicinski #undef BPF_PROG_TYPE 11877de16e3aSJakub Kicinski #undef BPF_MAP_TYPE 11887de16e3aSJakub Kicinski }; 11897de16e3aSJakub Kicinski 119009756af4SAlexei Starovoitov static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 119109756af4SAlexei Starovoitov { 1192d0f1a451SDaniel Borkmann const struct bpf_prog_ops *ops; 1193d0f1a451SDaniel Borkmann 1194d0f1a451SDaniel Borkmann if (type >= ARRAY_SIZE(bpf_prog_types)) 1195d0f1a451SDaniel Borkmann return -EINVAL; 1196d0f1a451SDaniel Borkmann type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types)); 1197d0f1a451SDaniel Borkmann ops = bpf_prog_types[type]; 1198d0f1a451SDaniel Borkmann if (!ops) 1199be9370a7SJohannes Berg return -EINVAL; 120009756af4SAlexei Starovoitov 1201ab3f0063SJakub Kicinski if (!bpf_prog_is_dev_bound(prog->aux)) 1202d0f1a451SDaniel Borkmann prog->aux->ops = ops; 1203ab3f0063SJakub Kicinski else 1204ab3f0063SJakub Kicinski prog->aux->ops = &bpf_offload_prog_ops; 120524701eceSDaniel Borkmann prog->type = type; 120609756af4SAlexei Starovoitov return 0; 120709756af4SAlexei Starovoitov } 120809756af4SAlexei Starovoitov 120909756af4SAlexei Starovoitov /* drop refcnt on maps used by eBPF program and free auxilary data */ 121009756af4SAlexei Starovoitov static void free_used_maps(struct bpf_prog_aux *aux) 121109756af4SAlexei Starovoitov { 12128bad74f9SRoman Gushchin enum bpf_cgroup_storage_type stype; 121309756af4SAlexei Starovoitov int i; 121409756af4SAlexei Starovoitov 12158bad74f9SRoman Gushchin for_each_cgroup_storage_type(stype) { 12168bad74f9SRoman Gushchin if (!aux->cgroup_storage[stype]) 12178bad74f9SRoman Gushchin continue; 12188bad74f9SRoman Gushchin bpf_cgroup_storage_release(aux->prog, 12198bad74f9SRoman Gushchin aux->cgroup_storage[stype]); 12208bad74f9SRoman Gushchin } 1221de9cbbaaSRoman Gushchin 122209756af4SAlexei Starovoitov for (i = 0; i < aux->used_map_cnt; i++) 122309756af4SAlexei Starovoitov bpf_map_put(aux->used_maps[i]); 122409756af4SAlexei Starovoitov 122509756af4SAlexei Starovoitov kfree(aux->used_maps); 122609756af4SAlexei Starovoitov } 122709756af4SAlexei Starovoitov 12285ccb071eSDaniel Borkmann int __bpf_prog_charge(struct user_struct *user, u32 pages) 12295ccb071eSDaniel Borkmann { 12305ccb071eSDaniel Borkmann unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 12315ccb071eSDaniel Borkmann unsigned long user_bufs; 12325ccb071eSDaniel Borkmann 12335ccb071eSDaniel Borkmann if (user) { 12345ccb071eSDaniel Borkmann user_bufs = atomic_long_add_return(pages, &user->locked_vm); 12355ccb071eSDaniel Borkmann if (user_bufs > memlock_limit) { 12365ccb071eSDaniel Borkmann atomic_long_sub(pages, &user->locked_vm); 12375ccb071eSDaniel Borkmann return -EPERM; 12385ccb071eSDaniel Borkmann } 12395ccb071eSDaniel Borkmann } 12405ccb071eSDaniel Borkmann 12415ccb071eSDaniel Borkmann return 0; 12425ccb071eSDaniel Borkmann } 12435ccb071eSDaniel Borkmann 12445ccb071eSDaniel Borkmann void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 12455ccb071eSDaniel Borkmann { 12465ccb071eSDaniel Borkmann if (user) 12475ccb071eSDaniel Borkmann atomic_long_sub(pages, &user->locked_vm); 12485ccb071eSDaniel Borkmann } 12495ccb071eSDaniel Borkmann 1250aaac3ba9SAlexei Starovoitov static int bpf_prog_charge_memlock(struct bpf_prog *prog) 1251aaac3ba9SAlexei Starovoitov { 1252aaac3ba9SAlexei Starovoitov struct user_struct *user = get_current_user(); 12535ccb071eSDaniel Borkmann int ret; 1254aaac3ba9SAlexei Starovoitov 12555ccb071eSDaniel Borkmann ret = __bpf_prog_charge(user, prog->pages); 12565ccb071eSDaniel Borkmann if (ret) { 1257aaac3ba9SAlexei Starovoitov free_uid(user); 12585ccb071eSDaniel Borkmann return ret; 1259aaac3ba9SAlexei Starovoitov } 12605ccb071eSDaniel Borkmann 1261aaac3ba9SAlexei Starovoitov prog->aux->user = user; 1262aaac3ba9SAlexei Starovoitov return 0; 1263aaac3ba9SAlexei Starovoitov } 1264aaac3ba9SAlexei Starovoitov 1265aaac3ba9SAlexei Starovoitov static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 1266aaac3ba9SAlexei Starovoitov { 1267aaac3ba9SAlexei Starovoitov struct user_struct *user = prog->aux->user; 1268aaac3ba9SAlexei Starovoitov 12695ccb071eSDaniel Borkmann __bpf_prog_uncharge(user, prog->pages); 1270aaac3ba9SAlexei Starovoitov free_uid(user); 1271aaac3ba9SAlexei Starovoitov } 1272aaac3ba9SAlexei Starovoitov 1273dc4bb0e2SMartin KaFai Lau static int bpf_prog_alloc_id(struct bpf_prog *prog) 1274dc4bb0e2SMartin KaFai Lau { 1275dc4bb0e2SMartin KaFai Lau int id; 1276dc4bb0e2SMartin KaFai Lau 1277b76354cdSShaohua Li idr_preload(GFP_KERNEL); 1278dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1279dc4bb0e2SMartin KaFai Lau id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 1280dc4bb0e2SMartin KaFai Lau if (id > 0) 1281dc4bb0e2SMartin KaFai Lau prog->aux->id = id; 1282dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1283b76354cdSShaohua Li idr_preload_end(); 1284dc4bb0e2SMartin KaFai Lau 1285dc4bb0e2SMartin KaFai Lau /* id is in [1, INT_MAX) */ 1286dc4bb0e2SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 1287dc4bb0e2SMartin KaFai Lau return -ENOSPC; 1288dc4bb0e2SMartin KaFai Lau 1289dc4bb0e2SMartin KaFai Lau return id > 0 ? 0 : id; 1290dc4bb0e2SMartin KaFai Lau } 1291dc4bb0e2SMartin KaFai Lau 1292ad8ad79fSJakub Kicinski void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) 1293dc4bb0e2SMartin KaFai Lau { 1294ad8ad79fSJakub Kicinski /* cBPF to eBPF migrations are currently not in the idr store. 1295ad8ad79fSJakub Kicinski * Offloaded programs are removed from the store when their device 1296ad8ad79fSJakub Kicinski * disappears - even if someone grabs an fd to them they are unusable, 1297ad8ad79fSJakub Kicinski * simply waiting for refcnt to drop to be freed. 1298ad8ad79fSJakub Kicinski */ 1299dc4bb0e2SMartin KaFai Lau if (!prog->aux->id) 1300dc4bb0e2SMartin KaFai Lau return; 1301dc4bb0e2SMartin KaFai Lau 1302b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1303dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1304b16d9aa4SMartin KaFai Lau else 1305b16d9aa4SMartin KaFai Lau __acquire(&prog_idr_lock); 1306b16d9aa4SMartin KaFai Lau 1307dc4bb0e2SMartin KaFai Lau idr_remove(&prog_idr, prog->aux->id); 1308ad8ad79fSJakub Kicinski prog->aux->id = 0; 1309b16d9aa4SMartin KaFai Lau 1310b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1311dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1312b16d9aa4SMartin KaFai Lau else 1313b16d9aa4SMartin KaFai Lau __release(&prog_idr_lock); 1314dc4bb0e2SMartin KaFai Lau } 1315dc4bb0e2SMartin KaFai Lau 13161aacde3dSDaniel Borkmann static void __bpf_prog_put_rcu(struct rcu_head *rcu) 1317abf2e7d6SAlexei Starovoitov { 1318abf2e7d6SAlexei Starovoitov struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 1319abf2e7d6SAlexei Starovoitov 1320abf2e7d6SAlexei Starovoitov free_used_maps(aux); 1321aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(aux->prog); 1322afdb09c7SChenbo Feng security_bpf_prog_free(aux); 1323abf2e7d6SAlexei Starovoitov bpf_prog_free(aux->prog); 1324abf2e7d6SAlexei Starovoitov } 1325abf2e7d6SAlexei Starovoitov 1326b16d9aa4SMartin KaFai Lau static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 132709756af4SAlexei Starovoitov { 1328a67edbf4SDaniel Borkmann if (atomic_dec_and_test(&prog->aux->refcnt)) { 13296ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0); 133034ad5580SMartin KaFai Lau /* bpf_prog_free_id() must be called first */ 1331b16d9aa4SMartin KaFai Lau bpf_prog_free_id(prog, do_idr_lock); 13327d1982b4SDaniel Borkmann bpf_prog_kallsyms_del_all(prog); 1333838e9690SYonghong Song btf_put(prog->aux->btf); 1334ba64e7d8SYonghong Song kvfree(prog->aux->func_info); 1335c454a46bSMartin KaFai Lau bpf_prog_free_linfo(prog); 13364f74d809SDaniel Borkmann 13371aacde3dSDaniel Borkmann call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 133809756af4SAlexei Starovoitov } 1339a67edbf4SDaniel Borkmann } 1340b16d9aa4SMartin KaFai Lau 1341b16d9aa4SMartin KaFai Lau void bpf_prog_put(struct bpf_prog *prog) 1342b16d9aa4SMartin KaFai Lau { 1343b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, true); 1344b16d9aa4SMartin KaFai Lau } 1345e2e9b654SDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_put); 134609756af4SAlexei Starovoitov 134709756af4SAlexei Starovoitov static int bpf_prog_release(struct inode *inode, struct file *filp) 134809756af4SAlexei Starovoitov { 134909756af4SAlexei Starovoitov struct bpf_prog *prog = filp->private_data; 135009756af4SAlexei Starovoitov 13511aacde3dSDaniel Borkmann bpf_prog_put(prog); 135209756af4SAlexei Starovoitov return 0; 135309756af4SAlexei Starovoitov } 135409756af4SAlexei Starovoitov 1355492ecee8SAlexei Starovoitov static void bpf_prog_get_stats(const struct bpf_prog *prog, 1356492ecee8SAlexei Starovoitov struct bpf_prog_stats *stats) 1357492ecee8SAlexei Starovoitov { 1358492ecee8SAlexei Starovoitov u64 nsecs = 0, cnt = 0; 1359492ecee8SAlexei Starovoitov int cpu; 1360492ecee8SAlexei Starovoitov 1361492ecee8SAlexei Starovoitov for_each_possible_cpu(cpu) { 1362492ecee8SAlexei Starovoitov const struct bpf_prog_stats *st; 1363492ecee8SAlexei Starovoitov unsigned int start; 1364492ecee8SAlexei Starovoitov u64 tnsecs, tcnt; 1365492ecee8SAlexei Starovoitov 1366492ecee8SAlexei Starovoitov st = per_cpu_ptr(prog->aux->stats, cpu); 1367492ecee8SAlexei Starovoitov do { 1368492ecee8SAlexei Starovoitov start = u64_stats_fetch_begin_irq(&st->syncp); 1369492ecee8SAlexei Starovoitov tnsecs = st->nsecs; 1370492ecee8SAlexei Starovoitov tcnt = st->cnt; 1371492ecee8SAlexei Starovoitov } while (u64_stats_fetch_retry_irq(&st->syncp, start)); 1372492ecee8SAlexei Starovoitov nsecs += tnsecs; 1373492ecee8SAlexei Starovoitov cnt += tcnt; 1374492ecee8SAlexei Starovoitov } 1375492ecee8SAlexei Starovoitov stats->nsecs = nsecs; 1376492ecee8SAlexei Starovoitov stats->cnt = cnt; 1377492ecee8SAlexei Starovoitov } 1378492ecee8SAlexei Starovoitov 13797bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 13807bd509e3SDaniel Borkmann static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 13817bd509e3SDaniel Borkmann { 13827bd509e3SDaniel Borkmann const struct bpf_prog *prog = filp->private_data; 1383f1f7714eSDaniel Borkmann char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 1384492ecee8SAlexei Starovoitov struct bpf_prog_stats stats; 13857bd509e3SDaniel Borkmann 1386492ecee8SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 1387f1f7714eSDaniel Borkmann bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 13887bd509e3SDaniel Borkmann seq_printf(m, 13897bd509e3SDaniel Borkmann "prog_type:\t%u\n" 13907bd509e3SDaniel Borkmann "prog_jited:\t%u\n" 1391f1f7714eSDaniel Borkmann "prog_tag:\t%s\n" 13924316b409SDaniel Borkmann "memlock:\t%llu\n" 1393492ecee8SAlexei Starovoitov "prog_id:\t%u\n" 1394492ecee8SAlexei Starovoitov "run_time_ns:\t%llu\n" 1395492ecee8SAlexei Starovoitov "run_cnt:\t%llu\n", 13967bd509e3SDaniel Borkmann prog->type, 13977bd509e3SDaniel Borkmann prog->jited, 1398f1f7714eSDaniel Borkmann prog_tag, 13994316b409SDaniel Borkmann prog->pages * 1ULL << PAGE_SHIFT, 1400492ecee8SAlexei Starovoitov prog->aux->id, 1401492ecee8SAlexei Starovoitov stats.nsecs, 1402492ecee8SAlexei Starovoitov stats.cnt); 14037bd509e3SDaniel Borkmann } 14047bd509e3SDaniel Borkmann #endif 14057bd509e3SDaniel Borkmann 1406f66e448cSChenbo Feng const struct file_operations bpf_prog_fops = { 14077bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 14087bd509e3SDaniel Borkmann .show_fdinfo = bpf_prog_show_fdinfo, 14097bd509e3SDaniel Borkmann #endif 141009756af4SAlexei Starovoitov .release = bpf_prog_release, 14116e71b04aSChenbo Feng .read = bpf_dummy_read, 14126e71b04aSChenbo Feng .write = bpf_dummy_write, 141309756af4SAlexei Starovoitov }; 141409756af4SAlexei Starovoitov 1415b2197755SDaniel Borkmann int bpf_prog_new_fd(struct bpf_prog *prog) 1416aa79781bSDaniel Borkmann { 1417afdb09c7SChenbo Feng int ret; 1418afdb09c7SChenbo Feng 1419afdb09c7SChenbo Feng ret = security_bpf_prog(prog); 1420afdb09c7SChenbo Feng if (ret < 0) 1421afdb09c7SChenbo Feng return ret; 1422afdb09c7SChenbo Feng 1423aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 1424aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC); 1425aa79781bSDaniel Borkmann } 1426aa79781bSDaniel Borkmann 1427113214beSDaniel Borkmann static struct bpf_prog *____bpf_prog_get(struct fd f) 142809756af4SAlexei Starovoitov { 142909756af4SAlexei Starovoitov if (!f.file) 143009756af4SAlexei Starovoitov return ERR_PTR(-EBADF); 143109756af4SAlexei Starovoitov if (f.file->f_op != &bpf_prog_fops) { 143209756af4SAlexei Starovoitov fdput(f); 143309756af4SAlexei Starovoitov return ERR_PTR(-EINVAL); 143409756af4SAlexei Starovoitov } 143509756af4SAlexei Starovoitov 1436c2101297SDaniel Borkmann return f.file->private_data; 143709756af4SAlexei Starovoitov } 143809756af4SAlexei Starovoitov 143959d3656dSBrenden Blanco struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i) 144092117d84SAlexei Starovoitov { 144159d3656dSBrenden Blanco if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) { 144259d3656dSBrenden Blanco atomic_sub(i, &prog->aux->refcnt); 144392117d84SAlexei Starovoitov return ERR_PTR(-EBUSY); 144492117d84SAlexei Starovoitov } 144592117d84SAlexei Starovoitov return prog; 144692117d84SAlexei Starovoitov } 144759d3656dSBrenden Blanco EXPORT_SYMBOL_GPL(bpf_prog_add); 144859d3656dSBrenden Blanco 1449c540594fSDaniel Borkmann void bpf_prog_sub(struct bpf_prog *prog, int i) 1450c540594fSDaniel Borkmann { 1451c540594fSDaniel Borkmann /* Only to be used for undoing previous bpf_prog_add() in some 1452c540594fSDaniel Borkmann * error path. We still know that another entity in our call 1453c540594fSDaniel Borkmann * path holds a reference to the program, thus atomic_sub() can 1454c540594fSDaniel Borkmann * be safely used in such cases! 1455c540594fSDaniel Borkmann */ 1456c540594fSDaniel Borkmann WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0); 1457c540594fSDaniel Borkmann } 1458c540594fSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_sub); 1459c540594fSDaniel Borkmann 146059d3656dSBrenden Blanco struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog) 146159d3656dSBrenden Blanco { 146259d3656dSBrenden Blanco return bpf_prog_add(prog, 1); 146359d3656dSBrenden Blanco } 146497bc402dSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_inc); 146592117d84SAlexei Starovoitov 1466b16d9aa4SMartin KaFai Lau /* prog_idr_lock should have been held */ 1467a6f6df69SJohn Fastabend struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 1468b16d9aa4SMartin KaFai Lau { 1469b16d9aa4SMartin KaFai Lau int refold; 1470b16d9aa4SMartin KaFai Lau 1471bfc18e38SMark Rutland refold = atomic_fetch_add_unless(&prog->aux->refcnt, 1, 0); 1472b16d9aa4SMartin KaFai Lau 1473b16d9aa4SMartin KaFai Lau if (refold >= BPF_MAX_REFCNT) { 1474b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, false); 1475b16d9aa4SMartin KaFai Lau return ERR_PTR(-EBUSY); 1476b16d9aa4SMartin KaFai Lau } 1477b16d9aa4SMartin KaFai Lau 1478b16d9aa4SMartin KaFai Lau if (!refold) 1479b16d9aa4SMartin KaFai Lau return ERR_PTR(-ENOENT); 1480b16d9aa4SMartin KaFai Lau 1481b16d9aa4SMartin KaFai Lau return prog; 1482b16d9aa4SMartin KaFai Lau } 1483a6f6df69SJohn Fastabend EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 1484b16d9aa4SMartin KaFai Lau 1485040ee692SAl Viro bool bpf_prog_get_ok(struct bpf_prog *prog, 1486288b3de5SJakub Kicinski enum bpf_prog_type *attach_type, bool attach_drv) 1487248f346fSJakub Kicinski { 1488288b3de5SJakub Kicinski /* not an attachment, just a refcount inc, always allow */ 1489288b3de5SJakub Kicinski if (!attach_type) 1490288b3de5SJakub Kicinski return true; 1491248f346fSJakub Kicinski 1492248f346fSJakub Kicinski if (prog->type != *attach_type) 1493248f346fSJakub Kicinski return false; 1494288b3de5SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv) 1495248f346fSJakub Kicinski return false; 1496248f346fSJakub Kicinski 1497248f346fSJakub Kicinski return true; 1498248f346fSJakub Kicinski } 1499248f346fSJakub Kicinski 1500248f346fSJakub Kicinski static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, 1501288b3de5SJakub Kicinski bool attach_drv) 150209756af4SAlexei Starovoitov { 150309756af4SAlexei Starovoitov struct fd f = fdget(ufd); 150409756af4SAlexei Starovoitov struct bpf_prog *prog; 150509756af4SAlexei Starovoitov 1506113214beSDaniel Borkmann prog = ____bpf_prog_get(f); 150709756af4SAlexei Starovoitov if (IS_ERR(prog)) 150809756af4SAlexei Starovoitov return prog; 1509288b3de5SJakub Kicinski if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) { 1510113214beSDaniel Borkmann prog = ERR_PTR(-EINVAL); 1511113214beSDaniel Borkmann goto out; 1512113214beSDaniel Borkmann } 151309756af4SAlexei Starovoitov 151492117d84SAlexei Starovoitov prog = bpf_prog_inc(prog); 1515113214beSDaniel Borkmann out: 151609756af4SAlexei Starovoitov fdput(f); 151709756af4SAlexei Starovoitov return prog; 151809756af4SAlexei Starovoitov } 1519113214beSDaniel Borkmann 1520113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get(u32 ufd) 1521113214beSDaniel Borkmann { 1522288b3de5SJakub Kicinski return __bpf_prog_get(ufd, NULL, false); 1523113214beSDaniel Borkmann } 1524113214beSDaniel Borkmann 1525248f346fSJakub Kicinski struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 1526288b3de5SJakub Kicinski bool attach_drv) 1527248f346fSJakub Kicinski { 15284d220ed0SAlexei Starovoitov return __bpf_prog_get(ufd, &type, attach_drv); 1529248f346fSJakub Kicinski } 15306c8dfe21SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev); 1531248f346fSJakub Kicinski 1532aac3fc32SAndrey Ignatov /* Initially all BPF programs could be loaded w/o specifying 1533aac3fc32SAndrey Ignatov * expected_attach_type. Later for some of them specifying expected_attach_type 1534aac3fc32SAndrey Ignatov * at load time became required so that program could be validated properly. 1535aac3fc32SAndrey Ignatov * Programs of types that are allowed to be loaded both w/ and w/o (for 1536aac3fc32SAndrey Ignatov * backward compatibility) expected_attach_type, should have the default attach 1537aac3fc32SAndrey Ignatov * type assigned to expected_attach_type for the latter case, so that it can be 1538aac3fc32SAndrey Ignatov * validated later at attach time. 1539aac3fc32SAndrey Ignatov * 1540aac3fc32SAndrey Ignatov * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if 1541aac3fc32SAndrey Ignatov * prog type requires it but has some attach types that have to be backward 1542aac3fc32SAndrey Ignatov * compatible. 1543aac3fc32SAndrey Ignatov */ 1544aac3fc32SAndrey Ignatov static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) 1545aac3fc32SAndrey Ignatov { 1546aac3fc32SAndrey Ignatov switch (attr->prog_type) { 1547aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1548aac3fc32SAndrey Ignatov /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't 1549aac3fc32SAndrey Ignatov * exist so checking for non-zero is the way to go here. 1550aac3fc32SAndrey Ignatov */ 1551aac3fc32SAndrey Ignatov if (!attr->expected_attach_type) 1552aac3fc32SAndrey Ignatov attr->expected_attach_type = 1553aac3fc32SAndrey Ignatov BPF_CGROUP_INET_SOCK_CREATE; 1554aac3fc32SAndrey Ignatov break; 1555aac3fc32SAndrey Ignatov } 1556aac3fc32SAndrey Ignatov } 1557aac3fc32SAndrey Ignatov 15585e43f899SAndrey Ignatov static int 15595e43f899SAndrey Ignatov bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type, 15605e43f899SAndrey Ignatov enum bpf_attach_type expected_attach_type) 15615e43f899SAndrey Ignatov { 15624fbac77dSAndrey Ignatov switch (prog_type) { 1563aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1564aac3fc32SAndrey Ignatov switch (expected_attach_type) { 1565aac3fc32SAndrey Ignatov case BPF_CGROUP_INET_SOCK_CREATE: 1566aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1567aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 1568aac3fc32SAndrey Ignatov return 0; 1569aac3fc32SAndrey Ignatov default: 1570aac3fc32SAndrey Ignatov return -EINVAL; 1571aac3fc32SAndrey Ignatov } 15724fbac77dSAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 15734fbac77dSAndrey Ignatov switch (expected_attach_type) { 15744fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 15754fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1576d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1577d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 15781cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 15791cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 15805e43f899SAndrey Ignatov return 0; 15814fbac77dSAndrey Ignatov default: 15824fbac77dSAndrey Ignatov return -EINVAL; 15834fbac77dSAndrey Ignatov } 15844fbac77dSAndrey Ignatov default: 15854fbac77dSAndrey Ignatov return 0; 15864fbac77dSAndrey Ignatov } 15875e43f899SAndrey Ignatov } 15885e43f899SAndrey Ignatov 158909756af4SAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 1590c454a46bSMartin KaFai Lau #define BPF_PROG_LOAD_LAST_FIELD line_info_cnt 159109756af4SAlexei Starovoitov 1592838e9690SYonghong Song static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) 159309756af4SAlexei Starovoitov { 159409756af4SAlexei Starovoitov enum bpf_prog_type type = attr->prog_type; 159509756af4SAlexei Starovoitov struct bpf_prog *prog; 159609756af4SAlexei Starovoitov int err; 159709756af4SAlexei Starovoitov char license[128]; 159809756af4SAlexei Starovoitov bool is_gpl; 159909756af4SAlexei Starovoitov 160009756af4SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_LOAD)) 160109756af4SAlexei Starovoitov return -EINVAL; 160209756af4SAlexei Starovoitov 1603e9ee9efcSDavid Miller if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | BPF_F_ANY_ALIGNMENT)) 1604e07b98d9SDavid S. Miller return -EINVAL; 1605e07b98d9SDavid S. Miller 1606e9ee9efcSDavid Miller if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && 1607e9ee9efcSDavid Miller (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && 1608e9ee9efcSDavid Miller !capable(CAP_SYS_ADMIN)) 1609e9ee9efcSDavid Miller return -EPERM; 1610e9ee9efcSDavid Miller 161109756af4SAlexei Starovoitov /* copy eBPF program license from user space */ 1612535e7b4bSMickaël Salaün if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 161309756af4SAlexei Starovoitov sizeof(license) - 1) < 0) 161409756af4SAlexei Starovoitov return -EFAULT; 161509756af4SAlexei Starovoitov license[sizeof(license) - 1] = 0; 161609756af4SAlexei Starovoitov 161709756af4SAlexei Starovoitov /* eBPF programs must be GPL compatible to use GPL-ed functions */ 161809756af4SAlexei Starovoitov is_gpl = license_is_gpl_compatible(license); 161909756af4SAlexei Starovoitov 1620c04c0d2bSAlexei Starovoitov if (attr->insn_cnt == 0 || 1621c04c0d2bSAlexei Starovoitov attr->insn_cnt > (capable(CAP_SYS_ADMIN) ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) 1622ef0915caSDaniel Borkmann return -E2BIG; 162380b7d819SChenbo Feng if (type != BPF_PROG_TYPE_SOCKET_FILTER && 162480b7d819SChenbo Feng type != BPF_PROG_TYPE_CGROUP_SKB && 162580b7d819SChenbo Feng !capable(CAP_SYS_ADMIN)) 16261be7f75dSAlexei Starovoitov return -EPERM; 16271be7f75dSAlexei Starovoitov 1628aac3fc32SAndrey Ignatov bpf_prog_load_fixup_attach_type(attr); 16295e43f899SAndrey Ignatov if (bpf_prog_load_check_attach_type(type, attr->expected_attach_type)) 16305e43f899SAndrey Ignatov return -EINVAL; 16315e43f899SAndrey Ignatov 163209756af4SAlexei Starovoitov /* plain bpf_prog allocation */ 163309756af4SAlexei Starovoitov prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 163409756af4SAlexei Starovoitov if (!prog) 163509756af4SAlexei Starovoitov return -ENOMEM; 163609756af4SAlexei Starovoitov 16375e43f899SAndrey Ignatov prog->expected_attach_type = attr->expected_attach_type; 16385e43f899SAndrey Ignatov 16399a18eedbSJakub Kicinski prog->aux->offload_requested = !!attr->prog_ifindex; 16409a18eedbSJakub Kicinski 1641afdb09c7SChenbo Feng err = security_bpf_prog_alloc(prog->aux); 1642aaac3ba9SAlexei Starovoitov if (err) 1643aaac3ba9SAlexei Starovoitov goto free_prog_nouncharge; 1644aaac3ba9SAlexei Starovoitov 1645afdb09c7SChenbo Feng err = bpf_prog_charge_memlock(prog); 1646afdb09c7SChenbo Feng if (err) 1647afdb09c7SChenbo Feng goto free_prog_sec; 1648afdb09c7SChenbo Feng 164909756af4SAlexei Starovoitov prog->len = attr->insn_cnt; 165009756af4SAlexei Starovoitov 165109756af4SAlexei Starovoitov err = -EFAULT; 1652535e7b4bSMickaël Salaün if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 1653aafe6ae9SDaniel Borkmann bpf_prog_insn_size(prog)) != 0) 165409756af4SAlexei Starovoitov goto free_prog; 165509756af4SAlexei Starovoitov 165609756af4SAlexei Starovoitov prog->orig_prog = NULL; 1657a91263d5SDaniel Borkmann prog->jited = 0; 165809756af4SAlexei Starovoitov 165909756af4SAlexei Starovoitov atomic_set(&prog->aux->refcnt, 1); 1660a91263d5SDaniel Borkmann prog->gpl_compatible = is_gpl ? 1 : 0; 166109756af4SAlexei Starovoitov 16629a18eedbSJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 1663ab3f0063SJakub Kicinski err = bpf_prog_offload_init(prog, attr); 1664ab3f0063SJakub Kicinski if (err) 1665ab3f0063SJakub Kicinski goto free_prog; 1666ab3f0063SJakub Kicinski } 1667ab3f0063SJakub Kicinski 166809756af4SAlexei Starovoitov /* find program type: socket_filter vs tracing_filter */ 166909756af4SAlexei Starovoitov err = find_prog_type(type, prog); 167009756af4SAlexei Starovoitov if (err < 0) 167109756af4SAlexei Starovoitov goto free_prog; 167209756af4SAlexei Starovoitov 1673cb4d2b3fSMartin KaFai Lau prog->aux->load_time = ktime_get_boot_ns(); 1674cb4d2b3fSMartin KaFai Lau err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name); 1675cb4d2b3fSMartin KaFai Lau if (err) 1676cb4d2b3fSMartin KaFai Lau goto free_prog; 1677cb4d2b3fSMartin KaFai Lau 167809756af4SAlexei Starovoitov /* run eBPF verifier */ 1679838e9690SYonghong Song err = bpf_check(&prog, attr, uattr); 168009756af4SAlexei Starovoitov if (err < 0) 168109756af4SAlexei Starovoitov goto free_used_maps; 168209756af4SAlexei Starovoitov 1683d1c55ab5SDaniel Borkmann prog = bpf_prog_select_runtime(prog, &err); 168404fd61abSAlexei Starovoitov if (err < 0) 168504fd61abSAlexei Starovoitov goto free_used_maps; 168609756af4SAlexei Starovoitov 1687dc4bb0e2SMartin KaFai Lau err = bpf_prog_alloc_id(prog); 1688dc4bb0e2SMartin KaFai Lau if (err) 1689dc4bb0e2SMartin KaFai Lau goto free_used_maps; 1690dc4bb0e2SMartin KaFai Lau 1691aa79781bSDaniel Borkmann err = bpf_prog_new_fd(prog); 1692b16d9aa4SMartin KaFai Lau if (err < 0) { 1693b16d9aa4SMartin KaFai Lau /* failed to allocate fd. 1694b16d9aa4SMartin KaFai Lau * bpf_prog_put() is needed because the above 1695b16d9aa4SMartin KaFai Lau * bpf_prog_alloc_id() has published the prog 1696b16d9aa4SMartin KaFai Lau * to the userspace and the userspace may 1697b16d9aa4SMartin KaFai Lau * have refcnt-ed it through BPF_PROG_GET_FD_BY_ID. 1698b16d9aa4SMartin KaFai Lau */ 1699b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 1700b16d9aa4SMartin KaFai Lau return err; 1701b16d9aa4SMartin KaFai Lau } 170209756af4SAlexei Starovoitov 170374451e66SDaniel Borkmann bpf_prog_kallsyms_add(prog); 17046ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); 170509756af4SAlexei Starovoitov return err; 170609756af4SAlexei Starovoitov 170709756af4SAlexei Starovoitov free_used_maps: 1708c454a46bSMartin KaFai Lau bpf_prog_free_linfo(prog); 17095482e9a9SMartin KaFai Lau kvfree(prog->aux->func_info); 17105482e9a9SMartin KaFai Lau btf_put(prog->aux->btf); 17117d1982b4SDaniel Borkmann bpf_prog_kallsyms_del_subprogs(prog); 171209756af4SAlexei Starovoitov free_used_maps(prog->aux); 171309756af4SAlexei Starovoitov free_prog: 1714aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(prog); 1715afdb09c7SChenbo Feng free_prog_sec: 1716afdb09c7SChenbo Feng security_bpf_prog_free(prog->aux); 1717aaac3ba9SAlexei Starovoitov free_prog_nouncharge: 171809756af4SAlexei Starovoitov bpf_prog_free(prog); 171909756af4SAlexei Starovoitov return err; 172009756af4SAlexei Starovoitov } 172109756af4SAlexei Starovoitov 17226e71b04aSChenbo Feng #define BPF_OBJ_LAST_FIELD file_flags 1723b2197755SDaniel Borkmann 1724b2197755SDaniel Borkmann static int bpf_obj_pin(const union bpf_attr *attr) 1725b2197755SDaniel Borkmann { 17266e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0) 1727b2197755SDaniel Borkmann return -EINVAL; 1728b2197755SDaniel Borkmann 1729535e7b4bSMickaël Salaün return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 1730b2197755SDaniel Borkmann } 1731b2197755SDaniel Borkmann 1732b2197755SDaniel Borkmann static int bpf_obj_get(const union bpf_attr *attr) 1733b2197755SDaniel Borkmann { 17346e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || 17356e71b04aSChenbo Feng attr->file_flags & ~BPF_OBJ_FLAG_MASK) 1736b2197755SDaniel Borkmann return -EINVAL; 1737b2197755SDaniel Borkmann 17386e71b04aSChenbo Feng return bpf_obj_get_user(u64_to_user_ptr(attr->pathname), 17396e71b04aSChenbo Feng attr->file_flags); 1740b2197755SDaniel Borkmann } 1741b2197755SDaniel Borkmann 1742c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint { 1743c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 1744c4f6699dSAlexei Starovoitov struct bpf_prog *prog; 1745c4f6699dSAlexei Starovoitov }; 1746c4f6699dSAlexei Starovoitov 1747c4f6699dSAlexei Starovoitov static int bpf_raw_tracepoint_release(struct inode *inode, struct file *filp) 1748c4f6699dSAlexei Starovoitov { 1749c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint *raw_tp = filp->private_data; 1750c4f6699dSAlexei Starovoitov 1751c4f6699dSAlexei Starovoitov if (raw_tp->prog) { 1752c4f6699dSAlexei Starovoitov bpf_probe_unregister(raw_tp->btp, raw_tp->prog); 1753c4f6699dSAlexei Starovoitov bpf_prog_put(raw_tp->prog); 1754c4f6699dSAlexei Starovoitov } 1755a38d1107SMatt Mullins bpf_put_raw_tracepoint(raw_tp->btp); 1756c4f6699dSAlexei Starovoitov kfree(raw_tp); 1757c4f6699dSAlexei Starovoitov return 0; 1758c4f6699dSAlexei Starovoitov } 1759c4f6699dSAlexei Starovoitov 1760c4f6699dSAlexei Starovoitov static const struct file_operations bpf_raw_tp_fops = { 1761c4f6699dSAlexei Starovoitov .release = bpf_raw_tracepoint_release, 1762c4f6699dSAlexei Starovoitov .read = bpf_dummy_read, 1763c4f6699dSAlexei Starovoitov .write = bpf_dummy_write, 1764c4f6699dSAlexei Starovoitov }; 1765c4f6699dSAlexei Starovoitov 1766c4f6699dSAlexei Starovoitov #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd 1767c4f6699dSAlexei Starovoitov 1768c4f6699dSAlexei Starovoitov static int bpf_raw_tracepoint_open(const union bpf_attr *attr) 1769c4f6699dSAlexei Starovoitov { 1770c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint *raw_tp; 1771c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 1772c4f6699dSAlexei Starovoitov struct bpf_prog *prog; 1773c4f6699dSAlexei Starovoitov char tp_name[128]; 1774c4f6699dSAlexei Starovoitov int tp_fd, err; 1775c4f6699dSAlexei Starovoitov 1776c4f6699dSAlexei Starovoitov if (strncpy_from_user(tp_name, u64_to_user_ptr(attr->raw_tracepoint.name), 1777c4f6699dSAlexei Starovoitov sizeof(tp_name) - 1) < 0) 1778c4f6699dSAlexei Starovoitov return -EFAULT; 1779c4f6699dSAlexei Starovoitov tp_name[sizeof(tp_name) - 1] = 0; 1780c4f6699dSAlexei Starovoitov 1781a38d1107SMatt Mullins btp = bpf_get_raw_tracepoint(tp_name); 1782c4f6699dSAlexei Starovoitov if (!btp) 1783c4f6699dSAlexei Starovoitov return -ENOENT; 1784c4f6699dSAlexei Starovoitov 1785c4f6699dSAlexei Starovoitov raw_tp = kzalloc(sizeof(*raw_tp), GFP_USER); 1786a38d1107SMatt Mullins if (!raw_tp) { 1787a38d1107SMatt Mullins err = -ENOMEM; 1788a38d1107SMatt Mullins goto out_put_btp; 1789a38d1107SMatt Mullins } 1790c4f6699dSAlexei Starovoitov raw_tp->btp = btp; 1791c4f6699dSAlexei Starovoitov 1792c4f6699dSAlexei Starovoitov prog = bpf_prog_get_type(attr->raw_tracepoint.prog_fd, 1793c4f6699dSAlexei Starovoitov BPF_PROG_TYPE_RAW_TRACEPOINT); 1794c4f6699dSAlexei Starovoitov if (IS_ERR(prog)) { 1795c4f6699dSAlexei Starovoitov err = PTR_ERR(prog); 1796c4f6699dSAlexei Starovoitov goto out_free_tp; 1797c4f6699dSAlexei Starovoitov } 1798c4f6699dSAlexei Starovoitov 1799c4f6699dSAlexei Starovoitov err = bpf_probe_register(raw_tp->btp, prog); 1800c4f6699dSAlexei Starovoitov if (err) 1801c4f6699dSAlexei Starovoitov goto out_put_prog; 1802c4f6699dSAlexei Starovoitov 1803c4f6699dSAlexei Starovoitov raw_tp->prog = prog; 1804c4f6699dSAlexei Starovoitov tp_fd = anon_inode_getfd("bpf-raw-tracepoint", &bpf_raw_tp_fops, raw_tp, 1805c4f6699dSAlexei Starovoitov O_CLOEXEC); 1806c4f6699dSAlexei Starovoitov if (tp_fd < 0) { 1807c4f6699dSAlexei Starovoitov bpf_probe_unregister(raw_tp->btp, prog); 1808c4f6699dSAlexei Starovoitov err = tp_fd; 1809c4f6699dSAlexei Starovoitov goto out_put_prog; 1810c4f6699dSAlexei Starovoitov } 1811c4f6699dSAlexei Starovoitov return tp_fd; 1812c4f6699dSAlexei Starovoitov 1813c4f6699dSAlexei Starovoitov out_put_prog: 1814c4f6699dSAlexei Starovoitov bpf_prog_put(prog); 1815c4f6699dSAlexei Starovoitov out_free_tp: 1816c4f6699dSAlexei Starovoitov kfree(raw_tp); 1817a38d1107SMatt Mullins out_put_btp: 1818a38d1107SMatt Mullins bpf_put_raw_tracepoint(btp); 1819c4f6699dSAlexei Starovoitov return err; 1820c4f6699dSAlexei Starovoitov } 1821c4f6699dSAlexei Starovoitov 182233491588SAnders Roxell static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, 182333491588SAnders Roxell enum bpf_attach_type attach_type) 182433491588SAnders Roxell { 182533491588SAnders Roxell switch (prog->type) { 182633491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK: 182733491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 182833491588SAnders Roxell return attach_type == prog->expected_attach_type ? 0 : -EINVAL; 182933491588SAnders Roxell default: 183033491588SAnders Roxell return 0; 183133491588SAnders Roxell } 183233491588SAnders Roxell } 183333491588SAnders Roxell 1834464bc0fdSJohn Fastabend #define BPF_PROG_ATTACH_LAST_FIELD attach_flags 1835174a79ffSJohn Fastabend 1836324bda9eSAlexei Starovoitov #define BPF_F_ATTACH_MASK \ 1837324bda9eSAlexei Starovoitov (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI) 1838324bda9eSAlexei Starovoitov 1839f4324551SDaniel Mack static int bpf_prog_attach(const union bpf_attr *attr) 1840f4324551SDaniel Mack { 18417f677633SAlexei Starovoitov enum bpf_prog_type ptype; 1842f4324551SDaniel Mack struct bpf_prog *prog; 18437f677633SAlexei Starovoitov int ret; 1844f4324551SDaniel Mack 1845f4324551SDaniel Mack if (!capable(CAP_NET_ADMIN)) 1846f4324551SDaniel Mack return -EPERM; 1847f4324551SDaniel Mack 1848f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_ATTACH)) 1849f4324551SDaniel Mack return -EINVAL; 1850f4324551SDaniel Mack 1851324bda9eSAlexei Starovoitov if (attr->attach_flags & ~BPF_F_ATTACH_MASK) 18527f677633SAlexei Starovoitov return -EINVAL; 18537f677633SAlexei Starovoitov 1854f4324551SDaniel Mack switch (attr->attach_type) { 1855f4324551SDaniel Mack case BPF_CGROUP_INET_INGRESS: 1856f4324551SDaniel Mack case BPF_CGROUP_INET_EGRESS: 1857b2cd1257SDavid Ahern ptype = BPF_PROG_TYPE_CGROUP_SKB; 1858b2cd1257SDavid Ahern break; 185961023658SDavid Ahern case BPF_CGROUP_INET_SOCK_CREATE: 1860aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1861aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 186261023658SDavid Ahern ptype = BPF_PROG_TYPE_CGROUP_SOCK; 186361023658SDavid Ahern break; 18644fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 18654fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1866d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1867d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 18681cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 18691cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 18704fbac77dSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 18714fbac77dSAndrey Ignatov break; 187240304b2aSLawrence Brakmo case BPF_CGROUP_SOCK_OPS: 187340304b2aSLawrence Brakmo ptype = BPF_PROG_TYPE_SOCK_OPS; 187440304b2aSLawrence Brakmo break; 1875ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 1876ebc614f6SRoman Gushchin ptype = BPF_PROG_TYPE_CGROUP_DEVICE; 1877ebc614f6SRoman Gushchin break; 18784f738adbSJohn Fastabend case BPF_SK_MSG_VERDICT: 1879fdb5c453SSean Young ptype = BPF_PROG_TYPE_SK_MSG; 1880fdb5c453SSean Young break; 1881464bc0fdSJohn Fastabend case BPF_SK_SKB_STREAM_PARSER: 1882464bc0fdSJohn Fastabend case BPF_SK_SKB_STREAM_VERDICT: 1883fdb5c453SSean Young ptype = BPF_PROG_TYPE_SK_SKB; 1884fdb5c453SSean Young break; 1885f4364dcfSSean Young case BPF_LIRC_MODE2: 1886fdb5c453SSean Young ptype = BPF_PROG_TYPE_LIRC_MODE2; 1887fdb5c453SSean Young break; 1888d58e468bSPetar Penkov case BPF_FLOW_DISSECTOR: 1889d58e468bSPetar Penkov ptype = BPF_PROG_TYPE_FLOW_DISSECTOR; 1890d58e468bSPetar Penkov break; 1891*7b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 1892*7b146cebSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SYSCTL; 1893*7b146cebSAndrey Ignatov break; 1894b2cd1257SDavid Ahern default: 1895b2cd1257SDavid Ahern return -EINVAL; 1896b2cd1257SDavid Ahern } 1897b2cd1257SDavid Ahern 1898b2cd1257SDavid Ahern prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 1899f4324551SDaniel Mack if (IS_ERR(prog)) 1900f4324551SDaniel Mack return PTR_ERR(prog); 1901f4324551SDaniel Mack 19025e43f899SAndrey Ignatov if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) { 19035e43f899SAndrey Ignatov bpf_prog_put(prog); 19045e43f899SAndrey Ignatov return -EINVAL; 19055e43f899SAndrey Ignatov } 19065e43f899SAndrey Ignatov 1907fdb5c453SSean Young switch (ptype) { 1908fdb5c453SSean Young case BPF_PROG_TYPE_SK_SKB: 1909fdb5c453SSean Young case BPF_PROG_TYPE_SK_MSG: 1910604326b4SDaniel Borkmann ret = sock_map_get_from_fd(attr, prog); 1911fdb5c453SSean Young break; 1912fdb5c453SSean Young case BPF_PROG_TYPE_LIRC_MODE2: 1913fdb5c453SSean Young ret = lirc_prog_attach(attr, prog); 1914fdb5c453SSean Young break; 1915d58e468bSPetar Penkov case BPF_PROG_TYPE_FLOW_DISSECTOR: 1916d58e468bSPetar Penkov ret = skb_flow_dissector_bpf_prog_attach(attr, prog); 1917d58e468bSPetar Penkov break; 1918fdb5c453SSean Young default: 1919fdb5c453SSean Young ret = cgroup_bpf_prog_attach(attr, ptype, prog); 1920f4324551SDaniel Mack } 1921f4324551SDaniel Mack 19227f677633SAlexei Starovoitov if (ret) 19237f677633SAlexei Starovoitov bpf_prog_put(prog); 19247f677633SAlexei Starovoitov return ret; 1925f4324551SDaniel Mack } 1926f4324551SDaniel Mack 1927f4324551SDaniel Mack #define BPF_PROG_DETACH_LAST_FIELD attach_type 1928f4324551SDaniel Mack 1929f4324551SDaniel Mack static int bpf_prog_detach(const union bpf_attr *attr) 1930f4324551SDaniel Mack { 1931324bda9eSAlexei Starovoitov enum bpf_prog_type ptype; 1932f4324551SDaniel Mack 1933f4324551SDaniel Mack if (!capable(CAP_NET_ADMIN)) 1934f4324551SDaniel Mack return -EPERM; 1935f4324551SDaniel Mack 1936f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_DETACH)) 1937f4324551SDaniel Mack return -EINVAL; 1938f4324551SDaniel Mack 1939f4324551SDaniel Mack switch (attr->attach_type) { 1940f4324551SDaniel Mack case BPF_CGROUP_INET_INGRESS: 1941f4324551SDaniel Mack case BPF_CGROUP_INET_EGRESS: 1942324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_CGROUP_SKB; 1943324bda9eSAlexei Starovoitov break; 194461023658SDavid Ahern case BPF_CGROUP_INET_SOCK_CREATE: 1945aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1946aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 1947324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_CGROUP_SOCK; 1948324bda9eSAlexei Starovoitov break; 19494fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 19504fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1951d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1952d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 19531cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 19541cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 19554fbac77dSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 19564fbac77dSAndrey Ignatov break; 195740304b2aSLawrence Brakmo case BPF_CGROUP_SOCK_OPS: 1958324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_SOCK_OPS; 1959f4324551SDaniel Mack break; 1960ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 1961ebc614f6SRoman Gushchin ptype = BPF_PROG_TYPE_CGROUP_DEVICE; 1962ebc614f6SRoman Gushchin break; 19634f738adbSJohn Fastabend case BPF_SK_MSG_VERDICT: 1964604326b4SDaniel Borkmann return sock_map_get_from_fd(attr, NULL); 19655a67da2aSJohn Fastabend case BPF_SK_SKB_STREAM_PARSER: 19665a67da2aSJohn Fastabend case BPF_SK_SKB_STREAM_VERDICT: 1967604326b4SDaniel Borkmann return sock_map_get_from_fd(attr, NULL); 1968f4364dcfSSean Young case BPF_LIRC_MODE2: 1969f4364dcfSSean Young return lirc_prog_detach(attr); 1970d58e468bSPetar Penkov case BPF_FLOW_DISSECTOR: 1971d58e468bSPetar Penkov return skb_flow_dissector_bpf_prog_detach(attr); 1972*7b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 1973*7b146cebSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SYSCTL; 1974*7b146cebSAndrey Ignatov break; 1975f4324551SDaniel Mack default: 1976f4324551SDaniel Mack return -EINVAL; 1977f4324551SDaniel Mack } 1978f4324551SDaniel Mack 1979fdb5c453SSean Young return cgroup_bpf_prog_detach(attr, ptype); 1980f4324551SDaniel Mack } 198140304b2aSLawrence Brakmo 1982468e2f64SAlexei Starovoitov #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt 1983468e2f64SAlexei Starovoitov 1984468e2f64SAlexei Starovoitov static int bpf_prog_query(const union bpf_attr *attr, 1985468e2f64SAlexei Starovoitov union bpf_attr __user *uattr) 1986468e2f64SAlexei Starovoitov { 1987468e2f64SAlexei Starovoitov if (!capable(CAP_NET_ADMIN)) 1988468e2f64SAlexei Starovoitov return -EPERM; 1989468e2f64SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_QUERY)) 1990468e2f64SAlexei Starovoitov return -EINVAL; 1991468e2f64SAlexei Starovoitov if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE) 1992468e2f64SAlexei Starovoitov return -EINVAL; 1993468e2f64SAlexei Starovoitov 1994468e2f64SAlexei Starovoitov switch (attr->query.attach_type) { 1995468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_INGRESS: 1996468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_EGRESS: 1997468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_SOCK_CREATE: 19984fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 19994fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 2000aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 2001aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 2002d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 2003d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 20041cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 20051cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 2006468e2f64SAlexei Starovoitov case BPF_CGROUP_SOCK_OPS: 2007ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 2008*7b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 2009468e2f64SAlexei Starovoitov break; 2010f4364dcfSSean Young case BPF_LIRC_MODE2: 2011f4364dcfSSean Young return lirc_prog_query(attr, uattr); 2012468e2f64SAlexei Starovoitov default: 2013468e2f64SAlexei Starovoitov return -EINVAL; 2014468e2f64SAlexei Starovoitov } 2015fdb5c453SSean Young 2016fdb5c453SSean Young return cgroup_bpf_prog_query(attr, uattr); 2017468e2f64SAlexei Starovoitov } 2018f4324551SDaniel Mack 2019b0b9395dSStanislav Fomichev #define BPF_PROG_TEST_RUN_LAST_FIELD test.ctx_out 20201cf1cae9SAlexei Starovoitov 20211cf1cae9SAlexei Starovoitov static int bpf_prog_test_run(const union bpf_attr *attr, 20221cf1cae9SAlexei Starovoitov union bpf_attr __user *uattr) 20231cf1cae9SAlexei Starovoitov { 20241cf1cae9SAlexei Starovoitov struct bpf_prog *prog; 20251cf1cae9SAlexei Starovoitov int ret = -ENOTSUPP; 20261cf1cae9SAlexei Starovoitov 202761f3c964SAlexei Starovoitov if (!capable(CAP_SYS_ADMIN)) 202861f3c964SAlexei Starovoitov return -EPERM; 20291cf1cae9SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 20301cf1cae9SAlexei Starovoitov return -EINVAL; 20311cf1cae9SAlexei Starovoitov 2032b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_in && !attr->test.ctx_in) || 2033b0b9395dSStanislav Fomichev (!attr->test.ctx_size_in && attr->test.ctx_in)) 2034b0b9395dSStanislav Fomichev return -EINVAL; 2035b0b9395dSStanislav Fomichev 2036b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_out && !attr->test.ctx_out) || 2037b0b9395dSStanislav Fomichev (!attr->test.ctx_size_out && attr->test.ctx_out)) 2038b0b9395dSStanislav Fomichev return -EINVAL; 2039b0b9395dSStanislav Fomichev 20401cf1cae9SAlexei Starovoitov prog = bpf_prog_get(attr->test.prog_fd); 20411cf1cae9SAlexei Starovoitov if (IS_ERR(prog)) 20421cf1cae9SAlexei Starovoitov return PTR_ERR(prog); 20431cf1cae9SAlexei Starovoitov 20441cf1cae9SAlexei Starovoitov if (prog->aux->ops->test_run) 20451cf1cae9SAlexei Starovoitov ret = prog->aux->ops->test_run(prog, attr, uattr); 20461cf1cae9SAlexei Starovoitov 20471cf1cae9SAlexei Starovoitov bpf_prog_put(prog); 20481cf1cae9SAlexei Starovoitov return ret; 20491cf1cae9SAlexei Starovoitov } 20501cf1cae9SAlexei Starovoitov 205134ad5580SMartin KaFai Lau #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 205234ad5580SMartin KaFai Lau 205334ad5580SMartin KaFai Lau static int bpf_obj_get_next_id(const union bpf_attr *attr, 205434ad5580SMartin KaFai Lau union bpf_attr __user *uattr, 205534ad5580SMartin KaFai Lau struct idr *idr, 205634ad5580SMartin KaFai Lau spinlock_t *lock) 205734ad5580SMartin KaFai Lau { 205834ad5580SMartin KaFai Lau u32 next_id = attr->start_id; 205934ad5580SMartin KaFai Lau int err = 0; 206034ad5580SMartin KaFai Lau 206134ad5580SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 206234ad5580SMartin KaFai Lau return -EINVAL; 206334ad5580SMartin KaFai Lau 206434ad5580SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 206534ad5580SMartin KaFai Lau return -EPERM; 206634ad5580SMartin KaFai Lau 206734ad5580SMartin KaFai Lau next_id++; 206834ad5580SMartin KaFai Lau spin_lock_bh(lock); 206934ad5580SMartin KaFai Lau if (!idr_get_next(idr, &next_id)) 207034ad5580SMartin KaFai Lau err = -ENOENT; 207134ad5580SMartin KaFai Lau spin_unlock_bh(lock); 207234ad5580SMartin KaFai Lau 207334ad5580SMartin KaFai Lau if (!err) 207434ad5580SMartin KaFai Lau err = put_user(next_id, &uattr->next_id); 207534ad5580SMartin KaFai Lau 207634ad5580SMartin KaFai Lau return err; 207734ad5580SMartin KaFai Lau } 207834ad5580SMartin KaFai Lau 2079b16d9aa4SMartin KaFai Lau #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 2080b16d9aa4SMartin KaFai Lau 2081b16d9aa4SMartin KaFai Lau static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 2082b16d9aa4SMartin KaFai Lau { 2083b16d9aa4SMartin KaFai Lau struct bpf_prog *prog; 2084b16d9aa4SMartin KaFai Lau u32 id = attr->prog_id; 2085b16d9aa4SMartin KaFai Lau int fd; 2086b16d9aa4SMartin KaFai Lau 2087b16d9aa4SMartin KaFai Lau if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 2088b16d9aa4SMartin KaFai Lau return -EINVAL; 2089b16d9aa4SMartin KaFai Lau 2090b16d9aa4SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 2091b16d9aa4SMartin KaFai Lau return -EPERM; 2092b16d9aa4SMartin KaFai Lau 2093b16d9aa4SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 2094b16d9aa4SMartin KaFai Lau prog = idr_find(&prog_idr, id); 2095b16d9aa4SMartin KaFai Lau if (prog) 2096b16d9aa4SMartin KaFai Lau prog = bpf_prog_inc_not_zero(prog); 2097b16d9aa4SMartin KaFai Lau else 2098b16d9aa4SMartin KaFai Lau prog = ERR_PTR(-ENOENT); 2099b16d9aa4SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 2100b16d9aa4SMartin KaFai Lau 2101b16d9aa4SMartin KaFai Lau if (IS_ERR(prog)) 2102b16d9aa4SMartin KaFai Lau return PTR_ERR(prog); 2103b16d9aa4SMartin KaFai Lau 2104b16d9aa4SMartin KaFai Lau fd = bpf_prog_new_fd(prog); 2105b16d9aa4SMartin KaFai Lau if (fd < 0) 2106b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 2107b16d9aa4SMartin KaFai Lau 2108b16d9aa4SMartin KaFai Lau return fd; 2109b16d9aa4SMartin KaFai Lau } 2110b16d9aa4SMartin KaFai Lau 21116e71b04aSChenbo Feng #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags 2112bd5f5f4eSMartin KaFai Lau 2113bd5f5f4eSMartin KaFai Lau static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 2114bd5f5f4eSMartin KaFai Lau { 2115bd5f5f4eSMartin KaFai Lau struct bpf_map *map; 2116bd5f5f4eSMartin KaFai Lau u32 id = attr->map_id; 21176e71b04aSChenbo Feng int f_flags; 2118bd5f5f4eSMartin KaFai Lau int fd; 2119bd5f5f4eSMartin KaFai Lau 21206e71b04aSChenbo Feng if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || 21216e71b04aSChenbo Feng attr->open_flags & ~BPF_OBJ_FLAG_MASK) 2122bd5f5f4eSMartin KaFai Lau return -EINVAL; 2123bd5f5f4eSMartin KaFai Lau 2124bd5f5f4eSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 2125bd5f5f4eSMartin KaFai Lau return -EPERM; 2126bd5f5f4eSMartin KaFai Lau 21276e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->open_flags); 21286e71b04aSChenbo Feng if (f_flags < 0) 21296e71b04aSChenbo Feng return f_flags; 21306e71b04aSChenbo Feng 2131bd5f5f4eSMartin KaFai Lau spin_lock_bh(&map_idr_lock); 2132bd5f5f4eSMartin KaFai Lau map = idr_find(&map_idr, id); 2133bd5f5f4eSMartin KaFai Lau if (map) 2134bd5f5f4eSMartin KaFai Lau map = bpf_map_inc_not_zero(map, true); 2135bd5f5f4eSMartin KaFai Lau else 2136bd5f5f4eSMartin KaFai Lau map = ERR_PTR(-ENOENT); 2137bd5f5f4eSMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 2138bd5f5f4eSMartin KaFai Lau 2139bd5f5f4eSMartin KaFai Lau if (IS_ERR(map)) 2140bd5f5f4eSMartin KaFai Lau return PTR_ERR(map); 2141bd5f5f4eSMartin KaFai Lau 21426e71b04aSChenbo Feng fd = bpf_map_new_fd(map, f_flags); 2143bd5f5f4eSMartin KaFai Lau if (fd < 0) 2144781e6282SPeng Sun bpf_map_put_with_uref(map); 2145bd5f5f4eSMartin KaFai Lau 2146bd5f5f4eSMartin KaFai Lau return fd; 2147bd5f5f4eSMartin KaFai Lau } 2148bd5f5f4eSMartin KaFai Lau 21497105e828SDaniel Borkmann static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, 2150d8eca5bbSDaniel Borkmann unsigned long addr, u32 *off, 2151d8eca5bbSDaniel Borkmann u32 *type) 21527105e828SDaniel Borkmann { 2153d8eca5bbSDaniel Borkmann const struct bpf_map *map; 21547105e828SDaniel Borkmann int i; 21557105e828SDaniel Borkmann 2156d8eca5bbSDaniel Borkmann for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { 2157d8eca5bbSDaniel Borkmann map = prog->aux->used_maps[i]; 2158d8eca5bbSDaniel Borkmann if (map == (void *)addr) { 2159d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_FD; 2160d8eca5bbSDaniel Borkmann return map; 2161d8eca5bbSDaniel Borkmann } 2162d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta) 2163d8eca5bbSDaniel Borkmann continue; 2164d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta(map, addr, off)) { 2165d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_VALUE; 2166d8eca5bbSDaniel Borkmann return map; 2167d8eca5bbSDaniel Borkmann } 2168d8eca5bbSDaniel Borkmann } 2169d8eca5bbSDaniel Borkmann 21707105e828SDaniel Borkmann return NULL; 21717105e828SDaniel Borkmann } 21727105e828SDaniel Borkmann 21737105e828SDaniel Borkmann static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog) 21747105e828SDaniel Borkmann { 21757105e828SDaniel Borkmann const struct bpf_map *map; 21767105e828SDaniel Borkmann struct bpf_insn *insns; 2177d8eca5bbSDaniel Borkmann u32 off, type; 21787105e828SDaniel Borkmann u64 imm; 21797105e828SDaniel Borkmann int i; 21807105e828SDaniel Borkmann 21817105e828SDaniel Borkmann insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), 21827105e828SDaniel Borkmann GFP_USER); 21837105e828SDaniel Borkmann if (!insns) 21847105e828SDaniel Borkmann return insns; 21857105e828SDaniel Borkmann 21867105e828SDaniel Borkmann for (i = 0; i < prog->len; i++) { 21877105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_TAIL_CALL)) { 21887105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 21897105e828SDaniel Borkmann insns[i].imm = BPF_FUNC_tail_call; 21907105e828SDaniel Borkmann /* fall-through */ 21917105e828SDaniel Borkmann } 21927105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_CALL) || 21937105e828SDaniel Borkmann insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) { 21947105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) 21957105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 21967105e828SDaniel Borkmann if (!bpf_dump_raw_ok()) 21977105e828SDaniel Borkmann insns[i].imm = 0; 21987105e828SDaniel Borkmann continue; 21997105e828SDaniel Borkmann } 22007105e828SDaniel Borkmann 22017105e828SDaniel Borkmann if (insns[i].code != (BPF_LD | BPF_IMM | BPF_DW)) 22027105e828SDaniel Borkmann continue; 22037105e828SDaniel Borkmann 22047105e828SDaniel Borkmann imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; 2205d8eca5bbSDaniel Borkmann map = bpf_map_from_imm(prog, imm, &off, &type); 22067105e828SDaniel Borkmann if (map) { 2207d8eca5bbSDaniel Borkmann insns[i].src_reg = type; 22087105e828SDaniel Borkmann insns[i].imm = map->id; 2209d8eca5bbSDaniel Borkmann insns[i + 1].imm = off; 22107105e828SDaniel Borkmann continue; 22117105e828SDaniel Borkmann } 22127105e828SDaniel Borkmann } 22137105e828SDaniel Borkmann 22147105e828SDaniel Borkmann return insns; 22157105e828SDaniel Borkmann } 22167105e828SDaniel Borkmann 2217c454a46bSMartin KaFai Lau static int set_info_rec_size(struct bpf_prog_info *info) 2218c454a46bSMartin KaFai Lau { 2219c454a46bSMartin KaFai Lau /* 2220c454a46bSMartin KaFai Lau * Ensure info.*_rec_size is the same as kernel expected size 2221c454a46bSMartin KaFai Lau * 2222c454a46bSMartin KaFai Lau * or 2223c454a46bSMartin KaFai Lau * 2224c454a46bSMartin KaFai Lau * Only allow zero *_rec_size if both _rec_size and _cnt are 2225c454a46bSMartin KaFai Lau * zero. In this case, the kernel will set the expected 2226c454a46bSMartin KaFai Lau * _rec_size back to the info. 2227c454a46bSMartin KaFai Lau */ 2228c454a46bSMartin KaFai Lau 222911d8b82dSYonghong Song if ((info->nr_func_info || info->func_info_rec_size) && 2230c454a46bSMartin KaFai Lau info->func_info_rec_size != sizeof(struct bpf_func_info)) 2231c454a46bSMartin KaFai Lau return -EINVAL; 2232c454a46bSMartin KaFai Lau 223311d8b82dSYonghong Song if ((info->nr_line_info || info->line_info_rec_size) && 2234c454a46bSMartin KaFai Lau info->line_info_rec_size != sizeof(struct bpf_line_info)) 2235c454a46bSMartin KaFai Lau return -EINVAL; 2236c454a46bSMartin KaFai Lau 223711d8b82dSYonghong Song if ((info->nr_jited_line_info || info->jited_line_info_rec_size) && 2238c454a46bSMartin KaFai Lau info->jited_line_info_rec_size != sizeof(__u64)) 2239c454a46bSMartin KaFai Lau return -EINVAL; 2240c454a46bSMartin KaFai Lau 2241c454a46bSMartin KaFai Lau info->func_info_rec_size = sizeof(struct bpf_func_info); 2242c454a46bSMartin KaFai Lau info->line_info_rec_size = sizeof(struct bpf_line_info); 2243c454a46bSMartin KaFai Lau info->jited_line_info_rec_size = sizeof(__u64); 2244c454a46bSMartin KaFai Lau 2245c454a46bSMartin KaFai Lau return 0; 2246c454a46bSMartin KaFai Lau } 2247c454a46bSMartin KaFai Lau 22481e270976SMartin KaFai Lau static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, 22491e270976SMartin KaFai Lau const union bpf_attr *attr, 22501e270976SMartin KaFai Lau union bpf_attr __user *uattr) 22511e270976SMartin KaFai Lau { 22521e270976SMartin KaFai Lau struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 22531e270976SMartin KaFai Lau struct bpf_prog_info info = {}; 22541e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 22555f8f8b93SAlexei Starovoitov struct bpf_prog_stats stats; 22561e270976SMartin KaFai Lau char __user *uinsns; 22571e270976SMartin KaFai Lau u32 ulen; 22581e270976SMartin KaFai Lau int err; 22591e270976SMartin KaFai Lau 2260dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 22611e270976SMartin KaFai Lau if (err) 22621e270976SMartin KaFai Lau return err; 22631e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 22641e270976SMartin KaFai Lau 22651e270976SMartin KaFai Lau if (copy_from_user(&info, uinfo, info_len)) 226689b09689SDaniel Borkmann return -EFAULT; 22671e270976SMartin KaFai Lau 22681e270976SMartin KaFai Lau info.type = prog->type; 22691e270976SMartin KaFai Lau info.id = prog->aux->id; 2270cb4d2b3fSMartin KaFai Lau info.load_time = prog->aux->load_time; 2271cb4d2b3fSMartin KaFai Lau info.created_by_uid = from_kuid_munged(current_user_ns(), 2272cb4d2b3fSMartin KaFai Lau prog->aux->user->uid); 2273b85fab0eSJiri Olsa info.gpl_compatible = prog->gpl_compatible; 22741e270976SMartin KaFai Lau 22751e270976SMartin KaFai Lau memcpy(info.tag, prog->tag, sizeof(prog->tag)); 2276cb4d2b3fSMartin KaFai Lau memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); 2277cb4d2b3fSMartin KaFai Lau 2278cb4d2b3fSMartin KaFai Lau ulen = info.nr_map_ids; 2279cb4d2b3fSMartin KaFai Lau info.nr_map_ids = prog->aux->used_map_cnt; 2280cb4d2b3fSMartin KaFai Lau ulen = min_t(u32, info.nr_map_ids, ulen); 2281cb4d2b3fSMartin KaFai Lau if (ulen) { 2282721e08daSMartin KaFai Lau u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); 2283cb4d2b3fSMartin KaFai Lau u32 i; 2284cb4d2b3fSMartin KaFai Lau 2285cb4d2b3fSMartin KaFai Lau for (i = 0; i < ulen; i++) 2286cb4d2b3fSMartin KaFai Lau if (put_user(prog->aux->used_maps[i]->id, 2287cb4d2b3fSMartin KaFai Lau &user_map_ids[i])) 2288cb4d2b3fSMartin KaFai Lau return -EFAULT; 2289cb4d2b3fSMartin KaFai Lau } 22901e270976SMartin KaFai Lau 2291c454a46bSMartin KaFai Lau err = set_info_rec_size(&info); 2292c454a46bSMartin KaFai Lau if (err) 2293c454a46bSMartin KaFai Lau return err; 22947337224fSMartin KaFai Lau 22955f8f8b93SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 22965f8f8b93SAlexei Starovoitov info.run_time_ns = stats.nsecs; 22975f8f8b93SAlexei Starovoitov info.run_cnt = stats.cnt; 22985f8f8b93SAlexei Starovoitov 22991e270976SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) { 23001e270976SMartin KaFai Lau info.jited_prog_len = 0; 23011e270976SMartin KaFai Lau info.xlated_prog_len = 0; 2302dbecd738SSandipan Das info.nr_jited_ksyms = 0; 230328c2fae7SDaniel Borkmann info.nr_jited_func_lens = 0; 230411d8b82dSYonghong Song info.nr_func_info = 0; 230511d8b82dSYonghong Song info.nr_line_info = 0; 230611d8b82dSYonghong Song info.nr_jited_line_info = 0; 23071e270976SMartin KaFai Lau goto done; 23081e270976SMartin KaFai Lau } 23091e270976SMartin KaFai Lau 23101e270976SMartin KaFai Lau ulen = info.xlated_prog_len; 23119975a54bSDaniel Borkmann info.xlated_prog_len = bpf_prog_insn_size(prog); 23121e270976SMartin KaFai Lau if (info.xlated_prog_len && ulen) { 23137105e828SDaniel Borkmann struct bpf_insn *insns_sanitized; 23147105e828SDaniel Borkmann bool fault; 23157105e828SDaniel Borkmann 23167105e828SDaniel Borkmann if (prog->blinded && !bpf_dump_raw_ok()) { 23177105e828SDaniel Borkmann info.xlated_prog_insns = 0; 23187105e828SDaniel Borkmann goto done; 23197105e828SDaniel Borkmann } 23207105e828SDaniel Borkmann insns_sanitized = bpf_insn_prepare_dump(prog); 23217105e828SDaniel Borkmann if (!insns_sanitized) 23227105e828SDaniel Borkmann return -ENOMEM; 23231e270976SMartin KaFai Lau uinsns = u64_to_user_ptr(info.xlated_prog_insns); 23241e270976SMartin KaFai Lau ulen = min_t(u32, info.xlated_prog_len, ulen); 23257105e828SDaniel Borkmann fault = copy_to_user(uinsns, insns_sanitized, ulen); 23267105e828SDaniel Borkmann kfree(insns_sanitized); 23277105e828SDaniel Borkmann if (fault) 23281e270976SMartin KaFai Lau return -EFAULT; 23291e270976SMartin KaFai Lau } 23301e270976SMartin KaFai Lau 2331675fc275SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 2332675fc275SJakub Kicinski err = bpf_prog_offload_info_fill(&info, prog); 2333675fc275SJakub Kicinski if (err) 2334675fc275SJakub Kicinski return err; 2335fcfb126dSJiong Wang goto done; 2336fcfb126dSJiong Wang } 2337fcfb126dSJiong Wang 2338fcfb126dSJiong Wang /* NOTE: the following code is supposed to be skipped for offload. 2339fcfb126dSJiong Wang * bpf_prog_offload_info_fill() is the place to fill similar fields 2340fcfb126dSJiong Wang * for offload. 2341fcfb126dSJiong Wang */ 2342fcfb126dSJiong Wang ulen = info.jited_prog_len; 23434d56a76eSSandipan Das if (prog->aux->func_cnt) { 23444d56a76eSSandipan Das u32 i; 23454d56a76eSSandipan Das 23464d56a76eSSandipan Das info.jited_prog_len = 0; 23474d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) 23484d56a76eSSandipan Das info.jited_prog_len += prog->aux->func[i]->jited_len; 23494d56a76eSSandipan Das } else { 2350fcfb126dSJiong Wang info.jited_prog_len = prog->jited_len; 23514d56a76eSSandipan Das } 23524d56a76eSSandipan Das 2353fcfb126dSJiong Wang if (info.jited_prog_len && ulen) { 2354fcfb126dSJiong Wang if (bpf_dump_raw_ok()) { 2355fcfb126dSJiong Wang uinsns = u64_to_user_ptr(info.jited_prog_insns); 2356fcfb126dSJiong Wang ulen = min_t(u32, info.jited_prog_len, ulen); 23574d56a76eSSandipan Das 23584d56a76eSSandipan Das /* for multi-function programs, copy the JITed 23594d56a76eSSandipan Das * instructions for all the functions 23604d56a76eSSandipan Das */ 23614d56a76eSSandipan Das if (prog->aux->func_cnt) { 23624d56a76eSSandipan Das u32 len, free, i; 23634d56a76eSSandipan Das u8 *img; 23644d56a76eSSandipan Das 23654d56a76eSSandipan Das free = ulen; 23664d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) { 23674d56a76eSSandipan Das len = prog->aux->func[i]->jited_len; 23684d56a76eSSandipan Das len = min_t(u32, len, free); 23694d56a76eSSandipan Das img = (u8 *) prog->aux->func[i]->bpf_func; 23704d56a76eSSandipan Das if (copy_to_user(uinsns, img, len)) 23714d56a76eSSandipan Das return -EFAULT; 23724d56a76eSSandipan Das uinsns += len; 23734d56a76eSSandipan Das free -= len; 23744d56a76eSSandipan Das if (!free) 23754d56a76eSSandipan Das break; 23764d56a76eSSandipan Das } 23774d56a76eSSandipan Das } else { 2378fcfb126dSJiong Wang if (copy_to_user(uinsns, prog->bpf_func, ulen)) 2379fcfb126dSJiong Wang return -EFAULT; 23804d56a76eSSandipan Das } 2381fcfb126dSJiong Wang } else { 2382fcfb126dSJiong Wang info.jited_prog_insns = 0; 2383fcfb126dSJiong Wang } 2384675fc275SJakub Kicinski } 2385675fc275SJakub Kicinski 2386dbecd738SSandipan Das ulen = info.nr_jited_ksyms; 2387ff1889fcSSong Liu info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; 23887a5725ddSSong Liu if (ulen) { 2389dbecd738SSandipan Das if (bpf_dump_raw_ok()) { 2390ff1889fcSSong Liu unsigned long ksym_addr; 2391dbecd738SSandipan Das u64 __user *user_ksyms; 2392dbecd738SSandipan Das u32 i; 2393dbecd738SSandipan Das 2394dbecd738SSandipan Das /* copy the address of the kernel symbol 2395dbecd738SSandipan Das * corresponding to each function 2396dbecd738SSandipan Das */ 2397dbecd738SSandipan Das ulen = min_t(u32, info.nr_jited_ksyms, ulen); 2398dbecd738SSandipan Das user_ksyms = u64_to_user_ptr(info.jited_ksyms); 2399ff1889fcSSong Liu if (prog->aux->func_cnt) { 2400dbecd738SSandipan Das for (i = 0; i < ulen; i++) { 2401ff1889fcSSong Liu ksym_addr = (unsigned long) 2402ff1889fcSSong Liu prog->aux->func[i]->bpf_func; 2403ff1889fcSSong Liu if (put_user((u64) ksym_addr, 2404ff1889fcSSong Liu &user_ksyms[i])) 2405ff1889fcSSong Liu return -EFAULT; 2406ff1889fcSSong Liu } 2407ff1889fcSSong Liu } else { 2408ff1889fcSSong Liu ksym_addr = (unsigned long) prog->bpf_func; 2409ff1889fcSSong Liu if (put_user((u64) ksym_addr, &user_ksyms[0])) 2410dbecd738SSandipan Das return -EFAULT; 2411dbecd738SSandipan Das } 2412dbecd738SSandipan Das } else { 2413dbecd738SSandipan Das info.jited_ksyms = 0; 2414dbecd738SSandipan Das } 2415dbecd738SSandipan Das } 2416dbecd738SSandipan Das 2417815581c1SSandipan Das ulen = info.nr_jited_func_lens; 2418ff1889fcSSong Liu info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; 24197a5725ddSSong Liu if (ulen) { 2420815581c1SSandipan Das if (bpf_dump_raw_ok()) { 2421815581c1SSandipan Das u32 __user *user_lens; 2422815581c1SSandipan Das u32 func_len, i; 2423815581c1SSandipan Das 2424815581c1SSandipan Das /* copy the JITed image lengths for each function */ 2425815581c1SSandipan Das ulen = min_t(u32, info.nr_jited_func_lens, ulen); 2426815581c1SSandipan Das user_lens = u64_to_user_ptr(info.jited_func_lens); 2427ff1889fcSSong Liu if (prog->aux->func_cnt) { 2428815581c1SSandipan Das for (i = 0; i < ulen; i++) { 2429ff1889fcSSong Liu func_len = 2430ff1889fcSSong Liu prog->aux->func[i]->jited_len; 2431815581c1SSandipan Das if (put_user(func_len, &user_lens[i])) 2432815581c1SSandipan Das return -EFAULT; 2433815581c1SSandipan Das } 2434815581c1SSandipan Das } else { 2435ff1889fcSSong Liu func_len = prog->jited_len; 2436ff1889fcSSong Liu if (put_user(func_len, &user_lens[0])) 2437ff1889fcSSong Liu return -EFAULT; 2438ff1889fcSSong Liu } 2439ff1889fcSSong Liu } else { 2440815581c1SSandipan Das info.jited_func_lens = 0; 2441815581c1SSandipan Das } 2442815581c1SSandipan Das } 2443815581c1SSandipan Das 24447337224fSMartin KaFai Lau if (prog->aux->btf) 2445838e9690SYonghong Song info.btf_id = btf_id(prog->aux->btf); 2446838e9690SYonghong Song 244711d8b82dSYonghong Song ulen = info.nr_func_info; 244811d8b82dSYonghong Song info.nr_func_info = prog->aux->func_info_cnt; 244911d8b82dSYonghong Song if (info.nr_func_info && ulen) { 2450838e9690SYonghong Song char __user *user_finfo; 2451838e9690SYonghong Song 2452838e9690SYonghong Song user_finfo = u64_to_user_ptr(info.func_info); 245311d8b82dSYonghong Song ulen = min_t(u32, info.nr_func_info, ulen); 2454ba64e7d8SYonghong Song if (copy_to_user(user_finfo, prog->aux->func_info, 24557337224fSMartin KaFai Lau info.func_info_rec_size * ulen)) 2456838e9690SYonghong Song return -EFAULT; 2457838e9690SYonghong Song } 2458838e9690SYonghong Song 245911d8b82dSYonghong Song ulen = info.nr_line_info; 246011d8b82dSYonghong Song info.nr_line_info = prog->aux->nr_linfo; 246111d8b82dSYonghong Song if (info.nr_line_info && ulen) { 2462c454a46bSMartin KaFai Lau __u8 __user *user_linfo; 2463c454a46bSMartin KaFai Lau 2464c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.line_info); 246511d8b82dSYonghong Song ulen = min_t(u32, info.nr_line_info, ulen); 2466c454a46bSMartin KaFai Lau if (copy_to_user(user_linfo, prog->aux->linfo, 2467c454a46bSMartin KaFai Lau info.line_info_rec_size * ulen)) 2468c454a46bSMartin KaFai Lau return -EFAULT; 2469c454a46bSMartin KaFai Lau } 2470c454a46bSMartin KaFai Lau 247111d8b82dSYonghong Song ulen = info.nr_jited_line_info; 2472c454a46bSMartin KaFai Lau if (prog->aux->jited_linfo) 247311d8b82dSYonghong Song info.nr_jited_line_info = prog->aux->nr_linfo; 2474c454a46bSMartin KaFai Lau else 247511d8b82dSYonghong Song info.nr_jited_line_info = 0; 247611d8b82dSYonghong Song if (info.nr_jited_line_info && ulen) { 2477c454a46bSMartin KaFai Lau if (bpf_dump_raw_ok()) { 2478c454a46bSMartin KaFai Lau __u64 __user *user_linfo; 2479c454a46bSMartin KaFai Lau u32 i; 2480c454a46bSMartin KaFai Lau 2481c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.jited_line_info); 248211d8b82dSYonghong Song ulen = min_t(u32, info.nr_jited_line_info, ulen); 2483c454a46bSMartin KaFai Lau for (i = 0; i < ulen; i++) { 2484c454a46bSMartin KaFai Lau if (put_user((__u64)(long)prog->aux->jited_linfo[i], 2485c454a46bSMartin KaFai Lau &user_linfo[i])) 2486c454a46bSMartin KaFai Lau return -EFAULT; 2487c454a46bSMartin KaFai Lau } 2488c454a46bSMartin KaFai Lau } else { 2489c454a46bSMartin KaFai Lau info.jited_line_info = 0; 2490c454a46bSMartin KaFai Lau } 2491c454a46bSMartin KaFai Lau } 2492c454a46bSMartin KaFai Lau 2493c872bdb3SSong Liu ulen = info.nr_prog_tags; 2494c872bdb3SSong Liu info.nr_prog_tags = prog->aux->func_cnt ? : 1; 2495c872bdb3SSong Liu if (ulen) { 2496c872bdb3SSong Liu __u8 __user (*user_prog_tags)[BPF_TAG_SIZE]; 2497c872bdb3SSong Liu u32 i; 2498c872bdb3SSong Liu 2499c872bdb3SSong Liu user_prog_tags = u64_to_user_ptr(info.prog_tags); 2500c872bdb3SSong Liu ulen = min_t(u32, info.nr_prog_tags, ulen); 2501c872bdb3SSong Liu if (prog->aux->func_cnt) { 2502c872bdb3SSong Liu for (i = 0; i < ulen; i++) { 2503c872bdb3SSong Liu if (copy_to_user(user_prog_tags[i], 2504c872bdb3SSong Liu prog->aux->func[i]->tag, 2505c872bdb3SSong Liu BPF_TAG_SIZE)) 2506c872bdb3SSong Liu return -EFAULT; 2507c872bdb3SSong Liu } 2508c872bdb3SSong Liu } else { 2509c872bdb3SSong Liu if (copy_to_user(user_prog_tags[0], 2510c872bdb3SSong Liu prog->tag, BPF_TAG_SIZE)) 2511c872bdb3SSong Liu return -EFAULT; 2512c872bdb3SSong Liu } 2513c872bdb3SSong Liu } 2514c872bdb3SSong Liu 25151e270976SMartin KaFai Lau done: 25161e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 25171e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 25181e270976SMartin KaFai Lau return -EFAULT; 25191e270976SMartin KaFai Lau 25201e270976SMartin KaFai Lau return 0; 25211e270976SMartin KaFai Lau } 25221e270976SMartin KaFai Lau 25231e270976SMartin KaFai Lau static int bpf_map_get_info_by_fd(struct bpf_map *map, 25241e270976SMartin KaFai Lau const union bpf_attr *attr, 25251e270976SMartin KaFai Lau union bpf_attr __user *uattr) 25261e270976SMartin KaFai Lau { 25271e270976SMartin KaFai Lau struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 25281e270976SMartin KaFai Lau struct bpf_map_info info = {}; 25291e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 25301e270976SMartin KaFai Lau int err; 25311e270976SMartin KaFai Lau 2532dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 25331e270976SMartin KaFai Lau if (err) 25341e270976SMartin KaFai Lau return err; 25351e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 25361e270976SMartin KaFai Lau 25371e270976SMartin KaFai Lau info.type = map->map_type; 25381e270976SMartin KaFai Lau info.id = map->id; 25391e270976SMartin KaFai Lau info.key_size = map->key_size; 25401e270976SMartin KaFai Lau info.value_size = map->value_size; 25411e270976SMartin KaFai Lau info.max_entries = map->max_entries; 25421e270976SMartin KaFai Lau info.map_flags = map->map_flags; 2543ad5b177bSMartin KaFai Lau memcpy(info.name, map->name, sizeof(map->name)); 25441e270976SMartin KaFai Lau 254578958fcaSMartin KaFai Lau if (map->btf) { 254678958fcaSMartin KaFai Lau info.btf_id = btf_id(map->btf); 25479b2cf328SMartin KaFai Lau info.btf_key_type_id = map->btf_key_type_id; 25489b2cf328SMartin KaFai Lau info.btf_value_type_id = map->btf_value_type_id; 254978958fcaSMartin KaFai Lau } 255078958fcaSMartin KaFai Lau 255152775b33SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 255252775b33SJakub Kicinski err = bpf_map_offload_info_fill(&info, map); 255352775b33SJakub Kicinski if (err) 255452775b33SJakub Kicinski return err; 255552775b33SJakub Kicinski } 255652775b33SJakub Kicinski 25571e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 25581e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 25591e270976SMartin KaFai Lau return -EFAULT; 25601e270976SMartin KaFai Lau 25611e270976SMartin KaFai Lau return 0; 25621e270976SMartin KaFai Lau } 25631e270976SMartin KaFai Lau 256462dab84cSMartin KaFai Lau static int bpf_btf_get_info_by_fd(struct btf *btf, 256562dab84cSMartin KaFai Lau const union bpf_attr *attr, 256662dab84cSMartin KaFai Lau union bpf_attr __user *uattr) 256762dab84cSMartin KaFai Lau { 256862dab84cSMartin KaFai Lau struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info); 256962dab84cSMartin KaFai Lau u32 info_len = attr->info.info_len; 257062dab84cSMartin KaFai Lau int err; 257162dab84cSMartin KaFai Lau 2572dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(*uinfo), info_len); 257362dab84cSMartin KaFai Lau if (err) 257462dab84cSMartin KaFai Lau return err; 257562dab84cSMartin KaFai Lau 257662dab84cSMartin KaFai Lau return btf_get_info_by_fd(btf, attr, uattr); 257762dab84cSMartin KaFai Lau } 257862dab84cSMartin KaFai Lau 25791e270976SMartin KaFai Lau #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 25801e270976SMartin KaFai Lau 25811e270976SMartin KaFai Lau static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 25821e270976SMartin KaFai Lau union bpf_attr __user *uattr) 25831e270976SMartin KaFai Lau { 25841e270976SMartin KaFai Lau int ufd = attr->info.bpf_fd; 25851e270976SMartin KaFai Lau struct fd f; 25861e270976SMartin KaFai Lau int err; 25871e270976SMartin KaFai Lau 25881e270976SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 25891e270976SMartin KaFai Lau return -EINVAL; 25901e270976SMartin KaFai Lau 25911e270976SMartin KaFai Lau f = fdget(ufd); 25921e270976SMartin KaFai Lau if (!f.file) 25931e270976SMartin KaFai Lau return -EBADFD; 25941e270976SMartin KaFai Lau 25951e270976SMartin KaFai Lau if (f.file->f_op == &bpf_prog_fops) 25961e270976SMartin KaFai Lau err = bpf_prog_get_info_by_fd(f.file->private_data, attr, 25971e270976SMartin KaFai Lau uattr); 25981e270976SMartin KaFai Lau else if (f.file->f_op == &bpf_map_fops) 25991e270976SMartin KaFai Lau err = bpf_map_get_info_by_fd(f.file->private_data, attr, 26001e270976SMartin KaFai Lau uattr); 260160197cfbSMartin KaFai Lau else if (f.file->f_op == &btf_fops) 260262dab84cSMartin KaFai Lau err = bpf_btf_get_info_by_fd(f.file->private_data, attr, uattr); 26031e270976SMartin KaFai Lau else 26041e270976SMartin KaFai Lau err = -EINVAL; 26051e270976SMartin KaFai Lau 26061e270976SMartin KaFai Lau fdput(f); 26071e270976SMartin KaFai Lau return err; 26081e270976SMartin KaFai Lau } 26091e270976SMartin KaFai Lau 2610f56a653cSMartin KaFai Lau #define BPF_BTF_LOAD_LAST_FIELD btf_log_level 2611f56a653cSMartin KaFai Lau 2612f56a653cSMartin KaFai Lau static int bpf_btf_load(const union bpf_attr *attr) 2613f56a653cSMartin KaFai Lau { 2614f56a653cSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_LOAD)) 2615f56a653cSMartin KaFai Lau return -EINVAL; 2616f56a653cSMartin KaFai Lau 2617f56a653cSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 2618f56a653cSMartin KaFai Lau return -EPERM; 2619f56a653cSMartin KaFai Lau 2620f56a653cSMartin KaFai Lau return btf_new_fd(attr); 2621f56a653cSMartin KaFai Lau } 2622f56a653cSMartin KaFai Lau 262378958fcaSMartin KaFai Lau #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id 262478958fcaSMartin KaFai Lau 262578958fcaSMartin KaFai Lau static int bpf_btf_get_fd_by_id(const union bpf_attr *attr) 262678958fcaSMartin KaFai Lau { 262778958fcaSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID)) 262878958fcaSMartin KaFai Lau return -EINVAL; 262978958fcaSMartin KaFai Lau 263078958fcaSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 263178958fcaSMartin KaFai Lau return -EPERM; 263278958fcaSMartin KaFai Lau 263378958fcaSMartin KaFai Lau return btf_get_fd_by_id(attr->btf_id); 263478958fcaSMartin KaFai Lau } 263578958fcaSMartin KaFai Lau 263641bdc4b4SYonghong Song static int bpf_task_fd_query_copy(const union bpf_attr *attr, 263741bdc4b4SYonghong Song union bpf_attr __user *uattr, 263841bdc4b4SYonghong Song u32 prog_id, u32 fd_type, 263941bdc4b4SYonghong Song const char *buf, u64 probe_offset, 264041bdc4b4SYonghong Song u64 probe_addr) 264141bdc4b4SYonghong Song { 264241bdc4b4SYonghong Song char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf); 264341bdc4b4SYonghong Song u32 len = buf ? strlen(buf) : 0, input_len; 264441bdc4b4SYonghong Song int err = 0; 264541bdc4b4SYonghong Song 264641bdc4b4SYonghong Song if (put_user(len, &uattr->task_fd_query.buf_len)) 264741bdc4b4SYonghong Song return -EFAULT; 264841bdc4b4SYonghong Song input_len = attr->task_fd_query.buf_len; 264941bdc4b4SYonghong Song if (input_len && ubuf) { 265041bdc4b4SYonghong Song if (!len) { 265141bdc4b4SYonghong Song /* nothing to copy, just make ubuf NULL terminated */ 265241bdc4b4SYonghong Song char zero = '\0'; 265341bdc4b4SYonghong Song 265441bdc4b4SYonghong Song if (put_user(zero, ubuf)) 265541bdc4b4SYonghong Song return -EFAULT; 265641bdc4b4SYonghong Song } else if (input_len >= len + 1) { 265741bdc4b4SYonghong Song /* ubuf can hold the string with NULL terminator */ 265841bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, len + 1)) 265941bdc4b4SYonghong Song return -EFAULT; 266041bdc4b4SYonghong Song } else { 266141bdc4b4SYonghong Song /* ubuf cannot hold the string with NULL terminator, 266241bdc4b4SYonghong Song * do a partial copy with NULL terminator. 266341bdc4b4SYonghong Song */ 266441bdc4b4SYonghong Song char zero = '\0'; 266541bdc4b4SYonghong Song 266641bdc4b4SYonghong Song err = -ENOSPC; 266741bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, input_len - 1)) 266841bdc4b4SYonghong Song return -EFAULT; 266941bdc4b4SYonghong Song if (put_user(zero, ubuf + input_len - 1)) 267041bdc4b4SYonghong Song return -EFAULT; 267141bdc4b4SYonghong Song } 267241bdc4b4SYonghong Song } 267341bdc4b4SYonghong Song 267441bdc4b4SYonghong Song if (put_user(prog_id, &uattr->task_fd_query.prog_id) || 267541bdc4b4SYonghong Song put_user(fd_type, &uattr->task_fd_query.fd_type) || 267641bdc4b4SYonghong Song put_user(probe_offset, &uattr->task_fd_query.probe_offset) || 267741bdc4b4SYonghong Song put_user(probe_addr, &uattr->task_fd_query.probe_addr)) 267841bdc4b4SYonghong Song return -EFAULT; 267941bdc4b4SYonghong Song 268041bdc4b4SYonghong Song return err; 268141bdc4b4SYonghong Song } 268241bdc4b4SYonghong Song 268341bdc4b4SYonghong Song #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr 268441bdc4b4SYonghong Song 268541bdc4b4SYonghong Song static int bpf_task_fd_query(const union bpf_attr *attr, 268641bdc4b4SYonghong Song union bpf_attr __user *uattr) 268741bdc4b4SYonghong Song { 268841bdc4b4SYonghong Song pid_t pid = attr->task_fd_query.pid; 268941bdc4b4SYonghong Song u32 fd = attr->task_fd_query.fd; 269041bdc4b4SYonghong Song const struct perf_event *event; 269141bdc4b4SYonghong Song struct files_struct *files; 269241bdc4b4SYonghong Song struct task_struct *task; 269341bdc4b4SYonghong Song struct file *file; 269441bdc4b4SYonghong Song int err; 269541bdc4b4SYonghong Song 269641bdc4b4SYonghong Song if (CHECK_ATTR(BPF_TASK_FD_QUERY)) 269741bdc4b4SYonghong Song return -EINVAL; 269841bdc4b4SYonghong Song 269941bdc4b4SYonghong Song if (!capable(CAP_SYS_ADMIN)) 270041bdc4b4SYonghong Song return -EPERM; 270141bdc4b4SYonghong Song 270241bdc4b4SYonghong Song if (attr->task_fd_query.flags != 0) 270341bdc4b4SYonghong Song return -EINVAL; 270441bdc4b4SYonghong Song 270541bdc4b4SYonghong Song task = get_pid_task(find_vpid(pid), PIDTYPE_PID); 270641bdc4b4SYonghong Song if (!task) 270741bdc4b4SYonghong Song return -ENOENT; 270841bdc4b4SYonghong Song 270941bdc4b4SYonghong Song files = get_files_struct(task); 271041bdc4b4SYonghong Song put_task_struct(task); 271141bdc4b4SYonghong Song if (!files) 271241bdc4b4SYonghong Song return -ENOENT; 271341bdc4b4SYonghong Song 271441bdc4b4SYonghong Song err = 0; 271541bdc4b4SYonghong Song spin_lock(&files->file_lock); 271641bdc4b4SYonghong Song file = fcheck_files(files, fd); 271741bdc4b4SYonghong Song if (!file) 271841bdc4b4SYonghong Song err = -EBADF; 271941bdc4b4SYonghong Song else 272041bdc4b4SYonghong Song get_file(file); 272141bdc4b4SYonghong Song spin_unlock(&files->file_lock); 272241bdc4b4SYonghong Song put_files_struct(files); 272341bdc4b4SYonghong Song 272441bdc4b4SYonghong Song if (err) 272541bdc4b4SYonghong Song goto out; 272641bdc4b4SYonghong Song 272741bdc4b4SYonghong Song if (file->f_op == &bpf_raw_tp_fops) { 272841bdc4b4SYonghong Song struct bpf_raw_tracepoint *raw_tp = file->private_data; 272941bdc4b4SYonghong Song struct bpf_raw_event_map *btp = raw_tp->btp; 273041bdc4b4SYonghong Song 273141bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, 273241bdc4b4SYonghong Song raw_tp->prog->aux->id, 273341bdc4b4SYonghong Song BPF_FD_TYPE_RAW_TRACEPOINT, 273441bdc4b4SYonghong Song btp->tp->name, 0, 0); 273541bdc4b4SYonghong Song goto put_file; 273641bdc4b4SYonghong Song } 273741bdc4b4SYonghong Song 273841bdc4b4SYonghong Song event = perf_get_event(file); 273941bdc4b4SYonghong Song if (!IS_ERR(event)) { 274041bdc4b4SYonghong Song u64 probe_offset, probe_addr; 274141bdc4b4SYonghong Song u32 prog_id, fd_type; 274241bdc4b4SYonghong Song const char *buf; 274341bdc4b4SYonghong Song 274441bdc4b4SYonghong Song err = bpf_get_perf_event_info(event, &prog_id, &fd_type, 274541bdc4b4SYonghong Song &buf, &probe_offset, 274641bdc4b4SYonghong Song &probe_addr); 274741bdc4b4SYonghong Song if (!err) 274841bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, prog_id, 274941bdc4b4SYonghong Song fd_type, buf, 275041bdc4b4SYonghong Song probe_offset, 275141bdc4b4SYonghong Song probe_addr); 275241bdc4b4SYonghong Song goto put_file; 275341bdc4b4SYonghong Song } 275441bdc4b4SYonghong Song 275541bdc4b4SYonghong Song err = -ENOTSUPP; 275641bdc4b4SYonghong Song put_file: 275741bdc4b4SYonghong Song fput(file); 275841bdc4b4SYonghong Song out: 275941bdc4b4SYonghong Song return err; 276041bdc4b4SYonghong Song } 276141bdc4b4SYonghong Song 276299c55f7dSAlexei Starovoitov SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 276399c55f7dSAlexei Starovoitov { 276499c55f7dSAlexei Starovoitov union bpf_attr attr = {}; 276599c55f7dSAlexei Starovoitov int err; 276699c55f7dSAlexei Starovoitov 27670fa4fe85SChenbo Feng if (sysctl_unprivileged_bpf_disabled && !capable(CAP_SYS_ADMIN)) 276899c55f7dSAlexei Starovoitov return -EPERM; 276999c55f7dSAlexei Starovoitov 2770dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size); 277199c55f7dSAlexei Starovoitov if (err) 277299c55f7dSAlexei Starovoitov return err; 27731e270976SMartin KaFai Lau size = min_t(u32, size, sizeof(attr)); 277499c55f7dSAlexei Starovoitov 277599c55f7dSAlexei Starovoitov /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 277699c55f7dSAlexei Starovoitov if (copy_from_user(&attr, uattr, size) != 0) 277799c55f7dSAlexei Starovoitov return -EFAULT; 277899c55f7dSAlexei Starovoitov 2779afdb09c7SChenbo Feng err = security_bpf(cmd, &attr, size); 2780afdb09c7SChenbo Feng if (err < 0) 2781afdb09c7SChenbo Feng return err; 2782afdb09c7SChenbo Feng 278399c55f7dSAlexei Starovoitov switch (cmd) { 278499c55f7dSAlexei Starovoitov case BPF_MAP_CREATE: 278599c55f7dSAlexei Starovoitov err = map_create(&attr); 278699c55f7dSAlexei Starovoitov break; 2787db20fd2bSAlexei Starovoitov case BPF_MAP_LOOKUP_ELEM: 2788db20fd2bSAlexei Starovoitov err = map_lookup_elem(&attr); 2789db20fd2bSAlexei Starovoitov break; 2790db20fd2bSAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 2791db20fd2bSAlexei Starovoitov err = map_update_elem(&attr); 2792db20fd2bSAlexei Starovoitov break; 2793db20fd2bSAlexei Starovoitov case BPF_MAP_DELETE_ELEM: 2794db20fd2bSAlexei Starovoitov err = map_delete_elem(&attr); 2795db20fd2bSAlexei Starovoitov break; 2796db20fd2bSAlexei Starovoitov case BPF_MAP_GET_NEXT_KEY: 2797db20fd2bSAlexei Starovoitov err = map_get_next_key(&attr); 2798db20fd2bSAlexei Starovoitov break; 279987df15deSDaniel Borkmann case BPF_MAP_FREEZE: 280087df15deSDaniel Borkmann err = map_freeze(&attr); 280187df15deSDaniel Borkmann break; 280209756af4SAlexei Starovoitov case BPF_PROG_LOAD: 2803838e9690SYonghong Song err = bpf_prog_load(&attr, uattr); 280409756af4SAlexei Starovoitov break; 2805b2197755SDaniel Borkmann case BPF_OBJ_PIN: 2806b2197755SDaniel Borkmann err = bpf_obj_pin(&attr); 2807b2197755SDaniel Borkmann break; 2808b2197755SDaniel Borkmann case BPF_OBJ_GET: 2809b2197755SDaniel Borkmann err = bpf_obj_get(&attr); 2810b2197755SDaniel Borkmann break; 2811f4324551SDaniel Mack case BPF_PROG_ATTACH: 2812f4324551SDaniel Mack err = bpf_prog_attach(&attr); 2813f4324551SDaniel Mack break; 2814f4324551SDaniel Mack case BPF_PROG_DETACH: 2815f4324551SDaniel Mack err = bpf_prog_detach(&attr); 2816f4324551SDaniel Mack break; 2817468e2f64SAlexei Starovoitov case BPF_PROG_QUERY: 2818468e2f64SAlexei Starovoitov err = bpf_prog_query(&attr, uattr); 2819468e2f64SAlexei Starovoitov break; 28201cf1cae9SAlexei Starovoitov case BPF_PROG_TEST_RUN: 28211cf1cae9SAlexei Starovoitov err = bpf_prog_test_run(&attr, uattr); 28221cf1cae9SAlexei Starovoitov break; 282334ad5580SMartin KaFai Lau case BPF_PROG_GET_NEXT_ID: 282434ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 282534ad5580SMartin KaFai Lau &prog_idr, &prog_idr_lock); 282634ad5580SMartin KaFai Lau break; 282734ad5580SMartin KaFai Lau case BPF_MAP_GET_NEXT_ID: 282834ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 282934ad5580SMartin KaFai Lau &map_idr, &map_idr_lock); 283034ad5580SMartin KaFai Lau break; 2831b16d9aa4SMartin KaFai Lau case BPF_PROG_GET_FD_BY_ID: 2832b16d9aa4SMartin KaFai Lau err = bpf_prog_get_fd_by_id(&attr); 2833b16d9aa4SMartin KaFai Lau break; 2834bd5f5f4eSMartin KaFai Lau case BPF_MAP_GET_FD_BY_ID: 2835bd5f5f4eSMartin KaFai Lau err = bpf_map_get_fd_by_id(&attr); 2836bd5f5f4eSMartin KaFai Lau break; 28371e270976SMartin KaFai Lau case BPF_OBJ_GET_INFO_BY_FD: 28381e270976SMartin KaFai Lau err = bpf_obj_get_info_by_fd(&attr, uattr); 28391e270976SMartin KaFai Lau break; 2840c4f6699dSAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN: 2841c4f6699dSAlexei Starovoitov err = bpf_raw_tracepoint_open(&attr); 2842c4f6699dSAlexei Starovoitov break; 2843f56a653cSMartin KaFai Lau case BPF_BTF_LOAD: 2844f56a653cSMartin KaFai Lau err = bpf_btf_load(&attr); 2845f56a653cSMartin KaFai Lau break; 284678958fcaSMartin KaFai Lau case BPF_BTF_GET_FD_BY_ID: 284778958fcaSMartin KaFai Lau err = bpf_btf_get_fd_by_id(&attr); 284878958fcaSMartin KaFai Lau break; 284941bdc4b4SYonghong Song case BPF_TASK_FD_QUERY: 285041bdc4b4SYonghong Song err = bpf_task_fd_query(&attr, uattr); 285141bdc4b4SYonghong Song break; 2852bd513cd0SMauricio Vasquez B case BPF_MAP_LOOKUP_AND_DELETE_ELEM: 2853bd513cd0SMauricio Vasquez B err = map_lookup_and_delete_elem(&attr); 2854bd513cd0SMauricio Vasquez B break; 285599c55f7dSAlexei Starovoitov default: 285699c55f7dSAlexei Starovoitov err = -EINVAL; 285799c55f7dSAlexei Starovoitov break; 285899c55f7dSAlexei Starovoitov } 285999c55f7dSAlexei Starovoitov 286099c55f7dSAlexei Starovoitov return err; 286199c55f7dSAlexei Starovoitov } 2862