15b497af4SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 299c55f7dSAlexei Starovoitov /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 399c55f7dSAlexei Starovoitov */ 499c55f7dSAlexei Starovoitov #include <linux/bpf.h> 5a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 6f4364dcfSSean Young #include <linux/bpf_lirc.h> 7f56a653cSMartin KaFai Lau #include <linux/btf.h> 899c55f7dSAlexei Starovoitov #include <linux/syscalls.h> 999c55f7dSAlexei Starovoitov #include <linux/slab.h> 103f07c014SIngo Molnar #include <linux/sched/signal.h> 11d407bd25SDaniel Borkmann #include <linux/vmalloc.h> 12d407bd25SDaniel Borkmann #include <linux/mmzone.h> 1399c55f7dSAlexei Starovoitov #include <linux/anon_inodes.h> 1441bdc4b4SYonghong Song #include <linux/fdtable.h> 15db20fd2bSAlexei Starovoitov #include <linux/file.h> 1641bdc4b4SYonghong Song #include <linux/fs.h> 1709756af4SAlexei Starovoitov #include <linux/license.h> 1809756af4SAlexei Starovoitov #include <linux/filter.h> 192541517cSAlexei Starovoitov #include <linux/version.h> 20535e7b4bSMickaël Salaün #include <linux/kernel.h> 21dc4bb0e2SMartin KaFai Lau #include <linux/idr.h> 22cb4d2b3fSMartin KaFai Lau #include <linux/cred.h> 23cb4d2b3fSMartin KaFai Lau #include <linux/timekeeping.h> 24cb4d2b3fSMartin KaFai Lau #include <linux/ctype.h> 259ef09e35SMark Rutland #include <linux/nospec.h> 2699c55f7dSAlexei Starovoitov 2714dc6f04SMartin KaFai Lau #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ 2814dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 2914dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 3014dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 3114dc6f04SMartin KaFai Lau #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 3214dc6f04SMartin KaFai Lau #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map)) 3314dc6f04SMartin KaFai Lau 346e71b04aSChenbo Feng #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) 356e71b04aSChenbo Feng 36b121d1e7SAlexei Starovoitov DEFINE_PER_CPU(int, bpf_prog_active); 37dc4bb0e2SMartin KaFai Lau static DEFINE_IDR(prog_idr); 38dc4bb0e2SMartin KaFai Lau static DEFINE_SPINLOCK(prog_idr_lock); 39f3f1c054SMartin KaFai Lau static DEFINE_IDR(map_idr); 40f3f1c054SMartin KaFai Lau static DEFINE_SPINLOCK(map_idr_lock); 41b121d1e7SAlexei Starovoitov 421be7f75dSAlexei Starovoitov int sysctl_unprivileged_bpf_disabled __read_mostly; 431be7f75dSAlexei Starovoitov 4440077e0cSJohannes Berg static const struct bpf_map_ops * const bpf_map_types[] = { 4540077e0cSJohannes Berg #define BPF_PROG_TYPE(_id, _ops) 4640077e0cSJohannes Berg #define BPF_MAP_TYPE(_id, _ops) \ 4740077e0cSJohannes Berg [_id] = &_ops, 4840077e0cSJohannes Berg #include <linux/bpf_types.h> 4940077e0cSJohannes Berg #undef BPF_PROG_TYPE 5040077e0cSJohannes Berg #undef BPF_MAP_TYPE 5140077e0cSJohannes Berg }; 5299c55f7dSAlexei Starovoitov 53752ba56fSMickaël Salaün /* 54752ba56fSMickaël Salaün * If we're handed a bigger struct than we know of, ensure all the unknown bits 55752ba56fSMickaël Salaün * are 0 - i.e. new user-space does not rely on any kernel feature extensions 56752ba56fSMickaël Salaün * we don't know about yet. 57752ba56fSMickaël Salaün * 58752ba56fSMickaël Salaün * There is a ToCToU between this function call and the following 59752ba56fSMickaël Salaün * copy_from_user() call. However, this is not a concern since this function is 60752ba56fSMickaël Salaün * meant to be a future-proofing of bits. 61752ba56fSMickaël Salaün */ 62dcab51f1SMartin KaFai Lau int bpf_check_uarg_tail_zero(void __user *uaddr, 6358291a74SMickaël Salaün size_t expected_size, 6458291a74SMickaël Salaün size_t actual_size) 6558291a74SMickaël Salaün { 6658291a74SMickaël Salaün unsigned char __user *addr; 6758291a74SMickaël Salaün unsigned char __user *end; 6858291a74SMickaël Salaün unsigned char val; 6958291a74SMickaël Salaün int err; 7058291a74SMickaël Salaün 71752ba56fSMickaël Salaün if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 72752ba56fSMickaël Salaün return -E2BIG; 73752ba56fSMickaël Salaün 7496d4f267SLinus Torvalds if (unlikely(!access_ok(uaddr, actual_size))) 75752ba56fSMickaël Salaün return -EFAULT; 76752ba56fSMickaël Salaün 7758291a74SMickaël Salaün if (actual_size <= expected_size) 7858291a74SMickaël Salaün return 0; 7958291a74SMickaël Salaün 8058291a74SMickaël Salaün addr = uaddr + expected_size; 8158291a74SMickaël Salaün end = uaddr + actual_size; 8258291a74SMickaël Salaün 8358291a74SMickaël Salaün for (; addr < end; addr++) { 8458291a74SMickaël Salaün err = get_user(val, addr); 8558291a74SMickaël Salaün if (err) 8658291a74SMickaël Salaün return err; 8758291a74SMickaël Salaün if (val) 8858291a74SMickaël Salaün return -E2BIG; 8958291a74SMickaël Salaün } 9058291a74SMickaël Salaün 9158291a74SMickaël Salaün return 0; 9258291a74SMickaël Salaün } 9358291a74SMickaël Salaün 94a3884572SJakub Kicinski const struct bpf_map_ops bpf_map_offload_ops = { 95a3884572SJakub Kicinski .map_alloc = bpf_map_offload_map_alloc, 96a3884572SJakub Kicinski .map_free = bpf_map_offload_map_free, 97e8d2bec0SDaniel Borkmann .map_check_btf = map_check_no_btf, 98a3884572SJakub Kicinski }; 99a3884572SJakub Kicinski 10099c55f7dSAlexei Starovoitov static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 10199c55f7dSAlexei Starovoitov { 1021110f3a9SJakub Kicinski const struct bpf_map_ops *ops; 1039ef09e35SMark Rutland u32 type = attr->map_type; 10499c55f7dSAlexei Starovoitov struct bpf_map *map; 1051110f3a9SJakub Kicinski int err; 10699c55f7dSAlexei Starovoitov 1079ef09e35SMark Rutland if (type >= ARRAY_SIZE(bpf_map_types)) 1081110f3a9SJakub Kicinski return ERR_PTR(-EINVAL); 1099ef09e35SMark Rutland type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types)); 1109ef09e35SMark Rutland ops = bpf_map_types[type]; 1111110f3a9SJakub Kicinski if (!ops) 11240077e0cSJohannes Berg return ERR_PTR(-EINVAL); 11340077e0cSJohannes Berg 1141110f3a9SJakub Kicinski if (ops->map_alloc_check) { 1151110f3a9SJakub Kicinski err = ops->map_alloc_check(attr); 1161110f3a9SJakub Kicinski if (err) 1171110f3a9SJakub Kicinski return ERR_PTR(err); 1181110f3a9SJakub Kicinski } 119a3884572SJakub Kicinski if (attr->map_ifindex) 120a3884572SJakub Kicinski ops = &bpf_map_offload_ops; 1211110f3a9SJakub Kicinski map = ops->map_alloc(attr); 12299c55f7dSAlexei Starovoitov if (IS_ERR(map)) 12399c55f7dSAlexei Starovoitov return map; 1241110f3a9SJakub Kicinski map->ops = ops; 1259ef09e35SMark Rutland map->map_type = type; 12699c55f7dSAlexei Starovoitov return map; 12799c55f7dSAlexei Starovoitov } 12899c55f7dSAlexei Starovoitov 12996eabe7aSMartin KaFai Lau void *bpf_map_area_alloc(size_t size, int numa_node) 130d407bd25SDaniel Borkmann { 131f01a7dbeSMartynas Pumputis /* We really just want to fail instead of triggering OOM killer 132f01a7dbeSMartynas Pumputis * under memory pressure, therefore we set __GFP_NORETRY to kmalloc, 133f01a7dbeSMartynas Pumputis * which is used for lower order allocation requests. 134f01a7dbeSMartynas Pumputis * 135f01a7dbeSMartynas Pumputis * It has been observed that higher order allocation requests done by 136f01a7dbeSMartynas Pumputis * vmalloc with __GFP_NORETRY being set might fail due to not trying 137f01a7dbeSMartynas Pumputis * to reclaim memory from the page cache, thus we set 138f01a7dbeSMartynas Pumputis * __GFP_RETRY_MAYFAIL to avoid such situations. 139d407bd25SDaniel Borkmann */ 140f01a7dbeSMartynas Pumputis 141f01a7dbeSMartynas Pumputis const gfp_t flags = __GFP_NOWARN | __GFP_ZERO; 142d407bd25SDaniel Borkmann void *area; 143d407bd25SDaniel Borkmann 144d407bd25SDaniel Borkmann if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 145f01a7dbeSMartynas Pumputis area = kmalloc_node(size, GFP_USER | __GFP_NORETRY | flags, 146f01a7dbeSMartynas Pumputis numa_node); 147d407bd25SDaniel Borkmann if (area != NULL) 148d407bd25SDaniel Borkmann return area; 149d407bd25SDaniel Borkmann } 150d407bd25SDaniel Borkmann 151f01a7dbeSMartynas Pumputis return __vmalloc_node_flags_caller(size, numa_node, 152f01a7dbeSMartynas Pumputis GFP_KERNEL | __GFP_RETRY_MAYFAIL | 153f01a7dbeSMartynas Pumputis flags, __builtin_return_address(0)); 154d407bd25SDaniel Borkmann } 155d407bd25SDaniel Borkmann 156d407bd25SDaniel Borkmann void bpf_map_area_free(void *area) 157d407bd25SDaniel Borkmann { 158d407bd25SDaniel Borkmann kvfree(area); 159d407bd25SDaniel Borkmann } 160d407bd25SDaniel Borkmann 161be70bcd5SDaniel Borkmann static u32 bpf_map_flags_retain_permanent(u32 flags) 162be70bcd5SDaniel Borkmann { 163be70bcd5SDaniel Borkmann /* Some map creation flags are not tied to the map object but 164be70bcd5SDaniel Borkmann * rather to the map fd instead, so they have no meaning upon 165be70bcd5SDaniel Borkmann * map object inspection since multiple file descriptors with 166be70bcd5SDaniel Borkmann * different (access) properties can exist here. Thus, given 167be70bcd5SDaniel Borkmann * this has zero meaning for the map itself, lets clear these 168be70bcd5SDaniel Borkmann * from here. 169be70bcd5SDaniel Borkmann */ 170be70bcd5SDaniel Borkmann return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY); 171be70bcd5SDaniel Borkmann } 172be70bcd5SDaniel Borkmann 173bd475643SJakub Kicinski void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) 174bd475643SJakub Kicinski { 175bd475643SJakub Kicinski map->map_type = attr->map_type; 176bd475643SJakub Kicinski map->key_size = attr->key_size; 177bd475643SJakub Kicinski map->value_size = attr->value_size; 178bd475643SJakub Kicinski map->max_entries = attr->max_entries; 179be70bcd5SDaniel Borkmann map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags); 180bd475643SJakub Kicinski map->numa_node = bpf_map_attr_numa_node(attr); 181bd475643SJakub Kicinski } 182bd475643SJakub Kicinski 1836c905981SAlexei Starovoitov int bpf_map_precharge_memlock(u32 pages) 1846c905981SAlexei Starovoitov { 1856c905981SAlexei Starovoitov struct user_struct *user = get_current_user(); 1866c905981SAlexei Starovoitov unsigned long memlock_limit, cur; 1876c905981SAlexei Starovoitov 1886c905981SAlexei Starovoitov memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 1896c905981SAlexei Starovoitov cur = atomic_long_read(&user->locked_vm); 1906c905981SAlexei Starovoitov free_uid(user); 1916c905981SAlexei Starovoitov if (cur + pages > memlock_limit) 1926c905981SAlexei Starovoitov return -EPERM; 1936c905981SAlexei Starovoitov return 0; 1946c905981SAlexei Starovoitov } 1956c905981SAlexei Starovoitov 1960a4c58f5SRoman Gushchin static int bpf_charge_memlock(struct user_struct *user, u32 pages) 197aaac3ba9SAlexei Starovoitov { 1980a4c58f5SRoman Gushchin unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 199aaac3ba9SAlexei Starovoitov 2000a4c58f5SRoman Gushchin if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) { 2010a4c58f5SRoman Gushchin atomic_long_sub(pages, &user->locked_vm); 202aaac3ba9SAlexei Starovoitov return -EPERM; 203aaac3ba9SAlexei Starovoitov } 204aaac3ba9SAlexei Starovoitov return 0; 205aaac3ba9SAlexei Starovoitov } 206aaac3ba9SAlexei Starovoitov 2070a4c58f5SRoman Gushchin static void bpf_uncharge_memlock(struct user_struct *user, u32 pages) 2080a4c58f5SRoman Gushchin { 2090a4c58f5SRoman Gushchin atomic_long_sub(pages, &user->locked_vm); 2100a4c58f5SRoman Gushchin } 2110a4c58f5SRoman Gushchin 2120a4c58f5SRoman Gushchin static int bpf_map_init_memlock(struct bpf_map *map) 2130a4c58f5SRoman Gushchin { 2140a4c58f5SRoman Gushchin struct user_struct *user = get_current_user(); 2150a4c58f5SRoman Gushchin int ret; 2160a4c58f5SRoman Gushchin 2170a4c58f5SRoman Gushchin ret = bpf_charge_memlock(user, map->pages); 2180a4c58f5SRoman Gushchin if (ret) { 2190a4c58f5SRoman Gushchin free_uid(user); 2200a4c58f5SRoman Gushchin return ret; 2210a4c58f5SRoman Gushchin } 2220a4c58f5SRoman Gushchin map->user = user; 2230a4c58f5SRoman Gushchin return ret; 2240a4c58f5SRoman Gushchin } 2250a4c58f5SRoman Gushchin 2260a4c58f5SRoman Gushchin static void bpf_map_release_memlock(struct bpf_map *map) 227aaac3ba9SAlexei Starovoitov { 228aaac3ba9SAlexei Starovoitov struct user_struct *user = map->user; 2290a4c58f5SRoman Gushchin bpf_uncharge_memlock(user, map->pages); 230aaac3ba9SAlexei Starovoitov free_uid(user); 231aaac3ba9SAlexei Starovoitov } 232aaac3ba9SAlexei Starovoitov 2330a4c58f5SRoman Gushchin int bpf_map_charge_memlock(struct bpf_map *map, u32 pages) 2340a4c58f5SRoman Gushchin { 2350a4c58f5SRoman Gushchin int ret; 2360a4c58f5SRoman Gushchin 2370a4c58f5SRoman Gushchin ret = bpf_charge_memlock(map->user, pages); 2380a4c58f5SRoman Gushchin if (ret) 2390a4c58f5SRoman Gushchin return ret; 2400a4c58f5SRoman Gushchin map->pages += pages; 2410a4c58f5SRoman Gushchin return ret; 2420a4c58f5SRoman Gushchin } 2430a4c58f5SRoman Gushchin 2440a4c58f5SRoman Gushchin void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages) 2450a4c58f5SRoman Gushchin { 2460a4c58f5SRoman Gushchin bpf_uncharge_memlock(map->user, pages); 2470a4c58f5SRoman Gushchin map->pages -= pages; 2480a4c58f5SRoman Gushchin } 2490a4c58f5SRoman Gushchin 250f3f1c054SMartin KaFai Lau static int bpf_map_alloc_id(struct bpf_map *map) 251f3f1c054SMartin KaFai Lau { 252f3f1c054SMartin KaFai Lau int id; 253f3f1c054SMartin KaFai Lau 254b76354cdSShaohua Li idr_preload(GFP_KERNEL); 255f3f1c054SMartin KaFai Lau spin_lock_bh(&map_idr_lock); 256f3f1c054SMartin KaFai Lau id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 257f3f1c054SMartin KaFai Lau if (id > 0) 258f3f1c054SMartin KaFai Lau map->id = id; 259f3f1c054SMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 260b76354cdSShaohua Li idr_preload_end(); 261f3f1c054SMartin KaFai Lau 262f3f1c054SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 263f3f1c054SMartin KaFai Lau return -ENOSPC; 264f3f1c054SMartin KaFai Lau 265f3f1c054SMartin KaFai Lau return id > 0 ? 0 : id; 266f3f1c054SMartin KaFai Lau } 267f3f1c054SMartin KaFai Lau 268a3884572SJakub Kicinski void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) 269f3f1c054SMartin KaFai Lau { 270930651a7SEric Dumazet unsigned long flags; 271930651a7SEric Dumazet 272a3884572SJakub Kicinski /* Offloaded maps are removed from the IDR store when their device 273a3884572SJakub Kicinski * disappears - even if someone holds an fd to them they are unusable, 274a3884572SJakub Kicinski * the memory is gone, all ops will fail; they are simply waiting for 275a3884572SJakub Kicinski * refcnt to drop to be freed. 276a3884572SJakub Kicinski */ 277a3884572SJakub Kicinski if (!map->id) 278a3884572SJakub Kicinski return; 279a3884572SJakub Kicinski 280bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 281930651a7SEric Dumazet spin_lock_irqsave(&map_idr_lock, flags); 282bd5f5f4eSMartin KaFai Lau else 283bd5f5f4eSMartin KaFai Lau __acquire(&map_idr_lock); 284bd5f5f4eSMartin KaFai Lau 285f3f1c054SMartin KaFai Lau idr_remove(&map_idr, map->id); 286a3884572SJakub Kicinski map->id = 0; 287bd5f5f4eSMartin KaFai Lau 288bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 289930651a7SEric Dumazet spin_unlock_irqrestore(&map_idr_lock, flags); 290bd5f5f4eSMartin KaFai Lau else 291bd5f5f4eSMartin KaFai Lau __release(&map_idr_lock); 292f3f1c054SMartin KaFai Lau } 293f3f1c054SMartin KaFai Lau 29499c55f7dSAlexei Starovoitov /* called from workqueue */ 29599c55f7dSAlexei Starovoitov static void bpf_map_free_deferred(struct work_struct *work) 29699c55f7dSAlexei Starovoitov { 29799c55f7dSAlexei Starovoitov struct bpf_map *map = container_of(work, struct bpf_map, work); 29899c55f7dSAlexei Starovoitov 2990a4c58f5SRoman Gushchin bpf_map_release_memlock(map); 300afdb09c7SChenbo Feng security_bpf_map_free(map); 30199c55f7dSAlexei Starovoitov /* implementation dependent freeing */ 30299c55f7dSAlexei Starovoitov map->ops->map_free(map); 30399c55f7dSAlexei Starovoitov } 30499c55f7dSAlexei Starovoitov 305c9da161cSDaniel Borkmann static void bpf_map_put_uref(struct bpf_map *map) 306c9da161cSDaniel Borkmann { 307c9da161cSDaniel Borkmann if (atomic_dec_and_test(&map->usercnt)) { 308ba6b8de4SJohn Fastabend if (map->ops->map_release_uref) 309ba6b8de4SJohn Fastabend map->ops->map_release_uref(map); 310c9da161cSDaniel Borkmann } 311c9da161cSDaniel Borkmann } 312c9da161cSDaniel Borkmann 31399c55f7dSAlexei Starovoitov /* decrement map refcnt and schedule it for freeing via workqueue 31499c55f7dSAlexei Starovoitov * (unrelying map implementation ops->map_free() might sleep) 31599c55f7dSAlexei Starovoitov */ 316bd5f5f4eSMartin KaFai Lau static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) 31799c55f7dSAlexei Starovoitov { 31899c55f7dSAlexei Starovoitov if (atomic_dec_and_test(&map->refcnt)) { 31934ad5580SMartin KaFai Lau /* bpf_map_free_id() must be called first */ 320bd5f5f4eSMartin KaFai Lau bpf_map_free_id(map, do_idr_lock); 32178958fcaSMartin KaFai Lau btf_put(map->btf); 32299c55f7dSAlexei Starovoitov INIT_WORK(&map->work, bpf_map_free_deferred); 32399c55f7dSAlexei Starovoitov schedule_work(&map->work); 32499c55f7dSAlexei Starovoitov } 32599c55f7dSAlexei Starovoitov } 32699c55f7dSAlexei Starovoitov 327bd5f5f4eSMartin KaFai Lau void bpf_map_put(struct bpf_map *map) 328bd5f5f4eSMartin KaFai Lau { 329bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, true); 330bd5f5f4eSMartin KaFai Lau } 331630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_put); 332bd5f5f4eSMartin KaFai Lau 333c9da161cSDaniel Borkmann void bpf_map_put_with_uref(struct bpf_map *map) 334c9da161cSDaniel Borkmann { 335c9da161cSDaniel Borkmann bpf_map_put_uref(map); 336c9da161cSDaniel Borkmann bpf_map_put(map); 337c9da161cSDaniel Borkmann } 338c9da161cSDaniel Borkmann 33999c55f7dSAlexei Starovoitov static int bpf_map_release(struct inode *inode, struct file *filp) 34099c55f7dSAlexei Starovoitov { 34161d1b6a4SDaniel Borkmann struct bpf_map *map = filp->private_data; 34261d1b6a4SDaniel Borkmann 34361d1b6a4SDaniel Borkmann if (map->ops->map_release) 34461d1b6a4SDaniel Borkmann map->ops->map_release(map, filp); 34561d1b6a4SDaniel Borkmann 34661d1b6a4SDaniel Borkmann bpf_map_put_with_uref(map); 34799c55f7dSAlexei Starovoitov return 0; 34899c55f7dSAlexei Starovoitov } 34999c55f7dSAlexei Starovoitov 35087df15deSDaniel Borkmann static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f) 35187df15deSDaniel Borkmann { 35287df15deSDaniel Borkmann fmode_t mode = f.file->f_mode; 35387df15deSDaniel Borkmann 35487df15deSDaniel Borkmann /* Our file permissions may have been overridden by global 35587df15deSDaniel Borkmann * map permissions facing syscall side. 35687df15deSDaniel Borkmann */ 35787df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) 35887df15deSDaniel Borkmann mode &= ~FMODE_CAN_WRITE; 35987df15deSDaniel Borkmann return mode; 36087df15deSDaniel Borkmann } 36187df15deSDaniel Borkmann 362f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 363f99bf205SDaniel Borkmann static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 364f99bf205SDaniel Borkmann { 365f99bf205SDaniel Borkmann const struct bpf_map *map = filp->private_data; 36621116b70SDaniel Borkmann const struct bpf_array *array; 36721116b70SDaniel Borkmann u32 owner_prog_type = 0; 3689780c0abSDaniel Borkmann u32 owner_jited = 0; 36921116b70SDaniel Borkmann 37021116b70SDaniel Borkmann if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 37121116b70SDaniel Borkmann array = container_of(map, struct bpf_array, map); 37221116b70SDaniel Borkmann owner_prog_type = array->owner_prog_type; 3739780c0abSDaniel Borkmann owner_jited = array->owner_jited; 37421116b70SDaniel Borkmann } 375f99bf205SDaniel Borkmann 376f99bf205SDaniel Borkmann seq_printf(m, 377f99bf205SDaniel Borkmann "map_type:\t%u\n" 378f99bf205SDaniel Borkmann "key_size:\t%u\n" 379f99bf205SDaniel Borkmann "value_size:\t%u\n" 380322cea2fSDaniel Borkmann "max_entries:\t%u\n" 38121116b70SDaniel Borkmann "map_flags:\t%#x\n" 3824316b409SDaniel Borkmann "memlock:\t%llu\n" 38387df15deSDaniel Borkmann "map_id:\t%u\n" 38487df15deSDaniel Borkmann "frozen:\t%u\n", 385f99bf205SDaniel Borkmann map->map_type, 386f99bf205SDaniel Borkmann map->key_size, 387f99bf205SDaniel Borkmann map->value_size, 388322cea2fSDaniel Borkmann map->max_entries, 38921116b70SDaniel Borkmann map->map_flags, 3904316b409SDaniel Borkmann map->pages * 1ULL << PAGE_SHIFT, 39187df15deSDaniel Borkmann map->id, 39287df15deSDaniel Borkmann READ_ONCE(map->frozen)); 39321116b70SDaniel Borkmann 3949780c0abSDaniel Borkmann if (owner_prog_type) { 39521116b70SDaniel Borkmann seq_printf(m, "owner_prog_type:\t%u\n", 39621116b70SDaniel Borkmann owner_prog_type); 3979780c0abSDaniel Borkmann seq_printf(m, "owner_jited:\t%u\n", 3989780c0abSDaniel Borkmann owner_jited); 3999780c0abSDaniel Borkmann } 400f99bf205SDaniel Borkmann } 401f99bf205SDaniel Borkmann #endif 402f99bf205SDaniel Borkmann 4036e71b04aSChenbo Feng static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, 4046e71b04aSChenbo Feng loff_t *ppos) 4056e71b04aSChenbo Feng { 4066e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 4076e71b04aSChenbo Feng * f_mode with FMODE_CAN_READ. 4086e71b04aSChenbo Feng */ 4096e71b04aSChenbo Feng return -EINVAL; 4106e71b04aSChenbo Feng } 4116e71b04aSChenbo Feng 4126e71b04aSChenbo Feng static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, 4136e71b04aSChenbo Feng size_t siz, loff_t *ppos) 4146e71b04aSChenbo Feng { 4156e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 4166e71b04aSChenbo Feng * f_mode with FMODE_CAN_WRITE. 4176e71b04aSChenbo Feng */ 4186e71b04aSChenbo Feng return -EINVAL; 4196e71b04aSChenbo Feng } 4206e71b04aSChenbo Feng 421f66e448cSChenbo Feng const struct file_operations bpf_map_fops = { 422f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 423f99bf205SDaniel Borkmann .show_fdinfo = bpf_map_show_fdinfo, 424f99bf205SDaniel Borkmann #endif 42599c55f7dSAlexei Starovoitov .release = bpf_map_release, 4266e71b04aSChenbo Feng .read = bpf_dummy_read, 4276e71b04aSChenbo Feng .write = bpf_dummy_write, 42899c55f7dSAlexei Starovoitov }; 42999c55f7dSAlexei Starovoitov 4306e71b04aSChenbo Feng int bpf_map_new_fd(struct bpf_map *map, int flags) 431aa79781bSDaniel Borkmann { 432afdb09c7SChenbo Feng int ret; 433afdb09c7SChenbo Feng 434afdb09c7SChenbo Feng ret = security_bpf_map(map, OPEN_FMODE(flags)); 435afdb09c7SChenbo Feng if (ret < 0) 436afdb09c7SChenbo Feng return ret; 437afdb09c7SChenbo Feng 438aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 4396e71b04aSChenbo Feng flags | O_CLOEXEC); 4406e71b04aSChenbo Feng } 4416e71b04aSChenbo Feng 4426e71b04aSChenbo Feng int bpf_get_file_flag(int flags) 4436e71b04aSChenbo Feng { 4446e71b04aSChenbo Feng if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) 4456e71b04aSChenbo Feng return -EINVAL; 4466e71b04aSChenbo Feng if (flags & BPF_F_RDONLY) 4476e71b04aSChenbo Feng return O_RDONLY; 4486e71b04aSChenbo Feng if (flags & BPF_F_WRONLY) 4496e71b04aSChenbo Feng return O_WRONLY; 4506e71b04aSChenbo Feng return O_RDWR; 451aa79781bSDaniel Borkmann } 452aa79781bSDaniel Borkmann 45399c55f7dSAlexei Starovoitov /* helper macro to check that unused fields 'union bpf_attr' are zero */ 45499c55f7dSAlexei Starovoitov #define CHECK_ATTR(CMD) \ 45599c55f7dSAlexei Starovoitov memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 45699c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD), 0, \ 45799c55f7dSAlexei Starovoitov sizeof(*attr) - \ 45899c55f7dSAlexei Starovoitov offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 45999c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD)) != NULL 46099c55f7dSAlexei Starovoitov 461cb4d2b3fSMartin KaFai Lau /* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes. 462cb4d2b3fSMartin KaFai Lau * Return 0 on success and < 0 on error. 463cb4d2b3fSMartin KaFai Lau */ 464cb4d2b3fSMartin KaFai Lau static int bpf_obj_name_cpy(char *dst, const char *src) 465cb4d2b3fSMartin KaFai Lau { 466cb4d2b3fSMartin KaFai Lau const char *end = src + BPF_OBJ_NAME_LEN; 467cb4d2b3fSMartin KaFai Lau 468473d9734SMartin KaFai Lau memset(dst, 0, BPF_OBJ_NAME_LEN); 4693e0ddc4fSDaniel Borkmann /* Copy all isalnum(), '_' and '.' chars. */ 470cb4d2b3fSMartin KaFai Lau while (src < end && *src) { 4713e0ddc4fSDaniel Borkmann if (!isalnum(*src) && 4723e0ddc4fSDaniel Borkmann *src != '_' && *src != '.') 473cb4d2b3fSMartin KaFai Lau return -EINVAL; 474cb4d2b3fSMartin KaFai Lau *dst++ = *src++; 475cb4d2b3fSMartin KaFai Lau } 476cb4d2b3fSMartin KaFai Lau 477cb4d2b3fSMartin KaFai Lau /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */ 478cb4d2b3fSMartin KaFai Lau if (src == end) 479cb4d2b3fSMartin KaFai Lau return -EINVAL; 480cb4d2b3fSMartin KaFai Lau 481cb4d2b3fSMartin KaFai Lau return 0; 482cb4d2b3fSMartin KaFai Lau } 483cb4d2b3fSMartin KaFai Lau 484e8d2bec0SDaniel Borkmann int map_check_no_btf(const struct bpf_map *map, 4851b2b234bSRoman Gushchin const struct btf *btf, 486e8d2bec0SDaniel Borkmann const struct btf_type *key_type, 487e8d2bec0SDaniel Borkmann const struct btf_type *value_type) 488e8d2bec0SDaniel Borkmann { 489e8d2bec0SDaniel Borkmann return -ENOTSUPP; 490e8d2bec0SDaniel Borkmann } 491e8d2bec0SDaniel Borkmann 492d83525caSAlexei Starovoitov static int map_check_btf(struct bpf_map *map, const struct btf *btf, 493e8d2bec0SDaniel Borkmann u32 btf_key_id, u32 btf_value_id) 494e8d2bec0SDaniel Borkmann { 495e8d2bec0SDaniel Borkmann const struct btf_type *key_type, *value_type; 496e8d2bec0SDaniel Borkmann u32 key_size, value_size; 497e8d2bec0SDaniel Borkmann int ret = 0; 498e8d2bec0SDaniel Borkmann 4992824ecb7SDaniel Borkmann /* Some maps allow key to be unspecified. */ 5002824ecb7SDaniel Borkmann if (btf_key_id) { 501e8d2bec0SDaniel Borkmann key_type = btf_type_id_size(btf, &btf_key_id, &key_size); 502e8d2bec0SDaniel Borkmann if (!key_type || key_size != map->key_size) 503e8d2bec0SDaniel Borkmann return -EINVAL; 5042824ecb7SDaniel Borkmann } else { 5052824ecb7SDaniel Borkmann key_type = btf_type_by_id(btf, 0); 5062824ecb7SDaniel Borkmann if (!map->ops->map_check_btf) 5072824ecb7SDaniel Borkmann return -EINVAL; 5082824ecb7SDaniel Borkmann } 509e8d2bec0SDaniel Borkmann 510e8d2bec0SDaniel Borkmann value_type = btf_type_id_size(btf, &btf_value_id, &value_size); 511e8d2bec0SDaniel Borkmann if (!value_type || value_size != map->value_size) 512e8d2bec0SDaniel Borkmann return -EINVAL; 513e8d2bec0SDaniel Borkmann 514d83525caSAlexei Starovoitov map->spin_lock_off = btf_find_spin_lock(btf, value_type); 515d83525caSAlexei Starovoitov 516d83525caSAlexei Starovoitov if (map_value_has_spin_lock(map)) { 517591fe988SDaniel Borkmann if (map->map_flags & BPF_F_RDONLY_PROG) 518591fe988SDaniel Borkmann return -EACCES; 519d83525caSAlexei Starovoitov if (map->map_type != BPF_MAP_TYPE_HASH && 520e16d2f1aSAlexei Starovoitov map->map_type != BPF_MAP_TYPE_ARRAY && 5216ac99e8fSMartin KaFai Lau map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 5226ac99e8fSMartin KaFai Lau map->map_type != BPF_MAP_TYPE_SK_STORAGE) 523d83525caSAlexei Starovoitov return -ENOTSUPP; 524d83525caSAlexei Starovoitov if (map->spin_lock_off + sizeof(struct bpf_spin_lock) > 525d83525caSAlexei Starovoitov map->value_size) { 526d83525caSAlexei Starovoitov WARN_ONCE(1, 527d83525caSAlexei Starovoitov "verifier bug spin_lock_off %d value_size %d\n", 528d83525caSAlexei Starovoitov map->spin_lock_off, map->value_size); 529d83525caSAlexei Starovoitov return -EFAULT; 530d83525caSAlexei Starovoitov } 531d83525caSAlexei Starovoitov } 532d83525caSAlexei Starovoitov 533e8d2bec0SDaniel Borkmann if (map->ops->map_check_btf) 5341b2b234bSRoman Gushchin ret = map->ops->map_check_btf(map, btf, key_type, value_type); 535e8d2bec0SDaniel Borkmann 536e8d2bec0SDaniel Borkmann return ret; 537e8d2bec0SDaniel Borkmann } 538e8d2bec0SDaniel Borkmann 5399b2cf328SMartin KaFai Lau #define BPF_MAP_CREATE_LAST_FIELD btf_value_type_id 54099c55f7dSAlexei Starovoitov /* called via syscall */ 54199c55f7dSAlexei Starovoitov static int map_create(union bpf_attr *attr) 54299c55f7dSAlexei Starovoitov { 54396eabe7aSMartin KaFai Lau int numa_node = bpf_map_attr_numa_node(attr); 54499c55f7dSAlexei Starovoitov struct bpf_map *map; 5456e71b04aSChenbo Feng int f_flags; 54699c55f7dSAlexei Starovoitov int err; 54799c55f7dSAlexei Starovoitov 54899c55f7dSAlexei Starovoitov err = CHECK_ATTR(BPF_MAP_CREATE); 54999c55f7dSAlexei Starovoitov if (err) 55099c55f7dSAlexei Starovoitov return -EINVAL; 55199c55f7dSAlexei Starovoitov 5526e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->map_flags); 5536e71b04aSChenbo Feng if (f_flags < 0) 5546e71b04aSChenbo Feng return f_flags; 5556e71b04aSChenbo Feng 55696eabe7aSMartin KaFai Lau if (numa_node != NUMA_NO_NODE && 55796e5ae4eSEric Dumazet ((unsigned int)numa_node >= nr_node_ids || 55896e5ae4eSEric Dumazet !node_online(numa_node))) 55996eabe7aSMartin KaFai Lau return -EINVAL; 56096eabe7aSMartin KaFai Lau 56199c55f7dSAlexei Starovoitov /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 56299c55f7dSAlexei Starovoitov map = find_and_alloc_map(attr); 56399c55f7dSAlexei Starovoitov if (IS_ERR(map)) 56499c55f7dSAlexei Starovoitov return PTR_ERR(map); 56599c55f7dSAlexei Starovoitov 566ad5b177bSMartin KaFai Lau err = bpf_obj_name_cpy(map->name, attr->map_name); 567ad5b177bSMartin KaFai Lau if (err) 568ad5b177bSMartin KaFai Lau goto free_map_nouncharge; 569ad5b177bSMartin KaFai Lau 57099c55f7dSAlexei Starovoitov atomic_set(&map->refcnt, 1); 571c9da161cSDaniel Borkmann atomic_set(&map->usercnt, 1); 57299c55f7dSAlexei Starovoitov 573e8d2bec0SDaniel Borkmann if (attr->btf_key_type_id || attr->btf_value_type_id) { 574a26ca7c9SMartin KaFai Lau struct btf *btf; 575a26ca7c9SMartin KaFai Lau 5762824ecb7SDaniel Borkmann if (!attr->btf_value_type_id) { 577a26ca7c9SMartin KaFai Lau err = -EINVAL; 578a26ca7c9SMartin KaFai Lau goto free_map_nouncharge; 579a26ca7c9SMartin KaFai Lau } 580a26ca7c9SMartin KaFai Lau 581a26ca7c9SMartin KaFai Lau btf = btf_get_by_fd(attr->btf_fd); 582a26ca7c9SMartin KaFai Lau if (IS_ERR(btf)) { 583a26ca7c9SMartin KaFai Lau err = PTR_ERR(btf); 584a26ca7c9SMartin KaFai Lau goto free_map_nouncharge; 585a26ca7c9SMartin KaFai Lau } 586a26ca7c9SMartin KaFai Lau 587e8d2bec0SDaniel Borkmann err = map_check_btf(map, btf, attr->btf_key_type_id, 5889b2cf328SMartin KaFai Lau attr->btf_value_type_id); 589a26ca7c9SMartin KaFai Lau if (err) { 590a26ca7c9SMartin KaFai Lau btf_put(btf); 591a26ca7c9SMartin KaFai Lau goto free_map_nouncharge; 592a26ca7c9SMartin KaFai Lau } 593a26ca7c9SMartin KaFai Lau 594a26ca7c9SMartin KaFai Lau map->btf = btf; 5959b2cf328SMartin KaFai Lau map->btf_key_type_id = attr->btf_key_type_id; 5969b2cf328SMartin KaFai Lau map->btf_value_type_id = attr->btf_value_type_id; 597d83525caSAlexei Starovoitov } else { 598d83525caSAlexei Starovoitov map->spin_lock_off = -EINVAL; 599a26ca7c9SMartin KaFai Lau } 600a26ca7c9SMartin KaFai Lau 601afdb09c7SChenbo Feng err = security_bpf_map_alloc(map); 602aaac3ba9SAlexei Starovoitov if (err) 60320b2b24fSDaniel Borkmann goto free_map_nouncharge; 604aaac3ba9SAlexei Starovoitov 6050a4c58f5SRoman Gushchin err = bpf_map_init_memlock(map); 606afdb09c7SChenbo Feng if (err) 607afdb09c7SChenbo Feng goto free_map_sec; 608afdb09c7SChenbo Feng 609f3f1c054SMartin KaFai Lau err = bpf_map_alloc_id(map); 610f3f1c054SMartin KaFai Lau if (err) 611f3f1c054SMartin KaFai Lau goto free_map; 612f3f1c054SMartin KaFai Lau 6136e71b04aSChenbo Feng err = bpf_map_new_fd(map, f_flags); 614bd5f5f4eSMartin KaFai Lau if (err < 0) { 615bd5f5f4eSMartin KaFai Lau /* failed to allocate fd. 616352d20d6SPeng Sun * bpf_map_put_with_uref() is needed because the above 617bd5f5f4eSMartin KaFai Lau * bpf_map_alloc_id() has published the map 618bd5f5f4eSMartin KaFai Lau * to the userspace and the userspace may 619bd5f5f4eSMartin KaFai Lau * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 620bd5f5f4eSMartin KaFai Lau */ 621352d20d6SPeng Sun bpf_map_put_with_uref(map); 622bd5f5f4eSMartin KaFai Lau return err; 623bd5f5f4eSMartin KaFai Lau } 62499c55f7dSAlexei Starovoitov 62599c55f7dSAlexei Starovoitov return err; 62699c55f7dSAlexei Starovoitov 62799c55f7dSAlexei Starovoitov free_map: 6280a4c58f5SRoman Gushchin bpf_map_release_memlock(map); 629afdb09c7SChenbo Feng free_map_sec: 630afdb09c7SChenbo Feng security_bpf_map_free(map); 63120b2b24fSDaniel Borkmann free_map_nouncharge: 632a26ca7c9SMartin KaFai Lau btf_put(map->btf); 63399c55f7dSAlexei Starovoitov map->ops->map_free(map); 63499c55f7dSAlexei Starovoitov return err; 63599c55f7dSAlexei Starovoitov } 63699c55f7dSAlexei Starovoitov 637db20fd2bSAlexei Starovoitov /* if error is returned, fd is released. 638db20fd2bSAlexei Starovoitov * On success caller should complete fd access with matching fdput() 639db20fd2bSAlexei Starovoitov */ 640c2101297SDaniel Borkmann struct bpf_map *__bpf_map_get(struct fd f) 641db20fd2bSAlexei Starovoitov { 642db20fd2bSAlexei Starovoitov if (!f.file) 643db20fd2bSAlexei Starovoitov return ERR_PTR(-EBADF); 644db20fd2bSAlexei Starovoitov if (f.file->f_op != &bpf_map_fops) { 645db20fd2bSAlexei Starovoitov fdput(f); 646db20fd2bSAlexei Starovoitov return ERR_PTR(-EINVAL); 647db20fd2bSAlexei Starovoitov } 648db20fd2bSAlexei Starovoitov 649c2101297SDaniel Borkmann return f.file->private_data; 650c2101297SDaniel Borkmann } 651c2101297SDaniel Borkmann 65292117d84SAlexei Starovoitov /* prog's and map's refcnt limit */ 65392117d84SAlexei Starovoitov #define BPF_MAX_REFCNT 32768 65492117d84SAlexei Starovoitov 65592117d84SAlexei Starovoitov struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref) 656c9da161cSDaniel Borkmann { 65792117d84SAlexei Starovoitov if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) { 65892117d84SAlexei Starovoitov atomic_dec(&map->refcnt); 65992117d84SAlexei Starovoitov return ERR_PTR(-EBUSY); 66092117d84SAlexei Starovoitov } 661c9da161cSDaniel Borkmann if (uref) 662c9da161cSDaniel Borkmann atomic_inc(&map->usercnt); 66392117d84SAlexei Starovoitov return map; 664c9da161cSDaniel Borkmann } 665630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_inc); 666c9da161cSDaniel Borkmann 667c9da161cSDaniel Borkmann struct bpf_map *bpf_map_get_with_uref(u32 ufd) 668c2101297SDaniel Borkmann { 669c2101297SDaniel Borkmann struct fd f = fdget(ufd); 670c2101297SDaniel Borkmann struct bpf_map *map; 671c2101297SDaniel Borkmann 672c2101297SDaniel Borkmann map = __bpf_map_get(f); 673c2101297SDaniel Borkmann if (IS_ERR(map)) 674c2101297SDaniel Borkmann return map; 675c2101297SDaniel Borkmann 67692117d84SAlexei Starovoitov map = bpf_map_inc(map, true); 677c2101297SDaniel Borkmann fdput(f); 678db20fd2bSAlexei Starovoitov 679db20fd2bSAlexei Starovoitov return map; 680db20fd2bSAlexei Starovoitov } 681db20fd2bSAlexei Starovoitov 682bd5f5f4eSMartin KaFai Lau /* map_idr_lock should have been held */ 683bd5f5f4eSMartin KaFai Lau static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map, 684bd5f5f4eSMartin KaFai Lau bool uref) 685bd5f5f4eSMartin KaFai Lau { 686bd5f5f4eSMartin KaFai Lau int refold; 687bd5f5f4eSMartin KaFai Lau 688bfc18e38SMark Rutland refold = atomic_fetch_add_unless(&map->refcnt, 1, 0); 689bd5f5f4eSMartin KaFai Lau 690bd5f5f4eSMartin KaFai Lau if (refold >= BPF_MAX_REFCNT) { 691bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, false); 692bd5f5f4eSMartin KaFai Lau return ERR_PTR(-EBUSY); 693bd5f5f4eSMartin KaFai Lau } 694bd5f5f4eSMartin KaFai Lau 695bd5f5f4eSMartin KaFai Lau if (!refold) 696bd5f5f4eSMartin KaFai Lau return ERR_PTR(-ENOENT); 697bd5f5f4eSMartin KaFai Lau 698bd5f5f4eSMartin KaFai Lau if (uref) 699bd5f5f4eSMartin KaFai Lau atomic_inc(&map->usercnt); 700bd5f5f4eSMartin KaFai Lau 701bd5f5f4eSMartin KaFai Lau return map; 702bd5f5f4eSMartin KaFai Lau } 703bd5f5f4eSMartin KaFai Lau 704b8cdc051SAlexei Starovoitov int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 705b8cdc051SAlexei Starovoitov { 706b8cdc051SAlexei Starovoitov return -ENOTSUPP; 707b8cdc051SAlexei Starovoitov } 708b8cdc051SAlexei Starovoitov 709c9d29f46SMauricio Vasquez B static void *__bpf_copy_key(void __user *ukey, u64 key_size) 710c9d29f46SMauricio Vasquez B { 711c9d29f46SMauricio Vasquez B if (key_size) 712c9d29f46SMauricio Vasquez B return memdup_user(ukey, key_size); 713c9d29f46SMauricio Vasquez B 714c9d29f46SMauricio Vasquez B if (ukey) 715c9d29f46SMauricio Vasquez B return ERR_PTR(-EINVAL); 716c9d29f46SMauricio Vasquez B 717c9d29f46SMauricio Vasquez B return NULL; 718c9d29f46SMauricio Vasquez B } 719c9d29f46SMauricio Vasquez B 720db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 72196049f3aSAlexei Starovoitov #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags 722db20fd2bSAlexei Starovoitov 723db20fd2bSAlexei Starovoitov static int map_lookup_elem(union bpf_attr *attr) 724db20fd2bSAlexei Starovoitov { 725535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 726535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 727db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 728db20fd2bSAlexei Starovoitov struct bpf_map *map; 7298ebe667cSAlexei Starovoitov void *key, *value, *ptr; 73015a07b33SAlexei Starovoitov u32 value_size; 731592867bfSDaniel Borkmann struct fd f; 732db20fd2bSAlexei Starovoitov int err; 733db20fd2bSAlexei Starovoitov 734db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 735db20fd2bSAlexei Starovoitov return -EINVAL; 736db20fd2bSAlexei Starovoitov 73796049f3aSAlexei Starovoitov if (attr->flags & ~BPF_F_LOCK) 73896049f3aSAlexei Starovoitov return -EINVAL; 73996049f3aSAlexei Starovoitov 740592867bfSDaniel Borkmann f = fdget(ufd); 741c2101297SDaniel Borkmann map = __bpf_map_get(f); 742db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 743db20fd2bSAlexei Starovoitov return PTR_ERR(map); 74487df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 7456e71b04aSChenbo Feng err = -EPERM; 7466e71b04aSChenbo Feng goto err_put; 7476e71b04aSChenbo Feng } 7486e71b04aSChenbo Feng 74996049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 75096049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 75196049f3aSAlexei Starovoitov err = -EINVAL; 75296049f3aSAlexei Starovoitov goto err_put; 75396049f3aSAlexei Starovoitov } 75496049f3aSAlexei Starovoitov 755c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 756e4448ed8SAl Viro if (IS_ERR(key)) { 757e4448ed8SAl Viro err = PTR_ERR(key); 758db20fd2bSAlexei Starovoitov goto err_put; 759e4448ed8SAl Viro } 760db20fd2bSAlexei Starovoitov 76115a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 7628f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 763b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 764b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 76515a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 76614dc6f04SMartin KaFai Lau else if (IS_FD_MAP(map)) 76714dc6f04SMartin KaFai Lau value_size = sizeof(u32); 76815a07b33SAlexei Starovoitov else 76915a07b33SAlexei Starovoitov value_size = map->value_size; 77015a07b33SAlexei Starovoitov 7718ebe667cSAlexei Starovoitov err = -ENOMEM; 77215a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 773db20fd2bSAlexei Starovoitov if (!value) 7748ebe667cSAlexei Starovoitov goto free_key; 7758ebe667cSAlexei Starovoitov 776a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 777a3884572SJakub Kicinski err = bpf_map_offload_lookup_elem(map, key, value); 7787c4cd051SMartin KaFai Lau goto done; 7797c4cd051SMartin KaFai Lau } 7807c4cd051SMartin KaFai Lau 7817c4cd051SMartin KaFai Lau preempt_disable(); 7827c4cd051SMartin KaFai Lau this_cpu_inc(bpf_prog_active); 7837c4cd051SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 7848f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 78515a07b33SAlexei Starovoitov err = bpf_percpu_hash_copy(map, key, value); 78615a07b33SAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 78715a07b33SAlexei Starovoitov err = bpf_percpu_array_copy(map, key, value); 788b741f163SRoman Gushchin } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 789b741f163SRoman Gushchin err = bpf_percpu_cgroup_storage_copy(map, key, value); 790557c0c6eSAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 791557c0c6eSAlexei Starovoitov err = bpf_stackmap_copy(map, key, value); 79214dc6f04SMartin KaFai Lau } else if (IS_FD_ARRAY(map)) { 79314dc6f04SMartin KaFai Lau err = bpf_fd_array_map_lookup_elem(map, key, value); 79414dc6f04SMartin KaFai Lau } else if (IS_FD_HASH(map)) { 79514dc6f04SMartin KaFai Lau err = bpf_fd_htab_map_lookup_elem(map, key, value); 7965dc4c4b7SMartin KaFai Lau } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 7975dc4c4b7SMartin KaFai Lau err = bpf_fd_reuseport_array_lookup_elem(map, key, value); 798f1a2e44aSMauricio Vasquez B } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 799f1a2e44aSMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 800f1a2e44aSMauricio Vasquez B err = map->ops->map_peek_elem(map, value); 80115a07b33SAlexei Starovoitov } else { 8028ebe667cSAlexei Starovoitov rcu_read_lock(); 803c6110222SDaniel Borkmann if (map->ops->map_lookup_elem_sys_only) 804c6110222SDaniel Borkmann ptr = map->ops->map_lookup_elem_sys_only(map, key); 805c6110222SDaniel Borkmann else 8068ebe667cSAlexei Starovoitov ptr = map->ops->map_lookup_elem(map, key); 807509db283SPrashant Bhole if (IS_ERR(ptr)) { 808509db283SPrashant Bhole err = PTR_ERR(ptr); 809509db283SPrashant Bhole } else if (!ptr) { 810509db283SPrashant Bhole err = -ENOENT; 811509db283SPrashant Bhole } else { 812509db283SPrashant Bhole err = 0; 81396049f3aSAlexei Starovoitov if (attr->flags & BPF_F_LOCK) 81496049f3aSAlexei Starovoitov /* lock 'ptr' and copy everything but lock */ 81596049f3aSAlexei Starovoitov copy_map_value_locked(map, value, ptr, true); 81696049f3aSAlexei Starovoitov else 817d83525caSAlexei Starovoitov copy_map_value(map, value, ptr); 81896049f3aSAlexei Starovoitov /* mask lock, since value wasn't zero inited */ 81996049f3aSAlexei Starovoitov check_and_init_map_lock(map, value); 820509db283SPrashant Bhole } 8218ebe667cSAlexei Starovoitov rcu_read_unlock(); 82215a07b33SAlexei Starovoitov } 8237c4cd051SMartin KaFai Lau this_cpu_dec(bpf_prog_active); 8247c4cd051SMartin KaFai Lau preempt_enable(); 8258ebe667cSAlexei Starovoitov 8267c4cd051SMartin KaFai Lau done: 82715a07b33SAlexei Starovoitov if (err) 8288ebe667cSAlexei Starovoitov goto free_value; 829db20fd2bSAlexei Starovoitov 830db20fd2bSAlexei Starovoitov err = -EFAULT; 83115a07b33SAlexei Starovoitov if (copy_to_user(uvalue, value, value_size) != 0) 8328ebe667cSAlexei Starovoitov goto free_value; 833db20fd2bSAlexei Starovoitov 834db20fd2bSAlexei Starovoitov err = 0; 835db20fd2bSAlexei Starovoitov 8368ebe667cSAlexei Starovoitov free_value: 8378ebe667cSAlexei Starovoitov kfree(value); 838db20fd2bSAlexei Starovoitov free_key: 839db20fd2bSAlexei Starovoitov kfree(key); 840db20fd2bSAlexei Starovoitov err_put: 841db20fd2bSAlexei Starovoitov fdput(f); 842db20fd2bSAlexei Starovoitov return err; 843db20fd2bSAlexei Starovoitov } 844db20fd2bSAlexei Starovoitov 8451ae80cf3SDaniel Colascione static void maybe_wait_bpf_programs(struct bpf_map *map) 8461ae80cf3SDaniel Colascione { 8471ae80cf3SDaniel Colascione /* Wait for any running BPF programs to complete so that 8481ae80cf3SDaniel Colascione * userspace, when we return to it, knows that all programs 8491ae80cf3SDaniel Colascione * that could be running use the new map value. 8501ae80cf3SDaniel Colascione */ 8511ae80cf3SDaniel Colascione if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS || 8521ae80cf3SDaniel Colascione map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 8531ae80cf3SDaniel Colascione synchronize_rcu(); 8541ae80cf3SDaniel Colascione } 8551ae80cf3SDaniel Colascione 8563274f520SAlexei Starovoitov #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 857db20fd2bSAlexei Starovoitov 858db20fd2bSAlexei Starovoitov static int map_update_elem(union bpf_attr *attr) 859db20fd2bSAlexei Starovoitov { 860535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 861535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 862db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 863db20fd2bSAlexei Starovoitov struct bpf_map *map; 864db20fd2bSAlexei Starovoitov void *key, *value; 86515a07b33SAlexei Starovoitov u32 value_size; 866592867bfSDaniel Borkmann struct fd f; 867db20fd2bSAlexei Starovoitov int err; 868db20fd2bSAlexei Starovoitov 869db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 870db20fd2bSAlexei Starovoitov return -EINVAL; 871db20fd2bSAlexei Starovoitov 872592867bfSDaniel Borkmann f = fdget(ufd); 873c2101297SDaniel Borkmann map = __bpf_map_get(f); 874db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 875db20fd2bSAlexei Starovoitov return PTR_ERR(map); 87687df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 8776e71b04aSChenbo Feng err = -EPERM; 8786e71b04aSChenbo Feng goto err_put; 8796e71b04aSChenbo Feng } 8806e71b04aSChenbo Feng 88196049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 88296049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 88396049f3aSAlexei Starovoitov err = -EINVAL; 88496049f3aSAlexei Starovoitov goto err_put; 88596049f3aSAlexei Starovoitov } 88696049f3aSAlexei Starovoitov 887c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 888e4448ed8SAl Viro if (IS_ERR(key)) { 889e4448ed8SAl Viro err = PTR_ERR(key); 890db20fd2bSAlexei Starovoitov goto err_put; 891e4448ed8SAl Viro } 892db20fd2bSAlexei Starovoitov 89315a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 8948f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 895b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 896b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 89715a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 89815a07b33SAlexei Starovoitov else 89915a07b33SAlexei Starovoitov value_size = map->value_size; 90015a07b33SAlexei Starovoitov 901db20fd2bSAlexei Starovoitov err = -ENOMEM; 90215a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 903db20fd2bSAlexei Starovoitov if (!value) 904db20fd2bSAlexei Starovoitov goto free_key; 905db20fd2bSAlexei Starovoitov 906db20fd2bSAlexei Starovoitov err = -EFAULT; 90715a07b33SAlexei Starovoitov if (copy_from_user(value, uvalue, value_size) != 0) 908db20fd2bSAlexei Starovoitov goto free_value; 909db20fd2bSAlexei Starovoitov 9106710e112SJesper Dangaard Brouer /* Need to create a kthread, thus must support schedule */ 911a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 912a3884572SJakub Kicinski err = bpf_map_offload_update_elem(map, key, value, attr->flags); 913a3884572SJakub Kicinski goto out; 91499ba2b5aSJohn Fastabend } else if (map->map_type == BPF_MAP_TYPE_CPUMAP || 91599ba2b5aSJohn Fastabend map->map_type == BPF_MAP_TYPE_SOCKHASH || 91699ba2b5aSJohn Fastabend map->map_type == BPF_MAP_TYPE_SOCKMAP) { 9176710e112SJesper Dangaard Brouer err = map->ops->map_update_elem(map, key, value, attr->flags); 9186710e112SJesper Dangaard Brouer goto out; 9196710e112SJesper Dangaard Brouer } 9206710e112SJesper Dangaard Brouer 921b121d1e7SAlexei Starovoitov /* must increment bpf_prog_active to avoid kprobe+bpf triggering from 922b121d1e7SAlexei Starovoitov * inside bpf map update or delete otherwise deadlocks are possible 923b121d1e7SAlexei Starovoitov */ 924b121d1e7SAlexei Starovoitov preempt_disable(); 925b121d1e7SAlexei Starovoitov __this_cpu_inc(bpf_prog_active); 9268f844938SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 9278f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 92815a07b33SAlexei Starovoitov err = bpf_percpu_hash_update(map, key, value, attr->flags); 92915a07b33SAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 93015a07b33SAlexei Starovoitov err = bpf_percpu_array_update(map, key, value, attr->flags); 931b741f163SRoman Gushchin } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 932b741f163SRoman Gushchin err = bpf_percpu_cgroup_storage_update(map, key, value, 933b741f163SRoman Gushchin attr->flags); 9349c147b56SMickaël Salaün } else if (IS_FD_ARRAY(map)) { 935d056a788SDaniel Borkmann rcu_read_lock(); 936d056a788SDaniel Borkmann err = bpf_fd_array_map_update_elem(map, f.file, key, value, 937d056a788SDaniel Borkmann attr->flags); 938d056a788SDaniel Borkmann rcu_read_unlock(); 939bcc6b1b7SMartin KaFai Lau } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 940bcc6b1b7SMartin KaFai Lau rcu_read_lock(); 941bcc6b1b7SMartin KaFai Lau err = bpf_fd_htab_map_update_elem(map, f.file, key, value, 942bcc6b1b7SMartin KaFai Lau attr->flags); 943bcc6b1b7SMartin KaFai Lau rcu_read_unlock(); 9445dc4c4b7SMartin KaFai Lau } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 9455dc4c4b7SMartin KaFai Lau /* rcu_read_lock() is not needed */ 9465dc4c4b7SMartin KaFai Lau err = bpf_fd_reuseport_array_update_elem(map, key, value, 9475dc4c4b7SMartin KaFai Lau attr->flags); 948f1a2e44aSMauricio Vasquez B } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 949f1a2e44aSMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 950f1a2e44aSMauricio Vasquez B err = map->ops->map_push_elem(map, value, attr->flags); 95115a07b33SAlexei Starovoitov } else { 952db20fd2bSAlexei Starovoitov rcu_read_lock(); 9533274f520SAlexei Starovoitov err = map->ops->map_update_elem(map, key, value, attr->flags); 954db20fd2bSAlexei Starovoitov rcu_read_unlock(); 95515a07b33SAlexei Starovoitov } 956b121d1e7SAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 957b121d1e7SAlexei Starovoitov preempt_enable(); 9581ae80cf3SDaniel Colascione maybe_wait_bpf_programs(map); 9596710e112SJesper Dangaard Brouer out: 960db20fd2bSAlexei Starovoitov free_value: 961db20fd2bSAlexei Starovoitov kfree(value); 962db20fd2bSAlexei Starovoitov free_key: 963db20fd2bSAlexei Starovoitov kfree(key); 964db20fd2bSAlexei Starovoitov err_put: 965db20fd2bSAlexei Starovoitov fdput(f); 966db20fd2bSAlexei Starovoitov return err; 967db20fd2bSAlexei Starovoitov } 968db20fd2bSAlexei Starovoitov 969db20fd2bSAlexei Starovoitov #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 970db20fd2bSAlexei Starovoitov 971db20fd2bSAlexei Starovoitov static int map_delete_elem(union bpf_attr *attr) 972db20fd2bSAlexei Starovoitov { 973535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 974db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 975db20fd2bSAlexei Starovoitov struct bpf_map *map; 976592867bfSDaniel Borkmann struct fd f; 977db20fd2bSAlexei Starovoitov void *key; 978db20fd2bSAlexei Starovoitov int err; 979db20fd2bSAlexei Starovoitov 980db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 981db20fd2bSAlexei Starovoitov return -EINVAL; 982db20fd2bSAlexei Starovoitov 983592867bfSDaniel Borkmann f = fdget(ufd); 984c2101297SDaniel Borkmann map = __bpf_map_get(f); 985db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 986db20fd2bSAlexei Starovoitov return PTR_ERR(map); 98787df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 9886e71b04aSChenbo Feng err = -EPERM; 9896e71b04aSChenbo Feng goto err_put; 9906e71b04aSChenbo Feng } 9916e71b04aSChenbo Feng 992c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 993e4448ed8SAl Viro if (IS_ERR(key)) { 994e4448ed8SAl Viro err = PTR_ERR(key); 995db20fd2bSAlexei Starovoitov goto err_put; 996e4448ed8SAl Viro } 997db20fd2bSAlexei Starovoitov 998a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 999a3884572SJakub Kicinski err = bpf_map_offload_delete_elem(map, key); 1000a3884572SJakub Kicinski goto out; 1001a3884572SJakub Kicinski } 1002a3884572SJakub Kicinski 1003b121d1e7SAlexei Starovoitov preempt_disable(); 1004b121d1e7SAlexei Starovoitov __this_cpu_inc(bpf_prog_active); 1005db20fd2bSAlexei Starovoitov rcu_read_lock(); 1006db20fd2bSAlexei Starovoitov err = map->ops->map_delete_elem(map, key); 1007db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1008b121d1e7SAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 1009b121d1e7SAlexei Starovoitov preempt_enable(); 10101ae80cf3SDaniel Colascione maybe_wait_bpf_programs(map); 1011a3884572SJakub Kicinski out: 1012db20fd2bSAlexei Starovoitov kfree(key); 1013db20fd2bSAlexei Starovoitov err_put: 1014db20fd2bSAlexei Starovoitov fdput(f); 1015db20fd2bSAlexei Starovoitov return err; 1016db20fd2bSAlexei Starovoitov } 1017db20fd2bSAlexei Starovoitov 1018db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 1019db20fd2bSAlexei Starovoitov #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 1020db20fd2bSAlexei Starovoitov 1021db20fd2bSAlexei Starovoitov static int map_get_next_key(union bpf_attr *attr) 1022db20fd2bSAlexei Starovoitov { 1023535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1024535e7b4bSMickaël Salaün void __user *unext_key = u64_to_user_ptr(attr->next_key); 1025db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1026db20fd2bSAlexei Starovoitov struct bpf_map *map; 1027db20fd2bSAlexei Starovoitov void *key, *next_key; 1028592867bfSDaniel Borkmann struct fd f; 1029db20fd2bSAlexei Starovoitov int err; 1030db20fd2bSAlexei Starovoitov 1031db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 1032db20fd2bSAlexei Starovoitov return -EINVAL; 1033db20fd2bSAlexei Starovoitov 1034592867bfSDaniel Borkmann f = fdget(ufd); 1035c2101297SDaniel Borkmann map = __bpf_map_get(f); 1036db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1037db20fd2bSAlexei Starovoitov return PTR_ERR(map); 103887df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 10396e71b04aSChenbo Feng err = -EPERM; 10406e71b04aSChenbo Feng goto err_put; 10416e71b04aSChenbo Feng } 10426e71b04aSChenbo Feng 10438fe45924STeng Qin if (ukey) { 1044c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1045e4448ed8SAl Viro if (IS_ERR(key)) { 1046e4448ed8SAl Viro err = PTR_ERR(key); 1047db20fd2bSAlexei Starovoitov goto err_put; 1048e4448ed8SAl Viro } 10498fe45924STeng Qin } else { 10508fe45924STeng Qin key = NULL; 10518fe45924STeng Qin } 1052db20fd2bSAlexei Starovoitov 1053db20fd2bSAlexei Starovoitov err = -ENOMEM; 1054db20fd2bSAlexei Starovoitov next_key = kmalloc(map->key_size, GFP_USER); 1055db20fd2bSAlexei Starovoitov if (!next_key) 1056db20fd2bSAlexei Starovoitov goto free_key; 1057db20fd2bSAlexei Starovoitov 1058a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1059a3884572SJakub Kicinski err = bpf_map_offload_get_next_key(map, key, next_key); 1060a3884572SJakub Kicinski goto out; 1061a3884572SJakub Kicinski } 1062a3884572SJakub Kicinski 1063db20fd2bSAlexei Starovoitov rcu_read_lock(); 1064db20fd2bSAlexei Starovoitov err = map->ops->map_get_next_key(map, key, next_key); 1065db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1066a3884572SJakub Kicinski out: 1067db20fd2bSAlexei Starovoitov if (err) 1068db20fd2bSAlexei Starovoitov goto free_next_key; 1069db20fd2bSAlexei Starovoitov 1070db20fd2bSAlexei Starovoitov err = -EFAULT; 1071db20fd2bSAlexei Starovoitov if (copy_to_user(unext_key, next_key, map->key_size) != 0) 1072db20fd2bSAlexei Starovoitov goto free_next_key; 1073db20fd2bSAlexei Starovoitov 1074db20fd2bSAlexei Starovoitov err = 0; 1075db20fd2bSAlexei Starovoitov 1076db20fd2bSAlexei Starovoitov free_next_key: 1077db20fd2bSAlexei Starovoitov kfree(next_key); 1078db20fd2bSAlexei Starovoitov free_key: 1079db20fd2bSAlexei Starovoitov kfree(key); 1080db20fd2bSAlexei Starovoitov err_put: 1081db20fd2bSAlexei Starovoitov fdput(f); 1082db20fd2bSAlexei Starovoitov return err; 1083db20fd2bSAlexei Starovoitov } 1084db20fd2bSAlexei Starovoitov 1085bd513cd0SMauricio Vasquez B #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value 1086bd513cd0SMauricio Vasquez B 1087bd513cd0SMauricio Vasquez B static int map_lookup_and_delete_elem(union bpf_attr *attr) 1088bd513cd0SMauricio Vasquez B { 1089bd513cd0SMauricio Vasquez B void __user *ukey = u64_to_user_ptr(attr->key); 1090bd513cd0SMauricio Vasquez B void __user *uvalue = u64_to_user_ptr(attr->value); 1091bd513cd0SMauricio Vasquez B int ufd = attr->map_fd; 1092bd513cd0SMauricio Vasquez B struct bpf_map *map; 1093540fefc0SAlexei Starovoitov void *key, *value; 1094bd513cd0SMauricio Vasquez B u32 value_size; 1095bd513cd0SMauricio Vasquez B struct fd f; 1096bd513cd0SMauricio Vasquez B int err; 1097bd513cd0SMauricio Vasquez B 1098bd513cd0SMauricio Vasquez B if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) 1099bd513cd0SMauricio Vasquez B return -EINVAL; 1100bd513cd0SMauricio Vasquez B 1101bd513cd0SMauricio Vasquez B f = fdget(ufd); 1102bd513cd0SMauricio Vasquez B map = __bpf_map_get(f); 1103bd513cd0SMauricio Vasquez B if (IS_ERR(map)) 1104bd513cd0SMauricio Vasquez B return PTR_ERR(map); 110587df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 1106bd513cd0SMauricio Vasquez B err = -EPERM; 1107bd513cd0SMauricio Vasquez B goto err_put; 1108bd513cd0SMauricio Vasquez B } 1109bd513cd0SMauricio Vasquez B 1110bd513cd0SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1111bd513cd0SMauricio Vasquez B if (IS_ERR(key)) { 1112bd513cd0SMauricio Vasquez B err = PTR_ERR(key); 1113bd513cd0SMauricio Vasquez B goto err_put; 1114bd513cd0SMauricio Vasquez B } 1115bd513cd0SMauricio Vasquez B 1116bd513cd0SMauricio Vasquez B value_size = map->value_size; 1117bd513cd0SMauricio Vasquez B 1118bd513cd0SMauricio Vasquez B err = -ENOMEM; 1119bd513cd0SMauricio Vasquez B value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1120bd513cd0SMauricio Vasquez B if (!value) 1121bd513cd0SMauricio Vasquez B goto free_key; 1122bd513cd0SMauricio Vasquez B 1123bd513cd0SMauricio Vasquez B if (map->map_type == BPF_MAP_TYPE_QUEUE || 1124bd513cd0SMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 1125bd513cd0SMauricio Vasquez B err = map->ops->map_pop_elem(map, value); 1126bd513cd0SMauricio Vasquez B } else { 1127bd513cd0SMauricio Vasquez B err = -ENOTSUPP; 1128bd513cd0SMauricio Vasquez B } 1129bd513cd0SMauricio Vasquez B 1130bd513cd0SMauricio Vasquez B if (err) 1131bd513cd0SMauricio Vasquez B goto free_value; 1132bd513cd0SMauricio Vasquez B 1133bd513cd0SMauricio Vasquez B if (copy_to_user(uvalue, value, value_size) != 0) 1134bd513cd0SMauricio Vasquez B goto free_value; 1135bd513cd0SMauricio Vasquez B 1136bd513cd0SMauricio Vasquez B err = 0; 1137bd513cd0SMauricio Vasquez B 1138bd513cd0SMauricio Vasquez B free_value: 1139bd513cd0SMauricio Vasquez B kfree(value); 1140bd513cd0SMauricio Vasquez B free_key: 1141bd513cd0SMauricio Vasquez B kfree(key); 1142bd513cd0SMauricio Vasquez B err_put: 1143bd513cd0SMauricio Vasquez B fdput(f); 1144bd513cd0SMauricio Vasquez B return err; 1145bd513cd0SMauricio Vasquez B } 1146bd513cd0SMauricio Vasquez B 114787df15deSDaniel Borkmann #define BPF_MAP_FREEZE_LAST_FIELD map_fd 114887df15deSDaniel Borkmann 114987df15deSDaniel Borkmann static int map_freeze(const union bpf_attr *attr) 115087df15deSDaniel Borkmann { 115187df15deSDaniel Borkmann int err = 0, ufd = attr->map_fd; 115287df15deSDaniel Borkmann struct bpf_map *map; 115387df15deSDaniel Borkmann struct fd f; 115487df15deSDaniel Borkmann 115587df15deSDaniel Borkmann if (CHECK_ATTR(BPF_MAP_FREEZE)) 115687df15deSDaniel Borkmann return -EINVAL; 115787df15deSDaniel Borkmann 115887df15deSDaniel Borkmann f = fdget(ufd); 115987df15deSDaniel Borkmann map = __bpf_map_get(f); 116087df15deSDaniel Borkmann if (IS_ERR(map)) 116187df15deSDaniel Borkmann return PTR_ERR(map); 116287df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) { 116387df15deSDaniel Borkmann err = -EBUSY; 116487df15deSDaniel Borkmann goto err_put; 116587df15deSDaniel Borkmann } 116687df15deSDaniel Borkmann if (!capable(CAP_SYS_ADMIN)) { 116787df15deSDaniel Borkmann err = -EPERM; 116887df15deSDaniel Borkmann goto err_put; 116987df15deSDaniel Borkmann } 117087df15deSDaniel Borkmann 117187df15deSDaniel Borkmann WRITE_ONCE(map->frozen, true); 117287df15deSDaniel Borkmann err_put: 117387df15deSDaniel Borkmann fdput(f); 117487df15deSDaniel Borkmann return err; 117587df15deSDaniel Borkmann } 117687df15deSDaniel Borkmann 11777de16e3aSJakub Kicinski static const struct bpf_prog_ops * const bpf_prog_types[] = { 11787de16e3aSJakub Kicinski #define BPF_PROG_TYPE(_id, _name) \ 11797de16e3aSJakub Kicinski [_id] = & _name ## _prog_ops, 11807de16e3aSJakub Kicinski #define BPF_MAP_TYPE(_id, _ops) 11817de16e3aSJakub Kicinski #include <linux/bpf_types.h> 11827de16e3aSJakub Kicinski #undef BPF_PROG_TYPE 11837de16e3aSJakub Kicinski #undef BPF_MAP_TYPE 11847de16e3aSJakub Kicinski }; 11857de16e3aSJakub Kicinski 118609756af4SAlexei Starovoitov static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 118709756af4SAlexei Starovoitov { 1188d0f1a451SDaniel Borkmann const struct bpf_prog_ops *ops; 1189d0f1a451SDaniel Borkmann 1190d0f1a451SDaniel Borkmann if (type >= ARRAY_SIZE(bpf_prog_types)) 1191d0f1a451SDaniel Borkmann return -EINVAL; 1192d0f1a451SDaniel Borkmann type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types)); 1193d0f1a451SDaniel Borkmann ops = bpf_prog_types[type]; 1194d0f1a451SDaniel Borkmann if (!ops) 1195be9370a7SJohannes Berg return -EINVAL; 119609756af4SAlexei Starovoitov 1197ab3f0063SJakub Kicinski if (!bpf_prog_is_dev_bound(prog->aux)) 1198d0f1a451SDaniel Borkmann prog->aux->ops = ops; 1199ab3f0063SJakub Kicinski else 1200ab3f0063SJakub Kicinski prog->aux->ops = &bpf_offload_prog_ops; 120124701eceSDaniel Borkmann prog->type = type; 120209756af4SAlexei Starovoitov return 0; 120309756af4SAlexei Starovoitov } 120409756af4SAlexei Starovoitov 120509756af4SAlexei Starovoitov /* drop refcnt on maps used by eBPF program and free auxilary data */ 120609756af4SAlexei Starovoitov static void free_used_maps(struct bpf_prog_aux *aux) 120709756af4SAlexei Starovoitov { 12088bad74f9SRoman Gushchin enum bpf_cgroup_storage_type stype; 120909756af4SAlexei Starovoitov int i; 121009756af4SAlexei Starovoitov 12118bad74f9SRoman Gushchin for_each_cgroup_storage_type(stype) { 12128bad74f9SRoman Gushchin if (!aux->cgroup_storage[stype]) 12138bad74f9SRoman Gushchin continue; 12148bad74f9SRoman Gushchin bpf_cgroup_storage_release(aux->prog, 12158bad74f9SRoman Gushchin aux->cgroup_storage[stype]); 12168bad74f9SRoman Gushchin } 1217de9cbbaaSRoman Gushchin 121809756af4SAlexei Starovoitov for (i = 0; i < aux->used_map_cnt; i++) 121909756af4SAlexei Starovoitov bpf_map_put(aux->used_maps[i]); 122009756af4SAlexei Starovoitov 122109756af4SAlexei Starovoitov kfree(aux->used_maps); 122209756af4SAlexei Starovoitov } 122309756af4SAlexei Starovoitov 12245ccb071eSDaniel Borkmann int __bpf_prog_charge(struct user_struct *user, u32 pages) 12255ccb071eSDaniel Borkmann { 12265ccb071eSDaniel Borkmann unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 12275ccb071eSDaniel Borkmann unsigned long user_bufs; 12285ccb071eSDaniel Borkmann 12295ccb071eSDaniel Borkmann if (user) { 12305ccb071eSDaniel Borkmann user_bufs = atomic_long_add_return(pages, &user->locked_vm); 12315ccb071eSDaniel Borkmann if (user_bufs > memlock_limit) { 12325ccb071eSDaniel Borkmann atomic_long_sub(pages, &user->locked_vm); 12335ccb071eSDaniel Borkmann return -EPERM; 12345ccb071eSDaniel Borkmann } 12355ccb071eSDaniel Borkmann } 12365ccb071eSDaniel Borkmann 12375ccb071eSDaniel Borkmann return 0; 12385ccb071eSDaniel Borkmann } 12395ccb071eSDaniel Borkmann 12405ccb071eSDaniel Borkmann void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 12415ccb071eSDaniel Borkmann { 12425ccb071eSDaniel Borkmann if (user) 12435ccb071eSDaniel Borkmann atomic_long_sub(pages, &user->locked_vm); 12445ccb071eSDaniel Borkmann } 12455ccb071eSDaniel Borkmann 1246aaac3ba9SAlexei Starovoitov static int bpf_prog_charge_memlock(struct bpf_prog *prog) 1247aaac3ba9SAlexei Starovoitov { 1248aaac3ba9SAlexei Starovoitov struct user_struct *user = get_current_user(); 12495ccb071eSDaniel Borkmann int ret; 1250aaac3ba9SAlexei Starovoitov 12515ccb071eSDaniel Borkmann ret = __bpf_prog_charge(user, prog->pages); 12525ccb071eSDaniel Borkmann if (ret) { 1253aaac3ba9SAlexei Starovoitov free_uid(user); 12545ccb071eSDaniel Borkmann return ret; 1255aaac3ba9SAlexei Starovoitov } 12565ccb071eSDaniel Borkmann 1257aaac3ba9SAlexei Starovoitov prog->aux->user = user; 1258aaac3ba9SAlexei Starovoitov return 0; 1259aaac3ba9SAlexei Starovoitov } 1260aaac3ba9SAlexei Starovoitov 1261aaac3ba9SAlexei Starovoitov static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 1262aaac3ba9SAlexei Starovoitov { 1263aaac3ba9SAlexei Starovoitov struct user_struct *user = prog->aux->user; 1264aaac3ba9SAlexei Starovoitov 12655ccb071eSDaniel Borkmann __bpf_prog_uncharge(user, prog->pages); 1266aaac3ba9SAlexei Starovoitov free_uid(user); 1267aaac3ba9SAlexei Starovoitov } 1268aaac3ba9SAlexei Starovoitov 1269dc4bb0e2SMartin KaFai Lau static int bpf_prog_alloc_id(struct bpf_prog *prog) 1270dc4bb0e2SMartin KaFai Lau { 1271dc4bb0e2SMartin KaFai Lau int id; 1272dc4bb0e2SMartin KaFai Lau 1273b76354cdSShaohua Li idr_preload(GFP_KERNEL); 1274dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1275dc4bb0e2SMartin KaFai Lau id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 1276dc4bb0e2SMartin KaFai Lau if (id > 0) 1277dc4bb0e2SMartin KaFai Lau prog->aux->id = id; 1278dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1279b76354cdSShaohua Li idr_preload_end(); 1280dc4bb0e2SMartin KaFai Lau 1281dc4bb0e2SMartin KaFai Lau /* id is in [1, INT_MAX) */ 1282dc4bb0e2SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 1283dc4bb0e2SMartin KaFai Lau return -ENOSPC; 1284dc4bb0e2SMartin KaFai Lau 1285dc4bb0e2SMartin KaFai Lau return id > 0 ? 0 : id; 1286dc4bb0e2SMartin KaFai Lau } 1287dc4bb0e2SMartin KaFai Lau 1288ad8ad79fSJakub Kicinski void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) 1289dc4bb0e2SMartin KaFai Lau { 1290ad8ad79fSJakub Kicinski /* cBPF to eBPF migrations are currently not in the idr store. 1291ad8ad79fSJakub Kicinski * Offloaded programs are removed from the store when their device 1292ad8ad79fSJakub Kicinski * disappears - even if someone grabs an fd to them they are unusable, 1293ad8ad79fSJakub Kicinski * simply waiting for refcnt to drop to be freed. 1294ad8ad79fSJakub Kicinski */ 1295dc4bb0e2SMartin KaFai Lau if (!prog->aux->id) 1296dc4bb0e2SMartin KaFai Lau return; 1297dc4bb0e2SMartin KaFai Lau 1298b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1299dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1300b16d9aa4SMartin KaFai Lau else 1301b16d9aa4SMartin KaFai Lau __acquire(&prog_idr_lock); 1302b16d9aa4SMartin KaFai Lau 1303dc4bb0e2SMartin KaFai Lau idr_remove(&prog_idr, prog->aux->id); 1304ad8ad79fSJakub Kicinski prog->aux->id = 0; 1305b16d9aa4SMartin KaFai Lau 1306b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1307dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1308b16d9aa4SMartin KaFai Lau else 1309b16d9aa4SMartin KaFai Lau __release(&prog_idr_lock); 1310dc4bb0e2SMartin KaFai Lau } 1311dc4bb0e2SMartin KaFai Lau 13121aacde3dSDaniel Borkmann static void __bpf_prog_put_rcu(struct rcu_head *rcu) 1313abf2e7d6SAlexei Starovoitov { 1314abf2e7d6SAlexei Starovoitov struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 1315abf2e7d6SAlexei Starovoitov 1316abf2e7d6SAlexei Starovoitov free_used_maps(aux); 1317aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(aux->prog); 1318afdb09c7SChenbo Feng security_bpf_prog_free(aux); 1319abf2e7d6SAlexei Starovoitov bpf_prog_free(aux->prog); 1320abf2e7d6SAlexei Starovoitov } 1321abf2e7d6SAlexei Starovoitov 1322b16d9aa4SMartin KaFai Lau static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 132309756af4SAlexei Starovoitov { 1324a67edbf4SDaniel Borkmann if (atomic_dec_and_test(&prog->aux->refcnt)) { 13256ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0); 132634ad5580SMartin KaFai Lau /* bpf_prog_free_id() must be called first */ 1327b16d9aa4SMartin KaFai Lau bpf_prog_free_id(prog, do_idr_lock); 13287d1982b4SDaniel Borkmann bpf_prog_kallsyms_del_all(prog); 1329838e9690SYonghong Song btf_put(prog->aux->btf); 1330ba64e7d8SYonghong Song kvfree(prog->aux->func_info); 1331c454a46bSMartin KaFai Lau bpf_prog_free_linfo(prog); 13324f74d809SDaniel Borkmann 13331aacde3dSDaniel Borkmann call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 133409756af4SAlexei Starovoitov } 1335a67edbf4SDaniel Borkmann } 1336b16d9aa4SMartin KaFai Lau 1337b16d9aa4SMartin KaFai Lau void bpf_prog_put(struct bpf_prog *prog) 1338b16d9aa4SMartin KaFai Lau { 1339b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, true); 1340b16d9aa4SMartin KaFai Lau } 1341e2e9b654SDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_put); 134209756af4SAlexei Starovoitov 134309756af4SAlexei Starovoitov static int bpf_prog_release(struct inode *inode, struct file *filp) 134409756af4SAlexei Starovoitov { 134509756af4SAlexei Starovoitov struct bpf_prog *prog = filp->private_data; 134609756af4SAlexei Starovoitov 13471aacde3dSDaniel Borkmann bpf_prog_put(prog); 134809756af4SAlexei Starovoitov return 0; 134909756af4SAlexei Starovoitov } 135009756af4SAlexei Starovoitov 1351492ecee8SAlexei Starovoitov static void bpf_prog_get_stats(const struct bpf_prog *prog, 1352492ecee8SAlexei Starovoitov struct bpf_prog_stats *stats) 1353492ecee8SAlexei Starovoitov { 1354492ecee8SAlexei Starovoitov u64 nsecs = 0, cnt = 0; 1355492ecee8SAlexei Starovoitov int cpu; 1356492ecee8SAlexei Starovoitov 1357492ecee8SAlexei Starovoitov for_each_possible_cpu(cpu) { 1358492ecee8SAlexei Starovoitov const struct bpf_prog_stats *st; 1359492ecee8SAlexei Starovoitov unsigned int start; 1360492ecee8SAlexei Starovoitov u64 tnsecs, tcnt; 1361492ecee8SAlexei Starovoitov 1362492ecee8SAlexei Starovoitov st = per_cpu_ptr(prog->aux->stats, cpu); 1363492ecee8SAlexei Starovoitov do { 1364492ecee8SAlexei Starovoitov start = u64_stats_fetch_begin_irq(&st->syncp); 1365492ecee8SAlexei Starovoitov tnsecs = st->nsecs; 1366492ecee8SAlexei Starovoitov tcnt = st->cnt; 1367492ecee8SAlexei Starovoitov } while (u64_stats_fetch_retry_irq(&st->syncp, start)); 1368492ecee8SAlexei Starovoitov nsecs += tnsecs; 1369492ecee8SAlexei Starovoitov cnt += tcnt; 1370492ecee8SAlexei Starovoitov } 1371492ecee8SAlexei Starovoitov stats->nsecs = nsecs; 1372492ecee8SAlexei Starovoitov stats->cnt = cnt; 1373492ecee8SAlexei Starovoitov } 1374492ecee8SAlexei Starovoitov 13757bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 13767bd509e3SDaniel Borkmann static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 13777bd509e3SDaniel Borkmann { 13787bd509e3SDaniel Borkmann const struct bpf_prog *prog = filp->private_data; 1379f1f7714eSDaniel Borkmann char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 1380492ecee8SAlexei Starovoitov struct bpf_prog_stats stats; 13817bd509e3SDaniel Borkmann 1382492ecee8SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 1383f1f7714eSDaniel Borkmann bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 13847bd509e3SDaniel Borkmann seq_printf(m, 13857bd509e3SDaniel Borkmann "prog_type:\t%u\n" 13867bd509e3SDaniel Borkmann "prog_jited:\t%u\n" 1387f1f7714eSDaniel Borkmann "prog_tag:\t%s\n" 13884316b409SDaniel Borkmann "memlock:\t%llu\n" 1389492ecee8SAlexei Starovoitov "prog_id:\t%u\n" 1390492ecee8SAlexei Starovoitov "run_time_ns:\t%llu\n" 1391492ecee8SAlexei Starovoitov "run_cnt:\t%llu\n", 13927bd509e3SDaniel Borkmann prog->type, 13937bd509e3SDaniel Borkmann prog->jited, 1394f1f7714eSDaniel Borkmann prog_tag, 13954316b409SDaniel Borkmann prog->pages * 1ULL << PAGE_SHIFT, 1396492ecee8SAlexei Starovoitov prog->aux->id, 1397492ecee8SAlexei Starovoitov stats.nsecs, 1398492ecee8SAlexei Starovoitov stats.cnt); 13997bd509e3SDaniel Borkmann } 14007bd509e3SDaniel Borkmann #endif 14017bd509e3SDaniel Borkmann 1402f66e448cSChenbo Feng const struct file_operations bpf_prog_fops = { 14037bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 14047bd509e3SDaniel Borkmann .show_fdinfo = bpf_prog_show_fdinfo, 14057bd509e3SDaniel Borkmann #endif 140609756af4SAlexei Starovoitov .release = bpf_prog_release, 14076e71b04aSChenbo Feng .read = bpf_dummy_read, 14086e71b04aSChenbo Feng .write = bpf_dummy_write, 140909756af4SAlexei Starovoitov }; 141009756af4SAlexei Starovoitov 1411b2197755SDaniel Borkmann int bpf_prog_new_fd(struct bpf_prog *prog) 1412aa79781bSDaniel Borkmann { 1413afdb09c7SChenbo Feng int ret; 1414afdb09c7SChenbo Feng 1415afdb09c7SChenbo Feng ret = security_bpf_prog(prog); 1416afdb09c7SChenbo Feng if (ret < 0) 1417afdb09c7SChenbo Feng return ret; 1418afdb09c7SChenbo Feng 1419aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 1420aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC); 1421aa79781bSDaniel Borkmann } 1422aa79781bSDaniel Borkmann 1423113214beSDaniel Borkmann static struct bpf_prog *____bpf_prog_get(struct fd f) 142409756af4SAlexei Starovoitov { 142509756af4SAlexei Starovoitov if (!f.file) 142609756af4SAlexei Starovoitov return ERR_PTR(-EBADF); 142709756af4SAlexei Starovoitov if (f.file->f_op != &bpf_prog_fops) { 142809756af4SAlexei Starovoitov fdput(f); 142909756af4SAlexei Starovoitov return ERR_PTR(-EINVAL); 143009756af4SAlexei Starovoitov } 143109756af4SAlexei Starovoitov 1432c2101297SDaniel Borkmann return f.file->private_data; 143309756af4SAlexei Starovoitov } 143409756af4SAlexei Starovoitov 143559d3656dSBrenden Blanco struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i) 143692117d84SAlexei Starovoitov { 143759d3656dSBrenden Blanco if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) { 143859d3656dSBrenden Blanco atomic_sub(i, &prog->aux->refcnt); 143992117d84SAlexei Starovoitov return ERR_PTR(-EBUSY); 144092117d84SAlexei Starovoitov } 144192117d84SAlexei Starovoitov return prog; 144292117d84SAlexei Starovoitov } 144359d3656dSBrenden Blanco EXPORT_SYMBOL_GPL(bpf_prog_add); 144459d3656dSBrenden Blanco 1445c540594fSDaniel Borkmann void bpf_prog_sub(struct bpf_prog *prog, int i) 1446c540594fSDaniel Borkmann { 1447c540594fSDaniel Borkmann /* Only to be used for undoing previous bpf_prog_add() in some 1448c540594fSDaniel Borkmann * error path. We still know that another entity in our call 1449c540594fSDaniel Borkmann * path holds a reference to the program, thus atomic_sub() can 1450c540594fSDaniel Borkmann * be safely used in such cases! 1451c540594fSDaniel Borkmann */ 1452c540594fSDaniel Borkmann WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0); 1453c540594fSDaniel Borkmann } 1454c540594fSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_sub); 1455c540594fSDaniel Borkmann 145659d3656dSBrenden Blanco struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog) 145759d3656dSBrenden Blanco { 145859d3656dSBrenden Blanco return bpf_prog_add(prog, 1); 145959d3656dSBrenden Blanco } 146097bc402dSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_inc); 146192117d84SAlexei Starovoitov 1462b16d9aa4SMartin KaFai Lau /* prog_idr_lock should have been held */ 1463a6f6df69SJohn Fastabend struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 1464b16d9aa4SMartin KaFai Lau { 1465b16d9aa4SMartin KaFai Lau int refold; 1466b16d9aa4SMartin KaFai Lau 1467bfc18e38SMark Rutland refold = atomic_fetch_add_unless(&prog->aux->refcnt, 1, 0); 1468b16d9aa4SMartin KaFai Lau 1469b16d9aa4SMartin KaFai Lau if (refold >= BPF_MAX_REFCNT) { 1470b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, false); 1471b16d9aa4SMartin KaFai Lau return ERR_PTR(-EBUSY); 1472b16d9aa4SMartin KaFai Lau } 1473b16d9aa4SMartin KaFai Lau 1474b16d9aa4SMartin KaFai Lau if (!refold) 1475b16d9aa4SMartin KaFai Lau return ERR_PTR(-ENOENT); 1476b16d9aa4SMartin KaFai Lau 1477b16d9aa4SMartin KaFai Lau return prog; 1478b16d9aa4SMartin KaFai Lau } 1479a6f6df69SJohn Fastabend EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 1480b16d9aa4SMartin KaFai Lau 1481040ee692SAl Viro bool bpf_prog_get_ok(struct bpf_prog *prog, 1482288b3de5SJakub Kicinski enum bpf_prog_type *attach_type, bool attach_drv) 1483248f346fSJakub Kicinski { 1484288b3de5SJakub Kicinski /* not an attachment, just a refcount inc, always allow */ 1485288b3de5SJakub Kicinski if (!attach_type) 1486288b3de5SJakub Kicinski return true; 1487248f346fSJakub Kicinski 1488248f346fSJakub Kicinski if (prog->type != *attach_type) 1489248f346fSJakub Kicinski return false; 1490288b3de5SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv) 1491248f346fSJakub Kicinski return false; 1492248f346fSJakub Kicinski 1493248f346fSJakub Kicinski return true; 1494248f346fSJakub Kicinski } 1495248f346fSJakub Kicinski 1496248f346fSJakub Kicinski static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, 1497288b3de5SJakub Kicinski bool attach_drv) 149809756af4SAlexei Starovoitov { 149909756af4SAlexei Starovoitov struct fd f = fdget(ufd); 150009756af4SAlexei Starovoitov struct bpf_prog *prog; 150109756af4SAlexei Starovoitov 1502113214beSDaniel Borkmann prog = ____bpf_prog_get(f); 150309756af4SAlexei Starovoitov if (IS_ERR(prog)) 150409756af4SAlexei Starovoitov return prog; 1505288b3de5SJakub Kicinski if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) { 1506113214beSDaniel Borkmann prog = ERR_PTR(-EINVAL); 1507113214beSDaniel Borkmann goto out; 1508113214beSDaniel Borkmann } 150909756af4SAlexei Starovoitov 151092117d84SAlexei Starovoitov prog = bpf_prog_inc(prog); 1511113214beSDaniel Borkmann out: 151209756af4SAlexei Starovoitov fdput(f); 151309756af4SAlexei Starovoitov return prog; 151409756af4SAlexei Starovoitov } 1515113214beSDaniel Borkmann 1516113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get(u32 ufd) 1517113214beSDaniel Borkmann { 1518288b3de5SJakub Kicinski return __bpf_prog_get(ufd, NULL, false); 1519113214beSDaniel Borkmann } 1520113214beSDaniel Borkmann 1521248f346fSJakub Kicinski struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 1522288b3de5SJakub Kicinski bool attach_drv) 1523248f346fSJakub Kicinski { 15244d220ed0SAlexei Starovoitov return __bpf_prog_get(ufd, &type, attach_drv); 1525248f346fSJakub Kicinski } 15266c8dfe21SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev); 1527248f346fSJakub Kicinski 1528aac3fc32SAndrey Ignatov /* Initially all BPF programs could be loaded w/o specifying 1529aac3fc32SAndrey Ignatov * expected_attach_type. Later for some of them specifying expected_attach_type 1530aac3fc32SAndrey Ignatov * at load time became required so that program could be validated properly. 1531aac3fc32SAndrey Ignatov * Programs of types that are allowed to be loaded both w/ and w/o (for 1532aac3fc32SAndrey Ignatov * backward compatibility) expected_attach_type, should have the default attach 1533aac3fc32SAndrey Ignatov * type assigned to expected_attach_type for the latter case, so that it can be 1534aac3fc32SAndrey Ignatov * validated later at attach time. 1535aac3fc32SAndrey Ignatov * 1536aac3fc32SAndrey Ignatov * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if 1537aac3fc32SAndrey Ignatov * prog type requires it but has some attach types that have to be backward 1538aac3fc32SAndrey Ignatov * compatible. 1539aac3fc32SAndrey Ignatov */ 1540aac3fc32SAndrey Ignatov static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) 1541aac3fc32SAndrey Ignatov { 1542aac3fc32SAndrey Ignatov switch (attr->prog_type) { 1543aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1544aac3fc32SAndrey Ignatov /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't 1545aac3fc32SAndrey Ignatov * exist so checking for non-zero is the way to go here. 1546aac3fc32SAndrey Ignatov */ 1547aac3fc32SAndrey Ignatov if (!attr->expected_attach_type) 1548aac3fc32SAndrey Ignatov attr->expected_attach_type = 1549aac3fc32SAndrey Ignatov BPF_CGROUP_INET_SOCK_CREATE; 1550aac3fc32SAndrey Ignatov break; 1551aac3fc32SAndrey Ignatov } 1552aac3fc32SAndrey Ignatov } 1553aac3fc32SAndrey Ignatov 15545e43f899SAndrey Ignatov static int 15555e43f899SAndrey Ignatov bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type, 15565e43f899SAndrey Ignatov enum bpf_attach_type expected_attach_type) 15575e43f899SAndrey Ignatov { 15584fbac77dSAndrey Ignatov switch (prog_type) { 1559aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1560aac3fc32SAndrey Ignatov switch (expected_attach_type) { 1561aac3fc32SAndrey Ignatov case BPF_CGROUP_INET_SOCK_CREATE: 1562aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1563aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 1564aac3fc32SAndrey Ignatov return 0; 1565aac3fc32SAndrey Ignatov default: 1566aac3fc32SAndrey Ignatov return -EINVAL; 1567aac3fc32SAndrey Ignatov } 15684fbac77dSAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 15694fbac77dSAndrey Ignatov switch (expected_attach_type) { 15704fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 15714fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1572d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1573d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 15741cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 15751cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 15765e43f899SAndrey Ignatov return 0; 15774fbac77dSAndrey Ignatov default: 15784fbac77dSAndrey Ignatov return -EINVAL; 15794fbac77dSAndrey Ignatov } 15804fbac77dSAndrey Ignatov default: 15814fbac77dSAndrey Ignatov return 0; 15824fbac77dSAndrey Ignatov } 15835e43f899SAndrey Ignatov } 15845e43f899SAndrey Ignatov 158509756af4SAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 1586c454a46bSMartin KaFai Lau #define BPF_PROG_LOAD_LAST_FIELD line_info_cnt 158709756af4SAlexei Starovoitov 1588838e9690SYonghong Song static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) 158909756af4SAlexei Starovoitov { 159009756af4SAlexei Starovoitov enum bpf_prog_type type = attr->prog_type; 159109756af4SAlexei Starovoitov struct bpf_prog *prog; 159209756af4SAlexei Starovoitov int err; 159309756af4SAlexei Starovoitov char license[128]; 159409756af4SAlexei Starovoitov bool is_gpl; 159509756af4SAlexei Starovoitov 159609756af4SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_LOAD)) 159709756af4SAlexei Starovoitov return -EINVAL; 159809756af4SAlexei Starovoitov 1599e9ee9efcSDavid Miller if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | BPF_F_ANY_ALIGNMENT)) 1600e07b98d9SDavid S. Miller return -EINVAL; 1601e07b98d9SDavid S. Miller 1602e9ee9efcSDavid Miller if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && 1603e9ee9efcSDavid Miller (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && 1604e9ee9efcSDavid Miller !capable(CAP_SYS_ADMIN)) 1605e9ee9efcSDavid Miller return -EPERM; 1606e9ee9efcSDavid Miller 160709756af4SAlexei Starovoitov /* copy eBPF program license from user space */ 1608535e7b4bSMickaël Salaün if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 160909756af4SAlexei Starovoitov sizeof(license) - 1) < 0) 161009756af4SAlexei Starovoitov return -EFAULT; 161109756af4SAlexei Starovoitov license[sizeof(license) - 1] = 0; 161209756af4SAlexei Starovoitov 161309756af4SAlexei Starovoitov /* eBPF programs must be GPL compatible to use GPL-ed functions */ 161409756af4SAlexei Starovoitov is_gpl = license_is_gpl_compatible(license); 161509756af4SAlexei Starovoitov 1616c04c0d2bSAlexei Starovoitov if (attr->insn_cnt == 0 || 1617c04c0d2bSAlexei Starovoitov attr->insn_cnt > (capable(CAP_SYS_ADMIN) ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) 1618ef0915caSDaniel Borkmann return -E2BIG; 161980b7d819SChenbo Feng if (type != BPF_PROG_TYPE_SOCKET_FILTER && 162080b7d819SChenbo Feng type != BPF_PROG_TYPE_CGROUP_SKB && 162180b7d819SChenbo Feng !capable(CAP_SYS_ADMIN)) 16221be7f75dSAlexei Starovoitov return -EPERM; 16231be7f75dSAlexei Starovoitov 1624aac3fc32SAndrey Ignatov bpf_prog_load_fixup_attach_type(attr); 16255e43f899SAndrey Ignatov if (bpf_prog_load_check_attach_type(type, attr->expected_attach_type)) 16265e43f899SAndrey Ignatov return -EINVAL; 16275e43f899SAndrey Ignatov 162809756af4SAlexei Starovoitov /* plain bpf_prog allocation */ 162909756af4SAlexei Starovoitov prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 163009756af4SAlexei Starovoitov if (!prog) 163109756af4SAlexei Starovoitov return -ENOMEM; 163209756af4SAlexei Starovoitov 16335e43f899SAndrey Ignatov prog->expected_attach_type = attr->expected_attach_type; 16345e43f899SAndrey Ignatov 16359a18eedbSJakub Kicinski prog->aux->offload_requested = !!attr->prog_ifindex; 16369a18eedbSJakub Kicinski 1637afdb09c7SChenbo Feng err = security_bpf_prog_alloc(prog->aux); 1638aaac3ba9SAlexei Starovoitov if (err) 1639aaac3ba9SAlexei Starovoitov goto free_prog_nouncharge; 1640aaac3ba9SAlexei Starovoitov 1641afdb09c7SChenbo Feng err = bpf_prog_charge_memlock(prog); 1642afdb09c7SChenbo Feng if (err) 1643afdb09c7SChenbo Feng goto free_prog_sec; 1644afdb09c7SChenbo Feng 164509756af4SAlexei Starovoitov prog->len = attr->insn_cnt; 164609756af4SAlexei Starovoitov 164709756af4SAlexei Starovoitov err = -EFAULT; 1648535e7b4bSMickaël Salaün if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 1649aafe6ae9SDaniel Borkmann bpf_prog_insn_size(prog)) != 0) 165009756af4SAlexei Starovoitov goto free_prog; 165109756af4SAlexei Starovoitov 165209756af4SAlexei Starovoitov prog->orig_prog = NULL; 1653a91263d5SDaniel Borkmann prog->jited = 0; 165409756af4SAlexei Starovoitov 165509756af4SAlexei Starovoitov atomic_set(&prog->aux->refcnt, 1); 1656a91263d5SDaniel Borkmann prog->gpl_compatible = is_gpl ? 1 : 0; 165709756af4SAlexei Starovoitov 16589a18eedbSJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 1659ab3f0063SJakub Kicinski err = bpf_prog_offload_init(prog, attr); 1660ab3f0063SJakub Kicinski if (err) 1661ab3f0063SJakub Kicinski goto free_prog; 1662ab3f0063SJakub Kicinski } 1663ab3f0063SJakub Kicinski 166409756af4SAlexei Starovoitov /* find program type: socket_filter vs tracing_filter */ 166509756af4SAlexei Starovoitov err = find_prog_type(type, prog); 166609756af4SAlexei Starovoitov if (err < 0) 166709756af4SAlexei Starovoitov goto free_prog; 166809756af4SAlexei Starovoitov 1669*9285ec4cSJason A. Donenfeld prog->aux->load_time = ktime_get_boottime_ns(); 1670cb4d2b3fSMartin KaFai Lau err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name); 1671cb4d2b3fSMartin KaFai Lau if (err) 1672cb4d2b3fSMartin KaFai Lau goto free_prog; 1673cb4d2b3fSMartin KaFai Lau 167409756af4SAlexei Starovoitov /* run eBPF verifier */ 1675838e9690SYonghong Song err = bpf_check(&prog, attr, uattr); 167609756af4SAlexei Starovoitov if (err < 0) 167709756af4SAlexei Starovoitov goto free_used_maps; 167809756af4SAlexei Starovoitov 1679d1c55ab5SDaniel Borkmann prog = bpf_prog_select_runtime(prog, &err); 168004fd61abSAlexei Starovoitov if (err < 0) 168104fd61abSAlexei Starovoitov goto free_used_maps; 168209756af4SAlexei Starovoitov 1683dc4bb0e2SMartin KaFai Lau err = bpf_prog_alloc_id(prog); 1684dc4bb0e2SMartin KaFai Lau if (err) 1685dc4bb0e2SMartin KaFai Lau goto free_used_maps; 1686dc4bb0e2SMartin KaFai Lau 1687aa79781bSDaniel Borkmann err = bpf_prog_new_fd(prog); 1688b16d9aa4SMartin KaFai Lau if (err < 0) { 1689b16d9aa4SMartin KaFai Lau /* failed to allocate fd. 1690b16d9aa4SMartin KaFai Lau * bpf_prog_put() is needed because the above 1691b16d9aa4SMartin KaFai Lau * bpf_prog_alloc_id() has published the prog 1692b16d9aa4SMartin KaFai Lau * to the userspace and the userspace may 1693b16d9aa4SMartin KaFai Lau * have refcnt-ed it through BPF_PROG_GET_FD_BY_ID. 1694b16d9aa4SMartin KaFai Lau */ 1695b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 1696b16d9aa4SMartin KaFai Lau return err; 1697b16d9aa4SMartin KaFai Lau } 169809756af4SAlexei Starovoitov 169974451e66SDaniel Borkmann bpf_prog_kallsyms_add(prog); 17006ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); 170109756af4SAlexei Starovoitov return err; 170209756af4SAlexei Starovoitov 170309756af4SAlexei Starovoitov free_used_maps: 1704c454a46bSMartin KaFai Lau bpf_prog_free_linfo(prog); 17055482e9a9SMartin KaFai Lau kvfree(prog->aux->func_info); 17065482e9a9SMartin KaFai Lau btf_put(prog->aux->btf); 17077d1982b4SDaniel Borkmann bpf_prog_kallsyms_del_subprogs(prog); 170809756af4SAlexei Starovoitov free_used_maps(prog->aux); 170909756af4SAlexei Starovoitov free_prog: 1710aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(prog); 1711afdb09c7SChenbo Feng free_prog_sec: 1712afdb09c7SChenbo Feng security_bpf_prog_free(prog->aux); 1713aaac3ba9SAlexei Starovoitov free_prog_nouncharge: 171409756af4SAlexei Starovoitov bpf_prog_free(prog); 171509756af4SAlexei Starovoitov return err; 171609756af4SAlexei Starovoitov } 171709756af4SAlexei Starovoitov 17186e71b04aSChenbo Feng #define BPF_OBJ_LAST_FIELD file_flags 1719b2197755SDaniel Borkmann 1720b2197755SDaniel Borkmann static int bpf_obj_pin(const union bpf_attr *attr) 1721b2197755SDaniel Borkmann { 17226e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0) 1723b2197755SDaniel Borkmann return -EINVAL; 1724b2197755SDaniel Borkmann 1725535e7b4bSMickaël Salaün return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 1726b2197755SDaniel Borkmann } 1727b2197755SDaniel Borkmann 1728b2197755SDaniel Borkmann static int bpf_obj_get(const union bpf_attr *attr) 1729b2197755SDaniel Borkmann { 17306e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || 17316e71b04aSChenbo Feng attr->file_flags & ~BPF_OBJ_FLAG_MASK) 1732b2197755SDaniel Borkmann return -EINVAL; 1733b2197755SDaniel Borkmann 17346e71b04aSChenbo Feng return bpf_obj_get_user(u64_to_user_ptr(attr->pathname), 17356e71b04aSChenbo Feng attr->file_flags); 1736b2197755SDaniel Borkmann } 1737b2197755SDaniel Borkmann 1738c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint { 1739c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 1740c4f6699dSAlexei Starovoitov struct bpf_prog *prog; 1741c4f6699dSAlexei Starovoitov }; 1742c4f6699dSAlexei Starovoitov 1743c4f6699dSAlexei Starovoitov static int bpf_raw_tracepoint_release(struct inode *inode, struct file *filp) 1744c4f6699dSAlexei Starovoitov { 1745c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint *raw_tp = filp->private_data; 1746c4f6699dSAlexei Starovoitov 1747c4f6699dSAlexei Starovoitov if (raw_tp->prog) { 1748c4f6699dSAlexei Starovoitov bpf_probe_unregister(raw_tp->btp, raw_tp->prog); 1749c4f6699dSAlexei Starovoitov bpf_prog_put(raw_tp->prog); 1750c4f6699dSAlexei Starovoitov } 1751a38d1107SMatt Mullins bpf_put_raw_tracepoint(raw_tp->btp); 1752c4f6699dSAlexei Starovoitov kfree(raw_tp); 1753c4f6699dSAlexei Starovoitov return 0; 1754c4f6699dSAlexei Starovoitov } 1755c4f6699dSAlexei Starovoitov 1756c4f6699dSAlexei Starovoitov static const struct file_operations bpf_raw_tp_fops = { 1757c4f6699dSAlexei Starovoitov .release = bpf_raw_tracepoint_release, 1758c4f6699dSAlexei Starovoitov .read = bpf_dummy_read, 1759c4f6699dSAlexei Starovoitov .write = bpf_dummy_write, 1760c4f6699dSAlexei Starovoitov }; 1761c4f6699dSAlexei Starovoitov 1762c4f6699dSAlexei Starovoitov #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd 1763c4f6699dSAlexei Starovoitov 1764c4f6699dSAlexei Starovoitov static int bpf_raw_tracepoint_open(const union bpf_attr *attr) 1765c4f6699dSAlexei Starovoitov { 1766c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint *raw_tp; 1767c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 1768c4f6699dSAlexei Starovoitov struct bpf_prog *prog; 1769c4f6699dSAlexei Starovoitov char tp_name[128]; 1770c4f6699dSAlexei Starovoitov int tp_fd, err; 1771c4f6699dSAlexei Starovoitov 1772c4f6699dSAlexei Starovoitov if (strncpy_from_user(tp_name, u64_to_user_ptr(attr->raw_tracepoint.name), 1773c4f6699dSAlexei Starovoitov sizeof(tp_name) - 1) < 0) 1774c4f6699dSAlexei Starovoitov return -EFAULT; 1775c4f6699dSAlexei Starovoitov tp_name[sizeof(tp_name) - 1] = 0; 1776c4f6699dSAlexei Starovoitov 1777a38d1107SMatt Mullins btp = bpf_get_raw_tracepoint(tp_name); 1778c4f6699dSAlexei Starovoitov if (!btp) 1779c4f6699dSAlexei Starovoitov return -ENOENT; 1780c4f6699dSAlexei Starovoitov 1781c4f6699dSAlexei Starovoitov raw_tp = kzalloc(sizeof(*raw_tp), GFP_USER); 1782a38d1107SMatt Mullins if (!raw_tp) { 1783a38d1107SMatt Mullins err = -ENOMEM; 1784a38d1107SMatt Mullins goto out_put_btp; 1785a38d1107SMatt Mullins } 1786c4f6699dSAlexei Starovoitov raw_tp->btp = btp; 1787c4f6699dSAlexei Starovoitov 17889df1c28bSMatt Mullins prog = bpf_prog_get(attr->raw_tracepoint.prog_fd); 1789c4f6699dSAlexei Starovoitov if (IS_ERR(prog)) { 1790c4f6699dSAlexei Starovoitov err = PTR_ERR(prog); 1791c4f6699dSAlexei Starovoitov goto out_free_tp; 1792c4f6699dSAlexei Starovoitov } 17939df1c28bSMatt Mullins if (prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT && 17949df1c28bSMatt Mullins prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) { 17959df1c28bSMatt Mullins err = -EINVAL; 17969df1c28bSMatt Mullins goto out_put_prog; 17979df1c28bSMatt Mullins } 1798c4f6699dSAlexei Starovoitov 1799c4f6699dSAlexei Starovoitov err = bpf_probe_register(raw_tp->btp, prog); 1800c4f6699dSAlexei Starovoitov if (err) 1801c4f6699dSAlexei Starovoitov goto out_put_prog; 1802c4f6699dSAlexei Starovoitov 1803c4f6699dSAlexei Starovoitov raw_tp->prog = prog; 1804c4f6699dSAlexei Starovoitov tp_fd = anon_inode_getfd("bpf-raw-tracepoint", &bpf_raw_tp_fops, raw_tp, 1805c4f6699dSAlexei Starovoitov O_CLOEXEC); 1806c4f6699dSAlexei Starovoitov if (tp_fd < 0) { 1807c4f6699dSAlexei Starovoitov bpf_probe_unregister(raw_tp->btp, prog); 1808c4f6699dSAlexei Starovoitov err = tp_fd; 1809c4f6699dSAlexei Starovoitov goto out_put_prog; 1810c4f6699dSAlexei Starovoitov } 1811c4f6699dSAlexei Starovoitov return tp_fd; 1812c4f6699dSAlexei Starovoitov 1813c4f6699dSAlexei Starovoitov out_put_prog: 1814c4f6699dSAlexei Starovoitov bpf_prog_put(prog); 1815c4f6699dSAlexei Starovoitov out_free_tp: 1816c4f6699dSAlexei Starovoitov kfree(raw_tp); 1817a38d1107SMatt Mullins out_put_btp: 1818a38d1107SMatt Mullins bpf_put_raw_tracepoint(btp); 1819c4f6699dSAlexei Starovoitov return err; 1820c4f6699dSAlexei Starovoitov } 1821c4f6699dSAlexei Starovoitov 182233491588SAnders Roxell static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, 182333491588SAnders Roxell enum bpf_attach_type attach_type) 182433491588SAnders Roxell { 182533491588SAnders Roxell switch (prog->type) { 182633491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK: 182733491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 182833491588SAnders Roxell return attach_type == prog->expected_attach_type ? 0 : -EINVAL; 182933491588SAnders Roxell default: 183033491588SAnders Roxell return 0; 183133491588SAnders Roxell } 183233491588SAnders Roxell } 183333491588SAnders Roxell 1834464bc0fdSJohn Fastabend #define BPF_PROG_ATTACH_LAST_FIELD attach_flags 1835174a79ffSJohn Fastabend 1836324bda9eSAlexei Starovoitov #define BPF_F_ATTACH_MASK \ 1837324bda9eSAlexei Starovoitov (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI) 1838324bda9eSAlexei Starovoitov 1839f4324551SDaniel Mack static int bpf_prog_attach(const union bpf_attr *attr) 1840f4324551SDaniel Mack { 18417f677633SAlexei Starovoitov enum bpf_prog_type ptype; 1842f4324551SDaniel Mack struct bpf_prog *prog; 18437f677633SAlexei Starovoitov int ret; 1844f4324551SDaniel Mack 1845f4324551SDaniel Mack if (!capable(CAP_NET_ADMIN)) 1846f4324551SDaniel Mack return -EPERM; 1847f4324551SDaniel Mack 1848f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_ATTACH)) 1849f4324551SDaniel Mack return -EINVAL; 1850f4324551SDaniel Mack 1851324bda9eSAlexei Starovoitov if (attr->attach_flags & ~BPF_F_ATTACH_MASK) 18527f677633SAlexei Starovoitov return -EINVAL; 18537f677633SAlexei Starovoitov 1854f4324551SDaniel Mack switch (attr->attach_type) { 1855f4324551SDaniel Mack case BPF_CGROUP_INET_INGRESS: 1856f4324551SDaniel Mack case BPF_CGROUP_INET_EGRESS: 1857b2cd1257SDavid Ahern ptype = BPF_PROG_TYPE_CGROUP_SKB; 1858b2cd1257SDavid Ahern break; 185961023658SDavid Ahern case BPF_CGROUP_INET_SOCK_CREATE: 1860aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1861aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 186261023658SDavid Ahern ptype = BPF_PROG_TYPE_CGROUP_SOCK; 186361023658SDavid Ahern break; 18644fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 18654fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1866d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1867d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 18681cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 18691cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 18704fbac77dSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 18714fbac77dSAndrey Ignatov break; 187240304b2aSLawrence Brakmo case BPF_CGROUP_SOCK_OPS: 187340304b2aSLawrence Brakmo ptype = BPF_PROG_TYPE_SOCK_OPS; 187440304b2aSLawrence Brakmo break; 1875ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 1876ebc614f6SRoman Gushchin ptype = BPF_PROG_TYPE_CGROUP_DEVICE; 1877ebc614f6SRoman Gushchin break; 18784f738adbSJohn Fastabend case BPF_SK_MSG_VERDICT: 1879fdb5c453SSean Young ptype = BPF_PROG_TYPE_SK_MSG; 1880fdb5c453SSean Young break; 1881464bc0fdSJohn Fastabend case BPF_SK_SKB_STREAM_PARSER: 1882464bc0fdSJohn Fastabend case BPF_SK_SKB_STREAM_VERDICT: 1883fdb5c453SSean Young ptype = BPF_PROG_TYPE_SK_SKB; 1884fdb5c453SSean Young break; 1885f4364dcfSSean Young case BPF_LIRC_MODE2: 1886fdb5c453SSean Young ptype = BPF_PROG_TYPE_LIRC_MODE2; 1887fdb5c453SSean Young break; 1888d58e468bSPetar Penkov case BPF_FLOW_DISSECTOR: 1889d58e468bSPetar Penkov ptype = BPF_PROG_TYPE_FLOW_DISSECTOR; 1890d58e468bSPetar Penkov break; 18917b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 18927b146cebSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SYSCTL; 18937b146cebSAndrey Ignatov break; 1894b2cd1257SDavid Ahern default: 1895b2cd1257SDavid Ahern return -EINVAL; 1896b2cd1257SDavid Ahern } 1897b2cd1257SDavid Ahern 1898b2cd1257SDavid Ahern prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 1899f4324551SDaniel Mack if (IS_ERR(prog)) 1900f4324551SDaniel Mack return PTR_ERR(prog); 1901f4324551SDaniel Mack 19025e43f899SAndrey Ignatov if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) { 19035e43f899SAndrey Ignatov bpf_prog_put(prog); 19045e43f899SAndrey Ignatov return -EINVAL; 19055e43f899SAndrey Ignatov } 19065e43f899SAndrey Ignatov 1907fdb5c453SSean Young switch (ptype) { 1908fdb5c453SSean Young case BPF_PROG_TYPE_SK_SKB: 1909fdb5c453SSean Young case BPF_PROG_TYPE_SK_MSG: 1910604326b4SDaniel Borkmann ret = sock_map_get_from_fd(attr, prog); 1911fdb5c453SSean Young break; 1912fdb5c453SSean Young case BPF_PROG_TYPE_LIRC_MODE2: 1913fdb5c453SSean Young ret = lirc_prog_attach(attr, prog); 1914fdb5c453SSean Young break; 1915d58e468bSPetar Penkov case BPF_PROG_TYPE_FLOW_DISSECTOR: 1916d58e468bSPetar Penkov ret = skb_flow_dissector_bpf_prog_attach(attr, prog); 1917d58e468bSPetar Penkov break; 1918fdb5c453SSean Young default: 1919fdb5c453SSean Young ret = cgroup_bpf_prog_attach(attr, ptype, prog); 1920f4324551SDaniel Mack } 1921f4324551SDaniel Mack 19227f677633SAlexei Starovoitov if (ret) 19237f677633SAlexei Starovoitov bpf_prog_put(prog); 19247f677633SAlexei Starovoitov return ret; 1925f4324551SDaniel Mack } 1926f4324551SDaniel Mack 1927f4324551SDaniel Mack #define BPF_PROG_DETACH_LAST_FIELD attach_type 1928f4324551SDaniel Mack 1929f4324551SDaniel Mack static int bpf_prog_detach(const union bpf_attr *attr) 1930f4324551SDaniel Mack { 1931324bda9eSAlexei Starovoitov enum bpf_prog_type ptype; 1932f4324551SDaniel Mack 1933f4324551SDaniel Mack if (!capable(CAP_NET_ADMIN)) 1934f4324551SDaniel Mack return -EPERM; 1935f4324551SDaniel Mack 1936f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_DETACH)) 1937f4324551SDaniel Mack return -EINVAL; 1938f4324551SDaniel Mack 1939f4324551SDaniel Mack switch (attr->attach_type) { 1940f4324551SDaniel Mack case BPF_CGROUP_INET_INGRESS: 1941f4324551SDaniel Mack case BPF_CGROUP_INET_EGRESS: 1942324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_CGROUP_SKB; 1943324bda9eSAlexei Starovoitov break; 194461023658SDavid Ahern case BPF_CGROUP_INET_SOCK_CREATE: 1945aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1946aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 1947324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_CGROUP_SOCK; 1948324bda9eSAlexei Starovoitov break; 19494fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 19504fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1951d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1952d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 19531cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 19541cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 19554fbac77dSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 19564fbac77dSAndrey Ignatov break; 195740304b2aSLawrence Brakmo case BPF_CGROUP_SOCK_OPS: 1958324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_SOCK_OPS; 1959f4324551SDaniel Mack break; 1960ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 1961ebc614f6SRoman Gushchin ptype = BPF_PROG_TYPE_CGROUP_DEVICE; 1962ebc614f6SRoman Gushchin break; 19634f738adbSJohn Fastabend case BPF_SK_MSG_VERDICT: 1964604326b4SDaniel Borkmann return sock_map_get_from_fd(attr, NULL); 19655a67da2aSJohn Fastabend case BPF_SK_SKB_STREAM_PARSER: 19665a67da2aSJohn Fastabend case BPF_SK_SKB_STREAM_VERDICT: 1967604326b4SDaniel Borkmann return sock_map_get_from_fd(attr, NULL); 1968f4364dcfSSean Young case BPF_LIRC_MODE2: 1969f4364dcfSSean Young return lirc_prog_detach(attr); 1970d58e468bSPetar Penkov case BPF_FLOW_DISSECTOR: 1971d58e468bSPetar Penkov return skb_flow_dissector_bpf_prog_detach(attr); 19727b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 19737b146cebSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SYSCTL; 19747b146cebSAndrey Ignatov break; 1975f4324551SDaniel Mack default: 1976f4324551SDaniel Mack return -EINVAL; 1977f4324551SDaniel Mack } 1978f4324551SDaniel Mack 1979fdb5c453SSean Young return cgroup_bpf_prog_detach(attr, ptype); 1980f4324551SDaniel Mack } 198140304b2aSLawrence Brakmo 1982468e2f64SAlexei Starovoitov #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt 1983468e2f64SAlexei Starovoitov 1984468e2f64SAlexei Starovoitov static int bpf_prog_query(const union bpf_attr *attr, 1985468e2f64SAlexei Starovoitov union bpf_attr __user *uattr) 1986468e2f64SAlexei Starovoitov { 1987468e2f64SAlexei Starovoitov if (!capable(CAP_NET_ADMIN)) 1988468e2f64SAlexei Starovoitov return -EPERM; 1989468e2f64SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_QUERY)) 1990468e2f64SAlexei Starovoitov return -EINVAL; 1991468e2f64SAlexei Starovoitov if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE) 1992468e2f64SAlexei Starovoitov return -EINVAL; 1993468e2f64SAlexei Starovoitov 1994468e2f64SAlexei Starovoitov switch (attr->query.attach_type) { 1995468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_INGRESS: 1996468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_EGRESS: 1997468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_SOCK_CREATE: 19984fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 19994fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 2000aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 2001aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 2002d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 2003d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 20041cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 20051cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 2006468e2f64SAlexei Starovoitov case BPF_CGROUP_SOCK_OPS: 2007ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 20087b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 2009468e2f64SAlexei Starovoitov break; 2010f4364dcfSSean Young case BPF_LIRC_MODE2: 2011f4364dcfSSean Young return lirc_prog_query(attr, uattr); 2012118c8e9aSStanislav Fomichev case BPF_FLOW_DISSECTOR: 2013118c8e9aSStanislav Fomichev return skb_flow_dissector_prog_query(attr, uattr); 2014468e2f64SAlexei Starovoitov default: 2015468e2f64SAlexei Starovoitov return -EINVAL; 2016468e2f64SAlexei Starovoitov } 2017fdb5c453SSean Young 2018fdb5c453SSean Young return cgroup_bpf_prog_query(attr, uattr); 2019468e2f64SAlexei Starovoitov } 2020f4324551SDaniel Mack 2021b0b9395dSStanislav Fomichev #define BPF_PROG_TEST_RUN_LAST_FIELD test.ctx_out 20221cf1cae9SAlexei Starovoitov 20231cf1cae9SAlexei Starovoitov static int bpf_prog_test_run(const union bpf_attr *attr, 20241cf1cae9SAlexei Starovoitov union bpf_attr __user *uattr) 20251cf1cae9SAlexei Starovoitov { 20261cf1cae9SAlexei Starovoitov struct bpf_prog *prog; 20271cf1cae9SAlexei Starovoitov int ret = -ENOTSUPP; 20281cf1cae9SAlexei Starovoitov 202961f3c964SAlexei Starovoitov if (!capable(CAP_SYS_ADMIN)) 203061f3c964SAlexei Starovoitov return -EPERM; 20311cf1cae9SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 20321cf1cae9SAlexei Starovoitov return -EINVAL; 20331cf1cae9SAlexei Starovoitov 2034b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_in && !attr->test.ctx_in) || 2035b0b9395dSStanislav Fomichev (!attr->test.ctx_size_in && attr->test.ctx_in)) 2036b0b9395dSStanislav Fomichev return -EINVAL; 2037b0b9395dSStanislav Fomichev 2038b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_out && !attr->test.ctx_out) || 2039b0b9395dSStanislav Fomichev (!attr->test.ctx_size_out && attr->test.ctx_out)) 2040b0b9395dSStanislav Fomichev return -EINVAL; 2041b0b9395dSStanislav Fomichev 20421cf1cae9SAlexei Starovoitov prog = bpf_prog_get(attr->test.prog_fd); 20431cf1cae9SAlexei Starovoitov if (IS_ERR(prog)) 20441cf1cae9SAlexei Starovoitov return PTR_ERR(prog); 20451cf1cae9SAlexei Starovoitov 20461cf1cae9SAlexei Starovoitov if (prog->aux->ops->test_run) 20471cf1cae9SAlexei Starovoitov ret = prog->aux->ops->test_run(prog, attr, uattr); 20481cf1cae9SAlexei Starovoitov 20491cf1cae9SAlexei Starovoitov bpf_prog_put(prog); 20501cf1cae9SAlexei Starovoitov return ret; 20511cf1cae9SAlexei Starovoitov } 20521cf1cae9SAlexei Starovoitov 205334ad5580SMartin KaFai Lau #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 205434ad5580SMartin KaFai Lau 205534ad5580SMartin KaFai Lau static int bpf_obj_get_next_id(const union bpf_attr *attr, 205634ad5580SMartin KaFai Lau union bpf_attr __user *uattr, 205734ad5580SMartin KaFai Lau struct idr *idr, 205834ad5580SMartin KaFai Lau spinlock_t *lock) 205934ad5580SMartin KaFai Lau { 206034ad5580SMartin KaFai Lau u32 next_id = attr->start_id; 206134ad5580SMartin KaFai Lau int err = 0; 206234ad5580SMartin KaFai Lau 206334ad5580SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 206434ad5580SMartin KaFai Lau return -EINVAL; 206534ad5580SMartin KaFai Lau 206634ad5580SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 206734ad5580SMartin KaFai Lau return -EPERM; 206834ad5580SMartin KaFai Lau 206934ad5580SMartin KaFai Lau next_id++; 207034ad5580SMartin KaFai Lau spin_lock_bh(lock); 207134ad5580SMartin KaFai Lau if (!idr_get_next(idr, &next_id)) 207234ad5580SMartin KaFai Lau err = -ENOENT; 207334ad5580SMartin KaFai Lau spin_unlock_bh(lock); 207434ad5580SMartin KaFai Lau 207534ad5580SMartin KaFai Lau if (!err) 207634ad5580SMartin KaFai Lau err = put_user(next_id, &uattr->next_id); 207734ad5580SMartin KaFai Lau 207834ad5580SMartin KaFai Lau return err; 207934ad5580SMartin KaFai Lau } 208034ad5580SMartin KaFai Lau 2081b16d9aa4SMartin KaFai Lau #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 2082b16d9aa4SMartin KaFai Lau 2083b16d9aa4SMartin KaFai Lau static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 2084b16d9aa4SMartin KaFai Lau { 2085b16d9aa4SMartin KaFai Lau struct bpf_prog *prog; 2086b16d9aa4SMartin KaFai Lau u32 id = attr->prog_id; 2087b16d9aa4SMartin KaFai Lau int fd; 2088b16d9aa4SMartin KaFai Lau 2089b16d9aa4SMartin KaFai Lau if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 2090b16d9aa4SMartin KaFai Lau return -EINVAL; 2091b16d9aa4SMartin KaFai Lau 2092b16d9aa4SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 2093b16d9aa4SMartin KaFai Lau return -EPERM; 2094b16d9aa4SMartin KaFai Lau 2095b16d9aa4SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 2096b16d9aa4SMartin KaFai Lau prog = idr_find(&prog_idr, id); 2097b16d9aa4SMartin KaFai Lau if (prog) 2098b16d9aa4SMartin KaFai Lau prog = bpf_prog_inc_not_zero(prog); 2099b16d9aa4SMartin KaFai Lau else 2100b16d9aa4SMartin KaFai Lau prog = ERR_PTR(-ENOENT); 2101b16d9aa4SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 2102b16d9aa4SMartin KaFai Lau 2103b16d9aa4SMartin KaFai Lau if (IS_ERR(prog)) 2104b16d9aa4SMartin KaFai Lau return PTR_ERR(prog); 2105b16d9aa4SMartin KaFai Lau 2106b16d9aa4SMartin KaFai Lau fd = bpf_prog_new_fd(prog); 2107b16d9aa4SMartin KaFai Lau if (fd < 0) 2108b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 2109b16d9aa4SMartin KaFai Lau 2110b16d9aa4SMartin KaFai Lau return fd; 2111b16d9aa4SMartin KaFai Lau } 2112b16d9aa4SMartin KaFai Lau 21136e71b04aSChenbo Feng #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags 2114bd5f5f4eSMartin KaFai Lau 2115bd5f5f4eSMartin KaFai Lau static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 2116bd5f5f4eSMartin KaFai Lau { 2117bd5f5f4eSMartin KaFai Lau struct bpf_map *map; 2118bd5f5f4eSMartin KaFai Lau u32 id = attr->map_id; 21196e71b04aSChenbo Feng int f_flags; 2120bd5f5f4eSMartin KaFai Lau int fd; 2121bd5f5f4eSMartin KaFai Lau 21226e71b04aSChenbo Feng if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || 21236e71b04aSChenbo Feng attr->open_flags & ~BPF_OBJ_FLAG_MASK) 2124bd5f5f4eSMartin KaFai Lau return -EINVAL; 2125bd5f5f4eSMartin KaFai Lau 2126bd5f5f4eSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 2127bd5f5f4eSMartin KaFai Lau return -EPERM; 2128bd5f5f4eSMartin KaFai Lau 21296e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->open_flags); 21306e71b04aSChenbo Feng if (f_flags < 0) 21316e71b04aSChenbo Feng return f_flags; 21326e71b04aSChenbo Feng 2133bd5f5f4eSMartin KaFai Lau spin_lock_bh(&map_idr_lock); 2134bd5f5f4eSMartin KaFai Lau map = idr_find(&map_idr, id); 2135bd5f5f4eSMartin KaFai Lau if (map) 2136bd5f5f4eSMartin KaFai Lau map = bpf_map_inc_not_zero(map, true); 2137bd5f5f4eSMartin KaFai Lau else 2138bd5f5f4eSMartin KaFai Lau map = ERR_PTR(-ENOENT); 2139bd5f5f4eSMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 2140bd5f5f4eSMartin KaFai Lau 2141bd5f5f4eSMartin KaFai Lau if (IS_ERR(map)) 2142bd5f5f4eSMartin KaFai Lau return PTR_ERR(map); 2143bd5f5f4eSMartin KaFai Lau 21446e71b04aSChenbo Feng fd = bpf_map_new_fd(map, f_flags); 2145bd5f5f4eSMartin KaFai Lau if (fd < 0) 2146781e6282SPeng Sun bpf_map_put_with_uref(map); 2147bd5f5f4eSMartin KaFai Lau 2148bd5f5f4eSMartin KaFai Lau return fd; 2149bd5f5f4eSMartin KaFai Lau } 2150bd5f5f4eSMartin KaFai Lau 21517105e828SDaniel Borkmann static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, 2152d8eca5bbSDaniel Borkmann unsigned long addr, u32 *off, 2153d8eca5bbSDaniel Borkmann u32 *type) 21547105e828SDaniel Borkmann { 2155d8eca5bbSDaniel Borkmann const struct bpf_map *map; 21567105e828SDaniel Borkmann int i; 21577105e828SDaniel Borkmann 2158d8eca5bbSDaniel Borkmann for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { 2159d8eca5bbSDaniel Borkmann map = prog->aux->used_maps[i]; 2160d8eca5bbSDaniel Borkmann if (map == (void *)addr) { 2161d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_FD; 2162d8eca5bbSDaniel Borkmann return map; 2163d8eca5bbSDaniel Borkmann } 2164d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta) 2165d8eca5bbSDaniel Borkmann continue; 2166d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta(map, addr, off)) { 2167d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_VALUE; 2168d8eca5bbSDaniel Borkmann return map; 2169d8eca5bbSDaniel Borkmann } 2170d8eca5bbSDaniel Borkmann } 2171d8eca5bbSDaniel Borkmann 21727105e828SDaniel Borkmann return NULL; 21737105e828SDaniel Borkmann } 21747105e828SDaniel Borkmann 21757105e828SDaniel Borkmann static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog) 21767105e828SDaniel Borkmann { 21777105e828SDaniel Borkmann const struct bpf_map *map; 21787105e828SDaniel Borkmann struct bpf_insn *insns; 2179d8eca5bbSDaniel Borkmann u32 off, type; 21807105e828SDaniel Borkmann u64 imm; 21817105e828SDaniel Borkmann int i; 21827105e828SDaniel Borkmann 21837105e828SDaniel Borkmann insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), 21847105e828SDaniel Borkmann GFP_USER); 21857105e828SDaniel Borkmann if (!insns) 21867105e828SDaniel Borkmann return insns; 21877105e828SDaniel Borkmann 21887105e828SDaniel Borkmann for (i = 0; i < prog->len; i++) { 21897105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_TAIL_CALL)) { 21907105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 21917105e828SDaniel Borkmann insns[i].imm = BPF_FUNC_tail_call; 21927105e828SDaniel Borkmann /* fall-through */ 21937105e828SDaniel Borkmann } 21947105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_CALL) || 21957105e828SDaniel Borkmann insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) { 21967105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) 21977105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 21987105e828SDaniel Borkmann if (!bpf_dump_raw_ok()) 21997105e828SDaniel Borkmann insns[i].imm = 0; 22007105e828SDaniel Borkmann continue; 22017105e828SDaniel Borkmann } 22027105e828SDaniel Borkmann 22037105e828SDaniel Borkmann if (insns[i].code != (BPF_LD | BPF_IMM | BPF_DW)) 22047105e828SDaniel Borkmann continue; 22057105e828SDaniel Borkmann 22067105e828SDaniel Borkmann imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; 2207d8eca5bbSDaniel Borkmann map = bpf_map_from_imm(prog, imm, &off, &type); 22087105e828SDaniel Borkmann if (map) { 2209d8eca5bbSDaniel Borkmann insns[i].src_reg = type; 22107105e828SDaniel Borkmann insns[i].imm = map->id; 2211d8eca5bbSDaniel Borkmann insns[i + 1].imm = off; 22127105e828SDaniel Borkmann continue; 22137105e828SDaniel Borkmann } 22147105e828SDaniel Borkmann } 22157105e828SDaniel Borkmann 22167105e828SDaniel Borkmann return insns; 22177105e828SDaniel Borkmann } 22187105e828SDaniel Borkmann 2219c454a46bSMartin KaFai Lau static int set_info_rec_size(struct bpf_prog_info *info) 2220c454a46bSMartin KaFai Lau { 2221c454a46bSMartin KaFai Lau /* 2222c454a46bSMartin KaFai Lau * Ensure info.*_rec_size is the same as kernel expected size 2223c454a46bSMartin KaFai Lau * 2224c454a46bSMartin KaFai Lau * or 2225c454a46bSMartin KaFai Lau * 2226c454a46bSMartin KaFai Lau * Only allow zero *_rec_size if both _rec_size and _cnt are 2227c454a46bSMartin KaFai Lau * zero. In this case, the kernel will set the expected 2228c454a46bSMartin KaFai Lau * _rec_size back to the info. 2229c454a46bSMartin KaFai Lau */ 2230c454a46bSMartin KaFai Lau 223111d8b82dSYonghong Song if ((info->nr_func_info || info->func_info_rec_size) && 2232c454a46bSMartin KaFai Lau info->func_info_rec_size != sizeof(struct bpf_func_info)) 2233c454a46bSMartin KaFai Lau return -EINVAL; 2234c454a46bSMartin KaFai Lau 223511d8b82dSYonghong Song if ((info->nr_line_info || info->line_info_rec_size) && 2236c454a46bSMartin KaFai Lau info->line_info_rec_size != sizeof(struct bpf_line_info)) 2237c454a46bSMartin KaFai Lau return -EINVAL; 2238c454a46bSMartin KaFai Lau 223911d8b82dSYonghong Song if ((info->nr_jited_line_info || info->jited_line_info_rec_size) && 2240c454a46bSMartin KaFai Lau info->jited_line_info_rec_size != sizeof(__u64)) 2241c454a46bSMartin KaFai Lau return -EINVAL; 2242c454a46bSMartin KaFai Lau 2243c454a46bSMartin KaFai Lau info->func_info_rec_size = sizeof(struct bpf_func_info); 2244c454a46bSMartin KaFai Lau info->line_info_rec_size = sizeof(struct bpf_line_info); 2245c454a46bSMartin KaFai Lau info->jited_line_info_rec_size = sizeof(__u64); 2246c454a46bSMartin KaFai Lau 2247c454a46bSMartin KaFai Lau return 0; 2248c454a46bSMartin KaFai Lau } 2249c454a46bSMartin KaFai Lau 22501e270976SMartin KaFai Lau static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, 22511e270976SMartin KaFai Lau const union bpf_attr *attr, 22521e270976SMartin KaFai Lau union bpf_attr __user *uattr) 22531e270976SMartin KaFai Lau { 22541e270976SMartin KaFai Lau struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 22551e270976SMartin KaFai Lau struct bpf_prog_info info = {}; 22561e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 22575f8f8b93SAlexei Starovoitov struct bpf_prog_stats stats; 22581e270976SMartin KaFai Lau char __user *uinsns; 22591e270976SMartin KaFai Lau u32 ulen; 22601e270976SMartin KaFai Lau int err; 22611e270976SMartin KaFai Lau 2262dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 22631e270976SMartin KaFai Lau if (err) 22641e270976SMartin KaFai Lau return err; 22651e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 22661e270976SMartin KaFai Lau 22671e270976SMartin KaFai Lau if (copy_from_user(&info, uinfo, info_len)) 226889b09689SDaniel Borkmann return -EFAULT; 22691e270976SMartin KaFai Lau 22701e270976SMartin KaFai Lau info.type = prog->type; 22711e270976SMartin KaFai Lau info.id = prog->aux->id; 2272cb4d2b3fSMartin KaFai Lau info.load_time = prog->aux->load_time; 2273cb4d2b3fSMartin KaFai Lau info.created_by_uid = from_kuid_munged(current_user_ns(), 2274cb4d2b3fSMartin KaFai Lau prog->aux->user->uid); 2275b85fab0eSJiri Olsa info.gpl_compatible = prog->gpl_compatible; 22761e270976SMartin KaFai Lau 22771e270976SMartin KaFai Lau memcpy(info.tag, prog->tag, sizeof(prog->tag)); 2278cb4d2b3fSMartin KaFai Lau memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); 2279cb4d2b3fSMartin KaFai Lau 2280cb4d2b3fSMartin KaFai Lau ulen = info.nr_map_ids; 2281cb4d2b3fSMartin KaFai Lau info.nr_map_ids = prog->aux->used_map_cnt; 2282cb4d2b3fSMartin KaFai Lau ulen = min_t(u32, info.nr_map_ids, ulen); 2283cb4d2b3fSMartin KaFai Lau if (ulen) { 2284721e08daSMartin KaFai Lau u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); 2285cb4d2b3fSMartin KaFai Lau u32 i; 2286cb4d2b3fSMartin KaFai Lau 2287cb4d2b3fSMartin KaFai Lau for (i = 0; i < ulen; i++) 2288cb4d2b3fSMartin KaFai Lau if (put_user(prog->aux->used_maps[i]->id, 2289cb4d2b3fSMartin KaFai Lau &user_map_ids[i])) 2290cb4d2b3fSMartin KaFai Lau return -EFAULT; 2291cb4d2b3fSMartin KaFai Lau } 22921e270976SMartin KaFai Lau 2293c454a46bSMartin KaFai Lau err = set_info_rec_size(&info); 2294c454a46bSMartin KaFai Lau if (err) 2295c454a46bSMartin KaFai Lau return err; 22967337224fSMartin KaFai Lau 22975f8f8b93SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 22985f8f8b93SAlexei Starovoitov info.run_time_ns = stats.nsecs; 22995f8f8b93SAlexei Starovoitov info.run_cnt = stats.cnt; 23005f8f8b93SAlexei Starovoitov 23011e270976SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) { 23021e270976SMartin KaFai Lau info.jited_prog_len = 0; 23031e270976SMartin KaFai Lau info.xlated_prog_len = 0; 2304dbecd738SSandipan Das info.nr_jited_ksyms = 0; 230528c2fae7SDaniel Borkmann info.nr_jited_func_lens = 0; 230611d8b82dSYonghong Song info.nr_func_info = 0; 230711d8b82dSYonghong Song info.nr_line_info = 0; 230811d8b82dSYonghong Song info.nr_jited_line_info = 0; 23091e270976SMartin KaFai Lau goto done; 23101e270976SMartin KaFai Lau } 23111e270976SMartin KaFai Lau 23121e270976SMartin KaFai Lau ulen = info.xlated_prog_len; 23139975a54bSDaniel Borkmann info.xlated_prog_len = bpf_prog_insn_size(prog); 23141e270976SMartin KaFai Lau if (info.xlated_prog_len && ulen) { 23157105e828SDaniel Borkmann struct bpf_insn *insns_sanitized; 23167105e828SDaniel Borkmann bool fault; 23177105e828SDaniel Borkmann 23187105e828SDaniel Borkmann if (prog->blinded && !bpf_dump_raw_ok()) { 23197105e828SDaniel Borkmann info.xlated_prog_insns = 0; 23207105e828SDaniel Borkmann goto done; 23217105e828SDaniel Borkmann } 23227105e828SDaniel Borkmann insns_sanitized = bpf_insn_prepare_dump(prog); 23237105e828SDaniel Borkmann if (!insns_sanitized) 23247105e828SDaniel Borkmann return -ENOMEM; 23251e270976SMartin KaFai Lau uinsns = u64_to_user_ptr(info.xlated_prog_insns); 23261e270976SMartin KaFai Lau ulen = min_t(u32, info.xlated_prog_len, ulen); 23277105e828SDaniel Borkmann fault = copy_to_user(uinsns, insns_sanitized, ulen); 23287105e828SDaniel Borkmann kfree(insns_sanitized); 23297105e828SDaniel Borkmann if (fault) 23301e270976SMartin KaFai Lau return -EFAULT; 23311e270976SMartin KaFai Lau } 23321e270976SMartin KaFai Lau 2333675fc275SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 2334675fc275SJakub Kicinski err = bpf_prog_offload_info_fill(&info, prog); 2335675fc275SJakub Kicinski if (err) 2336675fc275SJakub Kicinski return err; 2337fcfb126dSJiong Wang goto done; 2338fcfb126dSJiong Wang } 2339fcfb126dSJiong Wang 2340fcfb126dSJiong Wang /* NOTE: the following code is supposed to be skipped for offload. 2341fcfb126dSJiong Wang * bpf_prog_offload_info_fill() is the place to fill similar fields 2342fcfb126dSJiong Wang * for offload. 2343fcfb126dSJiong Wang */ 2344fcfb126dSJiong Wang ulen = info.jited_prog_len; 23454d56a76eSSandipan Das if (prog->aux->func_cnt) { 23464d56a76eSSandipan Das u32 i; 23474d56a76eSSandipan Das 23484d56a76eSSandipan Das info.jited_prog_len = 0; 23494d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) 23504d56a76eSSandipan Das info.jited_prog_len += prog->aux->func[i]->jited_len; 23514d56a76eSSandipan Das } else { 2352fcfb126dSJiong Wang info.jited_prog_len = prog->jited_len; 23534d56a76eSSandipan Das } 23544d56a76eSSandipan Das 2355fcfb126dSJiong Wang if (info.jited_prog_len && ulen) { 2356fcfb126dSJiong Wang if (bpf_dump_raw_ok()) { 2357fcfb126dSJiong Wang uinsns = u64_to_user_ptr(info.jited_prog_insns); 2358fcfb126dSJiong Wang ulen = min_t(u32, info.jited_prog_len, ulen); 23594d56a76eSSandipan Das 23604d56a76eSSandipan Das /* for multi-function programs, copy the JITed 23614d56a76eSSandipan Das * instructions for all the functions 23624d56a76eSSandipan Das */ 23634d56a76eSSandipan Das if (prog->aux->func_cnt) { 23644d56a76eSSandipan Das u32 len, free, i; 23654d56a76eSSandipan Das u8 *img; 23664d56a76eSSandipan Das 23674d56a76eSSandipan Das free = ulen; 23684d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) { 23694d56a76eSSandipan Das len = prog->aux->func[i]->jited_len; 23704d56a76eSSandipan Das len = min_t(u32, len, free); 23714d56a76eSSandipan Das img = (u8 *) prog->aux->func[i]->bpf_func; 23724d56a76eSSandipan Das if (copy_to_user(uinsns, img, len)) 23734d56a76eSSandipan Das return -EFAULT; 23744d56a76eSSandipan Das uinsns += len; 23754d56a76eSSandipan Das free -= len; 23764d56a76eSSandipan Das if (!free) 23774d56a76eSSandipan Das break; 23784d56a76eSSandipan Das } 23794d56a76eSSandipan Das } else { 2380fcfb126dSJiong Wang if (copy_to_user(uinsns, prog->bpf_func, ulen)) 2381fcfb126dSJiong Wang return -EFAULT; 23824d56a76eSSandipan Das } 2383fcfb126dSJiong Wang } else { 2384fcfb126dSJiong Wang info.jited_prog_insns = 0; 2385fcfb126dSJiong Wang } 2386675fc275SJakub Kicinski } 2387675fc275SJakub Kicinski 2388dbecd738SSandipan Das ulen = info.nr_jited_ksyms; 2389ff1889fcSSong Liu info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; 23907a5725ddSSong Liu if (ulen) { 2391dbecd738SSandipan Das if (bpf_dump_raw_ok()) { 2392ff1889fcSSong Liu unsigned long ksym_addr; 2393dbecd738SSandipan Das u64 __user *user_ksyms; 2394dbecd738SSandipan Das u32 i; 2395dbecd738SSandipan Das 2396dbecd738SSandipan Das /* copy the address of the kernel symbol 2397dbecd738SSandipan Das * corresponding to each function 2398dbecd738SSandipan Das */ 2399dbecd738SSandipan Das ulen = min_t(u32, info.nr_jited_ksyms, ulen); 2400dbecd738SSandipan Das user_ksyms = u64_to_user_ptr(info.jited_ksyms); 2401ff1889fcSSong Liu if (prog->aux->func_cnt) { 2402dbecd738SSandipan Das for (i = 0; i < ulen; i++) { 2403ff1889fcSSong Liu ksym_addr = (unsigned long) 2404ff1889fcSSong Liu prog->aux->func[i]->bpf_func; 2405ff1889fcSSong Liu if (put_user((u64) ksym_addr, 2406ff1889fcSSong Liu &user_ksyms[i])) 2407ff1889fcSSong Liu return -EFAULT; 2408ff1889fcSSong Liu } 2409ff1889fcSSong Liu } else { 2410ff1889fcSSong Liu ksym_addr = (unsigned long) prog->bpf_func; 2411ff1889fcSSong Liu if (put_user((u64) ksym_addr, &user_ksyms[0])) 2412dbecd738SSandipan Das return -EFAULT; 2413dbecd738SSandipan Das } 2414dbecd738SSandipan Das } else { 2415dbecd738SSandipan Das info.jited_ksyms = 0; 2416dbecd738SSandipan Das } 2417dbecd738SSandipan Das } 2418dbecd738SSandipan Das 2419815581c1SSandipan Das ulen = info.nr_jited_func_lens; 2420ff1889fcSSong Liu info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; 24217a5725ddSSong Liu if (ulen) { 2422815581c1SSandipan Das if (bpf_dump_raw_ok()) { 2423815581c1SSandipan Das u32 __user *user_lens; 2424815581c1SSandipan Das u32 func_len, i; 2425815581c1SSandipan Das 2426815581c1SSandipan Das /* copy the JITed image lengths for each function */ 2427815581c1SSandipan Das ulen = min_t(u32, info.nr_jited_func_lens, ulen); 2428815581c1SSandipan Das user_lens = u64_to_user_ptr(info.jited_func_lens); 2429ff1889fcSSong Liu if (prog->aux->func_cnt) { 2430815581c1SSandipan Das for (i = 0; i < ulen; i++) { 2431ff1889fcSSong Liu func_len = 2432ff1889fcSSong Liu prog->aux->func[i]->jited_len; 2433815581c1SSandipan Das if (put_user(func_len, &user_lens[i])) 2434815581c1SSandipan Das return -EFAULT; 2435815581c1SSandipan Das } 2436815581c1SSandipan Das } else { 2437ff1889fcSSong Liu func_len = prog->jited_len; 2438ff1889fcSSong Liu if (put_user(func_len, &user_lens[0])) 2439ff1889fcSSong Liu return -EFAULT; 2440ff1889fcSSong Liu } 2441ff1889fcSSong Liu } else { 2442815581c1SSandipan Das info.jited_func_lens = 0; 2443815581c1SSandipan Das } 2444815581c1SSandipan Das } 2445815581c1SSandipan Das 24467337224fSMartin KaFai Lau if (prog->aux->btf) 2447838e9690SYonghong Song info.btf_id = btf_id(prog->aux->btf); 2448838e9690SYonghong Song 244911d8b82dSYonghong Song ulen = info.nr_func_info; 245011d8b82dSYonghong Song info.nr_func_info = prog->aux->func_info_cnt; 245111d8b82dSYonghong Song if (info.nr_func_info && ulen) { 2452838e9690SYonghong Song char __user *user_finfo; 2453838e9690SYonghong Song 2454838e9690SYonghong Song user_finfo = u64_to_user_ptr(info.func_info); 245511d8b82dSYonghong Song ulen = min_t(u32, info.nr_func_info, ulen); 2456ba64e7d8SYonghong Song if (copy_to_user(user_finfo, prog->aux->func_info, 24577337224fSMartin KaFai Lau info.func_info_rec_size * ulen)) 2458838e9690SYonghong Song return -EFAULT; 2459838e9690SYonghong Song } 2460838e9690SYonghong Song 246111d8b82dSYonghong Song ulen = info.nr_line_info; 246211d8b82dSYonghong Song info.nr_line_info = prog->aux->nr_linfo; 246311d8b82dSYonghong Song if (info.nr_line_info && ulen) { 2464c454a46bSMartin KaFai Lau __u8 __user *user_linfo; 2465c454a46bSMartin KaFai Lau 2466c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.line_info); 246711d8b82dSYonghong Song ulen = min_t(u32, info.nr_line_info, ulen); 2468c454a46bSMartin KaFai Lau if (copy_to_user(user_linfo, prog->aux->linfo, 2469c454a46bSMartin KaFai Lau info.line_info_rec_size * ulen)) 2470c454a46bSMartin KaFai Lau return -EFAULT; 2471c454a46bSMartin KaFai Lau } 2472c454a46bSMartin KaFai Lau 247311d8b82dSYonghong Song ulen = info.nr_jited_line_info; 2474c454a46bSMartin KaFai Lau if (prog->aux->jited_linfo) 247511d8b82dSYonghong Song info.nr_jited_line_info = prog->aux->nr_linfo; 2476c454a46bSMartin KaFai Lau else 247711d8b82dSYonghong Song info.nr_jited_line_info = 0; 247811d8b82dSYonghong Song if (info.nr_jited_line_info && ulen) { 2479c454a46bSMartin KaFai Lau if (bpf_dump_raw_ok()) { 2480c454a46bSMartin KaFai Lau __u64 __user *user_linfo; 2481c454a46bSMartin KaFai Lau u32 i; 2482c454a46bSMartin KaFai Lau 2483c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.jited_line_info); 248411d8b82dSYonghong Song ulen = min_t(u32, info.nr_jited_line_info, ulen); 2485c454a46bSMartin KaFai Lau for (i = 0; i < ulen; i++) { 2486c454a46bSMartin KaFai Lau if (put_user((__u64)(long)prog->aux->jited_linfo[i], 2487c454a46bSMartin KaFai Lau &user_linfo[i])) 2488c454a46bSMartin KaFai Lau return -EFAULT; 2489c454a46bSMartin KaFai Lau } 2490c454a46bSMartin KaFai Lau } else { 2491c454a46bSMartin KaFai Lau info.jited_line_info = 0; 2492c454a46bSMartin KaFai Lau } 2493c454a46bSMartin KaFai Lau } 2494c454a46bSMartin KaFai Lau 2495c872bdb3SSong Liu ulen = info.nr_prog_tags; 2496c872bdb3SSong Liu info.nr_prog_tags = prog->aux->func_cnt ? : 1; 2497c872bdb3SSong Liu if (ulen) { 2498c872bdb3SSong Liu __u8 __user (*user_prog_tags)[BPF_TAG_SIZE]; 2499c872bdb3SSong Liu u32 i; 2500c872bdb3SSong Liu 2501c872bdb3SSong Liu user_prog_tags = u64_to_user_ptr(info.prog_tags); 2502c872bdb3SSong Liu ulen = min_t(u32, info.nr_prog_tags, ulen); 2503c872bdb3SSong Liu if (prog->aux->func_cnt) { 2504c872bdb3SSong Liu for (i = 0; i < ulen; i++) { 2505c872bdb3SSong Liu if (copy_to_user(user_prog_tags[i], 2506c872bdb3SSong Liu prog->aux->func[i]->tag, 2507c872bdb3SSong Liu BPF_TAG_SIZE)) 2508c872bdb3SSong Liu return -EFAULT; 2509c872bdb3SSong Liu } 2510c872bdb3SSong Liu } else { 2511c872bdb3SSong Liu if (copy_to_user(user_prog_tags[0], 2512c872bdb3SSong Liu prog->tag, BPF_TAG_SIZE)) 2513c872bdb3SSong Liu return -EFAULT; 2514c872bdb3SSong Liu } 2515c872bdb3SSong Liu } 2516c872bdb3SSong Liu 25171e270976SMartin KaFai Lau done: 25181e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 25191e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 25201e270976SMartin KaFai Lau return -EFAULT; 25211e270976SMartin KaFai Lau 25221e270976SMartin KaFai Lau return 0; 25231e270976SMartin KaFai Lau } 25241e270976SMartin KaFai Lau 25251e270976SMartin KaFai Lau static int bpf_map_get_info_by_fd(struct bpf_map *map, 25261e270976SMartin KaFai Lau const union bpf_attr *attr, 25271e270976SMartin KaFai Lau union bpf_attr __user *uattr) 25281e270976SMartin KaFai Lau { 25291e270976SMartin KaFai Lau struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 25301e270976SMartin KaFai Lau struct bpf_map_info info = {}; 25311e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 25321e270976SMartin KaFai Lau int err; 25331e270976SMartin KaFai Lau 2534dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 25351e270976SMartin KaFai Lau if (err) 25361e270976SMartin KaFai Lau return err; 25371e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 25381e270976SMartin KaFai Lau 25391e270976SMartin KaFai Lau info.type = map->map_type; 25401e270976SMartin KaFai Lau info.id = map->id; 25411e270976SMartin KaFai Lau info.key_size = map->key_size; 25421e270976SMartin KaFai Lau info.value_size = map->value_size; 25431e270976SMartin KaFai Lau info.max_entries = map->max_entries; 25441e270976SMartin KaFai Lau info.map_flags = map->map_flags; 2545ad5b177bSMartin KaFai Lau memcpy(info.name, map->name, sizeof(map->name)); 25461e270976SMartin KaFai Lau 254778958fcaSMartin KaFai Lau if (map->btf) { 254878958fcaSMartin KaFai Lau info.btf_id = btf_id(map->btf); 25499b2cf328SMartin KaFai Lau info.btf_key_type_id = map->btf_key_type_id; 25509b2cf328SMartin KaFai Lau info.btf_value_type_id = map->btf_value_type_id; 255178958fcaSMartin KaFai Lau } 255278958fcaSMartin KaFai Lau 255352775b33SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 255452775b33SJakub Kicinski err = bpf_map_offload_info_fill(&info, map); 255552775b33SJakub Kicinski if (err) 255652775b33SJakub Kicinski return err; 255752775b33SJakub Kicinski } 255852775b33SJakub Kicinski 25591e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 25601e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 25611e270976SMartin KaFai Lau return -EFAULT; 25621e270976SMartin KaFai Lau 25631e270976SMartin KaFai Lau return 0; 25641e270976SMartin KaFai Lau } 25651e270976SMartin KaFai Lau 256662dab84cSMartin KaFai Lau static int bpf_btf_get_info_by_fd(struct btf *btf, 256762dab84cSMartin KaFai Lau const union bpf_attr *attr, 256862dab84cSMartin KaFai Lau union bpf_attr __user *uattr) 256962dab84cSMartin KaFai Lau { 257062dab84cSMartin KaFai Lau struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info); 257162dab84cSMartin KaFai Lau u32 info_len = attr->info.info_len; 257262dab84cSMartin KaFai Lau int err; 257362dab84cSMartin KaFai Lau 2574dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(*uinfo), info_len); 257562dab84cSMartin KaFai Lau if (err) 257662dab84cSMartin KaFai Lau return err; 257762dab84cSMartin KaFai Lau 257862dab84cSMartin KaFai Lau return btf_get_info_by_fd(btf, attr, uattr); 257962dab84cSMartin KaFai Lau } 258062dab84cSMartin KaFai Lau 25811e270976SMartin KaFai Lau #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 25821e270976SMartin KaFai Lau 25831e270976SMartin KaFai Lau static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 25841e270976SMartin KaFai Lau union bpf_attr __user *uattr) 25851e270976SMartin KaFai Lau { 25861e270976SMartin KaFai Lau int ufd = attr->info.bpf_fd; 25871e270976SMartin KaFai Lau struct fd f; 25881e270976SMartin KaFai Lau int err; 25891e270976SMartin KaFai Lau 25901e270976SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 25911e270976SMartin KaFai Lau return -EINVAL; 25921e270976SMartin KaFai Lau 25931e270976SMartin KaFai Lau f = fdget(ufd); 25941e270976SMartin KaFai Lau if (!f.file) 25951e270976SMartin KaFai Lau return -EBADFD; 25961e270976SMartin KaFai Lau 25971e270976SMartin KaFai Lau if (f.file->f_op == &bpf_prog_fops) 25981e270976SMartin KaFai Lau err = bpf_prog_get_info_by_fd(f.file->private_data, attr, 25991e270976SMartin KaFai Lau uattr); 26001e270976SMartin KaFai Lau else if (f.file->f_op == &bpf_map_fops) 26011e270976SMartin KaFai Lau err = bpf_map_get_info_by_fd(f.file->private_data, attr, 26021e270976SMartin KaFai Lau uattr); 260360197cfbSMartin KaFai Lau else if (f.file->f_op == &btf_fops) 260462dab84cSMartin KaFai Lau err = bpf_btf_get_info_by_fd(f.file->private_data, attr, uattr); 26051e270976SMartin KaFai Lau else 26061e270976SMartin KaFai Lau err = -EINVAL; 26071e270976SMartin KaFai Lau 26081e270976SMartin KaFai Lau fdput(f); 26091e270976SMartin KaFai Lau return err; 26101e270976SMartin KaFai Lau } 26111e270976SMartin KaFai Lau 2612f56a653cSMartin KaFai Lau #define BPF_BTF_LOAD_LAST_FIELD btf_log_level 2613f56a653cSMartin KaFai Lau 2614f56a653cSMartin KaFai Lau static int bpf_btf_load(const union bpf_attr *attr) 2615f56a653cSMartin KaFai Lau { 2616f56a653cSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_LOAD)) 2617f56a653cSMartin KaFai Lau return -EINVAL; 2618f56a653cSMartin KaFai Lau 2619f56a653cSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 2620f56a653cSMartin KaFai Lau return -EPERM; 2621f56a653cSMartin KaFai Lau 2622f56a653cSMartin KaFai Lau return btf_new_fd(attr); 2623f56a653cSMartin KaFai Lau } 2624f56a653cSMartin KaFai Lau 262578958fcaSMartin KaFai Lau #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id 262678958fcaSMartin KaFai Lau 262778958fcaSMartin KaFai Lau static int bpf_btf_get_fd_by_id(const union bpf_attr *attr) 262878958fcaSMartin KaFai Lau { 262978958fcaSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID)) 263078958fcaSMartin KaFai Lau return -EINVAL; 263178958fcaSMartin KaFai Lau 263278958fcaSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 263378958fcaSMartin KaFai Lau return -EPERM; 263478958fcaSMartin KaFai Lau 263578958fcaSMartin KaFai Lau return btf_get_fd_by_id(attr->btf_id); 263678958fcaSMartin KaFai Lau } 263778958fcaSMartin KaFai Lau 263841bdc4b4SYonghong Song static int bpf_task_fd_query_copy(const union bpf_attr *attr, 263941bdc4b4SYonghong Song union bpf_attr __user *uattr, 264041bdc4b4SYonghong Song u32 prog_id, u32 fd_type, 264141bdc4b4SYonghong Song const char *buf, u64 probe_offset, 264241bdc4b4SYonghong Song u64 probe_addr) 264341bdc4b4SYonghong Song { 264441bdc4b4SYonghong Song char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf); 264541bdc4b4SYonghong Song u32 len = buf ? strlen(buf) : 0, input_len; 264641bdc4b4SYonghong Song int err = 0; 264741bdc4b4SYonghong Song 264841bdc4b4SYonghong Song if (put_user(len, &uattr->task_fd_query.buf_len)) 264941bdc4b4SYonghong Song return -EFAULT; 265041bdc4b4SYonghong Song input_len = attr->task_fd_query.buf_len; 265141bdc4b4SYonghong Song if (input_len && ubuf) { 265241bdc4b4SYonghong Song if (!len) { 265341bdc4b4SYonghong Song /* nothing to copy, just make ubuf NULL terminated */ 265441bdc4b4SYonghong Song char zero = '\0'; 265541bdc4b4SYonghong Song 265641bdc4b4SYonghong Song if (put_user(zero, ubuf)) 265741bdc4b4SYonghong Song return -EFAULT; 265841bdc4b4SYonghong Song } else if (input_len >= len + 1) { 265941bdc4b4SYonghong Song /* ubuf can hold the string with NULL terminator */ 266041bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, len + 1)) 266141bdc4b4SYonghong Song return -EFAULT; 266241bdc4b4SYonghong Song } else { 266341bdc4b4SYonghong Song /* ubuf cannot hold the string with NULL terminator, 266441bdc4b4SYonghong Song * do a partial copy with NULL terminator. 266541bdc4b4SYonghong Song */ 266641bdc4b4SYonghong Song char zero = '\0'; 266741bdc4b4SYonghong Song 266841bdc4b4SYonghong Song err = -ENOSPC; 266941bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, input_len - 1)) 267041bdc4b4SYonghong Song return -EFAULT; 267141bdc4b4SYonghong Song if (put_user(zero, ubuf + input_len - 1)) 267241bdc4b4SYonghong Song return -EFAULT; 267341bdc4b4SYonghong Song } 267441bdc4b4SYonghong Song } 267541bdc4b4SYonghong Song 267641bdc4b4SYonghong Song if (put_user(prog_id, &uattr->task_fd_query.prog_id) || 267741bdc4b4SYonghong Song put_user(fd_type, &uattr->task_fd_query.fd_type) || 267841bdc4b4SYonghong Song put_user(probe_offset, &uattr->task_fd_query.probe_offset) || 267941bdc4b4SYonghong Song put_user(probe_addr, &uattr->task_fd_query.probe_addr)) 268041bdc4b4SYonghong Song return -EFAULT; 268141bdc4b4SYonghong Song 268241bdc4b4SYonghong Song return err; 268341bdc4b4SYonghong Song } 268441bdc4b4SYonghong Song 268541bdc4b4SYonghong Song #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr 268641bdc4b4SYonghong Song 268741bdc4b4SYonghong Song static int bpf_task_fd_query(const union bpf_attr *attr, 268841bdc4b4SYonghong Song union bpf_attr __user *uattr) 268941bdc4b4SYonghong Song { 269041bdc4b4SYonghong Song pid_t pid = attr->task_fd_query.pid; 269141bdc4b4SYonghong Song u32 fd = attr->task_fd_query.fd; 269241bdc4b4SYonghong Song const struct perf_event *event; 269341bdc4b4SYonghong Song struct files_struct *files; 269441bdc4b4SYonghong Song struct task_struct *task; 269541bdc4b4SYonghong Song struct file *file; 269641bdc4b4SYonghong Song int err; 269741bdc4b4SYonghong Song 269841bdc4b4SYonghong Song if (CHECK_ATTR(BPF_TASK_FD_QUERY)) 269941bdc4b4SYonghong Song return -EINVAL; 270041bdc4b4SYonghong Song 270141bdc4b4SYonghong Song if (!capable(CAP_SYS_ADMIN)) 270241bdc4b4SYonghong Song return -EPERM; 270341bdc4b4SYonghong Song 270441bdc4b4SYonghong Song if (attr->task_fd_query.flags != 0) 270541bdc4b4SYonghong Song return -EINVAL; 270641bdc4b4SYonghong Song 270741bdc4b4SYonghong Song task = get_pid_task(find_vpid(pid), PIDTYPE_PID); 270841bdc4b4SYonghong Song if (!task) 270941bdc4b4SYonghong Song return -ENOENT; 271041bdc4b4SYonghong Song 271141bdc4b4SYonghong Song files = get_files_struct(task); 271241bdc4b4SYonghong Song put_task_struct(task); 271341bdc4b4SYonghong Song if (!files) 271441bdc4b4SYonghong Song return -ENOENT; 271541bdc4b4SYonghong Song 271641bdc4b4SYonghong Song err = 0; 271741bdc4b4SYonghong Song spin_lock(&files->file_lock); 271841bdc4b4SYonghong Song file = fcheck_files(files, fd); 271941bdc4b4SYonghong Song if (!file) 272041bdc4b4SYonghong Song err = -EBADF; 272141bdc4b4SYonghong Song else 272241bdc4b4SYonghong Song get_file(file); 272341bdc4b4SYonghong Song spin_unlock(&files->file_lock); 272441bdc4b4SYonghong Song put_files_struct(files); 272541bdc4b4SYonghong Song 272641bdc4b4SYonghong Song if (err) 272741bdc4b4SYonghong Song goto out; 272841bdc4b4SYonghong Song 272941bdc4b4SYonghong Song if (file->f_op == &bpf_raw_tp_fops) { 273041bdc4b4SYonghong Song struct bpf_raw_tracepoint *raw_tp = file->private_data; 273141bdc4b4SYonghong Song struct bpf_raw_event_map *btp = raw_tp->btp; 273241bdc4b4SYonghong Song 273341bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, 273441bdc4b4SYonghong Song raw_tp->prog->aux->id, 273541bdc4b4SYonghong Song BPF_FD_TYPE_RAW_TRACEPOINT, 273641bdc4b4SYonghong Song btp->tp->name, 0, 0); 273741bdc4b4SYonghong Song goto put_file; 273841bdc4b4SYonghong Song } 273941bdc4b4SYonghong Song 274041bdc4b4SYonghong Song event = perf_get_event(file); 274141bdc4b4SYonghong Song if (!IS_ERR(event)) { 274241bdc4b4SYonghong Song u64 probe_offset, probe_addr; 274341bdc4b4SYonghong Song u32 prog_id, fd_type; 274441bdc4b4SYonghong Song const char *buf; 274541bdc4b4SYonghong Song 274641bdc4b4SYonghong Song err = bpf_get_perf_event_info(event, &prog_id, &fd_type, 274741bdc4b4SYonghong Song &buf, &probe_offset, 274841bdc4b4SYonghong Song &probe_addr); 274941bdc4b4SYonghong Song if (!err) 275041bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, prog_id, 275141bdc4b4SYonghong Song fd_type, buf, 275241bdc4b4SYonghong Song probe_offset, 275341bdc4b4SYonghong Song probe_addr); 275441bdc4b4SYonghong Song goto put_file; 275541bdc4b4SYonghong Song } 275641bdc4b4SYonghong Song 275741bdc4b4SYonghong Song err = -ENOTSUPP; 275841bdc4b4SYonghong Song put_file: 275941bdc4b4SYonghong Song fput(file); 276041bdc4b4SYonghong Song out: 276141bdc4b4SYonghong Song return err; 276241bdc4b4SYonghong Song } 276341bdc4b4SYonghong Song 276499c55f7dSAlexei Starovoitov SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 276599c55f7dSAlexei Starovoitov { 276699c55f7dSAlexei Starovoitov union bpf_attr attr = {}; 276799c55f7dSAlexei Starovoitov int err; 276899c55f7dSAlexei Starovoitov 27690fa4fe85SChenbo Feng if (sysctl_unprivileged_bpf_disabled && !capable(CAP_SYS_ADMIN)) 277099c55f7dSAlexei Starovoitov return -EPERM; 277199c55f7dSAlexei Starovoitov 2772dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size); 277399c55f7dSAlexei Starovoitov if (err) 277499c55f7dSAlexei Starovoitov return err; 27751e270976SMartin KaFai Lau size = min_t(u32, size, sizeof(attr)); 277699c55f7dSAlexei Starovoitov 277799c55f7dSAlexei Starovoitov /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 277899c55f7dSAlexei Starovoitov if (copy_from_user(&attr, uattr, size) != 0) 277999c55f7dSAlexei Starovoitov return -EFAULT; 278099c55f7dSAlexei Starovoitov 2781afdb09c7SChenbo Feng err = security_bpf(cmd, &attr, size); 2782afdb09c7SChenbo Feng if (err < 0) 2783afdb09c7SChenbo Feng return err; 2784afdb09c7SChenbo Feng 278599c55f7dSAlexei Starovoitov switch (cmd) { 278699c55f7dSAlexei Starovoitov case BPF_MAP_CREATE: 278799c55f7dSAlexei Starovoitov err = map_create(&attr); 278899c55f7dSAlexei Starovoitov break; 2789db20fd2bSAlexei Starovoitov case BPF_MAP_LOOKUP_ELEM: 2790db20fd2bSAlexei Starovoitov err = map_lookup_elem(&attr); 2791db20fd2bSAlexei Starovoitov break; 2792db20fd2bSAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 2793db20fd2bSAlexei Starovoitov err = map_update_elem(&attr); 2794db20fd2bSAlexei Starovoitov break; 2795db20fd2bSAlexei Starovoitov case BPF_MAP_DELETE_ELEM: 2796db20fd2bSAlexei Starovoitov err = map_delete_elem(&attr); 2797db20fd2bSAlexei Starovoitov break; 2798db20fd2bSAlexei Starovoitov case BPF_MAP_GET_NEXT_KEY: 2799db20fd2bSAlexei Starovoitov err = map_get_next_key(&attr); 2800db20fd2bSAlexei Starovoitov break; 280187df15deSDaniel Borkmann case BPF_MAP_FREEZE: 280287df15deSDaniel Borkmann err = map_freeze(&attr); 280387df15deSDaniel Borkmann break; 280409756af4SAlexei Starovoitov case BPF_PROG_LOAD: 2805838e9690SYonghong Song err = bpf_prog_load(&attr, uattr); 280609756af4SAlexei Starovoitov break; 2807b2197755SDaniel Borkmann case BPF_OBJ_PIN: 2808b2197755SDaniel Borkmann err = bpf_obj_pin(&attr); 2809b2197755SDaniel Borkmann break; 2810b2197755SDaniel Borkmann case BPF_OBJ_GET: 2811b2197755SDaniel Borkmann err = bpf_obj_get(&attr); 2812b2197755SDaniel Borkmann break; 2813f4324551SDaniel Mack case BPF_PROG_ATTACH: 2814f4324551SDaniel Mack err = bpf_prog_attach(&attr); 2815f4324551SDaniel Mack break; 2816f4324551SDaniel Mack case BPF_PROG_DETACH: 2817f4324551SDaniel Mack err = bpf_prog_detach(&attr); 2818f4324551SDaniel Mack break; 2819468e2f64SAlexei Starovoitov case BPF_PROG_QUERY: 2820468e2f64SAlexei Starovoitov err = bpf_prog_query(&attr, uattr); 2821468e2f64SAlexei Starovoitov break; 28221cf1cae9SAlexei Starovoitov case BPF_PROG_TEST_RUN: 28231cf1cae9SAlexei Starovoitov err = bpf_prog_test_run(&attr, uattr); 28241cf1cae9SAlexei Starovoitov break; 282534ad5580SMartin KaFai Lau case BPF_PROG_GET_NEXT_ID: 282634ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 282734ad5580SMartin KaFai Lau &prog_idr, &prog_idr_lock); 282834ad5580SMartin KaFai Lau break; 282934ad5580SMartin KaFai Lau case BPF_MAP_GET_NEXT_ID: 283034ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 283134ad5580SMartin KaFai Lau &map_idr, &map_idr_lock); 283234ad5580SMartin KaFai Lau break; 2833b16d9aa4SMartin KaFai Lau case BPF_PROG_GET_FD_BY_ID: 2834b16d9aa4SMartin KaFai Lau err = bpf_prog_get_fd_by_id(&attr); 2835b16d9aa4SMartin KaFai Lau break; 2836bd5f5f4eSMartin KaFai Lau case BPF_MAP_GET_FD_BY_ID: 2837bd5f5f4eSMartin KaFai Lau err = bpf_map_get_fd_by_id(&attr); 2838bd5f5f4eSMartin KaFai Lau break; 28391e270976SMartin KaFai Lau case BPF_OBJ_GET_INFO_BY_FD: 28401e270976SMartin KaFai Lau err = bpf_obj_get_info_by_fd(&attr, uattr); 28411e270976SMartin KaFai Lau break; 2842c4f6699dSAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN: 2843c4f6699dSAlexei Starovoitov err = bpf_raw_tracepoint_open(&attr); 2844c4f6699dSAlexei Starovoitov break; 2845f56a653cSMartin KaFai Lau case BPF_BTF_LOAD: 2846f56a653cSMartin KaFai Lau err = bpf_btf_load(&attr); 2847f56a653cSMartin KaFai Lau break; 284878958fcaSMartin KaFai Lau case BPF_BTF_GET_FD_BY_ID: 284978958fcaSMartin KaFai Lau err = bpf_btf_get_fd_by_id(&attr); 285078958fcaSMartin KaFai Lau break; 285141bdc4b4SYonghong Song case BPF_TASK_FD_QUERY: 285241bdc4b4SYonghong Song err = bpf_task_fd_query(&attr, uattr); 285341bdc4b4SYonghong Song break; 2854bd513cd0SMauricio Vasquez B case BPF_MAP_LOOKUP_AND_DELETE_ELEM: 2855bd513cd0SMauricio Vasquez B err = map_lookup_and_delete_elem(&attr); 2856bd513cd0SMauricio Vasquez B break; 285799c55f7dSAlexei Starovoitov default: 285899c55f7dSAlexei Starovoitov err = -EINVAL; 285999c55f7dSAlexei Starovoitov break; 286099c55f7dSAlexei Starovoitov } 286199c55f7dSAlexei Starovoitov 286299c55f7dSAlexei Starovoitov return err; 286399c55f7dSAlexei Starovoitov } 2864