199c55f7dSAlexei Starovoitov /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 299c55f7dSAlexei Starovoitov * 399c55f7dSAlexei Starovoitov * This program is free software; you can redistribute it and/or 499c55f7dSAlexei Starovoitov * modify it under the terms of version 2 of the GNU General Public 599c55f7dSAlexei Starovoitov * License as published by the Free Software Foundation. 699c55f7dSAlexei Starovoitov * 799c55f7dSAlexei Starovoitov * This program is distributed in the hope that it will be useful, but 899c55f7dSAlexei Starovoitov * WITHOUT ANY WARRANTY; without even the implied warranty of 999c55f7dSAlexei Starovoitov * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 1099c55f7dSAlexei Starovoitov * General Public License for more details. 1199c55f7dSAlexei Starovoitov */ 1299c55f7dSAlexei Starovoitov #include <linux/bpf.h> 13a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 14f4364dcfSSean Young #include <linux/bpf_lirc.h> 15f56a653cSMartin KaFai Lau #include <linux/btf.h> 1699c55f7dSAlexei Starovoitov #include <linux/syscalls.h> 1799c55f7dSAlexei Starovoitov #include <linux/slab.h> 183f07c014SIngo Molnar #include <linux/sched/signal.h> 19d407bd25SDaniel Borkmann #include <linux/vmalloc.h> 20d407bd25SDaniel Borkmann #include <linux/mmzone.h> 2199c55f7dSAlexei Starovoitov #include <linux/anon_inodes.h> 2241bdc4b4SYonghong Song #include <linux/fdtable.h> 23db20fd2bSAlexei Starovoitov #include <linux/file.h> 2441bdc4b4SYonghong Song #include <linux/fs.h> 2509756af4SAlexei Starovoitov #include <linux/license.h> 2609756af4SAlexei Starovoitov #include <linux/filter.h> 272541517cSAlexei Starovoitov #include <linux/version.h> 28535e7b4bSMickaël Salaün #include <linux/kernel.h> 29dc4bb0e2SMartin KaFai Lau #include <linux/idr.h> 30cb4d2b3fSMartin KaFai Lau #include <linux/cred.h> 31cb4d2b3fSMartin KaFai Lau #include <linux/timekeeping.h> 32cb4d2b3fSMartin KaFai Lau #include <linux/ctype.h> 339ef09e35SMark Rutland #include <linux/nospec.h> 3499c55f7dSAlexei Starovoitov 3514dc6f04SMartin KaFai Lau #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ 3614dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 3714dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 3814dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 3914dc6f04SMartin KaFai Lau #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 4014dc6f04SMartin KaFai Lau #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map)) 4114dc6f04SMartin KaFai Lau 426e71b04aSChenbo Feng #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) 436e71b04aSChenbo Feng 44b121d1e7SAlexei Starovoitov DEFINE_PER_CPU(int, bpf_prog_active); 45dc4bb0e2SMartin KaFai Lau static DEFINE_IDR(prog_idr); 46dc4bb0e2SMartin KaFai Lau static DEFINE_SPINLOCK(prog_idr_lock); 47f3f1c054SMartin KaFai Lau static DEFINE_IDR(map_idr); 48f3f1c054SMartin KaFai Lau static DEFINE_SPINLOCK(map_idr_lock); 49b121d1e7SAlexei Starovoitov 501be7f75dSAlexei Starovoitov int sysctl_unprivileged_bpf_disabled __read_mostly; 511be7f75dSAlexei Starovoitov 5240077e0cSJohannes Berg static const struct bpf_map_ops * const bpf_map_types[] = { 5340077e0cSJohannes Berg #define BPF_PROG_TYPE(_id, _ops) 5440077e0cSJohannes Berg #define BPF_MAP_TYPE(_id, _ops) \ 5540077e0cSJohannes Berg [_id] = &_ops, 5640077e0cSJohannes Berg #include <linux/bpf_types.h> 5740077e0cSJohannes Berg #undef BPF_PROG_TYPE 5840077e0cSJohannes Berg #undef BPF_MAP_TYPE 5940077e0cSJohannes Berg }; 6099c55f7dSAlexei Starovoitov 61752ba56fSMickaël Salaün /* 62752ba56fSMickaël Salaün * If we're handed a bigger struct than we know of, ensure all the unknown bits 63752ba56fSMickaël Salaün * are 0 - i.e. new user-space does not rely on any kernel feature extensions 64752ba56fSMickaël Salaün * we don't know about yet. 65752ba56fSMickaël Salaün * 66752ba56fSMickaël Salaün * There is a ToCToU between this function call and the following 67752ba56fSMickaël Salaün * copy_from_user() call. However, this is not a concern since this function is 68752ba56fSMickaël Salaün * meant to be a future-proofing of bits. 69752ba56fSMickaël Salaün */ 70dcab51f1SMartin KaFai Lau int bpf_check_uarg_tail_zero(void __user *uaddr, 7158291a74SMickaël Salaün size_t expected_size, 7258291a74SMickaël Salaün size_t actual_size) 7358291a74SMickaël Salaün { 7458291a74SMickaël Salaün unsigned char __user *addr; 7558291a74SMickaël Salaün unsigned char __user *end; 7658291a74SMickaël Salaün unsigned char val; 7758291a74SMickaël Salaün int err; 7858291a74SMickaël Salaün 79752ba56fSMickaël Salaün if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 80752ba56fSMickaël Salaün return -E2BIG; 81752ba56fSMickaël Salaün 8296d4f267SLinus Torvalds if (unlikely(!access_ok(uaddr, actual_size))) 83752ba56fSMickaël Salaün return -EFAULT; 84752ba56fSMickaël Salaün 8558291a74SMickaël Salaün if (actual_size <= expected_size) 8658291a74SMickaël Salaün return 0; 8758291a74SMickaël Salaün 8858291a74SMickaël Salaün addr = uaddr + expected_size; 8958291a74SMickaël Salaün end = uaddr + actual_size; 9058291a74SMickaël Salaün 9158291a74SMickaël Salaün for (; addr < end; addr++) { 9258291a74SMickaël Salaün err = get_user(val, addr); 9358291a74SMickaël Salaün if (err) 9458291a74SMickaël Salaün return err; 9558291a74SMickaël Salaün if (val) 9658291a74SMickaël Salaün return -E2BIG; 9758291a74SMickaël Salaün } 9858291a74SMickaël Salaün 9958291a74SMickaël Salaün return 0; 10058291a74SMickaël Salaün } 10158291a74SMickaël Salaün 102a3884572SJakub Kicinski const struct bpf_map_ops bpf_map_offload_ops = { 103a3884572SJakub Kicinski .map_alloc = bpf_map_offload_map_alloc, 104a3884572SJakub Kicinski .map_free = bpf_map_offload_map_free, 105e8d2bec0SDaniel Borkmann .map_check_btf = map_check_no_btf, 106a3884572SJakub Kicinski }; 107a3884572SJakub Kicinski 10899c55f7dSAlexei Starovoitov static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 10999c55f7dSAlexei Starovoitov { 1101110f3a9SJakub Kicinski const struct bpf_map_ops *ops; 1119ef09e35SMark Rutland u32 type = attr->map_type; 11299c55f7dSAlexei Starovoitov struct bpf_map *map; 1131110f3a9SJakub Kicinski int err; 11499c55f7dSAlexei Starovoitov 1159ef09e35SMark Rutland if (type >= ARRAY_SIZE(bpf_map_types)) 1161110f3a9SJakub Kicinski return ERR_PTR(-EINVAL); 1179ef09e35SMark Rutland type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types)); 1189ef09e35SMark Rutland ops = bpf_map_types[type]; 1191110f3a9SJakub Kicinski if (!ops) 12040077e0cSJohannes Berg return ERR_PTR(-EINVAL); 12140077e0cSJohannes Berg 1221110f3a9SJakub Kicinski if (ops->map_alloc_check) { 1231110f3a9SJakub Kicinski err = ops->map_alloc_check(attr); 1241110f3a9SJakub Kicinski if (err) 1251110f3a9SJakub Kicinski return ERR_PTR(err); 1261110f3a9SJakub Kicinski } 127a3884572SJakub Kicinski if (attr->map_ifindex) 128a3884572SJakub Kicinski ops = &bpf_map_offload_ops; 1291110f3a9SJakub Kicinski map = ops->map_alloc(attr); 13099c55f7dSAlexei Starovoitov if (IS_ERR(map)) 13199c55f7dSAlexei Starovoitov return map; 1321110f3a9SJakub Kicinski map->ops = ops; 1339ef09e35SMark Rutland map->map_type = type; 13499c55f7dSAlexei Starovoitov return map; 13599c55f7dSAlexei Starovoitov } 13699c55f7dSAlexei Starovoitov 13796eabe7aSMartin KaFai Lau void *bpf_map_area_alloc(size_t size, int numa_node) 138d407bd25SDaniel Borkmann { 139f01a7dbeSMartynas Pumputis /* We really just want to fail instead of triggering OOM killer 140f01a7dbeSMartynas Pumputis * under memory pressure, therefore we set __GFP_NORETRY to kmalloc, 141f01a7dbeSMartynas Pumputis * which is used for lower order allocation requests. 142f01a7dbeSMartynas Pumputis * 143f01a7dbeSMartynas Pumputis * It has been observed that higher order allocation requests done by 144f01a7dbeSMartynas Pumputis * vmalloc with __GFP_NORETRY being set might fail due to not trying 145f01a7dbeSMartynas Pumputis * to reclaim memory from the page cache, thus we set 146f01a7dbeSMartynas Pumputis * __GFP_RETRY_MAYFAIL to avoid such situations. 147d407bd25SDaniel Borkmann */ 148f01a7dbeSMartynas Pumputis 149f01a7dbeSMartynas Pumputis const gfp_t flags = __GFP_NOWARN | __GFP_ZERO; 150d407bd25SDaniel Borkmann void *area; 151d407bd25SDaniel Borkmann 152d407bd25SDaniel Borkmann if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 153f01a7dbeSMartynas Pumputis area = kmalloc_node(size, GFP_USER | __GFP_NORETRY | flags, 154f01a7dbeSMartynas Pumputis numa_node); 155d407bd25SDaniel Borkmann if (area != NULL) 156d407bd25SDaniel Borkmann return area; 157d407bd25SDaniel Borkmann } 158d407bd25SDaniel Borkmann 159f01a7dbeSMartynas Pumputis return __vmalloc_node_flags_caller(size, numa_node, 160f01a7dbeSMartynas Pumputis GFP_KERNEL | __GFP_RETRY_MAYFAIL | 161f01a7dbeSMartynas Pumputis flags, __builtin_return_address(0)); 162d407bd25SDaniel Borkmann } 163d407bd25SDaniel Borkmann 164d407bd25SDaniel Borkmann void bpf_map_area_free(void *area) 165d407bd25SDaniel Borkmann { 166d407bd25SDaniel Borkmann kvfree(area); 167d407bd25SDaniel Borkmann } 168d407bd25SDaniel Borkmann 169be70bcd5SDaniel Borkmann static u32 bpf_map_flags_retain_permanent(u32 flags) 170be70bcd5SDaniel Borkmann { 171be70bcd5SDaniel Borkmann /* Some map creation flags are not tied to the map object but 172be70bcd5SDaniel Borkmann * rather to the map fd instead, so they have no meaning upon 173be70bcd5SDaniel Borkmann * map object inspection since multiple file descriptors with 174be70bcd5SDaniel Borkmann * different (access) properties can exist here. Thus, given 175be70bcd5SDaniel Borkmann * this has zero meaning for the map itself, lets clear these 176be70bcd5SDaniel Borkmann * from here. 177be70bcd5SDaniel Borkmann */ 178be70bcd5SDaniel Borkmann return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY); 179be70bcd5SDaniel Borkmann } 180be70bcd5SDaniel Borkmann 181bd475643SJakub Kicinski void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) 182bd475643SJakub Kicinski { 183bd475643SJakub Kicinski map->map_type = attr->map_type; 184bd475643SJakub Kicinski map->key_size = attr->key_size; 185bd475643SJakub Kicinski map->value_size = attr->value_size; 186bd475643SJakub Kicinski map->max_entries = attr->max_entries; 187be70bcd5SDaniel Borkmann map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags); 188bd475643SJakub Kicinski map->numa_node = bpf_map_attr_numa_node(attr); 189bd475643SJakub Kicinski } 190bd475643SJakub Kicinski 1916c905981SAlexei Starovoitov int bpf_map_precharge_memlock(u32 pages) 1926c905981SAlexei Starovoitov { 1936c905981SAlexei Starovoitov struct user_struct *user = get_current_user(); 1946c905981SAlexei Starovoitov unsigned long memlock_limit, cur; 1956c905981SAlexei Starovoitov 1966c905981SAlexei Starovoitov memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 1976c905981SAlexei Starovoitov cur = atomic_long_read(&user->locked_vm); 1986c905981SAlexei Starovoitov free_uid(user); 1996c905981SAlexei Starovoitov if (cur + pages > memlock_limit) 2006c905981SAlexei Starovoitov return -EPERM; 2016c905981SAlexei Starovoitov return 0; 2026c905981SAlexei Starovoitov } 2036c905981SAlexei Starovoitov 2040a4c58f5SRoman Gushchin static int bpf_charge_memlock(struct user_struct *user, u32 pages) 205aaac3ba9SAlexei Starovoitov { 2060a4c58f5SRoman Gushchin unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 207aaac3ba9SAlexei Starovoitov 2080a4c58f5SRoman Gushchin if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) { 2090a4c58f5SRoman Gushchin atomic_long_sub(pages, &user->locked_vm); 210aaac3ba9SAlexei Starovoitov return -EPERM; 211aaac3ba9SAlexei Starovoitov } 212aaac3ba9SAlexei Starovoitov return 0; 213aaac3ba9SAlexei Starovoitov } 214aaac3ba9SAlexei Starovoitov 2150a4c58f5SRoman Gushchin static void bpf_uncharge_memlock(struct user_struct *user, u32 pages) 2160a4c58f5SRoman Gushchin { 2170a4c58f5SRoman Gushchin atomic_long_sub(pages, &user->locked_vm); 2180a4c58f5SRoman Gushchin } 2190a4c58f5SRoman Gushchin 2200a4c58f5SRoman Gushchin static int bpf_map_init_memlock(struct bpf_map *map) 2210a4c58f5SRoman Gushchin { 2220a4c58f5SRoman Gushchin struct user_struct *user = get_current_user(); 2230a4c58f5SRoman Gushchin int ret; 2240a4c58f5SRoman Gushchin 2250a4c58f5SRoman Gushchin ret = bpf_charge_memlock(user, map->pages); 2260a4c58f5SRoman Gushchin if (ret) { 2270a4c58f5SRoman Gushchin free_uid(user); 2280a4c58f5SRoman Gushchin return ret; 2290a4c58f5SRoman Gushchin } 2300a4c58f5SRoman Gushchin map->user = user; 2310a4c58f5SRoman Gushchin return ret; 2320a4c58f5SRoman Gushchin } 2330a4c58f5SRoman Gushchin 2340a4c58f5SRoman Gushchin static void bpf_map_release_memlock(struct bpf_map *map) 235aaac3ba9SAlexei Starovoitov { 236aaac3ba9SAlexei Starovoitov struct user_struct *user = map->user; 2370a4c58f5SRoman Gushchin bpf_uncharge_memlock(user, map->pages); 238aaac3ba9SAlexei Starovoitov free_uid(user); 239aaac3ba9SAlexei Starovoitov } 240aaac3ba9SAlexei Starovoitov 2410a4c58f5SRoman Gushchin int bpf_map_charge_memlock(struct bpf_map *map, u32 pages) 2420a4c58f5SRoman Gushchin { 2430a4c58f5SRoman Gushchin int ret; 2440a4c58f5SRoman Gushchin 2450a4c58f5SRoman Gushchin ret = bpf_charge_memlock(map->user, pages); 2460a4c58f5SRoman Gushchin if (ret) 2470a4c58f5SRoman Gushchin return ret; 2480a4c58f5SRoman Gushchin map->pages += pages; 2490a4c58f5SRoman Gushchin return ret; 2500a4c58f5SRoman Gushchin } 2510a4c58f5SRoman Gushchin 2520a4c58f5SRoman Gushchin void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages) 2530a4c58f5SRoman Gushchin { 2540a4c58f5SRoman Gushchin bpf_uncharge_memlock(map->user, pages); 2550a4c58f5SRoman Gushchin map->pages -= pages; 2560a4c58f5SRoman Gushchin } 2570a4c58f5SRoman Gushchin 258f3f1c054SMartin KaFai Lau static int bpf_map_alloc_id(struct bpf_map *map) 259f3f1c054SMartin KaFai Lau { 260f3f1c054SMartin KaFai Lau int id; 261f3f1c054SMartin KaFai Lau 262b76354cdSShaohua Li idr_preload(GFP_KERNEL); 263f3f1c054SMartin KaFai Lau spin_lock_bh(&map_idr_lock); 264f3f1c054SMartin KaFai Lau id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 265f3f1c054SMartin KaFai Lau if (id > 0) 266f3f1c054SMartin KaFai Lau map->id = id; 267f3f1c054SMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 268b76354cdSShaohua Li idr_preload_end(); 269f3f1c054SMartin KaFai Lau 270f3f1c054SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 271f3f1c054SMartin KaFai Lau return -ENOSPC; 272f3f1c054SMartin KaFai Lau 273f3f1c054SMartin KaFai Lau return id > 0 ? 0 : id; 274f3f1c054SMartin KaFai Lau } 275f3f1c054SMartin KaFai Lau 276a3884572SJakub Kicinski void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) 277f3f1c054SMartin KaFai Lau { 278930651a7SEric Dumazet unsigned long flags; 279930651a7SEric Dumazet 280a3884572SJakub Kicinski /* Offloaded maps are removed from the IDR store when their device 281a3884572SJakub Kicinski * disappears - even if someone holds an fd to them they are unusable, 282a3884572SJakub Kicinski * the memory is gone, all ops will fail; they are simply waiting for 283a3884572SJakub Kicinski * refcnt to drop to be freed. 284a3884572SJakub Kicinski */ 285a3884572SJakub Kicinski if (!map->id) 286a3884572SJakub Kicinski return; 287a3884572SJakub Kicinski 288bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 289930651a7SEric Dumazet spin_lock_irqsave(&map_idr_lock, flags); 290bd5f5f4eSMartin KaFai Lau else 291bd5f5f4eSMartin KaFai Lau __acquire(&map_idr_lock); 292bd5f5f4eSMartin KaFai Lau 293f3f1c054SMartin KaFai Lau idr_remove(&map_idr, map->id); 294a3884572SJakub Kicinski map->id = 0; 295bd5f5f4eSMartin KaFai Lau 296bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 297930651a7SEric Dumazet spin_unlock_irqrestore(&map_idr_lock, flags); 298bd5f5f4eSMartin KaFai Lau else 299bd5f5f4eSMartin KaFai Lau __release(&map_idr_lock); 300f3f1c054SMartin KaFai Lau } 301f3f1c054SMartin KaFai Lau 30299c55f7dSAlexei Starovoitov /* called from workqueue */ 30399c55f7dSAlexei Starovoitov static void bpf_map_free_deferred(struct work_struct *work) 30499c55f7dSAlexei Starovoitov { 30599c55f7dSAlexei Starovoitov struct bpf_map *map = container_of(work, struct bpf_map, work); 30699c55f7dSAlexei Starovoitov 3070a4c58f5SRoman Gushchin bpf_map_release_memlock(map); 308afdb09c7SChenbo Feng security_bpf_map_free(map); 30999c55f7dSAlexei Starovoitov /* implementation dependent freeing */ 31099c55f7dSAlexei Starovoitov map->ops->map_free(map); 31199c55f7dSAlexei Starovoitov } 31299c55f7dSAlexei Starovoitov 313c9da161cSDaniel Borkmann static void bpf_map_put_uref(struct bpf_map *map) 314c9da161cSDaniel Borkmann { 315c9da161cSDaniel Borkmann if (atomic_dec_and_test(&map->usercnt)) { 316ba6b8de4SJohn Fastabend if (map->ops->map_release_uref) 317ba6b8de4SJohn Fastabend map->ops->map_release_uref(map); 318c9da161cSDaniel Borkmann } 319c9da161cSDaniel Borkmann } 320c9da161cSDaniel Borkmann 32199c55f7dSAlexei Starovoitov /* decrement map refcnt and schedule it for freeing via workqueue 32299c55f7dSAlexei Starovoitov * (unrelying map implementation ops->map_free() might sleep) 32399c55f7dSAlexei Starovoitov */ 324bd5f5f4eSMartin KaFai Lau static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) 32599c55f7dSAlexei Starovoitov { 32699c55f7dSAlexei Starovoitov if (atomic_dec_and_test(&map->refcnt)) { 32734ad5580SMartin KaFai Lau /* bpf_map_free_id() must be called first */ 328bd5f5f4eSMartin KaFai Lau bpf_map_free_id(map, do_idr_lock); 32978958fcaSMartin KaFai Lau btf_put(map->btf); 33099c55f7dSAlexei Starovoitov INIT_WORK(&map->work, bpf_map_free_deferred); 33199c55f7dSAlexei Starovoitov schedule_work(&map->work); 33299c55f7dSAlexei Starovoitov } 33399c55f7dSAlexei Starovoitov } 33499c55f7dSAlexei Starovoitov 335bd5f5f4eSMartin KaFai Lau void bpf_map_put(struct bpf_map *map) 336bd5f5f4eSMartin KaFai Lau { 337bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, true); 338bd5f5f4eSMartin KaFai Lau } 339630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_put); 340bd5f5f4eSMartin KaFai Lau 341c9da161cSDaniel Borkmann void bpf_map_put_with_uref(struct bpf_map *map) 342c9da161cSDaniel Borkmann { 343c9da161cSDaniel Borkmann bpf_map_put_uref(map); 344c9da161cSDaniel Borkmann bpf_map_put(map); 345c9da161cSDaniel Borkmann } 346c9da161cSDaniel Borkmann 34799c55f7dSAlexei Starovoitov static int bpf_map_release(struct inode *inode, struct file *filp) 34899c55f7dSAlexei Starovoitov { 34961d1b6a4SDaniel Borkmann struct bpf_map *map = filp->private_data; 35061d1b6a4SDaniel Borkmann 35161d1b6a4SDaniel Borkmann if (map->ops->map_release) 35261d1b6a4SDaniel Borkmann map->ops->map_release(map, filp); 35361d1b6a4SDaniel Borkmann 35461d1b6a4SDaniel Borkmann bpf_map_put_with_uref(map); 35599c55f7dSAlexei Starovoitov return 0; 35699c55f7dSAlexei Starovoitov } 35799c55f7dSAlexei Starovoitov 35887df15deSDaniel Borkmann static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f) 35987df15deSDaniel Borkmann { 36087df15deSDaniel Borkmann fmode_t mode = f.file->f_mode; 36187df15deSDaniel Borkmann 36287df15deSDaniel Borkmann /* Our file permissions may have been overridden by global 36387df15deSDaniel Borkmann * map permissions facing syscall side. 36487df15deSDaniel Borkmann */ 36587df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) 36687df15deSDaniel Borkmann mode &= ~FMODE_CAN_WRITE; 36787df15deSDaniel Borkmann return mode; 36887df15deSDaniel Borkmann } 36987df15deSDaniel Borkmann 370f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 371f99bf205SDaniel Borkmann static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 372f99bf205SDaniel Borkmann { 373f99bf205SDaniel Borkmann const struct bpf_map *map = filp->private_data; 37421116b70SDaniel Borkmann const struct bpf_array *array; 37521116b70SDaniel Borkmann u32 owner_prog_type = 0; 3769780c0abSDaniel Borkmann u32 owner_jited = 0; 37721116b70SDaniel Borkmann 37821116b70SDaniel Borkmann if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 37921116b70SDaniel Borkmann array = container_of(map, struct bpf_array, map); 38021116b70SDaniel Borkmann owner_prog_type = array->owner_prog_type; 3819780c0abSDaniel Borkmann owner_jited = array->owner_jited; 38221116b70SDaniel Borkmann } 383f99bf205SDaniel Borkmann 384f99bf205SDaniel Borkmann seq_printf(m, 385f99bf205SDaniel Borkmann "map_type:\t%u\n" 386f99bf205SDaniel Borkmann "key_size:\t%u\n" 387f99bf205SDaniel Borkmann "value_size:\t%u\n" 388322cea2fSDaniel Borkmann "max_entries:\t%u\n" 38921116b70SDaniel Borkmann "map_flags:\t%#x\n" 3904316b409SDaniel Borkmann "memlock:\t%llu\n" 39187df15deSDaniel Borkmann "map_id:\t%u\n" 39287df15deSDaniel Borkmann "frozen:\t%u\n", 393f99bf205SDaniel Borkmann map->map_type, 394f99bf205SDaniel Borkmann map->key_size, 395f99bf205SDaniel Borkmann map->value_size, 396322cea2fSDaniel Borkmann map->max_entries, 39721116b70SDaniel Borkmann map->map_flags, 3984316b409SDaniel Borkmann map->pages * 1ULL << PAGE_SHIFT, 39987df15deSDaniel Borkmann map->id, 40087df15deSDaniel Borkmann READ_ONCE(map->frozen)); 40121116b70SDaniel Borkmann 4029780c0abSDaniel Borkmann if (owner_prog_type) { 40321116b70SDaniel Borkmann seq_printf(m, "owner_prog_type:\t%u\n", 40421116b70SDaniel Borkmann owner_prog_type); 4059780c0abSDaniel Borkmann seq_printf(m, "owner_jited:\t%u\n", 4069780c0abSDaniel Borkmann owner_jited); 4079780c0abSDaniel Borkmann } 408f99bf205SDaniel Borkmann } 409f99bf205SDaniel Borkmann #endif 410f99bf205SDaniel Borkmann 4116e71b04aSChenbo Feng static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, 4126e71b04aSChenbo Feng loff_t *ppos) 4136e71b04aSChenbo Feng { 4146e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 4156e71b04aSChenbo Feng * f_mode with FMODE_CAN_READ. 4166e71b04aSChenbo Feng */ 4176e71b04aSChenbo Feng return -EINVAL; 4186e71b04aSChenbo Feng } 4196e71b04aSChenbo Feng 4206e71b04aSChenbo Feng static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, 4216e71b04aSChenbo Feng size_t siz, loff_t *ppos) 4226e71b04aSChenbo Feng { 4236e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 4246e71b04aSChenbo Feng * f_mode with FMODE_CAN_WRITE. 4256e71b04aSChenbo Feng */ 4266e71b04aSChenbo Feng return -EINVAL; 4276e71b04aSChenbo Feng } 4286e71b04aSChenbo Feng 429f66e448cSChenbo Feng const struct file_operations bpf_map_fops = { 430f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 431f99bf205SDaniel Borkmann .show_fdinfo = bpf_map_show_fdinfo, 432f99bf205SDaniel Borkmann #endif 43399c55f7dSAlexei Starovoitov .release = bpf_map_release, 4346e71b04aSChenbo Feng .read = bpf_dummy_read, 4356e71b04aSChenbo Feng .write = bpf_dummy_write, 43699c55f7dSAlexei Starovoitov }; 43799c55f7dSAlexei Starovoitov 4386e71b04aSChenbo Feng int bpf_map_new_fd(struct bpf_map *map, int flags) 439aa79781bSDaniel Borkmann { 440afdb09c7SChenbo Feng int ret; 441afdb09c7SChenbo Feng 442afdb09c7SChenbo Feng ret = security_bpf_map(map, OPEN_FMODE(flags)); 443afdb09c7SChenbo Feng if (ret < 0) 444afdb09c7SChenbo Feng return ret; 445afdb09c7SChenbo Feng 446aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 4476e71b04aSChenbo Feng flags | O_CLOEXEC); 4486e71b04aSChenbo Feng } 4496e71b04aSChenbo Feng 4506e71b04aSChenbo Feng int bpf_get_file_flag(int flags) 4516e71b04aSChenbo Feng { 4526e71b04aSChenbo Feng if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) 4536e71b04aSChenbo Feng return -EINVAL; 4546e71b04aSChenbo Feng if (flags & BPF_F_RDONLY) 4556e71b04aSChenbo Feng return O_RDONLY; 4566e71b04aSChenbo Feng if (flags & BPF_F_WRONLY) 4576e71b04aSChenbo Feng return O_WRONLY; 4586e71b04aSChenbo Feng return O_RDWR; 459aa79781bSDaniel Borkmann } 460aa79781bSDaniel Borkmann 46199c55f7dSAlexei Starovoitov /* helper macro to check that unused fields 'union bpf_attr' are zero */ 46299c55f7dSAlexei Starovoitov #define CHECK_ATTR(CMD) \ 46399c55f7dSAlexei Starovoitov memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 46499c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD), 0, \ 46599c55f7dSAlexei Starovoitov sizeof(*attr) - \ 46699c55f7dSAlexei Starovoitov offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 46799c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD)) != NULL 46899c55f7dSAlexei Starovoitov 469cb4d2b3fSMartin KaFai Lau /* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes. 470cb4d2b3fSMartin KaFai Lau * Return 0 on success and < 0 on error. 471cb4d2b3fSMartin KaFai Lau */ 472cb4d2b3fSMartin KaFai Lau static int bpf_obj_name_cpy(char *dst, const char *src) 473cb4d2b3fSMartin KaFai Lau { 474cb4d2b3fSMartin KaFai Lau const char *end = src + BPF_OBJ_NAME_LEN; 475cb4d2b3fSMartin KaFai Lau 476473d9734SMartin KaFai Lau memset(dst, 0, BPF_OBJ_NAME_LEN); 4773e0ddc4fSDaniel Borkmann /* Copy all isalnum(), '_' and '.' chars. */ 478cb4d2b3fSMartin KaFai Lau while (src < end && *src) { 4793e0ddc4fSDaniel Borkmann if (!isalnum(*src) && 4803e0ddc4fSDaniel Borkmann *src != '_' && *src != '.') 481cb4d2b3fSMartin KaFai Lau return -EINVAL; 482cb4d2b3fSMartin KaFai Lau *dst++ = *src++; 483cb4d2b3fSMartin KaFai Lau } 484cb4d2b3fSMartin KaFai Lau 485cb4d2b3fSMartin KaFai Lau /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */ 486cb4d2b3fSMartin KaFai Lau if (src == end) 487cb4d2b3fSMartin KaFai Lau return -EINVAL; 488cb4d2b3fSMartin KaFai Lau 489cb4d2b3fSMartin KaFai Lau return 0; 490cb4d2b3fSMartin KaFai Lau } 491cb4d2b3fSMartin KaFai Lau 492e8d2bec0SDaniel Borkmann int map_check_no_btf(const struct bpf_map *map, 4931b2b234bSRoman Gushchin const struct btf *btf, 494e8d2bec0SDaniel Borkmann const struct btf_type *key_type, 495e8d2bec0SDaniel Borkmann const struct btf_type *value_type) 496e8d2bec0SDaniel Borkmann { 497e8d2bec0SDaniel Borkmann return -ENOTSUPP; 498e8d2bec0SDaniel Borkmann } 499e8d2bec0SDaniel Borkmann 500d83525caSAlexei Starovoitov static int map_check_btf(struct bpf_map *map, const struct btf *btf, 501e8d2bec0SDaniel Borkmann u32 btf_key_id, u32 btf_value_id) 502e8d2bec0SDaniel Borkmann { 503e8d2bec0SDaniel Borkmann const struct btf_type *key_type, *value_type; 504e8d2bec0SDaniel Borkmann u32 key_size, value_size; 505e8d2bec0SDaniel Borkmann int ret = 0; 506e8d2bec0SDaniel Borkmann 5072824ecb7SDaniel Borkmann /* Some maps allow key to be unspecified. */ 5082824ecb7SDaniel Borkmann if (btf_key_id) { 509e8d2bec0SDaniel Borkmann key_type = btf_type_id_size(btf, &btf_key_id, &key_size); 510e8d2bec0SDaniel Borkmann if (!key_type || key_size != map->key_size) 511e8d2bec0SDaniel Borkmann return -EINVAL; 5122824ecb7SDaniel Borkmann } else { 5132824ecb7SDaniel Borkmann key_type = btf_type_by_id(btf, 0); 5142824ecb7SDaniel Borkmann if (!map->ops->map_check_btf) 5152824ecb7SDaniel Borkmann return -EINVAL; 5162824ecb7SDaniel Borkmann } 517e8d2bec0SDaniel Borkmann 518e8d2bec0SDaniel Borkmann value_type = btf_type_id_size(btf, &btf_value_id, &value_size); 519e8d2bec0SDaniel Borkmann if (!value_type || value_size != map->value_size) 520e8d2bec0SDaniel Borkmann return -EINVAL; 521e8d2bec0SDaniel Borkmann 522d83525caSAlexei Starovoitov map->spin_lock_off = btf_find_spin_lock(btf, value_type); 523d83525caSAlexei Starovoitov 524d83525caSAlexei Starovoitov if (map_value_has_spin_lock(map)) { 525591fe988SDaniel Borkmann if (map->map_flags & BPF_F_RDONLY_PROG) 526591fe988SDaniel Borkmann return -EACCES; 527d83525caSAlexei Starovoitov if (map->map_type != BPF_MAP_TYPE_HASH && 528e16d2f1aSAlexei Starovoitov map->map_type != BPF_MAP_TYPE_ARRAY && 5296ac99e8fSMartin KaFai Lau map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 5306ac99e8fSMartin KaFai Lau map->map_type != BPF_MAP_TYPE_SK_STORAGE) 531d83525caSAlexei Starovoitov return -ENOTSUPP; 532d83525caSAlexei Starovoitov if (map->spin_lock_off + sizeof(struct bpf_spin_lock) > 533d83525caSAlexei Starovoitov map->value_size) { 534d83525caSAlexei Starovoitov WARN_ONCE(1, 535d83525caSAlexei Starovoitov "verifier bug spin_lock_off %d value_size %d\n", 536d83525caSAlexei Starovoitov map->spin_lock_off, map->value_size); 537d83525caSAlexei Starovoitov return -EFAULT; 538d83525caSAlexei Starovoitov } 539d83525caSAlexei Starovoitov } 540d83525caSAlexei Starovoitov 541e8d2bec0SDaniel Borkmann if (map->ops->map_check_btf) 5421b2b234bSRoman Gushchin ret = map->ops->map_check_btf(map, btf, key_type, value_type); 543e8d2bec0SDaniel Borkmann 544e8d2bec0SDaniel Borkmann return ret; 545e8d2bec0SDaniel Borkmann } 546e8d2bec0SDaniel Borkmann 5479b2cf328SMartin KaFai Lau #define BPF_MAP_CREATE_LAST_FIELD btf_value_type_id 54899c55f7dSAlexei Starovoitov /* called via syscall */ 54999c55f7dSAlexei Starovoitov static int map_create(union bpf_attr *attr) 55099c55f7dSAlexei Starovoitov { 55196eabe7aSMartin KaFai Lau int numa_node = bpf_map_attr_numa_node(attr); 55299c55f7dSAlexei Starovoitov struct bpf_map *map; 5536e71b04aSChenbo Feng int f_flags; 55499c55f7dSAlexei Starovoitov int err; 55599c55f7dSAlexei Starovoitov 55699c55f7dSAlexei Starovoitov err = CHECK_ATTR(BPF_MAP_CREATE); 55799c55f7dSAlexei Starovoitov if (err) 55899c55f7dSAlexei Starovoitov return -EINVAL; 55999c55f7dSAlexei Starovoitov 5606e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->map_flags); 5616e71b04aSChenbo Feng if (f_flags < 0) 5626e71b04aSChenbo Feng return f_flags; 5636e71b04aSChenbo Feng 56496eabe7aSMartin KaFai Lau if (numa_node != NUMA_NO_NODE && 56596e5ae4eSEric Dumazet ((unsigned int)numa_node >= nr_node_ids || 56696e5ae4eSEric Dumazet !node_online(numa_node))) 56796eabe7aSMartin KaFai Lau return -EINVAL; 56896eabe7aSMartin KaFai Lau 56999c55f7dSAlexei Starovoitov /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 57099c55f7dSAlexei Starovoitov map = find_and_alloc_map(attr); 57199c55f7dSAlexei Starovoitov if (IS_ERR(map)) 57299c55f7dSAlexei Starovoitov return PTR_ERR(map); 57399c55f7dSAlexei Starovoitov 574ad5b177bSMartin KaFai Lau err = bpf_obj_name_cpy(map->name, attr->map_name); 575ad5b177bSMartin KaFai Lau if (err) 576ad5b177bSMartin KaFai Lau goto free_map_nouncharge; 577ad5b177bSMartin KaFai Lau 57899c55f7dSAlexei Starovoitov atomic_set(&map->refcnt, 1); 579c9da161cSDaniel Borkmann atomic_set(&map->usercnt, 1); 58099c55f7dSAlexei Starovoitov 581e8d2bec0SDaniel Borkmann if (attr->btf_key_type_id || attr->btf_value_type_id) { 582a26ca7c9SMartin KaFai Lau struct btf *btf; 583a26ca7c9SMartin KaFai Lau 5842824ecb7SDaniel Borkmann if (!attr->btf_value_type_id) { 585a26ca7c9SMartin KaFai Lau err = -EINVAL; 586a26ca7c9SMartin KaFai Lau goto free_map_nouncharge; 587a26ca7c9SMartin KaFai Lau } 588a26ca7c9SMartin KaFai Lau 589a26ca7c9SMartin KaFai Lau btf = btf_get_by_fd(attr->btf_fd); 590a26ca7c9SMartin KaFai Lau if (IS_ERR(btf)) { 591a26ca7c9SMartin KaFai Lau err = PTR_ERR(btf); 592a26ca7c9SMartin KaFai Lau goto free_map_nouncharge; 593a26ca7c9SMartin KaFai Lau } 594a26ca7c9SMartin KaFai Lau 595e8d2bec0SDaniel Borkmann err = map_check_btf(map, btf, attr->btf_key_type_id, 5969b2cf328SMartin KaFai Lau attr->btf_value_type_id); 597a26ca7c9SMartin KaFai Lau if (err) { 598a26ca7c9SMartin KaFai Lau btf_put(btf); 599a26ca7c9SMartin KaFai Lau goto free_map_nouncharge; 600a26ca7c9SMartin KaFai Lau } 601a26ca7c9SMartin KaFai Lau 602a26ca7c9SMartin KaFai Lau map->btf = btf; 6039b2cf328SMartin KaFai Lau map->btf_key_type_id = attr->btf_key_type_id; 6049b2cf328SMartin KaFai Lau map->btf_value_type_id = attr->btf_value_type_id; 605d83525caSAlexei Starovoitov } else { 606d83525caSAlexei Starovoitov map->spin_lock_off = -EINVAL; 607a26ca7c9SMartin KaFai Lau } 608a26ca7c9SMartin KaFai Lau 609afdb09c7SChenbo Feng err = security_bpf_map_alloc(map); 610aaac3ba9SAlexei Starovoitov if (err) 61120b2b24fSDaniel Borkmann goto free_map_nouncharge; 612aaac3ba9SAlexei Starovoitov 6130a4c58f5SRoman Gushchin err = bpf_map_init_memlock(map); 614afdb09c7SChenbo Feng if (err) 615afdb09c7SChenbo Feng goto free_map_sec; 616afdb09c7SChenbo Feng 617f3f1c054SMartin KaFai Lau err = bpf_map_alloc_id(map); 618f3f1c054SMartin KaFai Lau if (err) 619f3f1c054SMartin KaFai Lau goto free_map; 620f3f1c054SMartin KaFai Lau 6216e71b04aSChenbo Feng err = bpf_map_new_fd(map, f_flags); 622bd5f5f4eSMartin KaFai Lau if (err < 0) { 623bd5f5f4eSMartin KaFai Lau /* failed to allocate fd. 624352d20d6SPeng Sun * bpf_map_put_with_uref() is needed because the above 625bd5f5f4eSMartin KaFai Lau * bpf_map_alloc_id() has published the map 626bd5f5f4eSMartin KaFai Lau * to the userspace and the userspace may 627bd5f5f4eSMartin KaFai Lau * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 628bd5f5f4eSMartin KaFai Lau */ 629352d20d6SPeng Sun bpf_map_put_with_uref(map); 630bd5f5f4eSMartin KaFai Lau return err; 631bd5f5f4eSMartin KaFai Lau } 63299c55f7dSAlexei Starovoitov 63399c55f7dSAlexei Starovoitov return err; 63499c55f7dSAlexei Starovoitov 63599c55f7dSAlexei Starovoitov free_map: 6360a4c58f5SRoman Gushchin bpf_map_release_memlock(map); 637afdb09c7SChenbo Feng free_map_sec: 638afdb09c7SChenbo Feng security_bpf_map_free(map); 63920b2b24fSDaniel Borkmann free_map_nouncharge: 640a26ca7c9SMartin KaFai Lau btf_put(map->btf); 64199c55f7dSAlexei Starovoitov map->ops->map_free(map); 64299c55f7dSAlexei Starovoitov return err; 64399c55f7dSAlexei Starovoitov } 64499c55f7dSAlexei Starovoitov 645db20fd2bSAlexei Starovoitov /* if error is returned, fd is released. 646db20fd2bSAlexei Starovoitov * On success caller should complete fd access with matching fdput() 647db20fd2bSAlexei Starovoitov */ 648c2101297SDaniel Borkmann struct bpf_map *__bpf_map_get(struct fd f) 649db20fd2bSAlexei Starovoitov { 650db20fd2bSAlexei Starovoitov if (!f.file) 651db20fd2bSAlexei Starovoitov return ERR_PTR(-EBADF); 652db20fd2bSAlexei Starovoitov if (f.file->f_op != &bpf_map_fops) { 653db20fd2bSAlexei Starovoitov fdput(f); 654db20fd2bSAlexei Starovoitov return ERR_PTR(-EINVAL); 655db20fd2bSAlexei Starovoitov } 656db20fd2bSAlexei Starovoitov 657c2101297SDaniel Borkmann return f.file->private_data; 658c2101297SDaniel Borkmann } 659c2101297SDaniel Borkmann 66092117d84SAlexei Starovoitov /* prog's and map's refcnt limit */ 66192117d84SAlexei Starovoitov #define BPF_MAX_REFCNT 32768 66292117d84SAlexei Starovoitov 66392117d84SAlexei Starovoitov struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref) 664c9da161cSDaniel Borkmann { 66592117d84SAlexei Starovoitov if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) { 66692117d84SAlexei Starovoitov atomic_dec(&map->refcnt); 66792117d84SAlexei Starovoitov return ERR_PTR(-EBUSY); 66892117d84SAlexei Starovoitov } 669c9da161cSDaniel Borkmann if (uref) 670c9da161cSDaniel Borkmann atomic_inc(&map->usercnt); 67192117d84SAlexei Starovoitov return map; 672c9da161cSDaniel Borkmann } 673630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_inc); 674c9da161cSDaniel Borkmann 675c9da161cSDaniel Borkmann struct bpf_map *bpf_map_get_with_uref(u32 ufd) 676c2101297SDaniel Borkmann { 677c2101297SDaniel Borkmann struct fd f = fdget(ufd); 678c2101297SDaniel Borkmann struct bpf_map *map; 679c2101297SDaniel Borkmann 680c2101297SDaniel Borkmann map = __bpf_map_get(f); 681c2101297SDaniel Borkmann if (IS_ERR(map)) 682c2101297SDaniel Borkmann return map; 683c2101297SDaniel Borkmann 68492117d84SAlexei Starovoitov map = bpf_map_inc(map, true); 685c2101297SDaniel Borkmann fdput(f); 686db20fd2bSAlexei Starovoitov 687db20fd2bSAlexei Starovoitov return map; 688db20fd2bSAlexei Starovoitov } 689db20fd2bSAlexei Starovoitov 690bd5f5f4eSMartin KaFai Lau /* map_idr_lock should have been held */ 691bd5f5f4eSMartin KaFai Lau static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map, 692bd5f5f4eSMartin KaFai Lau bool uref) 693bd5f5f4eSMartin KaFai Lau { 694bd5f5f4eSMartin KaFai Lau int refold; 695bd5f5f4eSMartin KaFai Lau 696bfc18e38SMark Rutland refold = atomic_fetch_add_unless(&map->refcnt, 1, 0); 697bd5f5f4eSMartin KaFai Lau 698bd5f5f4eSMartin KaFai Lau if (refold >= BPF_MAX_REFCNT) { 699bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, false); 700bd5f5f4eSMartin KaFai Lau return ERR_PTR(-EBUSY); 701bd5f5f4eSMartin KaFai Lau } 702bd5f5f4eSMartin KaFai Lau 703bd5f5f4eSMartin KaFai Lau if (!refold) 704bd5f5f4eSMartin KaFai Lau return ERR_PTR(-ENOENT); 705bd5f5f4eSMartin KaFai Lau 706bd5f5f4eSMartin KaFai Lau if (uref) 707bd5f5f4eSMartin KaFai Lau atomic_inc(&map->usercnt); 708bd5f5f4eSMartin KaFai Lau 709bd5f5f4eSMartin KaFai Lau return map; 710bd5f5f4eSMartin KaFai Lau } 711bd5f5f4eSMartin KaFai Lau 712b8cdc051SAlexei Starovoitov int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 713b8cdc051SAlexei Starovoitov { 714b8cdc051SAlexei Starovoitov return -ENOTSUPP; 715b8cdc051SAlexei Starovoitov } 716b8cdc051SAlexei Starovoitov 717c9d29f46SMauricio Vasquez B static void *__bpf_copy_key(void __user *ukey, u64 key_size) 718c9d29f46SMauricio Vasquez B { 719c9d29f46SMauricio Vasquez B if (key_size) 720c9d29f46SMauricio Vasquez B return memdup_user(ukey, key_size); 721c9d29f46SMauricio Vasquez B 722c9d29f46SMauricio Vasquez B if (ukey) 723c9d29f46SMauricio Vasquez B return ERR_PTR(-EINVAL); 724c9d29f46SMauricio Vasquez B 725c9d29f46SMauricio Vasquez B return NULL; 726c9d29f46SMauricio Vasquez B } 727c9d29f46SMauricio Vasquez B 728db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 72996049f3aSAlexei Starovoitov #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags 730db20fd2bSAlexei Starovoitov 731db20fd2bSAlexei Starovoitov static int map_lookup_elem(union bpf_attr *attr) 732db20fd2bSAlexei Starovoitov { 733535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 734535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 735db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 736db20fd2bSAlexei Starovoitov struct bpf_map *map; 7378ebe667cSAlexei Starovoitov void *key, *value, *ptr; 73815a07b33SAlexei Starovoitov u32 value_size; 739592867bfSDaniel Borkmann struct fd f; 740db20fd2bSAlexei Starovoitov int err; 741db20fd2bSAlexei Starovoitov 742db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 743db20fd2bSAlexei Starovoitov return -EINVAL; 744db20fd2bSAlexei Starovoitov 74596049f3aSAlexei Starovoitov if (attr->flags & ~BPF_F_LOCK) 74696049f3aSAlexei Starovoitov return -EINVAL; 74796049f3aSAlexei Starovoitov 748592867bfSDaniel Borkmann f = fdget(ufd); 749c2101297SDaniel Borkmann map = __bpf_map_get(f); 750db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 751db20fd2bSAlexei Starovoitov return PTR_ERR(map); 75287df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 7536e71b04aSChenbo Feng err = -EPERM; 7546e71b04aSChenbo Feng goto err_put; 7556e71b04aSChenbo Feng } 7566e71b04aSChenbo Feng 75796049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 75896049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 75996049f3aSAlexei Starovoitov err = -EINVAL; 76096049f3aSAlexei Starovoitov goto err_put; 76196049f3aSAlexei Starovoitov } 76296049f3aSAlexei Starovoitov 763c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 764e4448ed8SAl Viro if (IS_ERR(key)) { 765e4448ed8SAl Viro err = PTR_ERR(key); 766db20fd2bSAlexei Starovoitov goto err_put; 767e4448ed8SAl Viro } 768db20fd2bSAlexei Starovoitov 76915a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 7708f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 771b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 772b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 77315a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 77414dc6f04SMartin KaFai Lau else if (IS_FD_MAP(map)) 77514dc6f04SMartin KaFai Lau value_size = sizeof(u32); 77615a07b33SAlexei Starovoitov else 77715a07b33SAlexei Starovoitov value_size = map->value_size; 77815a07b33SAlexei Starovoitov 7798ebe667cSAlexei Starovoitov err = -ENOMEM; 78015a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 781db20fd2bSAlexei Starovoitov if (!value) 7828ebe667cSAlexei Starovoitov goto free_key; 7838ebe667cSAlexei Starovoitov 784a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 785a3884572SJakub Kicinski err = bpf_map_offload_lookup_elem(map, key, value); 7867c4cd051SMartin KaFai Lau goto done; 7877c4cd051SMartin KaFai Lau } 7887c4cd051SMartin KaFai Lau 7897c4cd051SMartin KaFai Lau preempt_disable(); 7907c4cd051SMartin KaFai Lau this_cpu_inc(bpf_prog_active); 7917c4cd051SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 7928f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 79315a07b33SAlexei Starovoitov err = bpf_percpu_hash_copy(map, key, value); 79415a07b33SAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 79515a07b33SAlexei Starovoitov err = bpf_percpu_array_copy(map, key, value); 796b741f163SRoman Gushchin } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 797b741f163SRoman Gushchin err = bpf_percpu_cgroup_storage_copy(map, key, value); 798557c0c6eSAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 799557c0c6eSAlexei Starovoitov err = bpf_stackmap_copy(map, key, value); 80014dc6f04SMartin KaFai Lau } else if (IS_FD_ARRAY(map)) { 80114dc6f04SMartin KaFai Lau err = bpf_fd_array_map_lookup_elem(map, key, value); 80214dc6f04SMartin KaFai Lau } else if (IS_FD_HASH(map)) { 80314dc6f04SMartin KaFai Lau err = bpf_fd_htab_map_lookup_elem(map, key, value); 8045dc4c4b7SMartin KaFai Lau } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 8055dc4c4b7SMartin KaFai Lau err = bpf_fd_reuseport_array_lookup_elem(map, key, value); 806f1a2e44aSMauricio Vasquez B } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 807f1a2e44aSMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 808f1a2e44aSMauricio Vasquez B err = map->ops->map_peek_elem(map, value); 80915a07b33SAlexei Starovoitov } else { 8108ebe667cSAlexei Starovoitov rcu_read_lock(); 811*c6110222SDaniel Borkmann if (map->ops->map_lookup_elem_sys_only) 812*c6110222SDaniel Borkmann ptr = map->ops->map_lookup_elem_sys_only(map, key); 813*c6110222SDaniel Borkmann else 8148ebe667cSAlexei Starovoitov ptr = map->ops->map_lookup_elem(map, key); 815509db283SPrashant Bhole if (IS_ERR(ptr)) { 816509db283SPrashant Bhole err = PTR_ERR(ptr); 817509db283SPrashant Bhole } else if (!ptr) { 818509db283SPrashant Bhole err = -ENOENT; 819509db283SPrashant Bhole } else { 820509db283SPrashant Bhole err = 0; 82196049f3aSAlexei Starovoitov if (attr->flags & BPF_F_LOCK) 82296049f3aSAlexei Starovoitov /* lock 'ptr' and copy everything but lock */ 82396049f3aSAlexei Starovoitov copy_map_value_locked(map, value, ptr, true); 82496049f3aSAlexei Starovoitov else 825d83525caSAlexei Starovoitov copy_map_value(map, value, ptr); 82696049f3aSAlexei Starovoitov /* mask lock, since value wasn't zero inited */ 82796049f3aSAlexei Starovoitov check_and_init_map_lock(map, value); 828509db283SPrashant Bhole } 8298ebe667cSAlexei Starovoitov rcu_read_unlock(); 83015a07b33SAlexei Starovoitov } 8317c4cd051SMartin KaFai Lau this_cpu_dec(bpf_prog_active); 8327c4cd051SMartin KaFai Lau preempt_enable(); 8338ebe667cSAlexei Starovoitov 8347c4cd051SMartin KaFai Lau done: 83515a07b33SAlexei Starovoitov if (err) 8368ebe667cSAlexei Starovoitov goto free_value; 837db20fd2bSAlexei Starovoitov 838db20fd2bSAlexei Starovoitov err = -EFAULT; 83915a07b33SAlexei Starovoitov if (copy_to_user(uvalue, value, value_size) != 0) 8408ebe667cSAlexei Starovoitov goto free_value; 841db20fd2bSAlexei Starovoitov 842db20fd2bSAlexei Starovoitov err = 0; 843db20fd2bSAlexei Starovoitov 8448ebe667cSAlexei Starovoitov free_value: 8458ebe667cSAlexei Starovoitov kfree(value); 846db20fd2bSAlexei Starovoitov free_key: 847db20fd2bSAlexei Starovoitov kfree(key); 848db20fd2bSAlexei Starovoitov err_put: 849db20fd2bSAlexei Starovoitov fdput(f); 850db20fd2bSAlexei Starovoitov return err; 851db20fd2bSAlexei Starovoitov } 852db20fd2bSAlexei Starovoitov 8531ae80cf3SDaniel Colascione static void maybe_wait_bpf_programs(struct bpf_map *map) 8541ae80cf3SDaniel Colascione { 8551ae80cf3SDaniel Colascione /* Wait for any running BPF programs to complete so that 8561ae80cf3SDaniel Colascione * userspace, when we return to it, knows that all programs 8571ae80cf3SDaniel Colascione * that could be running use the new map value. 8581ae80cf3SDaniel Colascione */ 8591ae80cf3SDaniel Colascione if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS || 8601ae80cf3SDaniel Colascione map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 8611ae80cf3SDaniel Colascione synchronize_rcu(); 8621ae80cf3SDaniel Colascione } 8631ae80cf3SDaniel Colascione 8643274f520SAlexei Starovoitov #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 865db20fd2bSAlexei Starovoitov 866db20fd2bSAlexei Starovoitov static int map_update_elem(union bpf_attr *attr) 867db20fd2bSAlexei Starovoitov { 868535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 869535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 870db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 871db20fd2bSAlexei Starovoitov struct bpf_map *map; 872db20fd2bSAlexei Starovoitov void *key, *value; 87315a07b33SAlexei Starovoitov u32 value_size; 874592867bfSDaniel Borkmann struct fd f; 875db20fd2bSAlexei Starovoitov int err; 876db20fd2bSAlexei Starovoitov 877db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 878db20fd2bSAlexei Starovoitov return -EINVAL; 879db20fd2bSAlexei Starovoitov 880592867bfSDaniel Borkmann f = fdget(ufd); 881c2101297SDaniel Borkmann map = __bpf_map_get(f); 882db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 883db20fd2bSAlexei Starovoitov return PTR_ERR(map); 88487df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 8856e71b04aSChenbo Feng err = -EPERM; 8866e71b04aSChenbo Feng goto err_put; 8876e71b04aSChenbo Feng } 8886e71b04aSChenbo Feng 88996049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 89096049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 89196049f3aSAlexei Starovoitov err = -EINVAL; 89296049f3aSAlexei Starovoitov goto err_put; 89396049f3aSAlexei Starovoitov } 89496049f3aSAlexei Starovoitov 895c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 896e4448ed8SAl Viro if (IS_ERR(key)) { 897e4448ed8SAl Viro err = PTR_ERR(key); 898db20fd2bSAlexei Starovoitov goto err_put; 899e4448ed8SAl Viro } 900db20fd2bSAlexei Starovoitov 90115a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 9028f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 903b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 904b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 90515a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 90615a07b33SAlexei Starovoitov else 90715a07b33SAlexei Starovoitov value_size = map->value_size; 90815a07b33SAlexei Starovoitov 909db20fd2bSAlexei Starovoitov err = -ENOMEM; 91015a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 911db20fd2bSAlexei Starovoitov if (!value) 912db20fd2bSAlexei Starovoitov goto free_key; 913db20fd2bSAlexei Starovoitov 914db20fd2bSAlexei Starovoitov err = -EFAULT; 91515a07b33SAlexei Starovoitov if (copy_from_user(value, uvalue, value_size) != 0) 916db20fd2bSAlexei Starovoitov goto free_value; 917db20fd2bSAlexei Starovoitov 9186710e112SJesper Dangaard Brouer /* Need to create a kthread, thus must support schedule */ 919a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 920a3884572SJakub Kicinski err = bpf_map_offload_update_elem(map, key, value, attr->flags); 921a3884572SJakub Kicinski goto out; 92299ba2b5aSJohn Fastabend } else if (map->map_type == BPF_MAP_TYPE_CPUMAP || 92399ba2b5aSJohn Fastabend map->map_type == BPF_MAP_TYPE_SOCKHASH || 92499ba2b5aSJohn Fastabend map->map_type == BPF_MAP_TYPE_SOCKMAP) { 9256710e112SJesper Dangaard Brouer err = map->ops->map_update_elem(map, key, value, attr->flags); 9266710e112SJesper Dangaard Brouer goto out; 9276710e112SJesper Dangaard Brouer } 9286710e112SJesper Dangaard Brouer 929b121d1e7SAlexei Starovoitov /* must increment bpf_prog_active to avoid kprobe+bpf triggering from 930b121d1e7SAlexei Starovoitov * inside bpf map update or delete otherwise deadlocks are possible 931b121d1e7SAlexei Starovoitov */ 932b121d1e7SAlexei Starovoitov preempt_disable(); 933b121d1e7SAlexei Starovoitov __this_cpu_inc(bpf_prog_active); 9348f844938SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 9358f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 93615a07b33SAlexei Starovoitov err = bpf_percpu_hash_update(map, key, value, attr->flags); 93715a07b33SAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 93815a07b33SAlexei Starovoitov err = bpf_percpu_array_update(map, key, value, attr->flags); 939b741f163SRoman Gushchin } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 940b741f163SRoman Gushchin err = bpf_percpu_cgroup_storage_update(map, key, value, 941b741f163SRoman Gushchin attr->flags); 9429c147b56SMickaël Salaün } else if (IS_FD_ARRAY(map)) { 943d056a788SDaniel Borkmann rcu_read_lock(); 944d056a788SDaniel Borkmann err = bpf_fd_array_map_update_elem(map, f.file, key, value, 945d056a788SDaniel Borkmann attr->flags); 946d056a788SDaniel Borkmann rcu_read_unlock(); 947bcc6b1b7SMartin KaFai Lau } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 948bcc6b1b7SMartin KaFai Lau rcu_read_lock(); 949bcc6b1b7SMartin KaFai Lau err = bpf_fd_htab_map_update_elem(map, f.file, key, value, 950bcc6b1b7SMartin KaFai Lau attr->flags); 951bcc6b1b7SMartin KaFai Lau rcu_read_unlock(); 9525dc4c4b7SMartin KaFai Lau } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 9535dc4c4b7SMartin KaFai Lau /* rcu_read_lock() is not needed */ 9545dc4c4b7SMartin KaFai Lau err = bpf_fd_reuseport_array_update_elem(map, key, value, 9555dc4c4b7SMartin KaFai Lau attr->flags); 956f1a2e44aSMauricio Vasquez B } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 957f1a2e44aSMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 958f1a2e44aSMauricio Vasquez B err = map->ops->map_push_elem(map, value, attr->flags); 95915a07b33SAlexei Starovoitov } else { 960db20fd2bSAlexei Starovoitov rcu_read_lock(); 9613274f520SAlexei Starovoitov err = map->ops->map_update_elem(map, key, value, attr->flags); 962db20fd2bSAlexei Starovoitov rcu_read_unlock(); 96315a07b33SAlexei Starovoitov } 964b121d1e7SAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 965b121d1e7SAlexei Starovoitov preempt_enable(); 9661ae80cf3SDaniel Colascione maybe_wait_bpf_programs(map); 9676710e112SJesper Dangaard Brouer out: 968db20fd2bSAlexei Starovoitov free_value: 969db20fd2bSAlexei Starovoitov kfree(value); 970db20fd2bSAlexei Starovoitov free_key: 971db20fd2bSAlexei Starovoitov kfree(key); 972db20fd2bSAlexei Starovoitov err_put: 973db20fd2bSAlexei Starovoitov fdput(f); 974db20fd2bSAlexei Starovoitov return err; 975db20fd2bSAlexei Starovoitov } 976db20fd2bSAlexei Starovoitov 977db20fd2bSAlexei Starovoitov #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 978db20fd2bSAlexei Starovoitov 979db20fd2bSAlexei Starovoitov static int map_delete_elem(union bpf_attr *attr) 980db20fd2bSAlexei Starovoitov { 981535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 982db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 983db20fd2bSAlexei Starovoitov struct bpf_map *map; 984592867bfSDaniel Borkmann struct fd f; 985db20fd2bSAlexei Starovoitov void *key; 986db20fd2bSAlexei Starovoitov int err; 987db20fd2bSAlexei Starovoitov 988db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 989db20fd2bSAlexei Starovoitov return -EINVAL; 990db20fd2bSAlexei Starovoitov 991592867bfSDaniel Borkmann f = fdget(ufd); 992c2101297SDaniel Borkmann map = __bpf_map_get(f); 993db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 994db20fd2bSAlexei Starovoitov return PTR_ERR(map); 99587df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 9966e71b04aSChenbo Feng err = -EPERM; 9976e71b04aSChenbo Feng goto err_put; 9986e71b04aSChenbo Feng } 9996e71b04aSChenbo Feng 1000c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1001e4448ed8SAl Viro if (IS_ERR(key)) { 1002e4448ed8SAl Viro err = PTR_ERR(key); 1003db20fd2bSAlexei Starovoitov goto err_put; 1004e4448ed8SAl Viro } 1005db20fd2bSAlexei Starovoitov 1006a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1007a3884572SJakub Kicinski err = bpf_map_offload_delete_elem(map, key); 1008a3884572SJakub Kicinski goto out; 1009a3884572SJakub Kicinski } 1010a3884572SJakub Kicinski 1011b121d1e7SAlexei Starovoitov preempt_disable(); 1012b121d1e7SAlexei Starovoitov __this_cpu_inc(bpf_prog_active); 1013db20fd2bSAlexei Starovoitov rcu_read_lock(); 1014db20fd2bSAlexei Starovoitov err = map->ops->map_delete_elem(map, key); 1015db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1016b121d1e7SAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 1017b121d1e7SAlexei Starovoitov preempt_enable(); 10181ae80cf3SDaniel Colascione maybe_wait_bpf_programs(map); 1019a3884572SJakub Kicinski out: 1020db20fd2bSAlexei Starovoitov kfree(key); 1021db20fd2bSAlexei Starovoitov err_put: 1022db20fd2bSAlexei Starovoitov fdput(f); 1023db20fd2bSAlexei Starovoitov return err; 1024db20fd2bSAlexei Starovoitov } 1025db20fd2bSAlexei Starovoitov 1026db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 1027db20fd2bSAlexei Starovoitov #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 1028db20fd2bSAlexei Starovoitov 1029db20fd2bSAlexei Starovoitov static int map_get_next_key(union bpf_attr *attr) 1030db20fd2bSAlexei Starovoitov { 1031535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1032535e7b4bSMickaël Salaün void __user *unext_key = u64_to_user_ptr(attr->next_key); 1033db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1034db20fd2bSAlexei Starovoitov struct bpf_map *map; 1035db20fd2bSAlexei Starovoitov void *key, *next_key; 1036592867bfSDaniel Borkmann struct fd f; 1037db20fd2bSAlexei Starovoitov int err; 1038db20fd2bSAlexei Starovoitov 1039db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 1040db20fd2bSAlexei Starovoitov return -EINVAL; 1041db20fd2bSAlexei Starovoitov 1042592867bfSDaniel Borkmann f = fdget(ufd); 1043c2101297SDaniel Borkmann map = __bpf_map_get(f); 1044db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1045db20fd2bSAlexei Starovoitov return PTR_ERR(map); 104687df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 10476e71b04aSChenbo Feng err = -EPERM; 10486e71b04aSChenbo Feng goto err_put; 10496e71b04aSChenbo Feng } 10506e71b04aSChenbo Feng 10518fe45924STeng Qin if (ukey) { 1052c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1053e4448ed8SAl Viro if (IS_ERR(key)) { 1054e4448ed8SAl Viro err = PTR_ERR(key); 1055db20fd2bSAlexei Starovoitov goto err_put; 1056e4448ed8SAl Viro } 10578fe45924STeng Qin } else { 10588fe45924STeng Qin key = NULL; 10598fe45924STeng Qin } 1060db20fd2bSAlexei Starovoitov 1061db20fd2bSAlexei Starovoitov err = -ENOMEM; 1062db20fd2bSAlexei Starovoitov next_key = kmalloc(map->key_size, GFP_USER); 1063db20fd2bSAlexei Starovoitov if (!next_key) 1064db20fd2bSAlexei Starovoitov goto free_key; 1065db20fd2bSAlexei Starovoitov 1066a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1067a3884572SJakub Kicinski err = bpf_map_offload_get_next_key(map, key, next_key); 1068a3884572SJakub Kicinski goto out; 1069a3884572SJakub Kicinski } 1070a3884572SJakub Kicinski 1071db20fd2bSAlexei Starovoitov rcu_read_lock(); 1072db20fd2bSAlexei Starovoitov err = map->ops->map_get_next_key(map, key, next_key); 1073db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1074a3884572SJakub Kicinski out: 1075db20fd2bSAlexei Starovoitov if (err) 1076db20fd2bSAlexei Starovoitov goto free_next_key; 1077db20fd2bSAlexei Starovoitov 1078db20fd2bSAlexei Starovoitov err = -EFAULT; 1079db20fd2bSAlexei Starovoitov if (copy_to_user(unext_key, next_key, map->key_size) != 0) 1080db20fd2bSAlexei Starovoitov goto free_next_key; 1081db20fd2bSAlexei Starovoitov 1082db20fd2bSAlexei Starovoitov err = 0; 1083db20fd2bSAlexei Starovoitov 1084db20fd2bSAlexei Starovoitov free_next_key: 1085db20fd2bSAlexei Starovoitov kfree(next_key); 1086db20fd2bSAlexei Starovoitov free_key: 1087db20fd2bSAlexei Starovoitov kfree(key); 1088db20fd2bSAlexei Starovoitov err_put: 1089db20fd2bSAlexei Starovoitov fdput(f); 1090db20fd2bSAlexei Starovoitov return err; 1091db20fd2bSAlexei Starovoitov } 1092db20fd2bSAlexei Starovoitov 1093bd513cd0SMauricio Vasquez B #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value 1094bd513cd0SMauricio Vasquez B 1095bd513cd0SMauricio Vasquez B static int map_lookup_and_delete_elem(union bpf_attr *attr) 1096bd513cd0SMauricio Vasquez B { 1097bd513cd0SMauricio Vasquez B void __user *ukey = u64_to_user_ptr(attr->key); 1098bd513cd0SMauricio Vasquez B void __user *uvalue = u64_to_user_ptr(attr->value); 1099bd513cd0SMauricio Vasquez B int ufd = attr->map_fd; 1100bd513cd0SMauricio Vasquez B struct bpf_map *map; 1101540fefc0SAlexei Starovoitov void *key, *value; 1102bd513cd0SMauricio Vasquez B u32 value_size; 1103bd513cd0SMauricio Vasquez B struct fd f; 1104bd513cd0SMauricio Vasquez B int err; 1105bd513cd0SMauricio Vasquez B 1106bd513cd0SMauricio Vasquez B if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) 1107bd513cd0SMauricio Vasquez B return -EINVAL; 1108bd513cd0SMauricio Vasquez B 1109bd513cd0SMauricio Vasquez B f = fdget(ufd); 1110bd513cd0SMauricio Vasquez B map = __bpf_map_get(f); 1111bd513cd0SMauricio Vasquez B if (IS_ERR(map)) 1112bd513cd0SMauricio Vasquez B return PTR_ERR(map); 111387df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 1114bd513cd0SMauricio Vasquez B err = -EPERM; 1115bd513cd0SMauricio Vasquez B goto err_put; 1116bd513cd0SMauricio Vasquez B } 1117bd513cd0SMauricio Vasquez B 1118bd513cd0SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1119bd513cd0SMauricio Vasquez B if (IS_ERR(key)) { 1120bd513cd0SMauricio Vasquez B err = PTR_ERR(key); 1121bd513cd0SMauricio Vasquez B goto err_put; 1122bd513cd0SMauricio Vasquez B } 1123bd513cd0SMauricio Vasquez B 1124bd513cd0SMauricio Vasquez B value_size = map->value_size; 1125bd513cd0SMauricio Vasquez B 1126bd513cd0SMauricio Vasquez B err = -ENOMEM; 1127bd513cd0SMauricio Vasquez B value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1128bd513cd0SMauricio Vasquez B if (!value) 1129bd513cd0SMauricio Vasquez B goto free_key; 1130bd513cd0SMauricio Vasquez B 1131bd513cd0SMauricio Vasquez B if (map->map_type == BPF_MAP_TYPE_QUEUE || 1132bd513cd0SMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 1133bd513cd0SMauricio Vasquez B err = map->ops->map_pop_elem(map, value); 1134bd513cd0SMauricio Vasquez B } else { 1135bd513cd0SMauricio Vasquez B err = -ENOTSUPP; 1136bd513cd0SMauricio Vasquez B } 1137bd513cd0SMauricio Vasquez B 1138bd513cd0SMauricio Vasquez B if (err) 1139bd513cd0SMauricio Vasquez B goto free_value; 1140bd513cd0SMauricio Vasquez B 1141bd513cd0SMauricio Vasquez B if (copy_to_user(uvalue, value, value_size) != 0) 1142bd513cd0SMauricio Vasquez B goto free_value; 1143bd513cd0SMauricio Vasquez B 1144bd513cd0SMauricio Vasquez B err = 0; 1145bd513cd0SMauricio Vasquez B 1146bd513cd0SMauricio Vasquez B free_value: 1147bd513cd0SMauricio Vasquez B kfree(value); 1148bd513cd0SMauricio Vasquez B free_key: 1149bd513cd0SMauricio Vasquez B kfree(key); 1150bd513cd0SMauricio Vasquez B err_put: 1151bd513cd0SMauricio Vasquez B fdput(f); 1152bd513cd0SMauricio Vasquez B return err; 1153bd513cd0SMauricio Vasquez B } 1154bd513cd0SMauricio Vasquez B 115587df15deSDaniel Borkmann #define BPF_MAP_FREEZE_LAST_FIELD map_fd 115687df15deSDaniel Borkmann 115787df15deSDaniel Borkmann static int map_freeze(const union bpf_attr *attr) 115887df15deSDaniel Borkmann { 115987df15deSDaniel Borkmann int err = 0, ufd = attr->map_fd; 116087df15deSDaniel Borkmann struct bpf_map *map; 116187df15deSDaniel Borkmann struct fd f; 116287df15deSDaniel Borkmann 116387df15deSDaniel Borkmann if (CHECK_ATTR(BPF_MAP_FREEZE)) 116487df15deSDaniel Borkmann return -EINVAL; 116587df15deSDaniel Borkmann 116687df15deSDaniel Borkmann f = fdget(ufd); 116787df15deSDaniel Borkmann map = __bpf_map_get(f); 116887df15deSDaniel Borkmann if (IS_ERR(map)) 116987df15deSDaniel Borkmann return PTR_ERR(map); 117087df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) { 117187df15deSDaniel Borkmann err = -EBUSY; 117287df15deSDaniel Borkmann goto err_put; 117387df15deSDaniel Borkmann } 117487df15deSDaniel Borkmann if (!capable(CAP_SYS_ADMIN)) { 117587df15deSDaniel Borkmann err = -EPERM; 117687df15deSDaniel Borkmann goto err_put; 117787df15deSDaniel Borkmann } 117887df15deSDaniel Borkmann 117987df15deSDaniel Borkmann WRITE_ONCE(map->frozen, true); 118087df15deSDaniel Borkmann err_put: 118187df15deSDaniel Borkmann fdput(f); 118287df15deSDaniel Borkmann return err; 118387df15deSDaniel Borkmann } 118487df15deSDaniel Borkmann 11857de16e3aSJakub Kicinski static const struct bpf_prog_ops * const bpf_prog_types[] = { 11867de16e3aSJakub Kicinski #define BPF_PROG_TYPE(_id, _name) \ 11877de16e3aSJakub Kicinski [_id] = & _name ## _prog_ops, 11887de16e3aSJakub Kicinski #define BPF_MAP_TYPE(_id, _ops) 11897de16e3aSJakub Kicinski #include <linux/bpf_types.h> 11907de16e3aSJakub Kicinski #undef BPF_PROG_TYPE 11917de16e3aSJakub Kicinski #undef BPF_MAP_TYPE 11927de16e3aSJakub Kicinski }; 11937de16e3aSJakub Kicinski 119409756af4SAlexei Starovoitov static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 119509756af4SAlexei Starovoitov { 1196d0f1a451SDaniel Borkmann const struct bpf_prog_ops *ops; 1197d0f1a451SDaniel Borkmann 1198d0f1a451SDaniel Borkmann if (type >= ARRAY_SIZE(bpf_prog_types)) 1199d0f1a451SDaniel Borkmann return -EINVAL; 1200d0f1a451SDaniel Borkmann type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types)); 1201d0f1a451SDaniel Borkmann ops = bpf_prog_types[type]; 1202d0f1a451SDaniel Borkmann if (!ops) 1203be9370a7SJohannes Berg return -EINVAL; 120409756af4SAlexei Starovoitov 1205ab3f0063SJakub Kicinski if (!bpf_prog_is_dev_bound(prog->aux)) 1206d0f1a451SDaniel Borkmann prog->aux->ops = ops; 1207ab3f0063SJakub Kicinski else 1208ab3f0063SJakub Kicinski prog->aux->ops = &bpf_offload_prog_ops; 120924701eceSDaniel Borkmann prog->type = type; 121009756af4SAlexei Starovoitov return 0; 121109756af4SAlexei Starovoitov } 121209756af4SAlexei Starovoitov 121309756af4SAlexei Starovoitov /* drop refcnt on maps used by eBPF program and free auxilary data */ 121409756af4SAlexei Starovoitov static void free_used_maps(struct bpf_prog_aux *aux) 121509756af4SAlexei Starovoitov { 12168bad74f9SRoman Gushchin enum bpf_cgroup_storage_type stype; 121709756af4SAlexei Starovoitov int i; 121809756af4SAlexei Starovoitov 12198bad74f9SRoman Gushchin for_each_cgroup_storage_type(stype) { 12208bad74f9SRoman Gushchin if (!aux->cgroup_storage[stype]) 12218bad74f9SRoman Gushchin continue; 12228bad74f9SRoman Gushchin bpf_cgroup_storage_release(aux->prog, 12238bad74f9SRoman Gushchin aux->cgroup_storage[stype]); 12248bad74f9SRoman Gushchin } 1225de9cbbaaSRoman Gushchin 122609756af4SAlexei Starovoitov for (i = 0; i < aux->used_map_cnt; i++) 122709756af4SAlexei Starovoitov bpf_map_put(aux->used_maps[i]); 122809756af4SAlexei Starovoitov 122909756af4SAlexei Starovoitov kfree(aux->used_maps); 123009756af4SAlexei Starovoitov } 123109756af4SAlexei Starovoitov 12325ccb071eSDaniel Borkmann int __bpf_prog_charge(struct user_struct *user, u32 pages) 12335ccb071eSDaniel Borkmann { 12345ccb071eSDaniel Borkmann unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 12355ccb071eSDaniel Borkmann unsigned long user_bufs; 12365ccb071eSDaniel Borkmann 12375ccb071eSDaniel Borkmann if (user) { 12385ccb071eSDaniel Borkmann user_bufs = atomic_long_add_return(pages, &user->locked_vm); 12395ccb071eSDaniel Borkmann if (user_bufs > memlock_limit) { 12405ccb071eSDaniel Borkmann atomic_long_sub(pages, &user->locked_vm); 12415ccb071eSDaniel Borkmann return -EPERM; 12425ccb071eSDaniel Borkmann } 12435ccb071eSDaniel Borkmann } 12445ccb071eSDaniel Borkmann 12455ccb071eSDaniel Borkmann return 0; 12465ccb071eSDaniel Borkmann } 12475ccb071eSDaniel Borkmann 12485ccb071eSDaniel Borkmann void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 12495ccb071eSDaniel Borkmann { 12505ccb071eSDaniel Borkmann if (user) 12515ccb071eSDaniel Borkmann atomic_long_sub(pages, &user->locked_vm); 12525ccb071eSDaniel Borkmann } 12535ccb071eSDaniel Borkmann 1254aaac3ba9SAlexei Starovoitov static int bpf_prog_charge_memlock(struct bpf_prog *prog) 1255aaac3ba9SAlexei Starovoitov { 1256aaac3ba9SAlexei Starovoitov struct user_struct *user = get_current_user(); 12575ccb071eSDaniel Borkmann int ret; 1258aaac3ba9SAlexei Starovoitov 12595ccb071eSDaniel Borkmann ret = __bpf_prog_charge(user, prog->pages); 12605ccb071eSDaniel Borkmann if (ret) { 1261aaac3ba9SAlexei Starovoitov free_uid(user); 12625ccb071eSDaniel Borkmann return ret; 1263aaac3ba9SAlexei Starovoitov } 12645ccb071eSDaniel Borkmann 1265aaac3ba9SAlexei Starovoitov prog->aux->user = user; 1266aaac3ba9SAlexei Starovoitov return 0; 1267aaac3ba9SAlexei Starovoitov } 1268aaac3ba9SAlexei Starovoitov 1269aaac3ba9SAlexei Starovoitov static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 1270aaac3ba9SAlexei Starovoitov { 1271aaac3ba9SAlexei Starovoitov struct user_struct *user = prog->aux->user; 1272aaac3ba9SAlexei Starovoitov 12735ccb071eSDaniel Borkmann __bpf_prog_uncharge(user, prog->pages); 1274aaac3ba9SAlexei Starovoitov free_uid(user); 1275aaac3ba9SAlexei Starovoitov } 1276aaac3ba9SAlexei Starovoitov 1277dc4bb0e2SMartin KaFai Lau static int bpf_prog_alloc_id(struct bpf_prog *prog) 1278dc4bb0e2SMartin KaFai Lau { 1279dc4bb0e2SMartin KaFai Lau int id; 1280dc4bb0e2SMartin KaFai Lau 1281b76354cdSShaohua Li idr_preload(GFP_KERNEL); 1282dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1283dc4bb0e2SMartin KaFai Lau id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 1284dc4bb0e2SMartin KaFai Lau if (id > 0) 1285dc4bb0e2SMartin KaFai Lau prog->aux->id = id; 1286dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1287b76354cdSShaohua Li idr_preload_end(); 1288dc4bb0e2SMartin KaFai Lau 1289dc4bb0e2SMartin KaFai Lau /* id is in [1, INT_MAX) */ 1290dc4bb0e2SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 1291dc4bb0e2SMartin KaFai Lau return -ENOSPC; 1292dc4bb0e2SMartin KaFai Lau 1293dc4bb0e2SMartin KaFai Lau return id > 0 ? 0 : id; 1294dc4bb0e2SMartin KaFai Lau } 1295dc4bb0e2SMartin KaFai Lau 1296ad8ad79fSJakub Kicinski void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) 1297dc4bb0e2SMartin KaFai Lau { 1298ad8ad79fSJakub Kicinski /* cBPF to eBPF migrations are currently not in the idr store. 1299ad8ad79fSJakub Kicinski * Offloaded programs are removed from the store when their device 1300ad8ad79fSJakub Kicinski * disappears - even if someone grabs an fd to them they are unusable, 1301ad8ad79fSJakub Kicinski * simply waiting for refcnt to drop to be freed. 1302ad8ad79fSJakub Kicinski */ 1303dc4bb0e2SMartin KaFai Lau if (!prog->aux->id) 1304dc4bb0e2SMartin KaFai Lau return; 1305dc4bb0e2SMartin KaFai Lau 1306b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1307dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1308b16d9aa4SMartin KaFai Lau else 1309b16d9aa4SMartin KaFai Lau __acquire(&prog_idr_lock); 1310b16d9aa4SMartin KaFai Lau 1311dc4bb0e2SMartin KaFai Lau idr_remove(&prog_idr, prog->aux->id); 1312ad8ad79fSJakub Kicinski prog->aux->id = 0; 1313b16d9aa4SMartin KaFai Lau 1314b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1315dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1316b16d9aa4SMartin KaFai Lau else 1317b16d9aa4SMartin KaFai Lau __release(&prog_idr_lock); 1318dc4bb0e2SMartin KaFai Lau } 1319dc4bb0e2SMartin KaFai Lau 13201aacde3dSDaniel Borkmann static void __bpf_prog_put_rcu(struct rcu_head *rcu) 1321abf2e7d6SAlexei Starovoitov { 1322abf2e7d6SAlexei Starovoitov struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 1323abf2e7d6SAlexei Starovoitov 1324abf2e7d6SAlexei Starovoitov free_used_maps(aux); 1325aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(aux->prog); 1326afdb09c7SChenbo Feng security_bpf_prog_free(aux); 1327abf2e7d6SAlexei Starovoitov bpf_prog_free(aux->prog); 1328abf2e7d6SAlexei Starovoitov } 1329abf2e7d6SAlexei Starovoitov 1330b16d9aa4SMartin KaFai Lau static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 133109756af4SAlexei Starovoitov { 1332a67edbf4SDaniel Borkmann if (atomic_dec_and_test(&prog->aux->refcnt)) { 13336ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0); 133434ad5580SMartin KaFai Lau /* bpf_prog_free_id() must be called first */ 1335b16d9aa4SMartin KaFai Lau bpf_prog_free_id(prog, do_idr_lock); 13367d1982b4SDaniel Borkmann bpf_prog_kallsyms_del_all(prog); 1337838e9690SYonghong Song btf_put(prog->aux->btf); 1338ba64e7d8SYonghong Song kvfree(prog->aux->func_info); 1339c454a46bSMartin KaFai Lau bpf_prog_free_linfo(prog); 13404f74d809SDaniel Borkmann 13411aacde3dSDaniel Borkmann call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 134209756af4SAlexei Starovoitov } 1343a67edbf4SDaniel Borkmann } 1344b16d9aa4SMartin KaFai Lau 1345b16d9aa4SMartin KaFai Lau void bpf_prog_put(struct bpf_prog *prog) 1346b16d9aa4SMartin KaFai Lau { 1347b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, true); 1348b16d9aa4SMartin KaFai Lau } 1349e2e9b654SDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_put); 135009756af4SAlexei Starovoitov 135109756af4SAlexei Starovoitov static int bpf_prog_release(struct inode *inode, struct file *filp) 135209756af4SAlexei Starovoitov { 135309756af4SAlexei Starovoitov struct bpf_prog *prog = filp->private_data; 135409756af4SAlexei Starovoitov 13551aacde3dSDaniel Borkmann bpf_prog_put(prog); 135609756af4SAlexei Starovoitov return 0; 135709756af4SAlexei Starovoitov } 135809756af4SAlexei Starovoitov 1359492ecee8SAlexei Starovoitov static void bpf_prog_get_stats(const struct bpf_prog *prog, 1360492ecee8SAlexei Starovoitov struct bpf_prog_stats *stats) 1361492ecee8SAlexei Starovoitov { 1362492ecee8SAlexei Starovoitov u64 nsecs = 0, cnt = 0; 1363492ecee8SAlexei Starovoitov int cpu; 1364492ecee8SAlexei Starovoitov 1365492ecee8SAlexei Starovoitov for_each_possible_cpu(cpu) { 1366492ecee8SAlexei Starovoitov const struct bpf_prog_stats *st; 1367492ecee8SAlexei Starovoitov unsigned int start; 1368492ecee8SAlexei Starovoitov u64 tnsecs, tcnt; 1369492ecee8SAlexei Starovoitov 1370492ecee8SAlexei Starovoitov st = per_cpu_ptr(prog->aux->stats, cpu); 1371492ecee8SAlexei Starovoitov do { 1372492ecee8SAlexei Starovoitov start = u64_stats_fetch_begin_irq(&st->syncp); 1373492ecee8SAlexei Starovoitov tnsecs = st->nsecs; 1374492ecee8SAlexei Starovoitov tcnt = st->cnt; 1375492ecee8SAlexei Starovoitov } while (u64_stats_fetch_retry_irq(&st->syncp, start)); 1376492ecee8SAlexei Starovoitov nsecs += tnsecs; 1377492ecee8SAlexei Starovoitov cnt += tcnt; 1378492ecee8SAlexei Starovoitov } 1379492ecee8SAlexei Starovoitov stats->nsecs = nsecs; 1380492ecee8SAlexei Starovoitov stats->cnt = cnt; 1381492ecee8SAlexei Starovoitov } 1382492ecee8SAlexei Starovoitov 13837bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 13847bd509e3SDaniel Borkmann static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 13857bd509e3SDaniel Borkmann { 13867bd509e3SDaniel Borkmann const struct bpf_prog *prog = filp->private_data; 1387f1f7714eSDaniel Borkmann char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 1388492ecee8SAlexei Starovoitov struct bpf_prog_stats stats; 13897bd509e3SDaniel Borkmann 1390492ecee8SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 1391f1f7714eSDaniel Borkmann bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 13927bd509e3SDaniel Borkmann seq_printf(m, 13937bd509e3SDaniel Borkmann "prog_type:\t%u\n" 13947bd509e3SDaniel Borkmann "prog_jited:\t%u\n" 1395f1f7714eSDaniel Borkmann "prog_tag:\t%s\n" 13964316b409SDaniel Borkmann "memlock:\t%llu\n" 1397492ecee8SAlexei Starovoitov "prog_id:\t%u\n" 1398492ecee8SAlexei Starovoitov "run_time_ns:\t%llu\n" 1399492ecee8SAlexei Starovoitov "run_cnt:\t%llu\n", 14007bd509e3SDaniel Borkmann prog->type, 14017bd509e3SDaniel Borkmann prog->jited, 1402f1f7714eSDaniel Borkmann prog_tag, 14034316b409SDaniel Borkmann prog->pages * 1ULL << PAGE_SHIFT, 1404492ecee8SAlexei Starovoitov prog->aux->id, 1405492ecee8SAlexei Starovoitov stats.nsecs, 1406492ecee8SAlexei Starovoitov stats.cnt); 14077bd509e3SDaniel Borkmann } 14087bd509e3SDaniel Borkmann #endif 14097bd509e3SDaniel Borkmann 1410f66e448cSChenbo Feng const struct file_operations bpf_prog_fops = { 14117bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 14127bd509e3SDaniel Borkmann .show_fdinfo = bpf_prog_show_fdinfo, 14137bd509e3SDaniel Borkmann #endif 141409756af4SAlexei Starovoitov .release = bpf_prog_release, 14156e71b04aSChenbo Feng .read = bpf_dummy_read, 14166e71b04aSChenbo Feng .write = bpf_dummy_write, 141709756af4SAlexei Starovoitov }; 141809756af4SAlexei Starovoitov 1419b2197755SDaniel Borkmann int bpf_prog_new_fd(struct bpf_prog *prog) 1420aa79781bSDaniel Borkmann { 1421afdb09c7SChenbo Feng int ret; 1422afdb09c7SChenbo Feng 1423afdb09c7SChenbo Feng ret = security_bpf_prog(prog); 1424afdb09c7SChenbo Feng if (ret < 0) 1425afdb09c7SChenbo Feng return ret; 1426afdb09c7SChenbo Feng 1427aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 1428aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC); 1429aa79781bSDaniel Borkmann } 1430aa79781bSDaniel Borkmann 1431113214beSDaniel Borkmann static struct bpf_prog *____bpf_prog_get(struct fd f) 143209756af4SAlexei Starovoitov { 143309756af4SAlexei Starovoitov if (!f.file) 143409756af4SAlexei Starovoitov return ERR_PTR(-EBADF); 143509756af4SAlexei Starovoitov if (f.file->f_op != &bpf_prog_fops) { 143609756af4SAlexei Starovoitov fdput(f); 143709756af4SAlexei Starovoitov return ERR_PTR(-EINVAL); 143809756af4SAlexei Starovoitov } 143909756af4SAlexei Starovoitov 1440c2101297SDaniel Borkmann return f.file->private_data; 144109756af4SAlexei Starovoitov } 144209756af4SAlexei Starovoitov 144359d3656dSBrenden Blanco struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i) 144492117d84SAlexei Starovoitov { 144559d3656dSBrenden Blanco if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) { 144659d3656dSBrenden Blanco atomic_sub(i, &prog->aux->refcnt); 144792117d84SAlexei Starovoitov return ERR_PTR(-EBUSY); 144892117d84SAlexei Starovoitov } 144992117d84SAlexei Starovoitov return prog; 145092117d84SAlexei Starovoitov } 145159d3656dSBrenden Blanco EXPORT_SYMBOL_GPL(bpf_prog_add); 145259d3656dSBrenden Blanco 1453c540594fSDaniel Borkmann void bpf_prog_sub(struct bpf_prog *prog, int i) 1454c540594fSDaniel Borkmann { 1455c540594fSDaniel Borkmann /* Only to be used for undoing previous bpf_prog_add() in some 1456c540594fSDaniel Borkmann * error path. We still know that another entity in our call 1457c540594fSDaniel Borkmann * path holds a reference to the program, thus atomic_sub() can 1458c540594fSDaniel Borkmann * be safely used in such cases! 1459c540594fSDaniel Borkmann */ 1460c540594fSDaniel Borkmann WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0); 1461c540594fSDaniel Borkmann } 1462c540594fSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_sub); 1463c540594fSDaniel Borkmann 146459d3656dSBrenden Blanco struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog) 146559d3656dSBrenden Blanco { 146659d3656dSBrenden Blanco return bpf_prog_add(prog, 1); 146759d3656dSBrenden Blanco } 146897bc402dSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_inc); 146992117d84SAlexei Starovoitov 1470b16d9aa4SMartin KaFai Lau /* prog_idr_lock should have been held */ 1471a6f6df69SJohn Fastabend struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 1472b16d9aa4SMartin KaFai Lau { 1473b16d9aa4SMartin KaFai Lau int refold; 1474b16d9aa4SMartin KaFai Lau 1475bfc18e38SMark Rutland refold = atomic_fetch_add_unless(&prog->aux->refcnt, 1, 0); 1476b16d9aa4SMartin KaFai Lau 1477b16d9aa4SMartin KaFai Lau if (refold >= BPF_MAX_REFCNT) { 1478b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, false); 1479b16d9aa4SMartin KaFai Lau return ERR_PTR(-EBUSY); 1480b16d9aa4SMartin KaFai Lau } 1481b16d9aa4SMartin KaFai Lau 1482b16d9aa4SMartin KaFai Lau if (!refold) 1483b16d9aa4SMartin KaFai Lau return ERR_PTR(-ENOENT); 1484b16d9aa4SMartin KaFai Lau 1485b16d9aa4SMartin KaFai Lau return prog; 1486b16d9aa4SMartin KaFai Lau } 1487a6f6df69SJohn Fastabend EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 1488b16d9aa4SMartin KaFai Lau 1489040ee692SAl Viro bool bpf_prog_get_ok(struct bpf_prog *prog, 1490288b3de5SJakub Kicinski enum bpf_prog_type *attach_type, bool attach_drv) 1491248f346fSJakub Kicinski { 1492288b3de5SJakub Kicinski /* not an attachment, just a refcount inc, always allow */ 1493288b3de5SJakub Kicinski if (!attach_type) 1494288b3de5SJakub Kicinski return true; 1495248f346fSJakub Kicinski 1496248f346fSJakub Kicinski if (prog->type != *attach_type) 1497248f346fSJakub Kicinski return false; 1498288b3de5SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv) 1499248f346fSJakub Kicinski return false; 1500248f346fSJakub Kicinski 1501248f346fSJakub Kicinski return true; 1502248f346fSJakub Kicinski } 1503248f346fSJakub Kicinski 1504248f346fSJakub Kicinski static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, 1505288b3de5SJakub Kicinski bool attach_drv) 150609756af4SAlexei Starovoitov { 150709756af4SAlexei Starovoitov struct fd f = fdget(ufd); 150809756af4SAlexei Starovoitov struct bpf_prog *prog; 150909756af4SAlexei Starovoitov 1510113214beSDaniel Borkmann prog = ____bpf_prog_get(f); 151109756af4SAlexei Starovoitov if (IS_ERR(prog)) 151209756af4SAlexei Starovoitov return prog; 1513288b3de5SJakub Kicinski if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) { 1514113214beSDaniel Borkmann prog = ERR_PTR(-EINVAL); 1515113214beSDaniel Borkmann goto out; 1516113214beSDaniel Borkmann } 151709756af4SAlexei Starovoitov 151892117d84SAlexei Starovoitov prog = bpf_prog_inc(prog); 1519113214beSDaniel Borkmann out: 152009756af4SAlexei Starovoitov fdput(f); 152109756af4SAlexei Starovoitov return prog; 152209756af4SAlexei Starovoitov } 1523113214beSDaniel Borkmann 1524113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get(u32 ufd) 1525113214beSDaniel Borkmann { 1526288b3de5SJakub Kicinski return __bpf_prog_get(ufd, NULL, false); 1527113214beSDaniel Borkmann } 1528113214beSDaniel Borkmann 1529248f346fSJakub Kicinski struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 1530288b3de5SJakub Kicinski bool attach_drv) 1531248f346fSJakub Kicinski { 15324d220ed0SAlexei Starovoitov return __bpf_prog_get(ufd, &type, attach_drv); 1533248f346fSJakub Kicinski } 15346c8dfe21SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev); 1535248f346fSJakub Kicinski 1536aac3fc32SAndrey Ignatov /* Initially all BPF programs could be loaded w/o specifying 1537aac3fc32SAndrey Ignatov * expected_attach_type. Later for some of them specifying expected_attach_type 1538aac3fc32SAndrey Ignatov * at load time became required so that program could be validated properly. 1539aac3fc32SAndrey Ignatov * Programs of types that are allowed to be loaded both w/ and w/o (for 1540aac3fc32SAndrey Ignatov * backward compatibility) expected_attach_type, should have the default attach 1541aac3fc32SAndrey Ignatov * type assigned to expected_attach_type for the latter case, so that it can be 1542aac3fc32SAndrey Ignatov * validated later at attach time. 1543aac3fc32SAndrey Ignatov * 1544aac3fc32SAndrey Ignatov * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if 1545aac3fc32SAndrey Ignatov * prog type requires it but has some attach types that have to be backward 1546aac3fc32SAndrey Ignatov * compatible. 1547aac3fc32SAndrey Ignatov */ 1548aac3fc32SAndrey Ignatov static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) 1549aac3fc32SAndrey Ignatov { 1550aac3fc32SAndrey Ignatov switch (attr->prog_type) { 1551aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1552aac3fc32SAndrey Ignatov /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't 1553aac3fc32SAndrey Ignatov * exist so checking for non-zero is the way to go here. 1554aac3fc32SAndrey Ignatov */ 1555aac3fc32SAndrey Ignatov if (!attr->expected_attach_type) 1556aac3fc32SAndrey Ignatov attr->expected_attach_type = 1557aac3fc32SAndrey Ignatov BPF_CGROUP_INET_SOCK_CREATE; 1558aac3fc32SAndrey Ignatov break; 1559aac3fc32SAndrey Ignatov } 1560aac3fc32SAndrey Ignatov } 1561aac3fc32SAndrey Ignatov 15625e43f899SAndrey Ignatov static int 15635e43f899SAndrey Ignatov bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type, 15645e43f899SAndrey Ignatov enum bpf_attach_type expected_attach_type) 15655e43f899SAndrey Ignatov { 15664fbac77dSAndrey Ignatov switch (prog_type) { 1567aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1568aac3fc32SAndrey Ignatov switch (expected_attach_type) { 1569aac3fc32SAndrey Ignatov case BPF_CGROUP_INET_SOCK_CREATE: 1570aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1571aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 1572aac3fc32SAndrey Ignatov return 0; 1573aac3fc32SAndrey Ignatov default: 1574aac3fc32SAndrey Ignatov return -EINVAL; 1575aac3fc32SAndrey Ignatov } 15764fbac77dSAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 15774fbac77dSAndrey Ignatov switch (expected_attach_type) { 15784fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 15794fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1580d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1581d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 15821cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 15831cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 15845e43f899SAndrey Ignatov return 0; 15854fbac77dSAndrey Ignatov default: 15864fbac77dSAndrey Ignatov return -EINVAL; 15874fbac77dSAndrey Ignatov } 15884fbac77dSAndrey Ignatov default: 15894fbac77dSAndrey Ignatov return 0; 15904fbac77dSAndrey Ignatov } 15915e43f899SAndrey Ignatov } 15925e43f899SAndrey Ignatov 159309756af4SAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 1594c454a46bSMartin KaFai Lau #define BPF_PROG_LOAD_LAST_FIELD line_info_cnt 159509756af4SAlexei Starovoitov 1596838e9690SYonghong Song static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) 159709756af4SAlexei Starovoitov { 159809756af4SAlexei Starovoitov enum bpf_prog_type type = attr->prog_type; 159909756af4SAlexei Starovoitov struct bpf_prog *prog; 160009756af4SAlexei Starovoitov int err; 160109756af4SAlexei Starovoitov char license[128]; 160209756af4SAlexei Starovoitov bool is_gpl; 160309756af4SAlexei Starovoitov 160409756af4SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_LOAD)) 160509756af4SAlexei Starovoitov return -EINVAL; 160609756af4SAlexei Starovoitov 1607e9ee9efcSDavid Miller if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | BPF_F_ANY_ALIGNMENT)) 1608e07b98d9SDavid S. Miller return -EINVAL; 1609e07b98d9SDavid S. Miller 1610e9ee9efcSDavid Miller if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && 1611e9ee9efcSDavid Miller (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && 1612e9ee9efcSDavid Miller !capable(CAP_SYS_ADMIN)) 1613e9ee9efcSDavid Miller return -EPERM; 1614e9ee9efcSDavid Miller 161509756af4SAlexei Starovoitov /* copy eBPF program license from user space */ 1616535e7b4bSMickaël Salaün if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 161709756af4SAlexei Starovoitov sizeof(license) - 1) < 0) 161809756af4SAlexei Starovoitov return -EFAULT; 161909756af4SAlexei Starovoitov license[sizeof(license) - 1] = 0; 162009756af4SAlexei Starovoitov 162109756af4SAlexei Starovoitov /* eBPF programs must be GPL compatible to use GPL-ed functions */ 162209756af4SAlexei Starovoitov is_gpl = license_is_gpl_compatible(license); 162309756af4SAlexei Starovoitov 1624c04c0d2bSAlexei Starovoitov if (attr->insn_cnt == 0 || 1625c04c0d2bSAlexei Starovoitov attr->insn_cnt > (capable(CAP_SYS_ADMIN) ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) 1626ef0915caSDaniel Borkmann return -E2BIG; 162780b7d819SChenbo Feng if (type != BPF_PROG_TYPE_SOCKET_FILTER && 162880b7d819SChenbo Feng type != BPF_PROG_TYPE_CGROUP_SKB && 162980b7d819SChenbo Feng !capable(CAP_SYS_ADMIN)) 16301be7f75dSAlexei Starovoitov return -EPERM; 16311be7f75dSAlexei Starovoitov 1632aac3fc32SAndrey Ignatov bpf_prog_load_fixup_attach_type(attr); 16335e43f899SAndrey Ignatov if (bpf_prog_load_check_attach_type(type, attr->expected_attach_type)) 16345e43f899SAndrey Ignatov return -EINVAL; 16355e43f899SAndrey Ignatov 163609756af4SAlexei Starovoitov /* plain bpf_prog allocation */ 163709756af4SAlexei Starovoitov prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 163809756af4SAlexei Starovoitov if (!prog) 163909756af4SAlexei Starovoitov return -ENOMEM; 164009756af4SAlexei Starovoitov 16415e43f899SAndrey Ignatov prog->expected_attach_type = attr->expected_attach_type; 16425e43f899SAndrey Ignatov 16439a18eedbSJakub Kicinski prog->aux->offload_requested = !!attr->prog_ifindex; 16449a18eedbSJakub Kicinski 1645afdb09c7SChenbo Feng err = security_bpf_prog_alloc(prog->aux); 1646aaac3ba9SAlexei Starovoitov if (err) 1647aaac3ba9SAlexei Starovoitov goto free_prog_nouncharge; 1648aaac3ba9SAlexei Starovoitov 1649afdb09c7SChenbo Feng err = bpf_prog_charge_memlock(prog); 1650afdb09c7SChenbo Feng if (err) 1651afdb09c7SChenbo Feng goto free_prog_sec; 1652afdb09c7SChenbo Feng 165309756af4SAlexei Starovoitov prog->len = attr->insn_cnt; 165409756af4SAlexei Starovoitov 165509756af4SAlexei Starovoitov err = -EFAULT; 1656535e7b4bSMickaël Salaün if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 1657aafe6ae9SDaniel Borkmann bpf_prog_insn_size(prog)) != 0) 165809756af4SAlexei Starovoitov goto free_prog; 165909756af4SAlexei Starovoitov 166009756af4SAlexei Starovoitov prog->orig_prog = NULL; 1661a91263d5SDaniel Borkmann prog->jited = 0; 166209756af4SAlexei Starovoitov 166309756af4SAlexei Starovoitov atomic_set(&prog->aux->refcnt, 1); 1664a91263d5SDaniel Borkmann prog->gpl_compatible = is_gpl ? 1 : 0; 166509756af4SAlexei Starovoitov 16669a18eedbSJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 1667ab3f0063SJakub Kicinski err = bpf_prog_offload_init(prog, attr); 1668ab3f0063SJakub Kicinski if (err) 1669ab3f0063SJakub Kicinski goto free_prog; 1670ab3f0063SJakub Kicinski } 1671ab3f0063SJakub Kicinski 167209756af4SAlexei Starovoitov /* find program type: socket_filter vs tracing_filter */ 167309756af4SAlexei Starovoitov err = find_prog_type(type, prog); 167409756af4SAlexei Starovoitov if (err < 0) 167509756af4SAlexei Starovoitov goto free_prog; 167609756af4SAlexei Starovoitov 1677cb4d2b3fSMartin KaFai Lau prog->aux->load_time = ktime_get_boot_ns(); 1678cb4d2b3fSMartin KaFai Lau err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name); 1679cb4d2b3fSMartin KaFai Lau if (err) 1680cb4d2b3fSMartin KaFai Lau goto free_prog; 1681cb4d2b3fSMartin KaFai Lau 168209756af4SAlexei Starovoitov /* run eBPF verifier */ 1683838e9690SYonghong Song err = bpf_check(&prog, attr, uattr); 168409756af4SAlexei Starovoitov if (err < 0) 168509756af4SAlexei Starovoitov goto free_used_maps; 168609756af4SAlexei Starovoitov 1687d1c55ab5SDaniel Borkmann prog = bpf_prog_select_runtime(prog, &err); 168804fd61abSAlexei Starovoitov if (err < 0) 168904fd61abSAlexei Starovoitov goto free_used_maps; 169009756af4SAlexei Starovoitov 1691dc4bb0e2SMartin KaFai Lau err = bpf_prog_alloc_id(prog); 1692dc4bb0e2SMartin KaFai Lau if (err) 1693dc4bb0e2SMartin KaFai Lau goto free_used_maps; 1694dc4bb0e2SMartin KaFai Lau 1695aa79781bSDaniel Borkmann err = bpf_prog_new_fd(prog); 1696b16d9aa4SMartin KaFai Lau if (err < 0) { 1697b16d9aa4SMartin KaFai Lau /* failed to allocate fd. 1698b16d9aa4SMartin KaFai Lau * bpf_prog_put() is needed because the above 1699b16d9aa4SMartin KaFai Lau * bpf_prog_alloc_id() has published the prog 1700b16d9aa4SMartin KaFai Lau * to the userspace and the userspace may 1701b16d9aa4SMartin KaFai Lau * have refcnt-ed it through BPF_PROG_GET_FD_BY_ID. 1702b16d9aa4SMartin KaFai Lau */ 1703b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 1704b16d9aa4SMartin KaFai Lau return err; 1705b16d9aa4SMartin KaFai Lau } 170609756af4SAlexei Starovoitov 170774451e66SDaniel Borkmann bpf_prog_kallsyms_add(prog); 17086ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); 170909756af4SAlexei Starovoitov return err; 171009756af4SAlexei Starovoitov 171109756af4SAlexei Starovoitov free_used_maps: 1712c454a46bSMartin KaFai Lau bpf_prog_free_linfo(prog); 17135482e9a9SMartin KaFai Lau kvfree(prog->aux->func_info); 17145482e9a9SMartin KaFai Lau btf_put(prog->aux->btf); 17157d1982b4SDaniel Borkmann bpf_prog_kallsyms_del_subprogs(prog); 171609756af4SAlexei Starovoitov free_used_maps(prog->aux); 171709756af4SAlexei Starovoitov free_prog: 1718aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(prog); 1719afdb09c7SChenbo Feng free_prog_sec: 1720afdb09c7SChenbo Feng security_bpf_prog_free(prog->aux); 1721aaac3ba9SAlexei Starovoitov free_prog_nouncharge: 172209756af4SAlexei Starovoitov bpf_prog_free(prog); 172309756af4SAlexei Starovoitov return err; 172409756af4SAlexei Starovoitov } 172509756af4SAlexei Starovoitov 17266e71b04aSChenbo Feng #define BPF_OBJ_LAST_FIELD file_flags 1727b2197755SDaniel Borkmann 1728b2197755SDaniel Borkmann static int bpf_obj_pin(const union bpf_attr *attr) 1729b2197755SDaniel Borkmann { 17306e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0) 1731b2197755SDaniel Borkmann return -EINVAL; 1732b2197755SDaniel Borkmann 1733535e7b4bSMickaël Salaün return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 1734b2197755SDaniel Borkmann } 1735b2197755SDaniel Borkmann 1736b2197755SDaniel Borkmann static int bpf_obj_get(const union bpf_attr *attr) 1737b2197755SDaniel Borkmann { 17386e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || 17396e71b04aSChenbo Feng attr->file_flags & ~BPF_OBJ_FLAG_MASK) 1740b2197755SDaniel Borkmann return -EINVAL; 1741b2197755SDaniel Borkmann 17426e71b04aSChenbo Feng return bpf_obj_get_user(u64_to_user_ptr(attr->pathname), 17436e71b04aSChenbo Feng attr->file_flags); 1744b2197755SDaniel Borkmann } 1745b2197755SDaniel Borkmann 1746c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint { 1747c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 1748c4f6699dSAlexei Starovoitov struct bpf_prog *prog; 1749c4f6699dSAlexei Starovoitov }; 1750c4f6699dSAlexei Starovoitov 1751c4f6699dSAlexei Starovoitov static int bpf_raw_tracepoint_release(struct inode *inode, struct file *filp) 1752c4f6699dSAlexei Starovoitov { 1753c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint *raw_tp = filp->private_data; 1754c4f6699dSAlexei Starovoitov 1755c4f6699dSAlexei Starovoitov if (raw_tp->prog) { 1756c4f6699dSAlexei Starovoitov bpf_probe_unregister(raw_tp->btp, raw_tp->prog); 1757c4f6699dSAlexei Starovoitov bpf_prog_put(raw_tp->prog); 1758c4f6699dSAlexei Starovoitov } 1759a38d1107SMatt Mullins bpf_put_raw_tracepoint(raw_tp->btp); 1760c4f6699dSAlexei Starovoitov kfree(raw_tp); 1761c4f6699dSAlexei Starovoitov return 0; 1762c4f6699dSAlexei Starovoitov } 1763c4f6699dSAlexei Starovoitov 1764c4f6699dSAlexei Starovoitov static const struct file_operations bpf_raw_tp_fops = { 1765c4f6699dSAlexei Starovoitov .release = bpf_raw_tracepoint_release, 1766c4f6699dSAlexei Starovoitov .read = bpf_dummy_read, 1767c4f6699dSAlexei Starovoitov .write = bpf_dummy_write, 1768c4f6699dSAlexei Starovoitov }; 1769c4f6699dSAlexei Starovoitov 1770c4f6699dSAlexei Starovoitov #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd 1771c4f6699dSAlexei Starovoitov 1772c4f6699dSAlexei Starovoitov static int bpf_raw_tracepoint_open(const union bpf_attr *attr) 1773c4f6699dSAlexei Starovoitov { 1774c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint *raw_tp; 1775c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 1776c4f6699dSAlexei Starovoitov struct bpf_prog *prog; 1777c4f6699dSAlexei Starovoitov char tp_name[128]; 1778c4f6699dSAlexei Starovoitov int tp_fd, err; 1779c4f6699dSAlexei Starovoitov 1780c4f6699dSAlexei Starovoitov if (strncpy_from_user(tp_name, u64_to_user_ptr(attr->raw_tracepoint.name), 1781c4f6699dSAlexei Starovoitov sizeof(tp_name) - 1) < 0) 1782c4f6699dSAlexei Starovoitov return -EFAULT; 1783c4f6699dSAlexei Starovoitov tp_name[sizeof(tp_name) - 1] = 0; 1784c4f6699dSAlexei Starovoitov 1785a38d1107SMatt Mullins btp = bpf_get_raw_tracepoint(tp_name); 1786c4f6699dSAlexei Starovoitov if (!btp) 1787c4f6699dSAlexei Starovoitov return -ENOENT; 1788c4f6699dSAlexei Starovoitov 1789c4f6699dSAlexei Starovoitov raw_tp = kzalloc(sizeof(*raw_tp), GFP_USER); 1790a38d1107SMatt Mullins if (!raw_tp) { 1791a38d1107SMatt Mullins err = -ENOMEM; 1792a38d1107SMatt Mullins goto out_put_btp; 1793a38d1107SMatt Mullins } 1794c4f6699dSAlexei Starovoitov raw_tp->btp = btp; 1795c4f6699dSAlexei Starovoitov 17969df1c28bSMatt Mullins prog = bpf_prog_get(attr->raw_tracepoint.prog_fd); 1797c4f6699dSAlexei Starovoitov if (IS_ERR(prog)) { 1798c4f6699dSAlexei Starovoitov err = PTR_ERR(prog); 1799c4f6699dSAlexei Starovoitov goto out_free_tp; 1800c4f6699dSAlexei Starovoitov } 18019df1c28bSMatt Mullins if (prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT && 18029df1c28bSMatt Mullins prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) { 18039df1c28bSMatt Mullins err = -EINVAL; 18049df1c28bSMatt Mullins goto out_put_prog; 18059df1c28bSMatt Mullins } 1806c4f6699dSAlexei Starovoitov 1807c4f6699dSAlexei Starovoitov err = bpf_probe_register(raw_tp->btp, prog); 1808c4f6699dSAlexei Starovoitov if (err) 1809c4f6699dSAlexei Starovoitov goto out_put_prog; 1810c4f6699dSAlexei Starovoitov 1811c4f6699dSAlexei Starovoitov raw_tp->prog = prog; 1812c4f6699dSAlexei Starovoitov tp_fd = anon_inode_getfd("bpf-raw-tracepoint", &bpf_raw_tp_fops, raw_tp, 1813c4f6699dSAlexei Starovoitov O_CLOEXEC); 1814c4f6699dSAlexei Starovoitov if (tp_fd < 0) { 1815c4f6699dSAlexei Starovoitov bpf_probe_unregister(raw_tp->btp, prog); 1816c4f6699dSAlexei Starovoitov err = tp_fd; 1817c4f6699dSAlexei Starovoitov goto out_put_prog; 1818c4f6699dSAlexei Starovoitov } 1819c4f6699dSAlexei Starovoitov return tp_fd; 1820c4f6699dSAlexei Starovoitov 1821c4f6699dSAlexei Starovoitov out_put_prog: 1822c4f6699dSAlexei Starovoitov bpf_prog_put(prog); 1823c4f6699dSAlexei Starovoitov out_free_tp: 1824c4f6699dSAlexei Starovoitov kfree(raw_tp); 1825a38d1107SMatt Mullins out_put_btp: 1826a38d1107SMatt Mullins bpf_put_raw_tracepoint(btp); 1827c4f6699dSAlexei Starovoitov return err; 1828c4f6699dSAlexei Starovoitov } 1829c4f6699dSAlexei Starovoitov 183033491588SAnders Roxell static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, 183133491588SAnders Roxell enum bpf_attach_type attach_type) 183233491588SAnders Roxell { 183333491588SAnders Roxell switch (prog->type) { 183433491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK: 183533491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 183633491588SAnders Roxell return attach_type == prog->expected_attach_type ? 0 : -EINVAL; 183733491588SAnders Roxell default: 183833491588SAnders Roxell return 0; 183933491588SAnders Roxell } 184033491588SAnders Roxell } 184133491588SAnders Roxell 1842464bc0fdSJohn Fastabend #define BPF_PROG_ATTACH_LAST_FIELD attach_flags 1843174a79ffSJohn Fastabend 1844324bda9eSAlexei Starovoitov #define BPF_F_ATTACH_MASK \ 1845324bda9eSAlexei Starovoitov (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI) 1846324bda9eSAlexei Starovoitov 1847f4324551SDaniel Mack static int bpf_prog_attach(const union bpf_attr *attr) 1848f4324551SDaniel Mack { 18497f677633SAlexei Starovoitov enum bpf_prog_type ptype; 1850f4324551SDaniel Mack struct bpf_prog *prog; 18517f677633SAlexei Starovoitov int ret; 1852f4324551SDaniel Mack 1853f4324551SDaniel Mack if (!capable(CAP_NET_ADMIN)) 1854f4324551SDaniel Mack return -EPERM; 1855f4324551SDaniel Mack 1856f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_ATTACH)) 1857f4324551SDaniel Mack return -EINVAL; 1858f4324551SDaniel Mack 1859324bda9eSAlexei Starovoitov if (attr->attach_flags & ~BPF_F_ATTACH_MASK) 18607f677633SAlexei Starovoitov return -EINVAL; 18617f677633SAlexei Starovoitov 1862f4324551SDaniel Mack switch (attr->attach_type) { 1863f4324551SDaniel Mack case BPF_CGROUP_INET_INGRESS: 1864f4324551SDaniel Mack case BPF_CGROUP_INET_EGRESS: 1865b2cd1257SDavid Ahern ptype = BPF_PROG_TYPE_CGROUP_SKB; 1866b2cd1257SDavid Ahern break; 186761023658SDavid Ahern case BPF_CGROUP_INET_SOCK_CREATE: 1868aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1869aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 187061023658SDavid Ahern ptype = BPF_PROG_TYPE_CGROUP_SOCK; 187161023658SDavid Ahern break; 18724fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 18734fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1874d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1875d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 18761cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 18771cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 18784fbac77dSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 18794fbac77dSAndrey Ignatov break; 188040304b2aSLawrence Brakmo case BPF_CGROUP_SOCK_OPS: 188140304b2aSLawrence Brakmo ptype = BPF_PROG_TYPE_SOCK_OPS; 188240304b2aSLawrence Brakmo break; 1883ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 1884ebc614f6SRoman Gushchin ptype = BPF_PROG_TYPE_CGROUP_DEVICE; 1885ebc614f6SRoman Gushchin break; 18864f738adbSJohn Fastabend case BPF_SK_MSG_VERDICT: 1887fdb5c453SSean Young ptype = BPF_PROG_TYPE_SK_MSG; 1888fdb5c453SSean Young break; 1889464bc0fdSJohn Fastabend case BPF_SK_SKB_STREAM_PARSER: 1890464bc0fdSJohn Fastabend case BPF_SK_SKB_STREAM_VERDICT: 1891fdb5c453SSean Young ptype = BPF_PROG_TYPE_SK_SKB; 1892fdb5c453SSean Young break; 1893f4364dcfSSean Young case BPF_LIRC_MODE2: 1894fdb5c453SSean Young ptype = BPF_PROG_TYPE_LIRC_MODE2; 1895fdb5c453SSean Young break; 1896d58e468bSPetar Penkov case BPF_FLOW_DISSECTOR: 1897d58e468bSPetar Penkov ptype = BPF_PROG_TYPE_FLOW_DISSECTOR; 1898d58e468bSPetar Penkov break; 18997b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 19007b146cebSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SYSCTL; 19017b146cebSAndrey Ignatov break; 1902b2cd1257SDavid Ahern default: 1903b2cd1257SDavid Ahern return -EINVAL; 1904b2cd1257SDavid Ahern } 1905b2cd1257SDavid Ahern 1906b2cd1257SDavid Ahern prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 1907f4324551SDaniel Mack if (IS_ERR(prog)) 1908f4324551SDaniel Mack return PTR_ERR(prog); 1909f4324551SDaniel Mack 19105e43f899SAndrey Ignatov if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) { 19115e43f899SAndrey Ignatov bpf_prog_put(prog); 19125e43f899SAndrey Ignatov return -EINVAL; 19135e43f899SAndrey Ignatov } 19145e43f899SAndrey Ignatov 1915fdb5c453SSean Young switch (ptype) { 1916fdb5c453SSean Young case BPF_PROG_TYPE_SK_SKB: 1917fdb5c453SSean Young case BPF_PROG_TYPE_SK_MSG: 1918604326b4SDaniel Borkmann ret = sock_map_get_from_fd(attr, prog); 1919fdb5c453SSean Young break; 1920fdb5c453SSean Young case BPF_PROG_TYPE_LIRC_MODE2: 1921fdb5c453SSean Young ret = lirc_prog_attach(attr, prog); 1922fdb5c453SSean Young break; 1923d58e468bSPetar Penkov case BPF_PROG_TYPE_FLOW_DISSECTOR: 1924d58e468bSPetar Penkov ret = skb_flow_dissector_bpf_prog_attach(attr, prog); 1925d58e468bSPetar Penkov break; 1926fdb5c453SSean Young default: 1927fdb5c453SSean Young ret = cgroup_bpf_prog_attach(attr, ptype, prog); 1928f4324551SDaniel Mack } 1929f4324551SDaniel Mack 19307f677633SAlexei Starovoitov if (ret) 19317f677633SAlexei Starovoitov bpf_prog_put(prog); 19327f677633SAlexei Starovoitov return ret; 1933f4324551SDaniel Mack } 1934f4324551SDaniel Mack 1935f4324551SDaniel Mack #define BPF_PROG_DETACH_LAST_FIELD attach_type 1936f4324551SDaniel Mack 1937f4324551SDaniel Mack static int bpf_prog_detach(const union bpf_attr *attr) 1938f4324551SDaniel Mack { 1939324bda9eSAlexei Starovoitov enum bpf_prog_type ptype; 1940f4324551SDaniel Mack 1941f4324551SDaniel Mack if (!capable(CAP_NET_ADMIN)) 1942f4324551SDaniel Mack return -EPERM; 1943f4324551SDaniel Mack 1944f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_DETACH)) 1945f4324551SDaniel Mack return -EINVAL; 1946f4324551SDaniel Mack 1947f4324551SDaniel Mack switch (attr->attach_type) { 1948f4324551SDaniel Mack case BPF_CGROUP_INET_INGRESS: 1949f4324551SDaniel Mack case BPF_CGROUP_INET_EGRESS: 1950324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_CGROUP_SKB; 1951324bda9eSAlexei Starovoitov break; 195261023658SDavid Ahern case BPF_CGROUP_INET_SOCK_CREATE: 1953aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1954aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 1955324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_CGROUP_SOCK; 1956324bda9eSAlexei Starovoitov break; 19574fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 19584fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1959d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1960d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 19611cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 19621cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 19634fbac77dSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 19644fbac77dSAndrey Ignatov break; 196540304b2aSLawrence Brakmo case BPF_CGROUP_SOCK_OPS: 1966324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_SOCK_OPS; 1967f4324551SDaniel Mack break; 1968ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 1969ebc614f6SRoman Gushchin ptype = BPF_PROG_TYPE_CGROUP_DEVICE; 1970ebc614f6SRoman Gushchin break; 19714f738adbSJohn Fastabend case BPF_SK_MSG_VERDICT: 1972604326b4SDaniel Borkmann return sock_map_get_from_fd(attr, NULL); 19735a67da2aSJohn Fastabend case BPF_SK_SKB_STREAM_PARSER: 19745a67da2aSJohn Fastabend case BPF_SK_SKB_STREAM_VERDICT: 1975604326b4SDaniel Borkmann return sock_map_get_from_fd(attr, NULL); 1976f4364dcfSSean Young case BPF_LIRC_MODE2: 1977f4364dcfSSean Young return lirc_prog_detach(attr); 1978d58e468bSPetar Penkov case BPF_FLOW_DISSECTOR: 1979d58e468bSPetar Penkov return skb_flow_dissector_bpf_prog_detach(attr); 19807b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 19817b146cebSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SYSCTL; 19827b146cebSAndrey Ignatov break; 1983f4324551SDaniel Mack default: 1984f4324551SDaniel Mack return -EINVAL; 1985f4324551SDaniel Mack } 1986f4324551SDaniel Mack 1987fdb5c453SSean Young return cgroup_bpf_prog_detach(attr, ptype); 1988f4324551SDaniel Mack } 198940304b2aSLawrence Brakmo 1990468e2f64SAlexei Starovoitov #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt 1991468e2f64SAlexei Starovoitov 1992468e2f64SAlexei Starovoitov static int bpf_prog_query(const union bpf_attr *attr, 1993468e2f64SAlexei Starovoitov union bpf_attr __user *uattr) 1994468e2f64SAlexei Starovoitov { 1995468e2f64SAlexei Starovoitov if (!capable(CAP_NET_ADMIN)) 1996468e2f64SAlexei Starovoitov return -EPERM; 1997468e2f64SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_QUERY)) 1998468e2f64SAlexei Starovoitov return -EINVAL; 1999468e2f64SAlexei Starovoitov if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE) 2000468e2f64SAlexei Starovoitov return -EINVAL; 2001468e2f64SAlexei Starovoitov 2002468e2f64SAlexei Starovoitov switch (attr->query.attach_type) { 2003468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_INGRESS: 2004468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_EGRESS: 2005468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_SOCK_CREATE: 20064fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 20074fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 2008aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 2009aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 2010d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 2011d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 20121cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 20131cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 2014468e2f64SAlexei Starovoitov case BPF_CGROUP_SOCK_OPS: 2015ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 20167b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 2017468e2f64SAlexei Starovoitov break; 2018f4364dcfSSean Young case BPF_LIRC_MODE2: 2019f4364dcfSSean Young return lirc_prog_query(attr, uattr); 2020118c8e9aSStanislav Fomichev case BPF_FLOW_DISSECTOR: 2021118c8e9aSStanislav Fomichev return skb_flow_dissector_prog_query(attr, uattr); 2022468e2f64SAlexei Starovoitov default: 2023468e2f64SAlexei Starovoitov return -EINVAL; 2024468e2f64SAlexei Starovoitov } 2025fdb5c453SSean Young 2026fdb5c453SSean Young return cgroup_bpf_prog_query(attr, uattr); 2027468e2f64SAlexei Starovoitov } 2028f4324551SDaniel Mack 2029b0b9395dSStanislav Fomichev #define BPF_PROG_TEST_RUN_LAST_FIELD test.ctx_out 20301cf1cae9SAlexei Starovoitov 20311cf1cae9SAlexei Starovoitov static int bpf_prog_test_run(const union bpf_attr *attr, 20321cf1cae9SAlexei Starovoitov union bpf_attr __user *uattr) 20331cf1cae9SAlexei Starovoitov { 20341cf1cae9SAlexei Starovoitov struct bpf_prog *prog; 20351cf1cae9SAlexei Starovoitov int ret = -ENOTSUPP; 20361cf1cae9SAlexei Starovoitov 203761f3c964SAlexei Starovoitov if (!capable(CAP_SYS_ADMIN)) 203861f3c964SAlexei Starovoitov return -EPERM; 20391cf1cae9SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 20401cf1cae9SAlexei Starovoitov return -EINVAL; 20411cf1cae9SAlexei Starovoitov 2042b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_in && !attr->test.ctx_in) || 2043b0b9395dSStanislav Fomichev (!attr->test.ctx_size_in && attr->test.ctx_in)) 2044b0b9395dSStanislav Fomichev return -EINVAL; 2045b0b9395dSStanislav Fomichev 2046b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_out && !attr->test.ctx_out) || 2047b0b9395dSStanislav Fomichev (!attr->test.ctx_size_out && attr->test.ctx_out)) 2048b0b9395dSStanislav Fomichev return -EINVAL; 2049b0b9395dSStanislav Fomichev 20501cf1cae9SAlexei Starovoitov prog = bpf_prog_get(attr->test.prog_fd); 20511cf1cae9SAlexei Starovoitov if (IS_ERR(prog)) 20521cf1cae9SAlexei Starovoitov return PTR_ERR(prog); 20531cf1cae9SAlexei Starovoitov 20541cf1cae9SAlexei Starovoitov if (prog->aux->ops->test_run) 20551cf1cae9SAlexei Starovoitov ret = prog->aux->ops->test_run(prog, attr, uattr); 20561cf1cae9SAlexei Starovoitov 20571cf1cae9SAlexei Starovoitov bpf_prog_put(prog); 20581cf1cae9SAlexei Starovoitov return ret; 20591cf1cae9SAlexei Starovoitov } 20601cf1cae9SAlexei Starovoitov 206134ad5580SMartin KaFai Lau #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 206234ad5580SMartin KaFai Lau 206334ad5580SMartin KaFai Lau static int bpf_obj_get_next_id(const union bpf_attr *attr, 206434ad5580SMartin KaFai Lau union bpf_attr __user *uattr, 206534ad5580SMartin KaFai Lau struct idr *idr, 206634ad5580SMartin KaFai Lau spinlock_t *lock) 206734ad5580SMartin KaFai Lau { 206834ad5580SMartin KaFai Lau u32 next_id = attr->start_id; 206934ad5580SMartin KaFai Lau int err = 0; 207034ad5580SMartin KaFai Lau 207134ad5580SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 207234ad5580SMartin KaFai Lau return -EINVAL; 207334ad5580SMartin KaFai Lau 207434ad5580SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 207534ad5580SMartin KaFai Lau return -EPERM; 207634ad5580SMartin KaFai Lau 207734ad5580SMartin KaFai Lau next_id++; 207834ad5580SMartin KaFai Lau spin_lock_bh(lock); 207934ad5580SMartin KaFai Lau if (!idr_get_next(idr, &next_id)) 208034ad5580SMartin KaFai Lau err = -ENOENT; 208134ad5580SMartin KaFai Lau spin_unlock_bh(lock); 208234ad5580SMartin KaFai Lau 208334ad5580SMartin KaFai Lau if (!err) 208434ad5580SMartin KaFai Lau err = put_user(next_id, &uattr->next_id); 208534ad5580SMartin KaFai Lau 208634ad5580SMartin KaFai Lau return err; 208734ad5580SMartin KaFai Lau } 208834ad5580SMartin KaFai Lau 2089b16d9aa4SMartin KaFai Lau #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 2090b16d9aa4SMartin KaFai Lau 2091b16d9aa4SMartin KaFai Lau static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 2092b16d9aa4SMartin KaFai Lau { 2093b16d9aa4SMartin KaFai Lau struct bpf_prog *prog; 2094b16d9aa4SMartin KaFai Lau u32 id = attr->prog_id; 2095b16d9aa4SMartin KaFai Lau int fd; 2096b16d9aa4SMartin KaFai Lau 2097b16d9aa4SMartin KaFai Lau if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 2098b16d9aa4SMartin KaFai Lau return -EINVAL; 2099b16d9aa4SMartin KaFai Lau 2100b16d9aa4SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 2101b16d9aa4SMartin KaFai Lau return -EPERM; 2102b16d9aa4SMartin KaFai Lau 2103b16d9aa4SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 2104b16d9aa4SMartin KaFai Lau prog = idr_find(&prog_idr, id); 2105b16d9aa4SMartin KaFai Lau if (prog) 2106b16d9aa4SMartin KaFai Lau prog = bpf_prog_inc_not_zero(prog); 2107b16d9aa4SMartin KaFai Lau else 2108b16d9aa4SMartin KaFai Lau prog = ERR_PTR(-ENOENT); 2109b16d9aa4SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 2110b16d9aa4SMartin KaFai Lau 2111b16d9aa4SMartin KaFai Lau if (IS_ERR(prog)) 2112b16d9aa4SMartin KaFai Lau return PTR_ERR(prog); 2113b16d9aa4SMartin KaFai Lau 2114b16d9aa4SMartin KaFai Lau fd = bpf_prog_new_fd(prog); 2115b16d9aa4SMartin KaFai Lau if (fd < 0) 2116b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 2117b16d9aa4SMartin KaFai Lau 2118b16d9aa4SMartin KaFai Lau return fd; 2119b16d9aa4SMartin KaFai Lau } 2120b16d9aa4SMartin KaFai Lau 21216e71b04aSChenbo Feng #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags 2122bd5f5f4eSMartin KaFai Lau 2123bd5f5f4eSMartin KaFai Lau static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 2124bd5f5f4eSMartin KaFai Lau { 2125bd5f5f4eSMartin KaFai Lau struct bpf_map *map; 2126bd5f5f4eSMartin KaFai Lau u32 id = attr->map_id; 21276e71b04aSChenbo Feng int f_flags; 2128bd5f5f4eSMartin KaFai Lau int fd; 2129bd5f5f4eSMartin KaFai Lau 21306e71b04aSChenbo Feng if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || 21316e71b04aSChenbo Feng attr->open_flags & ~BPF_OBJ_FLAG_MASK) 2132bd5f5f4eSMartin KaFai Lau return -EINVAL; 2133bd5f5f4eSMartin KaFai Lau 2134bd5f5f4eSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 2135bd5f5f4eSMartin KaFai Lau return -EPERM; 2136bd5f5f4eSMartin KaFai Lau 21376e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->open_flags); 21386e71b04aSChenbo Feng if (f_flags < 0) 21396e71b04aSChenbo Feng return f_flags; 21406e71b04aSChenbo Feng 2141bd5f5f4eSMartin KaFai Lau spin_lock_bh(&map_idr_lock); 2142bd5f5f4eSMartin KaFai Lau map = idr_find(&map_idr, id); 2143bd5f5f4eSMartin KaFai Lau if (map) 2144bd5f5f4eSMartin KaFai Lau map = bpf_map_inc_not_zero(map, true); 2145bd5f5f4eSMartin KaFai Lau else 2146bd5f5f4eSMartin KaFai Lau map = ERR_PTR(-ENOENT); 2147bd5f5f4eSMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 2148bd5f5f4eSMartin KaFai Lau 2149bd5f5f4eSMartin KaFai Lau if (IS_ERR(map)) 2150bd5f5f4eSMartin KaFai Lau return PTR_ERR(map); 2151bd5f5f4eSMartin KaFai Lau 21526e71b04aSChenbo Feng fd = bpf_map_new_fd(map, f_flags); 2153bd5f5f4eSMartin KaFai Lau if (fd < 0) 2154781e6282SPeng Sun bpf_map_put_with_uref(map); 2155bd5f5f4eSMartin KaFai Lau 2156bd5f5f4eSMartin KaFai Lau return fd; 2157bd5f5f4eSMartin KaFai Lau } 2158bd5f5f4eSMartin KaFai Lau 21597105e828SDaniel Borkmann static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, 2160d8eca5bbSDaniel Borkmann unsigned long addr, u32 *off, 2161d8eca5bbSDaniel Borkmann u32 *type) 21627105e828SDaniel Borkmann { 2163d8eca5bbSDaniel Borkmann const struct bpf_map *map; 21647105e828SDaniel Borkmann int i; 21657105e828SDaniel Borkmann 2166d8eca5bbSDaniel Borkmann for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { 2167d8eca5bbSDaniel Borkmann map = prog->aux->used_maps[i]; 2168d8eca5bbSDaniel Borkmann if (map == (void *)addr) { 2169d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_FD; 2170d8eca5bbSDaniel Borkmann return map; 2171d8eca5bbSDaniel Borkmann } 2172d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta) 2173d8eca5bbSDaniel Borkmann continue; 2174d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta(map, addr, off)) { 2175d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_VALUE; 2176d8eca5bbSDaniel Borkmann return map; 2177d8eca5bbSDaniel Borkmann } 2178d8eca5bbSDaniel Borkmann } 2179d8eca5bbSDaniel Borkmann 21807105e828SDaniel Borkmann return NULL; 21817105e828SDaniel Borkmann } 21827105e828SDaniel Borkmann 21837105e828SDaniel Borkmann static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog) 21847105e828SDaniel Borkmann { 21857105e828SDaniel Borkmann const struct bpf_map *map; 21867105e828SDaniel Borkmann struct bpf_insn *insns; 2187d8eca5bbSDaniel Borkmann u32 off, type; 21887105e828SDaniel Borkmann u64 imm; 21897105e828SDaniel Borkmann int i; 21907105e828SDaniel Borkmann 21917105e828SDaniel Borkmann insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), 21927105e828SDaniel Borkmann GFP_USER); 21937105e828SDaniel Borkmann if (!insns) 21947105e828SDaniel Borkmann return insns; 21957105e828SDaniel Borkmann 21967105e828SDaniel Borkmann for (i = 0; i < prog->len; i++) { 21977105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_TAIL_CALL)) { 21987105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 21997105e828SDaniel Borkmann insns[i].imm = BPF_FUNC_tail_call; 22007105e828SDaniel Borkmann /* fall-through */ 22017105e828SDaniel Borkmann } 22027105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_CALL) || 22037105e828SDaniel Borkmann insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) { 22047105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) 22057105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 22067105e828SDaniel Borkmann if (!bpf_dump_raw_ok()) 22077105e828SDaniel Borkmann insns[i].imm = 0; 22087105e828SDaniel Borkmann continue; 22097105e828SDaniel Borkmann } 22107105e828SDaniel Borkmann 22117105e828SDaniel Borkmann if (insns[i].code != (BPF_LD | BPF_IMM | BPF_DW)) 22127105e828SDaniel Borkmann continue; 22137105e828SDaniel Borkmann 22147105e828SDaniel Borkmann imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; 2215d8eca5bbSDaniel Borkmann map = bpf_map_from_imm(prog, imm, &off, &type); 22167105e828SDaniel Borkmann if (map) { 2217d8eca5bbSDaniel Borkmann insns[i].src_reg = type; 22187105e828SDaniel Borkmann insns[i].imm = map->id; 2219d8eca5bbSDaniel Borkmann insns[i + 1].imm = off; 22207105e828SDaniel Borkmann continue; 22217105e828SDaniel Borkmann } 22227105e828SDaniel Borkmann } 22237105e828SDaniel Borkmann 22247105e828SDaniel Borkmann return insns; 22257105e828SDaniel Borkmann } 22267105e828SDaniel Borkmann 2227c454a46bSMartin KaFai Lau static int set_info_rec_size(struct bpf_prog_info *info) 2228c454a46bSMartin KaFai Lau { 2229c454a46bSMartin KaFai Lau /* 2230c454a46bSMartin KaFai Lau * Ensure info.*_rec_size is the same as kernel expected size 2231c454a46bSMartin KaFai Lau * 2232c454a46bSMartin KaFai Lau * or 2233c454a46bSMartin KaFai Lau * 2234c454a46bSMartin KaFai Lau * Only allow zero *_rec_size if both _rec_size and _cnt are 2235c454a46bSMartin KaFai Lau * zero. In this case, the kernel will set the expected 2236c454a46bSMartin KaFai Lau * _rec_size back to the info. 2237c454a46bSMartin KaFai Lau */ 2238c454a46bSMartin KaFai Lau 223911d8b82dSYonghong Song if ((info->nr_func_info || info->func_info_rec_size) && 2240c454a46bSMartin KaFai Lau info->func_info_rec_size != sizeof(struct bpf_func_info)) 2241c454a46bSMartin KaFai Lau return -EINVAL; 2242c454a46bSMartin KaFai Lau 224311d8b82dSYonghong Song if ((info->nr_line_info || info->line_info_rec_size) && 2244c454a46bSMartin KaFai Lau info->line_info_rec_size != sizeof(struct bpf_line_info)) 2245c454a46bSMartin KaFai Lau return -EINVAL; 2246c454a46bSMartin KaFai Lau 224711d8b82dSYonghong Song if ((info->nr_jited_line_info || info->jited_line_info_rec_size) && 2248c454a46bSMartin KaFai Lau info->jited_line_info_rec_size != sizeof(__u64)) 2249c454a46bSMartin KaFai Lau return -EINVAL; 2250c454a46bSMartin KaFai Lau 2251c454a46bSMartin KaFai Lau info->func_info_rec_size = sizeof(struct bpf_func_info); 2252c454a46bSMartin KaFai Lau info->line_info_rec_size = sizeof(struct bpf_line_info); 2253c454a46bSMartin KaFai Lau info->jited_line_info_rec_size = sizeof(__u64); 2254c454a46bSMartin KaFai Lau 2255c454a46bSMartin KaFai Lau return 0; 2256c454a46bSMartin KaFai Lau } 2257c454a46bSMartin KaFai Lau 22581e270976SMartin KaFai Lau static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, 22591e270976SMartin KaFai Lau const union bpf_attr *attr, 22601e270976SMartin KaFai Lau union bpf_attr __user *uattr) 22611e270976SMartin KaFai Lau { 22621e270976SMartin KaFai Lau struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 22631e270976SMartin KaFai Lau struct bpf_prog_info info = {}; 22641e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 22655f8f8b93SAlexei Starovoitov struct bpf_prog_stats stats; 22661e270976SMartin KaFai Lau char __user *uinsns; 22671e270976SMartin KaFai Lau u32 ulen; 22681e270976SMartin KaFai Lau int err; 22691e270976SMartin KaFai Lau 2270dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 22711e270976SMartin KaFai Lau if (err) 22721e270976SMartin KaFai Lau return err; 22731e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 22741e270976SMartin KaFai Lau 22751e270976SMartin KaFai Lau if (copy_from_user(&info, uinfo, info_len)) 227689b09689SDaniel Borkmann return -EFAULT; 22771e270976SMartin KaFai Lau 22781e270976SMartin KaFai Lau info.type = prog->type; 22791e270976SMartin KaFai Lau info.id = prog->aux->id; 2280cb4d2b3fSMartin KaFai Lau info.load_time = prog->aux->load_time; 2281cb4d2b3fSMartin KaFai Lau info.created_by_uid = from_kuid_munged(current_user_ns(), 2282cb4d2b3fSMartin KaFai Lau prog->aux->user->uid); 2283b85fab0eSJiri Olsa info.gpl_compatible = prog->gpl_compatible; 22841e270976SMartin KaFai Lau 22851e270976SMartin KaFai Lau memcpy(info.tag, prog->tag, sizeof(prog->tag)); 2286cb4d2b3fSMartin KaFai Lau memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); 2287cb4d2b3fSMartin KaFai Lau 2288cb4d2b3fSMartin KaFai Lau ulen = info.nr_map_ids; 2289cb4d2b3fSMartin KaFai Lau info.nr_map_ids = prog->aux->used_map_cnt; 2290cb4d2b3fSMartin KaFai Lau ulen = min_t(u32, info.nr_map_ids, ulen); 2291cb4d2b3fSMartin KaFai Lau if (ulen) { 2292721e08daSMartin KaFai Lau u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); 2293cb4d2b3fSMartin KaFai Lau u32 i; 2294cb4d2b3fSMartin KaFai Lau 2295cb4d2b3fSMartin KaFai Lau for (i = 0; i < ulen; i++) 2296cb4d2b3fSMartin KaFai Lau if (put_user(prog->aux->used_maps[i]->id, 2297cb4d2b3fSMartin KaFai Lau &user_map_ids[i])) 2298cb4d2b3fSMartin KaFai Lau return -EFAULT; 2299cb4d2b3fSMartin KaFai Lau } 23001e270976SMartin KaFai Lau 2301c454a46bSMartin KaFai Lau err = set_info_rec_size(&info); 2302c454a46bSMartin KaFai Lau if (err) 2303c454a46bSMartin KaFai Lau return err; 23047337224fSMartin KaFai Lau 23055f8f8b93SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 23065f8f8b93SAlexei Starovoitov info.run_time_ns = stats.nsecs; 23075f8f8b93SAlexei Starovoitov info.run_cnt = stats.cnt; 23085f8f8b93SAlexei Starovoitov 23091e270976SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) { 23101e270976SMartin KaFai Lau info.jited_prog_len = 0; 23111e270976SMartin KaFai Lau info.xlated_prog_len = 0; 2312dbecd738SSandipan Das info.nr_jited_ksyms = 0; 231328c2fae7SDaniel Borkmann info.nr_jited_func_lens = 0; 231411d8b82dSYonghong Song info.nr_func_info = 0; 231511d8b82dSYonghong Song info.nr_line_info = 0; 231611d8b82dSYonghong Song info.nr_jited_line_info = 0; 23171e270976SMartin KaFai Lau goto done; 23181e270976SMartin KaFai Lau } 23191e270976SMartin KaFai Lau 23201e270976SMartin KaFai Lau ulen = info.xlated_prog_len; 23219975a54bSDaniel Borkmann info.xlated_prog_len = bpf_prog_insn_size(prog); 23221e270976SMartin KaFai Lau if (info.xlated_prog_len && ulen) { 23237105e828SDaniel Borkmann struct bpf_insn *insns_sanitized; 23247105e828SDaniel Borkmann bool fault; 23257105e828SDaniel Borkmann 23267105e828SDaniel Borkmann if (prog->blinded && !bpf_dump_raw_ok()) { 23277105e828SDaniel Borkmann info.xlated_prog_insns = 0; 23287105e828SDaniel Borkmann goto done; 23297105e828SDaniel Borkmann } 23307105e828SDaniel Borkmann insns_sanitized = bpf_insn_prepare_dump(prog); 23317105e828SDaniel Borkmann if (!insns_sanitized) 23327105e828SDaniel Borkmann return -ENOMEM; 23331e270976SMartin KaFai Lau uinsns = u64_to_user_ptr(info.xlated_prog_insns); 23341e270976SMartin KaFai Lau ulen = min_t(u32, info.xlated_prog_len, ulen); 23357105e828SDaniel Borkmann fault = copy_to_user(uinsns, insns_sanitized, ulen); 23367105e828SDaniel Borkmann kfree(insns_sanitized); 23377105e828SDaniel Borkmann if (fault) 23381e270976SMartin KaFai Lau return -EFAULT; 23391e270976SMartin KaFai Lau } 23401e270976SMartin KaFai Lau 2341675fc275SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 2342675fc275SJakub Kicinski err = bpf_prog_offload_info_fill(&info, prog); 2343675fc275SJakub Kicinski if (err) 2344675fc275SJakub Kicinski return err; 2345fcfb126dSJiong Wang goto done; 2346fcfb126dSJiong Wang } 2347fcfb126dSJiong Wang 2348fcfb126dSJiong Wang /* NOTE: the following code is supposed to be skipped for offload. 2349fcfb126dSJiong Wang * bpf_prog_offload_info_fill() is the place to fill similar fields 2350fcfb126dSJiong Wang * for offload. 2351fcfb126dSJiong Wang */ 2352fcfb126dSJiong Wang ulen = info.jited_prog_len; 23534d56a76eSSandipan Das if (prog->aux->func_cnt) { 23544d56a76eSSandipan Das u32 i; 23554d56a76eSSandipan Das 23564d56a76eSSandipan Das info.jited_prog_len = 0; 23574d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) 23584d56a76eSSandipan Das info.jited_prog_len += prog->aux->func[i]->jited_len; 23594d56a76eSSandipan Das } else { 2360fcfb126dSJiong Wang info.jited_prog_len = prog->jited_len; 23614d56a76eSSandipan Das } 23624d56a76eSSandipan Das 2363fcfb126dSJiong Wang if (info.jited_prog_len && ulen) { 2364fcfb126dSJiong Wang if (bpf_dump_raw_ok()) { 2365fcfb126dSJiong Wang uinsns = u64_to_user_ptr(info.jited_prog_insns); 2366fcfb126dSJiong Wang ulen = min_t(u32, info.jited_prog_len, ulen); 23674d56a76eSSandipan Das 23684d56a76eSSandipan Das /* for multi-function programs, copy the JITed 23694d56a76eSSandipan Das * instructions for all the functions 23704d56a76eSSandipan Das */ 23714d56a76eSSandipan Das if (prog->aux->func_cnt) { 23724d56a76eSSandipan Das u32 len, free, i; 23734d56a76eSSandipan Das u8 *img; 23744d56a76eSSandipan Das 23754d56a76eSSandipan Das free = ulen; 23764d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) { 23774d56a76eSSandipan Das len = prog->aux->func[i]->jited_len; 23784d56a76eSSandipan Das len = min_t(u32, len, free); 23794d56a76eSSandipan Das img = (u8 *) prog->aux->func[i]->bpf_func; 23804d56a76eSSandipan Das if (copy_to_user(uinsns, img, len)) 23814d56a76eSSandipan Das return -EFAULT; 23824d56a76eSSandipan Das uinsns += len; 23834d56a76eSSandipan Das free -= len; 23844d56a76eSSandipan Das if (!free) 23854d56a76eSSandipan Das break; 23864d56a76eSSandipan Das } 23874d56a76eSSandipan Das } else { 2388fcfb126dSJiong Wang if (copy_to_user(uinsns, prog->bpf_func, ulen)) 2389fcfb126dSJiong Wang return -EFAULT; 23904d56a76eSSandipan Das } 2391fcfb126dSJiong Wang } else { 2392fcfb126dSJiong Wang info.jited_prog_insns = 0; 2393fcfb126dSJiong Wang } 2394675fc275SJakub Kicinski } 2395675fc275SJakub Kicinski 2396dbecd738SSandipan Das ulen = info.nr_jited_ksyms; 2397ff1889fcSSong Liu info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; 23987a5725ddSSong Liu if (ulen) { 2399dbecd738SSandipan Das if (bpf_dump_raw_ok()) { 2400ff1889fcSSong Liu unsigned long ksym_addr; 2401dbecd738SSandipan Das u64 __user *user_ksyms; 2402dbecd738SSandipan Das u32 i; 2403dbecd738SSandipan Das 2404dbecd738SSandipan Das /* copy the address of the kernel symbol 2405dbecd738SSandipan Das * corresponding to each function 2406dbecd738SSandipan Das */ 2407dbecd738SSandipan Das ulen = min_t(u32, info.nr_jited_ksyms, ulen); 2408dbecd738SSandipan Das user_ksyms = u64_to_user_ptr(info.jited_ksyms); 2409ff1889fcSSong Liu if (prog->aux->func_cnt) { 2410dbecd738SSandipan Das for (i = 0; i < ulen; i++) { 2411ff1889fcSSong Liu ksym_addr = (unsigned long) 2412ff1889fcSSong Liu prog->aux->func[i]->bpf_func; 2413ff1889fcSSong Liu if (put_user((u64) ksym_addr, 2414ff1889fcSSong Liu &user_ksyms[i])) 2415ff1889fcSSong Liu return -EFAULT; 2416ff1889fcSSong Liu } 2417ff1889fcSSong Liu } else { 2418ff1889fcSSong Liu ksym_addr = (unsigned long) prog->bpf_func; 2419ff1889fcSSong Liu if (put_user((u64) ksym_addr, &user_ksyms[0])) 2420dbecd738SSandipan Das return -EFAULT; 2421dbecd738SSandipan Das } 2422dbecd738SSandipan Das } else { 2423dbecd738SSandipan Das info.jited_ksyms = 0; 2424dbecd738SSandipan Das } 2425dbecd738SSandipan Das } 2426dbecd738SSandipan Das 2427815581c1SSandipan Das ulen = info.nr_jited_func_lens; 2428ff1889fcSSong Liu info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; 24297a5725ddSSong Liu if (ulen) { 2430815581c1SSandipan Das if (bpf_dump_raw_ok()) { 2431815581c1SSandipan Das u32 __user *user_lens; 2432815581c1SSandipan Das u32 func_len, i; 2433815581c1SSandipan Das 2434815581c1SSandipan Das /* copy the JITed image lengths for each function */ 2435815581c1SSandipan Das ulen = min_t(u32, info.nr_jited_func_lens, ulen); 2436815581c1SSandipan Das user_lens = u64_to_user_ptr(info.jited_func_lens); 2437ff1889fcSSong Liu if (prog->aux->func_cnt) { 2438815581c1SSandipan Das for (i = 0; i < ulen; i++) { 2439ff1889fcSSong Liu func_len = 2440ff1889fcSSong Liu prog->aux->func[i]->jited_len; 2441815581c1SSandipan Das if (put_user(func_len, &user_lens[i])) 2442815581c1SSandipan Das return -EFAULT; 2443815581c1SSandipan Das } 2444815581c1SSandipan Das } else { 2445ff1889fcSSong Liu func_len = prog->jited_len; 2446ff1889fcSSong Liu if (put_user(func_len, &user_lens[0])) 2447ff1889fcSSong Liu return -EFAULT; 2448ff1889fcSSong Liu } 2449ff1889fcSSong Liu } else { 2450815581c1SSandipan Das info.jited_func_lens = 0; 2451815581c1SSandipan Das } 2452815581c1SSandipan Das } 2453815581c1SSandipan Das 24547337224fSMartin KaFai Lau if (prog->aux->btf) 2455838e9690SYonghong Song info.btf_id = btf_id(prog->aux->btf); 2456838e9690SYonghong Song 245711d8b82dSYonghong Song ulen = info.nr_func_info; 245811d8b82dSYonghong Song info.nr_func_info = prog->aux->func_info_cnt; 245911d8b82dSYonghong Song if (info.nr_func_info && ulen) { 2460838e9690SYonghong Song char __user *user_finfo; 2461838e9690SYonghong Song 2462838e9690SYonghong Song user_finfo = u64_to_user_ptr(info.func_info); 246311d8b82dSYonghong Song ulen = min_t(u32, info.nr_func_info, ulen); 2464ba64e7d8SYonghong Song if (copy_to_user(user_finfo, prog->aux->func_info, 24657337224fSMartin KaFai Lau info.func_info_rec_size * ulen)) 2466838e9690SYonghong Song return -EFAULT; 2467838e9690SYonghong Song } 2468838e9690SYonghong Song 246911d8b82dSYonghong Song ulen = info.nr_line_info; 247011d8b82dSYonghong Song info.nr_line_info = prog->aux->nr_linfo; 247111d8b82dSYonghong Song if (info.nr_line_info && ulen) { 2472c454a46bSMartin KaFai Lau __u8 __user *user_linfo; 2473c454a46bSMartin KaFai Lau 2474c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.line_info); 247511d8b82dSYonghong Song ulen = min_t(u32, info.nr_line_info, ulen); 2476c454a46bSMartin KaFai Lau if (copy_to_user(user_linfo, prog->aux->linfo, 2477c454a46bSMartin KaFai Lau info.line_info_rec_size * ulen)) 2478c454a46bSMartin KaFai Lau return -EFAULT; 2479c454a46bSMartin KaFai Lau } 2480c454a46bSMartin KaFai Lau 248111d8b82dSYonghong Song ulen = info.nr_jited_line_info; 2482c454a46bSMartin KaFai Lau if (prog->aux->jited_linfo) 248311d8b82dSYonghong Song info.nr_jited_line_info = prog->aux->nr_linfo; 2484c454a46bSMartin KaFai Lau else 248511d8b82dSYonghong Song info.nr_jited_line_info = 0; 248611d8b82dSYonghong Song if (info.nr_jited_line_info && ulen) { 2487c454a46bSMartin KaFai Lau if (bpf_dump_raw_ok()) { 2488c454a46bSMartin KaFai Lau __u64 __user *user_linfo; 2489c454a46bSMartin KaFai Lau u32 i; 2490c454a46bSMartin KaFai Lau 2491c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.jited_line_info); 249211d8b82dSYonghong Song ulen = min_t(u32, info.nr_jited_line_info, ulen); 2493c454a46bSMartin KaFai Lau for (i = 0; i < ulen; i++) { 2494c454a46bSMartin KaFai Lau if (put_user((__u64)(long)prog->aux->jited_linfo[i], 2495c454a46bSMartin KaFai Lau &user_linfo[i])) 2496c454a46bSMartin KaFai Lau return -EFAULT; 2497c454a46bSMartin KaFai Lau } 2498c454a46bSMartin KaFai Lau } else { 2499c454a46bSMartin KaFai Lau info.jited_line_info = 0; 2500c454a46bSMartin KaFai Lau } 2501c454a46bSMartin KaFai Lau } 2502c454a46bSMartin KaFai Lau 2503c872bdb3SSong Liu ulen = info.nr_prog_tags; 2504c872bdb3SSong Liu info.nr_prog_tags = prog->aux->func_cnt ? : 1; 2505c872bdb3SSong Liu if (ulen) { 2506c872bdb3SSong Liu __u8 __user (*user_prog_tags)[BPF_TAG_SIZE]; 2507c872bdb3SSong Liu u32 i; 2508c872bdb3SSong Liu 2509c872bdb3SSong Liu user_prog_tags = u64_to_user_ptr(info.prog_tags); 2510c872bdb3SSong Liu ulen = min_t(u32, info.nr_prog_tags, ulen); 2511c872bdb3SSong Liu if (prog->aux->func_cnt) { 2512c872bdb3SSong Liu for (i = 0; i < ulen; i++) { 2513c872bdb3SSong Liu if (copy_to_user(user_prog_tags[i], 2514c872bdb3SSong Liu prog->aux->func[i]->tag, 2515c872bdb3SSong Liu BPF_TAG_SIZE)) 2516c872bdb3SSong Liu return -EFAULT; 2517c872bdb3SSong Liu } 2518c872bdb3SSong Liu } else { 2519c872bdb3SSong Liu if (copy_to_user(user_prog_tags[0], 2520c872bdb3SSong Liu prog->tag, BPF_TAG_SIZE)) 2521c872bdb3SSong Liu return -EFAULT; 2522c872bdb3SSong Liu } 2523c872bdb3SSong Liu } 2524c872bdb3SSong Liu 25251e270976SMartin KaFai Lau done: 25261e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 25271e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 25281e270976SMartin KaFai Lau return -EFAULT; 25291e270976SMartin KaFai Lau 25301e270976SMartin KaFai Lau return 0; 25311e270976SMartin KaFai Lau } 25321e270976SMartin KaFai Lau 25331e270976SMartin KaFai Lau static int bpf_map_get_info_by_fd(struct bpf_map *map, 25341e270976SMartin KaFai Lau const union bpf_attr *attr, 25351e270976SMartin KaFai Lau union bpf_attr __user *uattr) 25361e270976SMartin KaFai Lau { 25371e270976SMartin KaFai Lau struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 25381e270976SMartin KaFai Lau struct bpf_map_info info = {}; 25391e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 25401e270976SMartin KaFai Lau int err; 25411e270976SMartin KaFai Lau 2542dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 25431e270976SMartin KaFai Lau if (err) 25441e270976SMartin KaFai Lau return err; 25451e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 25461e270976SMartin KaFai Lau 25471e270976SMartin KaFai Lau info.type = map->map_type; 25481e270976SMartin KaFai Lau info.id = map->id; 25491e270976SMartin KaFai Lau info.key_size = map->key_size; 25501e270976SMartin KaFai Lau info.value_size = map->value_size; 25511e270976SMartin KaFai Lau info.max_entries = map->max_entries; 25521e270976SMartin KaFai Lau info.map_flags = map->map_flags; 2553ad5b177bSMartin KaFai Lau memcpy(info.name, map->name, sizeof(map->name)); 25541e270976SMartin KaFai Lau 255578958fcaSMartin KaFai Lau if (map->btf) { 255678958fcaSMartin KaFai Lau info.btf_id = btf_id(map->btf); 25579b2cf328SMartin KaFai Lau info.btf_key_type_id = map->btf_key_type_id; 25589b2cf328SMartin KaFai Lau info.btf_value_type_id = map->btf_value_type_id; 255978958fcaSMartin KaFai Lau } 256078958fcaSMartin KaFai Lau 256152775b33SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 256252775b33SJakub Kicinski err = bpf_map_offload_info_fill(&info, map); 256352775b33SJakub Kicinski if (err) 256452775b33SJakub Kicinski return err; 256552775b33SJakub Kicinski } 256652775b33SJakub Kicinski 25671e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 25681e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 25691e270976SMartin KaFai Lau return -EFAULT; 25701e270976SMartin KaFai Lau 25711e270976SMartin KaFai Lau return 0; 25721e270976SMartin KaFai Lau } 25731e270976SMartin KaFai Lau 257462dab84cSMartin KaFai Lau static int bpf_btf_get_info_by_fd(struct btf *btf, 257562dab84cSMartin KaFai Lau const union bpf_attr *attr, 257662dab84cSMartin KaFai Lau union bpf_attr __user *uattr) 257762dab84cSMartin KaFai Lau { 257862dab84cSMartin KaFai Lau struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info); 257962dab84cSMartin KaFai Lau u32 info_len = attr->info.info_len; 258062dab84cSMartin KaFai Lau int err; 258162dab84cSMartin KaFai Lau 2582dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(*uinfo), info_len); 258362dab84cSMartin KaFai Lau if (err) 258462dab84cSMartin KaFai Lau return err; 258562dab84cSMartin KaFai Lau 258662dab84cSMartin KaFai Lau return btf_get_info_by_fd(btf, attr, uattr); 258762dab84cSMartin KaFai Lau } 258862dab84cSMartin KaFai Lau 25891e270976SMartin KaFai Lau #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 25901e270976SMartin KaFai Lau 25911e270976SMartin KaFai Lau static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 25921e270976SMartin KaFai Lau union bpf_attr __user *uattr) 25931e270976SMartin KaFai Lau { 25941e270976SMartin KaFai Lau int ufd = attr->info.bpf_fd; 25951e270976SMartin KaFai Lau struct fd f; 25961e270976SMartin KaFai Lau int err; 25971e270976SMartin KaFai Lau 25981e270976SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 25991e270976SMartin KaFai Lau return -EINVAL; 26001e270976SMartin KaFai Lau 26011e270976SMartin KaFai Lau f = fdget(ufd); 26021e270976SMartin KaFai Lau if (!f.file) 26031e270976SMartin KaFai Lau return -EBADFD; 26041e270976SMartin KaFai Lau 26051e270976SMartin KaFai Lau if (f.file->f_op == &bpf_prog_fops) 26061e270976SMartin KaFai Lau err = bpf_prog_get_info_by_fd(f.file->private_data, attr, 26071e270976SMartin KaFai Lau uattr); 26081e270976SMartin KaFai Lau else if (f.file->f_op == &bpf_map_fops) 26091e270976SMartin KaFai Lau err = bpf_map_get_info_by_fd(f.file->private_data, attr, 26101e270976SMartin KaFai Lau uattr); 261160197cfbSMartin KaFai Lau else if (f.file->f_op == &btf_fops) 261262dab84cSMartin KaFai Lau err = bpf_btf_get_info_by_fd(f.file->private_data, attr, uattr); 26131e270976SMartin KaFai Lau else 26141e270976SMartin KaFai Lau err = -EINVAL; 26151e270976SMartin KaFai Lau 26161e270976SMartin KaFai Lau fdput(f); 26171e270976SMartin KaFai Lau return err; 26181e270976SMartin KaFai Lau } 26191e270976SMartin KaFai Lau 2620f56a653cSMartin KaFai Lau #define BPF_BTF_LOAD_LAST_FIELD btf_log_level 2621f56a653cSMartin KaFai Lau 2622f56a653cSMartin KaFai Lau static int bpf_btf_load(const union bpf_attr *attr) 2623f56a653cSMartin KaFai Lau { 2624f56a653cSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_LOAD)) 2625f56a653cSMartin KaFai Lau return -EINVAL; 2626f56a653cSMartin KaFai Lau 2627f56a653cSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 2628f56a653cSMartin KaFai Lau return -EPERM; 2629f56a653cSMartin KaFai Lau 2630f56a653cSMartin KaFai Lau return btf_new_fd(attr); 2631f56a653cSMartin KaFai Lau } 2632f56a653cSMartin KaFai Lau 263378958fcaSMartin KaFai Lau #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id 263478958fcaSMartin KaFai Lau 263578958fcaSMartin KaFai Lau static int bpf_btf_get_fd_by_id(const union bpf_attr *attr) 263678958fcaSMartin KaFai Lau { 263778958fcaSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID)) 263878958fcaSMartin KaFai Lau return -EINVAL; 263978958fcaSMartin KaFai Lau 264078958fcaSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 264178958fcaSMartin KaFai Lau return -EPERM; 264278958fcaSMartin KaFai Lau 264378958fcaSMartin KaFai Lau return btf_get_fd_by_id(attr->btf_id); 264478958fcaSMartin KaFai Lau } 264578958fcaSMartin KaFai Lau 264641bdc4b4SYonghong Song static int bpf_task_fd_query_copy(const union bpf_attr *attr, 264741bdc4b4SYonghong Song union bpf_attr __user *uattr, 264841bdc4b4SYonghong Song u32 prog_id, u32 fd_type, 264941bdc4b4SYonghong Song const char *buf, u64 probe_offset, 265041bdc4b4SYonghong Song u64 probe_addr) 265141bdc4b4SYonghong Song { 265241bdc4b4SYonghong Song char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf); 265341bdc4b4SYonghong Song u32 len = buf ? strlen(buf) : 0, input_len; 265441bdc4b4SYonghong Song int err = 0; 265541bdc4b4SYonghong Song 265641bdc4b4SYonghong Song if (put_user(len, &uattr->task_fd_query.buf_len)) 265741bdc4b4SYonghong Song return -EFAULT; 265841bdc4b4SYonghong Song input_len = attr->task_fd_query.buf_len; 265941bdc4b4SYonghong Song if (input_len && ubuf) { 266041bdc4b4SYonghong Song if (!len) { 266141bdc4b4SYonghong Song /* nothing to copy, just make ubuf NULL terminated */ 266241bdc4b4SYonghong Song char zero = '\0'; 266341bdc4b4SYonghong Song 266441bdc4b4SYonghong Song if (put_user(zero, ubuf)) 266541bdc4b4SYonghong Song return -EFAULT; 266641bdc4b4SYonghong Song } else if (input_len >= len + 1) { 266741bdc4b4SYonghong Song /* ubuf can hold the string with NULL terminator */ 266841bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, len + 1)) 266941bdc4b4SYonghong Song return -EFAULT; 267041bdc4b4SYonghong Song } else { 267141bdc4b4SYonghong Song /* ubuf cannot hold the string with NULL terminator, 267241bdc4b4SYonghong Song * do a partial copy with NULL terminator. 267341bdc4b4SYonghong Song */ 267441bdc4b4SYonghong Song char zero = '\0'; 267541bdc4b4SYonghong Song 267641bdc4b4SYonghong Song err = -ENOSPC; 267741bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, input_len - 1)) 267841bdc4b4SYonghong Song return -EFAULT; 267941bdc4b4SYonghong Song if (put_user(zero, ubuf + input_len - 1)) 268041bdc4b4SYonghong Song return -EFAULT; 268141bdc4b4SYonghong Song } 268241bdc4b4SYonghong Song } 268341bdc4b4SYonghong Song 268441bdc4b4SYonghong Song if (put_user(prog_id, &uattr->task_fd_query.prog_id) || 268541bdc4b4SYonghong Song put_user(fd_type, &uattr->task_fd_query.fd_type) || 268641bdc4b4SYonghong Song put_user(probe_offset, &uattr->task_fd_query.probe_offset) || 268741bdc4b4SYonghong Song put_user(probe_addr, &uattr->task_fd_query.probe_addr)) 268841bdc4b4SYonghong Song return -EFAULT; 268941bdc4b4SYonghong Song 269041bdc4b4SYonghong Song return err; 269141bdc4b4SYonghong Song } 269241bdc4b4SYonghong Song 269341bdc4b4SYonghong Song #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr 269441bdc4b4SYonghong Song 269541bdc4b4SYonghong Song static int bpf_task_fd_query(const union bpf_attr *attr, 269641bdc4b4SYonghong Song union bpf_attr __user *uattr) 269741bdc4b4SYonghong Song { 269841bdc4b4SYonghong Song pid_t pid = attr->task_fd_query.pid; 269941bdc4b4SYonghong Song u32 fd = attr->task_fd_query.fd; 270041bdc4b4SYonghong Song const struct perf_event *event; 270141bdc4b4SYonghong Song struct files_struct *files; 270241bdc4b4SYonghong Song struct task_struct *task; 270341bdc4b4SYonghong Song struct file *file; 270441bdc4b4SYonghong Song int err; 270541bdc4b4SYonghong Song 270641bdc4b4SYonghong Song if (CHECK_ATTR(BPF_TASK_FD_QUERY)) 270741bdc4b4SYonghong Song return -EINVAL; 270841bdc4b4SYonghong Song 270941bdc4b4SYonghong Song if (!capable(CAP_SYS_ADMIN)) 271041bdc4b4SYonghong Song return -EPERM; 271141bdc4b4SYonghong Song 271241bdc4b4SYonghong Song if (attr->task_fd_query.flags != 0) 271341bdc4b4SYonghong Song return -EINVAL; 271441bdc4b4SYonghong Song 271541bdc4b4SYonghong Song task = get_pid_task(find_vpid(pid), PIDTYPE_PID); 271641bdc4b4SYonghong Song if (!task) 271741bdc4b4SYonghong Song return -ENOENT; 271841bdc4b4SYonghong Song 271941bdc4b4SYonghong Song files = get_files_struct(task); 272041bdc4b4SYonghong Song put_task_struct(task); 272141bdc4b4SYonghong Song if (!files) 272241bdc4b4SYonghong Song return -ENOENT; 272341bdc4b4SYonghong Song 272441bdc4b4SYonghong Song err = 0; 272541bdc4b4SYonghong Song spin_lock(&files->file_lock); 272641bdc4b4SYonghong Song file = fcheck_files(files, fd); 272741bdc4b4SYonghong Song if (!file) 272841bdc4b4SYonghong Song err = -EBADF; 272941bdc4b4SYonghong Song else 273041bdc4b4SYonghong Song get_file(file); 273141bdc4b4SYonghong Song spin_unlock(&files->file_lock); 273241bdc4b4SYonghong Song put_files_struct(files); 273341bdc4b4SYonghong Song 273441bdc4b4SYonghong Song if (err) 273541bdc4b4SYonghong Song goto out; 273641bdc4b4SYonghong Song 273741bdc4b4SYonghong Song if (file->f_op == &bpf_raw_tp_fops) { 273841bdc4b4SYonghong Song struct bpf_raw_tracepoint *raw_tp = file->private_data; 273941bdc4b4SYonghong Song struct bpf_raw_event_map *btp = raw_tp->btp; 274041bdc4b4SYonghong Song 274141bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, 274241bdc4b4SYonghong Song raw_tp->prog->aux->id, 274341bdc4b4SYonghong Song BPF_FD_TYPE_RAW_TRACEPOINT, 274441bdc4b4SYonghong Song btp->tp->name, 0, 0); 274541bdc4b4SYonghong Song goto put_file; 274641bdc4b4SYonghong Song } 274741bdc4b4SYonghong Song 274841bdc4b4SYonghong Song event = perf_get_event(file); 274941bdc4b4SYonghong Song if (!IS_ERR(event)) { 275041bdc4b4SYonghong Song u64 probe_offset, probe_addr; 275141bdc4b4SYonghong Song u32 prog_id, fd_type; 275241bdc4b4SYonghong Song const char *buf; 275341bdc4b4SYonghong Song 275441bdc4b4SYonghong Song err = bpf_get_perf_event_info(event, &prog_id, &fd_type, 275541bdc4b4SYonghong Song &buf, &probe_offset, 275641bdc4b4SYonghong Song &probe_addr); 275741bdc4b4SYonghong Song if (!err) 275841bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, prog_id, 275941bdc4b4SYonghong Song fd_type, buf, 276041bdc4b4SYonghong Song probe_offset, 276141bdc4b4SYonghong Song probe_addr); 276241bdc4b4SYonghong Song goto put_file; 276341bdc4b4SYonghong Song } 276441bdc4b4SYonghong Song 276541bdc4b4SYonghong Song err = -ENOTSUPP; 276641bdc4b4SYonghong Song put_file: 276741bdc4b4SYonghong Song fput(file); 276841bdc4b4SYonghong Song out: 276941bdc4b4SYonghong Song return err; 277041bdc4b4SYonghong Song } 277141bdc4b4SYonghong Song 277299c55f7dSAlexei Starovoitov SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 277399c55f7dSAlexei Starovoitov { 277499c55f7dSAlexei Starovoitov union bpf_attr attr = {}; 277599c55f7dSAlexei Starovoitov int err; 277699c55f7dSAlexei Starovoitov 27770fa4fe85SChenbo Feng if (sysctl_unprivileged_bpf_disabled && !capable(CAP_SYS_ADMIN)) 277899c55f7dSAlexei Starovoitov return -EPERM; 277999c55f7dSAlexei Starovoitov 2780dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size); 278199c55f7dSAlexei Starovoitov if (err) 278299c55f7dSAlexei Starovoitov return err; 27831e270976SMartin KaFai Lau size = min_t(u32, size, sizeof(attr)); 278499c55f7dSAlexei Starovoitov 278599c55f7dSAlexei Starovoitov /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 278699c55f7dSAlexei Starovoitov if (copy_from_user(&attr, uattr, size) != 0) 278799c55f7dSAlexei Starovoitov return -EFAULT; 278899c55f7dSAlexei Starovoitov 2789afdb09c7SChenbo Feng err = security_bpf(cmd, &attr, size); 2790afdb09c7SChenbo Feng if (err < 0) 2791afdb09c7SChenbo Feng return err; 2792afdb09c7SChenbo Feng 279399c55f7dSAlexei Starovoitov switch (cmd) { 279499c55f7dSAlexei Starovoitov case BPF_MAP_CREATE: 279599c55f7dSAlexei Starovoitov err = map_create(&attr); 279699c55f7dSAlexei Starovoitov break; 2797db20fd2bSAlexei Starovoitov case BPF_MAP_LOOKUP_ELEM: 2798db20fd2bSAlexei Starovoitov err = map_lookup_elem(&attr); 2799db20fd2bSAlexei Starovoitov break; 2800db20fd2bSAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 2801db20fd2bSAlexei Starovoitov err = map_update_elem(&attr); 2802db20fd2bSAlexei Starovoitov break; 2803db20fd2bSAlexei Starovoitov case BPF_MAP_DELETE_ELEM: 2804db20fd2bSAlexei Starovoitov err = map_delete_elem(&attr); 2805db20fd2bSAlexei Starovoitov break; 2806db20fd2bSAlexei Starovoitov case BPF_MAP_GET_NEXT_KEY: 2807db20fd2bSAlexei Starovoitov err = map_get_next_key(&attr); 2808db20fd2bSAlexei Starovoitov break; 280987df15deSDaniel Borkmann case BPF_MAP_FREEZE: 281087df15deSDaniel Borkmann err = map_freeze(&attr); 281187df15deSDaniel Borkmann break; 281209756af4SAlexei Starovoitov case BPF_PROG_LOAD: 2813838e9690SYonghong Song err = bpf_prog_load(&attr, uattr); 281409756af4SAlexei Starovoitov break; 2815b2197755SDaniel Borkmann case BPF_OBJ_PIN: 2816b2197755SDaniel Borkmann err = bpf_obj_pin(&attr); 2817b2197755SDaniel Borkmann break; 2818b2197755SDaniel Borkmann case BPF_OBJ_GET: 2819b2197755SDaniel Borkmann err = bpf_obj_get(&attr); 2820b2197755SDaniel Borkmann break; 2821f4324551SDaniel Mack case BPF_PROG_ATTACH: 2822f4324551SDaniel Mack err = bpf_prog_attach(&attr); 2823f4324551SDaniel Mack break; 2824f4324551SDaniel Mack case BPF_PROG_DETACH: 2825f4324551SDaniel Mack err = bpf_prog_detach(&attr); 2826f4324551SDaniel Mack break; 2827468e2f64SAlexei Starovoitov case BPF_PROG_QUERY: 2828468e2f64SAlexei Starovoitov err = bpf_prog_query(&attr, uattr); 2829468e2f64SAlexei Starovoitov break; 28301cf1cae9SAlexei Starovoitov case BPF_PROG_TEST_RUN: 28311cf1cae9SAlexei Starovoitov err = bpf_prog_test_run(&attr, uattr); 28321cf1cae9SAlexei Starovoitov break; 283334ad5580SMartin KaFai Lau case BPF_PROG_GET_NEXT_ID: 283434ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 283534ad5580SMartin KaFai Lau &prog_idr, &prog_idr_lock); 283634ad5580SMartin KaFai Lau break; 283734ad5580SMartin KaFai Lau case BPF_MAP_GET_NEXT_ID: 283834ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 283934ad5580SMartin KaFai Lau &map_idr, &map_idr_lock); 284034ad5580SMartin KaFai Lau break; 2841b16d9aa4SMartin KaFai Lau case BPF_PROG_GET_FD_BY_ID: 2842b16d9aa4SMartin KaFai Lau err = bpf_prog_get_fd_by_id(&attr); 2843b16d9aa4SMartin KaFai Lau break; 2844bd5f5f4eSMartin KaFai Lau case BPF_MAP_GET_FD_BY_ID: 2845bd5f5f4eSMartin KaFai Lau err = bpf_map_get_fd_by_id(&attr); 2846bd5f5f4eSMartin KaFai Lau break; 28471e270976SMartin KaFai Lau case BPF_OBJ_GET_INFO_BY_FD: 28481e270976SMartin KaFai Lau err = bpf_obj_get_info_by_fd(&attr, uattr); 28491e270976SMartin KaFai Lau break; 2850c4f6699dSAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN: 2851c4f6699dSAlexei Starovoitov err = bpf_raw_tracepoint_open(&attr); 2852c4f6699dSAlexei Starovoitov break; 2853f56a653cSMartin KaFai Lau case BPF_BTF_LOAD: 2854f56a653cSMartin KaFai Lau err = bpf_btf_load(&attr); 2855f56a653cSMartin KaFai Lau break; 285678958fcaSMartin KaFai Lau case BPF_BTF_GET_FD_BY_ID: 285778958fcaSMartin KaFai Lau err = bpf_btf_get_fd_by_id(&attr); 285878958fcaSMartin KaFai Lau break; 285941bdc4b4SYonghong Song case BPF_TASK_FD_QUERY: 286041bdc4b4SYonghong Song err = bpf_task_fd_query(&attr, uattr); 286141bdc4b4SYonghong Song break; 2862bd513cd0SMauricio Vasquez B case BPF_MAP_LOOKUP_AND_DELETE_ELEM: 2863bd513cd0SMauricio Vasquez B err = map_lookup_and_delete_elem(&attr); 2864bd513cd0SMauricio Vasquez B break; 286599c55f7dSAlexei Starovoitov default: 286699c55f7dSAlexei Starovoitov err = -EINVAL; 286799c55f7dSAlexei Starovoitov break; 286899c55f7dSAlexei Starovoitov } 286999c55f7dSAlexei Starovoitov 287099c55f7dSAlexei Starovoitov return err; 287199c55f7dSAlexei Starovoitov } 2872