199c55f7dSAlexei Starovoitov /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 299c55f7dSAlexei Starovoitov * 399c55f7dSAlexei Starovoitov * This program is free software; you can redistribute it and/or 499c55f7dSAlexei Starovoitov * modify it under the terms of version 2 of the GNU General Public 599c55f7dSAlexei Starovoitov * License as published by the Free Software Foundation. 699c55f7dSAlexei Starovoitov * 799c55f7dSAlexei Starovoitov * This program is distributed in the hope that it will be useful, but 899c55f7dSAlexei Starovoitov * WITHOUT ANY WARRANTY; without even the implied warranty of 999c55f7dSAlexei Starovoitov * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 1099c55f7dSAlexei Starovoitov * General Public License for more details. 1199c55f7dSAlexei Starovoitov */ 1299c55f7dSAlexei Starovoitov #include <linux/bpf.h> 13a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 14f4364dcfSSean Young #include <linux/bpf_lirc.h> 15f56a653cSMartin KaFai Lau #include <linux/btf.h> 1699c55f7dSAlexei Starovoitov #include <linux/syscalls.h> 1799c55f7dSAlexei Starovoitov #include <linux/slab.h> 183f07c014SIngo Molnar #include <linux/sched/signal.h> 19d407bd25SDaniel Borkmann #include <linux/vmalloc.h> 20d407bd25SDaniel Borkmann #include <linux/mmzone.h> 2199c55f7dSAlexei Starovoitov #include <linux/anon_inodes.h> 2241bdc4b4SYonghong Song #include <linux/fdtable.h> 23db20fd2bSAlexei Starovoitov #include <linux/file.h> 2441bdc4b4SYonghong Song #include <linux/fs.h> 2509756af4SAlexei Starovoitov #include <linux/license.h> 2609756af4SAlexei Starovoitov #include <linux/filter.h> 272541517cSAlexei Starovoitov #include <linux/version.h> 28535e7b4bSMickaël Salaün #include <linux/kernel.h> 29dc4bb0e2SMartin KaFai Lau #include <linux/idr.h> 30cb4d2b3fSMartin KaFai Lau #include <linux/cred.h> 31cb4d2b3fSMartin KaFai Lau #include <linux/timekeeping.h> 32cb4d2b3fSMartin KaFai Lau #include <linux/ctype.h> 339ef09e35SMark Rutland #include <linux/nospec.h> 3499c55f7dSAlexei Starovoitov 3514dc6f04SMartin KaFai Lau #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ 3614dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 3714dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 3814dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 3914dc6f04SMartin KaFai Lau #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 4014dc6f04SMartin KaFai Lau #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map)) 4114dc6f04SMartin KaFai Lau 426e71b04aSChenbo Feng #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) 436e71b04aSChenbo Feng 44b121d1e7SAlexei Starovoitov DEFINE_PER_CPU(int, bpf_prog_active); 45dc4bb0e2SMartin KaFai Lau static DEFINE_IDR(prog_idr); 46dc4bb0e2SMartin KaFai Lau static DEFINE_SPINLOCK(prog_idr_lock); 47f3f1c054SMartin KaFai Lau static DEFINE_IDR(map_idr); 48f3f1c054SMartin KaFai Lau static DEFINE_SPINLOCK(map_idr_lock); 49b121d1e7SAlexei Starovoitov 501be7f75dSAlexei Starovoitov int sysctl_unprivileged_bpf_disabled __read_mostly; 511be7f75dSAlexei Starovoitov 5240077e0cSJohannes Berg static const struct bpf_map_ops * const bpf_map_types[] = { 5340077e0cSJohannes Berg #define BPF_PROG_TYPE(_id, _ops) 5440077e0cSJohannes Berg #define BPF_MAP_TYPE(_id, _ops) \ 5540077e0cSJohannes Berg [_id] = &_ops, 5640077e0cSJohannes Berg #include <linux/bpf_types.h> 5740077e0cSJohannes Berg #undef BPF_PROG_TYPE 5840077e0cSJohannes Berg #undef BPF_MAP_TYPE 5940077e0cSJohannes Berg }; 6099c55f7dSAlexei Starovoitov 61752ba56fSMickaël Salaün /* 62752ba56fSMickaël Salaün * If we're handed a bigger struct than we know of, ensure all the unknown bits 63752ba56fSMickaël Salaün * are 0 - i.e. new user-space does not rely on any kernel feature extensions 64752ba56fSMickaël Salaün * we don't know about yet. 65752ba56fSMickaël Salaün * 66752ba56fSMickaël Salaün * There is a ToCToU between this function call and the following 67752ba56fSMickaël Salaün * copy_from_user() call. However, this is not a concern since this function is 68752ba56fSMickaël Salaün * meant to be a future-proofing of bits. 69752ba56fSMickaël Salaün */ 70dcab51f1SMartin KaFai Lau int bpf_check_uarg_tail_zero(void __user *uaddr, 7158291a74SMickaël Salaün size_t expected_size, 7258291a74SMickaël Salaün size_t actual_size) 7358291a74SMickaël Salaün { 7458291a74SMickaël Salaün unsigned char __user *addr; 7558291a74SMickaël Salaün unsigned char __user *end; 7658291a74SMickaël Salaün unsigned char val; 7758291a74SMickaël Salaün int err; 7858291a74SMickaël Salaün 79752ba56fSMickaël Salaün if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 80752ba56fSMickaël Salaün return -E2BIG; 81752ba56fSMickaël Salaün 8296d4f267SLinus Torvalds if (unlikely(!access_ok(uaddr, actual_size))) 83752ba56fSMickaël Salaün return -EFAULT; 84752ba56fSMickaël Salaün 8558291a74SMickaël Salaün if (actual_size <= expected_size) 8658291a74SMickaël Salaün return 0; 8758291a74SMickaël Salaün 8858291a74SMickaël Salaün addr = uaddr + expected_size; 8958291a74SMickaël Salaün end = uaddr + actual_size; 9058291a74SMickaël Salaün 9158291a74SMickaël Salaün for (; addr < end; addr++) { 9258291a74SMickaël Salaün err = get_user(val, addr); 9358291a74SMickaël Salaün if (err) 9458291a74SMickaël Salaün return err; 9558291a74SMickaël Salaün if (val) 9658291a74SMickaël Salaün return -E2BIG; 9758291a74SMickaël Salaün } 9858291a74SMickaël Salaün 9958291a74SMickaël Salaün return 0; 10058291a74SMickaël Salaün } 10158291a74SMickaël Salaün 102a3884572SJakub Kicinski const struct bpf_map_ops bpf_map_offload_ops = { 103a3884572SJakub Kicinski .map_alloc = bpf_map_offload_map_alloc, 104a3884572SJakub Kicinski .map_free = bpf_map_offload_map_free, 105e8d2bec0SDaniel Borkmann .map_check_btf = map_check_no_btf, 106a3884572SJakub Kicinski }; 107a3884572SJakub Kicinski 10899c55f7dSAlexei Starovoitov static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 10999c55f7dSAlexei Starovoitov { 1101110f3a9SJakub Kicinski const struct bpf_map_ops *ops; 1119ef09e35SMark Rutland u32 type = attr->map_type; 11299c55f7dSAlexei Starovoitov struct bpf_map *map; 1131110f3a9SJakub Kicinski int err; 11499c55f7dSAlexei Starovoitov 1159ef09e35SMark Rutland if (type >= ARRAY_SIZE(bpf_map_types)) 1161110f3a9SJakub Kicinski return ERR_PTR(-EINVAL); 1179ef09e35SMark Rutland type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types)); 1189ef09e35SMark Rutland ops = bpf_map_types[type]; 1191110f3a9SJakub Kicinski if (!ops) 12040077e0cSJohannes Berg return ERR_PTR(-EINVAL); 12140077e0cSJohannes Berg 1221110f3a9SJakub Kicinski if (ops->map_alloc_check) { 1231110f3a9SJakub Kicinski err = ops->map_alloc_check(attr); 1241110f3a9SJakub Kicinski if (err) 1251110f3a9SJakub Kicinski return ERR_PTR(err); 1261110f3a9SJakub Kicinski } 127a3884572SJakub Kicinski if (attr->map_ifindex) 128a3884572SJakub Kicinski ops = &bpf_map_offload_ops; 1291110f3a9SJakub Kicinski map = ops->map_alloc(attr); 13099c55f7dSAlexei Starovoitov if (IS_ERR(map)) 13199c55f7dSAlexei Starovoitov return map; 1321110f3a9SJakub Kicinski map->ops = ops; 1339ef09e35SMark Rutland map->map_type = type; 13499c55f7dSAlexei Starovoitov return map; 13599c55f7dSAlexei Starovoitov } 13699c55f7dSAlexei Starovoitov 13796eabe7aSMartin KaFai Lau void *bpf_map_area_alloc(size_t size, int numa_node) 138d407bd25SDaniel Borkmann { 139f01a7dbeSMartynas Pumputis /* We really just want to fail instead of triggering OOM killer 140f01a7dbeSMartynas Pumputis * under memory pressure, therefore we set __GFP_NORETRY to kmalloc, 141f01a7dbeSMartynas Pumputis * which is used for lower order allocation requests. 142f01a7dbeSMartynas Pumputis * 143f01a7dbeSMartynas Pumputis * It has been observed that higher order allocation requests done by 144f01a7dbeSMartynas Pumputis * vmalloc with __GFP_NORETRY being set might fail due to not trying 145f01a7dbeSMartynas Pumputis * to reclaim memory from the page cache, thus we set 146f01a7dbeSMartynas Pumputis * __GFP_RETRY_MAYFAIL to avoid such situations. 147d407bd25SDaniel Borkmann */ 148f01a7dbeSMartynas Pumputis 149f01a7dbeSMartynas Pumputis const gfp_t flags = __GFP_NOWARN | __GFP_ZERO; 150d407bd25SDaniel Borkmann void *area; 151d407bd25SDaniel Borkmann 152d407bd25SDaniel Borkmann if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 153f01a7dbeSMartynas Pumputis area = kmalloc_node(size, GFP_USER | __GFP_NORETRY | flags, 154f01a7dbeSMartynas Pumputis numa_node); 155d407bd25SDaniel Borkmann if (area != NULL) 156d407bd25SDaniel Borkmann return area; 157d407bd25SDaniel Borkmann } 158d407bd25SDaniel Borkmann 159f01a7dbeSMartynas Pumputis return __vmalloc_node_flags_caller(size, numa_node, 160f01a7dbeSMartynas Pumputis GFP_KERNEL | __GFP_RETRY_MAYFAIL | 161f01a7dbeSMartynas Pumputis flags, __builtin_return_address(0)); 162d407bd25SDaniel Borkmann } 163d407bd25SDaniel Borkmann 164d407bd25SDaniel Borkmann void bpf_map_area_free(void *area) 165d407bd25SDaniel Borkmann { 166d407bd25SDaniel Borkmann kvfree(area); 167d407bd25SDaniel Borkmann } 168d407bd25SDaniel Borkmann 169be70bcd5SDaniel Borkmann static u32 bpf_map_flags_retain_permanent(u32 flags) 170be70bcd5SDaniel Borkmann { 171be70bcd5SDaniel Borkmann /* Some map creation flags are not tied to the map object but 172be70bcd5SDaniel Borkmann * rather to the map fd instead, so they have no meaning upon 173be70bcd5SDaniel Borkmann * map object inspection since multiple file descriptors with 174be70bcd5SDaniel Borkmann * different (access) properties can exist here. Thus, given 175be70bcd5SDaniel Borkmann * this has zero meaning for the map itself, lets clear these 176be70bcd5SDaniel Borkmann * from here. 177be70bcd5SDaniel Borkmann */ 178be70bcd5SDaniel Borkmann return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY); 179be70bcd5SDaniel Borkmann } 180be70bcd5SDaniel Borkmann 181bd475643SJakub Kicinski void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) 182bd475643SJakub Kicinski { 183bd475643SJakub Kicinski map->map_type = attr->map_type; 184bd475643SJakub Kicinski map->key_size = attr->key_size; 185bd475643SJakub Kicinski map->value_size = attr->value_size; 186bd475643SJakub Kicinski map->max_entries = attr->max_entries; 187be70bcd5SDaniel Borkmann map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags); 188bd475643SJakub Kicinski map->numa_node = bpf_map_attr_numa_node(attr); 189bd475643SJakub Kicinski } 190bd475643SJakub Kicinski 1910a4c58f5SRoman Gushchin static int bpf_charge_memlock(struct user_struct *user, u32 pages) 192aaac3ba9SAlexei Starovoitov { 1930a4c58f5SRoman Gushchin unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 194aaac3ba9SAlexei Starovoitov 1950a4c58f5SRoman Gushchin if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) { 1960a4c58f5SRoman Gushchin atomic_long_sub(pages, &user->locked_vm); 197aaac3ba9SAlexei Starovoitov return -EPERM; 198aaac3ba9SAlexei Starovoitov } 199aaac3ba9SAlexei Starovoitov return 0; 200aaac3ba9SAlexei Starovoitov } 201aaac3ba9SAlexei Starovoitov 2020a4c58f5SRoman Gushchin static void bpf_uncharge_memlock(struct user_struct *user, u32 pages) 2030a4c58f5SRoman Gushchin { 204b936ca64SRoman Gushchin if (user) 2050a4c58f5SRoman Gushchin atomic_long_sub(pages, &user->locked_vm); 2060a4c58f5SRoman Gushchin } 2070a4c58f5SRoman Gushchin 208*c85d6913SRoman Gushchin int bpf_map_charge_init(struct bpf_map_memory *mem, size_t size) 2090a4c58f5SRoman Gushchin { 210*c85d6913SRoman Gushchin u32 pages = round_up(size, PAGE_SIZE) >> PAGE_SHIFT; 211*c85d6913SRoman Gushchin struct user_struct *user; 2120a4c58f5SRoman Gushchin int ret; 2130a4c58f5SRoman Gushchin 214*c85d6913SRoman Gushchin if (size >= U32_MAX - PAGE_SIZE) 215*c85d6913SRoman Gushchin return -E2BIG; 216*c85d6913SRoman Gushchin 217*c85d6913SRoman Gushchin user = get_current_user(); 218b936ca64SRoman Gushchin ret = bpf_charge_memlock(user, pages); 2190a4c58f5SRoman Gushchin if (ret) { 2200a4c58f5SRoman Gushchin free_uid(user); 2210a4c58f5SRoman Gushchin return ret; 2220a4c58f5SRoman Gushchin } 223b936ca64SRoman Gushchin 224b936ca64SRoman Gushchin mem->pages = pages; 225b936ca64SRoman Gushchin mem->user = user; 226b936ca64SRoman Gushchin 227b936ca64SRoman Gushchin return 0; 2280a4c58f5SRoman Gushchin } 2290a4c58f5SRoman Gushchin 230b936ca64SRoman Gushchin void bpf_map_charge_finish(struct bpf_map_memory *mem) 231aaac3ba9SAlexei Starovoitov { 232b936ca64SRoman Gushchin bpf_uncharge_memlock(mem->user, mem->pages); 233b936ca64SRoman Gushchin free_uid(mem->user); 234b936ca64SRoman Gushchin } 2353539b96eSRoman Gushchin 236b936ca64SRoman Gushchin void bpf_map_charge_move(struct bpf_map_memory *dst, 237b936ca64SRoman Gushchin struct bpf_map_memory *src) 238b936ca64SRoman Gushchin { 239b936ca64SRoman Gushchin *dst = *src; 240b936ca64SRoman Gushchin 241b936ca64SRoman Gushchin /* Make sure src will not be used for the redundant uncharging. */ 242b936ca64SRoman Gushchin memset(src, 0, sizeof(struct bpf_map_memory)); 243aaac3ba9SAlexei Starovoitov } 244aaac3ba9SAlexei Starovoitov 2450a4c58f5SRoman Gushchin int bpf_map_charge_memlock(struct bpf_map *map, u32 pages) 2460a4c58f5SRoman Gushchin { 2470a4c58f5SRoman Gushchin int ret; 2480a4c58f5SRoman Gushchin 2493539b96eSRoman Gushchin ret = bpf_charge_memlock(map->memory.user, pages); 2500a4c58f5SRoman Gushchin if (ret) 2510a4c58f5SRoman Gushchin return ret; 2523539b96eSRoman Gushchin map->memory.pages += pages; 2530a4c58f5SRoman Gushchin return ret; 2540a4c58f5SRoman Gushchin } 2550a4c58f5SRoman Gushchin 2560a4c58f5SRoman Gushchin void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages) 2570a4c58f5SRoman Gushchin { 2583539b96eSRoman Gushchin bpf_uncharge_memlock(map->memory.user, pages); 2593539b96eSRoman Gushchin map->memory.pages -= pages; 2600a4c58f5SRoman Gushchin } 2610a4c58f5SRoman Gushchin 262f3f1c054SMartin KaFai Lau static int bpf_map_alloc_id(struct bpf_map *map) 263f3f1c054SMartin KaFai Lau { 264f3f1c054SMartin KaFai Lau int id; 265f3f1c054SMartin KaFai Lau 266b76354cdSShaohua Li idr_preload(GFP_KERNEL); 267f3f1c054SMartin KaFai Lau spin_lock_bh(&map_idr_lock); 268f3f1c054SMartin KaFai Lau id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 269f3f1c054SMartin KaFai Lau if (id > 0) 270f3f1c054SMartin KaFai Lau map->id = id; 271f3f1c054SMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 272b76354cdSShaohua Li idr_preload_end(); 273f3f1c054SMartin KaFai Lau 274f3f1c054SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 275f3f1c054SMartin KaFai Lau return -ENOSPC; 276f3f1c054SMartin KaFai Lau 277f3f1c054SMartin KaFai Lau return id > 0 ? 0 : id; 278f3f1c054SMartin KaFai Lau } 279f3f1c054SMartin KaFai Lau 280a3884572SJakub Kicinski void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) 281f3f1c054SMartin KaFai Lau { 282930651a7SEric Dumazet unsigned long flags; 283930651a7SEric Dumazet 284a3884572SJakub Kicinski /* Offloaded maps are removed from the IDR store when their device 285a3884572SJakub Kicinski * disappears - even if someone holds an fd to them they are unusable, 286a3884572SJakub Kicinski * the memory is gone, all ops will fail; they are simply waiting for 287a3884572SJakub Kicinski * refcnt to drop to be freed. 288a3884572SJakub Kicinski */ 289a3884572SJakub Kicinski if (!map->id) 290a3884572SJakub Kicinski return; 291a3884572SJakub Kicinski 292bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 293930651a7SEric Dumazet spin_lock_irqsave(&map_idr_lock, flags); 294bd5f5f4eSMartin KaFai Lau else 295bd5f5f4eSMartin KaFai Lau __acquire(&map_idr_lock); 296bd5f5f4eSMartin KaFai Lau 297f3f1c054SMartin KaFai Lau idr_remove(&map_idr, map->id); 298a3884572SJakub Kicinski map->id = 0; 299bd5f5f4eSMartin KaFai Lau 300bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 301930651a7SEric Dumazet spin_unlock_irqrestore(&map_idr_lock, flags); 302bd5f5f4eSMartin KaFai Lau else 303bd5f5f4eSMartin KaFai Lau __release(&map_idr_lock); 304f3f1c054SMartin KaFai Lau } 305f3f1c054SMartin KaFai Lau 30699c55f7dSAlexei Starovoitov /* called from workqueue */ 30799c55f7dSAlexei Starovoitov static void bpf_map_free_deferred(struct work_struct *work) 30899c55f7dSAlexei Starovoitov { 30999c55f7dSAlexei Starovoitov struct bpf_map *map = container_of(work, struct bpf_map, work); 310b936ca64SRoman Gushchin struct bpf_map_memory mem; 31199c55f7dSAlexei Starovoitov 312b936ca64SRoman Gushchin bpf_map_charge_move(&mem, &map->memory); 313afdb09c7SChenbo Feng security_bpf_map_free(map); 31499c55f7dSAlexei Starovoitov /* implementation dependent freeing */ 31599c55f7dSAlexei Starovoitov map->ops->map_free(map); 316b936ca64SRoman Gushchin bpf_map_charge_finish(&mem); 31799c55f7dSAlexei Starovoitov } 31899c55f7dSAlexei Starovoitov 319c9da161cSDaniel Borkmann static void bpf_map_put_uref(struct bpf_map *map) 320c9da161cSDaniel Borkmann { 321c9da161cSDaniel Borkmann if (atomic_dec_and_test(&map->usercnt)) { 322ba6b8de4SJohn Fastabend if (map->ops->map_release_uref) 323ba6b8de4SJohn Fastabend map->ops->map_release_uref(map); 324c9da161cSDaniel Borkmann } 325c9da161cSDaniel Borkmann } 326c9da161cSDaniel Borkmann 32799c55f7dSAlexei Starovoitov /* decrement map refcnt and schedule it for freeing via workqueue 32899c55f7dSAlexei Starovoitov * (unrelying map implementation ops->map_free() might sleep) 32999c55f7dSAlexei Starovoitov */ 330bd5f5f4eSMartin KaFai Lau static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) 33199c55f7dSAlexei Starovoitov { 33299c55f7dSAlexei Starovoitov if (atomic_dec_and_test(&map->refcnt)) { 33334ad5580SMartin KaFai Lau /* bpf_map_free_id() must be called first */ 334bd5f5f4eSMartin KaFai Lau bpf_map_free_id(map, do_idr_lock); 33578958fcaSMartin KaFai Lau btf_put(map->btf); 33699c55f7dSAlexei Starovoitov INIT_WORK(&map->work, bpf_map_free_deferred); 33799c55f7dSAlexei Starovoitov schedule_work(&map->work); 33899c55f7dSAlexei Starovoitov } 33999c55f7dSAlexei Starovoitov } 34099c55f7dSAlexei Starovoitov 341bd5f5f4eSMartin KaFai Lau void bpf_map_put(struct bpf_map *map) 342bd5f5f4eSMartin KaFai Lau { 343bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, true); 344bd5f5f4eSMartin KaFai Lau } 345630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_put); 346bd5f5f4eSMartin KaFai Lau 347c9da161cSDaniel Borkmann void bpf_map_put_with_uref(struct bpf_map *map) 348c9da161cSDaniel Borkmann { 349c9da161cSDaniel Borkmann bpf_map_put_uref(map); 350c9da161cSDaniel Borkmann bpf_map_put(map); 351c9da161cSDaniel Borkmann } 352c9da161cSDaniel Borkmann 35399c55f7dSAlexei Starovoitov static int bpf_map_release(struct inode *inode, struct file *filp) 35499c55f7dSAlexei Starovoitov { 35561d1b6a4SDaniel Borkmann struct bpf_map *map = filp->private_data; 35661d1b6a4SDaniel Borkmann 35761d1b6a4SDaniel Borkmann if (map->ops->map_release) 35861d1b6a4SDaniel Borkmann map->ops->map_release(map, filp); 35961d1b6a4SDaniel Borkmann 36061d1b6a4SDaniel Borkmann bpf_map_put_with_uref(map); 36199c55f7dSAlexei Starovoitov return 0; 36299c55f7dSAlexei Starovoitov } 36399c55f7dSAlexei Starovoitov 36487df15deSDaniel Borkmann static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f) 36587df15deSDaniel Borkmann { 36687df15deSDaniel Borkmann fmode_t mode = f.file->f_mode; 36787df15deSDaniel Borkmann 36887df15deSDaniel Borkmann /* Our file permissions may have been overridden by global 36987df15deSDaniel Borkmann * map permissions facing syscall side. 37087df15deSDaniel Borkmann */ 37187df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) 37287df15deSDaniel Borkmann mode &= ~FMODE_CAN_WRITE; 37387df15deSDaniel Borkmann return mode; 37487df15deSDaniel Borkmann } 37587df15deSDaniel Borkmann 376f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 377f99bf205SDaniel Borkmann static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 378f99bf205SDaniel Borkmann { 379f99bf205SDaniel Borkmann const struct bpf_map *map = filp->private_data; 38021116b70SDaniel Borkmann const struct bpf_array *array; 38121116b70SDaniel Borkmann u32 owner_prog_type = 0; 3829780c0abSDaniel Borkmann u32 owner_jited = 0; 38321116b70SDaniel Borkmann 38421116b70SDaniel Borkmann if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 38521116b70SDaniel Borkmann array = container_of(map, struct bpf_array, map); 38621116b70SDaniel Borkmann owner_prog_type = array->owner_prog_type; 3879780c0abSDaniel Borkmann owner_jited = array->owner_jited; 38821116b70SDaniel Borkmann } 389f99bf205SDaniel Borkmann 390f99bf205SDaniel Borkmann seq_printf(m, 391f99bf205SDaniel Borkmann "map_type:\t%u\n" 392f99bf205SDaniel Borkmann "key_size:\t%u\n" 393f99bf205SDaniel Borkmann "value_size:\t%u\n" 394322cea2fSDaniel Borkmann "max_entries:\t%u\n" 39521116b70SDaniel Borkmann "map_flags:\t%#x\n" 3964316b409SDaniel Borkmann "memlock:\t%llu\n" 39787df15deSDaniel Borkmann "map_id:\t%u\n" 39887df15deSDaniel Borkmann "frozen:\t%u\n", 399f99bf205SDaniel Borkmann map->map_type, 400f99bf205SDaniel Borkmann map->key_size, 401f99bf205SDaniel Borkmann map->value_size, 402322cea2fSDaniel Borkmann map->max_entries, 40321116b70SDaniel Borkmann map->map_flags, 4043539b96eSRoman Gushchin map->memory.pages * 1ULL << PAGE_SHIFT, 40587df15deSDaniel Borkmann map->id, 40687df15deSDaniel Borkmann READ_ONCE(map->frozen)); 40721116b70SDaniel Borkmann 4089780c0abSDaniel Borkmann if (owner_prog_type) { 40921116b70SDaniel Borkmann seq_printf(m, "owner_prog_type:\t%u\n", 41021116b70SDaniel Borkmann owner_prog_type); 4119780c0abSDaniel Borkmann seq_printf(m, "owner_jited:\t%u\n", 4129780c0abSDaniel Borkmann owner_jited); 4139780c0abSDaniel Borkmann } 414f99bf205SDaniel Borkmann } 415f99bf205SDaniel Borkmann #endif 416f99bf205SDaniel Borkmann 4176e71b04aSChenbo Feng static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, 4186e71b04aSChenbo Feng loff_t *ppos) 4196e71b04aSChenbo Feng { 4206e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 4216e71b04aSChenbo Feng * f_mode with FMODE_CAN_READ. 4226e71b04aSChenbo Feng */ 4236e71b04aSChenbo Feng return -EINVAL; 4246e71b04aSChenbo Feng } 4256e71b04aSChenbo Feng 4266e71b04aSChenbo Feng static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, 4276e71b04aSChenbo Feng size_t siz, loff_t *ppos) 4286e71b04aSChenbo Feng { 4296e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 4306e71b04aSChenbo Feng * f_mode with FMODE_CAN_WRITE. 4316e71b04aSChenbo Feng */ 4326e71b04aSChenbo Feng return -EINVAL; 4336e71b04aSChenbo Feng } 4346e71b04aSChenbo Feng 435f66e448cSChenbo Feng const struct file_operations bpf_map_fops = { 436f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 437f99bf205SDaniel Borkmann .show_fdinfo = bpf_map_show_fdinfo, 438f99bf205SDaniel Borkmann #endif 43999c55f7dSAlexei Starovoitov .release = bpf_map_release, 4406e71b04aSChenbo Feng .read = bpf_dummy_read, 4416e71b04aSChenbo Feng .write = bpf_dummy_write, 44299c55f7dSAlexei Starovoitov }; 44399c55f7dSAlexei Starovoitov 4446e71b04aSChenbo Feng int bpf_map_new_fd(struct bpf_map *map, int flags) 445aa79781bSDaniel Borkmann { 446afdb09c7SChenbo Feng int ret; 447afdb09c7SChenbo Feng 448afdb09c7SChenbo Feng ret = security_bpf_map(map, OPEN_FMODE(flags)); 449afdb09c7SChenbo Feng if (ret < 0) 450afdb09c7SChenbo Feng return ret; 451afdb09c7SChenbo Feng 452aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 4536e71b04aSChenbo Feng flags | O_CLOEXEC); 4546e71b04aSChenbo Feng } 4556e71b04aSChenbo Feng 4566e71b04aSChenbo Feng int bpf_get_file_flag(int flags) 4576e71b04aSChenbo Feng { 4586e71b04aSChenbo Feng if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) 4596e71b04aSChenbo Feng return -EINVAL; 4606e71b04aSChenbo Feng if (flags & BPF_F_RDONLY) 4616e71b04aSChenbo Feng return O_RDONLY; 4626e71b04aSChenbo Feng if (flags & BPF_F_WRONLY) 4636e71b04aSChenbo Feng return O_WRONLY; 4646e71b04aSChenbo Feng return O_RDWR; 465aa79781bSDaniel Borkmann } 466aa79781bSDaniel Borkmann 46799c55f7dSAlexei Starovoitov /* helper macro to check that unused fields 'union bpf_attr' are zero */ 46899c55f7dSAlexei Starovoitov #define CHECK_ATTR(CMD) \ 46999c55f7dSAlexei Starovoitov memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 47099c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD), 0, \ 47199c55f7dSAlexei Starovoitov sizeof(*attr) - \ 47299c55f7dSAlexei Starovoitov offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 47399c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD)) != NULL 47499c55f7dSAlexei Starovoitov 475cb4d2b3fSMartin KaFai Lau /* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes. 476cb4d2b3fSMartin KaFai Lau * Return 0 on success and < 0 on error. 477cb4d2b3fSMartin KaFai Lau */ 478cb4d2b3fSMartin KaFai Lau static int bpf_obj_name_cpy(char *dst, const char *src) 479cb4d2b3fSMartin KaFai Lau { 480cb4d2b3fSMartin KaFai Lau const char *end = src + BPF_OBJ_NAME_LEN; 481cb4d2b3fSMartin KaFai Lau 482473d9734SMartin KaFai Lau memset(dst, 0, BPF_OBJ_NAME_LEN); 4833e0ddc4fSDaniel Borkmann /* Copy all isalnum(), '_' and '.' chars. */ 484cb4d2b3fSMartin KaFai Lau while (src < end && *src) { 4853e0ddc4fSDaniel Borkmann if (!isalnum(*src) && 4863e0ddc4fSDaniel Borkmann *src != '_' && *src != '.') 487cb4d2b3fSMartin KaFai Lau return -EINVAL; 488cb4d2b3fSMartin KaFai Lau *dst++ = *src++; 489cb4d2b3fSMartin KaFai Lau } 490cb4d2b3fSMartin KaFai Lau 491cb4d2b3fSMartin KaFai Lau /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */ 492cb4d2b3fSMartin KaFai Lau if (src == end) 493cb4d2b3fSMartin KaFai Lau return -EINVAL; 494cb4d2b3fSMartin KaFai Lau 495cb4d2b3fSMartin KaFai Lau return 0; 496cb4d2b3fSMartin KaFai Lau } 497cb4d2b3fSMartin KaFai Lau 498e8d2bec0SDaniel Borkmann int map_check_no_btf(const struct bpf_map *map, 4991b2b234bSRoman Gushchin const struct btf *btf, 500e8d2bec0SDaniel Borkmann const struct btf_type *key_type, 501e8d2bec0SDaniel Borkmann const struct btf_type *value_type) 502e8d2bec0SDaniel Borkmann { 503e8d2bec0SDaniel Borkmann return -ENOTSUPP; 504e8d2bec0SDaniel Borkmann } 505e8d2bec0SDaniel Borkmann 506d83525caSAlexei Starovoitov static int map_check_btf(struct bpf_map *map, const struct btf *btf, 507e8d2bec0SDaniel Borkmann u32 btf_key_id, u32 btf_value_id) 508e8d2bec0SDaniel Borkmann { 509e8d2bec0SDaniel Borkmann const struct btf_type *key_type, *value_type; 510e8d2bec0SDaniel Borkmann u32 key_size, value_size; 511e8d2bec0SDaniel Borkmann int ret = 0; 512e8d2bec0SDaniel Borkmann 5132824ecb7SDaniel Borkmann /* Some maps allow key to be unspecified. */ 5142824ecb7SDaniel Borkmann if (btf_key_id) { 515e8d2bec0SDaniel Borkmann key_type = btf_type_id_size(btf, &btf_key_id, &key_size); 516e8d2bec0SDaniel Borkmann if (!key_type || key_size != map->key_size) 517e8d2bec0SDaniel Borkmann return -EINVAL; 5182824ecb7SDaniel Borkmann } else { 5192824ecb7SDaniel Borkmann key_type = btf_type_by_id(btf, 0); 5202824ecb7SDaniel Borkmann if (!map->ops->map_check_btf) 5212824ecb7SDaniel Borkmann return -EINVAL; 5222824ecb7SDaniel Borkmann } 523e8d2bec0SDaniel Borkmann 524e8d2bec0SDaniel Borkmann value_type = btf_type_id_size(btf, &btf_value_id, &value_size); 525e8d2bec0SDaniel Borkmann if (!value_type || value_size != map->value_size) 526e8d2bec0SDaniel Borkmann return -EINVAL; 527e8d2bec0SDaniel Borkmann 528d83525caSAlexei Starovoitov map->spin_lock_off = btf_find_spin_lock(btf, value_type); 529d83525caSAlexei Starovoitov 530d83525caSAlexei Starovoitov if (map_value_has_spin_lock(map)) { 531591fe988SDaniel Borkmann if (map->map_flags & BPF_F_RDONLY_PROG) 532591fe988SDaniel Borkmann return -EACCES; 533d83525caSAlexei Starovoitov if (map->map_type != BPF_MAP_TYPE_HASH && 534e16d2f1aSAlexei Starovoitov map->map_type != BPF_MAP_TYPE_ARRAY && 5356ac99e8fSMartin KaFai Lau map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 5366ac99e8fSMartin KaFai Lau map->map_type != BPF_MAP_TYPE_SK_STORAGE) 537d83525caSAlexei Starovoitov return -ENOTSUPP; 538d83525caSAlexei Starovoitov if (map->spin_lock_off + sizeof(struct bpf_spin_lock) > 539d83525caSAlexei Starovoitov map->value_size) { 540d83525caSAlexei Starovoitov WARN_ONCE(1, 541d83525caSAlexei Starovoitov "verifier bug spin_lock_off %d value_size %d\n", 542d83525caSAlexei Starovoitov map->spin_lock_off, map->value_size); 543d83525caSAlexei Starovoitov return -EFAULT; 544d83525caSAlexei Starovoitov } 545d83525caSAlexei Starovoitov } 546d83525caSAlexei Starovoitov 547e8d2bec0SDaniel Borkmann if (map->ops->map_check_btf) 5481b2b234bSRoman Gushchin ret = map->ops->map_check_btf(map, btf, key_type, value_type); 549e8d2bec0SDaniel Borkmann 550e8d2bec0SDaniel Borkmann return ret; 551e8d2bec0SDaniel Borkmann } 552e8d2bec0SDaniel Borkmann 5539b2cf328SMartin KaFai Lau #define BPF_MAP_CREATE_LAST_FIELD btf_value_type_id 55499c55f7dSAlexei Starovoitov /* called via syscall */ 55599c55f7dSAlexei Starovoitov static int map_create(union bpf_attr *attr) 55699c55f7dSAlexei Starovoitov { 55796eabe7aSMartin KaFai Lau int numa_node = bpf_map_attr_numa_node(attr); 558b936ca64SRoman Gushchin struct bpf_map_memory mem; 55999c55f7dSAlexei Starovoitov struct bpf_map *map; 5606e71b04aSChenbo Feng int f_flags; 56199c55f7dSAlexei Starovoitov int err; 56299c55f7dSAlexei Starovoitov 56399c55f7dSAlexei Starovoitov err = CHECK_ATTR(BPF_MAP_CREATE); 56499c55f7dSAlexei Starovoitov if (err) 56599c55f7dSAlexei Starovoitov return -EINVAL; 56699c55f7dSAlexei Starovoitov 5676e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->map_flags); 5686e71b04aSChenbo Feng if (f_flags < 0) 5696e71b04aSChenbo Feng return f_flags; 5706e71b04aSChenbo Feng 57196eabe7aSMartin KaFai Lau if (numa_node != NUMA_NO_NODE && 57296e5ae4eSEric Dumazet ((unsigned int)numa_node >= nr_node_ids || 57396e5ae4eSEric Dumazet !node_online(numa_node))) 57496eabe7aSMartin KaFai Lau return -EINVAL; 57596eabe7aSMartin KaFai Lau 57699c55f7dSAlexei Starovoitov /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 57799c55f7dSAlexei Starovoitov map = find_and_alloc_map(attr); 57899c55f7dSAlexei Starovoitov if (IS_ERR(map)) 57999c55f7dSAlexei Starovoitov return PTR_ERR(map); 58099c55f7dSAlexei Starovoitov 581ad5b177bSMartin KaFai Lau err = bpf_obj_name_cpy(map->name, attr->map_name); 582ad5b177bSMartin KaFai Lau if (err) 583b936ca64SRoman Gushchin goto free_map; 584ad5b177bSMartin KaFai Lau 58599c55f7dSAlexei Starovoitov atomic_set(&map->refcnt, 1); 586c9da161cSDaniel Borkmann atomic_set(&map->usercnt, 1); 58799c55f7dSAlexei Starovoitov 588e8d2bec0SDaniel Borkmann if (attr->btf_key_type_id || attr->btf_value_type_id) { 589a26ca7c9SMartin KaFai Lau struct btf *btf; 590a26ca7c9SMartin KaFai Lau 5912824ecb7SDaniel Borkmann if (!attr->btf_value_type_id) { 592a26ca7c9SMartin KaFai Lau err = -EINVAL; 593b936ca64SRoman Gushchin goto free_map; 594a26ca7c9SMartin KaFai Lau } 595a26ca7c9SMartin KaFai Lau 596a26ca7c9SMartin KaFai Lau btf = btf_get_by_fd(attr->btf_fd); 597a26ca7c9SMartin KaFai Lau if (IS_ERR(btf)) { 598a26ca7c9SMartin KaFai Lau err = PTR_ERR(btf); 599b936ca64SRoman Gushchin goto free_map; 600a26ca7c9SMartin KaFai Lau } 601a26ca7c9SMartin KaFai Lau 602e8d2bec0SDaniel Borkmann err = map_check_btf(map, btf, attr->btf_key_type_id, 6039b2cf328SMartin KaFai Lau attr->btf_value_type_id); 604a26ca7c9SMartin KaFai Lau if (err) { 605a26ca7c9SMartin KaFai Lau btf_put(btf); 606b936ca64SRoman Gushchin goto free_map; 607a26ca7c9SMartin KaFai Lau } 608a26ca7c9SMartin KaFai Lau 609a26ca7c9SMartin KaFai Lau map->btf = btf; 6109b2cf328SMartin KaFai Lau map->btf_key_type_id = attr->btf_key_type_id; 6119b2cf328SMartin KaFai Lau map->btf_value_type_id = attr->btf_value_type_id; 612d83525caSAlexei Starovoitov } else { 613d83525caSAlexei Starovoitov map->spin_lock_off = -EINVAL; 614a26ca7c9SMartin KaFai Lau } 615a26ca7c9SMartin KaFai Lau 616afdb09c7SChenbo Feng err = security_bpf_map_alloc(map); 617aaac3ba9SAlexei Starovoitov if (err) 618b936ca64SRoman Gushchin goto free_map; 619afdb09c7SChenbo Feng 620f3f1c054SMartin KaFai Lau err = bpf_map_alloc_id(map); 621f3f1c054SMartin KaFai Lau if (err) 622b936ca64SRoman Gushchin goto free_map_sec; 623f3f1c054SMartin KaFai Lau 6246e71b04aSChenbo Feng err = bpf_map_new_fd(map, f_flags); 625bd5f5f4eSMartin KaFai Lau if (err < 0) { 626bd5f5f4eSMartin KaFai Lau /* failed to allocate fd. 627352d20d6SPeng Sun * bpf_map_put_with_uref() is needed because the above 628bd5f5f4eSMartin KaFai Lau * bpf_map_alloc_id() has published the map 629bd5f5f4eSMartin KaFai Lau * to the userspace and the userspace may 630bd5f5f4eSMartin KaFai Lau * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 631bd5f5f4eSMartin KaFai Lau */ 632352d20d6SPeng Sun bpf_map_put_with_uref(map); 633bd5f5f4eSMartin KaFai Lau return err; 634bd5f5f4eSMartin KaFai Lau } 63599c55f7dSAlexei Starovoitov 63699c55f7dSAlexei Starovoitov return err; 63799c55f7dSAlexei Starovoitov 638afdb09c7SChenbo Feng free_map_sec: 639afdb09c7SChenbo Feng security_bpf_map_free(map); 640b936ca64SRoman Gushchin free_map: 641a26ca7c9SMartin KaFai Lau btf_put(map->btf); 642b936ca64SRoman Gushchin bpf_map_charge_move(&mem, &map->memory); 64399c55f7dSAlexei Starovoitov map->ops->map_free(map); 644b936ca64SRoman Gushchin bpf_map_charge_finish(&mem); 64599c55f7dSAlexei Starovoitov return err; 64699c55f7dSAlexei Starovoitov } 64799c55f7dSAlexei Starovoitov 648db20fd2bSAlexei Starovoitov /* if error is returned, fd is released. 649db20fd2bSAlexei Starovoitov * On success caller should complete fd access with matching fdput() 650db20fd2bSAlexei Starovoitov */ 651c2101297SDaniel Borkmann struct bpf_map *__bpf_map_get(struct fd f) 652db20fd2bSAlexei Starovoitov { 653db20fd2bSAlexei Starovoitov if (!f.file) 654db20fd2bSAlexei Starovoitov return ERR_PTR(-EBADF); 655db20fd2bSAlexei Starovoitov if (f.file->f_op != &bpf_map_fops) { 656db20fd2bSAlexei Starovoitov fdput(f); 657db20fd2bSAlexei Starovoitov return ERR_PTR(-EINVAL); 658db20fd2bSAlexei Starovoitov } 659db20fd2bSAlexei Starovoitov 660c2101297SDaniel Borkmann return f.file->private_data; 661c2101297SDaniel Borkmann } 662c2101297SDaniel Borkmann 66392117d84SAlexei Starovoitov /* prog's and map's refcnt limit */ 66492117d84SAlexei Starovoitov #define BPF_MAX_REFCNT 32768 66592117d84SAlexei Starovoitov 66692117d84SAlexei Starovoitov struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref) 667c9da161cSDaniel Borkmann { 66892117d84SAlexei Starovoitov if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) { 66992117d84SAlexei Starovoitov atomic_dec(&map->refcnt); 67092117d84SAlexei Starovoitov return ERR_PTR(-EBUSY); 67192117d84SAlexei Starovoitov } 672c9da161cSDaniel Borkmann if (uref) 673c9da161cSDaniel Borkmann atomic_inc(&map->usercnt); 67492117d84SAlexei Starovoitov return map; 675c9da161cSDaniel Borkmann } 676630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_inc); 677c9da161cSDaniel Borkmann 678c9da161cSDaniel Borkmann struct bpf_map *bpf_map_get_with_uref(u32 ufd) 679c2101297SDaniel Borkmann { 680c2101297SDaniel Borkmann struct fd f = fdget(ufd); 681c2101297SDaniel Borkmann struct bpf_map *map; 682c2101297SDaniel Borkmann 683c2101297SDaniel Borkmann map = __bpf_map_get(f); 684c2101297SDaniel Borkmann if (IS_ERR(map)) 685c2101297SDaniel Borkmann return map; 686c2101297SDaniel Borkmann 68792117d84SAlexei Starovoitov map = bpf_map_inc(map, true); 688c2101297SDaniel Borkmann fdput(f); 689db20fd2bSAlexei Starovoitov 690db20fd2bSAlexei Starovoitov return map; 691db20fd2bSAlexei Starovoitov } 692db20fd2bSAlexei Starovoitov 693bd5f5f4eSMartin KaFai Lau /* map_idr_lock should have been held */ 694bd5f5f4eSMartin KaFai Lau static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map, 695bd5f5f4eSMartin KaFai Lau bool uref) 696bd5f5f4eSMartin KaFai Lau { 697bd5f5f4eSMartin KaFai Lau int refold; 698bd5f5f4eSMartin KaFai Lau 699bfc18e38SMark Rutland refold = atomic_fetch_add_unless(&map->refcnt, 1, 0); 700bd5f5f4eSMartin KaFai Lau 701bd5f5f4eSMartin KaFai Lau if (refold >= BPF_MAX_REFCNT) { 702bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, false); 703bd5f5f4eSMartin KaFai Lau return ERR_PTR(-EBUSY); 704bd5f5f4eSMartin KaFai Lau } 705bd5f5f4eSMartin KaFai Lau 706bd5f5f4eSMartin KaFai Lau if (!refold) 707bd5f5f4eSMartin KaFai Lau return ERR_PTR(-ENOENT); 708bd5f5f4eSMartin KaFai Lau 709bd5f5f4eSMartin KaFai Lau if (uref) 710bd5f5f4eSMartin KaFai Lau atomic_inc(&map->usercnt); 711bd5f5f4eSMartin KaFai Lau 712bd5f5f4eSMartin KaFai Lau return map; 713bd5f5f4eSMartin KaFai Lau } 714bd5f5f4eSMartin KaFai Lau 715b8cdc051SAlexei Starovoitov int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 716b8cdc051SAlexei Starovoitov { 717b8cdc051SAlexei Starovoitov return -ENOTSUPP; 718b8cdc051SAlexei Starovoitov } 719b8cdc051SAlexei Starovoitov 720c9d29f46SMauricio Vasquez B static void *__bpf_copy_key(void __user *ukey, u64 key_size) 721c9d29f46SMauricio Vasquez B { 722c9d29f46SMauricio Vasquez B if (key_size) 723c9d29f46SMauricio Vasquez B return memdup_user(ukey, key_size); 724c9d29f46SMauricio Vasquez B 725c9d29f46SMauricio Vasquez B if (ukey) 726c9d29f46SMauricio Vasquez B return ERR_PTR(-EINVAL); 727c9d29f46SMauricio Vasquez B 728c9d29f46SMauricio Vasquez B return NULL; 729c9d29f46SMauricio Vasquez B } 730c9d29f46SMauricio Vasquez B 731db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 73296049f3aSAlexei Starovoitov #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags 733db20fd2bSAlexei Starovoitov 734db20fd2bSAlexei Starovoitov static int map_lookup_elem(union bpf_attr *attr) 735db20fd2bSAlexei Starovoitov { 736535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 737535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 738db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 739db20fd2bSAlexei Starovoitov struct bpf_map *map; 7408ebe667cSAlexei Starovoitov void *key, *value, *ptr; 74115a07b33SAlexei Starovoitov u32 value_size; 742592867bfSDaniel Borkmann struct fd f; 743db20fd2bSAlexei Starovoitov int err; 744db20fd2bSAlexei Starovoitov 745db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 746db20fd2bSAlexei Starovoitov return -EINVAL; 747db20fd2bSAlexei Starovoitov 74896049f3aSAlexei Starovoitov if (attr->flags & ~BPF_F_LOCK) 74996049f3aSAlexei Starovoitov return -EINVAL; 75096049f3aSAlexei Starovoitov 751592867bfSDaniel Borkmann f = fdget(ufd); 752c2101297SDaniel Borkmann map = __bpf_map_get(f); 753db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 754db20fd2bSAlexei Starovoitov return PTR_ERR(map); 75587df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 7566e71b04aSChenbo Feng err = -EPERM; 7576e71b04aSChenbo Feng goto err_put; 7586e71b04aSChenbo Feng } 7596e71b04aSChenbo Feng 76096049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 76196049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 76296049f3aSAlexei Starovoitov err = -EINVAL; 76396049f3aSAlexei Starovoitov goto err_put; 76496049f3aSAlexei Starovoitov } 76596049f3aSAlexei Starovoitov 766c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 767e4448ed8SAl Viro if (IS_ERR(key)) { 768e4448ed8SAl Viro err = PTR_ERR(key); 769db20fd2bSAlexei Starovoitov goto err_put; 770e4448ed8SAl Viro } 771db20fd2bSAlexei Starovoitov 77215a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 7738f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 774b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 775b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 77615a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 77714dc6f04SMartin KaFai Lau else if (IS_FD_MAP(map)) 77814dc6f04SMartin KaFai Lau value_size = sizeof(u32); 77915a07b33SAlexei Starovoitov else 78015a07b33SAlexei Starovoitov value_size = map->value_size; 78115a07b33SAlexei Starovoitov 7828ebe667cSAlexei Starovoitov err = -ENOMEM; 78315a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 784db20fd2bSAlexei Starovoitov if (!value) 7858ebe667cSAlexei Starovoitov goto free_key; 7868ebe667cSAlexei Starovoitov 787a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 788a3884572SJakub Kicinski err = bpf_map_offload_lookup_elem(map, key, value); 7897c4cd051SMartin KaFai Lau goto done; 7907c4cd051SMartin KaFai Lau } 7917c4cd051SMartin KaFai Lau 7927c4cd051SMartin KaFai Lau preempt_disable(); 7937c4cd051SMartin KaFai Lau this_cpu_inc(bpf_prog_active); 7947c4cd051SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 7958f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 79615a07b33SAlexei Starovoitov err = bpf_percpu_hash_copy(map, key, value); 79715a07b33SAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 79815a07b33SAlexei Starovoitov err = bpf_percpu_array_copy(map, key, value); 799b741f163SRoman Gushchin } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 800b741f163SRoman Gushchin err = bpf_percpu_cgroup_storage_copy(map, key, value); 801557c0c6eSAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 802557c0c6eSAlexei Starovoitov err = bpf_stackmap_copy(map, key, value); 80314dc6f04SMartin KaFai Lau } else if (IS_FD_ARRAY(map)) { 80414dc6f04SMartin KaFai Lau err = bpf_fd_array_map_lookup_elem(map, key, value); 80514dc6f04SMartin KaFai Lau } else if (IS_FD_HASH(map)) { 80614dc6f04SMartin KaFai Lau err = bpf_fd_htab_map_lookup_elem(map, key, value); 8075dc4c4b7SMartin KaFai Lau } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 8085dc4c4b7SMartin KaFai Lau err = bpf_fd_reuseport_array_lookup_elem(map, key, value); 809f1a2e44aSMauricio Vasquez B } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 810f1a2e44aSMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 811f1a2e44aSMauricio Vasquez B err = map->ops->map_peek_elem(map, value); 81215a07b33SAlexei Starovoitov } else { 8138ebe667cSAlexei Starovoitov rcu_read_lock(); 814c6110222SDaniel Borkmann if (map->ops->map_lookup_elem_sys_only) 815c6110222SDaniel Borkmann ptr = map->ops->map_lookup_elem_sys_only(map, key); 816c6110222SDaniel Borkmann else 8178ebe667cSAlexei Starovoitov ptr = map->ops->map_lookup_elem(map, key); 818509db283SPrashant Bhole if (IS_ERR(ptr)) { 819509db283SPrashant Bhole err = PTR_ERR(ptr); 820509db283SPrashant Bhole } else if (!ptr) { 821509db283SPrashant Bhole err = -ENOENT; 822509db283SPrashant Bhole } else { 823509db283SPrashant Bhole err = 0; 82496049f3aSAlexei Starovoitov if (attr->flags & BPF_F_LOCK) 82596049f3aSAlexei Starovoitov /* lock 'ptr' and copy everything but lock */ 82696049f3aSAlexei Starovoitov copy_map_value_locked(map, value, ptr, true); 82796049f3aSAlexei Starovoitov else 828d83525caSAlexei Starovoitov copy_map_value(map, value, ptr); 82996049f3aSAlexei Starovoitov /* mask lock, since value wasn't zero inited */ 83096049f3aSAlexei Starovoitov check_and_init_map_lock(map, value); 831509db283SPrashant Bhole } 8328ebe667cSAlexei Starovoitov rcu_read_unlock(); 83315a07b33SAlexei Starovoitov } 8347c4cd051SMartin KaFai Lau this_cpu_dec(bpf_prog_active); 8357c4cd051SMartin KaFai Lau preempt_enable(); 8368ebe667cSAlexei Starovoitov 8377c4cd051SMartin KaFai Lau done: 83815a07b33SAlexei Starovoitov if (err) 8398ebe667cSAlexei Starovoitov goto free_value; 840db20fd2bSAlexei Starovoitov 841db20fd2bSAlexei Starovoitov err = -EFAULT; 84215a07b33SAlexei Starovoitov if (copy_to_user(uvalue, value, value_size) != 0) 8438ebe667cSAlexei Starovoitov goto free_value; 844db20fd2bSAlexei Starovoitov 845db20fd2bSAlexei Starovoitov err = 0; 846db20fd2bSAlexei Starovoitov 8478ebe667cSAlexei Starovoitov free_value: 8488ebe667cSAlexei Starovoitov kfree(value); 849db20fd2bSAlexei Starovoitov free_key: 850db20fd2bSAlexei Starovoitov kfree(key); 851db20fd2bSAlexei Starovoitov err_put: 852db20fd2bSAlexei Starovoitov fdput(f); 853db20fd2bSAlexei Starovoitov return err; 854db20fd2bSAlexei Starovoitov } 855db20fd2bSAlexei Starovoitov 8561ae80cf3SDaniel Colascione static void maybe_wait_bpf_programs(struct bpf_map *map) 8571ae80cf3SDaniel Colascione { 8581ae80cf3SDaniel Colascione /* Wait for any running BPF programs to complete so that 8591ae80cf3SDaniel Colascione * userspace, when we return to it, knows that all programs 8601ae80cf3SDaniel Colascione * that could be running use the new map value. 8611ae80cf3SDaniel Colascione */ 8621ae80cf3SDaniel Colascione if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS || 8631ae80cf3SDaniel Colascione map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 8641ae80cf3SDaniel Colascione synchronize_rcu(); 8651ae80cf3SDaniel Colascione } 8661ae80cf3SDaniel Colascione 8673274f520SAlexei Starovoitov #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 868db20fd2bSAlexei Starovoitov 869db20fd2bSAlexei Starovoitov static int map_update_elem(union bpf_attr *attr) 870db20fd2bSAlexei Starovoitov { 871535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 872535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 873db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 874db20fd2bSAlexei Starovoitov struct bpf_map *map; 875db20fd2bSAlexei Starovoitov void *key, *value; 87615a07b33SAlexei Starovoitov u32 value_size; 877592867bfSDaniel Borkmann struct fd f; 878db20fd2bSAlexei Starovoitov int err; 879db20fd2bSAlexei Starovoitov 880db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 881db20fd2bSAlexei Starovoitov return -EINVAL; 882db20fd2bSAlexei Starovoitov 883592867bfSDaniel Borkmann f = fdget(ufd); 884c2101297SDaniel Borkmann map = __bpf_map_get(f); 885db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 886db20fd2bSAlexei Starovoitov return PTR_ERR(map); 88787df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 8886e71b04aSChenbo Feng err = -EPERM; 8896e71b04aSChenbo Feng goto err_put; 8906e71b04aSChenbo Feng } 8916e71b04aSChenbo Feng 89296049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) && 89396049f3aSAlexei Starovoitov !map_value_has_spin_lock(map)) { 89496049f3aSAlexei Starovoitov err = -EINVAL; 89596049f3aSAlexei Starovoitov goto err_put; 89696049f3aSAlexei Starovoitov } 89796049f3aSAlexei Starovoitov 898c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 899e4448ed8SAl Viro if (IS_ERR(key)) { 900e4448ed8SAl Viro err = PTR_ERR(key); 901db20fd2bSAlexei Starovoitov goto err_put; 902e4448ed8SAl Viro } 903db20fd2bSAlexei Starovoitov 90415a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 9058f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 906b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || 907b741f163SRoman Gushchin map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 90815a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 90915a07b33SAlexei Starovoitov else 91015a07b33SAlexei Starovoitov value_size = map->value_size; 91115a07b33SAlexei Starovoitov 912db20fd2bSAlexei Starovoitov err = -ENOMEM; 91315a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 914db20fd2bSAlexei Starovoitov if (!value) 915db20fd2bSAlexei Starovoitov goto free_key; 916db20fd2bSAlexei Starovoitov 917db20fd2bSAlexei Starovoitov err = -EFAULT; 91815a07b33SAlexei Starovoitov if (copy_from_user(value, uvalue, value_size) != 0) 919db20fd2bSAlexei Starovoitov goto free_value; 920db20fd2bSAlexei Starovoitov 9216710e112SJesper Dangaard Brouer /* Need to create a kthread, thus must support schedule */ 922a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 923a3884572SJakub Kicinski err = bpf_map_offload_update_elem(map, key, value, attr->flags); 924a3884572SJakub Kicinski goto out; 92599ba2b5aSJohn Fastabend } else if (map->map_type == BPF_MAP_TYPE_CPUMAP || 92699ba2b5aSJohn Fastabend map->map_type == BPF_MAP_TYPE_SOCKHASH || 92799ba2b5aSJohn Fastabend map->map_type == BPF_MAP_TYPE_SOCKMAP) { 9286710e112SJesper Dangaard Brouer err = map->ops->map_update_elem(map, key, value, attr->flags); 9296710e112SJesper Dangaard Brouer goto out; 9306710e112SJesper Dangaard Brouer } 9316710e112SJesper Dangaard Brouer 932b121d1e7SAlexei Starovoitov /* must increment bpf_prog_active to avoid kprobe+bpf triggering from 933b121d1e7SAlexei Starovoitov * inside bpf map update or delete otherwise deadlocks are possible 934b121d1e7SAlexei Starovoitov */ 935b121d1e7SAlexei Starovoitov preempt_disable(); 936b121d1e7SAlexei Starovoitov __this_cpu_inc(bpf_prog_active); 9378f844938SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 9388f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 93915a07b33SAlexei Starovoitov err = bpf_percpu_hash_update(map, key, value, attr->flags); 94015a07b33SAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 94115a07b33SAlexei Starovoitov err = bpf_percpu_array_update(map, key, value, attr->flags); 942b741f163SRoman Gushchin } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 943b741f163SRoman Gushchin err = bpf_percpu_cgroup_storage_update(map, key, value, 944b741f163SRoman Gushchin attr->flags); 9459c147b56SMickaël Salaün } else if (IS_FD_ARRAY(map)) { 946d056a788SDaniel Borkmann rcu_read_lock(); 947d056a788SDaniel Borkmann err = bpf_fd_array_map_update_elem(map, f.file, key, value, 948d056a788SDaniel Borkmann attr->flags); 949d056a788SDaniel Borkmann rcu_read_unlock(); 950bcc6b1b7SMartin KaFai Lau } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 951bcc6b1b7SMartin KaFai Lau rcu_read_lock(); 952bcc6b1b7SMartin KaFai Lau err = bpf_fd_htab_map_update_elem(map, f.file, key, value, 953bcc6b1b7SMartin KaFai Lau attr->flags); 954bcc6b1b7SMartin KaFai Lau rcu_read_unlock(); 9555dc4c4b7SMartin KaFai Lau } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { 9565dc4c4b7SMartin KaFai Lau /* rcu_read_lock() is not needed */ 9575dc4c4b7SMartin KaFai Lau err = bpf_fd_reuseport_array_update_elem(map, key, value, 9585dc4c4b7SMartin KaFai Lau attr->flags); 959f1a2e44aSMauricio Vasquez B } else if (map->map_type == BPF_MAP_TYPE_QUEUE || 960f1a2e44aSMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 961f1a2e44aSMauricio Vasquez B err = map->ops->map_push_elem(map, value, attr->flags); 96215a07b33SAlexei Starovoitov } else { 963db20fd2bSAlexei Starovoitov rcu_read_lock(); 9643274f520SAlexei Starovoitov err = map->ops->map_update_elem(map, key, value, attr->flags); 965db20fd2bSAlexei Starovoitov rcu_read_unlock(); 96615a07b33SAlexei Starovoitov } 967b121d1e7SAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 968b121d1e7SAlexei Starovoitov preempt_enable(); 9691ae80cf3SDaniel Colascione maybe_wait_bpf_programs(map); 9706710e112SJesper Dangaard Brouer out: 971db20fd2bSAlexei Starovoitov free_value: 972db20fd2bSAlexei Starovoitov kfree(value); 973db20fd2bSAlexei Starovoitov free_key: 974db20fd2bSAlexei Starovoitov kfree(key); 975db20fd2bSAlexei Starovoitov err_put: 976db20fd2bSAlexei Starovoitov fdput(f); 977db20fd2bSAlexei Starovoitov return err; 978db20fd2bSAlexei Starovoitov } 979db20fd2bSAlexei Starovoitov 980db20fd2bSAlexei Starovoitov #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 981db20fd2bSAlexei Starovoitov 982db20fd2bSAlexei Starovoitov static int map_delete_elem(union bpf_attr *attr) 983db20fd2bSAlexei Starovoitov { 984535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 985db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 986db20fd2bSAlexei Starovoitov struct bpf_map *map; 987592867bfSDaniel Borkmann struct fd f; 988db20fd2bSAlexei Starovoitov void *key; 989db20fd2bSAlexei Starovoitov int err; 990db20fd2bSAlexei Starovoitov 991db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 992db20fd2bSAlexei Starovoitov return -EINVAL; 993db20fd2bSAlexei Starovoitov 994592867bfSDaniel Borkmann f = fdget(ufd); 995c2101297SDaniel Borkmann map = __bpf_map_get(f); 996db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 997db20fd2bSAlexei Starovoitov return PTR_ERR(map); 99887df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 9996e71b04aSChenbo Feng err = -EPERM; 10006e71b04aSChenbo Feng goto err_put; 10016e71b04aSChenbo Feng } 10026e71b04aSChenbo Feng 1003c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1004e4448ed8SAl Viro if (IS_ERR(key)) { 1005e4448ed8SAl Viro err = PTR_ERR(key); 1006db20fd2bSAlexei Starovoitov goto err_put; 1007e4448ed8SAl Viro } 1008db20fd2bSAlexei Starovoitov 1009a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1010a3884572SJakub Kicinski err = bpf_map_offload_delete_elem(map, key); 1011a3884572SJakub Kicinski goto out; 1012a3884572SJakub Kicinski } 1013a3884572SJakub Kicinski 1014b121d1e7SAlexei Starovoitov preempt_disable(); 1015b121d1e7SAlexei Starovoitov __this_cpu_inc(bpf_prog_active); 1016db20fd2bSAlexei Starovoitov rcu_read_lock(); 1017db20fd2bSAlexei Starovoitov err = map->ops->map_delete_elem(map, key); 1018db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1019b121d1e7SAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 1020b121d1e7SAlexei Starovoitov preempt_enable(); 10211ae80cf3SDaniel Colascione maybe_wait_bpf_programs(map); 1022a3884572SJakub Kicinski out: 1023db20fd2bSAlexei Starovoitov kfree(key); 1024db20fd2bSAlexei Starovoitov err_put: 1025db20fd2bSAlexei Starovoitov fdput(f); 1026db20fd2bSAlexei Starovoitov return err; 1027db20fd2bSAlexei Starovoitov } 1028db20fd2bSAlexei Starovoitov 1029db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 1030db20fd2bSAlexei Starovoitov #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 1031db20fd2bSAlexei Starovoitov 1032db20fd2bSAlexei Starovoitov static int map_get_next_key(union bpf_attr *attr) 1033db20fd2bSAlexei Starovoitov { 1034535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 1035535e7b4bSMickaël Salaün void __user *unext_key = u64_to_user_ptr(attr->next_key); 1036db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 1037db20fd2bSAlexei Starovoitov struct bpf_map *map; 1038db20fd2bSAlexei Starovoitov void *key, *next_key; 1039592867bfSDaniel Borkmann struct fd f; 1040db20fd2bSAlexei Starovoitov int err; 1041db20fd2bSAlexei Starovoitov 1042db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 1043db20fd2bSAlexei Starovoitov return -EINVAL; 1044db20fd2bSAlexei Starovoitov 1045592867bfSDaniel Borkmann f = fdget(ufd); 1046c2101297SDaniel Borkmann map = __bpf_map_get(f); 1047db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 1048db20fd2bSAlexei Starovoitov return PTR_ERR(map); 104987df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { 10506e71b04aSChenbo Feng err = -EPERM; 10516e71b04aSChenbo Feng goto err_put; 10526e71b04aSChenbo Feng } 10536e71b04aSChenbo Feng 10548fe45924STeng Qin if (ukey) { 1055c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1056e4448ed8SAl Viro if (IS_ERR(key)) { 1057e4448ed8SAl Viro err = PTR_ERR(key); 1058db20fd2bSAlexei Starovoitov goto err_put; 1059e4448ed8SAl Viro } 10608fe45924STeng Qin } else { 10618fe45924STeng Qin key = NULL; 10628fe45924STeng Qin } 1063db20fd2bSAlexei Starovoitov 1064db20fd2bSAlexei Starovoitov err = -ENOMEM; 1065db20fd2bSAlexei Starovoitov next_key = kmalloc(map->key_size, GFP_USER); 1066db20fd2bSAlexei Starovoitov if (!next_key) 1067db20fd2bSAlexei Starovoitov goto free_key; 1068db20fd2bSAlexei Starovoitov 1069a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 1070a3884572SJakub Kicinski err = bpf_map_offload_get_next_key(map, key, next_key); 1071a3884572SJakub Kicinski goto out; 1072a3884572SJakub Kicinski } 1073a3884572SJakub Kicinski 1074db20fd2bSAlexei Starovoitov rcu_read_lock(); 1075db20fd2bSAlexei Starovoitov err = map->ops->map_get_next_key(map, key, next_key); 1076db20fd2bSAlexei Starovoitov rcu_read_unlock(); 1077a3884572SJakub Kicinski out: 1078db20fd2bSAlexei Starovoitov if (err) 1079db20fd2bSAlexei Starovoitov goto free_next_key; 1080db20fd2bSAlexei Starovoitov 1081db20fd2bSAlexei Starovoitov err = -EFAULT; 1082db20fd2bSAlexei Starovoitov if (copy_to_user(unext_key, next_key, map->key_size) != 0) 1083db20fd2bSAlexei Starovoitov goto free_next_key; 1084db20fd2bSAlexei Starovoitov 1085db20fd2bSAlexei Starovoitov err = 0; 1086db20fd2bSAlexei Starovoitov 1087db20fd2bSAlexei Starovoitov free_next_key: 1088db20fd2bSAlexei Starovoitov kfree(next_key); 1089db20fd2bSAlexei Starovoitov free_key: 1090db20fd2bSAlexei Starovoitov kfree(key); 1091db20fd2bSAlexei Starovoitov err_put: 1092db20fd2bSAlexei Starovoitov fdput(f); 1093db20fd2bSAlexei Starovoitov return err; 1094db20fd2bSAlexei Starovoitov } 1095db20fd2bSAlexei Starovoitov 1096bd513cd0SMauricio Vasquez B #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value 1097bd513cd0SMauricio Vasquez B 1098bd513cd0SMauricio Vasquez B static int map_lookup_and_delete_elem(union bpf_attr *attr) 1099bd513cd0SMauricio Vasquez B { 1100bd513cd0SMauricio Vasquez B void __user *ukey = u64_to_user_ptr(attr->key); 1101bd513cd0SMauricio Vasquez B void __user *uvalue = u64_to_user_ptr(attr->value); 1102bd513cd0SMauricio Vasquez B int ufd = attr->map_fd; 1103bd513cd0SMauricio Vasquez B struct bpf_map *map; 1104540fefc0SAlexei Starovoitov void *key, *value; 1105bd513cd0SMauricio Vasquez B u32 value_size; 1106bd513cd0SMauricio Vasquez B struct fd f; 1107bd513cd0SMauricio Vasquez B int err; 1108bd513cd0SMauricio Vasquez B 1109bd513cd0SMauricio Vasquez B if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) 1110bd513cd0SMauricio Vasquez B return -EINVAL; 1111bd513cd0SMauricio Vasquez B 1112bd513cd0SMauricio Vasquez B f = fdget(ufd); 1113bd513cd0SMauricio Vasquez B map = __bpf_map_get(f); 1114bd513cd0SMauricio Vasquez B if (IS_ERR(map)) 1115bd513cd0SMauricio Vasquez B return PTR_ERR(map); 111687df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { 1117bd513cd0SMauricio Vasquez B err = -EPERM; 1118bd513cd0SMauricio Vasquez B goto err_put; 1119bd513cd0SMauricio Vasquez B } 1120bd513cd0SMauricio Vasquez B 1121bd513cd0SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size); 1122bd513cd0SMauricio Vasquez B if (IS_ERR(key)) { 1123bd513cd0SMauricio Vasquez B err = PTR_ERR(key); 1124bd513cd0SMauricio Vasquez B goto err_put; 1125bd513cd0SMauricio Vasquez B } 1126bd513cd0SMauricio Vasquez B 1127bd513cd0SMauricio Vasquez B value_size = map->value_size; 1128bd513cd0SMauricio Vasquez B 1129bd513cd0SMauricio Vasquez B err = -ENOMEM; 1130bd513cd0SMauricio Vasquez B value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 1131bd513cd0SMauricio Vasquez B if (!value) 1132bd513cd0SMauricio Vasquez B goto free_key; 1133bd513cd0SMauricio Vasquez B 1134bd513cd0SMauricio Vasquez B if (map->map_type == BPF_MAP_TYPE_QUEUE || 1135bd513cd0SMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) { 1136bd513cd0SMauricio Vasquez B err = map->ops->map_pop_elem(map, value); 1137bd513cd0SMauricio Vasquez B } else { 1138bd513cd0SMauricio Vasquez B err = -ENOTSUPP; 1139bd513cd0SMauricio Vasquez B } 1140bd513cd0SMauricio Vasquez B 1141bd513cd0SMauricio Vasquez B if (err) 1142bd513cd0SMauricio Vasquez B goto free_value; 1143bd513cd0SMauricio Vasquez B 1144bd513cd0SMauricio Vasquez B if (copy_to_user(uvalue, value, value_size) != 0) 1145bd513cd0SMauricio Vasquez B goto free_value; 1146bd513cd0SMauricio Vasquez B 1147bd513cd0SMauricio Vasquez B err = 0; 1148bd513cd0SMauricio Vasquez B 1149bd513cd0SMauricio Vasquez B free_value: 1150bd513cd0SMauricio Vasquez B kfree(value); 1151bd513cd0SMauricio Vasquez B free_key: 1152bd513cd0SMauricio Vasquez B kfree(key); 1153bd513cd0SMauricio Vasquez B err_put: 1154bd513cd0SMauricio Vasquez B fdput(f); 1155bd513cd0SMauricio Vasquez B return err; 1156bd513cd0SMauricio Vasquez B } 1157bd513cd0SMauricio Vasquez B 115887df15deSDaniel Borkmann #define BPF_MAP_FREEZE_LAST_FIELD map_fd 115987df15deSDaniel Borkmann 116087df15deSDaniel Borkmann static int map_freeze(const union bpf_attr *attr) 116187df15deSDaniel Borkmann { 116287df15deSDaniel Borkmann int err = 0, ufd = attr->map_fd; 116387df15deSDaniel Borkmann struct bpf_map *map; 116487df15deSDaniel Borkmann struct fd f; 116587df15deSDaniel Borkmann 116687df15deSDaniel Borkmann if (CHECK_ATTR(BPF_MAP_FREEZE)) 116787df15deSDaniel Borkmann return -EINVAL; 116887df15deSDaniel Borkmann 116987df15deSDaniel Borkmann f = fdget(ufd); 117087df15deSDaniel Borkmann map = __bpf_map_get(f); 117187df15deSDaniel Borkmann if (IS_ERR(map)) 117287df15deSDaniel Borkmann return PTR_ERR(map); 117387df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) { 117487df15deSDaniel Borkmann err = -EBUSY; 117587df15deSDaniel Borkmann goto err_put; 117687df15deSDaniel Borkmann } 117787df15deSDaniel Borkmann if (!capable(CAP_SYS_ADMIN)) { 117887df15deSDaniel Borkmann err = -EPERM; 117987df15deSDaniel Borkmann goto err_put; 118087df15deSDaniel Borkmann } 118187df15deSDaniel Borkmann 118287df15deSDaniel Borkmann WRITE_ONCE(map->frozen, true); 118387df15deSDaniel Borkmann err_put: 118487df15deSDaniel Borkmann fdput(f); 118587df15deSDaniel Borkmann return err; 118687df15deSDaniel Borkmann } 118787df15deSDaniel Borkmann 11887de16e3aSJakub Kicinski static const struct bpf_prog_ops * const bpf_prog_types[] = { 11897de16e3aSJakub Kicinski #define BPF_PROG_TYPE(_id, _name) \ 11907de16e3aSJakub Kicinski [_id] = & _name ## _prog_ops, 11917de16e3aSJakub Kicinski #define BPF_MAP_TYPE(_id, _ops) 11927de16e3aSJakub Kicinski #include <linux/bpf_types.h> 11937de16e3aSJakub Kicinski #undef BPF_PROG_TYPE 11947de16e3aSJakub Kicinski #undef BPF_MAP_TYPE 11957de16e3aSJakub Kicinski }; 11967de16e3aSJakub Kicinski 119709756af4SAlexei Starovoitov static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 119809756af4SAlexei Starovoitov { 1199d0f1a451SDaniel Borkmann const struct bpf_prog_ops *ops; 1200d0f1a451SDaniel Borkmann 1201d0f1a451SDaniel Borkmann if (type >= ARRAY_SIZE(bpf_prog_types)) 1202d0f1a451SDaniel Borkmann return -EINVAL; 1203d0f1a451SDaniel Borkmann type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types)); 1204d0f1a451SDaniel Borkmann ops = bpf_prog_types[type]; 1205d0f1a451SDaniel Borkmann if (!ops) 1206be9370a7SJohannes Berg return -EINVAL; 120709756af4SAlexei Starovoitov 1208ab3f0063SJakub Kicinski if (!bpf_prog_is_dev_bound(prog->aux)) 1209d0f1a451SDaniel Borkmann prog->aux->ops = ops; 1210ab3f0063SJakub Kicinski else 1211ab3f0063SJakub Kicinski prog->aux->ops = &bpf_offload_prog_ops; 121224701eceSDaniel Borkmann prog->type = type; 121309756af4SAlexei Starovoitov return 0; 121409756af4SAlexei Starovoitov } 121509756af4SAlexei Starovoitov 121609756af4SAlexei Starovoitov /* drop refcnt on maps used by eBPF program and free auxilary data */ 121709756af4SAlexei Starovoitov static void free_used_maps(struct bpf_prog_aux *aux) 121809756af4SAlexei Starovoitov { 12198bad74f9SRoman Gushchin enum bpf_cgroup_storage_type stype; 122009756af4SAlexei Starovoitov int i; 122109756af4SAlexei Starovoitov 12228bad74f9SRoman Gushchin for_each_cgroup_storage_type(stype) { 12238bad74f9SRoman Gushchin if (!aux->cgroup_storage[stype]) 12248bad74f9SRoman Gushchin continue; 12258bad74f9SRoman Gushchin bpf_cgroup_storage_release(aux->prog, 12268bad74f9SRoman Gushchin aux->cgroup_storage[stype]); 12278bad74f9SRoman Gushchin } 1228de9cbbaaSRoman Gushchin 122909756af4SAlexei Starovoitov for (i = 0; i < aux->used_map_cnt; i++) 123009756af4SAlexei Starovoitov bpf_map_put(aux->used_maps[i]); 123109756af4SAlexei Starovoitov 123209756af4SAlexei Starovoitov kfree(aux->used_maps); 123309756af4SAlexei Starovoitov } 123409756af4SAlexei Starovoitov 12355ccb071eSDaniel Borkmann int __bpf_prog_charge(struct user_struct *user, u32 pages) 12365ccb071eSDaniel Borkmann { 12375ccb071eSDaniel Borkmann unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 12385ccb071eSDaniel Borkmann unsigned long user_bufs; 12395ccb071eSDaniel Borkmann 12405ccb071eSDaniel Borkmann if (user) { 12415ccb071eSDaniel Borkmann user_bufs = atomic_long_add_return(pages, &user->locked_vm); 12425ccb071eSDaniel Borkmann if (user_bufs > memlock_limit) { 12435ccb071eSDaniel Borkmann atomic_long_sub(pages, &user->locked_vm); 12445ccb071eSDaniel Borkmann return -EPERM; 12455ccb071eSDaniel Borkmann } 12465ccb071eSDaniel Borkmann } 12475ccb071eSDaniel Borkmann 12485ccb071eSDaniel Borkmann return 0; 12495ccb071eSDaniel Borkmann } 12505ccb071eSDaniel Borkmann 12515ccb071eSDaniel Borkmann void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 12525ccb071eSDaniel Borkmann { 12535ccb071eSDaniel Borkmann if (user) 12545ccb071eSDaniel Borkmann atomic_long_sub(pages, &user->locked_vm); 12555ccb071eSDaniel Borkmann } 12565ccb071eSDaniel Borkmann 1257aaac3ba9SAlexei Starovoitov static int bpf_prog_charge_memlock(struct bpf_prog *prog) 1258aaac3ba9SAlexei Starovoitov { 1259aaac3ba9SAlexei Starovoitov struct user_struct *user = get_current_user(); 12605ccb071eSDaniel Borkmann int ret; 1261aaac3ba9SAlexei Starovoitov 12625ccb071eSDaniel Borkmann ret = __bpf_prog_charge(user, prog->pages); 12635ccb071eSDaniel Borkmann if (ret) { 1264aaac3ba9SAlexei Starovoitov free_uid(user); 12655ccb071eSDaniel Borkmann return ret; 1266aaac3ba9SAlexei Starovoitov } 12675ccb071eSDaniel Borkmann 1268aaac3ba9SAlexei Starovoitov prog->aux->user = user; 1269aaac3ba9SAlexei Starovoitov return 0; 1270aaac3ba9SAlexei Starovoitov } 1271aaac3ba9SAlexei Starovoitov 1272aaac3ba9SAlexei Starovoitov static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 1273aaac3ba9SAlexei Starovoitov { 1274aaac3ba9SAlexei Starovoitov struct user_struct *user = prog->aux->user; 1275aaac3ba9SAlexei Starovoitov 12765ccb071eSDaniel Borkmann __bpf_prog_uncharge(user, prog->pages); 1277aaac3ba9SAlexei Starovoitov free_uid(user); 1278aaac3ba9SAlexei Starovoitov } 1279aaac3ba9SAlexei Starovoitov 1280dc4bb0e2SMartin KaFai Lau static int bpf_prog_alloc_id(struct bpf_prog *prog) 1281dc4bb0e2SMartin KaFai Lau { 1282dc4bb0e2SMartin KaFai Lau int id; 1283dc4bb0e2SMartin KaFai Lau 1284b76354cdSShaohua Li idr_preload(GFP_KERNEL); 1285dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1286dc4bb0e2SMartin KaFai Lau id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 1287dc4bb0e2SMartin KaFai Lau if (id > 0) 1288dc4bb0e2SMartin KaFai Lau prog->aux->id = id; 1289dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1290b76354cdSShaohua Li idr_preload_end(); 1291dc4bb0e2SMartin KaFai Lau 1292dc4bb0e2SMartin KaFai Lau /* id is in [1, INT_MAX) */ 1293dc4bb0e2SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 1294dc4bb0e2SMartin KaFai Lau return -ENOSPC; 1295dc4bb0e2SMartin KaFai Lau 1296dc4bb0e2SMartin KaFai Lau return id > 0 ? 0 : id; 1297dc4bb0e2SMartin KaFai Lau } 1298dc4bb0e2SMartin KaFai Lau 1299ad8ad79fSJakub Kicinski void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) 1300dc4bb0e2SMartin KaFai Lau { 1301ad8ad79fSJakub Kicinski /* cBPF to eBPF migrations are currently not in the idr store. 1302ad8ad79fSJakub Kicinski * Offloaded programs are removed from the store when their device 1303ad8ad79fSJakub Kicinski * disappears - even if someone grabs an fd to them they are unusable, 1304ad8ad79fSJakub Kicinski * simply waiting for refcnt to drop to be freed. 1305ad8ad79fSJakub Kicinski */ 1306dc4bb0e2SMartin KaFai Lau if (!prog->aux->id) 1307dc4bb0e2SMartin KaFai Lau return; 1308dc4bb0e2SMartin KaFai Lau 1309b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1310dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1311b16d9aa4SMartin KaFai Lau else 1312b16d9aa4SMartin KaFai Lau __acquire(&prog_idr_lock); 1313b16d9aa4SMartin KaFai Lau 1314dc4bb0e2SMartin KaFai Lau idr_remove(&prog_idr, prog->aux->id); 1315ad8ad79fSJakub Kicinski prog->aux->id = 0; 1316b16d9aa4SMartin KaFai Lau 1317b16d9aa4SMartin KaFai Lau if (do_idr_lock) 1318dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1319b16d9aa4SMartin KaFai Lau else 1320b16d9aa4SMartin KaFai Lau __release(&prog_idr_lock); 1321dc4bb0e2SMartin KaFai Lau } 1322dc4bb0e2SMartin KaFai Lau 13231aacde3dSDaniel Borkmann static void __bpf_prog_put_rcu(struct rcu_head *rcu) 1324abf2e7d6SAlexei Starovoitov { 1325abf2e7d6SAlexei Starovoitov struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 1326abf2e7d6SAlexei Starovoitov 1327abf2e7d6SAlexei Starovoitov free_used_maps(aux); 1328aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(aux->prog); 1329afdb09c7SChenbo Feng security_bpf_prog_free(aux); 1330abf2e7d6SAlexei Starovoitov bpf_prog_free(aux->prog); 1331abf2e7d6SAlexei Starovoitov } 1332abf2e7d6SAlexei Starovoitov 1333b16d9aa4SMartin KaFai Lau static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 133409756af4SAlexei Starovoitov { 1335a67edbf4SDaniel Borkmann if (atomic_dec_and_test(&prog->aux->refcnt)) { 13366ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0); 133734ad5580SMartin KaFai Lau /* bpf_prog_free_id() must be called first */ 1338b16d9aa4SMartin KaFai Lau bpf_prog_free_id(prog, do_idr_lock); 13397d1982b4SDaniel Borkmann bpf_prog_kallsyms_del_all(prog); 1340838e9690SYonghong Song btf_put(prog->aux->btf); 1341ba64e7d8SYonghong Song kvfree(prog->aux->func_info); 1342c454a46bSMartin KaFai Lau bpf_prog_free_linfo(prog); 13434f74d809SDaniel Borkmann 13441aacde3dSDaniel Borkmann call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 134509756af4SAlexei Starovoitov } 1346a67edbf4SDaniel Borkmann } 1347b16d9aa4SMartin KaFai Lau 1348b16d9aa4SMartin KaFai Lau void bpf_prog_put(struct bpf_prog *prog) 1349b16d9aa4SMartin KaFai Lau { 1350b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, true); 1351b16d9aa4SMartin KaFai Lau } 1352e2e9b654SDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_put); 135309756af4SAlexei Starovoitov 135409756af4SAlexei Starovoitov static int bpf_prog_release(struct inode *inode, struct file *filp) 135509756af4SAlexei Starovoitov { 135609756af4SAlexei Starovoitov struct bpf_prog *prog = filp->private_data; 135709756af4SAlexei Starovoitov 13581aacde3dSDaniel Borkmann bpf_prog_put(prog); 135909756af4SAlexei Starovoitov return 0; 136009756af4SAlexei Starovoitov } 136109756af4SAlexei Starovoitov 1362492ecee8SAlexei Starovoitov static void bpf_prog_get_stats(const struct bpf_prog *prog, 1363492ecee8SAlexei Starovoitov struct bpf_prog_stats *stats) 1364492ecee8SAlexei Starovoitov { 1365492ecee8SAlexei Starovoitov u64 nsecs = 0, cnt = 0; 1366492ecee8SAlexei Starovoitov int cpu; 1367492ecee8SAlexei Starovoitov 1368492ecee8SAlexei Starovoitov for_each_possible_cpu(cpu) { 1369492ecee8SAlexei Starovoitov const struct bpf_prog_stats *st; 1370492ecee8SAlexei Starovoitov unsigned int start; 1371492ecee8SAlexei Starovoitov u64 tnsecs, tcnt; 1372492ecee8SAlexei Starovoitov 1373492ecee8SAlexei Starovoitov st = per_cpu_ptr(prog->aux->stats, cpu); 1374492ecee8SAlexei Starovoitov do { 1375492ecee8SAlexei Starovoitov start = u64_stats_fetch_begin_irq(&st->syncp); 1376492ecee8SAlexei Starovoitov tnsecs = st->nsecs; 1377492ecee8SAlexei Starovoitov tcnt = st->cnt; 1378492ecee8SAlexei Starovoitov } while (u64_stats_fetch_retry_irq(&st->syncp, start)); 1379492ecee8SAlexei Starovoitov nsecs += tnsecs; 1380492ecee8SAlexei Starovoitov cnt += tcnt; 1381492ecee8SAlexei Starovoitov } 1382492ecee8SAlexei Starovoitov stats->nsecs = nsecs; 1383492ecee8SAlexei Starovoitov stats->cnt = cnt; 1384492ecee8SAlexei Starovoitov } 1385492ecee8SAlexei Starovoitov 13867bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 13877bd509e3SDaniel Borkmann static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 13887bd509e3SDaniel Borkmann { 13897bd509e3SDaniel Borkmann const struct bpf_prog *prog = filp->private_data; 1390f1f7714eSDaniel Borkmann char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 1391492ecee8SAlexei Starovoitov struct bpf_prog_stats stats; 13927bd509e3SDaniel Borkmann 1393492ecee8SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 1394f1f7714eSDaniel Borkmann bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 13957bd509e3SDaniel Borkmann seq_printf(m, 13967bd509e3SDaniel Borkmann "prog_type:\t%u\n" 13977bd509e3SDaniel Borkmann "prog_jited:\t%u\n" 1398f1f7714eSDaniel Borkmann "prog_tag:\t%s\n" 13994316b409SDaniel Borkmann "memlock:\t%llu\n" 1400492ecee8SAlexei Starovoitov "prog_id:\t%u\n" 1401492ecee8SAlexei Starovoitov "run_time_ns:\t%llu\n" 1402492ecee8SAlexei Starovoitov "run_cnt:\t%llu\n", 14037bd509e3SDaniel Borkmann prog->type, 14047bd509e3SDaniel Borkmann prog->jited, 1405f1f7714eSDaniel Borkmann prog_tag, 14064316b409SDaniel Borkmann prog->pages * 1ULL << PAGE_SHIFT, 1407492ecee8SAlexei Starovoitov prog->aux->id, 1408492ecee8SAlexei Starovoitov stats.nsecs, 1409492ecee8SAlexei Starovoitov stats.cnt); 14107bd509e3SDaniel Borkmann } 14117bd509e3SDaniel Borkmann #endif 14127bd509e3SDaniel Borkmann 1413f66e448cSChenbo Feng const struct file_operations bpf_prog_fops = { 14147bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 14157bd509e3SDaniel Borkmann .show_fdinfo = bpf_prog_show_fdinfo, 14167bd509e3SDaniel Borkmann #endif 141709756af4SAlexei Starovoitov .release = bpf_prog_release, 14186e71b04aSChenbo Feng .read = bpf_dummy_read, 14196e71b04aSChenbo Feng .write = bpf_dummy_write, 142009756af4SAlexei Starovoitov }; 142109756af4SAlexei Starovoitov 1422b2197755SDaniel Borkmann int bpf_prog_new_fd(struct bpf_prog *prog) 1423aa79781bSDaniel Borkmann { 1424afdb09c7SChenbo Feng int ret; 1425afdb09c7SChenbo Feng 1426afdb09c7SChenbo Feng ret = security_bpf_prog(prog); 1427afdb09c7SChenbo Feng if (ret < 0) 1428afdb09c7SChenbo Feng return ret; 1429afdb09c7SChenbo Feng 1430aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 1431aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC); 1432aa79781bSDaniel Borkmann } 1433aa79781bSDaniel Borkmann 1434113214beSDaniel Borkmann static struct bpf_prog *____bpf_prog_get(struct fd f) 143509756af4SAlexei Starovoitov { 143609756af4SAlexei Starovoitov if (!f.file) 143709756af4SAlexei Starovoitov return ERR_PTR(-EBADF); 143809756af4SAlexei Starovoitov if (f.file->f_op != &bpf_prog_fops) { 143909756af4SAlexei Starovoitov fdput(f); 144009756af4SAlexei Starovoitov return ERR_PTR(-EINVAL); 144109756af4SAlexei Starovoitov } 144209756af4SAlexei Starovoitov 1443c2101297SDaniel Borkmann return f.file->private_data; 144409756af4SAlexei Starovoitov } 144509756af4SAlexei Starovoitov 144659d3656dSBrenden Blanco struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i) 144792117d84SAlexei Starovoitov { 144859d3656dSBrenden Blanco if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) { 144959d3656dSBrenden Blanco atomic_sub(i, &prog->aux->refcnt); 145092117d84SAlexei Starovoitov return ERR_PTR(-EBUSY); 145192117d84SAlexei Starovoitov } 145292117d84SAlexei Starovoitov return prog; 145392117d84SAlexei Starovoitov } 145459d3656dSBrenden Blanco EXPORT_SYMBOL_GPL(bpf_prog_add); 145559d3656dSBrenden Blanco 1456c540594fSDaniel Borkmann void bpf_prog_sub(struct bpf_prog *prog, int i) 1457c540594fSDaniel Borkmann { 1458c540594fSDaniel Borkmann /* Only to be used for undoing previous bpf_prog_add() in some 1459c540594fSDaniel Borkmann * error path. We still know that another entity in our call 1460c540594fSDaniel Borkmann * path holds a reference to the program, thus atomic_sub() can 1461c540594fSDaniel Borkmann * be safely used in such cases! 1462c540594fSDaniel Borkmann */ 1463c540594fSDaniel Borkmann WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0); 1464c540594fSDaniel Borkmann } 1465c540594fSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_sub); 1466c540594fSDaniel Borkmann 146759d3656dSBrenden Blanco struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog) 146859d3656dSBrenden Blanco { 146959d3656dSBrenden Blanco return bpf_prog_add(prog, 1); 147059d3656dSBrenden Blanco } 147197bc402dSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_inc); 147292117d84SAlexei Starovoitov 1473b16d9aa4SMartin KaFai Lau /* prog_idr_lock should have been held */ 1474a6f6df69SJohn Fastabend struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 1475b16d9aa4SMartin KaFai Lau { 1476b16d9aa4SMartin KaFai Lau int refold; 1477b16d9aa4SMartin KaFai Lau 1478bfc18e38SMark Rutland refold = atomic_fetch_add_unless(&prog->aux->refcnt, 1, 0); 1479b16d9aa4SMartin KaFai Lau 1480b16d9aa4SMartin KaFai Lau if (refold >= BPF_MAX_REFCNT) { 1481b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, false); 1482b16d9aa4SMartin KaFai Lau return ERR_PTR(-EBUSY); 1483b16d9aa4SMartin KaFai Lau } 1484b16d9aa4SMartin KaFai Lau 1485b16d9aa4SMartin KaFai Lau if (!refold) 1486b16d9aa4SMartin KaFai Lau return ERR_PTR(-ENOENT); 1487b16d9aa4SMartin KaFai Lau 1488b16d9aa4SMartin KaFai Lau return prog; 1489b16d9aa4SMartin KaFai Lau } 1490a6f6df69SJohn Fastabend EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 1491b16d9aa4SMartin KaFai Lau 1492040ee692SAl Viro bool bpf_prog_get_ok(struct bpf_prog *prog, 1493288b3de5SJakub Kicinski enum bpf_prog_type *attach_type, bool attach_drv) 1494248f346fSJakub Kicinski { 1495288b3de5SJakub Kicinski /* not an attachment, just a refcount inc, always allow */ 1496288b3de5SJakub Kicinski if (!attach_type) 1497288b3de5SJakub Kicinski return true; 1498248f346fSJakub Kicinski 1499248f346fSJakub Kicinski if (prog->type != *attach_type) 1500248f346fSJakub Kicinski return false; 1501288b3de5SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv) 1502248f346fSJakub Kicinski return false; 1503248f346fSJakub Kicinski 1504248f346fSJakub Kicinski return true; 1505248f346fSJakub Kicinski } 1506248f346fSJakub Kicinski 1507248f346fSJakub Kicinski static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, 1508288b3de5SJakub Kicinski bool attach_drv) 150909756af4SAlexei Starovoitov { 151009756af4SAlexei Starovoitov struct fd f = fdget(ufd); 151109756af4SAlexei Starovoitov struct bpf_prog *prog; 151209756af4SAlexei Starovoitov 1513113214beSDaniel Borkmann prog = ____bpf_prog_get(f); 151409756af4SAlexei Starovoitov if (IS_ERR(prog)) 151509756af4SAlexei Starovoitov return prog; 1516288b3de5SJakub Kicinski if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) { 1517113214beSDaniel Borkmann prog = ERR_PTR(-EINVAL); 1518113214beSDaniel Borkmann goto out; 1519113214beSDaniel Borkmann } 152009756af4SAlexei Starovoitov 152192117d84SAlexei Starovoitov prog = bpf_prog_inc(prog); 1522113214beSDaniel Borkmann out: 152309756af4SAlexei Starovoitov fdput(f); 152409756af4SAlexei Starovoitov return prog; 152509756af4SAlexei Starovoitov } 1526113214beSDaniel Borkmann 1527113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get(u32 ufd) 1528113214beSDaniel Borkmann { 1529288b3de5SJakub Kicinski return __bpf_prog_get(ufd, NULL, false); 1530113214beSDaniel Borkmann } 1531113214beSDaniel Borkmann 1532248f346fSJakub Kicinski struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 1533288b3de5SJakub Kicinski bool attach_drv) 1534248f346fSJakub Kicinski { 15354d220ed0SAlexei Starovoitov return __bpf_prog_get(ufd, &type, attach_drv); 1536248f346fSJakub Kicinski } 15376c8dfe21SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev); 1538248f346fSJakub Kicinski 1539aac3fc32SAndrey Ignatov /* Initially all BPF programs could be loaded w/o specifying 1540aac3fc32SAndrey Ignatov * expected_attach_type. Later for some of them specifying expected_attach_type 1541aac3fc32SAndrey Ignatov * at load time became required so that program could be validated properly. 1542aac3fc32SAndrey Ignatov * Programs of types that are allowed to be loaded both w/ and w/o (for 1543aac3fc32SAndrey Ignatov * backward compatibility) expected_attach_type, should have the default attach 1544aac3fc32SAndrey Ignatov * type assigned to expected_attach_type for the latter case, so that it can be 1545aac3fc32SAndrey Ignatov * validated later at attach time. 1546aac3fc32SAndrey Ignatov * 1547aac3fc32SAndrey Ignatov * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if 1548aac3fc32SAndrey Ignatov * prog type requires it but has some attach types that have to be backward 1549aac3fc32SAndrey Ignatov * compatible. 1550aac3fc32SAndrey Ignatov */ 1551aac3fc32SAndrey Ignatov static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) 1552aac3fc32SAndrey Ignatov { 1553aac3fc32SAndrey Ignatov switch (attr->prog_type) { 1554aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1555aac3fc32SAndrey Ignatov /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't 1556aac3fc32SAndrey Ignatov * exist so checking for non-zero is the way to go here. 1557aac3fc32SAndrey Ignatov */ 1558aac3fc32SAndrey Ignatov if (!attr->expected_attach_type) 1559aac3fc32SAndrey Ignatov attr->expected_attach_type = 1560aac3fc32SAndrey Ignatov BPF_CGROUP_INET_SOCK_CREATE; 1561aac3fc32SAndrey Ignatov break; 1562aac3fc32SAndrey Ignatov } 1563aac3fc32SAndrey Ignatov } 1564aac3fc32SAndrey Ignatov 15655e43f899SAndrey Ignatov static int 15665e43f899SAndrey Ignatov bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type, 15675e43f899SAndrey Ignatov enum bpf_attach_type expected_attach_type) 15685e43f899SAndrey Ignatov { 15694fbac77dSAndrey Ignatov switch (prog_type) { 1570aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1571aac3fc32SAndrey Ignatov switch (expected_attach_type) { 1572aac3fc32SAndrey Ignatov case BPF_CGROUP_INET_SOCK_CREATE: 1573aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1574aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 1575aac3fc32SAndrey Ignatov return 0; 1576aac3fc32SAndrey Ignatov default: 1577aac3fc32SAndrey Ignatov return -EINVAL; 1578aac3fc32SAndrey Ignatov } 15794fbac77dSAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 15804fbac77dSAndrey Ignatov switch (expected_attach_type) { 15814fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 15824fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1583d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1584d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 15851cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 15861cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 15875e43f899SAndrey Ignatov return 0; 15884fbac77dSAndrey Ignatov default: 15894fbac77dSAndrey Ignatov return -EINVAL; 15904fbac77dSAndrey Ignatov } 15915cf1e914Sbrakmo case BPF_PROG_TYPE_CGROUP_SKB: 15925cf1e914Sbrakmo switch (expected_attach_type) { 15935cf1e914Sbrakmo case BPF_CGROUP_INET_INGRESS: 15945cf1e914Sbrakmo case BPF_CGROUP_INET_EGRESS: 15955cf1e914Sbrakmo return 0; 15965cf1e914Sbrakmo default: 15975cf1e914Sbrakmo return -EINVAL; 15985cf1e914Sbrakmo } 15994fbac77dSAndrey Ignatov default: 16004fbac77dSAndrey Ignatov return 0; 16014fbac77dSAndrey Ignatov } 16025e43f899SAndrey Ignatov } 16035e43f899SAndrey Ignatov 160409756af4SAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 1605c454a46bSMartin KaFai Lau #define BPF_PROG_LOAD_LAST_FIELD line_info_cnt 160609756af4SAlexei Starovoitov 1607838e9690SYonghong Song static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) 160809756af4SAlexei Starovoitov { 160909756af4SAlexei Starovoitov enum bpf_prog_type type = attr->prog_type; 161009756af4SAlexei Starovoitov struct bpf_prog *prog; 161109756af4SAlexei Starovoitov int err; 161209756af4SAlexei Starovoitov char license[128]; 161309756af4SAlexei Starovoitov bool is_gpl; 161409756af4SAlexei Starovoitov 161509756af4SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_LOAD)) 161609756af4SAlexei Starovoitov return -EINVAL; 161709756af4SAlexei Starovoitov 1618c240eff6SJiong Wang if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | 1619c240eff6SJiong Wang BPF_F_ANY_ALIGNMENT | 1620c240eff6SJiong Wang BPF_F_TEST_RND_HI32)) 1621e07b98d9SDavid S. Miller return -EINVAL; 1622e07b98d9SDavid S. Miller 1623e9ee9efcSDavid Miller if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && 1624e9ee9efcSDavid Miller (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && 1625e9ee9efcSDavid Miller !capable(CAP_SYS_ADMIN)) 1626e9ee9efcSDavid Miller return -EPERM; 1627e9ee9efcSDavid Miller 162809756af4SAlexei Starovoitov /* copy eBPF program license from user space */ 1629535e7b4bSMickaël Salaün if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 163009756af4SAlexei Starovoitov sizeof(license) - 1) < 0) 163109756af4SAlexei Starovoitov return -EFAULT; 163209756af4SAlexei Starovoitov license[sizeof(license) - 1] = 0; 163309756af4SAlexei Starovoitov 163409756af4SAlexei Starovoitov /* eBPF programs must be GPL compatible to use GPL-ed functions */ 163509756af4SAlexei Starovoitov is_gpl = license_is_gpl_compatible(license); 163609756af4SAlexei Starovoitov 1637c04c0d2bSAlexei Starovoitov if (attr->insn_cnt == 0 || 1638c04c0d2bSAlexei Starovoitov attr->insn_cnt > (capable(CAP_SYS_ADMIN) ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) 1639ef0915caSDaniel Borkmann return -E2BIG; 164080b7d819SChenbo Feng if (type != BPF_PROG_TYPE_SOCKET_FILTER && 164180b7d819SChenbo Feng type != BPF_PROG_TYPE_CGROUP_SKB && 164280b7d819SChenbo Feng !capable(CAP_SYS_ADMIN)) 16431be7f75dSAlexei Starovoitov return -EPERM; 16441be7f75dSAlexei Starovoitov 1645aac3fc32SAndrey Ignatov bpf_prog_load_fixup_attach_type(attr); 16465e43f899SAndrey Ignatov if (bpf_prog_load_check_attach_type(type, attr->expected_attach_type)) 16475e43f899SAndrey Ignatov return -EINVAL; 16485e43f899SAndrey Ignatov 164909756af4SAlexei Starovoitov /* plain bpf_prog allocation */ 165009756af4SAlexei Starovoitov prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 165109756af4SAlexei Starovoitov if (!prog) 165209756af4SAlexei Starovoitov return -ENOMEM; 165309756af4SAlexei Starovoitov 16545e43f899SAndrey Ignatov prog->expected_attach_type = attr->expected_attach_type; 16555e43f899SAndrey Ignatov 16569a18eedbSJakub Kicinski prog->aux->offload_requested = !!attr->prog_ifindex; 16579a18eedbSJakub Kicinski 1658afdb09c7SChenbo Feng err = security_bpf_prog_alloc(prog->aux); 1659aaac3ba9SAlexei Starovoitov if (err) 1660aaac3ba9SAlexei Starovoitov goto free_prog_nouncharge; 1661aaac3ba9SAlexei Starovoitov 1662afdb09c7SChenbo Feng err = bpf_prog_charge_memlock(prog); 1663afdb09c7SChenbo Feng if (err) 1664afdb09c7SChenbo Feng goto free_prog_sec; 1665afdb09c7SChenbo Feng 166609756af4SAlexei Starovoitov prog->len = attr->insn_cnt; 166709756af4SAlexei Starovoitov 166809756af4SAlexei Starovoitov err = -EFAULT; 1669535e7b4bSMickaël Salaün if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 1670aafe6ae9SDaniel Borkmann bpf_prog_insn_size(prog)) != 0) 167109756af4SAlexei Starovoitov goto free_prog; 167209756af4SAlexei Starovoitov 167309756af4SAlexei Starovoitov prog->orig_prog = NULL; 1674a91263d5SDaniel Borkmann prog->jited = 0; 167509756af4SAlexei Starovoitov 167609756af4SAlexei Starovoitov atomic_set(&prog->aux->refcnt, 1); 1677a91263d5SDaniel Borkmann prog->gpl_compatible = is_gpl ? 1 : 0; 167809756af4SAlexei Starovoitov 16799a18eedbSJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 1680ab3f0063SJakub Kicinski err = bpf_prog_offload_init(prog, attr); 1681ab3f0063SJakub Kicinski if (err) 1682ab3f0063SJakub Kicinski goto free_prog; 1683ab3f0063SJakub Kicinski } 1684ab3f0063SJakub Kicinski 168509756af4SAlexei Starovoitov /* find program type: socket_filter vs tracing_filter */ 168609756af4SAlexei Starovoitov err = find_prog_type(type, prog); 168709756af4SAlexei Starovoitov if (err < 0) 168809756af4SAlexei Starovoitov goto free_prog; 168909756af4SAlexei Starovoitov 1690cb4d2b3fSMartin KaFai Lau prog->aux->load_time = ktime_get_boot_ns(); 1691cb4d2b3fSMartin KaFai Lau err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name); 1692cb4d2b3fSMartin KaFai Lau if (err) 1693cb4d2b3fSMartin KaFai Lau goto free_prog; 1694cb4d2b3fSMartin KaFai Lau 169509756af4SAlexei Starovoitov /* run eBPF verifier */ 1696838e9690SYonghong Song err = bpf_check(&prog, attr, uattr); 169709756af4SAlexei Starovoitov if (err < 0) 169809756af4SAlexei Starovoitov goto free_used_maps; 169909756af4SAlexei Starovoitov 1700d1c55ab5SDaniel Borkmann prog = bpf_prog_select_runtime(prog, &err); 170104fd61abSAlexei Starovoitov if (err < 0) 170204fd61abSAlexei Starovoitov goto free_used_maps; 170309756af4SAlexei Starovoitov 1704dc4bb0e2SMartin KaFai Lau err = bpf_prog_alloc_id(prog); 1705dc4bb0e2SMartin KaFai Lau if (err) 1706dc4bb0e2SMartin KaFai Lau goto free_used_maps; 1707dc4bb0e2SMartin KaFai Lau 1708aa79781bSDaniel Borkmann err = bpf_prog_new_fd(prog); 1709b16d9aa4SMartin KaFai Lau if (err < 0) { 1710b16d9aa4SMartin KaFai Lau /* failed to allocate fd. 1711b16d9aa4SMartin KaFai Lau * bpf_prog_put() is needed because the above 1712b16d9aa4SMartin KaFai Lau * bpf_prog_alloc_id() has published the prog 1713b16d9aa4SMartin KaFai Lau * to the userspace and the userspace may 1714b16d9aa4SMartin KaFai Lau * have refcnt-ed it through BPF_PROG_GET_FD_BY_ID. 1715b16d9aa4SMartin KaFai Lau */ 1716b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 1717b16d9aa4SMartin KaFai Lau return err; 1718b16d9aa4SMartin KaFai Lau } 171909756af4SAlexei Starovoitov 172074451e66SDaniel Borkmann bpf_prog_kallsyms_add(prog); 17216ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); 172209756af4SAlexei Starovoitov return err; 172309756af4SAlexei Starovoitov 172409756af4SAlexei Starovoitov free_used_maps: 1725c454a46bSMartin KaFai Lau bpf_prog_free_linfo(prog); 17265482e9a9SMartin KaFai Lau kvfree(prog->aux->func_info); 17275482e9a9SMartin KaFai Lau btf_put(prog->aux->btf); 17287d1982b4SDaniel Borkmann bpf_prog_kallsyms_del_subprogs(prog); 172909756af4SAlexei Starovoitov free_used_maps(prog->aux); 173009756af4SAlexei Starovoitov free_prog: 1731aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(prog); 1732afdb09c7SChenbo Feng free_prog_sec: 1733afdb09c7SChenbo Feng security_bpf_prog_free(prog->aux); 1734aaac3ba9SAlexei Starovoitov free_prog_nouncharge: 173509756af4SAlexei Starovoitov bpf_prog_free(prog); 173609756af4SAlexei Starovoitov return err; 173709756af4SAlexei Starovoitov } 173809756af4SAlexei Starovoitov 17396e71b04aSChenbo Feng #define BPF_OBJ_LAST_FIELD file_flags 1740b2197755SDaniel Borkmann 1741b2197755SDaniel Borkmann static int bpf_obj_pin(const union bpf_attr *attr) 1742b2197755SDaniel Borkmann { 17436e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0) 1744b2197755SDaniel Borkmann return -EINVAL; 1745b2197755SDaniel Borkmann 1746535e7b4bSMickaël Salaün return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 1747b2197755SDaniel Borkmann } 1748b2197755SDaniel Borkmann 1749b2197755SDaniel Borkmann static int bpf_obj_get(const union bpf_attr *attr) 1750b2197755SDaniel Borkmann { 17516e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || 17526e71b04aSChenbo Feng attr->file_flags & ~BPF_OBJ_FLAG_MASK) 1753b2197755SDaniel Borkmann return -EINVAL; 1754b2197755SDaniel Borkmann 17556e71b04aSChenbo Feng return bpf_obj_get_user(u64_to_user_ptr(attr->pathname), 17566e71b04aSChenbo Feng attr->file_flags); 1757b2197755SDaniel Borkmann } 1758b2197755SDaniel Borkmann 1759c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint { 1760c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 1761c4f6699dSAlexei Starovoitov struct bpf_prog *prog; 1762c4f6699dSAlexei Starovoitov }; 1763c4f6699dSAlexei Starovoitov 1764c4f6699dSAlexei Starovoitov static int bpf_raw_tracepoint_release(struct inode *inode, struct file *filp) 1765c4f6699dSAlexei Starovoitov { 1766c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint *raw_tp = filp->private_data; 1767c4f6699dSAlexei Starovoitov 1768c4f6699dSAlexei Starovoitov if (raw_tp->prog) { 1769c4f6699dSAlexei Starovoitov bpf_probe_unregister(raw_tp->btp, raw_tp->prog); 1770c4f6699dSAlexei Starovoitov bpf_prog_put(raw_tp->prog); 1771c4f6699dSAlexei Starovoitov } 1772a38d1107SMatt Mullins bpf_put_raw_tracepoint(raw_tp->btp); 1773c4f6699dSAlexei Starovoitov kfree(raw_tp); 1774c4f6699dSAlexei Starovoitov return 0; 1775c4f6699dSAlexei Starovoitov } 1776c4f6699dSAlexei Starovoitov 1777c4f6699dSAlexei Starovoitov static const struct file_operations bpf_raw_tp_fops = { 1778c4f6699dSAlexei Starovoitov .release = bpf_raw_tracepoint_release, 1779c4f6699dSAlexei Starovoitov .read = bpf_dummy_read, 1780c4f6699dSAlexei Starovoitov .write = bpf_dummy_write, 1781c4f6699dSAlexei Starovoitov }; 1782c4f6699dSAlexei Starovoitov 1783c4f6699dSAlexei Starovoitov #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd 1784c4f6699dSAlexei Starovoitov 1785c4f6699dSAlexei Starovoitov static int bpf_raw_tracepoint_open(const union bpf_attr *attr) 1786c4f6699dSAlexei Starovoitov { 1787c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint *raw_tp; 1788c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 1789c4f6699dSAlexei Starovoitov struct bpf_prog *prog; 1790c4f6699dSAlexei Starovoitov char tp_name[128]; 1791c4f6699dSAlexei Starovoitov int tp_fd, err; 1792c4f6699dSAlexei Starovoitov 1793c4f6699dSAlexei Starovoitov if (strncpy_from_user(tp_name, u64_to_user_ptr(attr->raw_tracepoint.name), 1794c4f6699dSAlexei Starovoitov sizeof(tp_name) - 1) < 0) 1795c4f6699dSAlexei Starovoitov return -EFAULT; 1796c4f6699dSAlexei Starovoitov tp_name[sizeof(tp_name) - 1] = 0; 1797c4f6699dSAlexei Starovoitov 1798a38d1107SMatt Mullins btp = bpf_get_raw_tracepoint(tp_name); 1799c4f6699dSAlexei Starovoitov if (!btp) 1800c4f6699dSAlexei Starovoitov return -ENOENT; 1801c4f6699dSAlexei Starovoitov 1802c4f6699dSAlexei Starovoitov raw_tp = kzalloc(sizeof(*raw_tp), GFP_USER); 1803a38d1107SMatt Mullins if (!raw_tp) { 1804a38d1107SMatt Mullins err = -ENOMEM; 1805a38d1107SMatt Mullins goto out_put_btp; 1806a38d1107SMatt Mullins } 1807c4f6699dSAlexei Starovoitov raw_tp->btp = btp; 1808c4f6699dSAlexei Starovoitov 18099df1c28bSMatt Mullins prog = bpf_prog_get(attr->raw_tracepoint.prog_fd); 1810c4f6699dSAlexei Starovoitov if (IS_ERR(prog)) { 1811c4f6699dSAlexei Starovoitov err = PTR_ERR(prog); 1812c4f6699dSAlexei Starovoitov goto out_free_tp; 1813c4f6699dSAlexei Starovoitov } 18149df1c28bSMatt Mullins if (prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT && 18159df1c28bSMatt Mullins prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) { 18169df1c28bSMatt Mullins err = -EINVAL; 18179df1c28bSMatt Mullins goto out_put_prog; 18189df1c28bSMatt Mullins } 1819c4f6699dSAlexei Starovoitov 1820c4f6699dSAlexei Starovoitov err = bpf_probe_register(raw_tp->btp, prog); 1821c4f6699dSAlexei Starovoitov if (err) 1822c4f6699dSAlexei Starovoitov goto out_put_prog; 1823c4f6699dSAlexei Starovoitov 1824c4f6699dSAlexei Starovoitov raw_tp->prog = prog; 1825c4f6699dSAlexei Starovoitov tp_fd = anon_inode_getfd("bpf-raw-tracepoint", &bpf_raw_tp_fops, raw_tp, 1826c4f6699dSAlexei Starovoitov O_CLOEXEC); 1827c4f6699dSAlexei Starovoitov if (tp_fd < 0) { 1828c4f6699dSAlexei Starovoitov bpf_probe_unregister(raw_tp->btp, prog); 1829c4f6699dSAlexei Starovoitov err = tp_fd; 1830c4f6699dSAlexei Starovoitov goto out_put_prog; 1831c4f6699dSAlexei Starovoitov } 1832c4f6699dSAlexei Starovoitov return tp_fd; 1833c4f6699dSAlexei Starovoitov 1834c4f6699dSAlexei Starovoitov out_put_prog: 1835c4f6699dSAlexei Starovoitov bpf_prog_put(prog); 1836c4f6699dSAlexei Starovoitov out_free_tp: 1837c4f6699dSAlexei Starovoitov kfree(raw_tp); 1838a38d1107SMatt Mullins out_put_btp: 1839a38d1107SMatt Mullins bpf_put_raw_tracepoint(btp); 1840c4f6699dSAlexei Starovoitov return err; 1841c4f6699dSAlexei Starovoitov } 1842c4f6699dSAlexei Starovoitov 184333491588SAnders Roxell static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, 184433491588SAnders Roxell enum bpf_attach_type attach_type) 184533491588SAnders Roxell { 184633491588SAnders Roxell switch (prog->type) { 184733491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK: 184833491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 184933491588SAnders Roxell return attach_type == prog->expected_attach_type ? 0 : -EINVAL; 18505cf1e914Sbrakmo case BPF_PROG_TYPE_CGROUP_SKB: 18515cf1e914Sbrakmo return prog->enforce_expected_attach_type && 18525cf1e914Sbrakmo prog->expected_attach_type != attach_type ? 18535cf1e914Sbrakmo -EINVAL : 0; 185433491588SAnders Roxell default: 185533491588SAnders Roxell return 0; 185633491588SAnders Roxell } 185733491588SAnders Roxell } 185833491588SAnders Roxell 1859464bc0fdSJohn Fastabend #define BPF_PROG_ATTACH_LAST_FIELD attach_flags 1860174a79ffSJohn Fastabend 1861324bda9eSAlexei Starovoitov #define BPF_F_ATTACH_MASK \ 1862324bda9eSAlexei Starovoitov (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI) 1863324bda9eSAlexei Starovoitov 1864f4324551SDaniel Mack static int bpf_prog_attach(const union bpf_attr *attr) 1865f4324551SDaniel Mack { 18667f677633SAlexei Starovoitov enum bpf_prog_type ptype; 1867f4324551SDaniel Mack struct bpf_prog *prog; 18687f677633SAlexei Starovoitov int ret; 1869f4324551SDaniel Mack 1870f4324551SDaniel Mack if (!capable(CAP_NET_ADMIN)) 1871f4324551SDaniel Mack return -EPERM; 1872f4324551SDaniel Mack 1873f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_ATTACH)) 1874f4324551SDaniel Mack return -EINVAL; 1875f4324551SDaniel Mack 1876324bda9eSAlexei Starovoitov if (attr->attach_flags & ~BPF_F_ATTACH_MASK) 18777f677633SAlexei Starovoitov return -EINVAL; 18787f677633SAlexei Starovoitov 1879f4324551SDaniel Mack switch (attr->attach_type) { 1880f4324551SDaniel Mack case BPF_CGROUP_INET_INGRESS: 1881f4324551SDaniel Mack case BPF_CGROUP_INET_EGRESS: 1882b2cd1257SDavid Ahern ptype = BPF_PROG_TYPE_CGROUP_SKB; 1883b2cd1257SDavid Ahern break; 188461023658SDavid Ahern case BPF_CGROUP_INET_SOCK_CREATE: 1885aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1886aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 188761023658SDavid Ahern ptype = BPF_PROG_TYPE_CGROUP_SOCK; 188861023658SDavid Ahern break; 18894fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 18904fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1891d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1892d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 18931cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 18941cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 18954fbac77dSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 18964fbac77dSAndrey Ignatov break; 189740304b2aSLawrence Brakmo case BPF_CGROUP_SOCK_OPS: 189840304b2aSLawrence Brakmo ptype = BPF_PROG_TYPE_SOCK_OPS; 189940304b2aSLawrence Brakmo break; 1900ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 1901ebc614f6SRoman Gushchin ptype = BPF_PROG_TYPE_CGROUP_DEVICE; 1902ebc614f6SRoman Gushchin break; 19034f738adbSJohn Fastabend case BPF_SK_MSG_VERDICT: 1904fdb5c453SSean Young ptype = BPF_PROG_TYPE_SK_MSG; 1905fdb5c453SSean Young break; 1906464bc0fdSJohn Fastabend case BPF_SK_SKB_STREAM_PARSER: 1907464bc0fdSJohn Fastabend case BPF_SK_SKB_STREAM_VERDICT: 1908fdb5c453SSean Young ptype = BPF_PROG_TYPE_SK_SKB; 1909fdb5c453SSean Young break; 1910f4364dcfSSean Young case BPF_LIRC_MODE2: 1911fdb5c453SSean Young ptype = BPF_PROG_TYPE_LIRC_MODE2; 1912fdb5c453SSean Young break; 1913d58e468bSPetar Penkov case BPF_FLOW_DISSECTOR: 1914d58e468bSPetar Penkov ptype = BPF_PROG_TYPE_FLOW_DISSECTOR; 1915d58e468bSPetar Penkov break; 19167b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 19177b146cebSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SYSCTL; 19187b146cebSAndrey Ignatov break; 1919b2cd1257SDavid Ahern default: 1920b2cd1257SDavid Ahern return -EINVAL; 1921b2cd1257SDavid Ahern } 1922b2cd1257SDavid Ahern 1923b2cd1257SDavid Ahern prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 1924f4324551SDaniel Mack if (IS_ERR(prog)) 1925f4324551SDaniel Mack return PTR_ERR(prog); 1926f4324551SDaniel Mack 19275e43f899SAndrey Ignatov if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) { 19285e43f899SAndrey Ignatov bpf_prog_put(prog); 19295e43f899SAndrey Ignatov return -EINVAL; 19305e43f899SAndrey Ignatov } 19315e43f899SAndrey Ignatov 1932fdb5c453SSean Young switch (ptype) { 1933fdb5c453SSean Young case BPF_PROG_TYPE_SK_SKB: 1934fdb5c453SSean Young case BPF_PROG_TYPE_SK_MSG: 1935604326b4SDaniel Borkmann ret = sock_map_get_from_fd(attr, prog); 1936fdb5c453SSean Young break; 1937fdb5c453SSean Young case BPF_PROG_TYPE_LIRC_MODE2: 1938fdb5c453SSean Young ret = lirc_prog_attach(attr, prog); 1939fdb5c453SSean Young break; 1940d58e468bSPetar Penkov case BPF_PROG_TYPE_FLOW_DISSECTOR: 1941d58e468bSPetar Penkov ret = skb_flow_dissector_bpf_prog_attach(attr, prog); 1942d58e468bSPetar Penkov break; 1943fdb5c453SSean Young default: 1944fdb5c453SSean Young ret = cgroup_bpf_prog_attach(attr, ptype, prog); 1945f4324551SDaniel Mack } 1946f4324551SDaniel Mack 19477f677633SAlexei Starovoitov if (ret) 19487f677633SAlexei Starovoitov bpf_prog_put(prog); 19497f677633SAlexei Starovoitov return ret; 1950f4324551SDaniel Mack } 1951f4324551SDaniel Mack 1952f4324551SDaniel Mack #define BPF_PROG_DETACH_LAST_FIELD attach_type 1953f4324551SDaniel Mack 1954f4324551SDaniel Mack static int bpf_prog_detach(const union bpf_attr *attr) 1955f4324551SDaniel Mack { 1956324bda9eSAlexei Starovoitov enum bpf_prog_type ptype; 1957f4324551SDaniel Mack 1958f4324551SDaniel Mack if (!capable(CAP_NET_ADMIN)) 1959f4324551SDaniel Mack return -EPERM; 1960f4324551SDaniel Mack 1961f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_DETACH)) 1962f4324551SDaniel Mack return -EINVAL; 1963f4324551SDaniel Mack 1964f4324551SDaniel Mack switch (attr->attach_type) { 1965f4324551SDaniel Mack case BPF_CGROUP_INET_INGRESS: 1966f4324551SDaniel Mack case BPF_CGROUP_INET_EGRESS: 1967324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_CGROUP_SKB; 1968324bda9eSAlexei Starovoitov break; 196961023658SDavid Ahern case BPF_CGROUP_INET_SOCK_CREATE: 1970aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1971aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 1972324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_CGROUP_SOCK; 1973324bda9eSAlexei Starovoitov break; 19744fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 19754fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1976d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1977d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 19781cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 19791cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 19804fbac77dSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 19814fbac77dSAndrey Ignatov break; 198240304b2aSLawrence Brakmo case BPF_CGROUP_SOCK_OPS: 1983324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_SOCK_OPS; 1984f4324551SDaniel Mack break; 1985ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 1986ebc614f6SRoman Gushchin ptype = BPF_PROG_TYPE_CGROUP_DEVICE; 1987ebc614f6SRoman Gushchin break; 19884f738adbSJohn Fastabend case BPF_SK_MSG_VERDICT: 1989604326b4SDaniel Borkmann return sock_map_get_from_fd(attr, NULL); 19905a67da2aSJohn Fastabend case BPF_SK_SKB_STREAM_PARSER: 19915a67da2aSJohn Fastabend case BPF_SK_SKB_STREAM_VERDICT: 1992604326b4SDaniel Borkmann return sock_map_get_from_fd(attr, NULL); 1993f4364dcfSSean Young case BPF_LIRC_MODE2: 1994f4364dcfSSean Young return lirc_prog_detach(attr); 1995d58e468bSPetar Penkov case BPF_FLOW_DISSECTOR: 1996d58e468bSPetar Penkov return skb_flow_dissector_bpf_prog_detach(attr); 19977b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 19987b146cebSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SYSCTL; 19997b146cebSAndrey Ignatov break; 2000f4324551SDaniel Mack default: 2001f4324551SDaniel Mack return -EINVAL; 2002f4324551SDaniel Mack } 2003f4324551SDaniel Mack 2004fdb5c453SSean Young return cgroup_bpf_prog_detach(attr, ptype); 2005f4324551SDaniel Mack } 200640304b2aSLawrence Brakmo 2007468e2f64SAlexei Starovoitov #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt 2008468e2f64SAlexei Starovoitov 2009468e2f64SAlexei Starovoitov static int bpf_prog_query(const union bpf_attr *attr, 2010468e2f64SAlexei Starovoitov union bpf_attr __user *uattr) 2011468e2f64SAlexei Starovoitov { 2012468e2f64SAlexei Starovoitov if (!capable(CAP_NET_ADMIN)) 2013468e2f64SAlexei Starovoitov return -EPERM; 2014468e2f64SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_QUERY)) 2015468e2f64SAlexei Starovoitov return -EINVAL; 2016468e2f64SAlexei Starovoitov if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE) 2017468e2f64SAlexei Starovoitov return -EINVAL; 2018468e2f64SAlexei Starovoitov 2019468e2f64SAlexei Starovoitov switch (attr->query.attach_type) { 2020468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_INGRESS: 2021468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_EGRESS: 2022468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_SOCK_CREATE: 20234fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 20244fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 2025aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 2026aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 2027d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 2028d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 20291cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG: 20301cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG: 2031468e2f64SAlexei Starovoitov case BPF_CGROUP_SOCK_OPS: 2032ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 20337b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL: 2034468e2f64SAlexei Starovoitov break; 2035f4364dcfSSean Young case BPF_LIRC_MODE2: 2036f4364dcfSSean Young return lirc_prog_query(attr, uattr); 2037118c8e9aSStanislav Fomichev case BPF_FLOW_DISSECTOR: 2038118c8e9aSStanislav Fomichev return skb_flow_dissector_prog_query(attr, uattr); 2039468e2f64SAlexei Starovoitov default: 2040468e2f64SAlexei Starovoitov return -EINVAL; 2041468e2f64SAlexei Starovoitov } 2042fdb5c453SSean Young 2043fdb5c453SSean Young return cgroup_bpf_prog_query(attr, uattr); 2044468e2f64SAlexei Starovoitov } 2045f4324551SDaniel Mack 2046b0b9395dSStanislav Fomichev #define BPF_PROG_TEST_RUN_LAST_FIELD test.ctx_out 20471cf1cae9SAlexei Starovoitov 20481cf1cae9SAlexei Starovoitov static int bpf_prog_test_run(const union bpf_attr *attr, 20491cf1cae9SAlexei Starovoitov union bpf_attr __user *uattr) 20501cf1cae9SAlexei Starovoitov { 20511cf1cae9SAlexei Starovoitov struct bpf_prog *prog; 20521cf1cae9SAlexei Starovoitov int ret = -ENOTSUPP; 20531cf1cae9SAlexei Starovoitov 205461f3c964SAlexei Starovoitov if (!capable(CAP_SYS_ADMIN)) 205561f3c964SAlexei Starovoitov return -EPERM; 20561cf1cae9SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 20571cf1cae9SAlexei Starovoitov return -EINVAL; 20581cf1cae9SAlexei Starovoitov 2059b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_in && !attr->test.ctx_in) || 2060b0b9395dSStanislav Fomichev (!attr->test.ctx_size_in && attr->test.ctx_in)) 2061b0b9395dSStanislav Fomichev return -EINVAL; 2062b0b9395dSStanislav Fomichev 2063b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_out && !attr->test.ctx_out) || 2064b0b9395dSStanislav Fomichev (!attr->test.ctx_size_out && attr->test.ctx_out)) 2065b0b9395dSStanislav Fomichev return -EINVAL; 2066b0b9395dSStanislav Fomichev 20671cf1cae9SAlexei Starovoitov prog = bpf_prog_get(attr->test.prog_fd); 20681cf1cae9SAlexei Starovoitov if (IS_ERR(prog)) 20691cf1cae9SAlexei Starovoitov return PTR_ERR(prog); 20701cf1cae9SAlexei Starovoitov 20711cf1cae9SAlexei Starovoitov if (prog->aux->ops->test_run) 20721cf1cae9SAlexei Starovoitov ret = prog->aux->ops->test_run(prog, attr, uattr); 20731cf1cae9SAlexei Starovoitov 20741cf1cae9SAlexei Starovoitov bpf_prog_put(prog); 20751cf1cae9SAlexei Starovoitov return ret; 20761cf1cae9SAlexei Starovoitov } 20771cf1cae9SAlexei Starovoitov 207834ad5580SMartin KaFai Lau #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 207934ad5580SMartin KaFai Lau 208034ad5580SMartin KaFai Lau static int bpf_obj_get_next_id(const union bpf_attr *attr, 208134ad5580SMartin KaFai Lau union bpf_attr __user *uattr, 208234ad5580SMartin KaFai Lau struct idr *idr, 208334ad5580SMartin KaFai Lau spinlock_t *lock) 208434ad5580SMartin KaFai Lau { 208534ad5580SMartin KaFai Lau u32 next_id = attr->start_id; 208634ad5580SMartin KaFai Lau int err = 0; 208734ad5580SMartin KaFai Lau 208834ad5580SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 208934ad5580SMartin KaFai Lau return -EINVAL; 209034ad5580SMartin KaFai Lau 209134ad5580SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 209234ad5580SMartin KaFai Lau return -EPERM; 209334ad5580SMartin KaFai Lau 209434ad5580SMartin KaFai Lau next_id++; 209534ad5580SMartin KaFai Lau spin_lock_bh(lock); 209634ad5580SMartin KaFai Lau if (!idr_get_next(idr, &next_id)) 209734ad5580SMartin KaFai Lau err = -ENOENT; 209834ad5580SMartin KaFai Lau spin_unlock_bh(lock); 209934ad5580SMartin KaFai Lau 210034ad5580SMartin KaFai Lau if (!err) 210134ad5580SMartin KaFai Lau err = put_user(next_id, &uattr->next_id); 210234ad5580SMartin KaFai Lau 210334ad5580SMartin KaFai Lau return err; 210434ad5580SMartin KaFai Lau } 210534ad5580SMartin KaFai Lau 2106b16d9aa4SMartin KaFai Lau #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 2107b16d9aa4SMartin KaFai Lau 2108b16d9aa4SMartin KaFai Lau static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 2109b16d9aa4SMartin KaFai Lau { 2110b16d9aa4SMartin KaFai Lau struct bpf_prog *prog; 2111b16d9aa4SMartin KaFai Lau u32 id = attr->prog_id; 2112b16d9aa4SMartin KaFai Lau int fd; 2113b16d9aa4SMartin KaFai Lau 2114b16d9aa4SMartin KaFai Lau if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 2115b16d9aa4SMartin KaFai Lau return -EINVAL; 2116b16d9aa4SMartin KaFai Lau 2117b16d9aa4SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 2118b16d9aa4SMartin KaFai Lau return -EPERM; 2119b16d9aa4SMartin KaFai Lau 2120b16d9aa4SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 2121b16d9aa4SMartin KaFai Lau prog = idr_find(&prog_idr, id); 2122b16d9aa4SMartin KaFai Lau if (prog) 2123b16d9aa4SMartin KaFai Lau prog = bpf_prog_inc_not_zero(prog); 2124b16d9aa4SMartin KaFai Lau else 2125b16d9aa4SMartin KaFai Lau prog = ERR_PTR(-ENOENT); 2126b16d9aa4SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 2127b16d9aa4SMartin KaFai Lau 2128b16d9aa4SMartin KaFai Lau if (IS_ERR(prog)) 2129b16d9aa4SMartin KaFai Lau return PTR_ERR(prog); 2130b16d9aa4SMartin KaFai Lau 2131b16d9aa4SMartin KaFai Lau fd = bpf_prog_new_fd(prog); 2132b16d9aa4SMartin KaFai Lau if (fd < 0) 2133b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 2134b16d9aa4SMartin KaFai Lau 2135b16d9aa4SMartin KaFai Lau return fd; 2136b16d9aa4SMartin KaFai Lau } 2137b16d9aa4SMartin KaFai Lau 21386e71b04aSChenbo Feng #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags 2139bd5f5f4eSMartin KaFai Lau 2140bd5f5f4eSMartin KaFai Lau static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 2141bd5f5f4eSMartin KaFai Lau { 2142bd5f5f4eSMartin KaFai Lau struct bpf_map *map; 2143bd5f5f4eSMartin KaFai Lau u32 id = attr->map_id; 21446e71b04aSChenbo Feng int f_flags; 2145bd5f5f4eSMartin KaFai Lau int fd; 2146bd5f5f4eSMartin KaFai Lau 21476e71b04aSChenbo Feng if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || 21486e71b04aSChenbo Feng attr->open_flags & ~BPF_OBJ_FLAG_MASK) 2149bd5f5f4eSMartin KaFai Lau return -EINVAL; 2150bd5f5f4eSMartin KaFai Lau 2151bd5f5f4eSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 2152bd5f5f4eSMartin KaFai Lau return -EPERM; 2153bd5f5f4eSMartin KaFai Lau 21546e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->open_flags); 21556e71b04aSChenbo Feng if (f_flags < 0) 21566e71b04aSChenbo Feng return f_flags; 21576e71b04aSChenbo Feng 2158bd5f5f4eSMartin KaFai Lau spin_lock_bh(&map_idr_lock); 2159bd5f5f4eSMartin KaFai Lau map = idr_find(&map_idr, id); 2160bd5f5f4eSMartin KaFai Lau if (map) 2161bd5f5f4eSMartin KaFai Lau map = bpf_map_inc_not_zero(map, true); 2162bd5f5f4eSMartin KaFai Lau else 2163bd5f5f4eSMartin KaFai Lau map = ERR_PTR(-ENOENT); 2164bd5f5f4eSMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 2165bd5f5f4eSMartin KaFai Lau 2166bd5f5f4eSMartin KaFai Lau if (IS_ERR(map)) 2167bd5f5f4eSMartin KaFai Lau return PTR_ERR(map); 2168bd5f5f4eSMartin KaFai Lau 21696e71b04aSChenbo Feng fd = bpf_map_new_fd(map, f_flags); 2170bd5f5f4eSMartin KaFai Lau if (fd < 0) 2171781e6282SPeng Sun bpf_map_put_with_uref(map); 2172bd5f5f4eSMartin KaFai Lau 2173bd5f5f4eSMartin KaFai Lau return fd; 2174bd5f5f4eSMartin KaFai Lau } 2175bd5f5f4eSMartin KaFai Lau 21767105e828SDaniel Borkmann static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, 2177d8eca5bbSDaniel Borkmann unsigned long addr, u32 *off, 2178d8eca5bbSDaniel Borkmann u32 *type) 21797105e828SDaniel Borkmann { 2180d8eca5bbSDaniel Borkmann const struct bpf_map *map; 21817105e828SDaniel Borkmann int i; 21827105e828SDaniel Borkmann 2183d8eca5bbSDaniel Borkmann for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { 2184d8eca5bbSDaniel Borkmann map = prog->aux->used_maps[i]; 2185d8eca5bbSDaniel Borkmann if (map == (void *)addr) { 2186d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_FD; 2187d8eca5bbSDaniel Borkmann return map; 2188d8eca5bbSDaniel Borkmann } 2189d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta) 2190d8eca5bbSDaniel Borkmann continue; 2191d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta(map, addr, off)) { 2192d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_VALUE; 2193d8eca5bbSDaniel Borkmann return map; 2194d8eca5bbSDaniel Borkmann } 2195d8eca5bbSDaniel Borkmann } 2196d8eca5bbSDaniel Borkmann 21977105e828SDaniel Borkmann return NULL; 21987105e828SDaniel Borkmann } 21997105e828SDaniel Borkmann 22007105e828SDaniel Borkmann static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog) 22017105e828SDaniel Borkmann { 22027105e828SDaniel Borkmann const struct bpf_map *map; 22037105e828SDaniel Borkmann struct bpf_insn *insns; 2204d8eca5bbSDaniel Borkmann u32 off, type; 22057105e828SDaniel Borkmann u64 imm; 22067105e828SDaniel Borkmann int i; 22077105e828SDaniel Borkmann 22087105e828SDaniel Borkmann insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), 22097105e828SDaniel Borkmann GFP_USER); 22107105e828SDaniel Borkmann if (!insns) 22117105e828SDaniel Borkmann return insns; 22127105e828SDaniel Borkmann 22137105e828SDaniel Borkmann for (i = 0; i < prog->len; i++) { 22147105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_TAIL_CALL)) { 22157105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 22167105e828SDaniel Borkmann insns[i].imm = BPF_FUNC_tail_call; 22177105e828SDaniel Borkmann /* fall-through */ 22187105e828SDaniel Borkmann } 22197105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_CALL) || 22207105e828SDaniel Borkmann insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) { 22217105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) 22227105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 22237105e828SDaniel Borkmann if (!bpf_dump_raw_ok()) 22247105e828SDaniel Borkmann insns[i].imm = 0; 22257105e828SDaniel Borkmann continue; 22267105e828SDaniel Borkmann } 22277105e828SDaniel Borkmann 22287105e828SDaniel Borkmann if (insns[i].code != (BPF_LD | BPF_IMM | BPF_DW)) 22297105e828SDaniel Borkmann continue; 22307105e828SDaniel Borkmann 22317105e828SDaniel Borkmann imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; 2232d8eca5bbSDaniel Borkmann map = bpf_map_from_imm(prog, imm, &off, &type); 22337105e828SDaniel Borkmann if (map) { 2234d8eca5bbSDaniel Borkmann insns[i].src_reg = type; 22357105e828SDaniel Borkmann insns[i].imm = map->id; 2236d8eca5bbSDaniel Borkmann insns[i + 1].imm = off; 22377105e828SDaniel Borkmann continue; 22387105e828SDaniel Borkmann } 22397105e828SDaniel Borkmann } 22407105e828SDaniel Borkmann 22417105e828SDaniel Borkmann return insns; 22427105e828SDaniel Borkmann } 22437105e828SDaniel Borkmann 2244c454a46bSMartin KaFai Lau static int set_info_rec_size(struct bpf_prog_info *info) 2245c454a46bSMartin KaFai Lau { 2246c454a46bSMartin KaFai Lau /* 2247c454a46bSMartin KaFai Lau * Ensure info.*_rec_size is the same as kernel expected size 2248c454a46bSMartin KaFai Lau * 2249c454a46bSMartin KaFai Lau * or 2250c454a46bSMartin KaFai Lau * 2251c454a46bSMartin KaFai Lau * Only allow zero *_rec_size if both _rec_size and _cnt are 2252c454a46bSMartin KaFai Lau * zero. In this case, the kernel will set the expected 2253c454a46bSMartin KaFai Lau * _rec_size back to the info. 2254c454a46bSMartin KaFai Lau */ 2255c454a46bSMartin KaFai Lau 225611d8b82dSYonghong Song if ((info->nr_func_info || info->func_info_rec_size) && 2257c454a46bSMartin KaFai Lau info->func_info_rec_size != sizeof(struct bpf_func_info)) 2258c454a46bSMartin KaFai Lau return -EINVAL; 2259c454a46bSMartin KaFai Lau 226011d8b82dSYonghong Song if ((info->nr_line_info || info->line_info_rec_size) && 2261c454a46bSMartin KaFai Lau info->line_info_rec_size != sizeof(struct bpf_line_info)) 2262c454a46bSMartin KaFai Lau return -EINVAL; 2263c454a46bSMartin KaFai Lau 226411d8b82dSYonghong Song if ((info->nr_jited_line_info || info->jited_line_info_rec_size) && 2265c454a46bSMartin KaFai Lau info->jited_line_info_rec_size != sizeof(__u64)) 2266c454a46bSMartin KaFai Lau return -EINVAL; 2267c454a46bSMartin KaFai Lau 2268c454a46bSMartin KaFai Lau info->func_info_rec_size = sizeof(struct bpf_func_info); 2269c454a46bSMartin KaFai Lau info->line_info_rec_size = sizeof(struct bpf_line_info); 2270c454a46bSMartin KaFai Lau info->jited_line_info_rec_size = sizeof(__u64); 2271c454a46bSMartin KaFai Lau 2272c454a46bSMartin KaFai Lau return 0; 2273c454a46bSMartin KaFai Lau } 2274c454a46bSMartin KaFai Lau 22751e270976SMartin KaFai Lau static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, 22761e270976SMartin KaFai Lau const union bpf_attr *attr, 22771e270976SMartin KaFai Lau union bpf_attr __user *uattr) 22781e270976SMartin KaFai Lau { 22791e270976SMartin KaFai Lau struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 22801e270976SMartin KaFai Lau struct bpf_prog_info info = {}; 22811e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 22825f8f8b93SAlexei Starovoitov struct bpf_prog_stats stats; 22831e270976SMartin KaFai Lau char __user *uinsns; 22841e270976SMartin KaFai Lau u32 ulen; 22851e270976SMartin KaFai Lau int err; 22861e270976SMartin KaFai Lau 2287dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 22881e270976SMartin KaFai Lau if (err) 22891e270976SMartin KaFai Lau return err; 22901e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 22911e270976SMartin KaFai Lau 22921e270976SMartin KaFai Lau if (copy_from_user(&info, uinfo, info_len)) 229389b09689SDaniel Borkmann return -EFAULT; 22941e270976SMartin KaFai Lau 22951e270976SMartin KaFai Lau info.type = prog->type; 22961e270976SMartin KaFai Lau info.id = prog->aux->id; 2297cb4d2b3fSMartin KaFai Lau info.load_time = prog->aux->load_time; 2298cb4d2b3fSMartin KaFai Lau info.created_by_uid = from_kuid_munged(current_user_ns(), 2299cb4d2b3fSMartin KaFai Lau prog->aux->user->uid); 2300b85fab0eSJiri Olsa info.gpl_compatible = prog->gpl_compatible; 23011e270976SMartin KaFai Lau 23021e270976SMartin KaFai Lau memcpy(info.tag, prog->tag, sizeof(prog->tag)); 2303cb4d2b3fSMartin KaFai Lau memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); 2304cb4d2b3fSMartin KaFai Lau 2305cb4d2b3fSMartin KaFai Lau ulen = info.nr_map_ids; 2306cb4d2b3fSMartin KaFai Lau info.nr_map_ids = prog->aux->used_map_cnt; 2307cb4d2b3fSMartin KaFai Lau ulen = min_t(u32, info.nr_map_ids, ulen); 2308cb4d2b3fSMartin KaFai Lau if (ulen) { 2309721e08daSMartin KaFai Lau u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); 2310cb4d2b3fSMartin KaFai Lau u32 i; 2311cb4d2b3fSMartin KaFai Lau 2312cb4d2b3fSMartin KaFai Lau for (i = 0; i < ulen; i++) 2313cb4d2b3fSMartin KaFai Lau if (put_user(prog->aux->used_maps[i]->id, 2314cb4d2b3fSMartin KaFai Lau &user_map_ids[i])) 2315cb4d2b3fSMartin KaFai Lau return -EFAULT; 2316cb4d2b3fSMartin KaFai Lau } 23171e270976SMartin KaFai Lau 2318c454a46bSMartin KaFai Lau err = set_info_rec_size(&info); 2319c454a46bSMartin KaFai Lau if (err) 2320c454a46bSMartin KaFai Lau return err; 23217337224fSMartin KaFai Lau 23225f8f8b93SAlexei Starovoitov bpf_prog_get_stats(prog, &stats); 23235f8f8b93SAlexei Starovoitov info.run_time_ns = stats.nsecs; 23245f8f8b93SAlexei Starovoitov info.run_cnt = stats.cnt; 23255f8f8b93SAlexei Starovoitov 23261e270976SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) { 23271e270976SMartin KaFai Lau info.jited_prog_len = 0; 23281e270976SMartin KaFai Lau info.xlated_prog_len = 0; 2329dbecd738SSandipan Das info.nr_jited_ksyms = 0; 233028c2fae7SDaniel Borkmann info.nr_jited_func_lens = 0; 233111d8b82dSYonghong Song info.nr_func_info = 0; 233211d8b82dSYonghong Song info.nr_line_info = 0; 233311d8b82dSYonghong Song info.nr_jited_line_info = 0; 23341e270976SMartin KaFai Lau goto done; 23351e270976SMartin KaFai Lau } 23361e270976SMartin KaFai Lau 23371e270976SMartin KaFai Lau ulen = info.xlated_prog_len; 23389975a54bSDaniel Borkmann info.xlated_prog_len = bpf_prog_insn_size(prog); 23391e270976SMartin KaFai Lau if (info.xlated_prog_len && ulen) { 23407105e828SDaniel Borkmann struct bpf_insn *insns_sanitized; 23417105e828SDaniel Borkmann bool fault; 23427105e828SDaniel Borkmann 23437105e828SDaniel Borkmann if (prog->blinded && !bpf_dump_raw_ok()) { 23447105e828SDaniel Borkmann info.xlated_prog_insns = 0; 23457105e828SDaniel Borkmann goto done; 23467105e828SDaniel Borkmann } 23477105e828SDaniel Borkmann insns_sanitized = bpf_insn_prepare_dump(prog); 23487105e828SDaniel Borkmann if (!insns_sanitized) 23497105e828SDaniel Borkmann return -ENOMEM; 23501e270976SMartin KaFai Lau uinsns = u64_to_user_ptr(info.xlated_prog_insns); 23511e270976SMartin KaFai Lau ulen = min_t(u32, info.xlated_prog_len, ulen); 23527105e828SDaniel Borkmann fault = copy_to_user(uinsns, insns_sanitized, ulen); 23537105e828SDaniel Borkmann kfree(insns_sanitized); 23547105e828SDaniel Borkmann if (fault) 23551e270976SMartin KaFai Lau return -EFAULT; 23561e270976SMartin KaFai Lau } 23571e270976SMartin KaFai Lau 2358675fc275SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 2359675fc275SJakub Kicinski err = bpf_prog_offload_info_fill(&info, prog); 2360675fc275SJakub Kicinski if (err) 2361675fc275SJakub Kicinski return err; 2362fcfb126dSJiong Wang goto done; 2363fcfb126dSJiong Wang } 2364fcfb126dSJiong Wang 2365fcfb126dSJiong Wang /* NOTE: the following code is supposed to be skipped for offload. 2366fcfb126dSJiong Wang * bpf_prog_offload_info_fill() is the place to fill similar fields 2367fcfb126dSJiong Wang * for offload. 2368fcfb126dSJiong Wang */ 2369fcfb126dSJiong Wang ulen = info.jited_prog_len; 23704d56a76eSSandipan Das if (prog->aux->func_cnt) { 23714d56a76eSSandipan Das u32 i; 23724d56a76eSSandipan Das 23734d56a76eSSandipan Das info.jited_prog_len = 0; 23744d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) 23754d56a76eSSandipan Das info.jited_prog_len += prog->aux->func[i]->jited_len; 23764d56a76eSSandipan Das } else { 2377fcfb126dSJiong Wang info.jited_prog_len = prog->jited_len; 23784d56a76eSSandipan Das } 23794d56a76eSSandipan Das 2380fcfb126dSJiong Wang if (info.jited_prog_len && ulen) { 2381fcfb126dSJiong Wang if (bpf_dump_raw_ok()) { 2382fcfb126dSJiong Wang uinsns = u64_to_user_ptr(info.jited_prog_insns); 2383fcfb126dSJiong Wang ulen = min_t(u32, info.jited_prog_len, ulen); 23844d56a76eSSandipan Das 23854d56a76eSSandipan Das /* for multi-function programs, copy the JITed 23864d56a76eSSandipan Das * instructions for all the functions 23874d56a76eSSandipan Das */ 23884d56a76eSSandipan Das if (prog->aux->func_cnt) { 23894d56a76eSSandipan Das u32 len, free, i; 23904d56a76eSSandipan Das u8 *img; 23914d56a76eSSandipan Das 23924d56a76eSSandipan Das free = ulen; 23934d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) { 23944d56a76eSSandipan Das len = prog->aux->func[i]->jited_len; 23954d56a76eSSandipan Das len = min_t(u32, len, free); 23964d56a76eSSandipan Das img = (u8 *) prog->aux->func[i]->bpf_func; 23974d56a76eSSandipan Das if (copy_to_user(uinsns, img, len)) 23984d56a76eSSandipan Das return -EFAULT; 23994d56a76eSSandipan Das uinsns += len; 24004d56a76eSSandipan Das free -= len; 24014d56a76eSSandipan Das if (!free) 24024d56a76eSSandipan Das break; 24034d56a76eSSandipan Das } 24044d56a76eSSandipan Das } else { 2405fcfb126dSJiong Wang if (copy_to_user(uinsns, prog->bpf_func, ulen)) 2406fcfb126dSJiong Wang return -EFAULT; 24074d56a76eSSandipan Das } 2408fcfb126dSJiong Wang } else { 2409fcfb126dSJiong Wang info.jited_prog_insns = 0; 2410fcfb126dSJiong Wang } 2411675fc275SJakub Kicinski } 2412675fc275SJakub Kicinski 2413dbecd738SSandipan Das ulen = info.nr_jited_ksyms; 2414ff1889fcSSong Liu info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; 24157a5725ddSSong Liu if (ulen) { 2416dbecd738SSandipan Das if (bpf_dump_raw_ok()) { 2417ff1889fcSSong Liu unsigned long ksym_addr; 2418dbecd738SSandipan Das u64 __user *user_ksyms; 2419dbecd738SSandipan Das u32 i; 2420dbecd738SSandipan Das 2421dbecd738SSandipan Das /* copy the address of the kernel symbol 2422dbecd738SSandipan Das * corresponding to each function 2423dbecd738SSandipan Das */ 2424dbecd738SSandipan Das ulen = min_t(u32, info.nr_jited_ksyms, ulen); 2425dbecd738SSandipan Das user_ksyms = u64_to_user_ptr(info.jited_ksyms); 2426ff1889fcSSong Liu if (prog->aux->func_cnt) { 2427dbecd738SSandipan Das for (i = 0; i < ulen; i++) { 2428ff1889fcSSong Liu ksym_addr = (unsigned long) 2429ff1889fcSSong Liu prog->aux->func[i]->bpf_func; 2430ff1889fcSSong Liu if (put_user((u64) ksym_addr, 2431ff1889fcSSong Liu &user_ksyms[i])) 2432ff1889fcSSong Liu return -EFAULT; 2433ff1889fcSSong Liu } 2434ff1889fcSSong Liu } else { 2435ff1889fcSSong Liu ksym_addr = (unsigned long) prog->bpf_func; 2436ff1889fcSSong Liu if (put_user((u64) ksym_addr, &user_ksyms[0])) 2437dbecd738SSandipan Das return -EFAULT; 2438dbecd738SSandipan Das } 2439dbecd738SSandipan Das } else { 2440dbecd738SSandipan Das info.jited_ksyms = 0; 2441dbecd738SSandipan Das } 2442dbecd738SSandipan Das } 2443dbecd738SSandipan Das 2444815581c1SSandipan Das ulen = info.nr_jited_func_lens; 2445ff1889fcSSong Liu info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; 24467a5725ddSSong Liu if (ulen) { 2447815581c1SSandipan Das if (bpf_dump_raw_ok()) { 2448815581c1SSandipan Das u32 __user *user_lens; 2449815581c1SSandipan Das u32 func_len, i; 2450815581c1SSandipan Das 2451815581c1SSandipan Das /* copy the JITed image lengths for each function */ 2452815581c1SSandipan Das ulen = min_t(u32, info.nr_jited_func_lens, ulen); 2453815581c1SSandipan Das user_lens = u64_to_user_ptr(info.jited_func_lens); 2454ff1889fcSSong Liu if (prog->aux->func_cnt) { 2455815581c1SSandipan Das for (i = 0; i < ulen; i++) { 2456ff1889fcSSong Liu func_len = 2457ff1889fcSSong Liu prog->aux->func[i]->jited_len; 2458815581c1SSandipan Das if (put_user(func_len, &user_lens[i])) 2459815581c1SSandipan Das return -EFAULT; 2460815581c1SSandipan Das } 2461815581c1SSandipan Das } else { 2462ff1889fcSSong Liu func_len = prog->jited_len; 2463ff1889fcSSong Liu if (put_user(func_len, &user_lens[0])) 2464ff1889fcSSong Liu return -EFAULT; 2465ff1889fcSSong Liu } 2466ff1889fcSSong Liu } else { 2467815581c1SSandipan Das info.jited_func_lens = 0; 2468815581c1SSandipan Das } 2469815581c1SSandipan Das } 2470815581c1SSandipan Das 24717337224fSMartin KaFai Lau if (prog->aux->btf) 2472838e9690SYonghong Song info.btf_id = btf_id(prog->aux->btf); 2473838e9690SYonghong Song 247411d8b82dSYonghong Song ulen = info.nr_func_info; 247511d8b82dSYonghong Song info.nr_func_info = prog->aux->func_info_cnt; 247611d8b82dSYonghong Song if (info.nr_func_info && ulen) { 2477838e9690SYonghong Song char __user *user_finfo; 2478838e9690SYonghong Song 2479838e9690SYonghong Song user_finfo = u64_to_user_ptr(info.func_info); 248011d8b82dSYonghong Song ulen = min_t(u32, info.nr_func_info, ulen); 2481ba64e7d8SYonghong Song if (copy_to_user(user_finfo, prog->aux->func_info, 24827337224fSMartin KaFai Lau info.func_info_rec_size * ulen)) 2483838e9690SYonghong Song return -EFAULT; 2484838e9690SYonghong Song } 2485838e9690SYonghong Song 248611d8b82dSYonghong Song ulen = info.nr_line_info; 248711d8b82dSYonghong Song info.nr_line_info = prog->aux->nr_linfo; 248811d8b82dSYonghong Song if (info.nr_line_info && ulen) { 2489c454a46bSMartin KaFai Lau __u8 __user *user_linfo; 2490c454a46bSMartin KaFai Lau 2491c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.line_info); 249211d8b82dSYonghong Song ulen = min_t(u32, info.nr_line_info, ulen); 2493c454a46bSMartin KaFai Lau if (copy_to_user(user_linfo, prog->aux->linfo, 2494c454a46bSMartin KaFai Lau info.line_info_rec_size * ulen)) 2495c454a46bSMartin KaFai Lau return -EFAULT; 2496c454a46bSMartin KaFai Lau } 2497c454a46bSMartin KaFai Lau 249811d8b82dSYonghong Song ulen = info.nr_jited_line_info; 2499c454a46bSMartin KaFai Lau if (prog->aux->jited_linfo) 250011d8b82dSYonghong Song info.nr_jited_line_info = prog->aux->nr_linfo; 2501c454a46bSMartin KaFai Lau else 250211d8b82dSYonghong Song info.nr_jited_line_info = 0; 250311d8b82dSYonghong Song if (info.nr_jited_line_info && ulen) { 2504c454a46bSMartin KaFai Lau if (bpf_dump_raw_ok()) { 2505c454a46bSMartin KaFai Lau __u64 __user *user_linfo; 2506c454a46bSMartin KaFai Lau u32 i; 2507c454a46bSMartin KaFai Lau 2508c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.jited_line_info); 250911d8b82dSYonghong Song ulen = min_t(u32, info.nr_jited_line_info, ulen); 2510c454a46bSMartin KaFai Lau for (i = 0; i < ulen; i++) { 2511c454a46bSMartin KaFai Lau if (put_user((__u64)(long)prog->aux->jited_linfo[i], 2512c454a46bSMartin KaFai Lau &user_linfo[i])) 2513c454a46bSMartin KaFai Lau return -EFAULT; 2514c454a46bSMartin KaFai Lau } 2515c454a46bSMartin KaFai Lau } else { 2516c454a46bSMartin KaFai Lau info.jited_line_info = 0; 2517c454a46bSMartin KaFai Lau } 2518c454a46bSMartin KaFai Lau } 2519c454a46bSMartin KaFai Lau 2520c872bdb3SSong Liu ulen = info.nr_prog_tags; 2521c872bdb3SSong Liu info.nr_prog_tags = prog->aux->func_cnt ? : 1; 2522c872bdb3SSong Liu if (ulen) { 2523c872bdb3SSong Liu __u8 __user (*user_prog_tags)[BPF_TAG_SIZE]; 2524c872bdb3SSong Liu u32 i; 2525c872bdb3SSong Liu 2526c872bdb3SSong Liu user_prog_tags = u64_to_user_ptr(info.prog_tags); 2527c872bdb3SSong Liu ulen = min_t(u32, info.nr_prog_tags, ulen); 2528c872bdb3SSong Liu if (prog->aux->func_cnt) { 2529c872bdb3SSong Liu for (i = 0; i < ulen; i++) { 2530c872bdb3SSong Liu if (copy_to_user(user_prog_tags[i], 2531c872bdb3SSong Liu prog->aux->func[i]->tag, 2532c872bdb3SSong Liu BPF_TAG_SIZE)) 2533c872bdb3SSong Liu return -EFAULT; 2534c872bdb3SSong Liu } 2535c872bdb3SSong Liu } else { 2536c872bdb3SSong Liu if (copy_to_user(user_prog_tags[0], 2537c872bdb3SSong Liu prog->tag, BPF_TAG_SIZE)) 2538c872bdb3SSong Liu return -EFAULT; 2539c872bdb3SSong Liu } 2540c872bdb3SSong Liu } 2541c872bdb3SSong Liu 25421e270976SMartin KaFai Lau done: 25431e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 25441e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 25451e270976SMartin KaFai Lau return -EFAULT; 25461e270976SMartin KaFai Lau 25471e270976SMartin KaFai Lau return 0; 25481e270976SMartin KaFai Lau } 25491e270976SMartin KaFai Lau 25501e270976SMartin KaFai Lau static int bpf_map_get_info_by_fd(struct bpf_map *map, 25511e270976SMartin KaFai Lau const union bpf_attr *attr, 25521e270976SMartin KaFai Lau union bpf_attr __user *uattr) 25531e270976SMartin KaFai Lau { 25541e270976SMartin KaFai Lau struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 25551e270976SMartin KaFai Lau struct bpf_map_info info = {}; 25561e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 25571e270976SMartin KaFai Lau int err; 25581e270976SMartin KaFai Lau 2559dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); 25601e270976SMartin KaFai Lau if (err) 25611e270976SMartin KaFai Lau return err; 25621e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 25631e270976SMartin KaFai Lau 25641e270976SMartin KaFai Lau info.type = map->map_type; 25651e270976SMartin KaFai Lau info.id = map->id; 25661e270976SMartin KaFai Lau info.key_size = map->key_size; 25671e270976SMartin KaFai Lau info.value_size = map->value_size; 25681e270976SMartin KaFai Lau info.max_entries = map->max_entries; 25691e270976SMartin KaFai Lau info.map_flags = map->map_flags; 2570ad5b177bSMartin KaFai Lau memcpy(info.name, map->name, sizeof(map->name)); 25711e270976SMartin KaFai Lau 257278958fcaSMartin KaFai Lau if (map->btf) { 257378958fcaSMartin KaFai Lau info.btf_id = btf_id(map->btf); 25749b2cf328SMartin KaFai Lau info.btf_key_type_id = map->btf_key_type_id; 25759b2cf328SMartin KaFai Lau info.btf_value_type_id = map->btf_value_type_id; 257678958fcaSMartin KaFai Lau } 257778958fcaSMartin KaFai Lau 257852775b33SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 257952775b33SJakub Kicinski err = bpf_map_offload_info_fill(&info, map); 258052775b33SJakub Kicinski if (err) 258152775b33SJakub Kicinski return err; 258252775b33SJakub Kicinski } 258352775b33SJakub Kicinski 25841e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 25851e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 25861e270976SMartin KaFai Lau return -EFAULT; 25871e270976SMartin KaFai Lau 25881e270976SMartin KaFai Lau return 0; 25891e270976SMartin KaFai Lau } 25901e270976SMartin KaFai Lau 259162dab84cSMartin KaFai Lau static int bpf_btf_get_info_by_fd(struct btf *btf, 259262dab84cSMartin KaFai Lau const union bpf_attr *attr, 259362dab84cSMartin KaFai Lau union bpf_attr __user *uattr) 259462dab84cSMartin KaFai Lau { 259562dab84cSMartin KaFai Lau struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info); 259662dab84cSMartin KaFai Lau u32 info_len = attr->info.info_len; 259762dab84cSMartin KaFai Lau int err; 259862dab84cSMartin KaFai Lau 2599dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uinfo, sizeof(*uinfo), info_len); 260062dab84cSMartin KaFai Lau if (err) 260162dab84cSMartin KaFai Lau return err; 260262dab84cSMartin KaFai Lau 260362dab84cSMartin KaFai Lau return btf_get_info_by_fd(btf, attr, uattr); 260462dab84cSMartin KaFai Lau } 260562dab84cSMartin KaFai Lau 26061e270976SMartin KaFai Lau #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 26071e270976SMartin KaFai Lau 26081e270976SMartin KaFai Lau static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 26091e270976SMartin KaFai Lau union bpf_attr __user *uattr) 26101e270976SMartin KaFai Lau { 26111e270976SMartin KaFai Lau int ufd = attr->info.bpf_fd; 26121e270976SMartin KaFai Lau struct fd f; 26131e270976SMartin KaFai Lau int err; 26141e270976SMartin KaFai Lau 26151e270976SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 26161e270976SMartin KaFai Lau return -EINVAL; 26171e270976SMartin KaFai Lau 26181e270976SMartin KaFai Lau f = fdget(ufd); 26191e270976SMartin KaFai Lau if (!f.file) 26201e270976SMartin KaFai Lau return -EBADFD; 26211e270976SMartin KaFai Lau 26221e270976SMartin KaFai Lau if (f.file->f_op == &bpf_prog_fops) 26231e270976SMartin KaFai Lau err = bpf_prog_get_info_by_fd(f.file->private_data, attr, 26241e270976SMartin KaFai Lau uattr); 26251e270976SMartin KaFai Lau else if (f.file->f_op == &bpf_map_fops) 26261e270976SMartin KaFai Lau err = bpf_map_get_info_by_fd(f.file->private_data, attr, 26271e270976SMartin KaFai Lau uattr); 262860197cfbSMartin KaFai Lau else if (f.file->f_op == &btf_fops) 262962dab84cSMartin KaFai Lau err = bpf_btf_get_info_by_fd(f.file->private_data, attr, uattr); 26301e270976SMartin KaFai Lau else 26311e270976SMartin KaFai Lau err = -EINVAL; 26321e270976SMartin KaFai Lau 26331e270976SMartin KaFai Lau fdput(f); 26341e270976SMartin KaFai Lau return err; 26351e270976SMartin KaFai Lau } 26361e270976SMartin KaFai Lau 2637f56a653cSMartin KaFai Lau #define BPF_BTF_LOAD_LAST_FIELD btf_log_level 2638f56a653cSMartin KaFai Lau 2639f56a653cSMartin KaFai Lau static int bpf_btf_load(const union bpf_attr *attr) 2640f56a653cSMartin KaFai Lau { 2641f56a653cSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_LOAD)) 2642f56a653cSMartin KaFai Lau return -EINVAL; 2643f56a653cSMartin KaFai Lau 2644f56a653cSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 2645f56a653cSMartin KaFai Lau return -EPERM; 2646f56a653cSMartin KaFai Lau 2647f56a653cSMartin KaFai Lau return btf_new_fd(attr); 2648f56a653cSMartin KaFai Lau } 2649f56a653cSMartin KaFai Lau 265078958fcaSMartin KaFai Lau #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id 265178958fcaSMartin KaFai Lau 265278958fcaSMartin KaFai Lau static int bpf_btf_get_fd_by_id(const union bpf_attr *attr) 265378958fcaSMartin KaFai Lau { 265478958fcaSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID)) 265578958fcaSMartin KaFai Lau return -EINVAL; 265678958fcaSMartin KaFai Lau 265778958fcaSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 265878958fcaSMartin KaFai Lau return -EPERM; 265978958fcaSMartin KaFai Lau 266078958fcaSMartin KaFai Lau return btf_get_fd_by_id(attr->btf_id); 266178958fcaSMartin KaFai Lau } 266278958fcaSMartin KaFai Lau 266341bdc4b4SYonghong Song static int bpf_task_fd_query_copy(const union bpf_attr *attr, 266441bdc4b4SYonghong Song union bpf_attr __user *uattr, 266541bdc4b4SYonghong Song u32 prog_id, u32 fd_type, 266641bdc4b4SYonghong Song const char *buf, u64 probe_offset, 266741bdc4b4SYonghong Song u64 probe_addr) 266841bdc4b4SYonghong Song { 266941bdc4b4SYonghong Song char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf); 267041bdc4b4SYonghong Song u32 len = buf ? strlen(buf) : 0, input_len; 267141bdc4b4SYonghong Song int err = 0; 267241bdc4b4SYonghong Song 267341bdc4b4SYonghong Song if (put_user(len, &uattr->task_fd_query.buf_len)) 267441bdc4b4SYonghong Song return -EFAULT; 267541bdc4b4SYonghong Song input_len = attr->task_fd_query.buf_len; 267641bdc4b4SYonghong Song if (input_len && ubuf) { 267741bdc4b4SYonghong Song if (!len) { 267841bdc4b4SYonghong Song /* nothing to copy, just make ubuf NULL terminated */ 267941bdc4b4SYonghong Song char zero = '\0'; 268041bdc4b4SYonghong Song 268141bdc4b4SYonghong Song if (put_user(zero, ubuf)) 268241bdc4b4SYonghong Song return -EFAULT; 268341bdc4b4SYonghong Song } else if (input_len >= len + 1) { 268441bdc4b4SYonghong Song /* ubuf can hold the string with NULL terminator */ 268541bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, len + 1)) 268641bdc4b4SYonghong Song return -EFAULT; 268741bdc4b4SYonghong Song } else { 268841bdc4b4SYonghong Song /* ubuf cannot hold the string with NULL terminator, 268941bdc4b4SYonghong Song * do a partial copy with NULL terminator. 269041bdc4b4SYonghong Song */ 269141bdc4b4SYonghong Song char zero = '\0'; 269241bdc4b4SYonghong Song 269341bdc4b4SYonghong Song err = -ENOSPC; 269441bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, input_len - 1)) 269541bdc4b4SYonghong Song return -EFAULT; 269641bdc4b4SYonghong Song if (put_user(zero, ubuf + input_len - 1)) 269741bdc4b4SYonghong Song return -EFAULT; 269841bdc4b4SYonghong Song } 269941bdc4b4SYonghong Song } 270041bdc4b4SYonghong Song 270141bdc4b4SYonghong Song if (put_user(prog_id, &uattr->task_fd_query.prog_id) || 270241bdc4b4SYonghong Song put_user(fd_type, &uattr->task_fd_query.fd_type) || 270341bdc4b4SYonghong Song put_user(probe_offset, &uattr->task_fd_query.probe_offset) || 270441bdc4b4SYonghong Song put_user(probe_addr, &uattr->task_fd_query.probe_addr)) 270541bdc4b4SYonghong Song return -EFAULT; 270641bdc4b4SYonghong Song 270741bdc4b4SYonghong Song return err; 270841bdc4b4SYonghong Song } 270941bdc4b4SYonghong Song 271041bdc4b4SYonghong Song #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr 271141bdc4b4SYonghong Song 271241bdc4b4SYonghong Song static int bpf_task_fd_query(const union bpf_attr *attr, 271341bdc4b4SYonghong Song union bpf_attr __user *uattr) 271441bdc4b4SYonghong Song { 271541bdc4b4SYonghong Song pid_t pid = attr->task_fd_query.pid; 271641bdc4b4SYonghong Song u32 fd = attr->task_fd_query.fd; 271741bdc4b4SYonghong Song const struct perf_event *event; 271841bdc4b4SYonghong Song struct files_struct *files; 271941bdc4b4SYonghong Song struct task_struct *task; 272041bdc4b4SYonghong Song struct file *file; 272141bdc4b4SYonghong Song int err; 272241bdc4b4SYonghong Song 272341bdc4b4SYonghong Song if (CHECK_ATTR(BPF_TASK_FD_QUERY)) 272441bdc4b4SYonghong Song return -EINVAL; 272541bdc4b4SYonghong Song 272641bdc4b4SYonghong Song if (!capable(CAP_SYS_ADMIN)) 272741bdc4b4SYonghong Song return -EPERM; 272841bdc4b4SYonghong Song 272941bdc4b4SYonghong Song if (attr->task_fd_query.flags != 0) 273041bdc4b4SYonghong Song return -EINVAL; 273141bdc4b4SYonghong Song 273241bdc4b4SYonghong Song task = get_pid_task(find_vpid(pid), PIDTYPE_PID); 273341bdc4b4SYonghong Song if (!task) 273441bdc4b4SYonghong Song return -ENOENT; 273541bdc4b4SYonghong Song 273641bdc4b4SYonghong Song files = get_files_struct(task); 273741bdc4b4SYonghong Song put_task_struct(task); 273841bdc4b4SYonghong Song if (!files) 273941bdc4b4SYonghong Song return -ENOENT; 274041bdc4b4SYonghong Song 274141bdc4b4SYonghong Song err = 0; 274241bdc4b4SYonghong Song spin_lock(&files->file_lock); 274341bdc4b4SYonghong Song file = fcheck_files(files, fd); 274441bdc4b4SYonghong Song if (!file) 274541bdc4b4SYonghong Song err = -EBADF; 274641bdc4b4SYonghong Song else 274741bdc4b4SYonghong Song get_file(file); 274841bdc4b4SYonghong Song spin_unlock(&files->file_lock); 274941bdc4b4SYonghong Song put_files_struct(files); 275041bdc4b4SYonghong Song 275141bdc4b4SYonghong Song if (err) 275241bdc4b4SYonghong Song goto out; 275341bdc4b4SYonghong Song 275441bdc4b4SYonghong Song if (file->f_op == &bpf_raw_tp_fops) { 275541bdc4b4SYonghong Song struct bpf_raw_tracepoint *raw_tp = file->private_data; 275641bdc4b4SYonghong Song struct bpf_raw_event_map *btp = raw_tp->btp; 275741bdc4b4SYonghong Song 275841bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, 275941bdc4b4SYonghong Song raw_tp->prog->aux->id, 276041bdc4b4SYonghong Song BPF_FD_TYPE_RAW_TRACEPOINT, 276141bdc4b4SYonghong Song btp->tp->name, 0, 0); 276241bdc4b4SYonghong Song goto put_file; 276341bdc4b4SYonghong Song } 276441bdc4b4SYonghong Song 276541bdc4b4SYonghong Song event = perf_get_event(file); 276641bdc4b4SYonghong Song if (!IS_ERR(event)) { 276741bdc4b4SYonghong Song u64 probe_offset, probe_addr; 276841bdc4b4SYonghong Song u32 prog_id, fd_type; 276941bdc4b4SYonghong Song const char *buf; 277041bdc4b4SYonghong Song 277141bdc4b4SYonghong Song err = bpf_get_perf_event_info(event, &prog_id, &fd_type, 277241bdc4b4SYonghong Song &buf, &probe_offset, 277341bdc4b4SYonghong Song &probe_addr); 277441bdc4b4SYonghong Song if (!err) 277541bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, prog_id, 277641bdc4b4SYonghong Song fd_type, buf, 277741bdc4b4SYonghong Song probe_offset, 277841bdc4b4SYonghong Song probe_addr); 277941bdc4b4SYonghong Song goto put_file; 278041bdc4b4SYonghong Song } 278141bdc4b4SYonghong Song 278241bdc4b4SYonghong Song err = -ENOTSUPP; 278341bdc4b4SYonghong Song put_file: 278441bdc4b4SYonghong Song fput(file); 278541bdc4b4SYonghong Song out: 278641bdc4b4SYonghong Song return err; 278741bdc4b4SYonghong Song } 278841bdc4b4SYonghong Song 278999c55f7dSAlexei Starovoitov SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 279099c55f7dSAlexei Starovoitov { 279199c55f7dSAlexei Starovoitov union bpf_attr attr = {}; 279299c55f7dSAlexei Starovoitov int err; 279399c55f7dSAlexei Starovoitov 27940fa4fe85SChenbo Feng if (sysctl_unprivileged_bpf_disabled && !capable(CAP_SYS_ADMIN)) 279599c55f7dSAlexei Starovoitov return -EPERM; 279699c55f7dSAlexei Starovoitov 2797dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size); 279899c55f7dSAlexei Starovoitov if (err) 279999c55f7dSAlexei Starovoitov return err; 28001e270976SMartin KaFai Lau size = min_t(u32, size, sizeof(attr)); 280199c55f7dSAlexei Starovoitov 280299c55f7dSAlexei Starovoitov /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 280399c55f7dSAlexei Starovoitov if (copy_from_user(&attr, uattr, size) != 0) 280499c55f7dSAlexei Starovoitov return -EFAULT; 280599c55f7dSAlexei Starovoitov 2806afdb09c7SChenbo Feng err = security_bpf(cmd, &attr, size); 2807afdb09c7SChenbo Feng if (err < 0) 2808afdb09c7SChenbo Feng return err; 2809afdb09c7SChenbo Feng 281099c55f7dSAlexei Starovoitov switch (cmd) { 281199c55f7dSAlexei Starovoitov case BPF_MAP_CREATE: 281299c55f7dSAlexei Starovoitov err = map_create(&attr); 281399c55f7dSAlexei Starovoitov break; 2814db20fd2bSAlexei Starovoitov case BPF_MAP_LOOKUP_ELEM: 2815db20fd2bSAlexei Starovoitov err = map_lookup_elem(&attr); 2816db20fd2bSAlexei Starovoitov break; 2817db20fd2bSAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 2818db20fd2bSAlexei Starovoitov err = map_update_elem(&attr); 2819db20fd2bSAlexei Starovoitov break; 2820db20fd2bSAlexei Starovoitov case BPF_MAP_DELETE_ELEM: 2821db20fd2bSAlexei Starovoitov err = map_delete_elem(&attr); 2822db20fd2bSAlexei Starovoitov break; 2823db20fd2bSAlexei Starovoitov case BPF_MAP_GET_NEXT_KEY: 2824db20fd2bSAlexei Starovoitov err = map_get_next_key(&attr); 2825db20fd2bSAlexei Starovoitov break; 282687df15deSDaniel Borkmann case BPF_MAP_FREEZE: 282787df15deSDaniel Borkmann err = map_freeze(&attr); 282887df15deSDaniel Borkmann break; 282909756af4SAlexei Starovoitov case BPF_PROG_LOAD: 2830838e9690SYonghong Song err = bpf_prog_load(&attr, uattr); 283109756af4SAlexei Starovoitov break; 2832b2197755SDaniel Borkmann case BPF_OBJ_PIN: 2833b2197755SDaniel Borkmann err = bpf_obj_pin(&attr); 2834b2197755SDaniel Borkmann break; 2835b2197755SDaniel Borkmann case BPF_OBJ_GET: 2836b2197755SDaniel Borkmann err = bpf_obj_get(&attr); 2837b2197755SDaniel Borkmann break; 2838f4324551SDaniel Mack case BPF_PROG_ATTACH: 2839f4324551SDaniel Mack err = bpf_prog_attach(&attr); 2840f4324551SDaniel Mack break; 2841f4324551SDaniel Mack case BPF_PROG_DETACH: 2842f4324551SDaniel Mack err = bpf_prog_detach(&attr); 2843f4324551SDaniel Mack break; 2844468e2f64SAlexei Starovoitov case BPF_PROG_QUERY: 2845468e2f64SAlexei Starovoitov err = bpf_prog_query(&attr, uattr); 2846468e2f64SAlexei Starovoitov break; 28471cf1cae9SAlexei Starovoitov case BPF_PROG_TEST_RUN: 28481cf1cae9SAlexei Starovoitov err = bpf_prog_test_run(&attr, uattr); 28491cf1cae9SAlexei Starovoitov break; 285034ad5580SMartin KaFai Lau case BPF_PROG_GET_NEXT_ID: 285134ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 285234ad5580SMartin KaFai Lau &prog_idr, &prog_idr_lock); 285334ad5580SMartin KaFai Lau break; 285434ad5580SMartin KaFai Lau case BPF_MAP_GET_NEXT_ID: 285534ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 285634ad5580SMartin KaFai Lau &map_idr, &map_idr_lock); 285734ad5580SMartin KaFai Lau break; 2858b16d9aa4SMartin KaFai Lau case BPF_PROG_GET_FD_BY_ID: 2859b16d9aa4SMartin KaFai Lau err = bpf_prog_get_fd_by_id(&attr); 2860b16d9aa4SMartin KaFai Lau break; 2861bd5f5f4eSMartin KaFai Lau case BPF_MAP_GET_FD_BY_ID: 2862bd5f5f4eSMartin KaFai Lau err = bpf_map_get_fd_by_id(&attr); 2863bd5f5f4eSMartin KaFai Lau break; 28641e270976SMartin KaFai Lau case BPF_OBJ_GET_INFO_BY_FD: 28651e270976SMartin KaFai Lau err = bpf_obj_get_info_by_fd(&attr, uattr); 28661e270976SMartin KaFai Lau break; 2867c4f6699dSAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN: 2868c4f6699dSAlexei Starovoitov err = bpf_raw_tracepoint_open(&attr); 2869c4f6699dSAlexei Starovoitov break; 2870f56a653cSMartin KaFai Lau case BPF_BTF_LOAD: 2871f56a653cSMartin KaFai Lau err = bpf_btf_load(&attr); 2872f56a653cSMartin KaFai Lau break; 287378958fcaSMartin KaFai Lau case BPF_BTF_GET_FD_BY_ID: 287478958fcaSMartin KaFai Lau err = bpf_btf_get_fd_by_id(&attr); 287578958fcaSMartin KaFai Lau break; 287641bdc4b4SYonghong Song case BPF_TASK_FD_QUERY: 287741bdc4b4SYonghong Song err = bpf_task_fd_query(&attr, uattr); 287841bdc4b4SYonghong Song break; 2879bd513cd0SMauricio Vasquez B case BPF_MAP_LOOKUP_AND_DELETE_ELEM: 2880bd513cd0SMauricio Vasquez B err = map_lookup_and_delete_elem(&attr); 2881bd513cd0SMauricio Vasquez B break; 288299c55f7dSAlexei Starovoitov default: 288399c55f7dSAlexei Starovoitov err = -EINVAL; 288499c55f7dSAlexei Starovoitov break; 288599c55f7dSAlexei Starovoitov } 288699c55f7dSAlexei Starovoitov 288799c55f7dSAlexei Starovoitov return err; 288899c55f7dSAlexei Starovoitov } 2889