199c55f7dSAlexei Starovoitov /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 299c55f7dSAlexei Starovoitov * 399c55f7dSAlexei Starovoitov * This program is free software; you can redistribute it and/or 499c55f7dSAlexei Starovoitov * modify it under the terms of version 2 of the GNU General Public 599c55f7dSAlexei Starovoitov * License as published by the Free Software Foundation. 699c55f7dSAlexei Starovoitov * 799c55f7dSAlexei Starovoitov * This program is distributed in the hope that it will be useful, but 899c55f7dSAlexei Starovoitov * WITHOUT ANY WARRANTY; without even the implied warranty of 999c55f7dSAlexei Starovoitov * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 1099c55f7dSAlexei Starovoitov * General Public License for more details. 1199c55f7dSAlexei Starovoitov */ 1299c55f7dSAlexei Starovoitov #include <linux/bpf.h> 13a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h> 1499c55f7dSAlexei Starovoitov #include <linux/syscalls.h> 1599c55f7dSAlexei Starovoitov #include <linux/slab.h> 163f07c014SIngo Molnar #include <linux/sched/signal.h> 17d407bd25SDaniel Borkmann #include <linux/vmalloc.h> 18d407bd25SDaniel Borkmann #include <linux/mmzone.h> 1999c55f7dSAlexei Starovoitov #include <linux/anon_inodes.h> 20db20fd2bSAlexei Starovoitov #include <linux/file.h> 2109756af4SAlexei Starovoitov #include <linux/license.h> 2209756af4SAlexei Starovoitov #include <linux/filter.h> 232541517cSAlexei Starovoitov #include <linux/version.h> 24535e7b4bSMickaël Salaün #include <linux/kernel.h> 25dc4bb0e2SMartin KaFai Lau #include <linux/idr.h> 26cb4d2b3fSMartin KaFai Lau #include <linux/cred.h> 27cb4d2b3fSMartin KaFai Lau #include <linux/timekeeping.h> 28cb4d2b3fSMartin KaFai Lau #include <linux/ctype.h> 2999c55f7dSAlexei Starovoitov 3014dc6f04SMartin KaFai Lau #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ 3114dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 3214dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 3314dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 3414dc6f04SMartin KaFai Lau #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 3514dc6f04SMartin KaFai Lau #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map)) 3614dc6f04SMartin KaFai Lau 376e71b04aSChenbo Feng #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) 386e71b04aSChenbo Feng 39b121d1e7SAlexei Starovoitov DEFINE_PER_CPU(int, bpf_prog_active); 40dc4bb0e2SMartin KaFai Lau static DEFINE_IDR(prog_idr); 41dc4bb0e2SMartin KaFai Lau static DEFINE_SPINLOCK(prog_idr_lock); 42f3f1c054SMartin KaFai Lau static DEFINE_IDR(map_idr); 43f3f1c054SMartin KaFai Lau static DEFINE_SPINLOCK(map_idr_lock); 44b121d1e7SAlexei Starovoitov 451be7f75dSAlexei Starovoitov int sysctl_unprivileged_bpf_disabled __read_mostly; 461be7f75dSAlexei Starovoitov 4740077e0cSJohannes Berg static const struct bpf_map_ops * const bpf_map_types[] = { 4840077e0cSJohannes Berg #define BPF_PROG_TYPE(_id, _ops) 4940077e0cSJohannes Berg #define BPF_MAP_TYPE(_id, _ops) \ 5040077e0cSJohannes Berg [_id] = &_ops, 5140077e0cSJohannes Berg #include <linux/bpf_types.h> 5240077e0cSJohannes Berg #undef BPF_PROG_TYPE 5340077e0cSJohannes Berg #undef BPF_MAP_TYPE 5440077e0cSJohannes Berg }; 5599c55f7dSAlexei Starovoitov 56752ba56fSMickaël Salaün /* 57752ba56fSMickaël Salaün * If we're handed a bigger struct than we know of, ensure all the unknown bits 58752ba56fSMickaël Salaün * are 0 - i.e. new user-space does not rely on any kernel feature extensions 59752ba56fSMickaël Salaün * we don't know about yet. 60752ba56fSMickaël Salaün * 61752ba56fSMickaël Salaün * There is a ToCToU between this function call and the following 62752ba56fSMickaël Salaün * copy_from_user() call. However, this is not a concern since this function is 63752ba56fSMickaël Salaün * meant to be a future-proofing of bits. 64752ba56fSMickaël Salaün */ 6558291a74SMickaël Salaün static int check_uarg_tail_zero(void __user *uaddr, 6658291a74SMickaël Salaün size_t expected_size, 6758291a74SMickaël Salaün size_t actual_size) 6858291a74SMickaël Salaün { 6958291a74SMickaël Salaün unsigned char __user *addr; 7058291a74SMickaël Salaün unsigned char __user *end; 7158291a74SMickaël Salaün unsigned char val; 7258291a74SMickaël Salaün int err; 7358291a74SMickaël Salaün 74752ba56fSMickaël Salaün if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 75752ba56fSMickaël Salaün return -E2BIG; 76752ba56fSMickaël Salaün 77752ba56fSMickaël Salaün if (unlikely(!access_ok(VERIFY_READ, uaddr, actual_size))) 78752ba56fSMickaël Salaün return -EFAULT; 79752ba56fSMickaël Salaün 8058291a74SMickaël Salaün if (actual_size <= expected_size) 8158291a74SMickaël Salaün return 0; 8258291a74SMickaël Salaün 8358291a74SMickaël Salaün addr = uaddr + expected_size; 8458291a74SMickaël Salaün end = uaddr + actual_size; 8558291a74SMickaël Salaün 8658291a74SMickaël Salaün for (; addr < end; addr++) { 8758291a74SMickaël Salaün err = get_user(val, addr); 8858291a74SMickaël Salaün if (err) 8958291a74SMickaël Salaün return err; 9058291a74SMickaël Salaün if (val) 9158291a74SMickaël Salaün return -E2BIG; 9258291a74SMickaël Salaün } 9358291a74SMickaël Salaün 9458291a74SMickaël Salaün return 0; 9558291a74SMickaël Salaün } 9658291a74SMickaël Salaün 97a3884572SJakub Kicinski const struct bpf_map_ops bpf_map_offload_ops = { 98a3884572SJakub Kicinski .map_alloc = bpf_map_offload_map_alloc, 99a3884572SJakub Kicinski .map_free = bpf_map_offload_map_free, 100a3884572SJakub Kicinski }; 101a3884572SJakub Kicinski 10299c55f7dSAlexei Starovoitov static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 10399c55f7dSAlexei Starovoitov { 1041110f3a9SJakub Kicinski const struct bpf_map_ops *ops; 10599c55f7dSAlexei Starovoitov struct bpf_map *map; 1061110f3a9SJakub Kicinski int err; 10799c55f7dSAlexei Starovoitov 1081110f3a9SJakub Kicinski if (attr->map_type >= ARRAY_SIZE(bpf_map_types)) 1091110f3a9SJakub Kicinski return ERR_PTR(-EINVAL); 1101110f3a9SJakub Kicinski ops = bpf_map_types[attr->map_type]; 1111110f3a9SJakub Kicinski if (!ops) 11240077e0cSJohannes Berg return ERR_PTR(-EINVAL); 11340077e0cSJohannes Berg 1141110f3a9SJakub Kicinski if (ops->map_alloc_check) { 1151110f3a9SJakub Kicinski err = ops->map_alloc_check(attr); 1161110f3a9SJakub Kicinski if (err) 1171110f3a9SJakub Kicinski return ERR_PTR(err); 1181110f3a9SJakub Kicinski } 119a3884572SJakub Kicinski if (attr->map_ifindex) 120a3884572SJakub Kicinski ops = &bpf_map_offload_ops; 1211110f3a9SJakub Kicinski map = ops->map_alloc(attr); 12299c55f7dSAlexei Starovoitov if (IS_ERR(map)) 12399c55f7dSAlexei Starovoitov return map; 1241110f3a9SJakub Kicinski map->ops = ops; 12599c55f7dSAlexei Starovoitov map->map_type = attr->map_type; 12699c55f7dSAlexei Starovoitov return map; 12799c55f7dSAlexei Starovoitov } 12899c55f7dSAlexei Starovoitov 12996eabe7aSMartin KaFai Lau void *bpf_map_area_alloc(size_t size, int numa_node) 130d407bd25SDaniel Borkmann { 131d407bd25SDaniel Borkmann /* We definitely need __GFP_NORETRY, so OOM killer doesn't 132d407bd25SDaniel Borkmann * trigger under memory pressure as we really just want to 133d407bd25SDaniel Borkmann * fail instead. 134d407bd25SDaniel Borkmann */ 135d407bd25SDaniel Borkmann const gfp_t flags = __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO; 136d407bd25SDaniel Borkmann void *area; 137d407bd25SDaniel Borkmann 138d407bd25SDaniel Borkmann if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 13996eabe7aSMartin KaFai Lau area = kmalloc_node(size, GFP_USER | flags, numa_node); 140d407bd25SDaniel Borkmann if (area != NULL) 141d407bd25SDaniel Borkmann return area; 142d407bd25SDaniel Borkmann } 143d407bd25SDaniel Borkmann 14496eabe7aSMartin KaFai Lau return __vmalloc_node_flags_caller(size, numa_node, GFP_KERNEL | flags, 14596eabe7aSMartin KaFai Lau __builtin_return_address(0)); 146d407bd25SDaniel Borkmann } 147d407bd25SDaniel Borkmann 148d407bd25SDaniel Borkmann void bpf_map_area_free(void *area) 149d407bd25SDaniel Borkmann { 150d407bd25SDaniel Borkmann kvfree(area); 151d407bd25SDaniel Borkmann } 152d407bd25SDaniel Borkmann 153bd475643SJakub Kicinski void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) 154bd475643SJakub Kicinski { 155bd475643SJakub Kicinski map->map_type = attr->map_type; 156bd475643SJakub Kicinski map->key_size = attr->key_size; 157bd475643SJakub Kicinski map->value_size = attr->value_size; 158bd475643SJakub Kicinski map->max_entries = attr->max_entries; 159bd475643SJakub Kicinski map->map_flags = attr->map_flags; 160bd475643SJakub Kicinski map->numa_node = bpf_map_attr_numa_node(attr); 161bd475643SJakub Kicinski } 162bd475643SJakub Kicinski 1636c905981SAlexei Starovoitov int bpf_map_precharge_memlock(u32 pages) 1646c905981SAlexei Starovoitov { 1656c905981SAlexei Starovoitov struct user_struct *user = get_current_user(); 1666c905981SAlexei Starovoitov unsigned long memlock_limit, cur; 1676c905981SAlexei Starovoitov 1686c905981SAlexei Starovoitov memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 1696c905981SAlexei Starovoitov cur = atomic_long_read(&user->locked_vm); 1706c905981SAlexei Starovoitov free_uid(user); 1716c905981SAlexei Starovoitov if (cur + pages > memlock_limit) 1726c905981SAlexei Starovoitov return -EPERM; 1736c905981SAlexei Starovoitov return 0; 1746c905981SAlexei Starovoitov } 1756c905981SAlexei Starovoitov 176aaac3ba9SAlexei Starovoitov static int bpf_map_charge_memlock(struct bpf_map *map) 177aaac3ba9SAlexei Starovoitov { 178aaac3ba9SAlexei Starovoitov struct user_struct *user = get_current_user(); 179aaac3ba9SAlexei Starovoitov unsigned long memlock_limit; 180aaac3ba9SAlexei Starovoitov 181aaac3ba9SAlexei Starovoitov memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 182aaac3ba9SAlexei Starovoitov 183aaac3ba9SAlexei Starovoitov atomic_long_add(map->pages, &user->locked_vm); 184aaac3ba9SAlexei Starovoitov 185aaac3ba9SAlexei Starovoitov if (atomic_long_read(&user->locked_vm) > memlock_limit) { 186aaac3ba9SAlexei Starovoitov atomic_long_sub(map->pages, &user->locked_vm); 187aaac3ba9SAlexei Starovoitov free_uid(user); 188aaac3ba9SAlexei Starovoitov return -EPERM; 189aaac3ba9SAlexei Starovoitov } 190aaac3ba9SAlexei Starovoitov map->user = user; 191aaac3ba9SAlexei Starovoitov return 0; 192aaac3ba9SAlexei Starovoitov } 193aaac3ba9SAlexei Starovoitov 194aaac3ba9SAlexei Starovoitov static void bpf_map_uncharge_memlock(struct bpf_map *map) 195aaac3ba9SAlexei Starovoitov { 196aaac3ba9SAlexei Starovoitov struct user_struct *user = map->user; 197aaac3ba9SAlexei Starovoitov 198aaac3ba9SAlexei Starovoitov atomic_long_sub(map->pages, &user->locked_vm); 199aaac3ba9SAlexei Starovoitov free_uid(user); 200aaac3ba9SAlexei Starovoitov } 201aaac3ba9SAlexei Starovoitov 202f3f1c054SMartin KaFai Lau static int bpf_map_alloc_id(struct bpf_map *map) 203f3f1c054SMartin KaFai Lau { 204f3f1c054SMartin KaFai Lau int id; 205f3f1c054SMartin KaFai Lau 206b76354cdSShaohua Li idr_preload(GFP_KERNEL); 207f3f1c054SMartin KaFai Lau spin_lock_bh(&map_idr_lock); 208f3f1c054SMartin KaFai Lau id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 209f3f1c054SMartin KaFai Lau if (id > 0) 210f3f1c054SMartin KaFai Lau map->id = id; 211f3f1c054SMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 212b76354cdSShaohua Li idr_preload_end(); 213f3f1c054SMartin KaFai Lau 214f3f1c054SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 215f3f1c054SMartin KaFai Lau return -ENOSPC; 216f3f1c054SMartin KaFai Lau 217f3f1c054SMartin KaFai Lau return id > 0 ? 0 : id; 218f3f1c054SMartin KaFai Lau } 219f3f1c054SMartin KaFai Lau 220a3884572SJakub Kicinski void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) 221f3f1c054SMartin KaFai Lau { 222930651a7SEric Dumazet unsigned long flags; 223930651a7SEric Dumazet 224a3884572SJakub Kicinski /* Offloaded maps are removed from the IDR store when their device 225a3884572SJakub Kicinski * disappears - even if someone holds an fd to them they are unusable, 226a3884572SJakub Kicinski * the memory is gone, all ops will fail; they are simply waiting for 227a3884572SJakub Kicinski * refcnt to drop to be freed. 228a3884572SJakub Kicinski */ 229a3884572SJakub Kicinski if (!map->id) 230a3884572SJakub Kicinski return; 231a3884572SJakub Kicinski 232bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 233930651a7SEric Dumazet spin_lock_irqsave(&map_idr_lock, flags); 234bd5f5f4eSMartin KaFai Lau else 235bd5f5f4eSMartin KaFai Lau __acquire(&map_idr_lock); 236bd5f5f4eSMartin KaFai Lau 237f3f1c054SMartin KaFai Lau idr_remove(&map_idr, map->id); 238a3884572SJakub Kicinski map->id = 0; 239bd5f5f4eSMartin KaFai Lau 240bd5f5f4eSMartin KaFai Lau if (do_idr_lock) 241930651a7SEric Dumazet spin_unlock_irqrestore(&map_idr_lock, flags); 242bd5f5f4eSMartin KaFai Lau else 243bd5f5f4eSMartin KaFai Lau __release(&map_idr_lock); 244f3f1c054SMartin KaFai Lau } 245f3f1c054SMartin KaFai Lau 24699c55f7dSAlexei Starovoitov /* called from workqueue */ 24799c55f7dSAlexei Starovoitov static void bpf_map_free_deferred(struct work_struct *work) 24899c55f7dSAlexei Starovoitov { 24999c55f7dSAlexei Starovoitov struct bpf_map *map = container_of(work, struct bpf_map, work); 25099c55f7dSAlexei Starovoitov 251aaac3ba9SAlexei Starovoitov bpf_map_uncharge_memlock(map); 252afdb09c7SChenbo Feng security_bpf_map_free(map); 25399c55f7dSAlexei Starovoitov /* implementation dependent freeing */ 25499c55f7dSAlexei Starovoitov map->ops->map_free(map); 25599c55f7dSAlexei Starovoitov } 25699c55f7dSAlexei Starovoitov 257c9da161cSDaniel Borkmann static void bpf_map_put_uref(struct bpf_map *map) 258c9da161cSDaniel Borkmann { 259c9da161cSDaniel Borkmann if (atomic_dec_and_test(&map->usercnt)) { 260c9da161cSDaniel Borkmann if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) 261c9da161cSDaniel Borkmann bpf_fd_array_map_clear(map); 262c9da161cSDaniel Borkmann } 263c9da161cSDaniel Borkmann } 264c9da161cSDaniel Borkmann 26599c55f7dSAlexei Starovoitov /* decrement map refcnt and schedule it for freeing via workqueue 26699c55f7dSAlexei Starovoitov * (unrelying map implementation ops->map_free() might sleep) 26799c55f7dSAlexei Starovoitov */ 268bd5f5f4eSMartin KaFai Lau static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) 26999c55f7dSAlexei Starovoitov { 27099c55f7dSAlexei Starovoitov if (atomic_dec_and_test(&map->refcnt)) { 27134ad5580SMartin KaFai Lau /* bpf_map_free_id() must be called first */ 272bd5f5f4eSMartin KaFai Lau bpf_map_free_id(map, do_idr_lock); 27399c55f7dSAlexei Starovoitov INIT_WORK(&map->work, bpf_map_free_deferred); 27499c55f7dSAlexei Starovoitov schedule_work(&map->work); 27599c55f7dSAlexei Starovoitov } 27699c55f7dSAlexei Starovoitov } 27799c55f7dSAlexei Starovoitov 278bd5f5f4eSMartin KaFai Lau void bpf_map_put(struct bpf_map *map) 279bd5f5f4eSMartin KaFai Lau { 280bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, true); 281bd5f5f4eSMartin KaFai Lau } 282bd5f5f4eSMartin KaFai Lau 283c9da161cSDaniel Borkmann void bpf_map_put_with_uref(struct bpf_map *map) 284c9da161cSDaniel Borkmann { 285c9da161cSDaniel Borkmann bpf_map_put_uref(map); 286c9da161cSDaniel Borkmann bpf_map_put(map); 287c9da161cSDaniel Borkmann } 288c9da161cSDaniel Borkmann 28999c55f7dSAlexei Starovoitov static int bpf_map_release(struct inode *inode, struct file *filp) 29099c55f7dSAlexei Starovoitov { 29161d1b6a4SDaniel Borkmann struct bpf_map *map = filp->private_data; 29261d1b6a4SDaniel Borkmann 29361d1b6a4SDaniel Borkmann if (map->ops->map_release) 29461d1b6a4SDaniel Borkmann map->ops->map_release(map, filp); 29561d1b6a4SDaniel Borkmann 29661d1b6a4SDaniel Borkmann bpf_map_put_with_uref(map); 29799c55f7dSAlexei Starovoitov return 0; 29899c55f7dSAlexei Starovoitov } 29999c55f7dSAlexei Starovoitov 300f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 301f99bf205SDaniel Borkmann static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 302f99bf205SDaniel Borkmann { 303f99bf205SDaniel Borkmann const struct bpf_map *map = filp->private_data; 30421116b70SDaniel Borkmann const struct bpf_array *array; 30521116b70SDaniel Borkmann u32 owner_prog_type = 0; 3069780c0abSDaniel Borkmann u32 owner_jited = 0; 30721116b70SDaniel Borkmann 30821116b70SDaniel Borkmann if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 30921116b70SDaniel Borkmann array = container_of(map, struct bpf_array, map); 31021116b70SDaniel Borkmann owner_prog_type = array->owner_prog_type; 3119780c0abSDaniel Borkmann owner_jited = array->owner_jited; 31221116b70SDaniel Borkmann } 313f99bf205SDaniel Borkmann 314f99bf205SDaniel Borkmann seq_printf(m, 315f99bf205SDaniel Borkmann "map_type:\t%u\n" 316f99bf205SDaniel Borkmann "key_size:\t%u\n" 317f99bf205SDaniel Borkmann "value_size:\t%u\n" 318322cea2fSDaniel Borkmann "max_entries:\t%u\n" 31921116b70SDaniel Borkmann "map_flags:\t%#x\n" 32021116b70SDaniel Borkmann "memlock:\t%llu\n", 321f99bf205SDaniel Borkmann map->map_type, 322f99bf205SDaniel Borkmann map->key_size, 323f99bf205SDaniel Borkmann map->value_size, 324322cea2fSDaniel Borkmann map->max_entries, 32521116b70SDaniel Borkmann map->map_flags, 32621116b70SDaniel Borkmann map->pages * 1ULL << PAGE_SHIFT); 32721116b70SDaniel Borkmann 3289780c0abSDaniel Borkmann if (owner_prog_type) { 32921116b70SDaniel Borkmann seq_printf(m, "owner_prog_type:\t%u\n", 33021116b70SDaniel Borkmann owner_prog_type); 3319780c0abSDaniel Borkmann seq_printf(m, "owner_jited:\t%u\n", 3329780c0abSDaniel Borkmann owner_jited); 3339780c0abSDaniel Borkmann } 334f99bf205SDaniel Borkmann } 335f99bf205SDaniel Borkmann #endif 336f99bf205SDaniel Borkmann 3376e71b04aSChenbo Feng static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, 3386e71b04aSChenbo Feng loff_t *ppos) 3396e71b04aSChenbo Feng { 3406e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 3416e71b04aSChenbo Feng * f_mode with FMODE_CAN_READ. 3426e71b04aSChenbo Feng */ 3436e71b04aSChenbo Feng return -EINVAL; 3446e71b04aSChenbo Feng } 3456e71b04aSChenbo Feng 3466e71b04aSChenbo Feng static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, 3476e71b04aSChenbo Feng size_t siz, loff_t *ppos) 3486e71b04aSChenbo Feng { 3496e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables 3506e71b04aSChenbo Feng * f_mode with FMODE_CAN_WRITE. 3516e71b04aSChenbo Feng */ 3526e71b04aSChenbo Feng return -EINVAL; 3536e71b04aSChenbo Feng } 3546e71b04aSChenbo Feng 355f66e448cSChenbo Feng const struct file_operations bpf_map_fops = { 356f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS 357f99bf205SDaniel Borkmann .show_fdinfo = bpf_map_show_fdinfo, 358f99bf205SDaniel Borkmann #endif 35999c55f7dSAlexei Starovoitov .release = bpf_map_release, 3606e71b04aSChenbo Feng .read = bpf_dummy_read, 3616e71b04aSChenbo Feng .write = bpf_dummy_write, 36299c55f7dSAlexei Starovoitov }; 36399c55f7dSAlexei Starovoitov 3646e71b04aSChenbo Feng int bpf_map_new_fd(struct bpf_map *map, int flags) 365aa79781bSDaniel Borkmann { 366afdb09c7SChenbo Feng int ret; 367afdb09c7SChenbo Feng 368afdb09c7SChenbo Feng ret = security_bpf_map(map, OPEN_FMODE(flags)); 369afdb09c7SChenbo Feng if (ret < 0) 370afdb09c7SChenbo Feng return ret; 371afdb09c7SChenbo Feng 372aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 3736e71b04aSChenbo Feng flags | O_CLOEXEC); 3746e71b04aSChenbo Feng } 3756e71b04aSChenbo Feng 3766e71b04aSChenbo Feng int bpf_get_file_flag(int flags) 3776e71b04aSChenbo Feng { 3786e71b04aSChenbo Feng if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) 3796e71b04aSChenbo Feng return -EINVAL; 3806e71b04aSChenbo Feng if (flags & BPF_F_RDONLY) 3816e71b04aSChenbo Feng return O_RDONLY; 3826e71b04aSChenbo Feng if (flags & BPF_F_WRONLY) 3836e71b04aSChenbo Feng return O_WRONLY; 3846e71b04aSChenbo Feng return O_RDWR; 385aa79781bSDaniel Borkmann } 386aa79781bSDaniel Borkmann 38799c55f7dSAlexei Starovoitov /* helper macro to check that unused fields 'union bpf_attr' are zero */ 38899c55f7dSAlexei Starovoitov #define CHECK_ATTR(CMD) \ 38999c55f7dSAlexei Starovoitov memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 39099c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD), 0, \ 39199c55f7dSAlexei Starovoitov sizeof(*attr) - \ 39299c55f7dSAlexei Starovoitov offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 39399c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD)) != NULL 39499c55f7dSAlexei Starovoitov 395cb4d2b3fSMartin KaFai Lau /* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes. 396cb4d2b3fSMartin KaFai Lau * Return 0 on success and < 0 on error. 397cb4d2b3fSMartin KaFai Lau */ 398cb4d2b3fSMartin KaFai Lau static int bpf_obj_name_cpy(char *dst, const char *src) 399cb4d2b3fSMartin KaFai Lau { 400cb4d2b3fSMartin KaFai Lau const char *end = src + BPF_OBJ_NAME_LEN; 401cb4d2b3fSMartin KaFai Lau 402473d9734SMartin KaFai Lau memset(dst, 0, BPF_OBJ_NAME_LEN); 403473d9734SMartin KaFai Lau 404cb4d2b3fSMartin KaFai Lau /* Copy all isalnum() and '_' char */ 405cb4d2b3fSMartin KaFai Lau while (src < end && *src) { 406cb4d2b3fSMartin KaFai Lau if (!isalnum(*src) && *src != '_') 407cb4d2b3fSMartin KaFai Lau return -EINVAL; 408cb4d2b3fSMartin KaFai Lau *dst++ = *src++; 409cb4d2b3fSMartin KaFai Lau } 410cb4d2b3fSMartin KaFai Lau 411cb4d2b3fSMartin KaFai Lau /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */ 412cb4d2b3fSMartin KaFai Lau if (src == end) 413cb4d2b3fSMartin KaFai Lau return -EINVAL; 414cb4d2b3fSMartin KaFai Lau 415cb4d2b3fSMartin KaFai Lau return 0; 416cb4d2b3fSMartin KaFai Lau } 417cb4d2b3fSMartin KaFai Lau 418a3884572SJakub Kicinski #define BPF_MAP_CREATE_LAST_FIELD map_ifindex 41999c55f7dSAlexei Starovoitov /* called via syscall */ 42099c55f7dSAlexei Starovoitov static int map_create(union bpf_attr *attr) 42199c55f7dSAlexei Starovoitov { 42296eabe7aSMartin KaFai Lau int numa_node = bpf_map_attr_numa_node(attr); 42399c55f7dSAlexei Starovoitov struct bpf_map *map; 4246e71b04aSChenbo Feng int f_flags; 42599c55f7dSAlexei Starovoitov int err; 42699c55f7dSAlexei Starovoitov 42799c55f7dSAlexei Starovoitov err = CHECK_ATTR(BPF_MAP_CREATE); 42899c55f7dSAlexei Starovoitov if (err) 42999c55f7dSAlexei Starovoitov return -EINVAL; 43099c55f7dSAlexei Starovoitov 4316e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->map_flags); 4326e71b04aSChenbo Feng if (f_flags < 0) 4336e71b04aSChenbo Feng return f_flags; 4346e71b04aSChenbo Feng 43596eabe7aSMartin KaFai Lau if (numa_node != NUMA_NO_NODE && 43696e5ae4eSEric Dumazet ((unsigned int)numa_node >= nr_node_ids || 43796e5ae4eSEric Dumazet !node_online(numa_node))) 43896eabe7aSMartin KaFai Lau return -EINVAL; 43996eabe7aSMartin KaFai Lau 44099c55f7dSAlexei Starovoitov /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 44199c55f7dSAlexei Starovoitov map = find_and_alloc_map(attr); 44299c55f7dSAlexei Starovoitov if (IS_ERR(map)) 44399c55f7dSAlexei Starovoitov return PTR_ERR(map); 44499c55f7dSAlexei Starovoitov 445ad5b177bSMartin KaFai Lau err = bpf_obj_name_cpy(map->name, attr->map_name); 446ad5b177bSMartin KaFai Lau if (err) 447ad5b177bSMartin KaFai Lau goto free_map_nouncharge; 448ad5b177bSMartin KaFai Lau 44999c55f7dSAlexei Starovoitov atomic_set(&map->refcnt, 1); 450c9da161cSDaniel Borkmann atomic_set(&map->usercnt, 1); 45199c55f7dSAlexei Starovoitov 452afdb09c7SChenbo Feng err = security_bpf_map_alloc(map); 453aaac3ba9SAlexei Starovoitov if (err) 45420b2b24fSDaniel Borkmann goto free_map_nouncharge; 455aaac3ba9SAlexei Starovoitov 456afdb09c7SChenbo Feng err = bpf_map_charge_memlock(map); 457afdb09c7SChenbo Feng if (err) 458afdb09c7SChenbo Feng goto free_map_sec; 459afdb09c7SChenbo Feng 460f3f1c054SMartin KaFai Lau err = bpf_map_alloc_id(map); 461f3f1c054SMartin KaFai Lau if (err) 462f3f1c054SMartin KaFai Lau goto free_map; 463f3f1c054SMartin KaFai Lau 4646e71b04aSChenbo Feng err = bpf_map_new_fd(map, f_flags); 465bd5f5f4eSMartin KaFai Lau if (err < 0) { 466bd5f5f4eSMartin KaFai Lau /* failed to allocate fd. 467bd5f5f4eSMartin KaFai Lau * bpf_map_put() is needed because the above 468bd5f5f4eSMartin KaFai Lau * bpf_map_alloc_id() has published the map 469bd5f5f4eSMartin KaFai Lau * to the userspace and the userspace may 470bd5f5f4eSMartin KaFai Lau * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 471bd5f5f4eSMartin KaFai Lau */ 472bd5f5f4eSMartin KaFai Lau bpf_map_put(map); 473bd5f5f4eSMartin KaFai Lau return err; 474bd5f5f4eSMartin KaFai Lau } 47599c55f7dSAlexei Starovoitov 476a67edbf4SDaniel Borkmann trace_bpf_map_create(map, err); 47799c55f7dSAlexei Starovoitov return err; 47899c55f7dSAlexei Starovoitov 47999c55f7dSAlexei Starovoitov free_map: 48020b2b24fSDaniel Borkmann bpf_map_uncharge_memlock(map); 481afdb09c7SChenbo Feng free_map_sec: 482afdb09c7SChenbo Feng security_bpf_map_free(map); 48320b2b24fSDaniel Borkmann free_map_nouncharge: 48499c55f7dSAlexei Starovoitov map->ops->map_free(map); 48599c55f7dSAlexei Starovoitov return err; 48699c55f7dSAlexei Starovoitov } 48799c55f7dSAlexei Starovoitov 488db20fd2bSAlexei Starovoitov /* if error is returned, fd is released. 489db20fd2bSAlexei Starovoitov * On success caller should complete fd access with matching fdput() 490db20fd2bSAlexei Starovoitov */ 491c2101297SDaniel Borkmann struct bpf_map *__bpf_map_get(struct fd f) 492db20fd2bSAlexei Starovoitov { 493db20fd2bSAlexei Starovoitov if (!f.file) 494db20fd2bSAlexei Starovoitov return ERR_PTR(-EBADF); 495db20fd2bSAlexei Starovoitov if (f.file->f_op != &bpf_map_fops) { 496db20fd2bSAlexei Starovoitov fdput(f); 497db20fd2bSAlexei Starovoitov return ERR_PTR(-EINVAL); 498db20fd2bSAlexei Starovoitov } 499db20fd2bSAlexei Starovoitov 500c2101297SDaniel Borkmann return f.file->private_data; 501c2101297SDaniel Borkmann } 502c2101297SDaniel Borkmann 50392117d84SAlexei Starovoitov /* prog's and map's refcnt limit */ 50492117d84SAlexei Starovoitov #define BPF_MAX_REFCNT 32768 50592117d84SAlexei Starovoitov 50692117d84SAlexei Starovoitov struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref) 507c9da161cSDaniel Borkmann { 50892117d84SAlexei Starovoitov if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) { 50992117d84SAlexei Starovoitov atomic_dec(&map->refcnt); 51092117d84SAlexei Starovoitov return ERR_PTR(-EBUSY); 51192117d84SAlexei Starovoitov } 512c9da161cSDaniel Borkmann if (uref) 513c9da161cSDaniel Borkmann atomic_inc(&map->usercnt); 51492117d84SAlexei Starovoitov return map; 515c9da161cSDaniel Borkmann } 516c9da161cSDaniel Borkmann 517c9da161cSDaniel Borkmann struct bpf_map *bpf_map_get_with_uref(u32 ufd) 518c2101297SDaniel Borkmann { 519c2101297SDaniel Borkmann struct fd f = fdget(ufd); 520c2101297SDaniel Borkmann struct bpf_map *map; 521c2101297SDaniel Borkmann 522c2101297SDaniel Borkmann map = __bpf_map_get(f); 523c2101297SDaniel Borkmann if (IS_ERR(map)) 524c2101297SDaniel Borkmann return map; 525c2101297SDaniel Borkmann 52692117d84SAlexei Starovoitov map = bpf_map_inc(map, true); 527c2101297SDaniel Borkmann fdput(f); 528db20fd2bSAlexei Starovoitov 529db20fd2bSAlexei Starovoitov return map; 530db20fd2bSAlexei Starovoitov } 531db20fd2bSAlexei Starovoitov 532bd5f5f4eSMartin KaFai Lau /* map_idr_lock should have been held */ 533bd5f5f4eSMartin KaFai Lau static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map, 534bd5f5f4eSMartin KaFai Lau bool uref) 535bd5f5f4eSMartin KaFai Lau { 536bd5f5f4eSMartin KaFai Lau int refold; 537bd5f5f4eSMartin KaFai Lau 538bd5f5f4eSMartin KaFai Lau refold = __atomic_add_unless(&map->refcnt, 1, 0); 539bd5f5f4eSMartin KaFai Lau 540bd5f5f4eSMartin KaFai Lau if (refold >= BPF_MAX_REFCNT) { 541bd5f5f4eSMartin KaFai Lau __bpf_map_put(map, false); 542bd5f5f4eSMartin KaFai Lau return ERR_PTR(-EBUSY); 543bd5f5f4eSMartin KaFai Lau } 544bd5f5f4eSMartin KaFai Lau 545bd5f5f4eSMartin KaFai Lau if (!refold) 546bd5f5f4eSMartin KaFai Lau return ERR_PTR(-ENOENT); 547bd5f5f4eSMartin KaFai Lau 548bd5f5f4eSMartin KaFai Lau if (uref) 549bd5f5f4eSMartin KaFai Lau atomic_inc(&map->usercnt); 550bd5f5f4eSMartin KaFai Lau 551bd5f5f4eSMartin KaFai Lau return map; 552bd5f5f4eSMartin KaFai Lau } 553bd5f5f4eSMartin KaFai Lau 554b8cdc051SAlexei Starovoitov int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 555b8cdc051SAlexei Starovoitov { 556b8cdc051SAlexei Starovoitov return -ENOTSUPP; 557b8cdc051SAlexei Starovoitov } 558b8cdc051SAlexei Starovoitov 559db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 560db20fd2bSAlexei Starovoitov #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value 561db20fd2bSAlexei Starovoitov 562db20fd2bSAlexei Starovoitov static int map_lookup_elem(union bpf_attr *attr) 563db20fd2bSAlexei Starovoitov { 564535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 565535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 566db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 567db20fd2bSAlexei Starovoitov struct bpf_map *map; 5688ebe667cSAlexei Starovoitov void *key, *value, *ptr; 56915a07b33SAlexei Starovoitov u32 value_size; 570592867bfSDaniel Borkmann struct fd f; 571db20fd2bSAlexei Starovoitov int err; 572db20fd2bSAlexei Starovoitov 573db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 574db20fd2bSAlexei Starovoitov return -EINVAL; 575db20fd2bSAlexei Starovoitov 576592867bfSDaniel Borkmann f = fdget(ufd); 577c2101297SDaniel Borkmann map = __bpf_map_get(f); 578db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 579db20fd2bSAlexei Starovoitov return PTR_ERR(map); 580db20fd2bSAlexei Starovoitov 5816e71b04aSChenbo Feng if (!(f.file->f_mode & FMODE_CAN_READ)) { 5826e71b04aSChenbo Feng err = -EPERM; 5836e71b04aSChenbo Feng goto err_put; 5846e71b04aSChenbo Feng } 5856e71b04aSChenbo Feng 586e4448ed8SAl Viro key = memdup_user(ukey, map->key_size); 587e4448ed8SAl Viro if (IS_ERR(key)) { 588e4448ed8SAl Viro err = PTR_ERR(key); 589db20fd2bSAlexei Starovoitov goto err_put; 590e4448ed8SAl Viro } 591db20fd2bSAlexei Starovoitov 59215a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 5938f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 59415a07b33SAlexei Starovoitov map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 59515a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 59614dc6f04SMartin KaFai Lau else if (IS_FD_MAP(map)) 59714dc6f04SMartin KaFai Lau value_size = sizeof(u32); 59815a07b33SAlexei Starovoitov else 59915a07b33SAlexei Starovoitov value_size = map->value_size; 60015a07b33SAlexei Starovoitov 6018ebe667cSAlexei Starovoitov err = -ENOMEM; 60215a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 603db20fd2bSAlexei Starovoitov if (!value) 6048ebe667cSAlexei Starovoitov goto free_key; 6058ebe667cSAlexei Starovoitov 606a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 607a3884572SJakub Kicinski err = bpf_map_offload_lookup_elem(map, key, value); 608a3884572SJakub Kicinski } else if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 6098f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 61015a07b33SAlexei Starovoitov err = bpf_percpu_hash_copy(map, key, value); 61115a07b33SAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 61215a07b33SAlexei Starovoitov err = bpf_percpu_array_copy(map, key, value); 613557c0c6eSAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 614557c0c6eSAlexei Starovoitov err = bpf_stackmap_copy(map, key, value); 61514dc6f04SMartin KaFai Lau } else if (IS_FD_ARRAY(map)) { 61614dc6f04SMartin KaFai Lau err = bpf_fd_array_map_lookup_elem(map, key, value); 61714dc6f04SMartin KaFai Lau } else if (IS_FD_HASH(map)) { 61814dc6f04SMartin KaFai Lau err = bpf_fd_htab_map_lookup_elem(map, key, value); 61915a07b33SAlexei Starovoitov } else { 6208ebe667cSAlexei Starovoitov rcu_read_lock(); 6218ebe667cSAlexei Starovoitov ptr = map->ops->map_lookup_elem(map, key); 6228ebe667cSAlexei Starovoitov if (ptr) 62315a07b33SAlexei Starovoitov memcpy(value, ptr, value_size); 6248ebe667cSAlexei Starovoitov rcu_read_unlock(); 62515a07b33SAlexei Starovoitov err = ptr ? 0 : -ENOENT; 62615a07b33SAlexei Starovoitov } 6278ebe667cSAlexei Starovoitov 62815a07b33SAlexei Starovoitov if (err) 6298ebe667cSAlexei Starovoitov goto free_value; 630db20fd2bSAlexei Starovoitov 631db20fd2bSAlexei Starovoitov err = -EFAULT; 63215a07b33SAlexei Starovoitov if (copy_to_user(uvalue, value, value_size) != 0) 6338ebe667cSAlexei Starovoitov goto free_value; 634db20fd2bSAlexei Starovoitov 635a67edbf4SDaniel Borkmann trace_bpf_map_lookup_elem(map, ufd, key, value); 636db20fd2bSAlexei Starovoitov err = 0; 637db20fd2bSAlexei Starovoitov 6388ebe667cSAlexei Starovoitov free_value: 6398ebe667cSAlexei Starovoitov kfree(value); 640db20fd2bSAlexei Starovoitov free_key: 641db20fd2bSAlexei Starovoitov kfree(key); 642db20fd2bSAlexei Starovoitov err_put: 643db20fd2bSAlexei Starovoitov fdput(f); 644db20fd2bSAlexei Starovoitov return err; 645db20fd2bSAlexei Starovoitov } 646db20fd2bSAlexei Starovoitov 6473274f520SAlexei Starovoitov #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 648db20fd2bSAlexei Starovoitov 649db20fd2bSAlexei Starovoitov static int map_update_elem(union bpf_attr *attr) 650db20fd2bSAlexei Starovoitov { 651535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 652535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value); 653db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 654db20fd2bSAlexei Starovoitov struct bpf_map *map; 655db20fd2bSAlexei Starovoitov void *key, *value; 65615a07b33SAlexei Starovoitov u32 value_size; 657592867bfSDaniel Borkmann struct fd f; 658db20fd2bSAlexei Starovoitov int err; 659db20fd2bSAlexei Starovoitov 660db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 661db20fd2bSAlexei Starovoitov return -EINVAL; 662db20fd2bSAlexei Starovoitov 663592867bfSDaniel Borkmann f = fdget(ufd); 664c2101297SDaniel Borkmann map = __bpf_map_get(f); 665db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 666db20fd2bSAlexei Starovoitov return PTR_ERR(map); 667db20fd2bSAlexei Starovoitov 6686e71b04aSChenbo Feng if (!(f.file->f_mode & FMODE_CAN_WRITE)) { 6696e71b04aSChenbo Feng err = -EPERM; 6706e71b04aSChenbo Feng goto err_put; 6716e71b04aSChenbo Feng } 6726e71b04aSChenbo Feng 673e4448ed8SAl Viro key = memdup_user(ukey, map->key_size); 674e4448ed8SAl Viro if (IS_ERR(key)) { 675e4448ed8SAl Viro err = PTR_ERR(key); 676db20fd2bSAlexei Starovoitov goto err_put; 677e4448ed8SAl Viro } 678db20fd2bSAlexei Starovoitov 67915a07b33SAlexei Starovoitov if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 6808f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 68115a07b33SAlexei Starovoitov map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 68215a07b33SAlexei Starovoitov value_size = round_up(map->value_size, 8) * num_possible_cpus(); 68315a07b33SAlexei Starovoitov else 68415a07b33SAlexei Starovoitov value_size = map->value_size; 68515a07b33SAlexei Starovoitov 686db20fd2bSAlexei Starovoitov err = -ENOMEM; 68715a07b33SAlexei Starovoitov value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 688db20fd2bSAlexei Starovoitov if (!value) 689db20fd2bSAlexei Starovoitov goto free_key; 690db20fd2bSAlexei Starovoitov 691db20fd2bSAlexei Starovoitov err = -EFAULT; 69215a07b33SAlexei Starovoitov if (copy_from_user(value, uvalue, value_size) != 0) 693db20fd2bSAlexei Starovoitov goto free_value; 694db20fd2bSAlexei Starovoitov 6956710e112SJesper Dangaard Brouer /* Need to create a kthread, thus must support schedule */ 696a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 697a3884572SJakub Kicinski err = bpf_map_offload_update_elem(map, key, value, attr->flags); 698a3884572SJakub Kicinski goto out; 699a3884572SJakub Kicinski } else if (map->map_type == BPF_MAP_TYPE_CPUMAP) { 7006710e112SJesper Dangaard Brouer err = map->ops->map_update_elem(map, key, value, attr->flags); 7016710e112SJesper Dangaard Brouer goto out; 7026710e112SJesper Dangaard Brouer } 7036710e112SJesper Dangaard Brouer 704b121d1e7SAlexei Starovoitov /* must increment bpf_prog_active to avoid kprobe+bpf triggering from 705b121d1e7SAlexei Starovoitov * inside bpf map update or delete otherwise deadlocks are possible 706b121d1e7SAlexei Starovoitov */ 707b121d1e7SAlexei Starovoitov preempt_disable(); 708b121d1e7SAlexei Starovoitov __this_cpu_inc(bpf_prog_active); 7098f844938SMartin KaFai Lau if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 7108f844938SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 71115a07b33SAlexei Starovoitov err = bpf_percpu_hash_update(map, key, value, attr->flags); 71215a07b33SAlexei Starovoitov } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 71315a07b33SAlexei Starovoitov err = bpf_percpu_array_update(map, key, value, attr->flags); 7149c147b56SMickaël Salaün } else if (IS_FD_ARRAY(map)) { 715d056a788SDaniel Borkmann rcu_read_lock(); 716d056a788SDaniel Borkmann err = bpf_fd_array_map_update_elem(map, f.file, key, value, 717d056a788SDaniel Borkmann attr->flags); 718d056a788SDaniel Borkmann rcu_read_unlock(); 719bcc6b1b7SMartin KaFai Lau } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 720bcc6b1b7SMartin KaFai Lau rcu_read_lock(); 721bcc6b1b7SMartin KaFai Lau err = bpf_fd_htab_map_update_elem(map, f.file, key, value, 722bcc6b1b7SMartin KaFai Lau attr->flags); 723bcc6b1b7SMartin KaFai Lau rcu_read_unlock(); 72415a07b33SAlexei Starovoitov } else { 725db20fd2bSAlexei Starovoitov rcu_read_lock(); 7263274f520SAlexei Starovoitov err = map->ops->map_update_elem(map, key, value, attr->flags); 727db20fd2bSAlexei Starovoitov rcu_read_unlock(); 72815a07b33SAlexei Starovoitov } 729b121d1e7SAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 730b121d1e7SAlexei Starovoitov preempt_enable(); 7316710e112SJesper Dangaard Brouer out: 732a67edbf4SDaniel Borkmann if (!err) 733a67edbf4SDaniel Borkmann trace_bpf_map_update_elem(map, ufd, key, value); 734db20fd2bSAlexei Starovoitov free_value: 735db20fd2bSAlexei Starovoitov kfree(value); 736db20fd2bSAlexei Starovoitov free_key: 737db20fd2bSAlexei Starovoitov kfree(key); 738db20fd2bSAlexei Starovoitov err_put: 739db20fd2bSAlexei Starovoitov fdput(f); 740db20fd2bSAlexei Starovoitov return err; 741db20fd2bSAlexei Starovoitov } 742db20fd2bSAlexei Starovoitov 743db20fd2bSAlexei Starovoitov #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 744db20fd2bSAlexei Starovoitov 745db20fd2bSAlexei Starovoitov static int map_delete_elem(union bpf_attr *attr) 746db20fd2bSAlexei Starovoitov { 747535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 748db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 749db20fd2bSAlexei Starovoitov struct bpf_map *map; 750592867bfSDaniel Borkmann struct fd f; 751db20fd2bSAlexei Starovoitov void *key; 752db20fd2bSAlexei Starovoitov int err; 753db20fd2bSAlexei Starovoitov 754db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 755db20fd2bSAlexei Starovoitov return -EINVAL; 756db20fd2bSAlexei Starovoitov 757592867bfSDaniel Borkmann f = fdget(ufd); 758c2101297SDaniel Borkmann map = __bpf_map_get(f); 759db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 760db20fd2bSAlexei Starovoitov return PTR_ERR(map); 761db20fd2bSAlexei Starovoitov 7626e71b04aSChenbo Feng if (!(f.file->f_mode & FMODE_CAN_WRITE)) { 7636e71b04aSChenbo Feng err = -EPERM; 7646e71b04aSChenbo Feng goto err_put; 7656e71b04aSChenbo Feng } 7666e71b04aSChenbo Feng 767e4448ed8SAl Viro key = memdup_user(ukey, map->key_size); 768e4448ed8SAl Viro if (IS_ERR(key)) { 769e4448ed8SAl Viro err = PTR_ERR(key); 770db20fd2bSAlexei Starovoitov goto err_put; 771e4448ed8SAl Viro } 772db20fd2bSAlexei Starovoitov 773a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 774a3884572SJakub Kicinski err = bpf_map_offload_delete_elem(map, key); 775a3884572SJakub Kicinski goto out; 776a3884572SJakub Kicinski } 777a3884572SJakub Kicinski 778b121d1e7SAlexei Starovoitov preempt_disable(); 779b121d1e7SAlexei Starovoitov __this_cpu_inc(bpf_prog_active); 780db20fd2bSAlexei Starovoitov rcu_read_lock(); 781db20fd2bSAlexei Starovoitov err = map->ops->map_delete_elem(map, key); 782db20fd2bSAlexei Starovoitov rcu_read_unlock(); 783b121d1e7SAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 784b121d1e7SAlexei Starovoitov preempt_enable(); 785a3884572SJakub Kicinski out: 786a67edbf4SDaniel Borkmann if (!err) 787a67edbf4SDaniel Borkmann trace_bpf_map_delete_elem(map, ufd, key); 788db20fd2bSAlexei Starovoitov kfree(key); 789db20fd2bSAlexei Starovoitov err_put: 790db20fd2bSAlexei Starovoitov fdput(f); 791db20fd2bSAlexei Starovoitov return err; 792db20fd2bSAlexei Starovoitov } 793db20fd2bSAlexei Starovoitov 794db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 795db20fd2bSAlexei Starovoitov #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 796db20fd2bSAlexei Starovoitov 797db20fd2bSAlexei Starovoitov static int map_get_next_key(union bpf_attr *attr) 798db20fd2bSAlexei Starovoitov { 799535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key); 800535e7b4bSMickaël Salaün void __user *unext_key = u64_to_user_ptr(attr->next_key); 801db20fd2bSAlexei Starovoitov int ufd = attr->map_fd; 802db20fd2bSAlexei Starovoitov struct bpf_map *map; 803db20fd2bSAlexei Starovoitov void *key, *next_key; 804592867bfSDaniel Borkmann struct fd f; 805db20fd2bSAlexei Starovoitov int err; 806db20fd2bSAlexei Starovoitov 807db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 808db20fd2bSAlexei Starovoitov return -EINVAL; 809db20fd2bSAlexei Starovoitov 810592867bfSDaniel Borkmann f = fdget(ufd); 811c2101297SDaniel Borkmann map = __bpf_map_get(f); 812db20fd2bSAlexei Starovoitov if (IS_ERR(map)) 813db20fd2bSAlexei Starovoitov return PTR_ERR(map); 814db20fd2bSAlexei Starovoitov 8156e71b04aSChenbo Feng if (!(f.file->f_mode & FMODE_CAN_READ)) { 8166e71b04aSChenbo Feng err = -EPERM; 8176e71b04aSChenbo Feng goto err_put; 8186e71b04aSChenbo Feng } 8196e71b04aSChenbo Feng 8208fe45924STeng Qin if (ukey) { 821e4448ed8SAl Viro key = memdup_user(ukey, map->key_size); 822e4448ed8SAl Viro if (IS_ERR(key)) { 823e4448ed8SAl Viro err = PTR_ERR(key); 824db20fd2bSAlexei Starovoitov goto err_put; 825e4448ed8SAl Viro } 8268fe45924STeng Qin } else { 8278fe45924STeng Qin key = NULL; 8288fe45924STeng Qin } 829db20fd2bSAlexei Starovoitov 830db20fd2bSAlexei Starovoitov err = -ENOMEM; 831db20fd2bSAlexei Starovoitov next_key = kmalloc(map->key_size, GFP_USER); 832db20fd2bSAlexei Starovoitov if (!next_key) 833db20fd2bSAlexei Starovoitov goto free_key; 834db20fd2bSAlexei Starovoitov 835a3884572SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 836a3884572SJakub Kicinski err = bpf_map_offload_get_next_key(map, key, next_key); 837a3884572SJakub Kicinski goto out; 838a3884572SJakub Kicinski } 839a3884572SJakub Kicinski 840db20fd2bSAlexei Starovoitov rcu_read_lock(); 841db20fd2bSAlexei Starovoitov err = map->ops->map_get_next_key(map, key, next_key); 842db20fd2bSAlexei Starovoitov rcu_read_unlock(); 843a3884572SJakub Kicinski out: 844db20fd2bSAlexei Starovoitov if (err) 845db20fd2bSAlexei Starovoitov goto free_next_key; 846db20fd2bSAlexei Starovoitov 847db20fd2bSAlexei Starovoitov err = -EFAULT; 848db20fd2bSAlexei Starovoitov if (copy_to_user(unext_key, next_key, map->key_size) != 0) 849db20fd2bSAlexei Starovoitov goto free_next_key; 850db20fd2bSAlexei Starovoitov 851a67edbf4SDaniel Borkmann trace_bpf_map_next_key(map, ufd, key, next_key); 852db20fd2bSAlexei Starovoitov err = 0; 853db20fd2bSAlexei Starovoitov 854db20fd2bSAlexei Starovoitov free_next_key: 855db20fd2bSAlexei Starovoitov kfree(next_key); 856db20fd2bSAlexei Starovoitov free_key: 857db20fd2bSAlexei Starovoitov kfree(key); 858db20fd2bSAlexei Starovoitov err_put: 859db20fd2bSAlexei Starovoitov fdput(f); 860db20fd2bSAlexei Starovoitov return err; 861db20fd2bSAlexei Starovoitov } 862db20fd2bSAlexei Starovoitov 8637de16e3aSJakub Kicinski static const struct bpf_prog_ops * const bpf_prog_types[] = { 8647de16e3aSJakub Kicinski #define BPF_PROG_TYPE(_id, _name) \ 8657de16e3aSJakub Kicinski [_id] = & _name ## _prog_ops, 8667de16e3aSJakub Kicinski #define BPF_MAP_TYPE(_id, _ops) 8677de16e3aSJakub Kicinski #include <linux/bpf_types.h> 8687de16e3aSJakub Kicinski #undef BPF_PROG_TYPE 8697de16e3aSJakub Kicinski #undef BPF_MAP_TYPE 8707de16e3aSJakub Kicinski }; 8717de16e3aSJakub Kicinski 87209756af4SAlexei Starovoitov static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 87309756af4SAlexei Starovoitov { 874be9370a7SJohannes Berg if (type >= ARRAY_SIZE(bpf_prog_types) || !bpf_prog_types[type]) 875be9370a7SJohannes Berg return -EINVAL; 87609756af4SAlexei Starovoitov 877ab3f0063SJakub Kicinski if (!bpf_prog_is_dev_bound(prog->aux)) 878be9370a7SJohannes Berg prog->aux->ops = bpf_prog_types[type]; 879ab3f0063SJakub Kicinski else 880ab3f0063SJakub Kicinski prog->aux->ops = &bpf_offload_prog_ops; 88124701eceSDaniel Borkmann prog->type = type; 88209756af4SAlexei Starovoitov return 0; 88309756af4SAlexei Starovoitov } 88409756af4SAlexei Starovoitov 88509756af4SAlexei Starovoitov /* drop refcnt on maps used by eBPF program and free auxilary data */ 88609756af4SAlexei Starovoitov static void free_used_maps(struct bpf_prog_aux *aux) 88709756af4SAlexei Starovoitov { 88809756af4SAlexei Starovoitov int i; 88909756af4SAlexei Starovoitov 89009756af4SAlexei Starovoitov for (i = 0; i < aux->used_map_cnt; i++) 89109756af4SAlexei Starovoitov bpf_map_put(aux->used_maps[i]); 89209756af4SAlexei Starovoitov 89309756af4SAlexei Starovoitov kfree(aux->used_maps); 89409756af4SAlexei Starovoitov } 89509756af4SAlexei Starovoitov 8965ccb071eSDaniel Borkmann int __bpf_prog_charge(struct user_struct *user, u32 pages) 8975ccb071eSDaniel Borkmann { 8985ccb071eSDaniel Borkmann unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 8995ccb071eSDaniel Borkmann unsigned long user_bufs; 9005ccb071eSDaniel Borkmann 9015ccb071eSDaniel Borkmann if (user) { 9025ccb071eSDaniel Borkmann user_bufs = atomic_long_add_return(pages, &user->locked_vm); 9035ccb071eSDaniel Borkmann if (user_bufs > memlock_limit) { 9045ccb071eSDaniel Borkmann atomic_long_sub(pages, &user->locked_vm); 9055ccb071eSDaniel Borkmann return -EPERM; 9065ccb071eSDaniel Borkmann } 9075ccb071eSDaniel Borkmann } 9085ccb071eSDaniel Borkmann 9095ccb071eSDaniel Borkmann return 0; 9105ccb071eSDaniel Borkmann } 9115ccb071eSDaniel Borkmann 9125ccb071eSDaniel Borkmann void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 9135ccb071eSDaniel Borkmann { 9145ccb071eSDaniel Borkmann if (user) 9155ccb071eSDaniel Borkmann atomic_long_sub(pages, &user->locked_vm); 9165ccb071eSDaniel Borkmann } 9175ccb071eSDaniel Borkmann 918aaac3ba9SAlexei Starovoitov static int bpf_prog_charge_memlock(struct bpf_prog *prog) 919aaac3ba9SAlexei Starovoitov { 920aaac3ba9SAlexei Starovoitov struct user_struct *user = get_current_user(); 9215ccb071eSDaniel Borkmann int ret; 922aaac3ba9SAlexei Starovoitov 9235ccb071eSDaniel Borkmann ret = __bpf_prog_charge(user, prog->pages); 9245ccb071eSDaniel Borkmann if (ret) { 925aaac3ba9SAlexei Starovoitov free_uid(user); 9265ccb071eSDaniel Borkmann return ret; 927aaac3ba9SAlexei Starovoitov } 9285ccb071eSDaniel Borkmann 929aaac3ba9SAlexei Starovoitov prog->aux->user = user; 930aaac3ba9SAlexei Starovoitov return 0; 931aaac3ba9SAlexei Starovoitov } 932aaac3ba9SAlexei Starovoitov 933aaac3ba9SAlexei Starovoitov static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 934aaac3ba9SAlexei Starovoitov { 935aaac3ba9SAlexei Starovoitov struct user_struct *user = prog->aux->user; 936aaac3ba9SAlexei Starovoitov 9375ccb071eSDaniel Borkmann __bpf_prog_uncharge(user, prog->pages); 938aaac3ba9SAlexei Starovoitov free_uid(user); 939aaac3ba9SAlexei Starovoitov } 940aaac3ba9SAlexei Starovoitov 941dc4bb0e2SMartin KaFai Lau static int bpf_prog_alloc_id(struct bpf_prog *prog) 942dc4bb0e2SMartin KaFai Lau { 943dc4bb0e2SMartin KaFai Lau int id; 944dc4bb0e2SMartin KaFai Lau 945b76354cdSShaohua Li idr_preload(GFP_KERNEL); 946dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 947dc4bb0e2SMartin KaFai Lau id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 948dc4bb0e2SMartin KaFai Lau if (id > 0) 949dc4bb0e2SMartin KaFai Lau prog->aux->id = id; 950dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 951b76354cdSShaohua Li idr_preload_end(); 952dc4bb0e2SMartin KaFai Lau 953dc4bb0e2SMartin KaFai Lau /* id is in [1, INT_MAX) */ 954dc4bb0e2SMartin KaFai Lau if (WARN_ON_ONCE(!id)) 955dc4bb0e2SMartin KaFai Lau return -ENOSPC; 956dc4bb0e2SMartin KaFai Lau 957dc4bb0e2SMartin KaFai Lau return id > 0 ? 0 : id; 958dc4bb0e2SMartin KaFai Lau } 959dc4bb0e2SMartin KaFai Lau 960ad8ad79fSJakub Kicinski void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) 961dc4bb0e2SMartin KaFai Lau { 962ad8ad79fSJakub Kicinski /* cBPF to eBPF migrations are currently not in the idr store. 963ad8ad79fSJakub Kicinski * Offloaded programs are removed from the store when their device 964ad8ad79fSJakub Kicinski * disappears - even if someone grabs an fd to them they are unusable, 965ad8ad79fSJakub Kicinski * simply waiting for refcnt to drop to be freed. 966ad8ad79fSJakub Kicinski */ 967dc4bb0e2SMartin KaFai Lau if (!prog->aux->id) 968dc4bb0e2SMartin KaFai Lau return; 969dc4bb0e2SMartin KaFai Lau 970b16d9aa4SMartin KaFai Lau if (do_idr_lock) 971dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 972b16d9aa4SMartin KaFai Lau else 973b16d9aa4SMartin KaFai Lau __acquire(&prog_idr_lock); 974b16d9aa4SMartin KaFai Lau 975dc4bb0e2SMartin KaFai Lau idr_remove(&prog_idr, prog->aux->id); 976ad8ad79fSJakub Kicinski prog->aux->id = 0; 977b16d9aa4SMartin KaFai Lau 978b16d9aa4SMartin KaFai Lau if (do_idr_lock) 979dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 980b16d9aa4SMartin KaFai Lau else 981b16d9aa4SMartin KaFai Lau __release(&prog_idr_lock); 982dc4bb0e2SMartin KaFai Lau } 983dc4bb0e2SMartin KaFai Lau 9841aacde3dSDaniel Borkmann static void __bpf_prog_put_rcu(struct rcu_head *rcu) 985abf2e7d6SAlexei Starovoitov { 986abf2e7d6SAlexei Starovoitov struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 987abf2e7d6SAlexei Starovoitov 988abf2e7d6SAlexei Starovoitov free_used_maps(aux); 989aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(aux->prog); 990afdb09c7SChenbo Feng security_bpf_prog_free(aux); 991abf2e7d6SAlexei Starovoitov bpf_prog_free(aux->prog); 992abf2e7d6SAlexei Starovoitov } 993abf2e7d6SAlexei Starovoitov 994b16d9aa4SMartin KaFai Lau static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 99509756af4SAlexei Starovoitov { 996a67edbf4SDaniel Borkmann if (atomic_dec_and_test(&prog->aux->refcnt)) { 9974f74d809SDaniel Borkmann int i; 9984f74d809SDaniel Borkmann 999a67edbf4SDaniel Borkmann trace_bpf_prog_put_rcu(prog); 100034ad5580SMartin KaFai Lau /* bpf_prog_free_id() must be called first */ 1001b16d9aa4SMartin KaFai Lau bpf_prog_free_id(prog, do_idr_lock); 10024f74d809SDaniel Borkmann 10034f74d809SDaniel Borkmann for (i = 0; i < prog->aux->func_cnt; i++) 10044f74d809SDaniel Borkmann bpf_prog_kallsyms_del(prog->aux->func[i]); 100574451e66SDaniel Borkmann bpf_prog_kallsyms_del(prog); 10064f74d809SDaniel Borkmann 10071aacde3dSDaniel Borkmann call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 100809756af4SAlexei Starovoitov } 1009a67edbf4SDaniel Borkmann } 1010b16d9aa4SMartin KaFai Lau 1011b16d9aa4SMartin KaFai Lau void bpf_prog_put(struct bpf_prog *prog) 1012b16d9aa4SMartin KaFai Lau { 1013b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, true); 1014b16d9aa4SMartin KaFai Lau } 1015e2e9b654SDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_put); 101609756af4SAlexei Starovoitov 101709756af4SAlexei Starovoitov static int bpf_prog_release(struct inode *inode, struct file *filp) 101809756af4SAlexei Starovoitov { 101909756af4SAlexei Starovoitov struct bpf_prog *prog = filp->private_data; 102009756af4SAlexei Starovoitov 10211aacde3dSDaniel Borkmann bpf_prog_put(prog); 102209756af4SAlexei Starovoitov return 0; 102309756af4SAlexei Starovoitov } 102409756af4SAlexei Starovoitov 10257bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 10267bd509e3SDaniel Borkmann static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 10277bd509e3SDaniel Borkmann { 10287bd509e3SDaniel Borkmann const struct bpf_prog *prog = filp->private_data; 1029f1f7714eSDaniel Borkmann char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 10307bd509e3SDaniel Borkmann 1031f1f7714eSDaniel Borkmann bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 10327bd509e3SDaniel Borkmann seq_printf(m, 10337bd509e3SDaniel Borkmann "prog_type:\t%u\n" 10347bd509e3SDaniel Borkmann "prog_jited:\t%u\n" 1035f1f7714eSDaniel Borkmann "prog_tag:\t%s\n" 10367bd509e3SDaniel Borkmann "memlock:\t%llu\n", 10377bd509e3SDaniel Borkmann prog->type, 10387bd509e3SDaniel Borkmann prog->jited, 1039f1f7714eSDaniel Borkmann prog_tag, 10407bd509e3SDaniel Borkmann prog->pages * 1ULL << PAGE_SHIFT); 10417bd509e3SDaniel Borkmann } 10427bd509e3SDaniel Borkmann #endif 10437bd509e3SDaniel Borkmann 1044f66e448cSChenbo Feng const struct file_operations bpf_prog_fops = { 10457bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS 10467bd509e3SDaniel Borkmann .show_fdinfo = bpf_prog_show_fdinfo, 10477bd509e3SDaniel Borkmann #endif 104809756af4SAlexei Starovoitov .release = bpf_prog_release, 10496e71b04aSChenbo Feng .read = bpf_dummy_read, 10506e71b04aSChenbo Feng .write = bpf_dummy_write, 105109756af4SAlexei Starovoitov }; 105209756af4SAlexei Starovoitov 1053b2197755SDaniel Borkmann int bpf_prog_new_fd(struct bpf_prog *prog) 1054aa79781bSDaniel Borkmann { 1055afdb09c7SChenbo Feng int ret; 1056afdb09c7SChenbo Feng 1057afdb09c7SChenbo Feng ret = security_bpf_prog(prog); 1058afdb09c7SChenbo Feng if (ret < 0) 1059afdb09c7SChenbo Feng return ret; 1060afdb09c7SChenbo Feng 1061aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 1062aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC); 1063aa79781bSDaniel Borkmann } 1064aa79781bSDaniel Borkmann 1065113214beSDaniel Borkmann static struct bpf_prog *____bpf_prog_get(struct fd f) 106609756af4SAlexei Starovoitov { 106709756af4SAlexei Starovoitov if (!f.file) 106809756af4SAlexei Starovoitov return ERR_PTR(-EBADF); 106909756af4SAlexei Starovoitov if (f.file->f_op != &bpf_prog_fops) { 107009756af4SAlexei Starovoitov fdput(f); 107109756af4SAlexei Starovoitov return ERR_PTR(-EINVAL); 107209756af4SAlexei Starovoitov } 107309756af4SAlexei Starovoitov 1074c2101297SDaniel Borkmann return f.file->private_data; 107509756af4SAlexei Starovoitov } 107609756af4SAlexei Starovoitov 107759d3656dSBrenden Blanco struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i) 107892117d84SAlexei Starovoitov { 107959d3656dSBrenden Blanco if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) { 108059d3656dSBrenden Blanco atomic_sub(i, &prog->aux->refcnt); 108192117d84SAlexei Starovoitov return ERR_PTR(-EBUSY); 108292117d84SAlexei Starovoitov } 108392117d84SAlexei Starovoitov return prog; 108492117d84SAlexei Starovoitov } 108559d3656dSBrenden Blanco EXPORT_SYMBOL_GPL(bpf_prog_add); 108659d3656dSBrenden Blanco 1087c540594fSDaniel Borkmann void bpf_prog_sub(struct bpf_prog *prog, int i) 1088c540594fSDaniel Borkmann { 1089c540594fSDaniel Borkmann /* Only to be used for undoing previous bpf_prog_add() in some 1090c540594fSDaniel Borkmann * error path. We still know that another entity in our call 1091c540594fSDaniel Borkmann * path holds a reference to the program, thus atomic_sub() can 1092c540594fSDaniel Borkmann * be safely used in such cases! 1093c540594fSDaniel Borkmann */ 1094c540594fSDaniel Borkmann WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0); 1095c540594fSDaniel Borkmann } 1096c540594fSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_sub); 1097c540594fSDaniel Borkmann 109859d3656dSBrenden Blanco struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog) 109959d3656dSBrenden Blanco { 110059d3656dSBrenden Blanco return bpf_prog_add(prog, 1); 110159d3656dSBrenden Blanco } 110297bc402dSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_inc); 110392117d84SAlexei Starovoitov 1104b16d9aa4SMartin KaFai Lau /* prog_idr_lock should have been held */ 1105a6f6df69SJohn Fastabend struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 1106b16d9aa4SMartin KaFai Lau { 1107b16d9aa4SMartin KaFai Lau int refold; 1108b16d9aa4SMartin KaFai Lau 1109b16d9aa4SMartin KaFai Lau refold = __atomic_add_unless(&prog->aux->refcnt, 1, 0); 1110b16d9aa4SMartin KaFai Lau 1111b16d9aa4SMartin KaFai Lau if (refold >= BPF_MAX_REFCNT) { 1112b16d9aa4SMartin KaFai Lau __bpf_prog_put(prog, false); 1113b16d9aa4SMartin KaFai Lau return ERR_PTR(-EBUSY); 1114b16d9aa4SMartin KaFai Lau } 1115b16d9aa4SMartin KaFai Lau 1116b16d9aa4SMartin KaFai Lau if (!refold) 1117b16d9aa4SMartin KaFai Lau return ERR_PTR(-ENOENT); 1118b16d9aa4SMartin KaFai Lau 1119b16d9aa4SMartin KaFai Lau return prog; 1120b16d9aa4SMartin KaFai Lau } 1121a6f6df69SJohn Fastabend EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 1122b16d9aa4SMartin KaFai Lau 1123040ee692SAl Viro bool bpf_prog_get_ok(struct bpf_prog *prog, 1124288b3de5SJakub Kicinski enum bpf_prog_type *attach_type, bool attach_drv) 1125248f346fSJakub Kicinski { 1126288b3de5SJakub Kicinski /* not an attachment, just a refcount inc, always allow */ 1127288b3de5SJakub Kicinski if (!attach_type) 1128288b3de5SJakub Kicinski return true; 1129248f346fSJakub Kicinski 1130248f346fSJakub Kicinski if (prog->type != *attach_type) 1131248f346fSJakub Kicinski return false; 1132288b3de5SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv) 1133248f346fSJakub Kicinski return false; 1134248f346fSJakub Kicinski 1135248f346fSJakub Kicinski return true; 1136248f346fSJakub Kicinski } 1137248f346fSJakub Kicinski 1138248f346fSJakub Kicinski static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, 1139288b3de5SJakub Kicinski bool attach_drv) 114009756af4SAlexei Starovoitov { 114109756af4SAlexei Starovoitov struct fd f = fdget(ufd); 114209756af4SAlexei Starovoitov struct bpf_prog *prog; 114309756af4SAlexei Starovoitov 1144113214beSDaniel Borkmann prog = ____bpf_prog_get(f); 114509756af4SAlexei Starovoitov if (IS_ERR(prog)) 114609756af4SAlexei Starovoitov return prog; 1147288b3de5SJakub Kicinski if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) { 1148113214beSDaniel Borkmann prog = ERR_PTR(-EINVAL); 1149113214beSDaniel Borkmann goto out; 1150113214beSDaniel Borkmann } 115109756af4SAlexei Starovoitov 115292117d84SAlexei Starovoitov prog = bpf_prog_inc(prog); 1153113214beSDaniel Borkmann out: 115409756af4SAlexei Starovoitov fdput(f); 115509756af4SAlexei Starovoitov return prog; 115609756af4SAlexei Starovoitov } 1157113214beSDaniel Borkmann 1158113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get(u32 ufd) 1159113214beSDaniel Borkmann { 1160288b3de5SJakub Kicinski return __bpf_prog_get(ufd, NULL, false); 1161113214beSDaniel Borkmann } 1162113214beSDaniel Borkmann 1163248f346fSJakub Kicinski struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 1164288b3de5SJakub Kicinski bool attach_drv) 1165248f346fSJakub Kicinski { 1166288b3de5SJakub Kicinski struct bpf_prog *prog = __bpf_prog_get(ufd, &type, attach_drv); 1167248f346fSJakub Kicinski 1168248f346fSJakub Kicinski if (!IS_ERR(prog)) 1169248f346fSJakub Kicinski trace_bpf_prog_get_type(prog); 1170248f346fSJakub Kicinski return prog; 1171248f346fSJakub Kicinski } 11726c8dfe21SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev); 1173248f346fSJakub Kicinski 1174aac3fc32SAndrey Ignatov /* Initially all BPF programs could be loaded w/o specifying 1175aac3fc32SAndrey Ignatov * expected_attach_type. Later for some of them specifying expected_attach_type 1176aac3fc32SAndrey Ignatov * at load time became required so that program could be validated properly. 1177aac3fc32SAndrey Ignatov * Programs of types that are allowed to be loaded both w/ and w/o (for 1178aac3fc32SAndrey Ignatov * backward compatibility) expected_attach_type, should have the default attach 1179aac3fc32SAndrey Ignatov * type assigned to expected_attach_type for the latter case, so that it can be 1180aac3fc32SAndrey Ignatov * validated later at attach time. 1181aac3fc32SAndrey Ignatov * 1182aac3fc32SAndrey Ignatov * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if 1183aac3fc32SAndrey Ignatov * prog type requires it but has some attach types that have to be backward 1184aac3fc32SAndrey Ignatov * compatible. 1185aac3fc32SAndrey Ignatov */ 1186aac3fc32SAndrey Ignatov static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) 1187aac3fc32SAndrey Ignatov { 1188aac3fc32SAndrey Ignatov switch (attr->prog_type) { 1189aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1190aac3fc32SAndrey Ignatov /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't 1191aac3fc32SAndrey Ignatov * exist so checking for non-zero is the way to go here. 1192aac3fc32SAndrey Ignatov */ 1193aac3fc32SAndrey Ignatov if (!attr->expected_attach_type) 1194aac3fc32SAndrey Ignatov attr->expected_attach_type = 1195aac3fc32SAndrey Ignatov BPF_CGROUP_INET_SOCK_CREATE; 1196aac3fc32SAndrey Ignatov break; 1197aac3fc32SAndrey Ignatov } 1198aac3fc32SAndrey Ignatov } 1199aac3fc32SAndrey Ignatov 12005e43f899SAndrey Ignatov static int 12015e43f899SAndrey Ignatov bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type, 12025e43f899SAndrey Ignatov enum bpf_attach_type expected_attach_type) 12035e43f899SAndrey Ignatov { 12044fbac77dSAndrey Ignatov switch (prog_type) { 1205aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK: 1206aac3fc32SAndrey Ignatov switch (expected_attach_type) { 1207aac3fc32SAndrey Ignatov case BPF_CGROUP_INET_SOCK_CREATE: 1208aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1209aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 1210aac3fc32SAndrey Ignatov return 0; 1211aac3fc32SAndrey Ignatov default: 1212aac3fc32SAndrey Ignatov return -EINVAL; 1213aac3fc32SAndrey Ignatov } 12144fbac77dSAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 12154fbac77dSAndrey Ignatov switch (expected_attach_type) { 12164fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 12174fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1218d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1219d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 12205e43f899SAndrey Ignatov return 0; 12214fbac77dSAndrey Ignatov default: 12224fbac77dSAndrey Ignatov return -EINVAL; 12234fbac77dSAndrey Ignatov } 12244fbac77dSAndrey Ignatov default: 12254fbac77dSAndrey Ignatov return 0; 12264fbac77dSAndrey Ignatov } 12275e43f899SAndrey Ignatov } 12285e43f899SAndrey Ignatov 122909756af4SAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */ 12305e43f899SAndrey Ignatov #define BPF_PROG_LOAD_LAST_FIELD expected_attach_type 123109756af4SAlexei Starovoitov 123209756af4SAlexei Starovoitov static int bpf_prog_load(union bpf_attr *attr) 123309756af4SAlexei Starovoitov { 123409756af4SAlexei Starovoitov enum bpf_prog_type type = attr->prog_type; 123509756af4SAlexei Starovoitov struct bpf_prog *prog; 123609756af4SAlexei Starovoitov int err; 123709756af4SAlexei Starovoitov char license[128]; 123809756af4SAlexei Starovoitov bool is_gpl; 123909756af4SAlexei Starovoitov 124009756af4SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_LOAD)) 124109756af4SAlexei Starovoitov return -EINVAL; 124209756af4SAlexei Starovoitov 1243e07b98d9SDavid S. Miller if (attr->prog_flags & ~BPF_F_STRICT_ALIGNMENT) 1244e07b98d9SDavid S. Miller return -EINVAL; 1245e07b98d9SDavid S. Miller 124609756af4SAlexei Starovoitov /* copy eBPF program license from user space */ 1247535e7b4bSMickaël Salaün if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 124809756af4SAlexei Starovoitov sizeof(license) - 1) < 0) 124909756af4SAlexei Starovoitov return -EFAULT; 125009756af4SAlexei Starovoitov license[sizeof(license) - 1] = 0; 125109756af4SAlexei Starovoitov 125209756af4SAlexei Starovoitov /* eBPF programs must be GPL compatible to use GPL-ed functions */ 125309756af4SAlexei Starovoitov is_gpl = license_is_gpl_compatible(license); 125409756af4SAlexei Starovoitov 1255ef0915caSDaniel Borkmann if (attr->insn_cnt == 0 || attr->insn_cnt > BPF_MAXINSNS) 1256ef0915caSDaniel Borkmann return -E2BIG; 125709756af4SAlexei Starovoitov 12582541517cSAlexei Starovoitov if (type == BPF_PROG_TYPE_KPROBE && 12592541517cSAlexei Starovoitov attr->kern_version != LINUX_VERSION_CODE) 12602541517cSAlexei Starovoitov return -EINVAL; 12612541517cSAlexei Starovoitov 126280b7d819SChenbo Feng if (type != BPF_PROG_TYPE_SOCKET_FILTER && 126380b7d819SChenbo Feng type != BPF_PROG_TYPE_CGROUP_SKB && 126480b7d819SChenbo Feng !capable(CAP_SYS_ADMIN)) 12651be7f75dSAlexei Starovoitov return -EPERM; 12661be7f75dSAlexei Starovoitov 1267aac3fc32SAndrey Ignatov bpf_prog_load_fixup_attach_type(attr); 12685e43f899SAndrey Ignatov if (bpf_prog_load_check_attach_type(type, attr->expected_attach_type)) 12695e43f899SAndrey Ignatov return -EINVAL; 12705e43f899SAndrey Ignatov 127109756af4SAlexei Starovoitov /* plain bpf_prog allocation */ 127209756af4SAlexei Starovoitov prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 127309756af4SAlexei Starovoitov if (!prog) 127409756af4SAlexei Starovoitov return -ENOMEM; 127509756af4SAlexei Starovoitov 12765e43f899SAndrey Ignatov prog->expected_attach_type = attr->expected_attach_type; 12775e43f899SAndrey Ignatov 12789a18eedbSJakub Kicinski prog->aux->offload_requested = !!attr->prog_ifindex; 12799a18eedbSJakub Kicinski 1280afdb09c7SChenbo Feng err = security_bpf_prog_alloc(prog->aux); 1281aaac3ba9SAlexei Starovoitov if (err) 1282aaac3ba9SAlexei Starovoitov goto free_prog_nouncharge; 1283aaac3ba9SAlexei Starovoitov 1284afdb09c7SChenbo Feng err = bpf_prog_charge_memlock(prog); 1285afdb09c7SChenbo Feng if (err) 1286afdb09c7SChenbo Feng goto free_prog_sec; 1287afdb09c7SChenbo Feng 128809756af4SAlexei Starovoitov prog->len = attr->insn_cnt; 128909756af4SAlexei Starovoitov 129009756af4SAlexei Starovoitov err = -EFAULT; 1291535e7b4bSMickaël Salaün if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 1292aafe6ae9SDaniel Borkmann bpf_prog_insn_size(prog)) != 0) 129309756af4SAlexei Starovoitov goto free_prog; 129409756af4SAlexei Starovoitov 129509756af4SAlexei Starovoitov prog->orig_prog = NULL; 1296a91263d5SDaniel Borkmann prog->jited = 0; 129709756af4SAlexei Starovoitov 129809756af4SAlexei Starovoitov atomic_set(&prog->aux->refcnt, 1); 1299a91263d5SDaniel Borkmann prog->gpl_compatible = is_gpl ? 1 : 0; 130009756af4SAlexei Starovoitov 13019a18eedbSJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 1302ab3f0063SJakub Kicinski err = bpf_prog_offload_init(prog, attr); 1303ab3f0063SJakub Kicinski if (err) 1304ab3f0063SJakub Kicinski goto free_prog; 1305ab3f0063SJakub Kicinski } 1306ab3f0063SJakub Kicinski 130709756af4SAlexei Starovoitov /* find program type: socket_filter vs tracing_filter */ 130809756af4SAlexei Starovoitov err = find_prog_type(type, prog); 130909756af4SAlexei Starovoitov if (err < 0) 131009756af4SAlexei Starovoitov goto free_prog; 131109756af4SAlexei Starovoitov 1312cb4d2b3fSMartin KaFai Lau prog->aux->load_time = ktime_get_boot_ns(); 1313cb4d2b3fSMartin KaFai Lau err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name); 1314cb4d2b3fSMartin KaFai Lau if (err) 1315cb4d2b3fSMartin KaFai Lau goto free_prog; 1316cb4d2b3fSMartin KaFai Lau 131709756af4SAlexei Starovoitov /* run eBPF verifier */ 13189bac3d6dSAlexei Starovoitov err = bpf_check(&prog, attr); 131909756af4SAlexei Starovoitov if (err < 0) 132009756af4SAlexei Starovoitov goto free_used_maps; 132109756af4SAlexei Starovoitov 132209756af4SAlexei Starovoitov /* eBPF program is ready to be JITed */ 13231c2a088aSAlexei Starovoitov if (!prog->bpf_func) 1324d1c55ab5SDaniel Borkmann prog = bpf_prog_select_runtime(prog, &err); 132504fd61abSAlexei Starovoitov if (err < 0) 132604fd61abSAlexei Starovoitov goto free_used_maps; 132709756af4SAlexei Starovoitov 1328dc4bb0e2SMartin KaFai Lau err = bpf_prog_alloc_id(prog); 1329dc4bb0e2SMartin KaFai Lau if (err) 1330dc4bb0e2SMartin KaFai Lau goto free_used_maps; 1331dc4bb0e2SMartin KaFai Lau 1332aa79781bSDaniel Borkmann err = bpf_prog_new_fd(prog); 1333b16d9aa4SMartin KaFai Lau if (err < 0) { 1334b16d9aa4SMartin KaFai Lau /* failed to allocate fd. 1335b16d9aa4SMartin KaFai Lau * bpf_prog_put() is needed because the above 1336b16d9aa4SMartin KaFai Lau * bpf_prog_alloc_id() has published the prog 1337b16d9aa4SMartin KaFai Lau * to the userspace and the userspace may 1338b16d9aa4SMartin KaFai Lau * have refcnt-ed it through BPF_PROG_GET_FD_BY_ID. 1339b16d9aa4SMartin KaFai Lau */ 1340b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 1341b16d9aa4SMartin KaFai Lau return err; 1342b16d9aa4SMartin KaFai Lau } 134309756af4SAlexei Starovoitov 134474451e66SDaniel Borkmann bpf_prog_kallsyms_add(prog); 1345a67edbf4SDaniel Borkmann trace_bpf_prog_load(prog, err); 134609756af4SAlexei Starovoitov return err; 134709756af4SAlexei Starovoitov 134809756af4SAlexei Starovoitov free_used_maps: 134909756af4SAlexei Starovoitov free_used_maps(prog->aux); 135009756af4SAlexei Starovoitov free_prog: 1351aaac3ba9SAlexei Starovoitov bpf_prog_uncharge_memlock(prog); 1352afdb09c7SChenbo Feng free_prog_sec: 1353afdb09c7SChenbo Feng security_bpf_prog_free(prog->aux); 1354aaac3ba9SAlexei Starovoitov free_prog_nouncharge: 135509756af4SAlexei Starovoitov bpf_prog_free(prog); 135609756af4SAlexei Starovoitov return err; 135709756af4SAlexei Starovoitov } 135809756af4SAlexei Starovoitov 13596e71b04aSChenbo Feng #define BPF_OBJ_LAST_FIELD file_flags 1360b2197755SDaniel Borkmann 1361b2197755SDaniel Borkmann static int bpf_obj_pin(const union bpf_attr *attr) 1362b2197755SDaniel Borkmann { 13636e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0) 1364b2197755SDaniel Borkmann return -EINVAL; 1365b2197755SDaniel Borkmann 1366535e7b4bSMickaël Salaün return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 1367b2197755SDaniel Borkmann } 1368b2197755SDaniel Borkmann 1369b2197755SDaniel Borkmann static int bpf_obj_get(const union bpf_attr *attr) 1370b2197755SDaniel Borkmann { 13716e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || 13726e71b04aSChenbo Feng attr->file_flags & ~BPF_OBJ_FLAG_MASK) 1373b2197755SDaniel Borkmann return -EINVAL; 1374b2197755SDaniel Borkmann 13756e71b04aSChenbo Feng return bpf_obj_get_user(u64_to_user_ptr(attr->pathname), 13766e71b04aSChenbo Feng attr->file_flags); 1377b2197755SDaniel Borkmann } 1378b2197755SDaniel Borkmann 1379c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint { 1380c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 1381c4f6699dSAlexei Starovoitov struct bpf_prog *prog; 1382c4f6699dSAlexei Starovoitov }; 1383c4f6699dSAlexei Starovoitov 1384c4f6699dSAlexei Starovoitov static int bpf_raw_tracepoint_release(struct inode *inode, struct file *filp) 1385c4f6699dSAlexei Starovoitov { 1386c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint *raw_tp = filp->private_data; 1387c4f6699dSAlexei Starovoitov 1388c4f6699dSAlexei Starovoitov if (raw_tp->prog) { 1389c4f6699dSAlexei Starovoitov bpf_probe_unregister(raw_tp->btp, raw_tp->prog); 1390c4f6699dSAlexei Starovoitov bpf_prog_put(raw_tp->prog); 1391c4f6699dSAlexei Starovoitov } 1392c4f6699dSAlexei Starovoitov kfree(raw_tp); 1393c4f6699dSAlexei Starovoitov return 0; 1394c4f6699dSAlexei Starovoitov } 1395c4f6699dSAlexei Starovoitov 1396c4f6699dSAlexei Starovoitov static const struct file_operations bpf_raw_tp_fops = { 1397c4f6699dSAlexei Starovoitov .release = bpf_raw_tracepoint_release, 1398c4f6699dSAlexei Starovoitov .read = bpf_dummy_read, 1399c4f6699dSAlexei Starovoitov .write = bpf_dummy_write, 1400c4f6699dSAlexei Starovoitov }; 1401c4f6699dSAlexei Starovoitov 1402c4f6699dSAlexei Starovoitov #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd 1403c4f6699dSAlexei Starovoitov 1404c4f6699dSAlexei Starovoitov static int bpf_raw_tracepoint_open(const union bpf_attr *attr) 1405c4f6699dSAlexei Starovoitov { 1406c4f6699dSAlexei Starovoitov struct bpf_raw_tracepoint *raw_tp; 1407c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp; 1408c4f6699dSAlexei Starovoitov struct bpf_prog *prog; 1409c4f6699dSAlexei Starovoitov char tp_name[128]; 1410c4f6699dSAlexei Starovoitov int tp_fd, err; 1411c4f6699dSAlexei Starovoitov 1412c4f6699dSAlexei Starovoitov if (strncpy_from_user(tp_name, u64_to_user_ptr(attr->raw_tracepoint.name), 1413c4f6699dSAlexei Starovoitov sizeof(tp_name) - 1) < 0) 1414c4f6699dSAlexei Starovoitov return -EFAULT; 1415c4f6699dSAlexei Starovoitov tp_name[sizeof(tp_name) - 1] = 0; 1416c4f6699dSAlexei Starovoitov 1417c4f6699dSAlexei Starovoitov btp = bpf_find_raw_tracepoint(tp_name); 1418c4f6699dSAlexei Starovoitov if (!btp) 1419c4f6699dSAlexei Starovoitov return -ENOENT; 1420c4f6699dSAlexei Starovoitov 1421c4f6699dSAlexei Starovoitov raw_tp = kzalloc(sizeof(*raw_tp), GFP_USER); 1422c4f6699dSAlexei Starovoitov if (!raw_tp) 1423c4f6699dSAlexei Starovoitov return -ENOMEM; 1424c4f6699dSAlexei Starovoitov raw_tp->btp = btp; 1425c4f6699dSAlexei Starovoitov 1426c4f6699dSAlexei Starovoitov prog = bpf_prog_get_type(attr->raw_tracepoint.prog_fd, 1427c4f6699dSAlexei Starovoitov BPF_PROG_TYPE_RAW_TRACEPOINT); 1428c4f6699dSAlexei Starovoitov if (IS_ERR(prog)) { 1429c4f6699dSAlexei Starovoitov err = PTR_ERR(prog); 1430c4f6699dSAlexei Starovoitov goto out_free_tp; 1431c4f6699dSAlexei Starovoitov } 1432c4f6699dSAlexei Starovoitov 1433c4f6699dSAlexei Starovoitov err = bpf_probe_register(raw_tp->btp, prog); 1434c4f6699dSAlexei Starovoitov if (err) 1435c4f6699dSAlexei Starovoitov goto out_put_prog; 1436c4f6699dSAlexei Starovoitov 1437c4f6699dSAlexei Starovoitov raw_tp->prog = prog; 1438c4f6699dSAlexei Starovoitov tp_fd = anon_inode_getfd("bpf-raw-tracepoint", &bpf_raw_tp_fops, raw_tp, 1439c4f6699dSAlexei Starovoitov O_CLOEXEC); 1440c4f6699dSAlexei Starovoitov if (tp_fd < 0) { 1441c4f6699dSAlexei Starovoitov bpf_probe_unregister(raw_tp->btp, prog); 1442c4f6699dSAlexei Starovoitov err = tp_fd; 1443c4f6699dSAlexei Starovoitov goto out_put_prog; 1444c4f6699dSAlexei Starovoitov } 1445c4f6699dSAlexei Starovoitov return tp_fd; 1446c4f6699dSAlexei Starovoitov 1447c4f6699dSAlexei Starovoitov out_put_prog: 1448c4f6699dSAlexei Starovoitov bpf_prog_put(prog); 1449c4f6699dSAlexei Starovoitov out_free_tp: 1450c4f6699dSAlexei Starovoitov kfree(raw_tp); 1451c4f6699dSAlexei Starovoitov return err; 1452c4f6699dSAlexei Starovoitov } 1453c4f6699dSAlexei Starovoitov 1454f4324551SDaniel Mack #ifdef CONFIG_CGROUP_BPF 1455f4324551SDaniel Mack 1456*33491588SAnders Roxell static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, 1457*33491588SAnders Roxell enum bpf_attach_type attach_type) 1458*33491588SAnders Roxell { 1459*33491588SAnders Roxell switch (prog->type) { 1460*33491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK: 1461*33491588SAnders Roxell case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 1462*33491588SAnders Roxell return attach_type == prog->expected_attach_type ? 0 : -EINVAL; 1463*33491588SAnders Roxell default: 1464*33491588SAnders Roxell return 0; 1465*33491588SAnders Roxell } 1466*33491588SAnders Roxell } 1467*33491588SAnders Roxell 1468464bc0fdSJohn Fastabend #define BPF_PROG_ATTACH_LAST_FIELD attach_flags 1469174a79ffSJohn Fastabend 14704f738adbSJohn Fastabend static int sockmap_get_from_fd(const union bpf_attr *attr, 14714f738adbSJohn Fastabend int type, bool attach) 1472174a79ffSJohn Fastabend { 14735a67da2aSJohn Fastabend struct bpf_prog *prog = NULL; 1474174a79ffSJohn Fastabend int ufd = attr->target_fd; 1475174a79ffSJohn Fastabend struct bpf_map *map; 1476174a79ffSJohn Fastabend struct fd f; 1477174a79ffSJohn Fastabend int err; 1478174a79ffSJohn Fastabend 1479174a79ffSJohn Fastabend f = fdget(ufd); 1480174a79ffSJohn Fastabend map = __bpf_map_get(f); 1481174a79ffSJohn Fastabend if (IS_ERR(map)) 1482174a79ffSJohn Fastabend return PTR_ERR(map); 1483174a79ffSJohn Fastabend 14845a67da2aSJohn Fastabend if (attach) { 14854f738adbSJohn Fastabend prog = bpf_prog_get_type(attr->attach_bpf_fd, type); 1486464bc0fdSJohn Fastabend if (IS_ERR(prog)) { 1487174a79ffSJohn Fastabend fdput(f); 1488464bc0fdSJohn Fastabend return PTR_ERR(prog); 1489174a79ffSJohn Fastabend } 14905a67da2aSJohn Fastabend } 1491174a79ffSJohn Fastabend 14925a67da2aSJohn Fastabend err = sock_map_prog(map, prog, attr->attach_type); 1493174a79ffSJohn Fastabend if (err) { 1494174a79ffSJohn Fastabend fdput(f); 14955a67da2aSJohn Fastabend if (prog) 1496464bc0fdSJohn Fastabend bpf_prog_put(prog); 1497ae2b27b8SDan Carpenter return err; 1498174a79ffSJohn Fastabend } 1499174a79ffSJohn Fastabend 1500174a79ffSJohn Fastabend fdput(f); 1501ae2b27b8SDan Carpenter return 0; 1502174a79ffSJohn Fastabend } 1503f4324551SDaniel Mack 1504324bda9eSAlexei Starovoitov #define BPF_F_ATTACH_MASK \ 1505324bda9eSAlexei Starovoitov (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI) 1506324bda9eSAlexei Starovoitov 1507f4324551SDaniel Mack static int bpf_prog_attach(const union bpf_attr *attr) 1508f4324551SDaniel Mack { 15097f677633SAlexei Starovoitov enum bpf_prog_type ptype; 1510f4324551SDaniel Mack struct bpf_prog *prog; 1511f4324551SDaniel Mack struct cgroup *cgrp; 15127f677633SAlexei Starovoitov int ret; 1513f4324551SDaniel Mack 1514f4324551SDaniel Mack if (!capable(CAP_NET_ADMIN)) 1515f4324551SDaniel Mack return -EPERM; 1516f4324551SDaniel Mack 1517f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_ATTACH)) 1518f4324551SDaniel Mack return -EINVAL; 1519f4324551SDaniel Mack 1520324bda9eSAlexei Starovoitov if (attr->attach_flags & ~BPF_F_ATTACH_MASK) 15217f677633SAlexei Starovoitov return -EINVAL; 15227f677633SAlexei Starovoitov 1523f4324551SDaniel Mack switch (attr->attach_type) { 1524f4324551SDaniel Mack case BPF_CGROUP_INET_INGRESS: 1525f4324551SDaniel Mack case BPF_CGROUP_INET_EGRESS: 1526b2cd1257SDavid Ahern ptype = BPF_PROG_TYPE_CGROUP_SKB; 1527b2cd1257SDavid Ahern break; 152861023658SDavid Ahern case BPF_CGROUP_INET_SOCK_CREATE: 1529aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1530aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 153161023658SDavid Ahern ptype = BPF_PROG_TYPE_CGROUP_SOCK; 153261023658SDavid Ahern break; 15334fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 15344fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1535d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1536d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 15374fbac77dSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 15384fbac77dSAndrey Ignatov break; 153940304b2aSLawrence Brakmo case BPF_CGROUP_SOCK_OPS: 154040304b2aSLawrence Brakmo ptype = BPF_PROG_TYPE_SOCK_OPS; 154140304b2aSLawrence Brakmo break; 1542ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 1543ebc614f6SRoman Gushchin ptype = BPF_PROG_TYPE_CGROUP_DEVICE; 1544ebc614f6SRoman Gushchin break; 15454f738adbSJohn Fastabend case BPF_SK_MSG_VERDICT: 15464f738adbSJohn Fastabend return sockmap_get_from_fd(attr, BPF_PROG_TYPE_SK_MSG, true); 1547464bc0fdSJohn Fastabend case BPF_SK_SKB_STREAM_PARSER: 1548464bc0fdSJohn Fastabend case BPF_SK_SKB_STREAM_VERDICT: 15494f738adbSJohn Fastabend return sockmap_get_from_fd(attr, BPF_PROG_TYPE_SK_SKB, true); 1550b2cd1257SDavid Ahern default: 1551b2cd1257SDavid Ahern return -EINVAL; 1552b2cd1257SDavid Ahern } 1553b2cd1257SDavid Ahern 1554b2cd1257SDavid Ahern prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 1555f4324551SDaniel Mack if (IS_ERR(prog)) 1556f4324551SDaniel Mack return PTR_ERR(prog); 1557f4324551SDaniel Mack 15585e43f899SAndrey Ignatov if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) { 15595e43f899SAndrey Ignatov bpf_prog_put(prog); 15605e43f899SAndrey Ignatov return -EINVAL; 15615e43f899SAndrey Ignatov } 15625e43f899SAndrey Ignatov 1563f4324551SDaniel Mack cgrp = cgroup_get_from_fd(attr->target_fd); 1564f4324551SDaniel Mack if (IS_ERR(cgrp)) { 1565f4324551SDaniel Mack bpf_prog_put(prog); 1566f4324551SDaniel Mack return PTR_ERR(cgrp); 1567f4324551SDaniel Mack } 1568f4324551SDaniel Mack 1569324bda9eSAlexei Starovoitov ret = cgroup_bpf_attach(cgrp, prog, attr->attach_type, 1570324bda9eSAlexei Starovoitov attr->attach_flags); 15717f677633SAlexei Starovoitov if (ret) 15727f677633SAlexei Starovoitov bpf_prog_put(prog); 1573f4324551SDaniel Mack cgroup_put(cgrp); 1574f4324551SDaniel Mack 15757f677633SAlexei Starovoitov return ret; 1576f4324551SDaniel Mack } 1577f4324551SDaniel Mack 1578f4324551SDaniel Mack #define BPF_PROG_DETACH_LAST_FIELD attach_type 1579f4324551SDaniel Mack 1580f4324551SDaniel Mack static int bpf_prog_detach(const union bpf_attr *attr) 1581f4324551SDaniel Mack { 1582324bda9eSAlexei Starovoitov enum bpf_prog_type ptype; 1583324bda9eSAlexei Starovoitov struct bpf_prog *prog; 1584f4324551SDaniel Mack struct cgroup *cgrp; 15857f677633SAlexei Starovoitov int ret; 1586f4324551SDaniel Mack 1587f4324551SDaniel Mack if (!capable(CAP_NET_ADMIN)) 1588f4324551SDaniel Mack return -EPERM; 1589f4324551SDaniel Mack 1590f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_DETACH)) 1591f4324551SDaniel Mack return -EINVAL; 1592f4324551SDaniel Mack 1593f4324551SDaniel Mack switch (attr->attach_type) { 1594f4324551SDaniel Mack case BPF_CGROUP_INET_INGRESS: 1595f4324551SDaniel Mack case BPF_CGROUP_INET_EGRESS: 1596324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_CGROUP_SKB; 1597324bda9eSAlexei Starovoitov break; 159861023658SDavid Ahern case BPF_CGROUP_INET_SOCK_CREATE: 1599aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1600aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 1601324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_CGROUP_SOCK; 1602324bda9eSAlexei Starovoitov break; 16034fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 16044fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1605d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1606d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 16074fbac77dSAndrey Ignatov ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 16084fbac77dSAndrey Ignatov break; 160940304b2aSLawrence Brakmo case BPF_CGROUP_SOCK_OPS: 1610324bda9eSAlexei Starovoitov ptype = BPF_PROG_TYPE_SOCK_OPS; 1611f4324551SDaniel Mack break; 1612ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 1613ebc614f6SRoman Gushchin ptype = BPF_PROG_TYPE_CGROUP_DEVICE; 1614ebc614f6SRoman Gushchin break; 16154f738adbSJohn Fastabend case BPF_SK_MSG_VERDICT: 16164f738adbSJohn Fastabend return sockmap_get_from_fd(attr, BPF_PROG_TYPE_SK_MSG, false); 16175a67da2aSJohn Fastabend case BPF_SK_SKB_STREAM_PARSER: 16185a67da2aSJohn Fastabend case BPF_SK_SKB_STREAM_VERDICT: 16194f738adbSJohn Fastabend return sockmap_get_from_fd(attr, BPF_PROG_TYPE_SK_SKB, false); 1620f4324551SDaniel Mack default: 1621f4324551SDaniel Mack return -EINVAL; 1622f4324551SDaniel Mack } 1623f4324551SDaniel Mack 1624324bda9eSAlexei Starovoitov cgrp = cgroup_get_from_fd(attr->target_fd); 1625324bda9eSAlexei Starovoitov if (IS_ERR(cgrp)) 1626324bda9eSAlexei Starovoitov return PTR_ERR(cgrp); 1627324bda9eSAlexei Starovoitov 1628324bda9eSAlexei Starovoitov prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 1629324bda9eSAlexei Starovoitov if (IS_ERR(prog)) 1630324bda9eSAlexei Starovoitov prog = NULL; 1631324bda9eSAlexei Starovoitov 1632324bda9eSAlexei Starovoitov ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type, 0); 1633324bda9eSAlexei Starovoitov if (prog) 1634324bda9eSAlexei Starovoitov bpf_prog_put(prog); 1635324bda9eSAlexei Starovoitov cgroup_put(cgrp); 16367f677633SAlexei Starovoitov return ret; 1637f4324551SDaniel Mack } 163840304b2aSLawrence Brakmo 1639468e2f64SAlexei Starovoitov #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt 1640468e2f64SAlexei Starovoitov 1641468e2f64SAlexei Starovoitov static int bpf_prog_query(const union bpf_attr *attr, 1642468e2f64SAlexei Starovoitov union bpf_attr __user *uattr) 1643468e2f64SAlexei Starovoitov { 1644468e2f64SAlexei Starovoitov struct cgroup *cgrp; 1645468e2f64SAlexei Starovoitov int ret; 1646468e2f64SAlexei Starovoitov 1647468e2f64SAlexei Starovoitov if (!capable(CAP_NET_ADMIN)) 1648468e2f64SAlexei Starovoitov return -EPERM; 1649468e2f64SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_QUERY)) 1650468e2f64SAlexei Starovoitov return -EINVAL; 1651468e2f64SAlexei Starovoitov if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE) 1652468e2f64SAlexei Starovoitov return -EINVAL; 1653468e2f64SAlexei Starovoitov 1654468e2f64SAlexei Starovoitov switch (attr->query.attach_type) { 1655468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_INGRESS: 1656468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_EGRESS: 1657468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_SOCK_CREATE: 16584fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND: 16594fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND: 1660aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND: 1661aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND: 1662d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT: 1663d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT: 1664468e2f64SAlexei Starovoitov case BPF_CGROUP_SOCK_OPS: 1665ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE: 1666468e2f64SAlexei Starovoitov break; 1667468e2f64SAlexei Starovoitov default: 1668468e2f64SAlexei Starovoitov return -EINVAL; 1669468e2f64SAlexei Starovoitov } 1670468e2f64SAlexei Starovoitov cgrp = cgroup_get_from_fd(attr->query.target_fd); 1671468e2f64SAlexei Starovoitov if (IS_ERR(cgrp)) 1672468e2f64SAlexei Starovoitov return PTR_ERR(cgrp); 1673468e2f64SAlexei Starovoitov ret = cgroup_bpf_query(cgrp, attr, uattr); 1674468e2f64SAlexei Starovoitov cgroup_put(cgrp); 1675468e2f64SAlexei Starovoitov return ret; 1676468e2f64SAlexei Starovoitov } 1677f4324551SDaniel Mack #endif /* CONFIG_CGROUP_BPF */ 1678f4324551SDaniel Mack 16791cf1cae9SAlexei Starovoitov #define BPF_PROG_TEST_RUN_LAST_FIELD test.duration 16801cf1cae9SAlexei Starovoitov 16811cf1cae9SAlexei Starovoitov static int bpf_prog_test_run(const union bpf_attr *attr, 16821cf1cae9SAlexei Starovoitov union bpf_attr __user *uattr) 16831cf1cae9SAlexei Starovoitov { 16841cf1cae9SAlexei Starovoitov struct bpf_prog *prog; 16851cf1cae9SAlexei Starovoitov int ret = -ENOTSUPP; 16861cf1cae9SAlexei Starovoitov 168761f3c964SAlexei Starovoitov if (!capable(CAP_SYS_ADMIN)) 168861f3c964SAlexei Starovoitov return -EPERM; 16891cf1cae9SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 16901cf1cae9SAlexei Starovoitov return -EINVAL; 16911cf1cae9SAlexei Starovoitov 16921cf1cae9SAlexei Starovoitov prog = bpf_prog_get(attr->test.prog_fd); 16931cf1cae9SAlexei Starovoitov if (IS_ERR(prog)) 16941cf1cae9SAlexei Starovoitov return PTR_ERR(prog); 16951cf1cae9SAlexei Starovoitov 16961cf1cae9SAlexei Starovoitov if (prog->aux->ops->test_run) 16971cf1cae9SAlexei Starovoitov ret = prog->aux->ops->test_run(prog, attr, uattr); 16981cf1cae9SAlexei Starovoitov 16991cf1cae9SAlexei Starovoitov bpf_prog_put(prog); 17001cf1cae9SAlexei Starovoitov return ret; 17011cf1cae9SAlexei Starovoitov } 17021cf1cae9SAlexei Starovoitov 170334ad5580SMartin KaFai Lau #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 170434ad5580SMartin KaFai Lau 170534ad5580SMartin KaFai Lau static int bpf_obj_get_next_id(const union bpf_attr *attr, 170634ad5580SMartin KaFai Lau union bpf_attr __user *uattr, 170734ad5580SMartin KaFai Lau struct idr *idr, 170834ad5580SMartin KaFai Lau spinlock_t *lock) 170934ad5580SMartin KaFai Lau { 171034ad5580SMartin KaFai Lau u32 next_id = attr->start_id; 171134ad5580SMartin KaFai Lau int err = 0; 171234ad5580SMartin KaFai Lau 171334ad5580SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 171434ad5580SMartin KaFai Lau return -EINVAL; 171534ad5580SMartin KaFai Lau 171634ad5580SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 171734ad5580SMartin KaFai Lau return -EPERM; 171834ad5580SMartin KaFai Lau 171934ad5580SMartin KaFai Lau next_id++; 172034ad5580SMartin KaFai Lau spin_lock_bh(lock); 172134ad5580SMartin KaFai Lau if (!idr_get_next(idr, &next_id)) 172234ad5580SMartin KaFai Lau err = -ENOENT; 172334ad5580SMartin KaFai Lau spin_unlock_bh(lock); 172434ad5580SMartin KaFai Lau 172534ad5580SMartin KaFai Lau if (!err) 172634ad5580SMartin KaFai Lau err = put_user(next_id, &uattr->next_id); 172734ad5580SMartin KaFai Lau 172834ad5580SMartin KaFai Lau return err; 172934ad5580SMartin KaFai Lau } 173034ad5580SMartin KaFai Lau 1731b16d9aa4SMartin KaFai Lau #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 1732b16d9aa4SMartin KaFai Lau 1733b16d9aa4SMartin KaFai Lau static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 1734b16d9aa4SMartin KaFai Lau { 1735b16d9aa4SMartin KaFai Lau struct bpf_prog *prog; 1736b16d9aa4SMartin KaFai Lau u32 id = attr->prog_id; 1737b16d9aa4SMartin KaFai Lau int fd; 1738b16d9aa4SMartin KaFai Lau 1739b16d9aa4SMartin KaFai Lau if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 1740b16d9aa4SMartin KaFai Lau return -EINVAL; 1741b16d9aa4SMartin KaFai Lau 1742b16d9aa4SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 1743b16d9aa4SMartin KaFai Lau return -EPERM; 1744b16d9aa4SMartin KaFai Lau 1745b16d9aa4SMartin KaFai Lau spin_lock_bh(&prog_idr_lock); 1746b16d9aa4SMartin KaFai Lau prog = idr_find(&prog_idr, id); 1747b16d9aa4SMartin KaFai Lau if (prog) 1748b16d9aa4SMartin KaFai Lau prog = bpf_prog_inc_not_zero(prog); 1749b16d9aa4SMartin KaFai Lau else 1750b16d9aa4SMartin KaFai Lau prog = ERR_PTR(-ENOENT); 1751b16d9aa4SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock); 1752b16d9aa4SMartin KaFai Lau 1753b16d9aa4SMartin KaFai Lau if (IS_ERR(prog)) 1754b16d9aa4SMartin KaFai Lau return PTR_ERR(prog); 1755b16d9aa4SMartin KaFai Lau 1756b16d9aa4SMartin KaFai Lau fd = bpf_prog_new_fd(prog); 1757b16d9aa4SMartin KaFai Lau if (fd < 0) 1758b16d9aa4SMartin KaFai Lau bpf_prog_put(prog); 1759b16d9aa4SMartin KaFai Lau 1760b16d9aa4SMartin KaFai Lau return fd; 1761b16d9aa4SMartin KaFai Lau } 1762b16d9aa4SMartin KaFai Lau 17636e71b04aSChenbo Feng #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags 1764bd5f5f4eSMartin KaFai Lau 1765bd5f5f4eSMartin KaFai Lau static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 1766bd5f5f4eSMartin KaFai Lau { 1767bd5f5f4eSMartin KaFai Lau struct bpf_map *map; 1768bd5f5f4eSMartin KaFai Lau u32 id = attr->map_id; 17696e71b04aSChenbo Feng int f_flags; 1770bd5f5f4eSMartin KaFai Lau int fd; 1771bd5f5f4eSMartin KaFai Lau 17726e71b04aSChenbo Feng if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || 17736e71b04aSChenbo Feng attr->open_flags & ~BPF_OBJ_FLAG_MASK) 1774bd5f5f4eSMartin KaFai Lau return -EINVAL; 1775bd5f5f4eSMartin KaFai Lau 1776bd5f5f4eSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) 1777bd5f5f4eSMartin KaFai Lau return -EPERM; 1778bd5f5f4eSMartin KaFai Lau 17796e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->open_flags); 17806e71b04aSChenbo Feng if (f_flags < 0) 17816e71b04aSChenbo Feng return f_flags; 17826e71b04aSChenbo Feng 1783bd5f5f4eSMartin KaFai Lau spin_lock_bh(&map_idr_lock); 1784bd5f5f4eSMartin KaFai Lau map = idr_find(&map_idr, id); 1785bd5f5f4eSMartin KaFai Lau if (map) 1786bd5f5f4eSMartin KaFai Lau map = bpf_map_inc_not_zero(map, true); 1787bd5f5f4eSMartin KaFai Lau else 1788bd5f5f4eSMartin KaFai Lau map = ERR_PTR(-ENOENT); 1789bd5f5f4eSMartin KaFai Lau spin_unlock_bh(&map_idr_lock); 1790bd5f5f4eSMartin KaFai Lau 1791bd5f5f4eSMartin KaFai Lau if (IS_ERR(map)) 1792bd5f5f4eSMartin KaFai Lau return PTR_ERR(map); 1793bd5f5f4eSMartin KaFai Lau 17946e71b04aSChenbo Feng fd = bpf_map_new_fd(map, f_flags); 1795bd5f5f4eSMartin KaFai Lau if (fd < 0) 1796bd5f5f4eSMartin KaFai Lau bpf_map_put(map); 1797bd5f5f4eSMartin KaFai Lau 1798bd5f5f4eSMartin KaFai Lau return fd; 1799bd5f5f4eSMartin KaFai Lau } 1800bd5f5f4eSMartin KaFai Lau 18017105e828SDaniel Borkmann static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, 18027105e828SDaniel Borkmann unsigned long addr) 18037105e828SDaniel Borkmann { 18047105e828SDaniel Borkmann int i; 18057105e828SDaniel Borkmann 18067105e828SDaniel Borkmann for (i = 0; i < prog->aux->used_map_cnt; i++) 18077105e828SDaniel Borkmann if (prog->aux->used_maps[i] == (void *)addr) 18087105e828SDaniel Borkmann return prog->aux->used_maps[i]; 18097105e828SDaniel Borkmann return NULL; 18107105e828SDaniel Borkmann } 18117105e828SDaniel Borkmann 18127105e828SDaniel Borkmann static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog) 18137105e828SDaniel Borkmann { 18147105e828SDaniel Borkmann const struct bpf_map *map; 18157105e828SDaniel Borkmann struct bpf_insn *insns; 18167105e828SDaniel Borkmann u64 imm; 18177105e828SDaniel Borkmann int i; 18187105e828SDaniel Borkmann 18197105e828SDaniel Borkmann insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), 18207105e828SDaniel Borkmann GFP_USER); 18217105e828SDaniel Borkmann if (!insns) 18227105e828SDaniel Borkmann return insns; 18237105e828SDaniel Borkmann 18247105e828SDaniel Borkmann for (i = 0; i < prog->len; i++) { 18257105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_TAIL_CALL)) { 18267105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 18277105e828SDaniel Borkmann insns[i].imm = BPF_FUNC_tail_call; 18287105e828SDaniel Borkmann /* fall-through */ 18297105e828SDaniel Borkmann } 18307105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_CALL) || 18317105e828SDaniel Borkmann insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) { 18327105e828SDaniel Borkmann if (insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) 18337105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL; 18347105e828SDaniel Borkmann if (!bpf_dump_raw_ok()) 18357105e828SDaniel Borkmann insns[i].imm = 0; 18367105e828SDaniel Borkmann continue; 18377105e828SDaniel Borkmann } 18387105e828SDaniel Borkmann 18397105e828SDaniel Borkmann if (insns[i].code != (BPF_LD | BPF_IMM | BPF_DW)) 18407105e828SDaniel Borkmann continue; 18417105e828SDaniel Borkmann 18427105e828SDaniel Borkmann imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; 18437105e828SDaniel Borkmann map = bpf_map_from_imm(prog, imm); 18447105e828SDaniel Borkmann if (map) { 18457105e828SDaniel Borkmann insns[i].src_reg = BPF_PSEUDO_MAP_FD; 18467105e828SDaniel Borkmann insns[i].imm = map->id; 18477105e828SDaniel Borkmann insns[i + 1].imm = 0; 18487105e828SDaniel Borkmann continue; 18497105e828SDaniel Borkmann } 18507105e828SDaniel Borkmann 18517105e828SDaniel Borkmann if (!bpf_dump_raw_ok() && 18527105e828SDaniel Borkmann imm == (unsigned long)prog->aux) { 18537105e828SDaniel Borkmann insns[i].imm = 0; 18547105e828SDaniel Borkmann insns[i + 1].imm = 0; 18557105e828SDaniel Borkmann continue; 18567105e828SDaniel Borkmann } 18577105e828SDaniel Borkmann } 18587105e828SDaniel Borkmann 18597105e828SDaniel Borkmann return insns; 18607105e828SDaniel Borkmann } 18617105e828SDaniel Borkmann 18621e270976SMartin KaFai Lau static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, 18631e270976SMartin KaFai Lau const union bpf_attr *attr, 18641e270976SMartin KaFai Lau union bpf_attr __user *uattr) 18651e270976SMartin KaFai Lau { 18661e270976SMartin KaFai Lau struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 18671e270976SMartin KaFai Lau struct bpf_prog_info info = {}; 18681e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 18691e270976SMartin KaFai Lau char __user *uinsns; 18701e270976SMartin KaFai Lau u32 ulen; 18711e270976SMartin KaFai Lau int err; 18721e270976SMartin KaFai Lau 18731e270976SMartin KaFai Lau err = check_uarg_tail_zero(uinfo, sizeof(info), info_len); 18741e270976SMartin KaFai Lau if (err) 18751e270976SMartin KaFai Lau return err; 18761e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 18771e270976SMartin KaFai Lau 18781e270976SMartin KaFai Lau if (copy_from_user(&info, uinfo, info_len)) 187989b09689SDaniel Borkmann return -EFAULT; 18801e270976SMartin KaFai Lau 18811e270976SMartin KaFai Lau info.type = prog->type; 18821e270976SMartin KaFai Lau info.id = prog->aux->id; 1883cb4d2b3fSMartin KaFai Lau info.load_time = prog->aux->load_time; 1884cb4d2b3fSMartin KaFai Lau info.created_by_uid = from_kuid_munged(current_user_ns(), 1885cb4d2b3fSMartin KaFai Lau prog->aux->user->uid); 18861e270976SMartin KaFai Lau 18871e270976SMartin KaFai Lau memcpy(info.tag, prog->tag, sizeof(prog->tag)); 1888cb4d2b3fSMartin KaFai Lau memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); 1889cb4d2b3fSMartin KaFai Lau 1890cb4d2b3fSMartin KaFai Lau ulen = info.nr_map_ids; 1891cb4d2b3fSMartin KaFai Lau info.nr_map_ids = prog->aux->used_map_cnt; 1892cb4d2b3fSMartin KaFai Lau ulen = min_t(u32, info.nr_map_ids, ulen); 1893cb4d2b3fSMartin KaFai Lau if (ulen) { 1894721e08daSMartin KaFai Lau u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); 1895cb4d2b3fSMartin KaFai Lau u32 i; 1896cb4d2b3fSMartin KaFai Lau 1897cb4d2b3fSMartin KaFai Lau for (i = 0; i < ulen; i++) 1898cb4d2b3fSMartin KaFai Lau if (put_user(prog->aux->used_maps[i]->id, 1899cb4d2b3fSMartin KaFai Lau &user_map_ids[i])) 1900cb4d2b3fSMartin KaFai Lau return -EFAULT; 1901cb4d2b3fSMartin KaFai Lau } 19021e270976SMartin KaFai Lau 19031e270976SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN)) { 19041e270976SMartin KaFai Lau info.jited_prog_len = 0; 19051e270976SMartin KaFai Lau info.xlated_prog_len = 0; 19061e270976SMartin KaFai Lau goto done; 19071e270976SMartin KaFai Lau } 19081e270976SMartin KaFai Lau 19091e270976SMartin KaFai Lau ulen = info.xlated_prog_len; 19109975a54bSDaniel Borkmann info.xlated_prog_len = bpf_prog_insn_size(prog); 19111e270976SMartin KaFai Lau if (info.xlated_prog_len && ulen) { 19127105e828SDaniel Borkmann struct bpf_insn *insns_sanitized; 19137105e828SDaniel Borkmann bool fault; 19147105e828SDaniel Borkmann 19157105e828SDaniel Borkmann if (prog->blinded && !bpf_dump_raw_ok()) { 19167105e828SDaniel Borkmann info.xlated_prog_insns = 0; 19177105e828SDaniel Borkmann goto done; 19187105e828SDaniel Borkmann } 19197105e828SDaniel Borkmann insns_sanitized = bpf_insn_prepare_dump(prog); 19207105e828SDaniel Borkmann if (!insns_sanitized) 19217105e828SDaniel Borkmann return -ENOMEM; 19221e270976SMartin KaFai Lau uinsns = u64_to_user_ptr(info.xlated_prog_insns); 19231e270976SMartin KaFai Lau ulen = min_t(u32, info.xlated_prog_len, ulen); 19247105e828SDaniel Borkmann fault = copy_to_user(uinsns, insns_sanitized, ulen); 19257105e828SDaniel Borkmann kfree(insns_sanitized); 19267105e828SDaniel Borkmann if (fault) 19271e270976SMartin KaFai Lau return -EFAULT; 19281e270976SMartin KaFai Lau } 19291e270976SMartin KaFai Lau 1930675fc275SJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) { 1931675fc275SJakub Kicinski err = bpf_prog_offload_info_fill(&info, prog); 1932675fc275SJakub Kicinski if (err) 1933675fc275SJakub Kicinski return err; 1934fcfb126dSJiong Wang goto done; 1935fcfb126dSJiong Wang } 1936fcfb126dSJiong Wang 1937fcfb126dSJiong Wang /* NOTE: the following code is supposed to be skipped for offload. 1938fcfb126dSJiong Wang * bpf_prog_offload_info_fill() is the place to fill similar fields 1939fcfb126dSJiong Wang * for offload. 1940fcfb126dSJiong Wang */ 1941fcfb126dSJiong Wang ulen = info.jited_prog_len; 1942fcfb126dSJiong Wang info.jited_prog_len = prog->jited_len; 1943fcfb126dSJiong Wang if (info.jited_prog_len && ulen) { 1944fcfb126dSJiong Wang if (bpf_dump_raw_ok()) { 1945fcfb126dSJiong Wang uinsns = u64_to_user_ptr(info.jited_prog_insns); 1946fcfb126dSJiong Wang ulen = min_t(u32, info.jited_prog_len, ulen); 1947fcfb126dSJiong Wang if (copy_to_user(uinsns, prog->bpf_func, ulen)) 1948fcfb126dSJiong Wang return -EFAULT; 1949fcfb126dSJiong Wang } else { 1950fcfb126dSJiong Wang info.jited_prog_insns = 0; 1951fcfb126dSJiong Wang } 1952675fc275SJakub Kicinski } 1953675fc275SJakub Kicinski 19541e270976SMartin KaFai Lau done: 19551e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 19561e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 19571e270976SMartin KaFai Lau return -EFAULT; 19581e270976SMartin KaFai Lau 19591e270976SMartin KaFai Lau return 0; 19601e270976SMartin KaFai Lau } 19611e270976SMartin KaFai Lau 19621e270976SMartin KaFai Lau static int bpf_map_get_info_by_fd(struct bpf_map *map, 19631e270976SMartin KaFai Lau const union bpf_attr *attr, 19641e270976SMartin KaFai Lau union bpf_attr __user *uattr) 19651e270976SMartin KaFai Lau { 19661e270976SMartin KaFai Lau struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 19671e270976SMartin KaFai Lau struct bpf_map_info info = {}; 19681e270976SMartin KaFai Lau u32 info_len = attr->info.info_len; 19691e270976SMartin KaFai Lau int err; 19701e270976SMartin KaFai Lau 19711e270976SMartin KaFai Lau err = check_uarg_tail_zero(uinfo, sizeof(info), info_len); 19721e270976SMartin KaFai Lau if (err) 19731e270976SMartin KaFai Lau return err; 19741e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len); 19751e270976SMartin KaFai Lau 19761e270976SMartin KaFai Lau info.type = map->map_type; 19771e270976SMartin KaFai Lau info.id = map->id; 19781e270976SMartin KaFai Lau info.key_size = map->key_size; 19791e270976SMartin KaFai Lau info.value_size = map->value_size; 19801e270976SMartin KaFai Lau info.max_entries = map->max_entries; 19811e270976SMartin KaFai Lau info.map_flags = map->map_flags; 1982ad5b177bSMartin KaFai Lau memcpy(info.name, map->name, sizeof(map->name)); 19831e270976SMartin KaFai Lau 198452775b33SJakub Kicinski if (bpf_map_is_dev_bound(map)) { 198552775b33SJakub Kicinski err = bpf_map_offload_info_fill(&info, map); 198652775b33SJakub Kicinski if (err) 198752775b33SJakub Kicinski return err; 198852775b33SJakub Kicinski } 198952775b33SJakub Kicinski 19901e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) || 19911e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len)) 19921e270976SMartin KaFai Lau return -EFAULT; 19931e270976SMartin KaFai Lau 19941e270976SMartin KaFai Lau return 0; 19951e270976SMartin KaFai Lau } 19961e270976SMartin KaFai Lau 19971e270976SMartin KaFai Lau #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 19981e270976SMartin KaFai Lau 19991e270976SMartin KaFai Lau static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 20001e270976SMartin KaFai Lau union bpf_attr __user *uattr) 20011e270976SMartin KaFai Lau { 20021e270976SMartin KaFai Lau int ufd = attr->info.bpf_fd; 20031e270976SMartin KaFai Lau struct fd f; 20041e270976SMartin KaFai Lau int err; 20051e270976SMartin KaFai Lau 20061e270976SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 20071e270976SMartin KaFai Lau return -EINVAL; 20081e270976SMartin KaFai Lau 20091e270976SMartin KaFai Lau f = fdget(ufd); 20101e270976SMartin KaFai Lau if (!f.file) 20111e270976SMartin KaFai Lau return -EBADFD; 20121e270976SMartin KaFai Lau 20131e270976SMartin KaFai Lau if (f.file->f_op == &bpf_prog_fops) 20141e270976SMartin KaFai Lau err = bpf_prog_get_info_by_fd(f.file->private_data, attr, 20151e270976SMartin KaFai Lau uattr); 20161e270976SMartin KaFai Lau else if (f.file->f_op == &bpf_map_fops) 20171e270976SMartin KaFai Lau err = bpf_map_get_info_by_fd(f.file->private_data, attr, 20181e270976SMartin KaFai Lau uattr); 20191e270976SMartin KaFai Lau else 20201e270976SMartin KaFai Lau err = -EINVAL; 20211e270976SMartin KaFai Lau 20221e270976SMartin KaFai Lau fdput(f); 20231e270976SMartin KaFai Lau return err; 20241e270976SMartin KaFai Lau } 20251e270976SMartin KaFai Lau 202699c55f7dSAlexei Starovoitov SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 202799c55f7dSAlexei Starovoitov { 202899c55f7dSAlexei Starovoitov union bpf_attr attr = {}; 202999c55f7dSAlexei Starovoitov int err; 203099c55f7dSAlexei Starovoitov 20310fa4fe85SChenbo Feng if (sysctl_unprivileged_bpf_disabled && !capable(CAP_SYS_ADMIN)) 203299c55f7dSAlexei Starovoitov return -EPERM; 203399c55f7dSAlexei Starovoitov 20341e270976SMartin KaFai Lau err = check_uarg_tail_zero(uattr, sizeof(attr), size); 203599c55f7dSAlexei Starovoitov if (err) 203699c55f7dSAlexei Starovoitov return err; 20371e270976SMartin KaFai Lau size = min_t(u32, size, sizeof(attr)); 203899c55f7dSAlexei Starovoitov 203999c55f7dSAlexei Starovoitov /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 204099c55f7dSAlexei Starovoitov if (copy_from_user(&attr, uattr, size) != 0) 204199c55f7dSAlexei Starovoitov return -EFAULT; 204299c55f7dSAlexei Starovoitov 2043afdb09c7SChenbo Feng err = security_bpf(cmd, &attr, size); 2044afdb09c7SChenbo Feng if (err < 0) 2045afdb09c7SChenbo Feng return err; 2046afdb09c7SChenbo Feng 204799c55f7dSAlexei Starovoitov switch (cmd) { 204899c55f7dSAlexei Starovoitov case BPF_MAP_CREATE: 204999c55f7dSAlexei Starovoitov err = map_create(&attr); 205099c55f7dSAlexei Starovoitov break; 2051db20fd2bSAlexei Starovoitov case BPF_MAP_LOOKUP_ELEM: 2052db20fd2bSAlexei Starovoitov err = map_lookup_elem(&attr); 2053db20fd2bSAlexei Starovoitov break; 2054db20fd2bSAlexei Starovoitov case BPF_MAP_UPDATE_ELEM: 2055db20fd2bSAlexei Starovoitov err = map_update_elem(&attr); 2056db20fd2bSAlexei Starovoitov break; 2057db20fd2bSAlexei Starovoitov case BPF_MAP_DELETE_ELEM: 2058db20fd2bSAlexei Starovoitov err = map_delete_elem(&attr); 2059db20fd2bSAlexei Starovoitov break; 2060db20fd2bSAlexei Starovoitov case BPF_MAP_GET_NEXT_KEY: 2061db20fd2bSAlexei Starovoitov err = map_get_next_key(&attr); 2062db20fd2bSAlexei Starovoitov break; 206309756af4SAlexei Starovoitov case BPF_PROG_LOAD: 206409756af4SAlexei Starovoitov err = bpf_prog_load(&attr); 206509756af4SAlexei Starovoitov break; 2066b2197755SDaniel Borkmann case BPF_OBJ_PIN: 2067b2197755SDaniel Borkmann err = bpf_obj_pin(&attr); 2068b2197755SDaniel Borkmann break; 2069b2197755SDaniel Borkmann case BPF_OBJ_GET: 2070b2197755SDaniel Borkmann err = bpf_obj_get(&attr); 2071b2197755SDaniel Borkmann break; 2072f4324551SDaniel Mack #ifdef CONFIG_CGROUP_BPF 2073f4324551SDaniel Mack case BPF_PROG_ATTACH: 2074f4324551SDaniel Mack err = bpf_prog_attach(&attr); 2075f4324551SDaniel Mack break; 2076f4324551SDaniel Mack case BPF_PROG_DETACH: 2077f4324551SDaniel Mack err = bpf_prog_detach(&attr); 2078f4324551SDaniel Mack break; 2079468e2f64SAlexei Starovoitov case BPF_PROG_QUERY: 2080468e2f64SAlexei Starovoitov err = bpf_prog_query(&attr, uattr); 2081468e2f64SAlexei Starovoitov break; 2082f4324551SDaniel Mack #endif 20831cf1cae9SAlexei Starovoitov case BPF_PROG_TEST_RUN: 20841cf1cae9SAlexei Starovoitov err = bpf_prog_test_run(&attr, uattr); 20851cf1cae9SAlexei Starovoitov break; 208634ad5580SMartin KaFai Lau case BPF_PROG_GET_NEXT_ID: 208734ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 208834ad5580SMartin KaFai Lau &prog_idr, &prog_idr_lock); 208934ad5580SMartin KaFai Lau break; 209034ad5580SMartin KaFai Lau case BPF_MAP_GET_NEXT_ID: 209134ad5580SMartin KaFai Lau err = bpf_obj_get_next_id(&attr, uattr, 209234ad5580SMartin KaFai Lau &map_idr, &map_idr_lock); 209334ad5580SMartin KaFai Lau break; 2094b16d9aa4SMartin KaFai Lau case BPF_PROG_GET_FD_BY_ID: 2095b16d9aa4SMartin KaFai Lau err = bpf_prog_get_fd_by_id(&attr); 2096b16d9aa4SMartin KaFai Lau break; 2097bd5f5f4eSMartin KaFai Lau case BPF_MAP_GET_FD_BY_ID: 2098bd5f5f4eSMartin KaFai Lau err = bpf_map_get_fd_by_id(&attr); 2099bd5f5f4eSMartin KaFai Lau break; 21001e270976SMartin KaFai Lau case BPF_OBJ_GET_INFO_BY_FD: 21011e270976SMartin KaFai Lau err = bpf_obj_get_info_by_fd(&attr, uattr); 21021e270976SMartin KaFai Lau break; 2103c4f6699dSAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN: 2104c4f6699dSAlexei Starovoitov err = bpf_raw_tracepoint_open(&attr); 2105c4f6699dSAlexei Starovoitov break; 210699c55f7dSAlexei Starovoitov default: 210799c55f7dSAlexei Starovoitov err = -EINVAL; 210899c55f7dSAlexei Starovoitov break; 210999c55f7dSAlexei Starovoitov } 211099c55f7dSAlexei Starovoitov 211199c55f7dSAlexei Starovoitov return err; 211299c55f7dSAlexei Starovoitov } 2113