1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2019 Facebook 4 * Copyright 2020 Google LLC. 5 */ 6 7 #include <linux/rculist.h> 8 #include <linux/list.h> 9 #include <linux/hash.h> 10 #include <linux/types.h> 11 #include <linux/spinlock.h> 12 #include <linux/bpf.h> 13 #include <linux/bpf_local_storage.h> 14 #include <net/sock.h> 15 #include <uapi/linux/sock_diag.h> 16 #include <uapi/linux/btf.h> 17 #include <linux/bpf_lsm.h> 18 #include <linux/btf_ids.h> 19 #include <linux/rcupdate_trace.h> 20 21 DEFINE_BPF_STORAGE_CACHE(inode_cache); 22 23 static struct bpf_local_storage __rcu ** 24 inode_storage_ptr(void *owner) 25 { 26 struct inode *inode = owner; 27 struct bpf_storage_blob *bsb; 28 29 bsb = bpf_inode(inode); 30 if (!bsb) 31 return NULL; 32 return &bsb->storage; 33 } 34 35 static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode, 36 struct bpf_map *map, 37 bool cacheit_lockit) 38 { 39 struct bpf_local_storage *inode_storage; 40 struct bpf_local_storage_map *smap; 41 struct bpf_storage_blob *bsb; 42 43 bsb = bpf_inode(inode); 44 if (!bsb) 45 return NULL; 46 47 inode_storage = 48 rcu_dereference_check(bsb->storage, bpf_rcu_lock_held()); 49 if (!inode_storage) 50 return NULL; 51 52 smap = (struct bpf_local_storage_map *)map; 53 return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit); 54 } 55 56 void bpf_inode_storage_free(struct inode *inode) 57 { 58 struct bpf_local_storage *local_storage; 59 struct bpf_storage_blob *bsb; 60 61 bsb = bpf_inode(inode); 62 if (!bsb) 63 return; 64 65 rcu_read_lock_dont_migrate(); 66 67 local_storage = rcu_dereference(bsb->storage); 68 if (!local_storage) 69 goto out; 70 71 bpf_local_storage_destroy(local_storage); 72 out: 73 rcu_read_unlock_migrate(); 74 } 75 76 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key) 77 { 78 struct bpf_local_storage_data *sdata; 79 CLASS(fd_raw, f)(*(int *)key); 80 81 if (fd_empty(f)) 82 return ERR_PTR(-EBADF); 83 84 sdata = inode_storage_lookup(file_inode(fd_file(f)), map, true); 85 return sdata ? sdata->data : NULL; 86 } 87 88 static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key, 89 void *value, u64 map_flags) 90 { 91 struct bpf_local_storage_data *sdata; 92 CLASS(fd_raw, f)(*(int *)key); 93 94 if (fd_empty(f)) 95 return -EBADF; 96 if (!inode_storage_ptr(file_inode(fd_file(f)))) 97 return -EBADF; 98 99 sdata = bpf_local_storage_update(file_inode(fd_file(f)), 100 (struct bpf_local_storage_map *)map, 101 value, map_flags, false, GFP_ATOMIC); 102 return PTR_ERR_OR_ZERO(sdata); 103 } 104 105 static int inode_storage_delete(struct inode *inode, struct bpf_map *map) 106 { 107 struct bpf_local_storage_data *sdata; 108 109 sdata = inode_storage_lookup(inode, map, false); 110 if (!sdata) 111 return -ENOENT; 112 113 bpf_selem_unlink(SELEM(sdata), false); 114 115 return 0; 116 } 117 118 static long bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key) 119 { 120 CLASS(fd_raw, f)(*(int *)key); 121 122 if (fd_empty(f)) 123 return -EBADF; 124 return inode_storage_delete(file_inode(fd_file(f)), map); 125 } 126 127 /* *gfp_flags* is a hidden argument provided by the verifier */ 128 BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode, 129 void *, value, u64, flags, gfp_t, gfp_flags) 130 { 131 struct bpf_local_storage_data *sdata; 132 133 WARN_ON_ONCE(!bpf_rcu_lock_held()); 134 if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE)) 135 return (unsigned long)NULL; 136 137 /* explicitly check that the inode_storage_ptr is not 138 * NULL as inode_storage_lookup returns NULL in this case and 139 * bpf_local_storage_update expects the owner to have a 140 * valid storage pointer. 141 */ 142 if (!inode || !inode_storage_ptr(inode)) 143 return (unsigned long)NULL; 144 145 sdata = inode_storage_lookup(inode, map, true); 146 if (sdata) 147 return (unsigned long)sdata->data; 148 149 /* This helper must only called from where the inode is guaranteed 150 * to have a refcount and cannot be freed. 151 */ 152 if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) { 153 sdata = bpf_local_storage_update( 154 inode, (struct bpf_local_storage_map *)map, value, 155 BPF_NOEXIST, false, gfp_flags); 156 return IS_ERR(sdata) ? (unsigned long)NULL : 157 (unsigned long)sdata->data; 158 } 159 160 return (unsigned long)NULL; 161 } 162 163 BPF_CALL_2(bpf_inode_storage_delete, 164 struct bpf_map *, map, struct inode *, inode) 165 { 166 WARN_ON_ONCE(!bpf_rcu_lock_held()); 167 if (!inode) 168 return -EINVAL; 169 170 /* This helper must only called from where the inode is guaranteed 171 * to have a refcount and cannot be freed. 172 */ 173 return inode_storage_delete(inode, map); 174 } 175 176 static int notsupp_get_next_key(struct bpf_map *map, void *key, 177 void *next_key) 178 { 179 return -ENOTSUPP; 180 } 181 182 static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr) 183 { 184 return bpf_local_storage_map_alloc(attr, &inode_cache, false); 185 } 186 187 static void inode_storage_map_free(struct bpf_map *map) 188 { 189 bpf_local_storage_map_free(map, &inode_cache, NULL); 190 } 191 192 const struct bpf_map_ops inode_storage_map_ops = { 193 .map_meta_equal = bpf_map_meta_equal, 194 .map_alloc_check = bpf_local_storage_map_alloc_check, 195 .map_alloc = inode_storage_map_alloc, 196 .map_free = inode_storage_map_free, 197 .map_get_next_key = notsupp_get_next_key, 198 .map_lookup_elem = bpf_fd_inode_storage_lookup_elem, 199 .map_update_elem = bpf_fd_inode_storage_update_elem, 200 .map_delete_elem = bpf_fd_inode_storage_delete_elem, 201 .map_check_btf = bpf_local_storage_map_check_btf, 202 .map_mem_usage = bpf_local_storage_map_mem_usage, 203 .map_btf_id = &bpf_local_storage_map_btf_id[0], 204 .map_owner_storage_ptr = inode_storage_ptr, 205 }; 206 207 BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode) 208 209 const struct bpf_func_proto bpf_inode_storage_get_proto = { 210 .func = bpf_inode_storage_get, 211 .gpl_only = false, 212 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 213 .arg1_type = ARG_CONST_MAP_PTR, 214 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, 215 .arg2_btf_id = &bpf_inode_storage_btf_ids[0], 216 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 217 .arg4_type = ARG_ANYTHING, 218 }; 219 220 const struct bpf_func_proto bpf_inode_storage_delete_proto = { 221 .func = bpf_inode_storage_delete, 222 .gpl_only = false, 223 .ret_type = RET_INTEGER, 224 .arg1_type = ARG_CONST_MAP_PTR, 225 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, 226 .arg2_btf_id = &bpf_inode_storage_btf_ids[0], 227 }; 228