1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2019 Facebook 4 * Copyright 2020 Google LLC. 5 */ 6 7 #include <linux/rculist.h> 8 #include <linux/list.h> 9 #include <linux/hash.h> 10 #include <linux/types.h> 11 #include <linux/spinlock.h> 12 #include <linux/bpf.h> 13 #include <linux/bpf_local_storage.h> 14 #include <net/sock.h> 15 #include <uapi/linux/sock_diag.h> 16 #include <uapi/linux/btf.h> 17 #include <linux/bpf_lsm.h> 18 #include <linux/btf_ids.h> 19 #include <linux/fdtable.h> 20 #include <linux/rcupdate_trace.h> 21 22 DEFINE_BPF_STORAGE_CACHE(inode_cache); 23 24 static struct bpf_local_storage __rcu ** 25 inode_storage_ptr(void *owner) 26 { 27 struct inode *inode = owner; 28 struct bpf_storage_blob *bsb; 29 30 bsb = bpf_inode(inode); 31 if (!bsb) 32 return NULL; 33 return &bsb->storage; 34 } 35 36 static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode, 37 struct bpf_map *map, 38 bool cacheit_lockit) 39 { 40 struct bpf_local_storage *inode_storage; 41 struct bpf_local_storage_map *smap; 42 struct bpf_storage_blob *bsb; 43 44 bsb = bpf_inode(inode); 45 if (!bsb) 46 return NULL; 47 48 inode_storage = 49 rcu_dereference_check(bsb->storage, bpf_rcu_lock_held()); 50 if (!inode_storage) 51 return NULL; 52 53 smap = (struct bpf_local_storage_map *)map; 54 return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit); 55 } 56 57 void bpf_inode_storage_free(struct inode *inode) 58 { 59 struct bpf_local_storage *local_storage; 60 bool free_inode_storage = false; 61 struct bpf_storage_blob *bsb; 62 63 bsb = bpf_inode(inode); 64 if (!bsb) 65 return; 66 67 rcu_read_lock(); 68 69 local_storage = rcu_dereference(bsb->storage); 70 if (!local_storage) { 71 rcu_read_unlock(); 72 return; 73 } 74 75 raw_spin_lock_bh(&local_storage->lock); 76 free_inode_storage = bpf_local_storage_unlink_nolock(local_storage); 77 raw_spin_unlock_bh(&local_storage->lock); 78 rcu_read_unlock(); 79 80 if (free_inode_storage) 81 kfree_rcu(local_storage, rcu); 82 } 83 84 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key) 85 { 86 struct bpf_local_storage_data *sdata; 87 struct file *f; 88 int fd; 89 90 fd = *(int *)key; 91 f = fget_raw(fd); 92 if (!f) 93 return ERR_PTR(-EBADF); 94 95 sdata = inode_storage_lookup(f->f_inode, map, true); 96 fput(f); 97 return sdata ? sdata->data : NULL; 98 } 99 100 static int bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key, 101 void *value, u64 map_flags) 102 { 103 struct bpf_local_storage_data *sdata; 104 struct file *f; 105 int fd; 106 107 fd = *(int *)key; 108 f = fget_raw(fd); 109 if (!f) 110 return -EBADF; 111 if (!inode_storage_ptr(f->f_inode)) { 112 fput(f); 113 return -EBADF; 114 } 115 116 sdata = bpf_local_storage_update(f->f_inode, 117 (struct bpf_local_storage_map *)map, 118 value, map_flags, GFP_ATOMIC); 119 fput(f); 120 return PTR_ERR_OR_ZERO(sdata); 121 } 122 123 static int inode_storage_delete(struct inode *inode, struct bpf_map *map) 124 { 125 struct bpf_local_storage_data *sdata; 126 127 sdata = inode_storage_lookup(inode, map, false); 128 if (!sdata) 129 return -ENOENT; 130 131 bpf_selem_unlink(SELEM(sdata), true); 132 133 return 0; 134 } 135 136 static int bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key) 137 { 138 struct file *f; 139 int fd, err; 140 141 fd = *(int *)key; 142 f = fget_raw(fd); 143 if (!f) 144 return -EBADF; 145 146 err = inode_storage_delete(f->f_inode, map); 147 fput(f); 148 return err; 149 } 150 151 /* *gfp_flags* is a hidden argument provided by the verifier */ 152 BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode, 153 void *, value, u64, flags, gfp_t, gfp_flags) 154 { 155 struct bpf_local_storage_data *sdata; 156 157 WARN_ON_ONCE(!bpf_rcu_lock_held()); 158 if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE)) 159 return (unsigned long)NULL; 160 161 /* explicitly check that the inode_storage_ptr is not 162 * NULL as inode_storage_lookup returns NULL in this case and 163 * bpf_local_storage_update expects the owner to have a 164 * valid storage pointer. 165 */ 166 if (!inode || !inode_storage_ptr(inode)) 167 return (unsigned long)NULL; 168 169 sdata = inode_storage_lookup(inode, map, true); 170 if (sdata) 171 return (unsigned long)sdata->data; 172 173 /* This helper must only called from where the inode is guaranteed 174 * to have a refcount and cannot be freed. 175 */ 176 if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) { 177 sdata = bpf_local_storage_update( 178 inode, (struct bpf_local_storage_map *)map, value, 179 BPF_NOEXIST, gfp_flags); 180 return IS_ERR(sdata) ? (unsigned long)NULL : 181 (unsigned long)sdata->data; 182 } 183 184 return (unsigned long)NULL; 185 } 186 187 BPF_CALL_2(bpf_inode_storage_delete, 188 struct bpf_map *, map, struct inode *, inode) 189 { 190 WARN_ON_ONCE(!bpf_rcu_lock_held()); 191 if (!inode) 192 return -EINVAL; 193 194 /* This helper must only called from where the inode is guaranteed 195 * to have a refcount and cannot be freed. 196 */ 197 return inode_storage_delete(inode, map); 198 } 199 200 static int notsupp_get_next_key(struct bpf_map *map, void *key, 201 void *next_key) 202 { 203 return -ENOTSUPP; 204 } 205 206 static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr) 207 { 208 return bpf_local_storage_map_alloc(attr, &inode_cache); 209 } 210 211 static void inode_storage_map_free(struct bpf_map *map) 212 { 213 bpf_local_storage_map_free(map, &inode_cache, NULL); 214 } 215 216 const struct bpf_map_ops inode_storage_map_ops = { 217 .map_meta_equal = bpf_map_meta_equal, 218 .map_alloc_check = bpf_local_storage_map_alloc_check, 219 .map_alloc = inode_storage_map_alloc, 220 .map_free = inode_storage_map_free, 221 .map_get_next_key = notsupp_get_next_key, 222 .map_lookup_elem = bpf_fd_inode_storage_lookup_elem, 223 .map_update_elem = bpf_fd_inode_storage_update_elem, 224 .map_delete_elem = bpf_fd_inode_storage_delete_elem, 225 .map_check_btf = bpf_local_storage_map_check_btf, 226 .map_btf_id = &bpf_local_storage_map_btf_id[0], 227 .map_owner_storage_ptr = inode_storage_ptr, 228 }; 229 230 BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode) 231 232 const struct bpf_func_proto bpf_inode_storage_get_proto = { 233 .func = bpf_inode_storage_get, 234 .gpl_only = false, 235 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 236 .arg1_type = ARG_CONST_MAP_PTR, 237 .arg2_type = ARG_PTR_TO_BTF_ID, 238 .arg2_btf_id = &bpf_inode_storage_btf_ids[0], 239 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 240 .arg4_type = ARG_ANYTHING, 241 }; 242 243 const struct bpf_func_proto bpf_inode_storage_delete_proto = { 244 .func = bpf_inode_storage_delete, 245 .gpl_only = false, 246 .ret_type = RET_INTEGER, 247 .arg1_type = ARG_CONST_MAP_PTR, 248 .arg2_type = ARG_PTR_TO_BTF_ID, 249 .arg2_btf_id = &bpf_inode_storage_btf_ids[0], 250 }; 251