1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2019 Facebook 4 * Copyright 2020 Google LLC. 5 */ 6 7 #include <linux/rculist.h> 8 #include <linux/list.h> 9 #include <linux/hash.h> 10 #include <linux/types.h> 11 #include <linux/spinlock.h> 12 #include <linux/bpf.h> 13 #include <linux/bpf_local_storage.h> 14 #include <net/sock.h> 15 #include <uapi/linux/sock_diag.h> 16 #include <uapi/linux/btf.h> 17 #include <linux/bpf_lsm.h> 18 #include <linux/btf_ids.h> 19 #include <linux/rcupdate_trace.h> 20 21 DEFINE_BPF_STORAGE_CACHE(inode_cache); 22 23 static struct bpf_local_storage __rcu ** 24 inode_storage_ptr(void *owner) 25 { 26 struct inode *inode = owner; 27 struct bpf_storage_blob *bsb; 28 29 bsb = bpf_inode(inode); 30 if (!bsb) 31 return NULL; 32 return &bsb->storage; 33 } 34 35 static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode, 36 struct bpf_map *map, 37 bool cacheit_lockit) 38 { 39 struct bpf_local_storage *inode_storage; 40 struct bpf_local_storage_map *smap; 41 struct bpf_storage_blob *bsb; 42 43 bsb = bpf_inode(inode); 44 if (!bsb) 45 return NULL; 46 47 inode_storage = 48 rcu_dereference_check(bsb->storage, bpf_rcu_lock_held()); 49 if (!inode_storage) 50 return NULL; 51 52 smap = (struct bpf_local_storage_map *)map; 53 return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit); 54 } 55 56 void bpf_inode_storage_free(struct inode *inode) 57 { 58 struct bpf_local_storage *local_storage; 59 struct bpf_storage_blob *bsb; 60 61 bsb = bpf_inode(inode); 62 if (!bsb) 63 return; 64 65 rcu_read_lock(); 66 67 local_storage = rcu_dereference(bsb->storage); 68 if (!local_storage) { 69 rcu_read_unlock(); 70 return; 71 } 72 73 bpf_local_storage_destroy(local_storage); 74 rcu_read_unlock(); 75 } 76 77 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key) 78 { 79 struct bpf_local_storage_data *sdata; 80 CLASS(fd_raw, f)(*(int *)key); 81 82 if (fd_empty(f)) 83 return ERR_PTR(-EBADF); 84 85 sdata = inode_storage_lookup(file_inode(fd_file(f)), map, true); 86 return sdata ? sdata->data : NULL; 87 } 88 89 static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key, 90 void *value, u64 map_flags) 91 { 92 struct bpf_local_storage_data *sdata; 93 CLASS(fd_raw, f)(*(int *)key); 94 95 if (fd_empty(f)) 96 return -EBADF; 97 if (!inode_storage_ptr(file_inode(fd_file(f)))) 98 return -EBADF; 99 100 sdata = bpf_local_storage_update(file_inode(fd_file(f)), 101 (struct bpf_local_storage_map *)map, 102 value, map_flags, false, GFP_ATOMIC); 103 return PTR_ERR_OR_ZERO(sdata); 104 } 105 106 static int inode_storage_delete(struct inode *inode, struct bpf_map *map) 107 { 108 struct bpf_local_storage_data *sdata; 109 110 sdata = inode_storage_lookup(inode, map, false); 111 if (!sdata) 112 return -ENOENT; 113 114 bpf_selem_unlink(SELEM(sdata), false); 115 116 return 0; 117 } 118 119 static long bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key) 120 { 121 CLASS(fd_raw, f)(*(int *)key); 122 123 if (fd_empty(f)) 124 return -EBADF; 125 return inode_storage_delete(file_inode(fd_file(f)), map); 126 } 127 128 /* *gfp_flags* is a hidden argument provided by the verifier */ 129 BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode, 130 void *, value, u64, flags, gfp_t, gfp_flags) 131 { 132 struct bpf_local_storage_data *sdata; 133 134 WARN_ON_ONCE(!bpf_rcu_lock_held()); 135 if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE)) 136 return (unsigned long)NULL; 137 138 /* explicitly check that the inode_storage_ptr is not 139 * NULL as inode_storage_lookup returns NULL in this case and 140 * bpf_local_storage_update expects the owner to have a 141 * valid storage pointer. 142 */ 143 if (!inode || !inode_storage_ptr(inode)) 144 return (unsigned long)NULL; 145 146 sdata = inode_storage_lookup(inode, map, true); 147 if (sdata) 148 return (unsigned long)sdata->data; 149 150 /* This helper must only called from where the inode is guaranteed 151 * to have a refcount and cannot be freed. 152 */ 153 if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) { 154 sdata = bpf_local_storage_update( 155 inode, (struct bpf_local_storage_map *)map, value, 156 BPF_NOEXIST, false, gfp_flags); 157 return IS_ERR(sdata) ? (unsigned long)NULL : 158 (unsigned long)sdata->data; 159 } 160 161 return (unsigned long)NULL; 162 } 163 164 BPF_CALL_2(bpf_inode_storage_delete, 165 struct bpf_map *, map, struct inode *, inode) 166 { 167 WARN_ON_ONCE(!bpf_rcu_lock_held()); 168 if (!inode) 169 return -EINVAL; 170 171 /* This helper must only called from where the inode is guaranteed 172 * to have a refcount and cannot be freed. 173 */ 174 return inode_storage_delete(inode, map); 175 } 176 177 static int notsupp_get_next_key(struct bpf_map *map, void *key, 178 void *next_key) 179 { 180 return -ENOTSUPP; 181 } 182 183 static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr) 184 { 185 return bpf_local_storage_map_alloc(attr, &inode_cache, false); 186 } 187 188 static void inode_storage_map_free(struct bpf_map *map) 189 { 190 bpf_local_storage_map_free(map, &inode_cache, NULL); 191 } 192 193 const struct bpf_map_ops inode_storage_map_ops = { 194 .map_meta_equal = bpf_map_meta_equal, 195 .map_alloc_check = bpf_local_storage_map_alloc_check, 196 .map_alloc = inode_storage_map_alloc, 197 .map_free = inode_storage_map_free, 198 .map_get_next_key = notsupp_get_next_key, 199 .map_lookup_elem = bpf_fd_inode_storage_lookup_elem, 200 .map_update_elem = bpf_fd_inode_storage_update_elem, 201 .map_delete_elem = bpf_fd_inode_storage_delete_elem, 202 .map_check_btf = bpf_local_storage_map_check_btf, 203 .map_mem_usage = bpf_local_storage_map_mem_usage, 204 .map_btf_id = &bpf_local_storage_map_btf_id[0], 205 .map_owner_storage_ptr = inode_storage_ptr, 206 }; 207 208 BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode) 209 210 const struct bpf_func_proto bpf_inode_storage_get_proto = { 211 .func = bpf_inode_storage_get, 212 .gpl_only = false, 213 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 214 .arg1_type = ARG_CONST_MAP_PTR, 215 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, 216 .arg2_btf_id = &bpf_inode_storage_btf_ids[0], 217 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 218 .arg4_type = ARG_ANYTHING, 219 }; 220 221 const struct bpf_func_proto bpf_inode_storage_delete_proto = { 222 .func = bpf_inode_storage_delete, 223 .gpl_only = false, 224 .ret_type = RET_INTEGER, 225 .arg1_type = ARG_CONST_MAP_PTR, 226 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, 227 .arg2_btf_id = &bpf_inode_storage_btf_ids[0], 228 }; 229