xref: /linux/kernel/bpf/bpf_inode_storage.c (revision d0d106a2bd21499901299160744e5fe9f4c83ddb)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2019 Facebook
4  * Copyright 2020 Google LLC.
5  */
6 
7 #include <linux/rculist.h>
8 #include <linux/list.h>
9 #include <linux/hash.h>
10 #include <linux/types.h>
11 #include <linux/spinlock.h>
12 #include <linux/bpf.h>
13 #include <linux/bpf_local_storage.h>
14 #include <net/sock.h>
15 #include <uapi/linux/sock_diag.h>
16 #include <uapi/linux/btf.h>
17 #include <linux/bpf_lsm.h>
18 #include <linux/btf_ids.h>
19 #include <linux/rcupdate_trace.h>
20 
21 DEFINE_BPF_STORAGE_CACHE(inode_cache);
22 
23 static struct bpf_local_storage __rcu **
inode_storage_ptr(void * owner)24 inode_storage_ptr(void *owner)
25 {
26 	struct inode *inode = owner;
27 	struct bpf_storage_blob *bsb;
28 
29 	bsb = bpf_inode(inode);
30 	if (!bsb)
31 		return NULL;
32 	return &bsb->storage;
33 }
34 
inode_storage_lookup(struct inode * inode,struct bpf_map * map,bool cacheit_lockit)35 static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
36 							   struct bpf_map *map,
37 							   bool cacheit_lockit)
38 {
39 	struct bpf_local_storage *inode_storage;
40 	struct bpf_local_storage_map *smap;
41 	struct bpf_storage_blob *bsb;
42 
43 	bsb = bpf_inode(inode);
44 	if (!bsb)
45 		return NULL;
46 
47 	inode_storage =
48 		rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
49 	if (!inode_storage)
50 		return NULL;
51 
52 	smap = (struct bpf_local_storage_map *)map;
53 	return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
54 }
55 
bpf_inode_storage_free(struct inode * inode)56 void bpf_inode_storage_free(struct inode *inode)
57 {
58 	struct bpf_local_storage *local_storage;
59 	struct bpf_storage_blob *bsb;
60 
61 	bsb = bpf_inode(inode);
62 	if (!bsb)
63 		return;
64 
65 	migrate_disable();
66 	rcu_read_lock();
67 
68 	local_storage = rcu_dereference(bsb->storage);
69 	if (!local_storage)
70 		goto out;
71 
72 	bpf_local_storage_destroy(local_storage);
73 out:
74 	rcu_read_unlock();
75 	migrate_enable();
76 }
77 
bpf_fd_inode_storage_lookup_elem(struct bpf_map * map,void * key)78 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
79 {
80 	struct bpf_local_storage_data *sdata;
81 	CLASS(fd_raw, f)(*(int *)key);
82 
83 	if (fd_empty(f))
84 		return ERR_PTR(-EBADF);
85 
86 	sdata = inode_storage_lookup(file_inode(fd_file(f)), map, true);
87 	return sdata ? sdata->data : NULL;
88 }
89 
bpf_fd_inode_storage_update_elem(struct bpf_map * map,void * key,void * value,u64 map_flags)90 static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
91 					     void *value, u64 map_flags)
92 {
93 	struct bpf_local_storage_data *sdata;
94 	CLASS(fd_raw, f)(*(int *)key);
95 
96 	if (fd_empty(f))
97 		return -EBADF;
98 	if (!inode_storage_ptr(file_inode(fd_file(f))))
99 		return -EBADF;
100 
101 	sdata = bpf_local_storage_update(file_inode(fd_file(f)),
102 					 (struct bpf_local_storage_map *)map,
103 					 value, map_flags, false, GFP_ATOMIC);
104 	return PTR_ERR_OR_ZERO(sdata);
105 }
106 
inode_storage_delete(struct inode * inode,struct bpf_map * map)107 static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
108 {
109 	struct bpf_local_storage_data *sdata;
110 
111 	sdata = inode_storage_lookup(inode, map, false);
112 	if (!sdata)
113 		return -ENOENT;
114 
115 	bpf_selem_unlink(SELEM(sdata), false);
116 
117 	return 0;
118 }
119 
bpf_fd_inode_storage_delete_elem(struct bpf_map * map,void * key)120 static long bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
121 {
122 	CLASS(fd_raw, f)(*(int *)key);
123 
124 	if (fd_empty(f))
125 		return -EBADF;
126 	return inode_storage_delete(file_inode(fd_file(f)), map);
127 }
128 
129 /* *gfp_flags* is a hidden argument provided by the verifier */
BPF_CALL_5(bpf_inode_storage_get,struct bpf_map *,map,struct inode *,inode,void *,value,u64,flags,gfp_t,gfp_flags)130 BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
131 	   void *, value, u64, flags, gfp_t, gfp_flags)
132 {
133 	struct bpf_local_storage_data *sdata;
134 
135 	WARN_ON_ONCE(!bpf_rcu_lock_held());
136 	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
137 		return (unsigned long)NULL;
138 
139 	/* explicitly check that the inode_storage_ptr is not
140 	 * NULL as inode_storage_lookup returns NULL in this case and
141 	 * bpf_local_storage_update expects the owner to have a
142 	 * valid storage pointer.
143 	 */
144 	if (!inode || !inode_storage_ptr(inode))
145 		return (unsigned long)NULL;
146 
147 	sdata = inode_storage_lookup(inode, map, true);
148 	if (sdata)
149 		return (unsigned long)sdata->data;
150 
151 	/* This helper must only called from where the inode is guaranteed
152 	 * to have a refcount and cannot be freed.
153 	 */
154 	if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
155 		sdata = bpf_local_storage_update(
156 			inode, (struct bpf_local_storage_map *)map, value,
157 			BPF_NOEXIST, false, gfp_flags);
158 		return IS_ERR(sdata) ? (unsigned long)NULL :
159 					     (unsigned long)sdata->data;
160 	}
161 
162 	return (unsigned long)NULL;
163 }
164 
BPF_CALL_2(bpf_inode_storage_delete,struct bpf_map *,map,struct inode *,inode)165 BPF_CALL_2(bpf_inode_storage_delete,
166 	   struct bpf_map *, map, struct inode *, inode)
167 {
168 	WARN_ON_ONCE(!bpf_rcu_lock_held());
169 	if (!inode)
170 		return -EINVAL;
171 
172 	/* This helper must only called from where the inode is guaranteed
173 	 * to have a refcount and cannot be freed.
174 	 */
175 	return inode_storage_delete(inode, map);
176 }
177 
notsupp_get_next_key(struct bpf_map * map,void * key,void * next_key)178 static int notsupp_get_next_key(struct bpf_map *map, void *key,
179 				void *next_key)
180 {
181 	return -ENOTSUPP;
182 }
183 
inode_storage_map_alloc(union bpf_attr * attr)184 static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
185 {
186 	return bpf_local_storage_map_alloc(attr, &inode_cache, false);
187 }
188 
inode_storage_map_free(struct bpf_map * map)189 static void inode_storage_map_free(struct bpf_map *map)
190 {
191 	bpf_local_storage_map_free(map, &inode_cache, NULL);
192 }
193 
194 const struct bpf_map_ops inode_storage_map_ops = {
195 	.map_meta_equal = bpf_map_meta_equal,
196 	.map_alloc_check = bpf_local_storage_map_alloc_check,
197 	.map_alloc = inode_storage_map_alloc,
198 	.map_free = inode_storage_map_free,
199 	.map_get_next_key = notsupp_get_next_key,
200 	.map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
201 	.map_update_elem = bpf_fd_inode_storage_update_elem,
202 	.map_delete_elem = bpf_fd_inode_storage_delete_elem,
203 	.map_check_btf = bpf_local_storage_map_check_btf,
204 	.map_mem_usage = bpf_local_storage_map_mem_usage,
205 	.map_btf_id = &bpf_local_storage_map_btf_id[0],
206 	.map_owner_storage_ptr = inode_storage_ptr,
207 };
208 
209 BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)
210 
211 const struct bpf_func_proto bpf_inode_storage_get_proto = {
212 	.func		= bpf_inode_storage_get,
213 	.gpl_only	= false,
214 	.ret_type	= RET_PTR_TO_MAP_VALUE_OR_NULL,
215 	.arg1_type	= ARG_CONST_MAP_PTR,
216 	.arg2_type	= ARG_PTR_TO_BTF_ID_OR_NULL,
217 	.arg2_btf_id	= &bpf_inode_storage_btf_ids[0],
218 	.arg3_type	= ARG_PTR_TO_MAP_VALUE_OR_NULL,
219 	.arg4_type	= ARG_ANYTHING,
220 };
221 
222 const struct bpf_func_proto bpf_inode_storage_delete_proto = {
223 	.func		= bpf_inode_storage_delete,
224 	.gpl_only	= false,
225 	.ret_type	= RET_INTEGER,
226 	.arg1_type	= ARG_CONST_MAP_PTR,
227 	.arg2_type	= ARG_PTR_TO_BTF_ID_OR_NULL,
228 	.arg2_btf_id	= &bpf_inode_storage_btf_ids[0],
229 };
230