1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
2 #ifndef __TASK_LOCAL_DATA_BPF_H
3 #define __TASK_LOCAL_DATA_BPF_H
4
5 /*
6 * Task local data is a library that facilitates sharing per-task data
7 * between user space and bpf programs.
8 *
9 *
10 * USAGE
11 *
12 * A TLD, an entry of data in task local data, first needs to be created by the
13 * user space. This is done by calling user space API, TLD_DEFINE_KEY() or
14 * tld_create_key(), with the name of the TLD and the size.
15 *
16 * TLD_DEFINE_KEY(prio, "priority", sizeof(int));
17 *
18 * or
19 *
20 * void func_call(...) {
21 * tld_key_t prio, in_cs;
22 *
23 * prio = tld_create_key("priority", sizeof(int));
24 * in_cs = tld_create_key("in_critical_section", sizeof(bool));
25 * ...
26 *
27 * A key associated with the TLD, which has an opaque type tld_key_t, will be
28 * initialized or returned. It can be used to get a pointer to the TLD in the
29 * user space by calling tld_get_data().
30 *
31 * In a bpf program, tld_object_init() first needs to be called to initialized a
32 * tld_object on the stack. Then, TLDs can be accessed by calling tld_get_data().
33 * The API will try to fetch the key by the name and use it to locate the data.
34 * A pointer to the TLD will be returned. It also caches the key in a task local
35 * storage map, tld_key_map, whose value type, struct tld_keys, must be defined
36 * by the developer.
37 *
38 * struct tld_keys {
39 * tld_key_t prio;
40 * tld_key_t in_cs;
41 * };
42 *
43 * SEC("struct_ops")
44 * void prog(struct task_struct task, ...)
45 * {
46 * struct tld_object tld_obj;
47 * int err, *p;
48 *
49 * err = tld_object_init(task, &tld_obj);
50 * if (err)
51 * return;
52 *
53 * p = tld_get_data(&tld_obj, prio, "priority", sizeof(int));
54 * if (p)
55 * // do something depending on *p
56 */
57 #include <errno.h>
58 #include <bpf/bpf_helpers.h>
59
60 #define TLD_ROUND_MASK(x, y) ((__typeof__(x))((y) - 1))
61 #define TLD_ROUND_UP(x, y) ((((x) - 1) | TLD_ROUND_MASK(x, y)) + 1)
62
63 #define TLD_MAX_DATA_CNT (__PAGE_SIZE / sizeof(struct tld_metadata) - 1)
64 #define TLD_DATA_SIZE (__PAGE_SIZE - sizeof(__u64))
65
66 #ifndef TLD_NAME_LEN
67 #define TLD_NAME_LEN 62
68 #endif
69
70 #ifndef TLD_KEY_MAP_CREATE_RETRY
71 #define TLD_KEY_MAP_CREATE_RETRY 10
72 #endif
73
74 typedef struct {
75 __s16 off;
76 } tld_key_t;
77
78 struct tld_metadata {
79 char name[TLD_NAME_LEN];
80 __u16 size;
81 };
82
83 struct tld_meta_u {
84 __u16 cnt;
85 __u16 size;
86 struct tld_metadata metadata[TLD_MAX_DATA_CNT];
87 };
88
89 struct tld_data_u {
90 __u64 unused;
91 char data[__PAGE_SIZE - sizeof(__u64)] __attribute__((aligned(8)));
92 };
93
94 struct tld_map_value {
95 struct tld_data_u __uptr *data;
96 struct tld_meta_u __uptr *meta;
97 __u16 start; /* offset of tld_data_u->data in a page */
98 };
99
100 typedef struct tld_uptr_dummy {
101 struct tld_data_u data[0];
102 struct tld_meta_u meta[0];
103 } *tld_uptr_dummy_t;
104
105 struct tld_object {
106 struct tld_map_value *data_map;
107 struct tld_keys *key_map;
108 /*
109 * Force the compiler to generate the actual definition of tld_meta_u
110 * and tld_data_u in BTF. Without it, tld_meta_u and u_tld_data will
111 * be BTF_KIND_FWD.
112 */
113 tld_uptr_dummy_t dummy[0];
114 };
115
116 /*
117 * Map value of tld_key_map for caching keys. Must be defined by the developer.
118 * Members should be tld_key_t and passed to the 3rd argument of tld_fetch_key().
119 */
120 struct tld_keys;
121
122 struct {
123 __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
124 __uint(map_flags, BPF_F_NO_PREALLOC);
125 __type(key, int);
126 __type(value, struct tld_map_value);
127 } tld_data_map SEC(".maps");
128
129 struct {
130 __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
131 __uint(map_flags, BPF_F_NO_PREALLOC);
132 __type(key, int);
133 __type(value, struct tld_keys);
134 } tld_key_map SEC(".maps");
135
136 /**
137 * tld_object_init() - Initialize a tld_object.
138 *
139 * @task: The task_struct of the target task
140 * @tld_obj: A pointer to a tld_object to be initialized
141 *
142 * Return 0 on success; -ENODATA if the user space did not initialize task local data
143 * for the current task through tld_get_data(); -ENOMEM if the creation of tld_key_map
144 * fails
145 */
146 __attribute__((unused))
tld_object_init(struct task_struct * task,struct tld_object * tld_obj)147 static int tld_object_init(struct task_struct *task, struct tld_object *tld_obj)
148 {
149 int i;
150
151 tld_obj->data_map = bpf_task_storage_get(&tld_data_map, task, 0, 0);
152 if (!tld_obj->data_map)
153 return -ENODATA;
154
155 bpf_for(i, 0, TLD_KEY_MAP_CREATE_RETRY) {
156 tld_obj->key_map = bpf_task_storage_get(&tld_key_map, task, 0,
157 BPF_LOCAL_STORAGE_GET_F_CREATE);
158 if (likely(tld_obj->key_map))
159 break;
160 }
161 if (!tld_obj->key_map)
162 return -ENOMEM;
163
164 return 0;
165 }
166
167 /*
168 * Return the offset of TLD if @name is found. Otherwise, return the current TLD count
169 * using the nonpositive range so that the next tld_get_data() can skip fetching key if
170 * no new TLD is added or start comparing name from the first newly added TLD.
171 */
172 __attribute__((unused))
__tld_fetch_key(struct tld_object * tld_obj,const char * name,int i_start)173 static int __tld_fetch_key(struct tld_object *tld_obj, const char *name, int i_start)
174 {
175 struct tld_metadata *metadata;
176 int i, cnt, start, off = 0;
177
178 if (!tld_obj->data_map || !tld_obj->data_map->data || !tld_obj->data_map->meta)
179 return 0;
180
181 start = tld_obj->data_map->start;
182 cnt = tld_obj->data_map->meta->cnt;
183 metadata = tld_obj->data_map->meta->metadata;
184
185 bpf_for(i, 0, cnt) {
186 if (i >= TLD_MAX_DATA_CNT)
187 break;
188
189 if (i >= i_start && !bpf_strncmp(metadata[i].name, TLD_NAME_LEN, name))
190 return start + off;
191
192 off += TLD_ROUND_UP(metadata[i].size, 8);
193 if (off > TLD_DATA_SIZE)
194 break;
195 }
196
197 return -cnt;
198 }
199
200 /**
201 * tld_get_data() - Retrieve a pointer to the TLD associated with the name.
202 *
203 * @tld_obj: A pointer to a valid tld_object initialized by tld_object_init()
204 * @key: The cached key of the TLD in tld_key_map
205 * @name: The name of the key associated with a TLD
206 * @size: The size of the TLD. Must be a known constant value
207 *
208 * Return a pointer to the TLD associated with @name; NULL if not found or @size is too
209 * big. @key is used to cache the key if the TLD is found to speed up subsequent calls.
210 * It should be defined as an member of tld_keys of tld_key_t type by the developer.
211 */
212 #define tld_get_data(tld_obj, key, name, size) \
213 ({ \
214 void *data = NULL, *_data = (tld_obj)->data_map->data; \
215 long off = (tld_obj)->key_map->key.off; \
216 int cnt; \
217 \
218 if (likely(_data)) { \
219 if (likely(off > 0)) { \
220 barrier_var(off); \
221 if (likely(off < __PAGE_SIZE - size)) \
222 data = _data + off; \
223 } else { \
224 cnt = -(off); \
225 if (likely((tld_obj)->data_map->meta) && \
226 cnt < (tld_obj)->data_map->meta->cnt) { \
227 off = __tld_fetch_key(tld_obj, name, cnt); \
228 (tld_obj)->key_map->key.off = off; \
229 \
230 if (likely(off < __PAGE_SIZE - size)) { \
231 barrier_var(off); \
232 if (off > 0) \
233 data = _data + off; \
234 } \
235 } \
236 } \
237 } \
238 data; \
239 })
240
241 #endif
242