1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
2 /* Copyright (c) 2021 Facebook */
3 #ifndef __SKEL_INTERNAL_H
4 #define __SKEL_INTERNAL_H
5
6 #ifdef __KERNEL__
7 #include <linux/fdtable.h>
8 #include <linux/mm.h>
9 #include <linux/mman.h>
10 #include <linux/slab.h>
11 #include <linux/bpf.h>
12 #else
13 #include <unistd.h>
14 #include <sys/syscall.h>
15 #include <sys/mman.h>
16 #include <linux/keyctl.h>
17 #include <stdlib.h>
18 #include "bpf.h"
19 #endif
20
21 #ifndef __NR_bpf
22 # if defined(__mips__) && defined(_ABIO32)
23 # define __NR_bpf 4355
24 # elif defined(__mips__) && defined(_ABIN32)
25 # define __NR_bpf 6319
26 # elif defined(__mips__) && defined(_ABI64)
27 # define __NR_bpf 5315
28 # endif
29 #endif
30
31 /* This file is a base header for auto-generated *.lskel.h files.
32 * Its contents will change and may become part of auto-generation in the future.
33 *
34 * The layout of bpf_[map|prog]_desc and bpf_loader_ctx is feature dependent
35 * and will change from one version of libbpf to another and features
36 * requested during loader program generation.
37 */
38 struct bpf_map_desc {
39 /* output of the loader prog */
40 int map_fd;
41 /* input for the loader prog */
42 __u32 max_entries;
43 __aligned_u64 initial_value;
44 };
45 struct bpf_prog_desc {
46 int prog_fd;
47 };
48
49 enum {
50 BPF_SKEL_KERNEL = (1ULL << 0),
51 };
52
53 struct bpf_loader_ctx {
54 __u32 sz;
55 __u32 flags;
56 __u32 log_level;
57 __u32 log_size;
58 __u64 log_buf;
59 };
60
61 struct bpf_load_and_run_opts {
62 struct bpf_loader_ctx *ctx;
63 const void *data;
64 const void *insns;
65 __u32 data_sz;
66 __u32 insns_sz;
67 const char *errstr;
68 void *signature;
69 __u32 signature_sz;
70 __s32 keyring_id;
71 void *excl_prog_hash;
72 __u32 excl_prog_hash_sz;
73 };
74
75 long kern_sys_bpf(__u32 cmd, void *attr, __u32 attr_size);
76
skel_sys_bpf(enum bpf_cmd cmd,union bpf_attr * attr,unsigned int size)77 static inline int skel_sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr,
78 unsigned int size)
79 {
80 #ifdef __KERNEL__
81 return kern_sys_bpf(cmd, attr, size);
82 #else
83 return syscall(__NR_bpf, cmd, attr, size);
84 #endif
85 }
86
87 #ifdef __KERNEL__
close(int fd)88 static inline int close(int fd)
89 {
90 return close_fd(fd);
91 }
92
skel_alloc(size_t size)93 static inline void *skel_alloc(size_t size)
94 {
95 struct bpf_loader_ctx *ctx = kzalloc(size, GFP_KERNEL);
96
97 if (!ctx)
98 return NULL;
99 ctx->flags |= BPF_SKEL_KERNEL;
100 return ctx;
101 }
102
skel_free(const void * p)103 static inline void skel_free(const void *p)
104 {
105 kfree(p);
106 }
107
108 /* skel->bss/rodata maps are populated the following way:
109 *
110 * For kernel use:
111 * skel_prep_map_data() allocates kernel memory that kernel module can directly access.
112 * Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value.
113 * The loader program will perform probe_read_kernel() from maps.rodata.initial_value.
114 * skel_finalize_map_data() sets skel->rodata to point to actual value in a bpf map and
115 * does maps.rodata.initial_value = ~0ULL to signal skel_free_map_data() that kvfree
116 * is not necessary.
117 *
118 * For user space:
119 * skel_prep_map_data() mmaps anon memory into skel->rodata that can be accessed directly.
120 * Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value.
121 * The loader program will perform copy_from_user() from maps.rodata.initial_value.
122 * skel_finalize_map_data() remaps bpf array map value from the kernel memory into
123 * skel->rodata address.
124 *
125 * The "bpftool gen skeleton -L" command generates lskel.h that is suitable for
126 * both kernel and user space. The generated loader program does
127 * either bpf_probe_read_kernel() or bpf_copy_from_user() from initial_value
128 * depending on bpf_loader_ctx->flags.
129 */
skel_free_map_data(void * p,__u64 addr,size_t sz)130 static inline void skel_free_map_data(void *p, __u64 addr, size_t sz)
131 {
132 if (addr != ~0ULL)
133 kvfree(p);
134 /*
135 * When addr == ~0ULL the init buffer has already been released.
136 * For skel_finalize_map_data(), 'p' points to
137 * ((struct bpf_array *)map)->value.
138 */
139 }
140
skel_prep_map_data(const void * val,size_t mmap_sz,size_t val_sz)141 static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz)
142 {
143 void *addr;
144
145 addr = kvmalloc(val_sz, GFP_KERNEL);
146 if (!addr)
147 return NULL;
148 memcpy(addr, val, val_sz);
149 return addr;
150 }
151
skel_finalize_map_data(__u64 * init_val,size_t mmap_sz,int flags,int fd)152 static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd)
153 {
154 struct bpf_map *map;
155 void *addr = NULL;
156
157 kvfree((void *) (long) *init_val);
158 *init_val = ~0ULL;
159
160 /* At this point bpf_load_and_run() finished without error and
161 * 'fd' is a valid bpf map FD. All sanity checks below should succeed.
162 */
163 map = bpf_map_get(fd);
164 if (IS_ERR(map))
165 return NULL;
166 if (map->map_type != BPF_MAP_TYPE_ARRAY)
167 goto out;
168 addr = ((struct bpf_array *)map)->value;
169 /* the addr stays valid, since FD is not closed */
170 out:
171 bpf_map_put(map);
172 return addr;
173 }
174
skel_protect_map_data(void * p,__u64 * init_val,size_t sz)175 static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t sz)
176 {
177 (void)sz;
178
179 kvfree(p);
180 *init_val = ~0ULL;
181 return 0;
182 }
183
184 #else
185
skel_alloc(size_t size)186 static inline void *skel_alloc(size_t size)
187 {
188 return calloc(1, size);
189 }
190
skel_free(void * p)191 static inline void skel_free(void *p)
192 {
193 free(p);
194 }
195
skel_free_map_data(void * p,__u64 addr,size_t sz)196 static inline void skel_free_map_data(void *p, __u64 addr, size_t sz)
197 {
198 munmap(p, sz);
199 }
200
skel_prep_map_data(const void * val,size_t mmap_sz,size_t val_sz)201 static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz)
202 {
203 void *addr;
204
205 addr = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
206 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
207 if (addr == (void *) -1)
208 return NULL;
209 memcpy(addr, val, val_sz);
210 return addr;
211 }
212
skel_finalize_map_data(__u64 * init_val,size_t mmap_sz,int flags,int fd)213 static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd)
214 {
215 void *addr;
216
217 addr = mmap((void *) (long) *init_val, mmap_sz, flags, MAP_SHARED | MAP_FIXED, fd, 0);
218 if (addr == (void *) -1)
219 return NULL;
220 return addr;
221 }
222
skel_protect_map_data(void * p,__u64 * init_val,size_t sz)223 static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t sz)
224 {
225 (void)init_val;
226
227 if (mprotect(p, sz, PROT_READ))
228 return -errno;
229 return 0;
230 }
231 #endif
232
skel_closenz(int fd)233 static inline int skel_closenz(int fd)
234 {
235 if (fd > 0)
236 return close(fd);
237 return -EINVAL;
238 }
239
240 #ifndef offsetofend
241 #define offsetofend(TYPE, MEMBER) \
242 (offsetof(TYPE, MEMBER) + sizeof((((TYPE *)0)->MEMBER)))
243 #endif
244
skel_map_create(enum bpf_map_type map_type,const char * map_name,__u32 key_size,__u32 value_size,__u32 max_entries,const void * excl_prog_hash,__u32 excl_prog_hash_sz)245 static inline int skel_map_create(enum bpf_map_type map_type,
246 const char *map_name,
247 __u32 key_size,
248 __u32 value_size,
249 __u32 max_entries,
250 const void *excl_prog_hash,
251 __u32 excl_prog_hash_sz)
252 {
253 const size_t attr_sz = offsetofend(union bpf_attr, excl_prog_hash_size);
254 union bpf_attr attr;
255
256 memset(&attr, 0, attr_sz);
257
258 attr.map_type = map_type;
259 attr.excl_prog_hash = (unsigned long) excl_prog_hash;
260 attr.excl_prog_hash_size = excl_prog_hash_sz;
261
262 #ifdef __KERNEL__
263 if (strscpy(attr.map_name, map_name) < 0)
264 return -EINVAL;
265 #else
266 strncpy(attr.map_name, map_name, sizeof(attr.map_name));
267 #endif
268 attr.key_size = key_size;
269 attr.value_size = value_size;
270 attr.max_entries = max_entries;
271
272 return skel_sys_bpf(BPF_MAP_CREATE, &attr, attr_sz);
273 }
274
skel_map_update_elem(int fd,const void * key,const void * value,__u64 flags)275 static inline int skel_map_update_elem(int fd, const void *key,
276 const void *value, __u64 flags)
277 {
278 const size_t attr_sz = offsetofend(union bpf_attr, flags);
279 union bpf_attr attr;
280
281 memset(&attr, 0, attr_sz);
282 attr.map_fd = fd;
283 attr.key = (long) key;
284 attr.value = (long) value;
285 attr.flags = flags;
286
287 return skel_sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz);
288 }
289
skel_map_delete_elem(int fd,const void * key)290 static inline int skel_map_delete_elem(int fd, const void *key)
291 {
292 const size_t attr_sz = offsetofend(union bpf_attr, flags);
293 union bpf_attr attr;
294
295 memset(&attr, 0, attr_sz);
296 attr.map_fd = fd;
297 attr.key = (long)key;
298
299 return skel_sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz);
300 }
301
skel_map_get_fd_by_id(__u32 id)302 static inline int skel_map_get_fd_by_id(__u32 id)
303 {
304 const size_t attr_sz = offsetofend(union bpf_attr, flags);
305 union bpf_attr attr;
306
307 memset(&attr, 0, attr_sz);
308 attr.map_id = id;
309
310 return skel_sys_bpf(BPF_MAP_GET_FD_BY_ID, &attr, attr_sz);
311 }
312
skel_raw_tracepoint_open(const char * name,int prog_fd)313 static inline int skel_raw_tracepoint_open(const char *name, int prog_fd)
314 {
315 const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint.prog_fd);
316 union bpf_attr attr;
317
318 memset(&attr, 0, attr_sz);
319 attr.raw_tracepoint.name = (long) name;
320 attr.raw_tracepoint.prog_fd = prog_fd;
321
322 return skel_sys_bpf(BPF_RAW_TRACEPOINT_OPEN, &attr, attr_sz);
323 }
324
skel_link_create(int prog_fd,int target_fd,enum bpf_attach_type attach_type)325 static inline int skel_link_create(int prog_fd, int target_fd,
326 enum bpf_attach_type attach_type)
327 {
328 const size_t attr_sz = offsetofend(union bpf_attr, link_create.iter_info_len);
329 union bpf_attr attr;
330
331 memset(&attr, 0, attr_sz);
332 attr.link_create.prog_fd = prog_fd;
333 attr.link_create.target_fd = target_fd;
334 attr.link_create.attach_type = attach_type;
335
336 return skel_sys_bpf(BPF_LINK_CREATE, &attr, attr_sz);
337 }
338
skel_map_freeze(int fd)339 static inline int skel_map_freeze(int fd)
340 {
341 const size_t attr_sz = offsetofend(union bpf_attr, map_fd);
342 union bpf_attr attr;
343
344 memset(&attr, 0, attr_sz);
345 attr.map_fd = fd;
346
347 return skel_sys_bpf(BPF_MAP_FREEZE, &attr, attr_sz);
348 }
349 #ifdef __KERNEL__
350 #define set_err
351 #else
352 #define set_err err = -errno
353 #endif
354
bpf_load_and_run(struct bpf_load_and_run_opts * opts)355 static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts)
356 {
357 const size_t prog_load_attr_sz = offsetofend(union bpf_attr, keyring_id);
358 const size_t test_run_attr_sz = offsetofend(union bpf_attr, test);
359 int map_fd = -1, prog_fd = -1, key = 0, err;
360 union bpf_attr attr;
361
362 err = map_fd = skel_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, opts->data_sz, 1,
363 opts->excl_prog_hash, opts->excl_prog_hash_sz);
364 if (map_fd < 0) {
365 opts->errstr = "failed to create loader map";
366 set_err;
367 goto out;
368 }
369
370 err = skel_map_update_elem(map_fd, &key, opts->data, 0);
371 if (err < 0) {
372 opts->errstr = "failed to update loader map";
373 set_err;
374 goto out;
375 }
376
377 #ifndef __KERNEL__
378 err = skel_map_freeze(map_fd);
379 if (err < 0) {
380 opts->errstr = "failed to freeze map";
381 set_err;
382 goto out;
383 }
384 #endif
385
386 memset(&attr, 0, prog_load_attr_sz);
387 attr.prog_type = BPF_PROG_TYPE_SYSCALL;
388 attr.insns = (long) opts->insns;
389 attr.insn_cnt = opts->insns_sz / sizeof(struct bpf_insn);
390 attr.license = (long) "Dual BSD/GPL";
391 #ifndef __KERNEL__
392 attr.signature = (long) opts->signature;
393 attr.signature_size = opts->signature_sz;
394 if (opts->signature)
395 attr.fd_array_cnt = 1;
396 #else
397 if (opts->signature || opts->signature_sz)
398 pr_warn("signatures are not supported from bpf_preload\n");
399 #endif
400 attr.keyring_id = opts->keyring_id;
401 memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog"));
402 attr.fd_array = (long) &map_fd;
403 attr.log_level = opts->ctx->log_level;
404 attr.log_size = opts->ctx->log_size;
405 attr.log_buf = opts->ctx->log_buf;
406 attr.prog_flags = BPF_F_SLEEPABLE;
407 err = prog_fd = skel_sys_bpf(BPF_PROG_LOAD, &attr, prog_load_attr_sz);
408 if (prog_fd < 0) {
409 opts->errstr = "failed to load loader prog";
410 set_err;
411 goto out;
412 }
413
414 memset(&attr, 0, test_run_attr_sz);
415 attr.test.prog_fd = prog_fd;
416 attr.test.ctx_in = (long) opts->ctx;
417 attr.test.ctx_size_in = opts->ctx->sz;
418 err = skel_sys_bpf(BPF_PROG_RUN, &attr, test_run_attr_sz);
419 if (err < 0 || (int)attr.test.retval < 0) {
420 if (err < 0) {
421 opts->errstr = "failed to execute loader prog";
422 set_err;
423 } else {
424 opts->errstr = "error returned by loader prog";
425 err = (int)attr.test.retval;
426 #ifndef __KERNEL__
427 errno = -err;
428 #endif
429 }
430 goto out;
431 }
432 err = 0;
433 out:
434 if (map_fd >= 0)
435 close(map_fd);
436 if (prog_fd >= 0)
437 close(prog_fd);
438 return err;
439 }
440
441 #endif
442