1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ 2 /* Copyright (c) 2021 Facebook */ 3 #ifndef __SKEL_INTERNAL_H 4 #define __SKEL_INTERNAL_H 5 6 #ifdef __KERNEL__ 7 #include <linux/fdtable.h> 8 #include <linux/mm.h> 9 #include <linux/mman.h> 10 #include <linux/slab.h> 11 #include <linux/bpf.h> 12 #else 13 #include <unistd.h> 14 #include <sys/syscall.h> 15 #include <sys/mman.h> 16 #include <linux/keyctl.h> 17 #include <stdlib.h> 18 #include "bpf.h" 19 #endif 20 21 #ifndef __NR_bpf 22 # if defined(__mips__) && defined(_ABIO32) 23 # define __NR_bpf 4355 24 # elif defined(__mips__) && defined(_ABIN32) 25 # define __NR_bpf 6319 26 # elif defined(__mips__) && defined(_ABI64) 27 # define __NR_bpf 5315 28 # endif 29 #endif 30 31 /* This file is a base header for auto-generated *.lskel.h files. 32 * Its contents will change and may become part of auto-generation in the future. 33 * 34 * The layout of bpf_[map|prog]_desc and bpf_loader_ctx is feature dependent 35 * and will change from one version of libbpf to another and features 36 * requested during loader program generation. 37 */ 38 struct bpf_map_desc { 39 /* output of the loader prog */ 40 int map_fd; 41 /* input for the loader prog */ 42 __u32 max_entries; 43 __aligned_u64 initial_value; 44 }; 45 struct bpf_prog_desc { 46 int prog_fd; 47 }; 48 49 enum { 50 BPF_SKEL_KERNEL = (1ULL << 0), 51 }; 52 53 struct bpf_loader_ctx { 54 __u32 sz; 55 __u32 flags; 56 __u32 log_level; 57 __u32 log_size; 58 __u64 log_buf; 59 }; 60 61 struct bpf_load_and_run_opts { 62 struct bpf_loader_ctx *ctx; 63 const void *data; 64 const void *insns; 65 __u32 data_sz; 66 __u32 insns_sz; 67 const char *errstr; 68 void *signature; 69 __u32 signature_sz; 70 __s32 keyring_id; 71 void *excl_prog_hash; 72 __u32 excl_prog_hash_sz; 73 }; 74 75 long kern_sys_bpf(__u32 cmd, void *attr, __u32 attr_size); 76 77 static inline int skel_sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr, 78 unsigned int size) 79 { 80 #ifdef __KERNEL__ 81 return kern_sys_bpf(cmd, attr, size); 82 #else 83 return syscall(__NR_bpf, cmd, attr, size); 84 #endif 85 } 86 87 #ifdef __KERNEL__ 88 static inline int close(int fd) 89 { 90 return close_fd(fd); 91 } 92 93 static inline void *skel_alloc(size_t size) 94 { 95 struct bpf_loader_ctx *ctx = kzalloc(size, GFP_KERNEL); 96 97 if (!ctx) 98 return NULL; 99 ctx->flags |= BPF_SKEL_KERNEL; 100 return ctx; 101 } 102 103 static inline void skel_free(const void *p) 104 { 105 kfree(p); 106 } 107 108 /* skel->bss/rodata maps are populated the following way: 109 * 110 * For kernel use: 111 * skel_prep_map_data() allocates kernel memory that kernel module can directly access. 112 * Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value. 113 * The loader program will perform probe_read_kernel() from maps.rodata.initial_value. 114 * skel_finalize_map_data() sets skel->rodata to point to actual value in a bpf map and 115 * does maps.rodata.initial_value = ~0ULL to signal skel_free_map_data() that kvfree 116 * is not necessary. 117 * 118 * For user space: 119 * skel_prep_map_data() mmaps anon memory into skel->rodata that can be accessed directly. 120 * Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value. 121 * The loader program will perform copy_from_user() from maps.rodata.initial_value. 122 * skel_finalize_map_data() remaps bpf array map value from the kernel memory into 123 * skel->rodata address. 124 * 125 * The "bpftool gen skeleton -L" command generates lskel.h that is suitable for 126 * both kernel and user space. The generated loader program does 127 * either bpf_probe_read_kernel() or bpf_copy_from_user() from initial_value 128 * depending on bpf_loader_ctx->flags. 129 */ 130 static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) 131 { 132 if (addr != ~0ULL) 133 kvfree(p); 134 /* 135 * When addr == ~0ULL the init buffer has already been released. 136 * For skel_finalize_map_data(), 'p' points to 137 * ((struct bpf_array *)map)->value. 138 */ 139 } 140 141 static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz) 142 { 143 void *addr; 144 145 addr = kvmalloc(val_sz, GFP_KERNEL); 146 if (!addr) 147 return NULL; 148 memcpy(addr, val, val_sz); 149 return addr; 150 } 151 152 static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd) 153 { 154 struct bpf_map *map; 155 void *addr = NULL; 156 157 kvfree((void *) (long) *init_val); 158 *init_val = ~0ULL; 159 160 /* At this point bpf_load_and_run() finished without error and 161 * 'fd' is a valid bpf map FD. All sanity checks below should succeed. 162 */ 163 map = bpf_map_get(fd); 164 if (IS_ERR(map)) 165 return NULL; 166 if (map->map_type != BPF_MAP_TYPE_ARRAY) 167 goto out; 168 addr = ((struct bpf_array *)map)->value; 169 /* the addr stays valid, since FD is not closed */ 170 out: 171 bpf_map_put(map); 172 return addr; 173 } 174 175 static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t sz) 176 { 177 (void)sz; 178 179 kvfree(p); 180 *init_val = ~0ULL; 181 return 0; 182 } 183 184 #else 185 186 static inline void *skel_alloc(size_t size) 187 { 188 return calloc(1, size); 189 } 190 191 static inline void skel_free(void *p) 192 { 193 free(p); 194 } 195 196 static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) 197 { 198 munmap(p, sz); 199 } 200 201 static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz) 202 { 203 void *addr; 204 205 addr = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, 206 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 207 if (addr == (void *) -1) 208 return NULL; 209 memcpy(addr, val, val_sz); 210 return addr; 211 } 212 213 static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd) 214 { 215 void *addr; 216 217 addr = mmap((void *) (long) *init_val, mmap_sz, flags, MAP_SHARED | MAP_FIXED, fd, 0); 218 if (addr == (void *) -1) 219 return NULL; 220 return addr; 221 } 222 223 static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t sz) 224 { 225 (void)init_val; 226 227 if (mprotect(p, sz, PROT_READ)) 228 return -errno; 229 return 0; 230 } 231 #endif 232 233 static inline int skel_closenz(int fd) 234 { 235 if (fd > 0) 236 return close(fd); 237 return -EINVAL; 238 } 239 240 #ifndef offsetofend 241 #define offsetofend(TYPE, MEMBER) \ 242 (offsetof(TYPE, MEMBER) + sizeof((((TYPE *)0)->MEMBER))) 243 #endif 244 245 static inline int skel_map_create(enum bpf_map_type map_type, 246 const char *map_name, 247 __u32 key_size, 248 __u32 value_size, 249 __u32 max_entries, 250 const void *excl_prog_hash, 251 __u32 excl_prog_hash_sz) 252 { 253 const size_t attr_sz = offsetofend(union bpf_attr, excl_prog_hash_size); 254 union bpf_attr attr; 255 256 memset(&attr, 0, attr_sz); 257 258 attr.map_type = map_type; 259 attr.excl_prog_hash = (unsigned long) excl_prog_hash; 260 attr.excl_prog_hash_size = excl_prog_hash_sz; 261 262 #ifdef __KERNEL__ 263 if (strscpy(attr.map_name, map_name) < 0) 264 return -EINVAL; 265 #else 266 strncpy(attr.map_name, map_name, sizeof(attr.map_name)); 267 #endif 268 attr.key_size = key_size; 269 attr.value_size = value_size; 270 attr.max_entries = max_entries; 271 272 return skel_sys_bpf(BPF_MAP_CREATE, &attr, attr_sz); 273 } 274 275 static inline int skel_map_update_elem(int fd, const void *key, 276 const void *value, __u64 flags) 277 { 278 const size_t attr_sz = offsetofend(union bpf_attr, flags); 279 union bpf_attr attr; 280 281 memset(&attr, 0, attr_sz); 282 attr.map_fd = fd; 283 attr.key = (long) key; 284 attr.value = (long) value; 285 attr.flags = flags; 286 287 return skel_sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz); 288 } 289 290 static inline int skel_map_delete_elem(int fd, const void *key) 291 { 292 const size_t attr_sz = offsetofend(union bpf_attr, flags); 293 union bpf_attr attr; 294 295 memset(&attr, 0, attr_sz); 296 attr.map_fd = fd; 297 attr.key = (long)key; 298 299 return skel_sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz); 300 } 301 302 static inline int skel_map_get_fd_by_id(__u32 id) 303 { 304 const size_t attr_sz = offsetofend(union bpf_attr, flags); 305 union bpf_attr attr; 306 307 memset(&attr, 0, attr_sz); 308 attr.map_id = id; 309 310 return skel_sys_bpf(BPF_MAP_GET_FD_BY_ID, &attr, attr_sz); 311 } 312 313 static inline int skel_raw_tracepoint_open(const char *name, int prog_fd) 314 { 315 const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint.prog_fd); 316 union bpf_attr attr; 317 318 memset(&attr, 0, attr_sz); 319 attr.raw_tracepoint.name = (long) name; 320 attr.raw_tracepoint.prog_fd = prog_fd; 321 322 return skel_sys_bpf(BPF_RAW_TRACEPOINT_OPEN, &attr, attr_sz); 323 } 324 325 static inline int skel_link_create(int prog_fd, int target_fd, 326 enum bpf_attach_type attach_type) 327 { 328 const size_t attr_sz = offsetofend(union bpf_attr, link_create.iter_info_len); 329 union bpf_attr attr; 330 331 memset(&attr, 0, attr_sz); 332 attr.link_create.prog_fd = prog_fd; 333 attr.link_create.target_fd = target_fd; 334 attr.link_create.attach_type = attach_type; 335 336 return skel_sys_bpf(BPF_LINK_CREATE, &attr, attr_sz); 337 } 338 339 static inline int skel_map_freeze(int fd) 340 { 341 const size_t attr_sz = offsetofend(union bpf_attr, map_fd); 342 union bpf_attr attr; 343 344 memset(&attr, 0, attr_sz); 345 attr.map_fd = fd; 346 347 return skel_sys_bpf(BPF_MAP_FREEZE, &attr, attr_sz); 348 } 349 #ifdef __KERNEL__ 350 #define set_err 351 #else 352 #define set_err err = -errno 353 #endif 354 355 static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) 356 { 357 const size_t prog_load_attr_sz = offsetofend(union bpf_attr, keyring_id); 358 const size_t test_run_attr_sz = offsetofend(union bpf_attr, test); 359 int map_fd = -1, prog_fd = -1, key = 0, err; 360 union bpf_attr attr; 361 362 err = map_fd = skel_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, opts->data_sz, 1, 363 opts->excl_prog_hash, opts->excl_prog_hash_sz); 364 if (map_fd < 0) { 365 opts->errstr = "failed to create loader map"; 366 set_err; 367 goto out; 368 } 369 370 err = skel_map_update_elem(map_fd, &key, opts->data, 0); 371 if (err < 0) { 372 opts->errstr = "failed to update loader map"; 373 set_err; 374 goto out; 375 } 376 377 #ifndef __KERNEL__ 378 err = skel_map_freeze(map_fd); 379 if (err < 0) { 380 opts->errstr = "failed to freeze map"; 381 set_err; 382 goto out; 383 } 384 #endif 385 386 memset(&attr, 0, prog_load_attr_sz); 387 attr.prog_type = BPF_PROG_TYPE_SYSCALL; 388 attr.insns = (long) opts->insns; 389 attr.insn_cnt = opts->insns_sz / sizeof(struct bpf_insn); 390 attr.license = (long) "Dual BSD/GPL"; 391 #ifndef __KERNEL__ 392 attr.signature = (long) opts->signature; 393 attr.signature_size = opts->signature_sz; 394 if (opts->signature) 395 attr.fd_array_cnt = 1; 396 #else 397 if (opts->signature || opts->signature_sz) 398 pr_warn("signatures are not supported from bpf_preload\n"); 399 #endif 400 attr.keyring_id = opts->keyring_id; 401 memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); 402 attr.fd_array = (long) &map_fd; 403 attr.log_level = opts->ctx->log_level; 404 attr.log_size = opts->ctx->log_size; 405 attr.log_buf = opts->ctx->log_buf; 406 attr.prog_flags = BPF_F_SLEEPABLE; 407 err = prog_fd = skel_sys_bpf(BPF_PROG_LOAD, &attr, prog_load_attr_sz); 408 if (prog_fd < 0) { 409 opts->errstr = "failed to load loader prog"; 410 set_err; 411 goto out; 412 } 413 414 memset(&attr, 0, test_run_attr_sz); 415 attr.test.prog_fd = prog_fd; 416 attr.test.ctx_in = (long) opts->ctx; 417 attr.test.ctx_size_in = opts->ctx->sz; 418 err = skel_sys_bpf(BPF_PROG_RUN, &attr, test_run_attr_sz); 419 if (err < 0 || (int)attr.test.retval < 0) { 420 if (err < 0) { 421 opts->errstr = "failed to execute loader prog"; 422 set_err; 423 } else { 424 opts->errstr = "error returned by loader prog"; 425 err = (int)attr.test.retval; 426 #ifndef __KERNEL__ 427 errno = -err; 428 #endif 429 } 430 goto out; 431 } 432 err = 0; 433 out: 434 if (map_fd >= 0) 435 close(map_fd); 436 if (prog_fd >= 0) 437 close(prog_fd); 438 return err; 439 } 440 441 #endif 442