1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ 2 /* Copyright (c) 2021 Facebook */ 3 #ifndef __SKEL_INTERNAL_H 4 #define __SKEL_INTERNAL_H 5 6 #ifdef __KERNEL__ 7 #include <linux/fdtable.h> 8 #include <linux/mm.h> 9 #include <linux/mman.h> 10 #include <linux/slab.h> 11 #include <linux/bpf.h> 12 #else 13 #include <unistd.h> 14 #include <sys/syscall.h> 15 #include <sys/mman.h> 16 #include <linux/keyctl.h> 17 #include <stdlib.h> 18 #include "bpf.h" 19 #endif 20 21 #ifndef SHA256_DIGEST_LENGTH 22 #define SHA256_DIGEST_LENGTH 32 23 #endif 24 25 #ifndef __NR_bpf 26 # if defined(__mips__) && defined(_ABIO32) 27 # define __NR_bpf 4355 28 # elif defined(__mips__) && defined(_ABIN32) 29 # define __NR_bpf 6319 30 # elif defined(__mips__) && defined(_ABI64) 31 # define __NR_bpf 5315 32 # endif 33 #endif 34 35 /* This file is a base header for auto-generated *.lskel.h files. 36 * Its contents will change and may become part of auto-generation in the future. 37 * 38 * The layout of bpf_[map|prog]_desc and bpf_loader_ctx is feature dependent 39 * and will change from one version of libbpf to another and features 40 * requested during loader program generation. 41 */ 42 struct bpf_map_desc { 43 /* output of the loader prog */ 44 int map_fd; 45 /* input for the loader prog */ 46 __u32 max_entries; 47 __aligned_u64 initial_value; 48 }; 49 struct bpf_prog_desc { 50 int prog_fd; 51 }; 52 53 enum { 54 BPF_SKEL_KERNEL = (1ULL << 0), 55 }; 56 57 struct bpf_loader_ctx { 58 __u32 sz; 59 __u32 flags; 60 __u32 log_level; 61 __u32 log_size; 62 __u64 log_buf; 63 }; 64 65 struct bpf_load_and_run_opts { 66 struct bpf_loader_ctx *ctx; 67 const void *data; 68 const void *insns; 69 __u32 data_sz; 70 __u32 insns_sz; 71 const char *errstr; 72 void *signature; 73 __u32 signature_sz; 74 __s32 keyring_id; 75 void *excl_prog_hash; 76 __u32 excl_prog_hash_sz; 77 }; 78 79 long kern_sys_bpf(__u32 cmd, void *attr, __u32 attr_size); 80 81 static inline int skel_sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr, 82 unsigned int size) 83 { 84 #ifdef __KERNEL__ 85 return kern_sys_bpf(cmd, attr, size); 86 #else 87 return syscall(__NR_bpf, cmd, attr, size); 88 #endif 89 } 90 91 #ifdef __KERNEL__ 92 static inline int close(int fd) 93 { 94 return close_fd(fd); 95 } 96 97 static inline void *skel_alloc(size_t size) 98 { 99 struct bpf_loader_ctx *ctx = kzalloc(size, GFP_KERNEL); 100 101 if (!ctx) 102 return NULL; 103 ctx->flags |= BPF_SKEL_KERNEL; 104 return ctx; 105 } 106 107 static inline void skel_free(const void *p) 108 { 109 kfree(p); 110 } 111 112 /* skel->bss/rodata maps are populated the following way: 113 * 114 * For kernel use: 115 * skel_prep_map_data() allocates kernel memory that kernel module can directly access. 116 * Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value. 117 * The loader program will perform probe_read_kernel() from maps.rodata.initial_value. 118 * skel_finalize_map_data() sets skel->rodata to point to actual value in a bpf map and 119 * does maps.rodata.initial_value = ~0ULL to signal skel_free_map_data() that kvfree 120 * is not necessary. 121 * 122 * For user space: 123 * skel_prep_map_data() mmaps anon memory into skel->rodata that can be accessed directly. 124 * Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value. 125 * The loader program will perform copy_from_user() from maps.rodata.initial_value. 126 * skel_finalize_map_data() remaps bpf array map value from the kernel memory into 127 * skel->rodata address. 128 * 129 * The "bpftool gen skeleton -L" command generates lskel.h that is suitable for 130 * both kernel and user space. The generated loader program does 131 * either bpf_probe_read_kernel() or bpf_copy_from_user() from initial_value 132 * depending on bpf_loader_ctx->flags. 133 */ 134 static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) 135 { 136 if (addr != ~0ULL) 137 kvfree(p); 138 /* When addr == ~0ULL the 'p' points to 139 * ((struct bpf_array *)map)->value. See skel_finalize_map_data. 140 */ 141 } 142 143 static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz) 144 { 145 void *addr; 146 147 addr = kvmalloc(val_sz, GFP_KERNEL); 148 if (!addr) 149 return NULL; 150 memcpy(addr, val, val_sz); 151 return addr; 152 } 153 154 static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd) 155 { 156 struct bpf_map *map; 157 void *addr = NULL; 158 159 kvfree((void *) (long) *init_val); 160 *init_val = ~0ULL; 161 162 /* At this point bpf_load_and_run() finished without error and 163 * 'fd' is a valid bpf map FD. All sanity checks below should succeed. 164 */ 165 map = bpf_map_get(fd); 166 if (IS_ERR(map)) 167 return NULL; 168 if (map->map_type != BPF_MAP_TYPE_ARRAY) 169 goto out; 170 addr = ((struct bpf_array *)map)->value; 171 /* the addr stays valid, since FD is not closed */ 172 out: 173 bpf_map_put(map); 174 return addr; 175 } 176 177 #else 178 179 static inline void *skel_alloc(size_t size) 180 { 181 return calloc(1, size); 182 } 183 184 static inline void skel_free(void *p) 185 { 186 free(p); 187 } 188 189 static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) 190 { 191 munmap(p, sz); 192 } 193 194 static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz) 195 { 196 void *addr; 197 198 addr = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, 199 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 200 if (addr == (void *) -1) 201 return NULL; 202 memcpy(addr, val, val_sz); 203 return addr; 204 } 205 206 static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd) 207 { 208 void *addr; 209 210 addr = mmap((void *) (long) *init_val, mmap_sz, flags, MAP_SHARED | MAP_FIXED, fd, 0); 211 if (addr == (void *) -1) 212 return NULL; 213 return addr; 214 } 215 #endif 216 217 static inline int skel_closenz(int fd) 218 { 219 if (fd > 0) 220 return close(fd); 221 return -EINVAL; 222 } 223 224 #ifndef offsetofend 225 #define offsetofend(TYPE, MEMBER) \ 226 (offsetof(TYPE, MEMBER) + sizeof((((TYPE *)0)->MEMBER))) 227 #endif 228 229 static inline int skel_map_create(enum bpf_map_type map_type, 230 const char *map_name, 231 __u32 key_size, 232 __u32 value_size, 233 __u32 max_entries, 234 const void *excl_prog_hash, 235 __u32 excl_prog_hash_sz) 236 { 237 const size_t attr_sz = offsetofend(union bpf_attr, excl_prog_hash_size); 238 union bpf_attr attr; 239 240 memset(&attr, 0, attr_sz); 241 242 attr.map_type = map_type; 243 attr.excl_prog_hash = (unsigned long) excl_prog_hash; 244 attr.excl_prog_hash_size = excl_prog_hash_sz; 245 246 #ifdef __KERNEL__ 247 if (strscpy(attr.map_name, map_name) < 0) 248 return -EINVAL; 249 #else 250 strncpy(attr.map_name, map_name, sizeof(attr.map_name)); 251 #endif 252 attr.key_size = key_size; 253 attr.value_size = value_size; 254 attr.max_entries = max_entries; 255 256 return skel_sys_bpf(BPF_MAP_CREATE, &attr, attr_sz); 257 } 258 259 static inline int skel_map_update_elem(int fd, const void *key, 260 const void *value, __u64 flags) 261 { 262 const size_t attr_sz = offsetofend(union bpf_attr, flags); 263 union bpf_attr attr; 264 265 memset(&attr, 0, attr_sz); 266 attr.map_fd = fd; 267 attr.key = (long) key; 268 attr.value = (long) value; 269 attr.flags = flags; 270 271 return skel_sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz); 272 } 273 274 static inline int skel_map_delete_elem(int fd, const void *key) 275 { 276 const size_t attr_sz = offsetofend(union bpf_attr, flags); 277 union bpf_attr attr; 278 279 memset(&attr, 0, attr_sz); 280 attr.map_fd = fd; 281 attr.key = (long)key; 282 283 return skel_sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz); 284 } 285 286 static inline int skel_map_get_fd_by_id(__u32 id) 287 { 288 const size_t attr_sz = offsetofend(union bpf_attr, flags); 289 union bpf_attr attr; 290 291 memset(&attr, 0, attr_sz); 292 attr.map_id = id; 293 294 return skel_sys_bpf(BPF_MAP_GET_FD_BY_ID, &attr, attr_sz); 295 } 296 297 static inline int skel_raw_tracepoint_open(const char *name, int prog_fd) 298 { 299 const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint.prog_fd); 300 union bpf_attr attr; 301 302 memset(&attr, 0, attr_sz); 303 attr.raw_tracepoint.name = (long) name; 304 attr.raw_tracepoint.prog_fd = prog_fd; 305 306 return skel_sys_bpf(BPF_RAW_TRACEPOINT_OPEN, &attr, attr_sz); 307 } 308 309 static inline int skel_link_create(int prog_fd, int target_fd, 310 enum bpf_attach_type attach_type) 311 { 312 const size_t attr_sz = offsetofend(union bpf_attr, link_create.iter_info_len); 313 union bpf_attr attr; 314 315 memset(&attr, 0, attr_sz); 316 attr.link_create.prog_fd = prog_fd; 317 attr.link_create.target_fd = target_fd; 318 attr.link_create.attach_type = attach_type; 319 320 return skel_sys_bpf(BPF_LINK_CREATE, &attr, attr_sz); 321 } 322 323 static inline int skel_obj_get_info_by_fd(int fd) 324 { 325 const size_t attr_sz = offsetofend(union bpf_attr, info); 326 __u8 sha[SHA256_DIGEST_LENGTH]; 327 struct bpf_map_info info; 328 __u32 info_len = sizeof(info); 329 union bpf_attr attr; 330 331 memset(&info, 0, sizeof(info)); 332 info.hash = (long) &sha; 333 info.hash_size = SHA256_DIGEST_LENGTH; 334 335 memset(&attr, 0, attr_sz); 336 attr.info.bpf_fd = fd; 337 attr.info.info = (long) &info; 338 attr.info.info_len = info_len; 339 return skel_sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, attr_sz); 340 } 341 342 static inline int skel_map_freeze(int fd) 343 { 344 const size_t attr_sz = offsetofend(union bpf_attr, map_fd); 345 union bpf_attr attr; 346 347 memset(&attr, 0, attr_sz); 348 attr.map_fd = fd; 349 350 return skel_sys_bpf(BPF_MAP_FREEZE, &attr, attr_sz); 351 } 352 #ifdef __KERNEL__ 353 #define set_err 354 #else 355 #define set_err err = -errno 356 #endif 357 358 static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) 359 { 360 const size_t prog_load_attr_sz = offsetofend(union bpf_attr, keyring_id); 361 const size_t test_run_attr_sz = offsetofend(union bpf_attr, test); 362 int map_fd = -1, prog_fd = -1, key = 0, err; 363 union bpf_attr attr; 364 365 err = map_fd = skel_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, opts->data_sz, 1, 366 opts->excl_prog_hash, opts->excl_prog_hash_sz); 367 if (map_fd < 0) { 368 opts->errstr = "failed to create loader map"; 369 set_err; 370 goto out; 371 } 372 373 err = skel_map_update_elem(map_fd, &key, opts->data, 0); 374 if (err < 0) { 375 opts->errstr = "failed to update loader map"; 376 set_err; 377 goto out; 378 } 379 380 #ifndef __KERNEL__ 381 err = skel_map_freeze(map_fd); 382 if (err < 0) { 383 opts->errstr = "failed to freeze map"; 384 set_err; 385 goto out; 386 } 387 err = skel_obj_get_info_by_fd(map_fd); 388 if (err < 0) { 389 opts->errstr = "failed to fetch obj info"; 390 set_err; 391 goto out; 392 } 393 #endif 394 395 memset(&attr, 0, prog_load_attr_sz); 396 attr.prog_type = BPF_PROG_TYPE_SYSCALL; 397 attr.insns = (long) opts->insns; 398 attr.insn_cnt = opts->insns_sz / sizeof(struct bpf_insn); 399 attr.license = (long) "Dual BSD/GPL"; 400 #ifndef __KERNEL__ 401 attr.signature = (long) opts->signature; 402 attr.signature_size = opts->signature_sz; 403 #else 404 if (opts->signature || opts->signature_sz) 405 pr_warn("signatures are not supported from bpf_preload\n"); 406 #endif 407 attr.keyring_id = opts->keyring_id; 408 memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); 409 attr.fd_array = (long) &map_fd; 410 attr.log_level = opts->ctx->log_level; 411 attr.log_size = opts->ctx->log_size; 412 attr.log_buf = opts->ctx->log_buf; 413 attr.prog_flags = BPF_F_SLEEPABLE; 414 err = prog_fd = skel_sys_bpf(BPF_PROG_LOAD, &attr, prog_load_attr_sz); 415 if (prog_fd < 0) { 416 opts->errstr = "failed to load loader prog"; 417 set_err; 418 goto out; 419 } 420 421 memset(&attr, 0, test_run_attr_sz); 422 attr.test.prog_fd = prog_fd; 423 attr.test.ctx_in = (long) opts->ctx; 424 attr.test.ctx_size_in = opts->ctx->sz; 425 err = skel_sys_bpf(BPF_PROG_RUN, &attr, test_run_attr_sz); 426 if (err < 0 || (int)attr.test.retval < 0) { 427 if (err < 0) { 428 opts->errstr = "failed to execute loader prog"; 429 set_err; 430 } else { 431 opts->errstr = "error returned by loader prog"; 432 err = (int)attr.test.retval; 433 #ifndef __KERNEL__ 434 errno = -err; 435 #endif 436 } 437 goto out; 438 } 439 err = 0; 440 out: 441 if (map_fd >= 0) 442 close(map_fd); 443 if (prog_fd >= 0) 444 close(prog_fd); 445 return err; 446 } 447 448 #endif 449