1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ 2 /* Copyright (c) 2021 Facebook */ 3 #ifndef __SKEL_INTERNAL_H 4 #define __SKEL_INTERNAL_H 5 6 #ifdef __KERNEL__ 7 #include <linux/fdtable.h> 8 #include <linux/mm.h> 9 #include <linux/mman.h> 10 #include <linux/slab.h> 11 #include <linux/bpf.h> 12 #else 13 #include <unistd.h> 14 #include <sys/syscall.h> 15 #include <sys/mman.h> 16 #include <linux/keyctl.h> 17 #include <stdlib.h> 18 #include "bpf.h" 19 #endif 20 21 #ifndef SHA256_DIGEST_LENGTH 22 #define SHA256_DIGEST_LENGTH 32 23 #endif 24 25 #ifndef __NR_bpf 26 # if defined(__mips__) && defined(_ABIO32) 27 # define __NR_bpf 4355 28 # elif defined(__mips__) && defined(_ABIN32) 29 # define __NR_bpf 6319 30 # elif defined(__mips__) && defined(_ABI64) 31 # define __NR_bpf 5315 32 # endif 33 #endif 34 35 /* This file is a base header for auto-generated *.lskel.h files. 36 * Its contents will change and may become part of auto-generation in the future. 37 * 38 * The layout of bpf_[map|prog]_desc and bpf_loader_ctx is feature dependent 39 * and will change from one version of libbpf to another and features 40 * requested during loader program generation. 41 */ 42 struct bpf_map_desc { 43 /* output of the loader prog */ 44 int map_fd; 45 /* input for the loader prog */ 46 __u32 max_entries; 47 __aligned_u64 initial_value; 48 }; 49 struct bpf_prog_desc { 50 int prog_fd; 51 }; 52 53 enum { 54 BPF_SKEL_KERNEL = (1ULL << 0), 55 }; 56 57 struct bpf_loader_ctx { 58 __u32 sz; 59 __u32 flags; 60 __u32 log_level; 61 __u32 log_size; 62 __u64 log_buf; 63 }; 64 65 struct bpf_load_and_run_opts { 66 struct bpf_loader_ctx *ctx; 67 const void *data; 68 const void *insns; 69 __u32 data_sz; 70 __u32 insns_sz; 71 const char *errstr; 72 void *signature; 73 __u32 signature_sz; 74 __s32 keyring_id; 75 void *excl_prog_hash; 76 __u32 excl_prog_hash_sz; 77 }; 78 79 long kern_sys_bpf(__u32 cmd, void *attr, __u32 attr_size); 80 81 static inline int skel_sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr, 82 unsigned int size) 83 { 84 #ifdef __KERNEL__ 85 return kern_sys_bpf(cmd, attr, size); 86 #else 87 return syscall(__NR_bpf, cmd, attr, size); 88 #endif 89 } 90 91 #ifdef __KERNEL__ 92 static inline int close(int fd) 93 { 94 return close_fd(fd); 95 } 96 97 static inline void *skel_alloc(size_t size) 98 { 99 struct bpf_loader_ctx *ctx = kzalloc(size, GFP_KERNEL); 100 101 if (!ctx) 102 return NULL; 103 ctx->flags |= BPF_SKEL_KERNEL; 104 return ctx; 105 } 106 107 static inline void skel_free(const void *p) 108 { 109 kfree(p); 110 } 111 112 /* skel->bss/rodata maps are populated the following way: 113 * 114 * For kernel use: 115 * skel_prep_map_data() allocates kernel memory that kernel module can directly access. 116 * Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value. 117 * The loader program will perform probe_read_kernel() from maps.rodata.initial_value. 118 * skel_finalize_map_data() sets skel->rodata to point to actual value in a bpf map and 119 * does maps.rodata.initial_value = ~0ULL to signal skel_free_map_data() that kvfree 120 * is not necessary. 121 * 122 * For user space: 123 * skel_prep_map_data() mmaps anon memory into skel->rodata that can be accessed directly. 124 * Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value. 125 * The loader program will perform copy_from_user() from maps.rodata.initial_value. 126 * skel_finalize_map_data() remaps bpf array map value from the kernel memory into 127 * skel->rodata address. 128 * 129 * The "bpftool gen skeleton -L" command generates lskel.h that is suitable for 130 * both kernel and user space. The generated loader program does 131 * either bpf_probe_read_kernel() or bpf_copy_from_user() from initial_value 132 * depending on bpf_loader_ctx->flags. 133 */ 134 static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) 135 { 136 if (addr != ~0ULL) 137 kvfree(p); 138 /* When addr == ~0ULL the 'p' points to 139 * ((struct bpf_array *)map)->value. See skel_finalize_map_data. 140 */ 141 } 142 143 static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz) 144 { 145 void *addr; 146 147 addr = kvmalloc(val_sz, GFP_KERNEL); 148 if (!addr) 149 return NULL; 150 memcpy(addr, val, val_sz); 151 return addr; 152 } 153 154 static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd) 155 { 156 struct bpf_map *map; 157 void *addr = NULL; 158 159 kvfree((void *) (long) *init_val); 160 *init_val = ~0ULL; 161 162 /* At this point bpf_load_and_run() finished without error and 163 * 'fd' is a valid bpf map FD. All sanity checks below should succeed. 164 */ 165 map = bpf_map_get(fd); 166 if (IS_ERR(map)) 167 return NULL; 168 if (map->map_type != BPF_MAP_TYPE_ARRAY) 169 goto out; 170 addr = ((struct bpf_array *)map)->value; 171 /* the addr stays valid, since FD is not closed */ 172 out: 173 bpf_map_put(map); 174 return addr; 175 } 176 177 #else 178 179 static inline void *skel_alloc(size_t size) 180 { 181 return calloc(1, size); 182 } 183 184 static inline void skel_free(void *p) 185 { 186 free(p); 187 } 188 189 static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) 190 { 191 munmap(p, sz); 192 } 193 194 static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz) 195 { 196 void *addr; 197 198 addr = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, 199 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 200 if (addr == (void *) -1) 201 return NULL; 202 memcpy(addr, val, val_sz); 203 return addr; 204 } 205 206 static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd) 207 { 208 void *addr; 209 210 addr = mmap((void *) (long) *init_val, mmap_sz, flags, MAP_SHARED | MAP_FIXED, fd, 0); 211 if (addr == (void *) -1) 212 return NULL; 213 return addr; 214 } 215 #endif 216 217 static inline int skel_closenz(int fd) 218 { 219 if (fd > 0) 220 return close(fd); 221 return -EINVAL; 222 } 223 224 #ifndef offsetofend 225 #define offsetofend(TYPE, MEMBER) \ 226 (offsetof(TYPE, MEMBER) + sizeof((((TYPE *)0)->MEMBER))) 227 #endif 228 229 static inline int skel_map_create(enum bpf_map_type map_type, 230 const char *map_name, 231 __u32 key_size, 232 __u32 value_size, 233 __u32 max_entries, 234 const void *excl_prog_hash, 235 __u32 excl_prog_hash_sz) 236 { 237 const size_t attr_sz = offsetofend(union bpf_attr, excl_prog_hash_size); 238 union bpf_attr attr; 239 240 memset(&attr, 0, attr_sz); 241 242 attr.map_type = map_type; 243 attr.excl_prog_hash = (unsigned long) excl_prog_hash; 244 attr.excl_prog_hash_size = excl_prog_hash_sz; 245 246 strncpy(attr.map_name, map_name, sizeof(attr.map_name)); 247 attr.key_size = key_size; 248 attr.value_size = value_size; 249 attr.max_entries = max_entries; 250 251 return skel_sys_bpf(BPF_MAP_CREATE, &attr, attr_sz); 252 } 253 254 static inline int skel_map_update_elem(int fd, const void *key, 255 const void *value, __u64 flags) 256 { 257 const size_t attr_sz = offsetofend(union bpf_attr, flags); 258 union bpf_attr attr; 259 260 memset(&attr, 0, attr_sz); 261 attr.map_fd = fd; 262 attr.key = (long) key; 263 attr.value = (long) value; 264 attr.flags = flags; 265 266 return skel_sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz); 267 } 268 269 static inline int skel_map_delete_elem(int fd, const void *key) 270 { 271 const size_t attr_sz = offsetofend(union bpf_attr, flags); 272 union bpf_attr attr; 273 274 memset(&attr, 0, attr_sz); 275 attr.map_fd = fd; 276 attr.key = (long)key; 277 278 return skel_sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz); 279 } 280 281 static inline int skel_map_get_fd_by_id(__u32 id) 282 { 283 const size_t attr_sz = offsetofend(union bpf_attr, flags); 284 union bpf_attr attr; 285 286 memset(&attr, 0, attr_sz); 287 attr.map_id = id; 288 289 return skel_sys_bpf(BPF_MAP_GET_FD_BY_ID, &attr, attr_sz); 290 } 291 292 static inline int skel_raw_tracepoint_open(const char *name, int prog_fd) 293 { 294 const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint.prog_fd); 295 union bpf_attr attr; 296 297 memset(&attr, 0, attr_sz); 298 attr.raw_tracepoint.name = (long) name; 299 attr.raw_tracepoint.prog_fd = prog_fd; 300 301 return skel_sys_bpf(BPF_RAW_TRACEPOINT_OPEN, &attr, attr_sz); 302 } 303 304 static inline int skel_link_create(int prog_fd, int target_fd, 305 enum bpf_attach_type attach_type) 306 { 307 const size_t attr_sz = offsetofend(union bpf_attr, link_create.iter_info_len); 308 union bpf_attr attr; 309 310 memset(&attr, 0, attr_sz); 311 attr.link_create.prog_fd = prog_fd; 312 attr.link_create.target_fd = target_fd; 313 attr.link_create.attach_type = attach_type; 314 315 return skel_sys_bpf(BPF_LINK_CREATE, &attr, attr_sz); 316 } 317 318 static inline int skel_obj_get_info_by_fd(int fd) 319 { 320 const size_t attr_sz = offsetofend(union bpf_attr, info); 321 __u8 sha[SHA256_DIGEST_LENGTH]; 322 struct bpf_map_info info; 323 __u32 info_len = sizeof(info); 324 union bpf_attr attr; 325 326 memset(&info, 0, sizeof(info)); 327 info.hash = (long) &sha; 328 info.hash_size = SHA256_DIGEST_LENGTH; 329 330 memset(&attr, 0, attr_sz); 331 attr.info.bpf_fd = fd; 332 attr.info.info = (long) &info; 333 attr.info.info_len = info_len; 334 return skel_sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, attr_sz); 335 } 336 337 static inline int skel_map_freeze(int fd) 338 { 339 const size_t attr_sz = offsetofend(union bpf_attr, map_fd); 340 union bpf_attr attr; 341 342 memset(&attr, 0, attr_sz); 343 attr.map_fd = fd; 344 345 return skel_sys_bpf(BPF_MAP_FREEZE, &attr, attr_sz); 346 } 347 #ifdef __KERNEL__ 348 #define set_err 349 #else 350 #define set_err err = -errno 351 #endif 352 353 static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) 354 { 355 const size_t prog_load_attr_sz = offsetofend(union bpf_attr, keyring_id); 356 const size_t test_run_attr_sz = offsetofend(union bpf_attr, test); 357 int map_fd = -1, prog_fd = -1, key = 0, err; 358 union bpf_attr attr; 359 360 err = map_fd = skel_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, opts->data_sz, 1, 361 opts->excl_prog_hash, opts->excl_prog_hash_sz); 362 if (map_fd < 0) { 363 opts->errstr = "failed to create loader map"; 364 set_err; 365 goto out; 366 } 367 368 err = skel_map_update_elem(map_fd, &key, opts->data, 0); 369 if (err < 0) { 370 opts->errstr = "failed to update loader map"; 371 set_err; 372 goto out; 373 } 374 375 #ifndef __KERNEL__ 376 err = skel_map_freeze(map_fd); 377 if (err < 0) { 378 opts->errstr = "failed to freeze map"; 379 set_err; 380 goto out; 381 } 382 err = skel_obj_get_info_by_fd(map_fd); 383 if (err < 0) { 384 opts->errstr = "failed to fetch obj info"; 385 set_err; 386 goto out; 387 } 388 #endif 389 390 memset(&attr, 0, prog_load_attr_sz); 391 attr.prog_type = BPF_PROG_TYPE_SYSCALL; 392 attr.insns = (long) opts->insns; 393 attr.insn_cnt = opts->insns_sz / sizeof(struct bpf_insn); 394 attr.license = (long) "Dual BSD/GPL"; 395 #ifndef __KERNEL__ 396 attr.signature = (long) opts->signature; 397 attr.signature_size = opts->signature_sz; 398 #else 399 if (opts->signature || opts->signature_sz) 400 pr_warn("signatures are not supported from bpf_preload\n"); 401 #endif 402 attr.keyring_id = opts->keyring_id; 403 memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); 404 attr.fd_array = (long) &map_fd; 405 attr.log_level = opts->ctx->log_level; 406 attr.log_size = opts->ctx->log_size; 407 attr.log_buf = opts->ctx->log_buf; 408 attr.prog_flags = BPF_F_SLEEPABLE; 409 err = prog_fd = skel_sys_bpf(BPF_PROG_LOAD, &attr, prog_load_attr_sz); 410 if (prog_fd < 0) { 411 opts->errstr = "failed to load loader prog"; 412 set_err; 413 goto out; 414 } 415 416 memset(&attr, 0, test_run_attr_sz); 417 attr.test.prog_fd = prog_fd; 418 attr.test.ctx_in = (long) opts->ctx; 419 attr.test.ctx_size_in = opts->ctx->sz; 420 err = skel_sys_bpf(BPF_PROG_RUN, &attr, test_run_attr_sz); 421 if (err < 0 || (int)attr.test.retval < 0) { 422 if (err < 0) { 423 opts->errstr = "failed to execute loader prog"; 424 set_err; 425 } else { 426 opts->errstr = "error returned by loader prog"; 427 err = (int)attr.test.retval; 428 #ifndef __KERNEL__ 429 errno = -err; 430 #endif 431 } 432 goto out; 433 } 434 err = 0; 435 out: 436 if (map_fd >= 0) 437 close(map_fd); 438 if (prog_fd >= 0) 439 close(prog_fd); 440 return err; 441 } 442 443 #endif 444