1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2019 Facebook */ 3 4 #ifndef _GNU_SOURCE 5 #define _GNU_SOURCE 6 #endif 7 #include <ctype.h> 8 #include <errno.h> 9 #include <fcntl.h> 10 #include <libgen.h> 11 #include <linux/err.h> 12 #include <stdbool.h> 13 #include <stdio.h> 14 #include <string.h> 15 #include <unistd.h> 16 #include <bpf/bpf.h> 17 #include <bpf/libbpf.h> 18 #include <bpf/libbpf_internal.h> 19 #include <sys/types.h> 20 #include <sys/stat.h> 21 #include <sys/mman.h> 22 #include <bpf/btf.h> 23 24 #include "json_writer.h" 25 #include "main.h" 26 27 #define MAX_OBJ_NAME_LEN 64 28 29 static void sanitize_identifier(char *name) 30 { 31 int i; 32 33 for (i = 0; name[i]; i++) 34 if (!isalnum(name[i]) && name[i] != '_') 35 name[i] = '_'; 36 } 37 38 static bool str_has_prefix(const char *str, const char *prefix) 39 { 40 return strncmp(str, prefix, strlen(prefix)) == 0; 41 } 42 43 static bool str_has_suffix(const char *str, const char *suffix) 44 { 45 size_t i, n1 = strlen(str), n2 = strlen(suffix); 46 47 if (n1 < n2) 48 return false; 49 50 for (i = 0; i < n2; i++) { 51 if (str[n1 - i - 1] != suffix[n2 - i - 1]) 52 return false; 53 } 54 55 return true; 56 } 57 58 static const struct btf_type * 59 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id) 60 { 61 const struct btf_type *t; 62 63 t = skip_mods_and_typedefs(btf, id, NULL); 64 if (!btf_is_ptr(t)) 65 return NULL; 66 67 t = skip_mods_and_typedefs(btf, t->type, res_id); 68 69 return btf_is_func_proto(t) ? t : NULL; 70 } 71 72 static void get_obj_name(char *name, const char *file) 73 { 74 char file_copy[PATH_MAX]; 75 76 /* Using basename() POSIX version to be more portable. */ 77 strncpy(file_copy, file, PATH_MAX - 1)[PATH_MAX - 1] = '\0'; 78 strncpy(name, basename(file_copy), MAX_OBJ_NAME_LEN - 1)[MAX_OBJ_NAME_LEN - 1] = '\0'; 79 if (str_has_suffix(name, ".o")) 80 name[strlen(name) - 2] = '\0'; 81 sanitize_identifier(name); 82 } 83 84 static void get_header_guard(char *guard, const char *obj_name, const char *suffix) 85 { 86 int i; 87 88 sprintf(guard, "__%s_%s__", obj_name, suffix); 89 for (i = 0; guard[i]; i++) 90 guard[i] = toupper(guard[i]); 91 } 92 93 static bool get_map_ident(const struct bpf_map *map, char *buf, size_t buf_sz) 94 { 95 static const char *sfxs[] = { ".data", ".rodata", ".bss", ".kconfig" }; 96 const char *name = bpf_map__name(map); 97 int i, n; 98 99 if (!bpf_map__is_internal(map)) { 100 snprintf(buf, buf_sz, "%s", name); 101 return true; 102 } 103 104 if (bpf_map__type(map) == BPF_MAP_TYPE_PERCPU_ARRAY) { 105 snprintf(buf, buf_sz, "%s", name + 1); 106 sanitize_identifier(buf); 107 return true; 108 } 109 110 for (i = 0, n = ARRAY_SIZE(sfxs); i < n; i++) { 111 const char *sfx = sfxs[i], *p; 112 113 p = strstr(name, sfx); 114 if (p) { 115 snprintf(buf, buf_sz, "%s", p + 1); 116 sanitize_identifier(buf); 117 return true; 118 } 119 } 120 121 return false; 122 } 123 124 static bool get_datasec_ident(const char *sec_name, char *buf, size_t buf_sz) 125 { 126 static const char *pfxs[] = { ".data", ".rodata", ".bss", ".percpu", ".kconfig" }; 127 int i, n; 128 129 /* recognize hard coded LLVM section name */ 130 if (strcmp(sec_name, ".addr_space.1") == 0) { 131 /* this is the name to use in skeleton */ 132 snprintf(buf, buf_sz, "arena"); 133 return true; 134 } 135 for (i = 0, n = ARRAY_SIZE(pfxs); i < n; i++) { 136 const char *pfx = pfxs[i]; 137 138 if (str_has_prefix(sec_name, pfx)) { 139 snprintf(buf, buf_sz, "%s", sec_name + 1); 140 sanitize_identifier(buf); 141 return true; 142 } 143 } 144 145 return false; 146 } 147 148 static void codegen_btf_dump_printf(void *ctx, const char *fmt, va_list args) 149 { 150 vprintf(fmt, args); 151 } 152 153 static int codegen_datasec_def(struct bpf_object *obj, 154 struct btf *btf, 155 struct btf_dump *d, 156 const struct btf_type *sec, 157 const char *obj_name) 158 { 159 const char *sec_name = btf__name_by_offset(btf, sec->name_off); 160 const struct btf_var_secinfo *sec_var = btf_var_secinfos(sec); 161 int i, err, off = 0, pad_cnt = 0, vlen = btf_vlen(sec); 162 char var_ident[256], sec_ident[256]; 163 bool strip_mods = false; 164 165 if (!get_datasec_ident(sec_name, sec_ident, sizeof(sec_ident))) 166 return 0; 167 168 if (strcmp(sec_name, ".kconfig") != 0) 169 strip_mods = true; 170 171 printf(" struct %s__%s {\n", obj_name, sec_ident); 172 for (i = 0; i < vlen; i++, sec_var++) { 173 const struct btf_type *var = btf__type_by_id(btf, sec_var->type); 174 const char *var_name = btf__name_by_offset(btf, var->name_off); 175 DECLARE_LIBBPF_OPTS(btf_dump_emit_type_decl_opts, opts, 176 .field_name = var_ident, 177 .indent_level = 2, 178 .strip_mods = strip_mods, 179 ); 180 int need_off = sec_var->offset, align_off, align; 181 __u32 var_type_id = var->type; 182 183 /* static variables are not exposed through BPF skeleton */ 184 if (btf_var(var)->linkage == BTF_VAR_STATIC) 185 continue; 186 187 if (off > need_off) { 188 p_err("Something is wrong for %s's variable #%d: need offset %d, already at %d.\n", 189 sec_name, i, need_off, off); 190 return -EINVAL; 191 } 192 193 align = btf__align_of(btf, var->type); 194 if (align <= 0) { 195 p_err("Failed to determine alignment of variable '%s': %d", 196 var_name, align); 197 return -EINVAL; 198 } 199 /* Assume 32-bit architectures when generating data section 200 * struct memory layout. Given bpftool can't know which target 201 * host architecture it's emitting skeleton for, we need to be 202 * conservative and assume 32-bit one to ensure enough padding 203 * bytes are generated for pointer and long types. This will 204 * still work correctly for 64-bit architectures, because in 205 * the worst case we'll generate unnecessary padding field, 206 * which on 64-bit architectures is not strictly necessary and 207 * would be handled by natural 8-byte alignment. But it still 208 * will be a correct memory layout, based on recorded offsets 209 * in BTF. 210 */ 211 if (align > 4) 212 align = 4; 213 214 align_off = (off + align - 1) / align * align; 215 if (align_off != need_off) { 216 printf("\t\tchar __pad%d[%d];\n", 217 pad_cnt, need_off - off); 218 pad_cnt++; 219 } 220 221 /* sanitize variable name, e.g., for static vars inside 222 * a function, it's name is '<function name>.<variable name>', 223 * which we'll turn into a '<function name>_<variable name>' 224 */ 225 var_ident[0] = '\0'; 226 strncat(var_ident, var_name, sizeof(var_ident) - 1); 227 sanitize_identifier(var_ident); 228 229 printf("\t\t"); 230 err = btf_dump__emit_type_decl(d, var_type_id, &opts); 231 if (err) 232 return err; 233 printf(";\n"); 234 235 off = sec_var->offset + sec_var->size; 236 } 237 printf(" } *%s;\n", sec_ident); 238 return 0; 239 } 240 241 static const struct btf_type *find_type_for_map(struct btf *btf, const char *map_ident) 242 { 243 int n = btf__type_cnt(btf), i; 244 char sec_ident[256]; 245 246 for (i = 1; i < n; i++) { 247 const struct btf_type *t = btf__type_by_id(btf, i); 248 const char *name; 249 250 if (!btf_is_datasec(t)) 251 continue; 252 253 name = btf__str_by_offset(btf, t->name_off); 254 if (!get_datasec_ident(name, sec_ident, sizeof(sec_ident))) 255 continue; 256 257 if (strcmp(sec_ident, map_ident) == 0) 258 return t; 259 } 260 return NULL; 261 } 262 263 static bool is_skel_data(const struct bpf_map *map, char *buf, size_t sz) 264 { 265 size_t tmp_sz; 266 267 if (bpf_map__type(map) == BPF_MAP_TYPE_ARENA && bpf_map__initial_value(map, &tmp_sz)) { 268 snprintf(buf, sz, "arena"); 269 return true; 270 } 271 272 if (!bpf_map__is_internal(map)) 273 return false; 274 275 if (!get_map_ident(map, buf, sz)) 276 return false; 277 278 if (bpf_map__map_flags(map) & BPF_F_MMAPABLE) 279 return true; 280 281 if (bpf_map__type(map) == BPF_MAP_TYPE_PERCPU_ARRAY) 282 return bpf_map__btf_value_type_id(map) != 0; 283 284 return false; 285 } 286 287 static bool is_mmapable_map(const struct bpf_map *map, char *buf, size_t sz) 288 { 289 return is_skel_data(map, buf, sz) && bpf_map__type(map) != BPF_MAP_TYPE_PERCPU_ARRAY; 290 } 291 292 static int codegen_datasecs(struct bpf_object *obj, const char *obj_name) 293 { 294 struct btf *btf = bpf_object__btf(obj); 295 struct btf_dump *d; 296 struct bpf_map *map; 297 const struct btf_type *sec; 298 char map_ident[256]; 299 int err = 0; 300 301 d = btf_dump__new(btf, codegen_btf_dump_printf, NULL, NULL); 302 if (!d) 303 return -errno; 304 305 bpf_object__for_each_map(map, obj) { 306 /* only generate definitions for memory-mapped internal maps */ 307 if (!is_skel_data(map, map_ident, sizeof(map_ident))) 308 continue; 309 310 sec = find_type_for_map(btf, map_ident); 311 312 /* In some cases (e.g., sections like .rodata.cst16 containing 313 * compiler allocated string constants only) there will be 314 * special internal maps with no corresponding DATASEC BTF 315 * type. In such case, generate empty structs for each such 316 * map. It will still be memory-mapped and its contents 317 * accessible from user-space through BPF skeleton. 318 */ 319 if (!sec) { 320 printf(" struct %s__%s {\n", obj_name, map_ident); 321 printf(" } *%s;\n", map_ident); 322 } else { 323 err = codegen_datasec_def(obj, btf, d, sec, obj_name); 324 if (err) 325 goto out; 326 } 327 } 328 329 330 out: 331 btf_dump__free(d); 332 return err; 333 } 334 335 static bool btf_is_ptr_to_func_proto(const struct btf *btf, 336 const struct btf_type *v) 337 { 338 return btf_is_ptr(v) && btf_is_func_proto(btf__type_by_id(btf, v->type)); 339 } 340 341 static int codegen_subskel_datasecs(struct bpf_object *obj, const char *obj_name) 342 { 343 struct btf *btf = bpf_object__btf(obj); 344 struct btf_dump *d; 345 struct bpf_map *map; 346 const struct btf_type *sec, *var; 347 const struct btf_var_secinfo *sec_var; 348 int i, err = 0, vlen; 349 char map_ident[256], sec_ident[256]; 350 bool strip_mods = false, needs_typeof = false; 351 const char *sec_name, *var_name; 352 __u32 var_type_id; 353 354 d = btf_dump__new(btf, codegen_btf_dump_printf, NULL, NULL); 355 if (!d) 356 return -errno; 357 358 bpf_object__for_each_map(map, obj) { 359 /* only generate definitions for memory-mapped internal maps */ 360 if (!is_mmapable_map(map, map_ident, sizeof(map_ident))) 361 continue; 362 363 sec = find_type_for_map(btf, map_ident); 364 if (!sec) 365 continue; 366 367 sec_name = btf__name_by_offset(btf, sec->name_off); 368 if (!get_datasec_ident(sec_name, sec_ident, sizeof(sec_ident))) 369 continue; 370 371 strip_mods = strcmp(sec_name, ".kconfig") != 0; 372 printf(" struct %s__%s {\n", obj_name, sec_ident); 373 374 sec_var = btf_var_secinfos(sec); 375 vlen = btf_vlen(sec); 376 for (i = 0; i < vlen; i++, sec_var++) { 377 DECLARE_LIBBPF_OPTS(btf_dump_emit_type_decl_opts, opts, 378 .indent_level = 2, 379 .strip_mods = strip_mods, 380 /* we'll print the name separately */ 381 .field_name = "", 382 ); 383 384 var = btf__type_by_id(btf, sec_var->type); 385 var_name = btf__name_by_offset(btf, var->name_off); 386 var_type_id = var->type; 387 388 /* static variables are not exposed through BPF skeleton */ 389 if (btf_var(var)->linkage == BTF_VAR_STATIC) 390 continue; 391 392 /* The datasec member has KIND_VAR but we want the 393 * underlying type of the variable (e.g. KIND_INT). 394 */ 395 var = skip_mods_and_typedefs(btf, var->type, NULL); 396 397 printf("\t\t"); 398 /* Func and array members require special handling. 399 * Instead of producing `typename *var`, they produce 400 * `typeof(typename) *var`. This allows us to keep a 401 * similar syntax where the identifier is just prefixed 402 * by *, allowing us to ignore C declaration minutiae. 403 */ 404 needs_typeof = btf_is_array(var) || btf_is_ptr_to_func_proto(btf, var); 405 if (needs_typeof) 406 printf("__typeof__("); 407 408 err = btf_dump__emit_type_decl(d, var_type_id, &opts); 409 if (err) 410 goto out; 411 412 if (needs_typeof) 413 printf(")"); 414 415 printf(" *%s;\n", var_name); 416 } 417 printf(" } %s;\n", sec_ident); 418 } 419 420 out: 421 btf_dump__free(d); 422 return err; 423 } 424 425 static void codegen(const char *template, ...) 426 { 427 const char *src, *end; 428 int skip_tabs = 0, n; 429 char *s, *dst; 430 va_list args; 431 char c; 432 433 n = strlen(template); 434 s = malloc(n + 1); 435 if (!s) 436 exit(-1); 437 src = template; 438 dst = s; 439 440 /* find out "baseline" indentation to skip */ 441 while ((c = *src++)) { 442 if (c == '\t') { 443 skip_tabs++; 444 } else if (c == '\n') { 445 break; 446 } else { 447 p_err("unrecognized character at pos %td in template '%s': '%c'", 448 src - template - 1, template, c); 449 free(s); 450 exit(-1); 451 } 452 } 453 454 while (*src) { 455 /* skip baseline indentation tabs */ 456 for (n = skip_tabs; n > 0; n--, src++) { 457 if (*src != '\t') { 458 p_err("not enough tabs at pos %td in template '%s'", 459 src - template - 1, template); 460 free(s); 461 exit(-1); 462 } 463 } 464 /* trim trailing whitespace */ 465 end = strchrnul(src, '\n'); 466 for (n = end - src; n > 0 && isspace(src[n - 1]); n--) 467 ; 468 memcpy(dst, src, n); 469 dst += n; 470 if (*end) 471 *dst++ = '\n'; 472 src = *end ? end + 1 : end; 473 } 474 *dst++ = '\0'; 475 476 /* print out using adjusted template */ 477 va_start(args, template); 478 n = vprintf(s, args); 479 va_end(args); 480 481 free(s); 482 } 483 484 static void print_hex(const char *data, int data_sz) 485 { 486 int i, len; 487 488 for (i = 0, len = 0; i < data_sz; i++) { 489 int w = data[i] ? 4 : 2; 490 491 len += w; 492 if (len > 78) { 493 printf("\\\n"); 494 len = w; 495 } 496 if (!data[i]) 497 printf("\\0"); 498 else 499 printf("\\x%02x", (unsigned char)data[i]); 500 } 501 } 502 503 /* Emit type size asserts for all top-level fields in memory-mapped internal maps. */ 504 static void codegen_asserts(struct bpf_object *obj, const char *obj_name) 505 { 506 struct btf *btf = bpf_object__btf(obj); 507 struct bpf_map *map; 508 struct btf_var_secinfo *sec_var; 509 int i, vlen; 510 const struct btf_type *sec; 511 char map_ident[256], var_ident[256]; 512 513 if (!btf) 514 return; 515 516 codegen("\ 517 \n\ 518 __attribute__((unused)) static void \n\ 519 %1$s__assert(struct %1$s *s __attribute__((unused))) \n\ 520 { \n\ 521 #ifdef __cplusplus \n\ 522 #define _Static_assert static_assert \n\ 523 #endif \n\ 524 ", obj_name); 525 526 bpf_object__for_each_map(map, obj) { 527 if (!is_skel_data(map, map_ident, sizeof(map_ident))) 528 continue; 529 530 sec = find_type_for_map(btf, map_ident); 531 if (!sec) { 532 /* best effort, couldn't find the type for this map */ 533 continue; 534 } 535 536 sec_var = btf_var_secinfos(sec); 537 vlen = btf_vlen(sec); 538 539 for (i = 0; i < vlen; i++, sec_var++) { 540 const struct btf_type *var = btf__type_by_id(btf, sec_var->type); 541 const char *var_name = btf__name_by_offset(btf, var->name_off); 542 long var_size; 543 544 /* static variables are not exposed through BPF skeleton */ 545 if (btf_var(var)->linkage == BTF_VAR_STATIC) 546 continue; 547 548 var_size = btf__resolve_size(btf, var->type); 549 if (var_size < 0) 550 continue; 551 552 var_ident[0] = '\0'; 553 strncat(var_ident, var_name, sizeof(var_ident) - 1); 554 sanitize_identifier(var_ident); 555 556 printf("\t_Static_assert(sizeof(s->%s->%s) == %ld, \"unexpected size of '%s'\");\n", 557 map_ident, var_ident, var_size, var_ident); 558 } 559 } 560 codegen("\ 561 \n\ 562 #ifdef __cplusplus \n\ 563 #undef _Static_assert \n\ 564 #endif \n\ 565 } \n\ 566 "); 567 } 568 569 static void codegen_attach_detach(struct bpf_object *obj, const char *obj_name) 570 { 571 struct bpf_program *prog; 572 573 bpf_object__for_each_program(prog, obj) { 574 const char *tp_name; 575 576 codegen("\ 577 \n\ 578 \n\ 579 static inline int \n\ 580 %1$s__%2$s__attach(struct %1$s *skel) \n\ 581 { \n\ 582 int prog_fd = skel->progs.%2$s.prog_fd; \n\ 583 ", obj_name, bpf_program__name(prog)); 584 585 switch (bpf_program__type(prog)) { 586 case BPF_PROG_TYPE_RAW_TRACEPOINT: 587 tp_name = strchr(bpf_program__section_name(prog), '/') + 1; 588 printf("\tint fd = skel_raw_tracepoint_open(\"%s\", prog_fd);\n", tp_name); 589 break; 590 case BPF_PROG_TYPE_TRACING: 591 case BPF_PROG_TYPE_LSM: 592 if (bpf_program__expected_attach_type(prog) == BPF_TRACE_ITER) 593 printf("\tint fd = skel_link_create(prog_fd, 0, BPF_TRACE_ITER);\n"); 594 else 595 printf("\tint fd = skel_raw_tracepoint_open(NULL, prog_fd);\n"); 596 break; 597 default: 598 printf("\tint fd = ((void)prog_fd, 0); /* auto-attach not supported */\n"); 599 break; 600 } 601 codegen("\ 602 \n\ 603 \n\ 604 if (fd > 0) \n\ 605 skel->links.%1$s_fd = fd; \n\ 606 return fd; \n\ 607 } \n\ 608 ", bpf_program__name(prog)); 609 } 610 611 codegen("\ 612 \n\ 613 \n\ 614 static inline int \n\ 615 %1$s__attach(struct %1$s *skel) \n\ 616 { \n\ 617 int ret = 0; \n\ 618 \n\ 619 ", obj_name); 620 621 bpf_object__for_each_program(prog, obj) { 622 codegen("\ 623 \n\ 624 ret = ret < 0 ? ret : %1$s__%2$s__attach(skel); \n\ 625 ", obj_name, bpf_program__name(prog)); 626 } 627 628 codegen("\ 629 \n\ 630 return ret < 0 ? ret : 0; \n\ 631 } \n\ 632 \n\ 633 static inline void \n\ 634 %1$s__detach(struct %1$s *skel) \n\ 635 { \n\ 636 ", obj_name); 637 638 bpf_object__for_each_program(prog, obj) { 639 codegen("\ 640 \n\ 641 skel_closenz(skel->links.%1$s_fd); \n\ 642 ", bpf_program__name(prog)); 643 } 644 645 codegen("\ 646 \n\ 647 } \n\ 648 "); 649 } 650 651 static void codegen_destroy(struct bpf_object *obj, const char *obj_name) 652 { 653 struct bpf_program *prog; 654 struct bpf_map *map; 655 char ident[256]; 656 657 codegen("\ 658 \n\ 659 static void \n\ 660 %1$s__destroy(struct %1$s *skel) \n\ 661 { \n\ 662 if (!skel) \n\ 663 return; \n\ 664 %1$s__detach(skel); \n\ 665 ", 666 obj_name); 667 668 bpf_object__for_each_program(prog, obj) { 669 codegen("\ 670 \n\ 671 skel_closenz(skel->progs.%1$s.prog_fd); \n\ 672 ", bpf_program__name(prog)); 673 } 674 675 bpf_object__for_each_map(map, obj) { 676 if (!get_map_ident(map, ident, sizeof(ident))) 677 continue; 678 if (is_skel_data(map, ident, sizeof(ident))) 679 printf("\tskel_free_map_data(skel->%1$s, skel->maps.%1$s.initial_value, %2$u, %3$u);\n", 680 ident, bpf_map__value_size(map), bpf_map__max_entries(map)); 681 codegen("\ 682 \n\ 683 skel_closenz(skel->maps.%1$s.map_fd); \n\ 684 ", ident); 685 } 686 codegen("\ 687 \n\ 688 skel_free(skel); \n\ 689 } \n\ 690 ", 691 obj_name); 692 } 693 694 static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *header_guard) 695 { 696 DECLARE_LIBBPF_OPTS(gen_loader_opts, opts); 697 struct bpf_load_and_run_opts sopts = {}; 698 char sig_buf[MAX_SIG_SIZE]; 699 __u8 prog_sha[SHA256_DIGEST_LENGTH]; 700 struct bpf_map *map; 701 702 char ident[256]; 703 int err = 0; 704 705 if (sign_progs) 706 opts.gen_hash = true; 707 708 err = bpf_object__gen_loader(obj, &opts); 709 if (err) 710 return err; 711 712 err = bpf_object__load(obj); 713 if (err) { 714 p_err("failed to load object file"); 715 goto out; 716 } 717 718 /* If there was no error during load then gen_loader_opts 719 * are populated with the loader program. 720 */ 721 722 /* finish generating 'struct skel' */ 723 codegen("\ 724 \n\ 725 }; \n\ 726 ", obj_name); 727 728 729 codegen_attach_detach(obj, obj_name); 730 731 codegen_destroy(obj, obj_name); 732 733 codegen("\ 734 \n\ 735 static inline struct %1$s * \n\ 736 %1$s__open(void) \n\ 737 { \n\ 738 struct %1$s *skel; \n\ 739 \n\ 740 skel = (struct %1$s *)skel_alloc(sizeof(*skel)); \n\ 741 if (!skel) \n\ 742 goto cleanup; \n\ 743 skel->ctx.sz = (char *)&skel->links - (char *)skel; \n\ 744 ", 745 obj_name, opts.data_sz); 746 bpf_object__for_each_map(map, obj) { 747 const void *mmap_data = NULL; 748 size_t mmap_size = 0; 749 750 if (!is_skel_data(map, ident, sizeof(ident))) 751 continue; 752 753 codegen("\ 754 \n\ 755 { \n\ 756 static const char data[] __attribute__((__aligned__(8))) = \"\\\n\ 757 "); 758 mmap_data = bpf_map__initial_value(map, &mmap_size); 759 print_hex(mmap_data, mmap_size); 760 codegen("\ 761 \n\ 762 \"; \n\ 763 \n\ 764 skel->%1$s = (__typeof__(skel->%1$s))skel_prep_map_data((void *)data, %2$u, %3$u,\n\ 765 sizeof(data) - 1);\n\ 766 if (!skel->%1$s) \n\ 767 goto cleanup; \n\ 768 skel->maps.%1$s.initial_value = (__u64) (long) skel->%1$s;\n\ 769 } \n\ 770 ", ident, bpf_map__value_size(map), bpf_map__max_entries(map)); 771 } 772 codegen("\ 773 \n\ 774 return skel; \n\ 775 cleanup: \n\ 776 %1$s__destroy(skel); \n\ 777 return NULL; \n\ 778 } \n\ 779 \n\ 780 static inline int \n\ 781 %1$s__load(struct %1$s *skel) \n\ 782 { \n\ 783 struct bpf_load_and_run_opts opts = {}; \n\ 784 int err; \n\ 785 static const char opts_data[] __attribute__((__aligned__(8))) = \"\\\n\ 786 ", 787 obj_name); 788 print_hex(opts.data, opts.data_sz); 789 codegen("\ 790 \n\ 791 \"; \n\ 792 static const char opts_insn[] __attribute__((__aligned__(8))) = \"\\\n\ 793 "); 794 print_hex(opts.insns, opts.insns_sz); 795 codegen("\ 796 \n\ 797 \";\n"); 798 799 if (sign_progs) { 800 sopts.insns = opts.insns; 801 sopts.insns_sz = opts.insns_sz; 802 sopts.data = opts.data; 803 sopts.data_sz = opts.data_sz; 804 sopts.excl_prog_hash = prog_sha; 805 sopts.excl_prog_hash_sz = sizeof(prog_sha); 806 sopts.signature = sig_buf; 807 sopts.signature_sz = MAX_SIG_SIZE; 808 809 err = bpftool_prog_sign(&sopts); 810 if (err < 0) { 811 p_err("failed to sign program"); 812 goto out; 813 } 814 815 codegen("\ 816 \n\ 817 static const char opts_sig[] __attribute__((__aligned__(8))) = \"\\\n\ 818 "); 819 print_hex((const void *)sig_buf, sopts.signature_sz); 820 codegen("\ 821 \n\ 822 \";\n"); 823 824 codegen("\ 825 \n\ 826 static const char opts_excl_hash[] __attribute__((__aligned__(8))) = \"\\\n\ 827 "); 828 print_hex((const void *)prog_sha, sizeof(prog_sha)); 829 codegen("\ 830 \n\ 831 \";\n"); 832 833 codegen("\ 834 \n\ 835 opts.signature = (void *)opts_sig; \n\ 836 opts.signature_sz = sizeof(opts_sig) - 1; \n\ 837 opts.excl_prog_hash = (void *)opts_excl_hash; \n\ 838 opts.excl_prog_hash_sz = sizeof(opts_excl_hash) - 1; \n\ 839 opts.keyring_id = skel->keyring_id; \n\ 840 "); 841 } 842 843 codegen("\ 844 \n\ 845 opts.ctx = (struct bpf_loader_ctx *)skel; \n\ 846 opts.data_sz = sizeof(opts_data) - 1; \n\ 847 opts.data = (void *)opts_data; \n\ 848 opts.insns_sz = sizeof(opts_insn) - 1; \n\ 849 opts.insns = (void *)opts_insn; \n\ 850 \n\ 851 err = bpf_load_and_run(&opts); \n\ 852 if (err < 0) \n\ 853 return err; \n\ 854 "); 855 bpf_object__for_each_map(map, obj) { 856 const char *mmap_flags; 857 858 if (!is_skel_data(map, ident, sizeof(ident))) 859 continue; 860 861 if (bpf_map__type(map) == BPF_MAP_TYPE_PERCPU_ARRAY) { 862 codegen("\ 863 \n\ 864 err = skel_protect_map_data(skel->%1$s, &skel->maps.%1$s.initial_value, %2$u, %3$u);\n\ 865 if (err) \n\ 866 return err; \n\ 867 #ifdef __KERNEL__ \n\ 868 skel->%1$s = NULL; \n\ 869 #endif \n\ 870 ", 871 ident, bpf_map__value_size(map), bpf_map__max_entries(map)); 872 continue; 873 } 874 875 if (bpf_map__map_flags(map) & BPF_F_RDONLY_PROG) 876 mmap_flags = "PROT_READ"; 877 else 878 mmap_flags = "PROT_READ | PROT_WRITE"; 879 880 codegen("\ 881 \n\ 882 skel->%1$s = (__typeof__(skel->%1$s))skel_finalize_map_data(&skel->maps.%1$s.initial_value,\n\ 883 %2$u, %3$u, %4$s, skel->maps.%1$s.map_fd);\n\ 884 if (!skel->%1$s) \n\ 885 return -ENOMEM; \n\ 886 ", 887 ident, bpf_map__value_size(map), bpf_map__max_entries(map), mmap_flags); 888 } 889 codegen("\ 890 \n\ 891 return 0; \n\ 892 } \n\ 893 \n\ 894 static inline struct %1$s * \n\ 895 %1$s__open_and_load(void) \n\ 896 { \n\ 897 struct %1$s *skel; \n\ 898 \n\ 899 skel = %1$s__open(); \n\ 900 if (!skel) \n\ 901 return NULL; \n\ 902 if (%1$s__load(skel)) { \n\ 903 %1$s__destroy(skel); \n\ 904 return NULL; \n\ 905 } \n\ 906 return skel; \n\ 907 } \n\ 908 \n\ 909 ", obj_name); 910 911 codegen_asserts(obj, obj_name); 912 913 codegen("\ 914 \n\ 915 \n\ 916 #endif /* %s */ \n\ 917 ", 918 header_guard); 919 err = 0; 920 out: 921 return err; 922 } 923 924 static void 925 codegen_maps_skeleton(struct bpf_object *obj, size_t map_cnt, bool mmaped, bool populate_links) 926 { 927 struct bpf_map *map; 928 char ident[256]; 929 size_t i, map_sz; 930 931 if (!map_cnt) 932 return; 933 934 /* for backward compatibility with old libbpf versions that don't 935 * handle new BPF skeleton with new struct bpf_map_skeleton definition 936 * that includes link field, avoid specifying new increased size, 937 * unless we absolutely have to (i.e., if there are struct_ops maps 938 * present) 939 */ 940 map_sz = offsetof(struct bpf_map_skeleton, link); 941 if (populate_links) { 942 bpf_object__for_each_map(map, obj) { 943 if (bpf_map__type(map) == BPF_MAP_TYPE_STRUCT_OPS) { 944 map_sz = sizeof(struct bpf_map_skeleton); 945 break; 946 } 947 } 948 } 949 950 codegen("\ 951 \n\ 952 \n\ 953 /* maps */ \n\ 954 s->map_cnt = %zu; \n\ 955 s->map_skel_sz = %zu; \n\ 956 s->maps = (struct bpf_map_skeleton *)calloc(s->map_cnt,\n\ 957 sizeof(*s->maps) > %zu ? sizeof(*s->maps) : %zu);\n\ 958 if (!s->maps) { \n\ 959 err = -ENOMEM; \n\ 960 goto err; \n\ 961 } \n\ 962 ", 963 map_cnt, map_sz, map_sz, map_sz 964 ); 965 i = 0; 966 bpf_object__for_each_map(map, obj) { 967 if (!get_map_ident(map, ident, sizeof(ident))) 968 continue; 969 970 codegen("\ 971 \n\ 972 \n\ 973 map = (struct bpf_map_skeleton *)((char *)s->maps + %zu * s->map_skel_sz);\n\ 974 map->name = \"%s\"; \n\ 975 map->map = &obj->maps.%s; \n\ 976 ", 977 i, bpf_map__name(map), ident); 978 if (mmaped && is_skel_data(map, ident, sizeof(ident))) { 979 printf("\tmap->mmaped = (void **)&obj->%s;\n", ident); 980 } 981 982 if (populate_links && bpf_map__type(map) == BPF_MAP_TYPE_STRUCT_OPS) { 983 codegen("\ 984 \n\ 985 map->link = &obj->links.%s; \n\ 986 ", ident); 987 } 988 i++; 989 } 990 } 991 992 static void 993 codegen_progs_skeleton(struct bpf_object *obj, size_t prog_cnt, bool populate_links) 994 { 995 struct bpf_program *prog; 996 int i; 997 998 if (!prog_cnt) 999 return; 1000 1001 codegen("\ 1002 \n\ 1003 \n\ 1004 /* programs */ \n\ 1005 s->prog_cnt = %zu; \n\ 1006 s->prog_skel_sz = sizeof(*s->progs); \n\ 1007 s->progs = (struct bpf_prog_skeleton *)calloc(s->prog_cnt, s->prog_skel_sz);\n\ 1008 if (!s->progs) { \n\ 1009 err = -ENOMEM; \n\ 1010 goto err; \n\ 1011 } \n\ 1012 ", 1013 prog_cnt 1014 ); 1015 i = 0; 1016 bpf_object__for_each_program(prog, obj) { 1017 codegen("\ 1018 \n\ 1019 \n\ 1020 s->progs[%1$zu].name = \"%2$s\"; \n\ 1021 s->progs[%1$zu].prog = &obj->progs.%2$s;\n\ 1022 ", 1023 i, bpf_program__name(prog)); 1024 1025 if (populate_links) { 1026 codegen("\ 1027 \n\ 1028 s->progs[%1$zu].link = &obj->links.%2$s;\n\ 1029 ", 1030 i, bpf_program__name(prog)); 1031 } 1032 i++; 1033 } 1034 } 1035 1036 static int walk_st_ops_shadow_vars(struct btf *btf, const char *ident, 1037 const struct btf_type *map_type, __u32 map_type_id) 1038 { 1039 LIBBPF_OPTS(btf_dump_emit_type_decl_opts, opts, .indent_level = 3); 1040 const struct btf_type *member_type; 1041 __u32 offset, next_offset = 0; 1042 const struct btf_member *m; 1043 struct btf_dump *d = NULL; 1044 const char *member_name; 1045 __u32 member_type_id; 1046 int i, err = 0, n; 1047 int size; 1048 1049 d = btf_dump__new(btf, codegen_btf_dump_printf, NULL, NULL); 1050 if (!d) 1051 return -errno; 1052 1053 n = btf_vlen(map_type); 1054 for (i = 0, m = btf_members(map_type); i < n; i++, m++) { 1055 member_type = skip_mods_and_typedefs(btf, m->type, &member_type_id); 1056 member_name = btf__name_by_offset(btf, m->name_off); 1057 1058 offset = m->offset / 8; 1059 if (next_offset < offset) 1060 printf("\t\t\tchar __padding_%d[%u];\n", i, offset - next_offset); 1061 1062 switch (btf_kind(member_type)) { 1063 case BTF_KIND_INT: 1064 case BTF_KIND_FLOAT: 1065 case BTF_KIND_ENUM: 1066 case BTF_KIND_ENUM64: 1067 /* scalar type */ 1068 printf("\t\t\t"); 1069 opts.field_name = member_name; 1070 err = btf_dump__emit_type_decl(d, member_type_id, &opts); 1071 if (err) { 1072 p_err("Failed to emit type declaration for %s: %d", member_name, err); 1073 goto out; 1074 } 1075 printf(";\n"); 1076 1077 size = btf__resolve_size(btf, member_type_id); 1078 if (size < 0) { 1079 p_err("Failed to resolve size of %s: %d\n", member_name, size); 1080 err = size; 1081 goto out; 1082 } 1083 1084 next_offset = offset + size; 1085 break; 1086 1087 case BTF_KIND_PTR: 1088 if (resolve_func_ptr(btf, m->type, NULL)) { 1089 /* Function pointer */ 1090 printf("\t\t\tstruct bpf_program *%s;\n", member_name); 1091 1092 next_offset = offset + sizeof(void *); 1093 break; 1094 } 1095 /* All pointer types are unsupported except for 1096 * function pointers. 1097 */ 1098 fallthrough; 1099 1100 default: 1101 /* Unsupported types 1102 * 1103 * Types other than scalar types and function 1104 * pointers are currently not supported in order to 1105 * prevent conflicts in the generated code caused 1106 * by multiple definitions. For instance, if the 1107 * struct type FOO is used in a struct_ops map, 1108 * bpftool has to generate definitions for FOO, 1109 * which may result in conflicts if FOO is defined 1110 * in different skeleton files. 1111 */ 1112 size = btf__resolve_size(btf, member_type_id); 1113 if (size < 0) { 1114 p_err("Failed to resolve size of %s: %d\n", member_name, size); 1115 err = size; 1116 goto out; 1117 } 1118 printf("\t\t\tchar __unsupported_%d[%d];\n", i, size); 1119 1120 next_offset = offset + size; 1121 break; 1122 } 1123 } 1124 1125 /* Cannot fail since it must be a struct type */ 1126 size = btf__resolve_size(btf, map_type_id); 1127 if (next_offset < (__u32)size) 1128 printf("\t\t\tchar __padding_end[%u];\n", size - next_offset); 1129 1130 out: 1131 btf_dump__free(d); 1132 1133 return err; 1134 } 1135 1136 /* Generate the pointer of the shadow type for a struct_ops map. 1137 * 1138 * This function adds a pointer of the shadow type for a struct_ops map. 1139 * The members of a struct_ops map can be exported through a pointer to a 1140 * shadow type. The user can access these members through the pointer. 1141 * 1142 * A shadow type includes not all members, only members of some types. 1143 * They are scalar types and function pointers. The function pointers are 1144 * translated to the pointer of the struct bpf_program. The scalar types 1145 * are translated to the original type without any modifiers. 1146 * 1147 * Unsupported types will be translated to a char array to occupy the same 1148 * space as the original field, being renamed as __unsupported_*. The user 1149 * should treat these fields as opaque data. 1150 */ 1151 static int gen_st_ops_shadow_type(const char *obj_name, struct btf *btf, const char *ident, 1152 const struct bpf_map *map) 1153 { 1154 const struct btf_type *map_type; 1155 const char *type_name; 1156 __u32 map_type_id; 1157 int err; 1158 1159 map_type_id = bpf_map__btf_value_type_id(map); 1160 if (map_type_id == 0) 1161 return -EINVAL; 1162 map_type = btf__type_by_id(btf, map_type_id); 1163 if (!map_type) 1164 return -EINVAL; 1165 1166 type_name = btf__name_by_offset(btf, map_type->name_off); 1167 1168 printf("\t\tstruct %s__%s__%s {\n", obj_name, ident, type_name); 1169 1170 err = walk_st_ops_shadow_vars(btf, ident, map_type, map_type_id); 1171 if (err) 1172 return err; 1173 1174 printf("\t\t} *%s;\n", ident); 1175 1176 return 0; 1177 } 1178 1179 static int gen_st_ops_shadow(const char *obj_name, struct btf *btf, struct bpf_object *obj) 1180 { 1181 int err, st_ops_cnt = 0; 1182 struct bpf_map *map; 1183 char ident[256]; 1184 1185 if (!btf) 1186 return 0; 1187 1188 /* Generate the pointers to shadow types of 1189 * struct_ops maps. 1190 */ 1191 bpf_object__for_each_map(map, obj) { 1192 if (bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS) 1193 continue; 1194 if (!get_map_ident(map, ident, sizeof(ident))) 1195 continue; 1196 1197 if (st_ops_cnt == 0) /* first struct_ops map */ 1198 printf("\tstruct {\n"); 1199 st_ops_cnt++; 1200 1201 err = gen_st_ops_shadow_type(obj_name, btf, ident, map); 1202 if (err) 1203 return err; 1204 } 1205 1206 if (st_ops_cnt) 1207 printf("\t} struct_ops;\n"); 1208 1209 return 0; 1210 } 1211 1212 /* Generate the code to initialize the pointers of shadow types. */ 1213 static void gen_st_ops_shadow_init(struct btf *btf, struct bpf_object *obj) 1214 { 1215 struct bpf_map *map; 1216 char ident[256]; 1217 1218 if (!btf) 1219 return; 1220 1221 /* Initialize the pointers to_ops shadow types of 1222 * struct_ops maps. 1223 */ 1224 bpf_object__for_each_map(map, obj) { 1225 if (bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS) 1226 continue; 1227 if (!get_map_ident(map, ident, sizeof(ident))) 1228 continue; 1229 codegen("\ 1230 \n\ 1231 obj->struct_ops.%1$s = (__typeof__(obj->struct_ops.%1$s))\n\ 1232 bpf_map__initial_value(obj->maps.%1$s, NULL);\n\ 1233 \n\ 1234 ", ident); 1235 } 1236 } 1237 1238 static int do_skeleton(int argc, char **argv) 1239 { 1240 char header_guard[MAX_OBJ_NAME_LEN + sizeof("__SKEL_H__")]; 1241 size_t map_cnt = 0, prog_cnt = 0, attach_map_cnt = 0, file_sz, mmap_sz; 1242 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts); 1243 char obj_name[MAX_OBJ_NAME_LEN] = "", *obj_data; 1244 struct bpf_object *obj = NULL; 1245 const char *file; 1246 char ident[256]; 1247 struct bpf_program *prog; 1248 int fd, err = -1; 1249 struct bpf_map *map; 1250 struct btf *btf; 1251 struct stat st; 1252 1253 if (!REQ_ARGS(1)) { 1254 usage(); 1255 return -1; 1256 } 1257 file = GET_ARG(); 1258 1259 while (argc) { 1260 if (!REQ_ARGS(2)) 1261 return -1; 1262 1263 if (is_prefix(*argv, "name")) { 1264 NEXT_ARG(); 1265 1266 if (obj_name[0] != '\0') { 1267 p_err("object name already specified"); 1268 return -1; 1269 } 1270 1271 strncpy(obj_name, *argv, MAX_OBJ_NAME_LEN - 1); 1272 obj_name[MAX_OBJ_NAME_LEN - 1] = '\0'; 1273 } else { 1274 p_err("unknown arg %s", *argv); 1275 return -1; 1276 } 1277 1278 NEXT_ARG(); 1279 } 1280 1281 if (argc) { 1282 p_err("extra unknown arguments"); 1283 return -1; 1284 } 1285 1286 if (stat(file, &st)) { 1287 p_err("failed to stat() %s: %s", file, strerror(errno)); 1288 return -1; 1289 } 1290 file_sz = st.st_size; 1291 mmap_sz = roundup(file_sz, sysconf(_SC_PAGE_SIZE)); 1292 fd = open(file, O_RDONLY); 1293 if (fd < 0) { 1294 p_err("failed to open() %s: %s", file, strerror(errno)); 1295 return -1; 1296 } 1297 obj_data = mmap(NULL, mmap_sz, PROT_READ, MAP_PRIVATE, fd, 0); 1298 if (obj_data == MAP_FAILED) { 1299 obj_data = NULL; 1300 p_err("failed to mmap() %s: %s", file, strerror(errno)); 1301 goto out; 1302 } 1303 if (obj_name[0] == '\0') 1304 get_obj_name(obj_name, file); 1305 opts.object_name = obj_name; 1306 if (verifier_logs) 1307 /* log_level1 + log_level2 + stats, but not stable UAPI */ 1308 opts.kernel_log_level = 1 + 2 + 4; 1309 obj = bpf_object__open_mem(obj_data, file_sz, &opts); 1310 if (!obj) { 1311 char err_buf[256]; 1312 1313 err = -errno; 1314 libbpf_strerror(err, err_buf, sizeof(err_buf)); 1315 p_err("failed to open BPF object file: %s", err_buf); 1316 goto out_obj; 1317 } 1318 1319 bpf_object__for_each_map(map, obj) { 1320 if (!get_map_ident(map, ident, sizeof(ident))) { 1321 p_err("ignoring unrecognized internal map '%s'...", 1322 bpf_map__name(map)); 1323 continue; 1324 } 1325 1326 if (bpf_map__type(map) == BPF_MAP_TYPE_STRUCT_OPS) 1327 attach_map_cnt++; 1328 1329 map_cnt++; 1330 } 1331 bpf_object__for_each_program(prog, obj) { 1332 prog_cnt++; 1333 } 1334 1335 get_header_guard(header_guard, obj_name, "SKEL_H"); 1336 if (use_loader) { 1337 codegen("\ 1338 \n\ 1339 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ \n\ 1340 /* THIS FILE IS AUTOGENERATED BY BPFTOOL! */ \n\ 1341 #ifndef %2$s \n\ 1342 #define %2$s \n\ 1343 \n\ 1344 #include <bpf/skel_internal.h> \n\ 1345 \n\ 1346 struct %1$s { \n\ 1347 struct bpf_loader_ctx ctx; \n\ 1348 ", 1349 obj_name, header_guard 1350 ); 1351 } else { 1352 codegen("\ 1353 \n\ 1354 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ \n\ 1355 \n\ 1356 /* THIS FILE IS AUTOGENERATED BY BPFTOOL! */ \n\ 1357 #ifndef %2$s \n\ 1358 #define %2$s \n\ 1359 \n\ 1360 #include <errno.h> \n\ 1361 #include <stdlib.h> \n\ 1362 #include <bpf/libbpf.h> \n\ 1363 \n\ 1364 #define BPF_SKEL_SUPPORTS_MAP_AUTO_ATTACH 1 \n\ 1365 \n\ 1366 struct %1$s { \n\ 1367 struct bpf_object_skeleton *skeleton; \n\ 1368 struct bpf_object *obj; \n\ 1369 ", 1370 obj_name, header_guard 1371 ); 1372 } 1373 1374 if (map_cnt) { 1375 printf("\tstruct {\n"); 1376 bpf_object__for_each_map(map, obj) { 1377 if (!get_map_ident(map, ident, sizeof(ident))) 1378 continue; 1379 if (use_loader) 1380 printf("\t\tstruct bpf_map_desc %s;\n", ident); 1381 else 1382 printf("\t\tstruct bpf_map *%s;\n", ident); 1383 } 1384 printf("\t} maps;\n"); 1385 } 1386 1387 btf = bpf_object__btf(obj); 1388 err = gen_st_ops_shadow(obj_name, btf, obj); 1389 if (err) 1390 goto out; 1391 1392 if (prog_cnt) { 1393 printf("\tstruct {\n"); 1394 bpf_object__for_each_program(prog, obj) { 1395 if (use_loader) 1396 printf("\t\tstruct bpf_prog_desc %s;\n", 1397 bpf_program__name(prog)); 1398 else 1399 printf("\t\tstruct bpf_program *%s;\n", 1400 bpf_program__name(prog)); 1401 } 1402 printf("\t} progs;\n"); 1403 } 1404 1405 if (prog_cnt + attach_map_cnt) { 1406 printf("\tstruct {\n"); 1407 bpf_object__for_each_program(prog, obj) { 1408 if (use_loader) 1409 printf("\t\tint %s_fd;\n", 1410 bpf_program__name(prog)); 1411 else 1412 printf("\t\tstruct bpf_link *%s;\n", 1413 bpf_program__name(prog)); 1414 } 1415 1416 bpf_object__for_each_map(map, obj) { 1417 if (!get_map_ident(map, ident, sizeof(ident))) 1418 continue; 1419 if (bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS) 1420 continue; 1421 1422 if (use_loader) 1423 printf("\t\tint %s_fd;\n", ident); 1424 else 1425 printf("\t\tstruct bpf_link *%s;\n", ident); 1426 } 1427 1428 printf("\t} links;\n"); 1429 } 1430 1431 if (sign_progs) { 1432 codegen("\ 1433 \n\ 1434 __s32 keyring_id; \n\ 1435 "); 1436 } 1437 1438 if (btf) { 1439 err = codegen_datasecs(obj, obj_name); 1440 if (err) 1441 goto out; 1442 } 1443 if (use_loader) { 1444 err = gen_trace(obj, obj_name, header_guard); 1445 goto out; 1446 } 1447 1448 codegen("\ 1449 \n\ 1450 \n\ 1451 #ifdef __cplusplus \n\ 1452 static inline struct %1$s *open(const struct bpf_object_open_opts *opts = nullptr);\n\ 1453 static inline struct %1$s *open_and_load(); \n\ 1454 static inline int load(struct %1$s *skel); \n\ 1455 static inline int attach(struct %1$s *skel); \n\ 1456 static inline void detach(struct %1$s *skel); \n\ 1457 static inline void destroy(struct %1$s *skel); \n\ 1458 static inline const void *elf_bytes(size_t *sz); \n\ 1459 #endif /* __cplusplus */ \n\ 1460 }; \n\ 1461 \n\ 1462 static void \n\ 1463 %1$s__destroy(struct %1$s *obj) \n\ 1464 { \n\ 1465 if (!obj) \n\ 1466 return; \n\ 1467 if (obj->skeleton) \n\ 1468 bpf_object__destroy_skeleton(obj->skeleton);\n\ 1469 free(obj); \n\ 1470 } \n\ 1471 \n\ 1472 static inline int \n\ 1473 %1$s__create_skeleton(struct %1$s *obj); \n\ 1474 \n\ 1475 static inline struct %1$s * \n\ 1476 %1$s__open_opts(const struct bpf_object_open_opts *opts) \n\ 1477 { \n\ 1478 struct %1$s *obj; \n\ 1479 int err; \n\ 1480 \n\ 1481 obj = (struct %1$s *)calloc(1, sizeof(*obj)); \n\ 1482 if (!obj) { \n\ 1483 errno = ENOMEM; \n\ 1484 return NULL; \n\ 1485 } \n\ 1486 \n\ 1487 err = %1$s__create_skeleton(obj); \n\ 1488 if (err) \n\ 1489 goto err_out; \n\ 1490 \n\ 1491 err = bpf_object__open_skeleton(obj->skeleton, opts);\n\ 1492 if (err) \n\ 1493 goto err_out; \n\ 1494 \n\ 1495 ", obj_name); 1496 1497 gen_st_ops_shadow_init(btf, obj); 1498 1499 codegen("\ 1500 \n\ 1501 return obj; \n\ 1502 err_out: \n\ 1503 %1$s__destroy(obj); \n\ 1504 errno = -err; \n\ 1505 return NULL; \n\ 1506 } \n\ 1507 \n\ 1508 static inline struct %1$s * \n\ 1509 %1$s__open(void) \n\ 1510 { \n\ 1511 return %1$s__open_opts(NULL); \n\ 1512 } \n\ 1513 \n\ 1514 static inline int \n\ 1515 %1$s__load(struct %1$s *obj) \n\ 1516 { \n\ 1517 return bpf_object__load_skeleton(obj->skeleton); \n\ 1518 } \n\ 1519 \n\ 1520 static inline struct %1$s * \n\ 1521 %1$s__open_and_load(void) \n\ 1522 { \n\ 1523 struct %1$s *obj; \n\ 1524 int err; \n\ 1525 \n\ 1526 obj = %1$s__open(); \n\ 1527 if (!obj) \n\ 1528 return NULL; \n\ 1529 err = %1$s__load(obj); \n\ 1530 if (err) { \n\ 1531 %1$s__destroy(obj); \n\ 1532 errno = -err; \n\ 1533 return NULL; \n\ 1534 } \n\ 1535 return obj; \n\ 1536 } \n\ 1537 \n\ 1538 static inline int \n\ 1539 %1$s__attach(struct %1$s *obj) \n\ 1540 { \n\ 1541 return bpf_object__attach_skeleton(obj->skeleton); \n\ 1542 } \n\ 1543 \n\ 1544 static inline void \n\ 1545 %1$s__detach(struct %1$s *obj) \n\ 1546 { \n\ 1547 bpf_object__detach_skeleton(obj->skeleton); \n\ 1548 } \n\ 1549 ", 1550 obj_name 1551 ); 1552 1553 codegen("\ 1554 \n\ 1555 \n\ 1556 static inline const void *%1$s__elf_bytes(size_t *sz); \n\ 1557 \n\ 1558 static inline int \n\ 1559 %1$s__create_skeleton(struct %1$s *obj) \n\ 1560 { \n\ 1561 struct bpf_object_skeleton *s; \n\ 1562 struct bpf_map_skeleton *map __attribute__((unused));\n\ 1563 int err; \n\ 1564 \n\ 1565 s = (struct bpf_object_skeleton *)calloc(1, sizeof(*s));\n\ 1566 if (!s) { \n\ 1567 err = -ENOMEM; \n\ 1568 goto err; \n\ 1569 } \n\ 1570 \n\ 1571 s->sz = sizeof(*s); \n\ 1572 s->name = \"%1$s\"; \n\ 1573 s->obj = &obj->obj; \n\ 1574 ", 1575 obj_name 1576 ); 1577 1578 codegen_maps_skeleton(obj, map_cnt, true /*mmaped*/, true /*links*/); 1579 codegen_progs_skeleton(obj, prog_cnt, true /*populate_links*/); 1580 1581 codegen("\ 1582 \n\ 1583 \n\ 1584 s->data = %1$s__elf_bytes(&s->data_sz); \n\ 1585 \n\ 1586 obj->skeleton = s; \n\ 1587 return 0; \n\ 1588 err: \n\ 1589 bpf_object__destroy_skeleton(s); \n\ 1590 return err; \n\ 1591 } \n\ 1592 \n\ 1593 static inline const void *%1$s__elf_bytes(size_t *sz) \n\ 1594 { \n\ 1595 static const char data[] __attribute__((__aligned__(8))) = \"\\\n\ 1596 ", 1597 obj_name 1598 ); 1599 1600 /* embed contents of BPF object file */ 1601 print_hex(obj_data, file_sz); 1602 1603 codegen("\ 1604 \n\ 1605 \"; \n\ 1606 \n\ 1607 *sz = sizeof(data) - 1; \n\ 1608 return (const void *)data; \n\ 1609 } \n\ 1610 \n\ 1611 #ifdef __cplusplus \n\ 1612 struct %1$s *%1$s::open(const struct bpf_object_open_opts *opts) { return %1$s__open_opts(opts); }\n\ 1613 struct %1$s *%1$s::open_and_load() { return %1$s__open_and_load(); } \n\ 1614 int %1$s::load(struct %1$s *skel) { return %1$s__load(skel); } \n\ 1615 int %1$s::attach(struct %1$s *skel) { return %1$s__attach(skel); } \n\ 1616 void %1$s::detach(struct %1$s *skel) { %1$s__detach(skel); } \n\ 1617 void %1$s::destroy(struct %1$s *skel) { %1$s__destroy(skel); } \n\ 1618 const void *%1$s::elf_bytes(size_t *sz) { return %1$s__elf_bytes(sz); } \n\ 1619 #endif /* __cplusplus */ \n\ 1620 \n\ 1621 ", 1622 obj_name); 1623 1624 codegen_asserts(obj, obj_name); 1625 1626 codegen("\ 1627 \n\ 1628 \n\ 1629 #endif /* %1$s */ \n\ 1630 ", 1631 header_guard); 1632 err = 0; 1633 out: 1634 bpf_object__close(obj); 1635 out_obj: 1636 if (obj_data) 1637 munmap(obj_data, mmap_sz); 1638 close(fd); 1639 return err; 1640 } 1641 1642 /* Subskeletons are like skeletons, except they don't own the bpf_object, 1643 * associated maps, links, etc. Instead, they know about the existence of 1644 * variables, maps, programs and are able to find their locations 1645 * _at runtime_ from an already loaded bpf_object. 1646 * 1647 * This allows for library-like BPF objects to have userspace counterparts 1648 * with access to their own items without having to know anything about the 1649 * final BPF object that the library was linked into. 1650 */ 1651 static int do_subskeleton(int argc, char **argv) 1652 { 1653 char header_guard[MAX_OBJ_NAME_LEN + sizeof("__SUBSKEL_H__")]; 1654 size_t i, len, file_sz, map_cnt = 0, prog_cnt = 0, mmap_sz, var_cnt = 0, var_idx = 0; 1655 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts); 1656 char obj_name[MAX_OBJ_NAME_LEN] = "", *obj_data; 1657 struct bpf_object *obj = NULL; 1658 const char *file, *var_name; 1659 char ident[256]; 1660 int fd, err = -1, map_type_id; 1661 const struct bpf_map *map; 1662 struct bpf_program *prog; 1663 struct btf *btf; 1664 const struct btf_type *map_type, *var_type; 1665 const struct btf_var_secinfo *var; 1666 struct stat st; 1667 1668 if (!REQ_ARGS(1)) { 1669 usage(); 1670 return -1; 1671 } 1672 file = GET_ARG(); 1673 1674 while (argc) { 1675 if (!REQ_ARGS(2)) 1676 return -1; 1677 1678 if (is_prefix(*argv, "name")) { 1679 NEXT_ARG(); 1680 1681 if (obj_name[0] != '\0') { 1682 p_err("object name already specified"); 1683 return -1; 1684 } 1685 1686 strncpy(obj_name, *argv, MAX_OBJ_NAME_LEN - 1); 1687 obj_name[MAX_OBJ_NAME_LEN - 1] = '\0'; 1688 } else { 1689 p_err("unknown arg %s", *argv); 1690 return -1; 1691 } 1692 1693 NEXT_ARG(); 1694 } 1695 1696 if (argc) { 1697 p_err("extra unknown arguments"); 1698 return -1; 1699 } 1700 1701 if (use_loader) { 1702 p_err("cannot use loader for subskeletons"); 1703 return -1; 1704 } 1705 1706 if (stat(file, &st)) { 1707 p_err("failed to stat() %s: %s", file, strerror(errno)); 1708 return -1; 1709 } 1710 file_sz = st.st_size; 1711 mmap_sz = roundup(file_sz, sysconf(_SC_PAGE_SIZE)); 1712 fd = open(file, O_RDONLY); 1713 if (fd < 0) { 1714 p_err("failed to open() %s: %s", file, strerror(errno)); 1715 return -1; 1716 } 1717 obj_data = mmap(NULL, mmap_sz, PROT_READ, MAP_PRIVATE, fd, 0); 1718 if (obj_data == MAP_FAILED) { 1719 obj_data = NULL; 1720 p_err("failed to mmap() %s: %s", file, strerror(errno)); 1721 goto out; 1722 } 1723 if (obj_name[0] == '\0') 1724 get_obj_name(obj_name, file); 1725 1726 /* The empty object name allows us to use bpf_map__name and produce 1727 * ELF section names out of it. (".data" instead of "obj.data") 1728 */ 1729 opts.object_name = ""; 1730 obj = bpf_object__open_mem(obj_data, file_sz, &opts); 1731 if (!obj) { 1732 char err_buf[256]; 1733 1734 libbpf_strerror(errno, err_buf, sizeof(err_buf)); 1735 p_err("failed to open BPF object file: %s", err_buf); 1736 obj = NULL; 1737 goto out; 1738 } 1739 1740 btf = bpf_object__btf(obj); 1741 if (!btf) { 1742 err = -1; 1743 p_err("need btf type information for %s", obj_name); 1744 goto out; 1745 } 1746 1747 bpf_object__for_each_program(prog, obj) { 1748 prog_cnt++; 1749 } 1750 1751 /* First, count how many variables we have to find. 1752 * We need this in advance so the subskel can allocate the right 1753 * amount of storage. 1754 */ 1755 bpf_object__for_each_map(map, obj) { 1756 if (!get_map_ident(map, ident, sizeof(ident))) 1757 continue; 1758 1759 /* Also count all maps that have a name */ 1760 map_cnt++; 1761 1762 if (!is_mmapable_map(map, ident, sizeof(ident))) 1763 continue; 1764 1765 map_type_id = bpf_map__btf_value_type_id(map); 1766 if (map_type_id <= 0) { 1767 err = map_type_id; 1768 goto out; 1769 } 1770 map_type = btf__type_by_id(btf, map_type_id); 1771 1772 var = btf_var_secinfos(map_type); 1773 len = btf_vlen(map_type); 1774 for (i = 0; i < len; i++, var++) { 1775 var_type = btf__type_by_id(btf, var->type); 1776 1777 if (btf_var(var_type)->linkage == BTF_VAR_STATIC) 1778 continue; 1779 1780 var_cnt++; 1781 } 1782 } 1783 1784 get_header_guard(header_guard, obj_name, "SUBSKEL_H"); 1785 codegen("\ 1786 \n\ 1787 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ \n\ 1788 \n\ 1789 /* THIS FILE IS AUTOGENERATED! */ \n\ 1790 #ifndef %2$s \n\ 1791 #define %2$s \n\ 1792 \n\ 1793 #include <errno.h> \n\ 1794 #include <stdlib.h> \n\ 1795 #include <bpf/libbpf.h> \n\ 1796 \n\ 1797 struct %1$s { \n\ 1798 struct bpf_object *obj; \n\ 1799 struct bpf_object_subskeleton *subskel; \n\ 1800 ", obj_name, header_guard); 1801 1802 if (map_cnt) { 1803 printf("\tstruct {\n"); 1804 bpf_object__for_each_map(map, obj) { 1805 if (!get_map_ident(map, ident, sizeof(ident))) 1806 continue; 1807 printf("\t\tstruct bpf_map *%s;\n", ident); 1808 } 1809 printf("\t} maps;\n"); 1810 } 1811 1812 err = gen_st_ops_shadow(obj_name, btf, obj); 1813 if (err) 1814 goto out; 1815 1816 if (prog_cnt) { 1817 printf("\tstruct {\n"); 1818 bpf_object__for_each_program(prog, obj) { 1819 printf("\t\tstruct bpf_program *%s;\n", 1820 bpf_program__name(prog)); 1821 } 1822 printf("\t} progs;\n"); 1823 } 1824 1825 err = codegen_subskel_datasecs(obj, obj_name); 1826 if (err) 1827 goto out; 1828 1829 /* emit code that will allocate enough storage for all symbols */ 1830 codegen("\ 1831 \n\ 1832 \n\ 1833 #ifdef __cplusplus \n\ 1834 static inline struct %1$s *open(const struct bpf_object *src);\n\ 1835 static inline void destroy(struct %1$s *skel); \n\ 1836 #endif /* __cplusplus */ \n\ 1837 }; \n\ 1838 \n\ 1839 static inline void \n\ 1840 %1$s__destroy(struct %1$s *skel) \n\ 1841 { \n\ 1842 if (!skel) \n\ 1843 return; \n\ 1844 if (skel->subskel) \n\ 1845 bpf_object__destroy_subskeleton(skel->subskel);\n\ 1846 free(skel); \n\ 1847 } \n\ 1848 \n\ 1849 static inline struct %1$s * \n\ 1850 %1$s__open(const struct bpf_object *src) \n\ 1851 { \n\ 1852 struct %1$s *obj; \n\ 1853 struct bpf_object_subskeleton *s; \n\ 1854 struct bpf_map_skeleton *map __attribute__((unused));\n\ 1855 int err; \n\ 1856 \n\ 1857 obj = (struct %1$s *)calloc(1, sizeof(*obj)); \n\ 1858 if (!obj) { \n\ 1859 err = -ENOMEM; \n\ 1860 goto err; \n\ 1861 } \n\ 1862 s = (struct bpf_object_subskeleton *)calloc(1, sizeof(*s));\n\ 1863 if (!s) { \n\ 1864 err = -ENOMEM; \n\ 1865 goto err; \n\ 1866 } \n\ 1867 s->sz = sizeof(*s); \n\ 1868 s->obj = src; \n\ 1869 s->var_skel_sz = sizeof(*s->vars); \n\ 1870 obj->subskel = s; \n\ 1871 \n\ 1872 /* vars */ \n\ 1873 s->var_cnt = %2$d; \n\ 1874 s->vars = (struct bpf_var_skeleton *)calloc(%2$d, sizeof(*s->vars));\n\ 1875 if (!s->vars) { \n\ 1876 err = -ENOMEM; \n\ 1877 goto err; \n\ 1878 } \n\ 1879 ", 1880 obj_name, var_cnt 1881 ); 1882 1883 /* walk through each symbol and emit the runtime representation */ 1884 bpf_object__for_each_map(map, obj) { 1885 if (!is_mmapable_map(map, ident, sizeof(ident))) 1886 continue; 1887 1888 map_type_id = bpf_map__btf_value_type_id(map); 1889 if (map_type_id <= 0) 1890 /* skip over internal maps with no type*/ 1891 continue; 1892 1893 map_type = btf__type_by_id(btf, map_type_id); 1894 var = btf_var_secinfos(map_type); 1895 len = btf_vlen(map_type); 1896 for (i = 0; i < len; i++, var++) { 1897 var_type = btf__type_by_id(btf, var->type); 1898 var_name = btf__name_by_offset(btf, var_type->name_off); 1899 1900 if (btf_var(var_type)->linkage == BTF_VAR_STATIC) 1901 continue; 1902 1903 /* Note that we use the dot prefix in .data as the 1904 * field access operator i.e. maps%s becomes maps.data 1905 */ 1906 codegen("\ 1907 \n\ 1908 \n\ 1909 s->vars[%3$d].name = \"%1$s\"; \n\ 1910 s->vars[%3$d].map = &obj->maps.%2$s; \n\ 1911 s->vars[%3$d].addr = (void **) &obj->%2$s.%1$s;\n\ 1912 ", var_name, ident, var_idx); 1913 1914 var_idx++; 1915 } 1916 } 1917 1918 codegen_maps_skeleton(obj, map_cnt, false /*mmaped*/, false /*links*/); 1919 codegen_progs_skeleton(obj, prog_cnt, false /*links*/); 1920 1921 codegen("\ 1922 \n\ 1923 \n\ 1924 err = bpf_object__open_subskeleton(s); \n\ 1925 if (err) \n\ 1926 goto err; \n\ 1927 \n\ 1928 "); 1929 1930 gen_st_ops_shadow_init(btf, obj); 1931 1932 codegen("\ 1933 \n\ 1934 return obj; \n\ 1935 err: \n\ 1936 %1$s__destroy(obj); \n\ 1937 errno = -err; \n\ 1938 return NULL; \n\ 1939 } \n\ 1940 \n\ 1941 #ifdef __cplusplus \n\ 1942 struct %1$s *%1$s::open(const struct bpf_object *src) { return %1$s__open(src); }\n\ 1943 void %1$s::destroy(struct %1$s *skel) { %1$s__destroy(skel); }\n\ 1944 #endif /* __cplusplus */ \n\ 1945 \n\ 1946 #endif /* %2$s */ \n\ 1947 ", 1948 obj_name, header_guard); 1949 err = 0; 1950 out: 1951 bpf_object__close(obj); 1952 if (obj_data) 1953 munmap(obj_data, mmap_sz); 1954 close(fd); 1955 return err; 1956 } 1957 1958 static int do_object(int argc, char **argv) 1959 { 1960 struct bpf_linker *linker; 1961 const char *output_file, *file; 1962 int err = 0; 1963 1964 if (!REQ_ARGS(2)) { 1965 usage(); 1966 return -1; 1967 } 1968 1969 output_file = GET_ARG(); 1970 1971 linker = bpf_linker__new(output_file, NULL); 1972 if (!linker) { 1973 p_err("failed to create BPF linker instance"); 1974 return -1; 1975 } 1976 1977 while (argc) { 1978 file = GET_ARG(); 1979 1980 err = bpf_linker__add_file(linker, file, NULL); 1981 if (err) { 1982 p_err("failed to link '%s': %s (%d)", file, strerror(errno), errno); 1983 goto out; 1984 } 1985 } 1986 1987 err = bpf_linker__finalize(linker); 1988 if (err) { 1989 p_err("failed to finalize ELF file: %s (%d)", strerror(errno), errno); 1990 goto out; 1991 } 1992 1993 err = 0; 1994 out: 1995 bpf_linker__free(linker); 1996 return err; 1997 } 1998 1999 static int do_help(int argc, char **argv) 2000 { 2001 if (json_output) { 2002 jsonw_null(json_wtr); 2003 return 0; 2004 } 2005 2006 fprintf(stderr, 2007 "Usage: %1$s %2$s object OUTPUT_FILE INPUT_FILE [INPUT_FILE...]\n" 2008 " %1$s %2$s skeleton FILE [name OBJECT_NAME]\n" 2009 " %1$s %2$s subskeleton FILE [name OBJECT_NAME]\n" 2010 " %1$s %2$s min_core_btf INPUT OUTPUT OBJECT [OBJECT...]\n" 2011 " %1$s %2$s help\n" 2012 "\n" 2013 " " HELP_SPEC_OPTIONS " |\n" 2014 " {-L|--use-loader} | [ {-S|--sign } {-k} <private_key.pem> {-i} <certificate.x509> ]}\n" 2015 "", 2016 bin_name, "gen"); 2017 2018 return 0; 2019 } 2020 2021 static int btf_save_raw(const struct btf *btf, const char *path) 2022 { 2023 const void *data; 2024 FILE *f = NULL; 2025 __u32 data_sz; 2026 int err = 0; 2027 2028 data = btf__raw_data(btf, &data_sz); 2029 if (!data) 2030 return -ENOMEM; 2031 2032 f = fopen(path, "wb"); 2033 if (!f) 2034 return -errno; 2035 2036 if (fwrite(data, 1, data_sz, f) != data_sz) 2037 err = -errno; 2038 2039 fclose(f); 2040 return err; 2041 } 2042 2043 struct btfgen_info { 2044 struct btf *src_btf; 2045 struct btf *marked_btf; /* btf structure used to mark used types */ 2046 }; 2047 2048 static size_t btfgen_hash_fn(long key, void *ctx) 2049 { 2050 return key; 2051 } 2052 2053 static bool btfgen_equal_fn(long k1, long k2, void *ctx) 2054 { 2055 return k1 == k2; 2056 } 2057 2058 static void btfgen_free_info(struct btfgen_info *info) 2059 { 2060 if (!info) 2061 return; 2062 2063 btf__free(info->src_btf); 2064 btf__free(info->marked_btf); 2065 2066 free(info); 2067 } 2068 2069 static struct btfgen_info * 2070 btfgen_new_info(const char *targ_btf_path) 2071 { 2072 struct btfgen_info *info; 2073 int err; 2074 2075 info = calloc(1, sizeof(*info)); 2076 if (!info) 2077 return NULL; 2078 2079 info->src_btf = btf__parse(targ_btf_path, NULL); 2080 if (!info->src_btf) { 2081 err = -errno; 2082 p_err("failed parsing '%s' BTF file: %s", targ_btf_path, strerror(errno)); 2083 goto err_out; 2084 } 2085 2086 info->marked_btf = btf__parse(targ_btf_path, NULL); 2087 if (!info->marked_btf) { 2088 err = -errno; 2089 p_err("failed parsing '%s' BTF file: %s", targ_btf_path, strerror(errno)); 2090 goto err_out; 2091 } 2092 2093 return info; 2094 2095 err_out: 2096 btfgen_free_info(info); 2097 errno = -err; 2098 return NULL; 2099 } 2100 2101 #define MARKED UINT32_MAX 2102 2103 static void btfgen_mark_member(struct btfgen_info *info, int type_id, int idx) 2104 { 2105 const struct btf_type *t = btf__type_by_id(info->marked_btf, type_id); 2106 struct btf_member *m = btf_members(t) + idx; 2107 2108 m->name_off = MARKED; 2109 } 2110 2111 static int 2112 btfgen_mark_type(struct btfgen_info *info, unsigned int type_id, bool follow_pointers) 2113 { 2114 const struct btf_type *btf_type = btf__type_by_id(info->src_btf, type_id); 2115 struct btf_type *cloned_type; 2116 struct btf_param *param; 2117 struct btf_array *array; 2118 __u32 i; 2119 int err; 2120 2121 if (type_id == 0) 2122 return 0; 2123 2124 /* mark type on cloned BTF as used */ 2125 cloned_type = (struct btf_type *) btf__type_by_id(info->marked_btf, type_id); 2126 cloned_type->name_off = MARKED; 2127 2128 /* recursively mark other types needed by it */ 2129 switch (btf_kind(btf_type)) { 2130 case BTF_KIND_UNKN: 2131 case BTF_KIND_INT: 2132 case BTF_KIND_FLOAT: 2133 case BTF_KIND_ENUM: 2134 case BTF_KIND_ENUM64: 2135 case BTF_KIND_STRUCT: 2136 case BTF_KIND_UNION: 2137 break; 2138 case BTF_KIND_PTR: 2139 if (follow_pointers) { 2140 err = btfgen_mark_type(info, btf_type->type, follow_pointers); 2141 if (err) 2142 return err; 2143 } 2144 break; 2145 case BTF_KIND_CONST: 2146 case BTF_KIND_RESTRICT: 2147 case BTF_KIND_VOLATILE: 2148 case BTF_KIND_TYPEDEF: 2149 err = btfgen_mark_type(info, btf_type->type, follow_pointers); 2150 if (err) 2151 return err; 2152 break; 2153 case BTF_KIND_ARRAY: 2154 array = btf_array(btf_type); 2155 2156 /* mark array type */ 2157 err = btfgen_mark_type(info, array->type, follow_pointers); 2158 /* mark array's index type */ 2159 err = err ? : btfgen_mark_type(info, array->index_type, follow_pointers); 2160 if (err) 2161 return err; 2162 break; 2163 case BTF_KIND_FUNC_PROTO: 2164 /* mark ret type */ 2165 err = btfgen_mark_type(info, btf_type->type, follow_pointers); 2166 if (err) 2167 return err; 2168 2169 /* mark parameters types */ 2170 param = btf_params(btf_type); 2171 for (i = 0; i < btf_vlen(btf_type); i++) { 2172 err = btfgen_mark_type(info, param->type, follow_pointers); 2173 if (err) 2174 return err; 2175 param++; 2176 } 2177 break; 2178 /* tells if some other type needs to be handled */ 2179 default: 2180 p_err("unsupported kind: %s (%u)", btf_kind_str(btf_type), type_id); 2181 return -EINVAL; 2182 } 2183 2184 return 0; 2185 } 2186 2187 static int btfgen_record_field_relo(struct btfgen_info *info, struct bpf_core_spec *targ_spec) 2188 { 2189 struct btf *btf = info->src_btf; 2190 const struct btf_type *btf_type; 2191 struct btf_member *btf_member; 2192 struct btf_array *array; 2193 unsigned int type_id = targ_spec->root_type_id; 2194 int idx, err; 2195 2196 /* mark root type */ 2197 btf_type = btf__type_by_id(btf, type_id); 2198 err = btfgen_mark_type(info, type_id, false); 2199 if (err) 2200 return err; 2201 2202 /* mark types for complex types (arrays, unions, structures) */ 2203 for (int i = 1; i < targ_spec->raw_len; i++) { 2204 /* skip typedefs and mods */ 2205 while (btf_is_mod(btf_type) || btf_is_typedef(btf_type)) { 2206 type_id = btf_type->type; 2207 btf_type = btf__type_by_id(btf, type_id); 2208 } 2209 2210 switch (btf_kind(btf_type)) { 2211 case BTF_KIND_STRUCT: 2212 case BTF_KIND_UNION: 2213 idx = targ_spec->raw_spec[i]; 2214 btf_member = btf_members(btf_type) + idx; 2215 2216 /* mark member */ 2217 btfgen_mark_member(info, type_id, idx); 2218 2219 /* mark member's type */ 2220 type_id = btf_member->type; 2221 btf_type = btf__type_by_id(btf, type_id); 2222 err = btfgen_mark_type(info, type_id, false); 2223 if (err) 2224 return err; 2225 break; 2226 case BTF_KIND_ARRAY: 2227 array = btf_array(btf_type); 2228 type_id = array->type; 2229 btf_type = btf__type_by_id(btf, type_id); 2230 break; 2231 default: 2232 p_err("unsupported kind: %s (%u)", 2233 btf_kind_str(btf_type), btf_type->type); 2234 return -EINVAL; 2235 } 2236 } 2237 2238 return 0; 2239 } 2240 2241 /* Mark types, members, and member types. Compared to btfgen_record_field_relo, 2242 * this function does not rely on the target spec for inferring members, but 2243 * uses the associated BTF. 2244 * 2245 * The `behind_ptr` argument is used to stop marking of composite types reached 2246 * through a pointer. This way, we can keep BTF size in check while providing 2247 * reasonable match semantics. 2248 */ 2249 static int btfgen_mark_type_match(struct btfgen_info *info, __u32 type_id, bool behind_ptr) 2250 { 2251 const struct btf_type *btf_type; 2252 struct btf *btf = info->src_btf; 2253 struct btf_type *cloned_type; 2254 int err; 2255 __u32 i; 2256 2257 if (type_id == 0) 2258 return 0; 2259 2260 btf_type = btf__type_by_id(btf, type_id); 2261 /* mark type on cloned BTF as used */ 2262 cloned_type = (struct btf_type *)btf__type_by_id(info->marked_btf, type_id); 2263 cloned_type->name_off = MARKED; 2264 2265 switch (btf_kind(btf_type)) { 2266 case BTF_KIND_UNKN: 2267 case BTF_KIND_INT: 2268 case BTF_KIND_FLOAT: 2269 case BTF_KIND_ENUM: 2270 case BTF_KIND_ENUM64: 2271 break; 2272 case BTF_KIND_STRUCT: 2273 case BTF_KIND_UNION: { 2274 struct btf_member *m = btf_members(btf_type); 2275 __u32 vlen = btf_vlen(btf_type); 2276 2277 if (behind_ptr) 2278 break; 2279 2280 for (i = 0; i < vlen; i++, m++) { 2281 /* mark member */ 2282 btfgen_mark_member(info, type_id, i); 2283 2284 /* mark member's type */ 2285 err = btfgen_mark_type_match(info, m->type, false); 2286 if (err) 2287 return err; 2288 } 2289 break; 2290 } 2291 case BTF_KIND_CONST: 2292 case BTF_KIND_FWD: 2293 case BTF_KIND_RESTRICT: 2294 case BTF_KIND_TYPEDEF: 2295 case BTF_KIND_VOLATILE: 2296 return btfgen_mark_type_match(info, btf_type->type, behind_ptr); 2297 case BTF_KIND_PTR: 2298 return btfgen_mark_type_match(info, btf_type->type, true); 2299 case BTF_KIND_ARRAY: { 2300 struct btf_array *array; 2301 2302 array = btf_array(btf_type); 2303 /* mark array type */ 2304 err = btfgen_mark_type_match(info, array->type, false); 2305 /* mark array's index type */ 2306 err = err ? : btfgen_mark_type_match(info, array->index_type, false); 2307 if (err) 2308 return err; 2309 break; 2310 } 2311 case BTF_KIND_FUNC_PROTO: { 2312 __u32 vlen = btf_vlen(btf_type); 2313 struct btf_param *param; 2314 2315 /* mark ret type */ 2316 err = btfgen_mark_type_match(info, btf_type->type, false); 2317 if (err) 2318 return err; 2319 2320 /* mark parameters types */ 2321 param = btf_params(btf_type); 2322 for (i = 0; i < vlen; i++) { 2323 err = btfgen_mark_type_match(info, param->type, false); 2324 if (err) 2325 return err; 2326 param++; 2327 } 2328 break; 2329 } 2330 /* tells if some other type needs to be handled */ 2331 default: 2332 p_err("unsupported kind: %s (%u)", btf_kind_str(btf_type), type_id); 2333 return -EINVAL; 2334 } 2335 2336 return 0; 2337 } 2338 2339 /* Mark types, members, and member types. Compared to btfgen_record_field_relo, 2340 * this function does not rely on the target spec for inferring members, but 2341 * uses the associated BTF. 2342 */ 2343 static int btfgen_record_type_match_relo(struct btfgen_info *info, struct bpf_core_spec *targ_spec) 2344 { 2345 return btfgen_mark_type_match(info, targ_spec->root_type_id, false); 2346 } 2347 2348 static int btfgen_record_type_relo(struct btfgen_info *info, struct bpf_core_spec *targ_spec) 2349 { 2350 return btfgen_mark_type(info, targ_spec->root_type_id, true); 2351 } 2352 2353 static int btfgen_record_enumval_relo(struct btfgen_info *info, struct bpf_core_spec *targ_spec) 2354 { 2355 return btfgen_mark_type(info, targ_spec->root_type_id, false); 2356 } 2357 2358 static int btfgen_record_reloc(struct btfgen_info *info, struct bpf_core_spec *res) 2359 { 2360 switch (res->relo_kind) { 2361 case BPF_CORE_FIELD_BYTE_OFFSET: 2362 case BPF_CORE_FIELD_BYTE_SIZE: 2363 case BPF_CORE_FIELD_EXISTS: 2364 case BPF_CORE_FIELD_SIGNED: 2365 case BPF_CORE_FIELD_LSHIFT_U64: 2366 case BPF_CORE_FIELD_RSHIFT_U64: 2367 return btfgen_record_field_relo(info, res); 2368 case BPF_CORE_TYPE_ID_LOCAL: /* BPF_CORE_TYPE_ID_LOCAL doesn't require kernel BTF */ 2369 return 0; 2370 case BPF_CORE_TYPE_ID_TARGET: 2371 case BPF_CORE_TYPE_EXISTS: 2372 case BPF_CORE_TYPE_SIZE: 2373 return btfgen_record_type_relo(info, res); 2374 case BPF_CORE_TYPE_MATCHES: 2375 return btfgen_record_type_match_relo(info, res); 2376 case BPF_CORE_ENUMVAL_EXISTS: 2377 case BPF_CORE_ENUMVAL_VALUE: 2378 return btfgen_record_enumval_relo(info, res); 2379 default: 2380 return -EINVAL; 2381 } 2382 } 2383 2384 static struct bpf_core_cand_list * 2385 btfgen_find_cands(const struct btf *local_btf, const struct btf *targ_btf, __u32 local_id) 2386 { 2387 const struct btf_type *local_type; 2388 struct bpf_core_cand_list *cands = NULL; 2389 struct bpf_core_cand local_cand = {}; 2390 size_t local_essent_len; 2391 const char *local_name; 2392 int err; 2393 2394 local_cand.btf = local_btf; 2395 local_cand.id = local_id; 2396 2397 local_type = btf__type_by_id(local_btf, local_id); 2398 if (!local_type) { 2399 err = -EINVAL; 2400 goto err_out; 2401 } 2402 2403 local_name = btf__name_by_offset(local_btf, local_type->name_off); 2404 if (!local_name) { 2405 err = -EINVAL; 2406 goto err_out; 2407 } 2408 local_essent_len = bpf_core_essential_name_len(local_name); 2409 2410 cands = calloc(1, sizeof(*cands)); 2411 if (!cands) 2412 return NULL; 2413 2414 err = bpf_core_add_cands(&local_cand, local_essent_len, targ_btf, "vmlinux", 1, cands); 2415 if (err) 2416 goto err_out; 2417 2418 return cands; 2419 2420 err_out: 2421 bpf_core_free_cands(cands); 2422 errno = -err; 2423 return NULL; 2424 } 2425 2426 /* Record relocation information for a single BPF object */ 2427 static int btfgen_record_obj(struct btfgen_info *info, const char *obj_path) 2428 { 2429 const struct btf_ext_info_sec *sec; 2430 const struct bpf_core_relo *relo; 2431 const struct btf_ext_info *seg; 2432 struct hashmap_entry *entry; 2433 struct hashmap *cand_cache = NULL; 2434 struct btf_ext *btf_ext = NULL; 2435 unsigned int relo_idx; 2436 struct btf *btf = NULL; 2437 size_t i; 2438 int err; 2439 2440 btf = btf__parse(obj_path, &btf_ext); 2441 if (!btf) { 2442 err = -errno; 2443 p_err("failed to parse BPF object '%s': %s", obj_path, strerror(errno)); 2444 return err; 2445 } 2446 2447 if (!btf_ext) { 2448 p_err("failed to parse BPF object '%s': section %s not found", 2449 obj_path, BTF_EXT_ELF_SEC); 2450 err = -EINVAL; 2451 goto out; 2452 } 2453 2454 if (btf_ext->core_relo_info.len == 0) { 2455 err = 0; 2456 goto out; 2457 } 2458 2459 cand_cache = hashmap__new(btfgen_hash_fn, btfgen_equal_fn, NULL); 2460 if (IS_ERR(cand_cache)) { 2461 err = PTR_ERR(cand_cache); 2462 goto out; 2463 } 2464 2465 seg = &btf_ext->core_relo_info; 2466 for_each_btf_ext_sec(seg, sec) { 2467 for_each_btf_ext_rec(seg, sec, relo_idx, relo) { 2468 struct bpf_core_spec specs_scratch[3] = {}; 2469 struct bpf_core_relo_res targ_res = {}; 2470 struct bpf_core_cand_list *cands = NULL; 2471 const char *sec_name = btf__name_by_offset(btf, sec->sec_name_off); 2472 2473 if (relo->kind != BPF_CORE_TYPE_ID_LOCAL && 2474 !hashmap__find(cand_cache, relo->type_id, &cands)) { 2475 cands = btfgen_find_cands(btf, info->src_btf, relo->type_id); 2476 if (!cands) { 2477 err = -errno; 2478 goto out; 2479 } 2480 2481 err = hashmap__set(cand_cache, relo->type_id, cands, 2482 NULL, NULL); 2483 if (err) 2484 goto out; 2485 } 2486 2487 err = bpf_core_calc_relo_insn(sec_name, relo, relo_idx, btf, cands, 2488 specs_scratch, &targ_res); 2489 if (err) 2490 goto out; 2491 2492 /* specs_scratch[2] is the target spec */ 2493 err = btfgen_record_reloc(info, &specs_scratch[2]); 2494 if (err) 2495 goto out; 2496 } 2497 } 2498 2499 out: 2500 btf__free(btf); 2501 btf_ext__free(btf_ext); 2502 2503 if (!IS_ERR_OR_NULL(cand_cache)) { 2504 hashmap__for_each_entry(cand_cache, entry, i) { 2505 bpf_core_free_cands(entry->pvalue); 2506 } 2507 hashmap__free(cand_cache); 2508 } 2509 2510 return err; 2511 } 2512 2513 /* Generate BTF from relocation information previously recorded */ 2514 static struct btf *btfgen_get_btf(struct btfgen_info *info) 2515 { 2516 struct btf *btf_new = NULL; 2517 unsigned int *ids = NULL; 2518 unsigned int n = btf__type_cnt(info->marked_btf); 2519 int err = 0; 2520 __u32 i; 2521 2522 btf_new = btf__new_empty(); 2523 if (!btf_new) { 2524 err = -errno; 2525 goto err_out; 2526 } 2527 2528 ids = calloc(n, sizeof(*ids)); 2529 if (!ids) { 2530 err = -errno; 2531 goto err_out; 2532 } 2533 2534 /* first pass: add all marked types to btf_new and add their new ids to the ids map */ 2535 for (i = 1; i < n; i++) { 2536 const struct btf_type *cloned_type, *type; 2537 const char *name; 2538 int new_id; 2539 2540 cloned_type = btf__type_by_id(info->marked_btf, i); 2541 2542 if (cloned_type->name_off != MARKED) 2543 continue; 2544 2545 type = btf__type_by_id(info->src_btf, i); 2546 2547 /* add members for struct and union */ 2548 if (btf_is_composite(type)) { 2549 struct btf_member *cloned_m, *m; 2550 __u32 vlen, idx_src; 2551 2552 name = btf__str_by_offset(info->src_btf, type->name_off); 2553 2554 if (btf_is_struct(type)) 2555 err = btf__add_struct(btf_new, name, type->size); 2556 else 2557 err = btf__add_union(btf_new, name, type->size); 2558 2559 if (err < 0) 2560 goto err_out; 2561 new_id = err; 2562 2563 cloned_m = btf_members(cloned_type); 2564 m = btf_members(type); 2565 vlen = btf_vlen(cloned_type); 2566 for (idx_src = 0; idx_src < vlen; idx_src++, cloned_m++, m++) { 2567 /* add only members that are marked as used */ 2568 if (cloned_m->name_off != MARKED) 2569 continue; 2570 2571 name = btf__str_by_offset(info->src_btf, m->name_off); 2572 err = btf__add_field(btf_new, name, m->type, 2573 btf_member_bit_offset(cloned_type, idx_src), 2574 btf_member_bitfield_size(cloned_type, idx_src)); 2575 if (err < 0) 2576 goto err_out; 2577 } 2578 } else { 2579 err = btf__add_type(btf_new, info->src_btf, type); 2580 if (err < 0) 2581 goto err_out; 2582 new_id = err; 2583 } 2584 2585 /* add ID mapping */ 2586 ids[i] = new_id; 2587 } 2588 2589 /* second pass: fix up type ids */ 2590 for (i = 1; i < btf__type_cnt(btf_new); i++) { 2591 struct btf_type *btf_type = (struct btf_type *) btf__type_by_id(btf_new, i); 2592 struct btf_field_iter it; 2593 __u32 *type_id; 2594 2595 err = btf_field_iter_init(&it, btf_type, BTF_FIELD_ITER_IDS); 2596 if (err) 2597 goto err_out; 2598 2599 while ((type_id = btf_field_iter_next(&it))) 2600 *type_id = ids[*type_id]; 2601 } 2602 2603 free(ids); 2604 return btf_new; 2605 2606 err_out: 2607 btf__free(btf_new); 2608 free(ids); 2609 errno = -err; 2610 return NULL; 2611 } 2612 2613 /* Create minimized BTF file for a set of BPF objects. 2614 * 2615 * The BTFGen algorithm is divided in two main parts: (1) collect the 2616 * BTF types that are involved in relocations and (2) generate the BTF 2617 * object using the collected types. 2618 * 2619 * In order to collect the types involved in the relocations, we parse 2620 * the BTF and BTF.ext sections of the BPF objects and use 2621 * bpf_core_calc_relo_insn() to get the target specification, this 2622 * indicates how the types and fields are used in a relocation. 2623 * 2624 * Types are recorded in different ways according to the kind of the 2625 * relocation. For field-based relocations only the members that are 2626 * actually used are saved in order to reduce the size of the generated 2627 * BTF file. For type-based relocations empty struct / unions are 2628 * generated and for enum-based relocations the whole type is saved. 2629 * 2630 * The second part of the algorithm generates the BTF object. It creates 2631 * an empty BTF object and fills it with the types recorded in the 2632 * previous step. This function takes care of only adding the structure 2633 * and union members that were marked as used and it also fixes up the 2634 * type IDs on the generated BTF object. 2635 */ 2636 static int minimize_btf(const char *src_btf, const char *dst_btf, const char *objspaths[]) 2637 { 2638 struct btfgen_info *info; 2639 struct btf *btf_new = NULL; 2640 int err, i; 2641 2642 info = btfgen_new_info(src_btf); 2643 if (!info) { 2644 err = -errno; 2645 p_err("failed to allocate info structure: %s", strerror(errno)); 2646 goto out; 2647 } 2648 2649 for (i = 0; objspaths[i] != NULL; i++) { 2650 err = btfgen_record_obj(info, objspaths[i]); 2651 if (err) { 2652 p_err("error recording relocations for %s: %s", objspaths[i], 2653 strerror(errno)); 2654 goto out; 2655 } 2656 } 2657 2658 btf_new = btfgen_get_btf(info); 2659 if (!btf_new) { 2660 err = -errno; 2661 p_err("error generating BTF: %s", strerror(errno)); 2662 goto out; 2663 } 2664 2665 err = btf_save_raw(btf_new, dst_btf); 2666 if (err) { 2667 p_err("error saving btf file: %s", strerror(errno)); 2668 goto out; 2669 } 2670 2671 out: 2672 btf__free(btf_new); 2673 btfgen_free_info(info); 2674 2675 return err; 2676 } 2677 2678 static int do_min_core_btf(int argc, char **argv) 2679 { 2680 const char *input, *output, **objs; 2681 int i, err; 2682 2683 if (!REQ_ARGS(3)) { 2684 usage(); 2685 return -1; 2686 } 2687 2688 input = GET_ARG(); 2689 output = GET_ARG(); 2690 2691 objs = (const char **) calloc(argc + 1, sizeof(*objs)); 2692 if (!objs) { 2693 p_err("failed to allocate array for object names"); 2694 return -ENOMEM; 2695 } 2696 2697 i = 0; 2698 while (argc) 2699 objs[i++] = GET_ARG(); 2700 2701 err = minimize_btf(input, output, objs); 2702 free(objs); 2703 return err; 2704 } 2705 2706 static const struct cmd cmds[] = { 2707 { "object", do_object }, 2708 { "skeleton", do_skeleton }, 2709 { "subskeleton", do_subskeleton }, 2710 { "min_core_btf", do_min_core_btf}, 2711 { "help", do_help }, 2712 { 0 } 2713 }; 2714 2715 int do_gen(int argc, char **argv) 2716 { 2717 return cmd_select(cmds, argc, argv, do_help); 2718 } 2719