1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* Copyright (c) 2018 Facebook */ 3 4 #include <byteswap.h> 5 #include <endian.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <fcntl.h> 10 #include <unistd.h> 11 #include <errno.h> 12 #include <sys/utsname.h> 13 #include <sys/param.h> 14 #include <sys/stat.h> 15 #include <sys/mman.h> 16 #include <linux/kernel.h> 17 #include <linux/err.h> 18 #include <linux/btf.h> 19 #include <gelf.h> 20 #include "btf.h" 21 #include "bpf.h" 22 #include "libbpf.h" 23 #include "libbpf_internal.h" 24 #include "hashmap.h" 25 #include "strset.h" 26 27 #define BTF_MAX_NR_TYPES 0x7fffffffU 28 #define BTF_MAX_STR_OFFSET 0x7fffffffU 29 30 static struct btf_type btf_void; 31 32 struct btf { 33 /* raw BTF data in native endianness */ 34 void *raw_data; 35 /* raw BTF data in non-native endianness */ 36 void *raw_data_swapped; 37 __u32 raw_size; 38 /* whether target endianness differs from the native one */ 39 bool swapped_endian; 40 41 /* 42 * When BTF is loaded from an ELF or raw memory it is stored 43 * in a contiguous memory block. The hdr, type_data, and, strs_data 44 * point inside that memory region to their respective parts of BTF 45 * representation: 46 * 47 * +--------------------------------+ 48 * | Header | Types | Strings | 49 * +--------------------------------+ 50 * ^ ^ ^ 51 * | | | 52 * hdr | | 53 * types_data-+ | 54 * strs_data------------+ 55 * 56 * If BTF data is later modified, e.g., due to types added or 57 * removed, BTF deduplication performed, etc, this contiguous 58 * representation is broken up into three independently allocated 59 * memory regions to be able to modify them independently. 60 * raw_data is nulled out at that point, but can be later allocated 61 * and cached again if user calls btf__raw_data(), at which point 62 * raw_data will contain a contiguous copy of header, types, and 63 * strings: 64 * 65 * +----------+ +---------+ +-----------+ 66 * | Header | | Types | | Strings | 67 * +----------+ +---------+ +-----------+ 68 * ^ ^ ^ 69 * | | | 70 * hdr | | 71 * types_data----+ | 72 * strset__data(strs_set)-----+ 73 * 74 * +----------+---------+-----------+ 75 * | Header | Types | Strings | 76 * raw_data----->+----------+---------+-----------+ 77 */ 78 struct btf_header *hdr; 79 80 void *types_data; 81 size_t types_data_cap; /* used size stored in hdr->type_len */ 82 83 /* type ID to `struct btf_type *` lookup index 84 * type_offs[0] corresponds to the first non-VOID type: 85 * - for base BTF it's type [1]; 86 * - for split BTF it's the first non-base BTF type. 87 */ 88 __u32 *type_offs; 89 size_t type_offs_cap; 90 /* number of types in this BTF instance: 91 * - doesn't include special [0] void type; 92 * - for split BTF counts number of types added on top of base BTF. 93 */ 94 __u32 nr_types; 95 /* the start IDs of named types in sorted BTF */ 96 int named_start_id; 97 /* if not NULL, points to the base BTF on top of which the current 98 * split BTF is based 99 */ 100 struct btf *base_btf; 101 /* BTF type ID of the first type in this BTF instance: 102 * - for base BTF it's equal to 1; 103 * - for split BTF it's equal to biggest type ID of base BTF plus 1. 104 */ 105 int start_id; 106 /* logical string offset of this BTF instance: 107 * - for base BTF it's equal to 0; 108 * - for split BTF it's equal to total size of base BTF's string section size. 109 */ 110 int start_str_off; 111 112 /* only one of strs_data or strs_set can be non-NULL, depending on 113 * whether BTF is in a modifiable state (strs_set is used) or not 114 * (strs_data points inside raw_data) 115 */ 116 void *strs_data; 117 /* a set of unique strings */ 118 struct strset *strs_set; 119 /* whether strings are already deduplicated */ 120 bool strs_deduped; 121 122 /* whether base_btf should be freed in btf_free for this instance */ 123 bool owns_base; 124 125 /* whether raw_data is a (read-only) mmap */ 126 bool raw_data_is_mmap; 127 128 /* BTF object FD, if loaded into kernel */ 129 int fd; 130 131 /* Pointer size (in bytes) for a target architecture of this BTF */ 132 int ptr_sz; 133 }; 134 135 static inline __u64 ptr_to_u64(const void *ptr) 136 { 137 return (__u64) (unsigned long) ptr; 138 } 139 140 /* Ensure given dynamically allocated memory region pointed to by *data* with 141 * capacity of *cap_cnt* elements each taking *elem_sz* bytes has enough 142 * memory to accommodate *add_cnt* new elements, assuming *cur_cnt* elements 143 * are already used. At most *max_cnt* elements can be ever allocated. 144 * If necessary, memory is reallocated and all existing data is copied over, 145 * new pointer to the memory region is stored at *data, new memory region 146 * capacity (in number of elements) is stored in *cap. 147 * On success, memory pointer to the beginning of unused memory is returned. 148 * On error, NULL is returned. 149 */ 150 void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, 151 size_t cur_cnt, size_t max_cnt, size_t add_cnt) 152 { 153 size_t new_cnt; 154 void *new_data; 155 156 if (cur_cnt + add_cnt <= *cap_cnt) 157 return *data + cur_cnt * elem_sz; 158 159 /* requested more than the set limit */ 160 if (cur_cnt + add_cnt > max_cnt) 161 return NULL; 162 163 new_cnt = *cap_cnt; 164 new_cnt += new_cnt / 4; /* expand by 25% */ 165 if (new_cnt < 16) /* but at least 16 elements */ 166 new_cnt = 16; 167 if (new_cnt > max_cnt) /* but not exceeding a set limit */ 168 new_cnt = max_cnt; 169 if (new_cnt < cur_cnt + add_cnt) /* also ensure we have enough memory */ 170 new_cnt = cur_cnt + add_cnt; 171 172 new_data = libbpf_reallocarray(*data, new_cnt, elem_sz); 173 if (!new_data) 174 return NULL; 175 176 /* zero out newly allocated portion of memory */ 177 memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz); 178 179 *data = new_data; 180 *cap_cnt = new_cnt; 181 return new_data + cur_cnt * elem_sz; 182 } 183 184 /* Ensure given dynamically allocated memory region has enough allocated space 185 * to accommodate *need_cnt* elements of size *elem_sz* bytes each 186 */ 187 int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt) 188 { 189 void *p; 190 191 if (need_cnt <= *cap_cnt) 192 return 0; 193 194 p = libbpf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt); 195 if (!p) 196 return -ENOMEM; 197 198 return 0; 199 } 200 201 static void *btf_add_type_offs_mem(struct btf *btf, size_t add_cnt) 202 { 203 return libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32), 204 btf->nr_types, BTF_MAX_NR_TYPES, add_cnt); 205 } 206 207 static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off) 208 { 209 __u32 *p; 210 211 p = btf_add_type_offs_mem(btf, 1); 212 if (!p) 213 return -ENOMEM; 214 215 *p = type_off; 216 return 0; 217 } 218 219 static void btf_bswap_hdr(struct btf_header *h) 220 { 221 h->magic = bswap_16(h->magic); 222 h->hdr_len = bswap_32(h->hdr_len); 223 h->type_off = bswap_32(h->type_off); 224 h->type_len = bswap_32(h->type_len); 225 h->str_off = bswap_32(h->str_off); 226 h->str_len = bswap_32(h->str_len); 227 } 228 229 static int btf_parse_hdr(struct btf *btf) 230 { 231 struct btf_header *hdr = btf->hdr; 232 __u32 meta_left; 233 234 if (btf->raw_size < sizeof(struct btf_header)) { 235 pr_debug("BTF header not found\n"); 236 return -EINVAL; 237 } 238 239 if (hdr->magic == bswap_16(BTF_MAGIC)) { 240 btf->swapped_endian = true; 241 if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) { 242 pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n", 243 bswap_32(hdr->hdr_len)); 244 return -ENOTSUP; 245 } 246 btf_bswap_hdr(hdr); 247 } else if (hdr->magic != BTF_MAGIC) { 248 pr_debug("Invalid BTF magic: %x\n", hdr->magic); 249 return -EINVAL; 250 } 251 252 if (btf->raw_size < hdr->hdr_len) { 253 pr_debug("BTF header len %u larger than data size %u\n", 254 hdr->hdr_len, btf->raw_size); 255 return -EINVAL; 256 } 257 258 meta_left = btf->raw_size - hdr->hdr_len; 259 if (meta_left < (long long)hdr->str_off + hdr->str_len) { 260 pr_debug("Invalid BTF total size: %u\n", btf->raw_size); 261 return -EINVAL; 262 } 263 264 if ((long long)hdr->type_off + hdr->type_len > hdr->str_off) { 265 pr_debug("Invalid BTF data sections layout: type data at %u + %u, strings data at %u + %u\n", 266 hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len); 267 return -EINVAL; 268 } 269 270 if (hdr->type_off % 4) { 271 pr_debug("BTF type section is not aligned to 4 bytes\n"); 272 return -EINVAL; 273 } 274 275 return 0; 276 } 277 278 static int btf_parse_str_sec(struct btf *btf) 279 { 280 const struct btf_header *hdr = btf->hdr; 281 const char *start = btf->strs_data; 282 const char *end = start + btf->hdr->str_len; 283 284 if (btf->base_btf && hdr->str_len == 0) 285 return 0; 286 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) { 287 pr_debug("Invalid BTF string section\n"); 288 return -EINVAL; 289 } 290 if (!btf->base_btf && start[0]) { 291 pr_debug("Malformed BTF string section, did you forget to provide base BTF?\n"); 292 return -EINVAL; 293 } 294 return 0; 295 } 296 297 static int btf_type_size(const struct btf_type *t) 298 { 299 const int base_size = sizeof(struct btf_type); 300 __u16 vlen = btf_vlen(t); 301 302 switch (btf_kind(t)) { 303 case BTF_KIND_FWD: 304 case BTF_KIND_CONST: 305 case BTF_KIND_VOLATILE: 306 case BTF_KIND_RESTRICT: 307 case BTF_KIND_PTR: 308 case BTF_KIND_TYPEDEF: 309 case BTF_KIND_FUNC: 310 case BTF_KIND_FLOAT: 311 case BTF_KIND_TYPE_TAG: 312 return base_size; 313 case BTF_KIND_INT: 314 return base_size + sizeof(__u32); 315 case BTF_KIND_ENUM: 316 return base_size + vlen * sizeof(struct btf_enum); 317 case BTF_KIND_ENUM64: 318 return base_size + vlen * sizeof(struct btf_enum64); 319 case BTF_KIND_ARRAY: 320 return base_size + sizeof(struct btf_array); 321 case BTF_KIND_STRUCT: 322 case BTF_KIND_UNION: 323 return base_size + vlen * sizeof(struct btf_member); 324 case BTF_KIND_FUNC_PROTO: 325 return base_size + vlen * sizeof(struct btf_param); 326 case BTF_KIND_VAR: 327 return base_size + sizeof(struct btf_var); 328 case BTF_KIND_DATASEC: 329 return base_size + vlen * sizeof(struct btf_var_secinfo); 330 case BTF_KIND_DECL_TAG: 331 return base_size + sizeof(struct btf_decl_tag); 332 default: 333 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t)); 334 return -EINVAL; 335 } 336 } 337 338 static void btf_bswap_type_base(struct btf_type *t) 339 { 340 t->name_off = bswap_32(t->name_off); 341 t->info = bswap_32(t->info); 342 t->type = bswap_32(t->type); 343 } 344 345 static int btf_bswap_type_rest(struct btf_type *t) 346 { 347 struct btf_var_secinfo *v; 348 struct btf_enum64 *e64; 349 struct btf_member *m; 350 struct btf_array *a; 351 struct btf_param *p; 352 struct btf_enum *e; 353 __u16 vlen = btf_vlen(t); 354 int i; 355 356 switch (btf_kind(t)) { 357 case BTF_KIND_FWD: 358 case BTF_KIND_CONST: 359 case BTF_KIND_VOLATILE: 360 case BTF_KIND_RESTRICT: 361 case BTF_KIND_PTR: 362 case BTF_KIND_TYPEDEF: 363 case BTF_KIND_FUNC: 364 case BTF_KIND_FLOAT: 365 case BTF_KIND_TYPE_TAG: 366 return 0; 367 case BTF_KIND_INT: 368 *(__u32 *)(t + 1) = bswap_32(*(__u32 *)(t + 1)); 369 return 0; 370 case BTF_KIND_ENUM: 371 for (i = 0, e = btf_enum(t); i < vlen; i++, e++) { 372 e->name_off = bswap_32(e->name_off); 373 e->val = bswap_32(e->val); 374 } 375 return 0; 376 case BTF_KIND_ENUM64: 377 for (i = 0, e64 = btf_enum64(t); i < vlen; i++, e64++) { 378 e64->name_off = bswap_32(e64->name_off); 379 e64->val_lo32 = bswap_32(e64->val_lo32); 380 e64->val_hi32 = bswap_32(e64->val_hi32); 381 } 382 return 0; 383 case BTF_KIND_ARRAY: 384 a = btf_array(t); 385 a->type = bswap_32(a->type); 386 a->index_type = bswap_32(a->index_type); 387 a->nelems = bswap_32(a->nelems); 388 return 0; 389 case BTF_KIND_STRUCT: 390 case BTF_KIND_UNION: 391 for (i = 0, m = btf_members(t); i < vlen; i++, m++) { 392 m->name_off = bswap_32(m->name_off); 393 m->type = bswap_32(m->type); 394 m->offset = bswap_32(m->offset); 395 } 396 return 0; 397 case BTF_KIND_FUNC_PROTO: 398 for (i = 0, p = btf_params(t); i < vlen; i++, p++) { 399 p->name_off = bswap_32(p->name_off); 400 p->type = bswap_32(p->type); 401 } 402 return 0; 403 case BTF_KIND_VAR: 404 btf_var(t)->linkage = bswap_32(btf_var(t)->linkage); 405 return 0; 406 case BTF_KIND_DATASEC: 407 for (i = 0, v = btf_var_secinfos(t); i < vlen; i++, v++) { 408 v->type = bswap_32(v->type); 409 v->offset = bswap_32(v->offset); 410 v->size = bswap_32(v->size); 411 } 412 return 0; 413 case BTF_KIND_DECL_TAG: 414 btf_decl_tag(t)->component_idx = bswap_32(btf_decl_tag(t)->component_idx); 415 return 0; 416 default: 417 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t)); 418 return -EINVAL; 419 } 420 } 421 422 static int btf_parse_type_sec(struct btf *btf) 423 { 424 struct btf_header *hdr = btf->hdr; 425 void *next_type = btf->types_data; 426 void *end_type = next_type + hdr->type_len; 427 int err, type_size; 428 429 while (next_type + sizeof(struct btf_type) <= end_type) { 430 if (btf->swapped_endian) 431 btf_bswap_type_base(next_type); 432 433 type_size = btf_type_size(next_type); 434 if (type_size < 0) 435 return type_size; 436 if (next_type + type_size > end_type) { 437 pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types); 438 return -EINVAL; 439 } 440 441 if (btf->swapped_endian && btf_bswap_type_rest(next_type)) 442 return -EINVAL; 443 444 err = btf_add_type_idx_entry(btf, next_type - btf->types_data); 445 if (err) 446 return err; 447 448 next_type += type_size; 449 btf->nr_types++; 450 } 451 452 if (next_type != end_type) { 453 pr_warn("BTF types data is malformed\n"); 454 return -EINVAL; 455 } 456 457 return 0; 458 } 459 460 static int btf_validate_str(const struct btf *btf, __u32 str_off, const char *what, __u32 type_id) 461 { 462 const char *s; 463 464 s = btf__str_by_offset(btf, str_off); 465 if (!s) { 466 pr_warn("btf: type [%u]: invalid %s (string offset %u)\n", type_id, what, str_off); 467 return -EINVAL; 468 } 469 470 return 0; 471 } 472 473 static int btf_validate_id(const struct btf *btf, __u32 id, __u32 ctx_id) 474 { 475 const struct btf_type *t; 476 477 t = btf__type_by_id(btf, id); 478 if (!t) { 479 pr_warn("btf: type [%u]: invalid referenced type ID %u\n", ctx_id, id); 480 return -EINVAL; 481 } 482 483 return 0; 484 } 485 486 static int btf_validate_type(const struct btf *btf, const struct btf_type *t, __u32 id) 487 { 488 __u32 kind = btf_kind(t); 489 int err, i, n; 490 491 err = btf_validate_str(btf, t->name_off, "type name", id); 492 if (err) 493 return err; 494 495 switch (kind) { 496 case BTF_KIND_UNKN: 497 case BTF_KIND_INT: 498 case BTF_KIND_FWD: 499 case BTF_KIND_FLOAT: 500 break; 501 case BTF_KIND_PTR: 502 case BTF_KIND_TYPEDEF: 503 case BTF_KIND_VOLATILE: 504 case BTF_KIND_CONST: 505 case BTF_KIND_RESTRICT: 506 case BTF_KIND_VAR: 507 case BTF_KIND_DECL_TAG: 508 case BTF_KIND_TYPE_TAG: 509 err = btf_validate_id(btf, t->type, id); 510 if (err) 511 return err; 512 break; 513 case BTF_KIND_ARRAY: { 514 const struct btf_array *a = btf_array(t); 515 516 err = btf_validate_id(btf, a->type, id); 517 err = err ?: btf_validate_id(btf, a->index_type, id); 518 if (err) 519 return err; 520 break; 521 } 522 case BTF_KIND_STRUCT: 523 case BTF_KIND_UNION: { 524 const struct btf_member *m = btf_members(t); 525 526 n = btf_vlen(t); 527 for (i = 0; i < n; i++, m++) { 528 err = btf_validate_str(btf, m->name_off, "field name", id); 529 err = err ?: btf_validate_id(btf, m->type, id); 530 if (err) 531 return err; 532 } 533 break; 534 } 535 case BTF_KIND_ENUM: { 536 const struct btf_enum *m = btf_enum(t); 537 538 n = btf_vlen(t); 539 for (i = 0; i < n; i++, m++) { 540 err = btf_validate_str(btf, m->name_off, "enum name", id); 541 if (err) 542 return err; 543 } 544 break; 545 } 546 case BTF_KIND_ENUM64: { 547 const struct btf_enum64 *m = btf_enum64(t); 548 549 n = btf_vlen(t); 550 for (i = 0; i < n; i++, m++) { 551 err = btf_validate_str(btf, m->name_off, "enum name", id); 552 if (err) 553 return err; 554 } 555 break; 556 } 557 case BTF_KIND_FUNC: { 558 const struct btf_type *ft; 559 560 err = btf_validate_id(btf, t->type, id); 561 if (err) 562 return err; 563 ft = btf__type_by_id(btf, t->type); 564 if (btf_kind(ft) != BTF_KIND_FUNC_PROTO) { 565 pr_warn("btf: type [%u]: referenced type [%u] is not FUNC_PROTO\n", id, t->type); 566 return -EINVAL; 567 } 568 break; 569 } 570 case BTF_KIND_FUNC_PROTO: { 571 const struct btf_param *m = btf_params(t); 572 573 n = btf_vlen(t); 574 for (i = 0; i < n; i++, m++) { 575 err = btf_validate_str(btf, m->name_off, "param name", id); 576 err = err ?: btf_validate_id(btf, m->type, id); 577 if (err) 578 return err; 579 } 580 break; 581 } 582 case BTF_KIND_DATASEC: { 583 const struct btf_var_secinfo *m = btf_var_secinfos(t); 584 585 n = btf_vlen(t); 586 for (i = 0; i < n; i++, m++) { 587 err = btf_validate_id(btf, m->type, id); 588 if (err) 589 return err; 590 } 591 break; 592 } 593 default: 594 pr_warn("btf: type [%u]: unrecognized kind %u\n", id, kind); 595 return -EINVAL; 596 } 597 return 0; 598 } 599 600 /* Validate basic sanity of BTF. It's intentionally less thorough than 601 * kernel's validation and validates only properties of BTF that libbpf relies 602 * on to be correct (e.g., valid type IDs, valid string offsets, etc) 603 */ 604 static int btf_sanity_check(const struct btf *btf) 605 { 606 const struct btf_type *t; 607 __u32 i, n = btf__type_cnt(btf); 608 int err; 609 610 for (i = btf->start_id; i < n; i++) { 611 t = btf_type_by_id(btf, i); 612 err = btf_validate_type(btf, t, i); 613 if (err) 614 return err; 615 } 616 return 0; 617 } 618 619 __u32 btf__type_cnt(const struct btf *btf) 620 { 621 return btf->start_id + btf->nr_types; 622 } 623 624 const struct btf *btf__base_btf(const struct btf *btf) 625 { 626 return btf->base_btf; 627 } 628 629 /* internal helper returning non-const pointer to a type */ 630 struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id) 631 { 632 if (type_id == 0) 633 return &btf_void; 634 if (type_id < btf->start_id) 635 return btf_type_by_id(btf->base_btf, type_id); 636 return btf->types_data + btf->type_offs[type_id - btf->start_id]; 637 } 638 639 const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id) 640 { 641 if (type_id >= btf->start_id + btf->nr_types) 642 return errno = EINVAL, NULL; 643 return btf_type_by_id((struct btf *)btf, type_id); 644 } 645 646 static int determine_ptr_size(const struct btf *btf) 647 { 648 static const char * const long_aliases[] = { 649 "long", 650 "long int", 651 "int long", 652 "unsigned long", 653 "long unsigned", 654 "unsigned long int", 655 "unsigned int long", 656 "long unsigned int", 657 "long int unsigned", 658 "int unsigned long", 659 "int long unsigned", 660 }; 661 const struct btf_type *t; 662 const char *name; 663 int i, j, n; 664 665 if (btf->base_btf && btf->base_btf->ptr_sz > 0) 666 return btf->base_btf->ptr_sz; 667 668 n = btf__type_cnt(btf); 669 for (i = 1; i < n; i++) { 670 t = btf__type_by_id(btf, i); 671 if (!btf_is_int(t)) 672 continue; 673 674 if (t->size != 4 && t->size != 8) 675 continue; 676 677 name = btf__name_by_offset(btf, t->name_off); 678 if (!name) 679 continue; 680 681 for (j = 0; j < ARRAY_SIZE(long_aliases); j++) { 682 if (strcmp(name, long_aliases[j]) == 0) 683 return t->size; 684 } 685 } 686 687 return -1; 688 } 689 690 static size_t btf_ptr_sz(const struct btf *btf) 691 { 692 if (!btf->ptr_sz) 693 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); 694 return btf->ptr_sz < 0 ? sizeof(void *) : btf->ptr_sz; 695 } 696 697 /* Return pointer size this BTF instance assumes. The size is heuristically 698 * determined by looking for 'long' or 'unsigned long' integer type and 699 * recording its size in bytes. If BTF type information doesn't have any such 700 * type, this function returns 0. In the latter case, native architecture's 701 * pointer size is assumed, so will be either 4 or 8, depending on 702 * architecture that libbpf was compiled for. It's possible to override 703 * guessed value by using btf__set_pointer_size() API. 704 */ 705 size_t btf__pointer_size(const struct btf *btf) 706 { 707 if (!btf->ptr_sz) 708 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); 709 710 if (btf->ptr_sz < 0) 711 /* not enough BTF type info to guess */ 712 return 0; 713 714 return btf->ptr_sz; 715 } 716 717 /* Override or set pointer size in bytes. Only values of 4 and 8 are 718 * supported. 719 */ 720 int btf__set_pointer_size(struct btf *btf, size_t ptr_sz) 721 { 722 if (ptr_sz != 4 && ptr_sz != 8) 723 return libbpf_err(-EINVAL); 724 btf->ptr_sz = ptr_sz; 725 return 0; 726 } 727 728 static bool is_host_big_endian(void) 729 { 730 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 731 return false; 732 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 733 return true; 734 #else 735 # error "Unrecognized __BYTE_ORDER__" 736 #endif 737 } 738 739 enum btf_endianness btf__endianness(const struct btf *btf) 740 { 741 if (is_host_big_endian()) 742 return btf->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN; 743 else 744 return btf->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN; 745 } 746 747 int btf__set_endianness(struct btf *btf, enum btf_endianness endian) 748 { 749 if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN) 750 return libbpf_err(-EINVAL); 751 752 btf->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN); 753 if (!btf->swapped_endian) { 754 free(btf->raw_data_swapped); 755 btf->raw_data_swapped = NULL; 756 } 757 return 0; 758 } 759 760 static bool btf_type_is_void(const struct btf_type *t) 761 { 762 return t == &btf_void || btf_is_fwd(t); 763 } 764 765 static bool btf_type_is_void_or_null(const struct btf_type *t) 766 { 767 return !t || btf_type_is_void(t); 768 } 769 770 #define MAX_RESOLVE_DEPTH 32 771 772 __s64 btf__resolve_size(const struct btf *btf, __u32 type_id) 773 { 774 const struct btf_array *array; 775 const struct btf_type *t; 776 __u32 nelems = 1; 777 __s64 size = -1; 778 int i; 779 780 t = btf__type_by_id(btf, type_id); 781 for (i = 0; i < MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null(t); i++) { 782 switch (btf_kind(t)) { 783 case BTF_KIND_INT: 784 case BTF_KIND_STRUCT: 785 case BTF_KIND_UNION: 786 case BTF_KIND_ENUM: 787 case BTF_KIND_ENUM64: 788 case BTF_KIND_DATASEC: 789 case BTF_KIND_FLOAT: 790 size = t->size; 791 goto done; 792 case BTF_KIND_PTR: 793 size = btf_ptr_sz(btf); 794 goto done; 795 case BTF_KIND_TYPEDEF: 796 case BTF_KIND_VOLATILE: 797 case BTF_KIND_CONST: 798 case BTF_KIND_RESTRICT: 799 case BTF_KIND_VAR: 800 case BTF_KIND_DECL_TAG: 801 case BTF_KIND_TYPE_TAG: 802 type_id = t->type; 803 break; 804 case BTF_KIND_ARRAY: 805 array = btf_array(t); 806 if (nelems && array->nelems > UINT32_MAX / nelems) 807 return libbpf_err(-E2BIG); 808 nelems *= array->nelems; 809 type_id = array->type; 810 break; 811 default: 812 return libbpf_err(-EINVAL); 813 } 814 815 t = btf__type_by_id(btf, type_id); 816 } 817 818 done: 819 if (size < 0) 820 return libbpf_err(-EINVAL); 821 if (nelems && size > UINT32_MAX / nelems) 822 return libbpf_err(-E2BIG); 823 824 return nelems * size; 825 } 826 827 int btf__align_of(const struct btf *btf, __u32 id) 828 { 829 const struct btf_type *t = btf__type_by_id(btf, id); 830 __u16 kind = btf_kind(t); 831 832 switch (kind) { 833 case BTF_KIND_INT: 834 case BTF_KIND_ENUM: 835 case BTF_KIND_ENUM64: 836 case BTF_KIND_FLOAT: 837 return min(btf_ptr_sz(btf), (size_t)t->size); 838 case BTF_KIND_PTR: 839 return btf_ptr_sz(btf); 840 case BTF_KIND_TYPEDEF: 841 case BTF_KIND_VOLATILE: 842 case BTF_KIND_CONST: 843 case BTF_KIND_RESTRICT: 844 case BTF_KIND_TYPE_TAG: 845 return btf__align_of(btf, t->type); 846 case BTF_KIND_ARRAY: 847 return btf__align_of(btf, btf_array(t)->type); 848 case BTF_KIND_STRUCT: 849 case BTF_KIND_UNION: { 850 const struct btf_member *m = btf_members(t); 851 __u16 vlen = btf_vlen(t); 852 int i, max_align = 1, align; 853 854 for (i = 0; i < vlen; i++, m++) { 855 align = btf__align_of(btf, m->type); 856 if (align <= 0) 857 return libbpf_err(align); 858 max_align = max(max_align, align); 859 860 /* if field offset isn't aligned according to field 861 * type's alignment, then struct must be packed 862 */ 863 if (btf_member_bitfield_size(t, i) == 0 && 864 (m->offset % (8 * align)) != 0) 865 return 1; 866 } 867 868 /* if struct/union size isn't a multiple of its alignment, 869 * then struct must be packed 870 */ 871 if ((t->size % max_align) != 0) 872 return 1; 873 874 return max_align; 875 } 876 default: 877 pr_warn("unsupported BTF_KIND:%u\n", btf_kind(t)); 878 return errno = EINVAL, 0; 879 } 880 } 881 882 int btf__resolve_type(const struct btf *btf, __u32 type_id) 883 { 884 const struct btf_type *t; 885 int depth = 0; 886 887 t = btf__type_by_id(btf, type_id); 888 while (depth < MAX_RESOLVE_DEPTH && 889 !btf_type_is_void_or_null(t) && 890 (btf_is_mod(t) || btf_is_typedef(t) || btf_is_var(t))) { 891 type_id = t->type; 892 t = btf__type_by_id(btf, type_id); 893 depth++; 894 } 895 896 if (depth == MAX_RESOLVE_DEPTH || btf_type_is_void_or_null(t)) 897 return libbpf_err(-EINVAL); 898 899 return type_id; 900 } 901 902 static void btf_check_sorted(struct btf *btf) 903 { 904 __u32 i, n, named_start_id = 0; 905 906 n = btf__type_cnt(btf); 907 for (i = btf->start_id + 1; i < n; i++) { 908 struct btf_type *ta = btf_type_by_id(btf, i - 1); 909 struct btf_type *tb = btf_type_by_id(btf, i); 910 const char *na = btf__str_by_offset(btf, ta->name_off); 911 const char *nb = btf__str_by_offset(btf, tb->name_off); 912 913 if (strcmp(na, nb) > 0) 914 return; 915 916 if (named_start_id == 0 && na[0] != '\0') 917 named_start_id = i - 1; 918 if (named_start_id == 0 && nb[0] != '\0') 919 named_start_id = i; 920 } 921 922 if (named_start_id) 923 btf->named_start_id = named_start_id; 924 } 925 926 static __s32 btf_find_type_by_name_bsearch(const struct btf *btf, const char *name, 927 __s32 start_id) 928 { 929 const struct btf_type *t; 930 const char *tname; 931 __s32 l, r, m; 932 933 l = start_id; 934 r = btf__type_cnt(btf) - 1; 935 while (l <= r) { 936 m = l + (r - l) / 2; 937 t = btf_type_by_id(btf, m); 938 tname = btf__str_by_offset(btf, t->name_off); 939 if (strcmp(tname, name) >= 0) { 940 if (l == r) 941 return r; 942 r = m; 943 } else { 944 l = m + 1; 945 } 946 } 947 948 return btf__type_cnt(btf); 949 } 950 951 static __s32 btf_find_by_name_kind(const struct btf *btf, int start_id, 952 const char *type_name, __s32 kind) 953 { 954 __u32 nr_types = btf__type_cnt(btf); 955 const struct btf_type *t; 956 const char *tname; 957 __s32 id; 958 959 if (start_id < btf->start_id) { 960 id = btf_find_by_name_kind(btf->base_btf, start_id, 961 type_name, kind); 962 if (id >= 0) 963 return id; 964 start_id = btf->start_id; 965 } 966 967 if (kind == BTF_KIND_UNKN || strcmp(type_name, "void") == 0) 968 return 0; 969 970 if (btf->named_start_id > 0 && type_name[0]) { 971 start_id = max(start_id, btf->named_start_id); 972 id = btf_find_type_by_name_bsearch(btf, type_name, start_id); 973 for (; id < nr_types; id++) { 974 t = btf__type_by_id(btf, id); 975 tname = btf__str_by_offset(btf, t->name_off); 976 if (strcmp(tname, type_name) != 0) 977 return libbpf_err(-ENOENT); 978 if (kind < 0 || btf_kind(t) == kind) 979 return id; 980 } 981 } else { 982 for (id = start_id; id < nr_types; id++) { 983 t = btf_type_by_id(btf, id); 984 if (kind > 0 && btf_kind(t) != kind) 985 continue; 986 tname = btf__str_by_offset(btf, t->name_off); 987 if (strcmp(tname, type_name) == 0) 988 return id; 989 } 990 } 991 992 return libbpf_err(-ENOENT); 993 } 994 995 /* the kind value of -1 indicates that kind matching should be skipped */ 996 __s32 btf__find_by_name(const struct btf *btf, const char *type_name) 997 { 998 return btf_find_by_name_kind(btf, 1, type_name, -1); 999 } 1000 1001 __s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name, 1002 __u32 kind) 1003 { 1004 return btf_find_by_name_kind(btf, btf->start_id, type_name, kind); 1005 } 1006 1007 __s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name, 1008 __u32 kind) 1009 { 1010 return btf_find_by_name_kind(btf, 1, type_name, kind); 1011 } 1012 1013 static bool btf_is_modifiable(const struct btf *btf) 1014 { 1015 return (void *)btf->hdr != btf->raw_data; 1016 } 1017 1018 static void btf_free_raw_data(struct btf *btf) 1019 { 1020 if (btf->raw_data_is_mmap) { 1021 munmap(btf->raw_data, btf->raw_size); 1022 btf->raw_data_is_mmap = false; 1023 } else { 1024 free(btf->raw_data); 1025 } 1026 btf->raw_data = NULL; 1027 } 1028 1029 void btf__free(struct btf *btf) 1030 { 1031 if (IS_ERR_OR_NULL(btf)) 1032 return; 1033 1034 if (btf->fd >= 0) 1035 close(btf->fd); 1036 1037 if (btf_is_modifiable(btf)) { 1038 /* if BTF was modified after loading, it will have a split 1039 * in-memory representation for header, types, and strings 1040 * sections, so we need to free all of them individually. It 1041 * might still have a cached contiguous raw data present, 1042 * which will be unconditionally freed below. 1043 */ 1044 free(btf->hdr); 1045 free(btf->types_data); 1046 strset__free(btf->strs_set); 1047 } 1048 btf_free_raw_data(btf); 1049 free(btf->raw_data_swapped); 1050 free(btf->type_offs); 1051 if (btf->owns_base) 1052 btf__free(btf->base_btf); 1053 free(btf); 1054 } 1055 1056 static struct btf *btf_new_empty(struct btf *base_btf) 1057 { 1058 struct btf *btf; 1059 1060 btf = calloc(1, sizeof(*btf)); 1061 if (!btf) 1062 return ERR_PTR(-ENOMEM); 1063 1064 btf->nr_types = 0; 1065 btf->start_id = 1; 1066 btf->start_str_off = 0; 1067 btf->fd = -1; 1068 btf->ptr_sz = sizeof(void *); 1069 btf->swapped_endian = false; 1070 btf->named_start_id = 0; 1071 1072 if (base_btf) { 1073 btf->base_btf = base_btf; 1074 btf->start_id = btf__type_cnt(base_btf); 1075 btf->start_str_off = base_btf->hdr->str_len + base_btf->start_str_off; 1076 btf->swapped_endian = base_btf->swapped_endian; 1077 } 1078 1079 /* +1 for empty string at offset 0 */ 1080 btf->raw_size = sizeof(struct btf_header) + (base_btf ? 0 : 1); 1081 btf->raw_data = calloc(1, btf->raw_size); 1082 if (!btf->raw_data) { 1083 free(btf); 1084 return ERR_PTR(-ENOMEM); 1085 } 1086 1087 btf->hdr = btf->raw_data; 1088 btf->hdr->hdr_len = sizeof(struct btf_header); 1089 btf->hdr->magic = BTF_MAGIC; 1090 btf->hdr->version = BTF_VERSION; 1091 1092 btf->types_data = btf->raw_data + btf->hdr->hdr_len; 1093 btf->strs_data = btf->raw_data + btf->hdr->hdr_len; 1094 btf->hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */ 1095 1096 return btf; 1097 } 1098 1099 struct btf *btf__new_empty(void) 1100 { 1101 return libbpf_ptr(btf_new_empty(NULL)); 1102 } 1103 1104 struct btf *btf__new_empty_split(struct btf *base_btf) 1105 { 1106 return libbpf_ptr(btf_new_empty(base_btf)); 1107 } 1108 1109 static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf, bool is_mmap) 1110 { 1111 struct btf *btf; 1112 int err; 1113 1114 btf = calloc(1, sizeof(struct btf)); 1115 if (!btf) 1116 return ERR_PTR(-ENOMEM); 1117 1118 btf->nr_types = 0; 1119 btf->start_id = 1; 1120 btf->start_str_off = 0; 1121 btf->fd = -1; 1122 btf->named_start_id = 0; 1123 1124 if (base_btf) { 1125 btf->base_btf = base_btf; 1126 btf->start_id = btf__type_cnt(base_btf); 1127 btf->start_str_off = base_btf->hdr->str_len + base_btf->start_str_off; 1128 } 1129 1130 if (is_mmap) { 1131 btf->raw_data = (void *)data; 1132 btf->raw_data_is_mmap = true; 1133 } else { 1134 btf->raw_data = malloc(size); 1135 if (!btf->raw_data) { 1136 err = -ENOMEM; 1137 goto done; 1138 } 1139 memcpy(btf->raw_data, data, size); 1140 } 1141 1142 btf->raw_size = size; 1143 1144 btf->hdr = btf->raw_data; 1145 err = btf_parse_hdr(btf); 1146 if (err) 1147 goto done; 1148 1149 btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off; 1150 btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off; 1151 1152 err = btf_parse_str_sec(btf); 1153 err = err ?: btf_parse_type_sec(btf); 1154 err = err ?: btf_sanity_check(btf); 1155 if (err) 1156 goto done; 1157 btf_check_sorted(btf); 1158 1159 done: 1160 if (err) { 1161 btf__free(btf); 1162 return ERR_PTR(err); 1163 } 1164 1165 return btf; 1166 } 1167 1168 struct btf *btf__new(const void *data, __u32 size) 1169 { 1170 return libbpf_ptr(btf_new(data, size, NULL, false)); 1171 } 1172 1173 struct btf *btf__new_split(const void *data, __u32 size, struct btf *base_btf) 1174 { 1175 return libbpf_ptr(btf_new(data, size, base_btf, false)); 1176 } 1177 1178 struct btf_elf_secs { 1179 Elf_Data *btf_data; 1180 Elf_Data *btf_ext_data; 1181 Elf_Data *btf_base_data; 1182 }; 1183 1184 static int btf_find_elf_sections(Elf *elf, const char *path, struct btf_elf_secs *secs) 1185 { 1186 Elf_Scn *scn = NULL; 1187 Elf_Data *data; 1188 GElf_Ehdr ehdr; 1189 size_t shstrndx; 1190 int idx = 0; 1191 1192 if (!gelf_getehdr(elf, &ehdr)) { 1193 pr_warn("failed to get EHDR from %s\n", path); 1194 goto err; 1195 } 1196 1197 if (elf_getshdrstrndx(elf, &shstrndx)) { 1198 pr_warn("failed to get section names section index for %s\n", 1199 path); 1200 goto err; 1201 } 1202 1203 if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL)) { 1204 pr_warn("failed to get e_shstrndx from %s\n", path); 1205 goto err; 1206 } 1207 1208 while ((scn = elf_nextscn(elf, scn)) != NULL) { 1209 Elf_Data **field; 1210 GElf_Shdr sh; 1211 char *name; 1212 1213 idx++; 1214 if (gelf_getshdr(scn, &sh) != &sh) { 1215 pr_warn("failed to get section(%d) header from %s\n", 1216 idx, path); 1217 goto err; 1218 } 1219 name = elf_strptr(elf, shstrndx, sh.sh_name); 1220 if (!name) { 1221 pr_warn("failed to get section(%d) name from %s\n", 1222 idx, path); 1223 goto err; 1224 } 1225 1226 if (strcmp(name, BTF_ELF_SEC) == 0) 1227 field = &secs->btf_data; 1228 else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) 1229 field = &secs->btf_ext_data; 1230 else if (strcmp(name, BTF_BASE_ELF_SEC) == 0) 1231 field = &secs->btf_base_data; 1232 else 1233 continue; 1234 1235 if (sh.sh_type != SHT_PROGBITS) { 1236 pr_warn("unexpected section type (%d) of section(%d, %s) from %s\n", 1237 sh.sh_type, idx, name, path); 1238 goto err; 1239 } 1240 1241 data = elf_getdata(scn, 0); 1242 if (!data) { 1243 pr_warn("failed to get section(%d, %s) data from %s\n", 1244 idx, name, path); 1245 goto err; 1246 } 1247 *field = data; 1248 } 1249 1250 return 0; 1251 1252 err: 1253 return -LIBBPF_ERRNO__FORMAT; 1254 } 1255 1256 static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, 1257 struct btf_ext **btf_ext) 1258 { 1259 struct btf_elf_secs secs = {}; 1260 struct btf *dist_base_btf = NULL; 1261 struct btf *btf = NULL; 1262 int err = 0, fd = -1; 1263 Elf *elf = NULL; 1264 1265 if (elf_version(EV_CURRENT) == EV_NONE) { 1266 pr_warn("failed to init libelf for %s\n", path); 1267 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 1268 } 1269 1270 fd = open(path, O_RDONLY | O_CLOEXEC); 1271 if (fd < 0) { 1272 err = -errno; 1273 pr_warn("failed to open %s: %s\n", path, errstr(err)); 1274 return ERR_PTR(err); 1275 } 1276 1277 elf = elf_begin(fd, ELF_C_READ, NULL); 1278 if (!elf) { 1279 err = -LIBBPF_ERRNO__FORMAT; 1280 pr_warn("failed to open %s as ELF file\n", path); 1281 goto done; 1282 } 1283 1284 err = btf_find_elf_sections(elf, path, &secs); 1285 if (err) 1286 goto done; 1287 1288 if (!secs.btf_data) { 1289 pr_warn("failed to find '%s' ELF section in %s\n", BTF_ELF_SEC, path); 1290 err = -ENODATA; 1291 goto done; 1292 } 1293 1294 if (secs.btf_base_data) { 1295 dist_base_btf = btf_new(secs.btf_base_data->d_buf, secs.btf_base_data->d_size, 1296 NULL, false); 1297 if (IS_ERR(dist_base_btf)) { 1298 err = PTR_ERR(dist_base_btf); 1299 dist_base_btf = NULL; 1300 goto done; 1301 } 1302 } 1303 1304 btf = btf_new(secs.btf_data->d_buf, secs.btf_data->d_size, 1305 dist_base_btf ?: base_btf, false); 1306 if (IS_ERR(btf)) { 1307 err = PTR_ERR(btf); 1308 goto done; 1309 } 1310 if (dist_base_btf && base_btf) { 1311 err = btf__relocate(btf, base_btf); 1312 if (err) 1313 goto done; 1314 btf__free(dist_base_btf); 1315 dist_base_btf = NULL; 1316 } 1317 1318 if (dist_base_btf) 1319 btf->owns_base = true; 1320 1321 switch (gelf_getclass(elf)) { 1322 case ELFCLASS32: 1323 btf__set_pointer_size(btf, 4); 1324 break; 1325 case ELFCLASS64: 1326 btf__set_pointer_size(btf, 8); 1327 break; 1328 default: 1329 pr_warn("failed to get ELF class (bitness) for %s\n", path); 1330 break; 1331 } 1332 1333 if (btf_ext && secs.btf_ext_data) { 1334 *btf_ext = btf_ext__new(secs.btf_ext_data->d_buf, secs.btf_ext_data->d_size); 1335 if (IS_ERR(*btf_ext)) { 1336 err = PTR_ERR(*btf_ext); 1337 goto done; 1338 } 1339 } else if (btf_ext) { 1340 *btf_ext = NULL; 1341 } 1342 done: 1343 if (elf) 1344 elf_end(elf); 1345 close(fd); 1346 1347 if (!err) 1348 return btf; 1349 1350 if (btf_ext) 1351 btf_ext__free(*btf_ext); 1352 btf__free(dist_base_btf); 1353 btf__free(btf); 1354 1355 return ERR_PTR(err); 1356 } 1357 1358 struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext) 1359 { 1360 return libbpf_ptr(btf_parse_elf(path, NULL, btf_ext)); 1361 } 1362 1363 struct btf *btf__parse_elf_split(const char *path, struct btf *base_btf) 1364 { 1365 return libbpf_ptr(btf_parse_elf(path, base_btf, NULL)); 1366 } 1367 1368 static struct btf *btf_parse_raw(const char *path, struct btf *base_btf) 1369 { 1370 struct btf *btf = NULL; 1371 void *data = NULL; 1372 FILE *f = NULL; 1373 __u16 magic; 1374 int err = 0; 1375 long sz; 1376 1377 f = fopen(path, "rbe"); 1378 if (!f) { 1379 err = -errno; 1380 goto err_out; 1381 } 1382 1383 /* check BTF magic */ 1384 if (fread(&magic, 1, sizeof(magic), f) < sizeof(magic)) { 1385 err = -EIO; 1386 goto err_out; 1387 } 1388 if (magic != BTF_MAGIC && magic != bswap_16(BTF_MAGIC)) { 1389 /* definitely not a raw BTF */ 1390 err = -EPROTO; 1391 goto err_out; 1392 } 1393 1394 /* get file size */ 1395 if (fseek(f, 0, SEEK_END)) { 1396 err = -errno; 1397 goto err_out; 1398 } 1399 sz = ftell(f); 1400 if (sz < 0) { 1401 err = -errno; 1402 goto err_out; 1403 } 1404 /* rewind to the start */ 1405 if (fseek(f, 0, SEEK_SET)) { 1406 err = -errno; 1407 goto err_out; 1408 } 1409 1410 /* pre-alloc memory and read all of BTF data */ 1411 data = malloc(sz); 1412 if (!data) { 1413 err = -ENOMEM; 1414 goto err_out; 1415 } 1416 if (fread(data, 1, sz, f) < sz) { 1417 err = -EIO; 1418 goto err_out; 1419 } 1420 1421 /* finally parse BTF data */ 1422 btf = btf_new(data, sz, base_btf, false); 1423 1424 err_out: 1425 free(data); 1426 if (f) 1427 fclose(f); 1428 return err ? ERR_PTR(err) : btf; 1429 } 1430 1431 struct btf *btf__parse_raw(const char *path) 1432 { 1433 return libbpf_ptr(btf_parse_raw(path, NULL)); 1434 } 1435 1436 struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf) 1437 { 1438 return libbpf_ptr(btf_parse_raw(path, base_btf)); 1439 } 1440 1441 static struct btf *btf_parse_raw_mmap(const char *path, struct btf *base_btf) 1442 { 1443 struct stat st; 1444 void *data; 1445 struct btf *btf; 1446 int fd, err; 1447 1448 fd = open(path, O_RDONLY); 1449 if (fd < 0) 1450 return ERR_PTR(-errno); 1451 1452 if (fstat(fd, &st) < 0) { 1453 err = -errno; 1454 close(fd); 1455 return ERR_PTR(err); 1456 } 1457 1458 data = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); 1459 err = -errno; 1460 close(fd); 1461 1462 if (data == MAP_FAILED) 1463 return ERR_PTR(err); 1464 1465 btf = btf_new(data, st.st_size, base_btf, true); 1466 if (IS_ERR(btf)) 1467 munmap(data, st.st_size); 1468 1469 return btf; 1470 } 1471 1472 static struct btf *btf_parse(const char *path, struct btf *base_btf, struct btf_ext **btf_ext) 1473 { 1474 struct btf *btf; 1475 int err; 1476 1477 if (btf_ext) 1478 *btf_ext = NULL; 1479 1480 btf = btf_parse_raw(path, base_btf); 1481 err = libbpf_get_error(btf); 1482 if (!err) 1483 return btf; 1484 if (err != -EPROTO) 1485 return ERR_PTR(err); 1486 return btf_parse_elf(path, base_btf, btf_ext); 1487 } 1488 1489 struct btf *btf__parse(const char *path, struct btf_ext **btf_ext) 1490 { 1491 return libbpf_ptr(btf_parse(path, NULL, btf_ext)); 1492 } 1493 1494 struct btf *btf__parse_split(const char *path, struct btf *base_btf) 1495 { 1496 return libbpf_ptr(btf_parse(path, base_btf, NULL)); 1497 } 1498 1499 static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian); 1500 1501 int btf_load_into_kernel(struct btf *btf, 1502 char *log_buf, size_t log_sz, __u32 log_level, 1503 int token_fd) 1504 { 1505 LIBBPF_OPTS(bpf_btf_load_opts, opts); 1506 __u32 buf_sz = 0, raw_size; 1507 char *buf = NULL, *tmp; 1508 void *raw_data; 1509 int err = 0; 1510 1511 if (btf->fd >= 0) 1512 return libbpf_err(-EEXIST); 1513 if (log_sz && !log_buf) 1514 return libbpf_err(-EINVAL); 1515 1516 /* cache native raw data representation */ 1517 raw_data = btf_get_raw_data(btf, &raw_size, false); 1518 if (!raw_data) { 1519 err = -ENOMEM; 1520 goto done; 1521 } 1522 btf->raw_size = raw_size; 1523 btf->raw_data = raw_data; 1524 1525 retry_load: 1526 /* if log_level is 0, we won't provide log_buf/log_size to the kernel, 1527 * initially. Only if BTF loading fails, we bump log_level to 1 and 1528 * retry, using either auto-allocated or custom log_buf. This way 1529 * non-NULL custom log_buf provides a buffer just in case, but hopes 1530 * for successful load and no need for log_buf. 1531 */ 1532 if (log_level) { 1533 /* if caller didn't provide custom log_buf, we'll keep 1534 * allocating our own progressively bigger buffers for BTF 1535 * verification log 1536 */ 1537 if (!log_buf) { 1538 buf_sz = max((__u32)BPF_LOG_BUF_SIZE, buf_sz * 2); 1539 tmp = realloc(buf, buf_sz); 1540 if (!tmp) { 1541 err = -ENOMEM; 1542 goto done; 1543 } 1544 buf = tmp; 1545 buf[0] = '\0'; 1546 } 1547 1548 opts.log_buf = log_buf ? log_buf : buf; 1549 opts.log_size = log_buf ? log_sz : buf_sz; 1550 opts.log_level = log_level; 1551 } 1552 1553 opts.token_fd = token_fd; 1554 if (token_fd) 1555 opts.btf_flags |= BPF_F_TOKEN_FD; 1556 1557 btf->fd = bpf_btf_load(raw_data, raw_size, &opts); 1558 if (btf->fd < 0) { 1559 /* time to turn on verbose mode and try again */ 1560 if (log_level == 0) { 1561 log_level = 1; 1562 goto retry_load; 1563 } 1564 /* only retry if caller didn't provide custom log_buf, but 1565 * make sure we can never overflow buf_sz 1566 */ 1567 if (!log_buf && errno == ENOSPC && buf_sz <= UINT_MAX / 2) 1568 goto retry_load; 1569 1570 err = -errno; 1571 pr_warn("BTF loading error: %s\n", errstr(err)); 1572 /* don't print out contents of custom log_buf */ 1573 if (!log_buf && buf[0]) 1574 pr_warn("-- BEGIN BTF LOAD LOG ---\n%s\n-- END BTF LOAD LOG --\n", buf); 1575 } 1576 1577 done: 1578 free(buf); 1579 return libbpf_err(err); 1580 } 1581 1582 int btf__load_into_kernel(struct btf *btf) 1583 { 1584 return btf_load_into_kernel(btf, NULL, 0, 0, 0); 1585 } 1586 1587 int btf__fd(const struct btf *btf) 1588 { 1589 return btf->fd; 1590 } 1591 1592 void btf__set_fd(struct btf *btf, int fd) 1593 { 1594 btf->fd = fd; 1595 } 1596 1597 static const void *btf_strs_data(const struct btf *btf) 1598 { 1599 return btf->strs_data ? btf->strs_data : strset__data(btf->strs_set); 1600 } 1601 1602 static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian) 1603 { 1604 struct btf_header *hdr = btf->hdr; 1605 struct btf_type *t; 1606 void *data, *p; 1607 __u32 data_sz; 1608 int i; 1609 1610 data = swap_endian ? btf->raw_data_swapped : btf->raw_data; 1611 if (data) { 1612 *size = btf->raw_size; 1613 return data; 1614 } 1615 1616 data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len; 1617 data = calloc(1, data_sz); 1618 if (!data) 1619 return NULL; 1620 p = data; 1621 1622 memcpy(p, hdr, hdr->hdr_len); 1623 if (swap_endian) 1624 btf_bswap_hdr(p); 1625 p += hdr->hdr_len; 1626 1627 memcpy(p, btf->types_data, hdr->type_len); 1628 if (swap_endian) { 1629 for (i = 0; i < btf->nr_types; i++) { 1630 t = p + btf->type_offs[i]; 1631 /* btf_bswap_type_rest() relies on native t->info, so 1632 * we swap base type info after we swapped all the 1633 * additional information 1634 */ 1635 if (btf_bswap_type_rest(t)) 1636 goto err_out; 1637 btf_bswap_type_base(t); 1638 } 1639 } 1640 p += hdr->type_len; 1641 1642 memcpy(p, btf_strs_data(btf), hdr->str_len); 1643 p += hdr->str_len; 1644 1645 *size = data_sz; 1646 return data; 1647 err_out: 1648 free(data); 1649 return NULL; 1650 } 1651 1652 const void *btf__raw_data(const struct btf *btf_ro, __u32 *size) 1653 { 1654 struct btf *btf = (struct btf *)btf_ro; 1655 __u32 data_sz; 1656 void *data; 1657 1658 data = btf_get_raw_data(btf, &data_sz, btf->swapped_endian); 1659 if (!data) 1660 return errno = ENOMEM, NULL; 1661 1662 btf->raw_size = data_sz; 1663 if (btf->swapped_endian) 1664 btf->raw_data_swapped = data; 1665 else 1666 btf->raw_data = data; 1667 *size = data_sz; 1668 return data; 1669 } 1670 1671 __attribute__((alias("btf__raw_data"))) 1672 const void *btf__get_raw_data(const struct btf *btf, __u32 *size); 1673 1674 const char *btf__str_by_offset(const struct btf *btf, __u32 offset) 1675 { 1676 if (offset < btf->start_str_off) 1677 return btf__str_by_offset(btf->base_btf, offset); 1678 else if (offset - btf->start_str_off < btf->hdr->str_len) 1679 return btf_strs_data(btf) + (offset - btf->start_str_off); 1680 else 1681 return errno = EINVAL, NULL; 1682 } 1683 1684 const char *btf__name_by_offset(const struct btf *btf, __u32 offset) 1685 { 1686 return btf__str_by_offset(btf, offset); 1687 } 1688 1689 struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf) 1690 { 1691 struct bpf_btf_info btf_info; 1692 __u32 len = sizeof(btf_info); 1693 __u32 last_size; 1694 struct btf *btf; 1695 void *ptr; 1696 int err; 1697 1698 /* we won't know btf_size until we call bpf_btf_get_info_by_fd(). so 1699 * let's start with a sane default - 4KiB here - and resize it only if 1700 * bpf_btf_get_info_by_fd() needs a bigger buffer. 1701 */ 1702 last_size = 4096; 1703 ptr = malloc(last_size); 1704 if (!ptr) 1705 return ERR_PTR(-ENOMEM); 1706 1707 memset(&btf_info, 0, sizeof(btf_info)); 1708 btf_info.btf = ptr_to_u64(ptr); 1709 btf_info.btf_size = last_size; 1710 err = bpf_btf_get_info_by_fd(btf_fd, &btf_info, &len); 1711 1712 if (!err && btf_info.btf_size > last_size) { 1713 void *temp_ptr; 1714 1715 last_size = btf_info.btf_size; 1716 temp_ptr = realloc(ptr, last_size); 1717 if (!temp_ptr) { 1718 btf = ERR_PTR(-ENOMEM); 1719 goto exit_free; 1720 } 1721 ptr = temp_ptr; 1722 1723 len = sizeof(btf_info); 1724 memset(&btf_info, 0, sizeof(btf_info)); 1725 btf_info.btf = ptr_to_u64(ptr); 1726 btf_info.btf_size = last_size; 1727 1728 err = bpf_btf_get_info_by_fd(btf_fd, &btf_info, &len); 1729 } 1730 1731 if (err || btf_info.btf_size > last_size) { 1732 btf = err ? ERR_PTR(-errno) : ERR_PTR(-E2BIG); 1733 goto exit_free; 1734 } 1735 1736 btf = btf_new(ptr, btf_info.btf_size, base_btf, false); 1737 1738 exit_free: 1739 free(ptr); 1740 return btf; 1741 } 1742 1743 struct btf *btf_load_from_kernel(__u32 id, struct btf *base_btf, int token_fd) 1744 { 1745 struct btf *btf; 1746 int btf_fd; 1747 LIBBPF_OPTS(bpf_get_fd_by_id_opts, opts); 1748 1749 if (token_fd) { 1750 opts.open_flags |= BPF_F_TOKEN_FD; 1751 opts.token_fd = token_fd; 1752 } 1753 1754 btf_fd = bpf_btf_get_fd_by_id_opts(id, &opts); 1755 if (btf_fd < 0) 1756 return libbpf_err_ptr(-errno); 1757 1758 btf = btf_get_from_fd(btf_fd, base_btf); 1759 close(btf_fd); 1760 1761 return libbpf_ptr(btf); 1762 } 1763 1764 struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf) 1765 { 1766 return btf_load_from_kernel(id, base_btf, 0); 1767 } 1768 1769 struct btf *btf__load_from_kernel_by_id(__u32 id) 1770 { 1771 return btf__load_from_kernel_by_id_split(id, NULL); 1772 } 1773 1774 static void btf_invalidate_raw_data(struct btf *btf) 1775 { 1776 if (btf->raw_data) 1777 btf_free_raw_data(btf); 1778 if (btf->raw_data_swapped) { 1779 free(btf->raw_data_swapped); 1780 btf->raw_data_swapped = NULL; 1781 } 1782 btf->named_start_id = 0; 1783 } 1784 1785 /* Ensure BTF is ready to be modified (by splitting into a three memory 1786 * regions for header, types, and strings). Also invalidate cached 1787 * raw_data, if any. 1788 */ 1789 static int btf_ensure_modifiable(struct btf *btf) 1790 { 1791 void *hdr, *types; 1792 struct strset *set = NULL; 1793 int err = -ENOMEM; 1794 1795 if (btf_is_modifiable(btf)) { 1796 /* any BTF modification invalidates raw_data */ 1797 btf_invalidate_raw_data(btf); 1798 return 0; 1799 } 1800 1801 /* split raw data into three memory regions */ 1802 hdr = malloc(btf->hdr->hdr_len); 1803 types = malloc(btf->hdr->type_len); 1804 if (!hdr || !types) 1805 goto err_out; 1806 1807 memcpy(hdr, btf->hdr, btf->hdr->hdr_len); 1808 memcpy(types, btf->types_data, btf->hdr->type_len); 1809 1810 /* build lookup index for all strings */ 1811 set = strset__new(BTF_MAX_STR_OFFSET, btf->strs_data, btf->hdr->str_len); 1812 if (IS_ERR(set)) { 1813 err = PTR_ERR(set); 1814 goto err_out; 1815 } 1816 1817 /* only when everything was successful, update internal state */ 1818 btf->hdr = hdr; 1819 btf->types_data = types; 1820 btf->types_data_cap = btf->hdr->type_len; 1821 btf->strs_data = NULL; 1822 btf->strs_set = set; 1823 /* if BTF was created from scratch, all strings are guaranteed to be 1824 * unique and deduplicated 1825 */ 1826 if (btf->hdr->str_len == 0) 1827 btf->strs_deduped = true; 1828 if (!btf->base_btf && btf->hdr->str_len == 1) 1829 btf->strs_deduped = true; 1830 1831 /* invalidate raw_data representation */ 1832 btf_invalidate_raw_data(btf); 1833 1834 return 0; 1835 1836 err_out: 1837 strset__free(set); 1838 free(hdr); 1839 free(types); 1840 return err; 1841 } 1842 1843 /* Find an offset in BTF string section that corresponds to a given string *s*. 1844 * Returns: 1845 * - >0 offset into string section, if string is found; 1846 * - -ENOENT, if string is not in the string section; 1847 * - <0, on any other error. 1848 */ 1849 int btf__find_str(struct btf *btf, const char *s) 1850 { 1851 int off; 1852 1853 if (btf->base_btf) { 1854 off = btf__find_str(btf->base_btf, s); 1855 if (off != -ENOENT) 1856 return off; 1857 } 1858 1859 /* BTF needs to be in a modifiable state to build string lookup index */ 1860 if (btf_ensure_modifiable(btf)) 1861 return libbpf_err(-ENOMEM); 1862 1863 off = strset__find_str(btf->strs_set, s); 1864 if (off < 0) 1865 return libbpf_err(off); 1866 1867 return btf->start_str_off + off; 1868 } 1869 1870 /* Add a string s to the BTF string section. 1871 * Returns: 1872 * - > 0 offset into string section, on success; 1873 * - < 0, on error. 1874 */ 1875 int btf__add_str(struct btf *btf, const char *s) 1876 { 1877 int off; 1878 1879 if (btf->base_btf) { 1880 off = btf__find_str(btf->base_btf, s); 1881 if (off != -ENOENT) 1882 return off; 1883 } 1884 1885 if (btf_ensure_modifiable(btf)) 1886 return libbpf_err(-ENOMEM); 1887 1888 off = strset__add_str(btf->strs_set, s); 1889 if (off < 0) 1890 return libbpf_err(off); 1891 1892 btf->hdr->str_len = strset__data_size(btf->strs_set); 1893 1894 return btf->start_str_off + off; 1895 } 1896 1897 static void *btf_add_type_mem(struct btf *btf, size_t add_sz) 1898 { 1899 return libbpf_add_mem(&btf->types_data, &btf->types_data_cap, 1, 1900 btf->hdr->type_len, UINT_MAX, add_sz); 1901 } 1902 1903 static void btf_type_inc_vlen(struct btf_type *t) 1904 { 1905 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t)); 1906 } 1907 1908 static int btf_commit_type(struct btf *btf, int data_sz) 1909 { 1910 int err; 1911 1912 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); 1913 if (err) 1914 return libbpf_err(err); 1915 1916 btf->hdr->type_len += data_sz; 1917 btf->hdr->str_off += data_sz; 1918 btf->nr_types++; 1919 return btf->start_id + btf->nr_types - 1; 1920 } 1921 1922 struct btf_pipe { 1923 const struct btf *src; 1924 struct btf *dst; 1925 struct hashmap *str_off_map; /* map string offsets from src to dst */ 1926 }; 1927 1928 static int btf_rewrite_str(struct btf_pipe *p, __u32 *str_off) 1929 { 1930 long mapped_off; 1931 int off, err; 1932 1933 if (!*str_off) /* nothing to do for empty strings */ 1934 return 0; 1935 1936 if (p->str_off_map && 1937 hashmap__find(p->str_off_map, *str_off, &mapped_off)) { 1938 *str_off = mapped_off; 1939 return 0; 1940 } 1941 1942 off = btf__add_str(p->dst, btf__str_by_offset(p->src, *str_off)); 1943 if (off < 0) 1944 return off; 1945 1946 /* Remember string mapping from src to dst. It avoids 1947 * performing expensive string comparisons. 1948 */ 1949 if (p->str_off_map) { 1950 err = hashmap__append(p->str_off_map, *str_off, off); 1951 if (err) 1952 return err; 1953 } 1954 1955 *str_off = off; 1956 return 0; 1957 } 1958 1959 static int btf_add_type(struct btf_pipe *p, const struct btf_type *src_type) 1960 { 1961 struct btf_field_iter it; 1962 struct btf_type *t; 1963 __u32 *str_off; 1964 int sz, err; 1965 1966 sz = btf_type_size(src_type); 1967 if (sz < 0) 1968 return libbpf_err(sz); 1969 1970 /* deconstruct BTF, if necessary, and invalidate raw_data */ 1971 if (btf_ensure_modifiable(p->dst)) 1972 return libbpf_err(-ENOMEM); 1973 1974 t = btf_add_type_mem(p->dst, sz); 1975 if (!t) 1976 return libbpf_err(-ENOMEM); 1977 1978 memcpy(t, src_type, sz); 1979 1980 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS); 1981 if (err) 1982 return libbpf_err(err); 1983 1984 while ((str_off = btf_field_iter_next(&it))) { 1985 err = btf_rewrite_str(p, str_off); 1986 if (err) 1987 return libbpf_err(err); 1988 } 1989 1990 return btf_commit_type(p->dst, sz); 1991 } 1992 1993 int btf__add_type(struct btf *btf, const struct btf *src_btf, const struct btf_type *src_type) 1994 { 1995 struct btf_pipe p = { .src = src_btf, .dst = btf }; 1996 1997 return btf_add_type(&p, src_type); 1998 } 1999 2000 static size_t btf_dedup_identity_hash_fn(long key, void *ctx); 2001 static bool btf_dedup_equal_fn(long k1, long k2, void *ctx); 2002 2003 int btf__add_btf(struct btf *btf, const struct btf *src_btf) 2004 { 2005 struct btf_pipe p = { .src = src_btf, .dst = btf }; 2006 int data_sz, sz, cnt, i, err, old_strs_len; 2007 __u32 *off; 2008 void *t; 2009 2010 /* appending split BTF isn't supported yet */ 2011 if (src_btf->base_btf) 2012 return libbpf_err(-ENOTSUP); 2013 2014 /* deconstruct BTF, if necessary, and invalidate raw_data */ 2015 if (btf_ensure_modifiable(btf)) 2016 return libbpf_err(-ENOMEM); 2017 2018 /* remember original strings section size if we have to roll back 2019 * partial strings section changes 2020 */ 2021 old_strs_len = btf->hdr->str_len; 2022 2023 data_sz = src_btf->hdr->type_len; 2024 cnt = btf__type_cnt(src_btf) - 1; 2025 2026 /* pre-allocate enough memory for new types */ 2027 t = btf_add_type_mem(btf, data_sz); 2028 if (!t) 2029 return libbpf_err(-ENOMEM); 2030 2031 /* pre-allocate enough memory for type offset index for new types */ 2032 off = btf_add_type_offs_mem(btf, cnt); 2033 if (!off) 2034 return libbpf_err(-ENOMEM); 2035 2036 /* Map the string offsets from src_btf to the offsets from btf to improve performance */ 2037 p.str_off_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL); 2038 if (IS_ERR(p.str_off_map)) 2039 return libbpf_err(-ENOMEM); 2040 2041 /* bulk copy types data for all types from src_btf */ 2042 memcpy(t, src_btf->types_data, data_sz); 2043 2044 for (i = 0; i < cnt; i++) { 2045 struct btf_field_iter it; 2046 __u32 *type_id, *str_off; 2047 2048 sz = btf_type_size(t); 2049 if (sz < 0) { 2050 /* unlikely, has to be corrupted src_btf */ 2051 err = sz; 2052 goto err_out; 2053 } 2054 2055 /* fill out type ID to type offset mapping for lookups by type ID */ 2056 *off = t - btf->types_data; 2057 2058 /* add, dedup, and remap strings referenced by this BTF type */ 2059 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS); 2060 if (err) 2061 goto err_out; 2062 while ((str_off = btf_field_iter_next(&it))) { 2063 err = btf_rewrite_str(&p, str_off); 2064 if (err) 2065 goto err_out; 2066 } 2067 2068 /* remap all type IDs referenced from this BTF type */ 2069 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS); 2070 if (err) 2071 goto err_out; 2072 2073 while ((type_id = btf_field_iter_next(&it))) { 2074 if (!*type_id) /* nothing to do for VOID references */ 2075 continue; 2076 2077 /* we haven't updated btf's type count yet, so 2078 * btf->start_id + btf->nr_types - 1 is the type ID offset we should 2079 * add to all newly added BTF types 2080 */ 2081 *type_id += btf->start_id + btf->nr_types - 1; 2082 } 2083 2084 /* go to next type data and type offset index entry */ 2085 t += sz; 2086 off++; 2087 } 2088 2089 /* Up until now any of the copied type data was effectively invisible, 2090 * so if we exited early before this point due to error, BTF would be 2091 * effectively unmodified. There would be extra internal memory 2092 * pre-allocated, but it would not be available for querying. But now 2093 * that we've copied and rewritten all the data successfully, we can 2094 * update type count and various internal offsets and sizes to 2095 * "commit" the changes and made them visible to the outside world. 2096 */ 2097 btf->hdr->type_len += data_sz; 2098 btf->hdr->str_off += data_sz; 2099 btf->nr_types += cnt; 2100 2101 hashmap__free(p.str_off_map); 2102 2103 /* return type ID of the first added BTF type */ 2104 return btf->start_id + btf->nr_types - cnt; 2105 err_out: 2106 /* zero out preallocated memory as if it was just allocated with 2107 * libbpf_add_mem() 2108 */ 2109 memset(btf->types_data + btf->hdr->type_len, 0, data_sz); 2110 memset(btf->strs_data + old_strs_len, 0, btf->hdr->str_len - old_strs_len); 2111 2112 /* and now restore original strings section size; types data size 2113 * wasn't modified, so doesn't need restoring, see big comment above 2114 */ 2115 btf->hdr->str_len = old_strs_len; 2116 2117 hashmap__free(p.str_off_map); 2118 2119 return libbpf_err(err); 2120 } 2121 2122 /* 2123 * Append new BTF_KIND_INT type with: 2124 * - *name* - non-empty, non-NULL type name; 2125 * - *sz* - power-of-2 (1, 2, 4, ..) size of the type, in bytes; 2126 * - encoding is a combination of BTF_INT_SIGNED, BTF_INT_CHAR, BTF_INT_BOOL. 2127 * Returns: 2128 * - >0, type ID of newly added BTF type; 2129 * - <0, on error. 2130 */ 2131 int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding) 2132 { 2133 struct btf_type *t; 2134 int sz, name_off; 2135 2136 /* non-empty name */ 2137 if (str_is_empty(name)) 2138 return libbpf_err(-EINVAL); 2139 /* byte_sz must be power of 2 */ 2140 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16) 2141 return libbpf_err(-EINVAL); 2142 if (encoding & ~(BTF_INT_SIGNED | BTF_INT_CHAR | BTF_INT_BOOL)) 2143 return libbpf_err(-EINVAL); 2144 2145 /* deconstruct BTF, if necessary, and invalidate raw_data */ 2146 if (btf_ensure_modifiable(btf)) 2147 return libbpf_err(-ENOMEM); 2148 2149 sz = sizeof(struct btf_type) + sizeof(int); 2150 t = btf_add_type_mem(btf, sz); 2151 if (!t) 2152 return libbpf_err(-ENOMEM); 2153 2154 /* if something goes wrong later, we might end up with an extra string, 2155 * but that shouldn't be a problem, because BTF can't be constructed 2156 * completely anyway and will most probably be just discarded 2157 */ 2158 name_off = btf__add_str(btf, name); 2159 if (name_off < 0) 2160 return name_off; 2161 2162 t->name_off = name_off; 2163 t->info = btf_type_info(BTF_KIND_INT, 0, 0); 2164 t->size = byte_sz; 2165 /* set INT info, we don't allow setting legacy bit offset/size */ 2166 *(__u32 *)(t + 1) = (encoding << 24) | (byte_sz * 8); 2167 2168 return btf_commit_type(btf, sz); 2169 } 2170 2171 /* 2172 * Append new BTF_KIND_FLOAT type with: 2173 * - *name* - non-empty, non-NULL type name; 2174 * - *sz* - size of the type, in bytes; 2175 * Returns: 2176 * - >0, type ID of newly added BTF type; 2177 * - <0, on error. 2178 */ 2179 int btf__add_float(struct btf *btf, const char *name, size_t byte_sz) 2180 { 2181 struct btf_type *t; 2182 int sz, name_off; 2183 2184 /* non-empty name */ 2185 if (str_is_empty(name)) 2186 return libbpf_err(-EINVAL); 2187 2188 /* byte_sz must be one of the explicitly allowed values */ 2189 if (byte_sz != 2 && byte_sz != 4 && byte_sz != 8 && byte_sz != 12 && 2190 byte_sz != 16) 2191 return libbpf_err(-EINVAL); 2192 2193 if (btf_ensure_modifiable(btf)) 2194 return libbpf_err(-ENOMEM); 2195 2196 sz = sizeof(struct btf_type); 2197 t = btf_add_type_mem(btf, sz); 2198 if (!t) 2199 return libbpf_err(-ENOMEM); 2200 2201 name_off = btf__add_str(btf, name); 2202 if (name_off < 0) 2203 return name_off; 2204 2205 t->name_off = name_off; 2206 t->info = btf_type_info(BTF_KIND_FLOAT, 0, 0); 2207 t->size = byte_sz; 2208 2209 return btf_commit_type(btf, sz); 2210 } 2211 2212 /* it's completely legal to append BTF types with type IDs pointing forward to 2213 * types that haven't been appended yet, so we only make sure that id looks 2214 * sane, we can't guarantee that ID will always be valid 2215 */ 2216 static int validate_type_id(int id) 2217 { 2218 if (id < 0 || id > BTF_MAX_NR_TYPES) 2219 return -EINVAL; 2220 return 0; 2221 } 2222 2223 /* generic append function for PTR, TYPEDEF, CONST/VOLATILE/RESTRICT */ 2224 static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref_type_id, int kflag) 2225 { 2226 struct btf_type *t; 2227 int sz, name_off = 0; 2228 2229 if (validate_type_id(ref_type_id)) 2230 return libbpf_err(-EINVAL); 2231 2232 if (btf_ensure_modifiable(btf)) 2233 return libbpf_err(-ENOMEM); 2234 2235 sz = sizeof(struct btf_type); 2236 t = btf_add_type_mem(btf, sz); 2237 if (!t) 2238 return libbpf_err(-ENOMEM); 2239 2240 if (!str_is_empty(name)) { 2241 name_off = btf__add_str(btf, name); 2242 if (name_off < 0) 2243 return name_off; 2244 } 2245 2246 t->name_off = name_off; 2247 t->info = btf_type_info(kind, 0, kflag); 2248 t->type = ref_type_id; 2249 2250 return btf_commit_type(btf, sz); 2251 } 2252 2253 /* 2254 * Append new BTF_KIND_PTR type with: 2255 * - *ref_type_id* - referenced type ID, it might not exist yet; 2256 * Returns: 2257 * - >0, type ID of newly added BTF type; 2258 * - <0, on error. 2259 */ 2260 int btf__add_ptr(struct btf *btf, int ref_type_id) 2261 { 2262 return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id, 0); 2263 } 2264 2265 /* 2266 * Append new BTF_KIND_ARRAY type with: 2267 * - *index_type_id* - type ID of the type describing array index; 2268 * - *elem_type_id* - type ID of the type describing array element; 2269 * - *nr_elems* - the size of the array; 2270 * Returns: 2271 * - >0, type ID of newly added BTF type; 2272 * - <0, on error. 2273 */ 2274 int btf__add_array(struct btf *btf, int index_type_id, int elem_type_id, __u32 nr_elems) 2275 { 2276 struct btf_type *t; 2277 struct btf_array *a; 2278 int sz; 2279 2280 if (validate_type_id(index_type_id) || validate_type_id(elem_type_id)) 2281 return libbpf_err(-EINVAL); 2282 2283 if (btf_ensure_modifiable(btf)) 2284 return libbpf_err(-ENOMEM); 2285 2286 sz = sizeof(struct btf_type) + sizeof(struct btf_array); 2287 t = btf_add_type_mem(btf, sz); 2288 if (!t) 2289 return libbpf_err(-ENOMEM); 2290 2291 t->name_off = 0; 2292 t->info = btf_type_info(BTF_KIND_ARRAY, 0, 0); 2293 t->size = 0; 2294 2295 a = btf_array(t); 2296 a->type = elem_type_id; 2297 a->index_type = index_type_id; 2298 a->nelems = nr_elems; 2299 2300 return btf_commit_type(btf, sz); 2301 } 2302 2303 /* generic STRUCT/UNION append function */ 2304 static int btf_add_composite(struct btf *btf, int kind, const char *name, __u32 bytes_sz) 2305 { 2306 struct btf_type *t; 2307 int sz, name_off = 0; 2308 2309 if (btf_ensure_modifiable(btf)) 2310 return libbpf_err(-ENOMEM); 2311 2312 sz = sizeof(struct btf_type); 2313 t = btf_add_type_mem(btf, sz); 2314 if (!t) 2315 return libbpf_err(-ENOMEM); 2316 2317 if (!str_is_empty(name)) { 2318 name_off = btf__add_str(btf, name); 2319 if (name_off < 0) 2320 return name_off; 2321 } 2322 2323 /* start out with vlen=0 and no kflag; this will be adjusted when 2324 * adding each member 2325 */ 2326 t->name_off = name_off; 2327 t->info = btf_type_info(kind, 0, 0); 2328 t->size = bytes_sz; 2329 2330 return btf_commit_type(btf, sz); 2331 } 2332 2333 /* 2334 * Append new BTF_KIND_STRUCT type with: 2335 * - *name* - name of the struct, can be NULL or empty for anonymous structs; 2336 * - *byte_sz* - size of the struct, in bytes; 2337 * 2338 * Struct initially has no fields in it. Fields can be added by 2339 * btf__add_field() right after btf__add_struct() succeeds. 2340 * 2341 * Returns: 2342 * - >0, type ID of newly added BTF type; 2343 * - <0, on error. 2344 */ 2345 int btf__add_struct(struct btf *btf, const char *name, __u32 byte_sz) 2346 { 2347 return btf_add_composite(btf, BTF_KIND_STRUCT, name, byte_sz); 2348 } 2349 2350 /* 2351 * Append new BTF_KIND_UNION type with: 2352 * - *name* - name of the union, can be NULL or empty for anonymous union; 2353 * - *byte_sz* - size of the union, in bytes; 2354 * 2355 * Union initially has no fields in it. Fields can be added by 2356 * btf__add_field() right after btf__add_union() succeeds. All fields 2357 * should have *bit_offset* of 0. 2358 * 2359 * Returns: 2360 * - >0, type ID of newly added BTF type; 2361 * - <0, on error. 2362 */ 2363 int btf__add_union(struct btf *btf, const char *name, __u32 byte_sz) 2364 { 2365 return btf_add_composite(btf, BTF_KIND_UNION, name, byte_sz); 2366 } 2367 2368 static struct btf_type *btf_last_type(struct btf *btf) 2369 { 2370 return btf_type_by_id(btf, btf__type_cnt(btf) - 1); 2371 } 2372 2373 /* 2374 * Append new field for the current STRUCT/UNION type with: 2375 * - *name* - name of the field, can be NULL or empty for anonymous field; 2376 * - *type_id* - type ID for the type describing field type; 2377 * - *bit_offset* - bit offset of the start of the field within struct/union; 2378 * - *bit_size* - bit size of a bitfield, 0 for non-bitfield fields; 2379 * Returns: 2380 * - 0, on success; 2381 * - <0, on error. 2382 */ 2383 int btf__add_field(struct btf *btf, const char *name, int type_id, 2384 __u32 bit_offset, __u32 bit_size) 2385 { 2386 struct btf_type *t; 2387 struct btf_member *m; 2388 bool is_bitfield; 2389 int sz, name_off = 0; 2390 2391 /* last type should be union/struct */ 2392 if (btf->nr_types == 0) 2393 return libbpf_err(-EINVAL); 2394 t = btf_last_type(btf); 2395 if (!btf_is_composite(t)) 2396 return libbpf_err(-EINVAL); 2397 2398 if (validate_type_id(type_id)) 2399 return libbpf_err(-EINVAL); 2400 /* best-effort bit field offset/size enforcement */ 2401 is_bitfield = bit_size || (bit_offset % 8 != 0); 2402 if (is_bitfield && (bit_size == 0 || bit_size > 255 || bit_offset > 0xffffff)) 2403 return libbpf_err(-EINVAL); 2404 2405 /* only offset 0 is allowed for unions */ 2406 if (btf_is_union(t) && bit_offset) 2407 return libbpf_err(-EINVAL); 2408 2409 /* decompose and invalidate raw data */ 2410 if (btf_ensure_modifiable(btf)) 2411 return libbpf_err(-ENOMEM); 2412 2413 sz = sizeof(struct btf_member); 2414 m = btf_add_type_mem(btf, sz); 2415 if (!m) 2416 return libbpf_err(-ENOMEM); 2417 2418 if (!str_is_empty(name)) { 2419 name_off = btf__add_str(btf, name); 2420 if (name_off < 0) 2421 return name_off; 2422 } 2423 2424 m->name_off = name_off; 2425 m->type = type_id; 2426 m->offset = bit_offset | (bit_size << 24); 2427 2428 /* btf_add_type_mem can invalidate t pointer */ 2429 t = btf_last_type(btf); 2430 /* update parent type's vlen and kflag */ 2431 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t)); 2432 2433 btf->hdr->type_len += sz; 2434 btf->hdr->str_off += sz; 2435 return 0; 2436 } 2437 2438 static int btf_add_enum_common(struct btf *btf, const char *name, __u32 byte_sz, 2439 bool is_signed, __u8 kind) 2440 { 2441 struct btf_type *t; 2442 int sz, name_off = 0; 2443 2444 /* byte_sz must be power of 2 */ 2445 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8) 2446 return libbpf_err(-EINVAL); 2447 2448 if (btf_ensure_modifiable(btf)) 2449 return libbpf_err(-ENOMEM); 2450 2451 sz = sizeof(struct btf_type); 2452 t = btf_add_type_mem(btf, sz); 2453 if (!t) 2454 return libbpf_err(-ENOMEM); 2455 2456 if (!str_is_empty(name)) { 2457 name_off = btf__add_str(btf, name); 2458 if (name_off < 0) 2459 return name_off; 2460 } 2461 2462 /* start out with vlen=0; it will be adjusted when adding enum values */ 2463 t->name_off = name_off; 2464 t->info = btf_type_info(kind, 0, is_signed); 2465 t->size = byte_sz; 2466 2467 return btf_commit_type(btf, sz); 2468 } 2469 2470 /* 2471 * Append new BTF_KIND_ENUM type with: 2472 * - *name* - name of the enum, can be NULL or empty for anonymous enums; 2473 * - *byte_sz* - size of the enum, in bytes. 2474 * 2475 * Enum initially has no enum values in it (and corresponds to enum forward 2476 * declaration). Enumerator values can be added by btf__add_enum_value() 2477 * immediately after btf__add_enum() succeeds. 2478 * 2479 * Returns: 2480 * - >0, type ID of newly added BTF type; 2481 * - <0, on error. 2482 */ 2483 int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz) 2484 { 2485 /* 2486 * set the signedness to be unsigned, it will change to signed 2487 * if any later enumerator is negative. 2488 */ 2489 return btf_add_enum_common(btf, name, byte_sz, false, BTF_KIND_ENUM); 2490 } 2491 2492 /* 2493 * Append new enum value for the current ENUM type with: 2494 * - *name* - name of the enumerator value, can't be NULL or empty; 2495 * - *value* - integer value corresponding to enum value *name*; 2496 * Returns: 2497 * - 0, on success; 2498 * - <0, on error. 2499 */ 2500 int btf__add_enum_value(struct btf *btf, const char *name, __s64 value) 2501 { 2502 struct btf_type *t; 2503 struct btf_enum *v; 2504 int sz, name_off; 2505 2506 /* last type should be BTF_KIND_ENUM */ 2507 if (btf->nr_types == 0) 2508 return libbpf_err(-EINVAL); 2509 t = btf_last_type(btf); 2510 if (!btf_is_enum(t)) 2511 return libbpf_err(-EINVAL); 2512 2513 /* non-empty name */ 2514 if (str_is_empty(name)) 2515 return libbpf_err(-EINVAL); 2516 if (value < INT_MIN || value > UINT_MAX) 2517 return libbpf_err(-E2BIG); 2518 2519 /* decompose and invalidate raw data */ 2520 if (btf_ensure_modifiable(btf)) 2521 return libbpf_err(-ENOMEM); 2522 2523 sz = sizeof(struct btf_enum); 2524 v = btf_add_type_mem(btf, sz); 2525 if (!v) 2526 return libbpf_err(-ENOMEM); 2527 2528 name_off = btf__add_str(btf, name); 2529 if (name_off < 0) 2530 return name_off; 2531 2532 v->name_off = name_off; 2533 v->val = value; 2534 2535 /* update parent type's vlen */ 2536 t = btf_last_type(btf); 2537 btf_type_inc_vlen(t); 2538 2539 /* if negative value, set signedness to signed */ 2540 if (value < 0) 2541 t->info = btf_type_info(btf_kind(t), btf_vlen(t), true); 2542 2543 btf->hdr->type_len += sz; 2544 btf->hdr->str_off += sz; 2545 return 0; 2546 } 2547 2548 /* 2549 * Append new BTF_KIND_ENUM64 type with: 2550 * - *name* - name of the enum, can be NULL or empty for anonymous enums; 2551 * - *byte_sz* - size of the enum, in bytes. 2552 * - *is_signed* - whether the enum values are signed or not; 2553 * 2554 * Enum initially has no enum values in it (and corresponds to enum forward 2555 * declaration). Enumerator values can be added by btf__add_enum64_value() 2556 * immediately after btf__add_enum64() succeeds. 2557 * 2558 * Returns: 2559 * - >0, type ID of newly added BTF type; 2560 * - <0, on error. 2561 */ 2562 int btf__add_enum64(struct btf *btf, const char *name, __u32 byte_sz, 2563 bool is_signed) 2564 { 2565 return btf_add_enum_common(btf, name, byte_sz, is_signed, 2566 BTF_KIND_ENUM64); 2567 } 2568 2569 /* 2570 * Append new enum value for the current ENUM64 type with: 2571 * - *name* - name of the enumerator value, can't be NULL or empty; 2572 * - *value* - integer value corresponding to enum value *name*; 2573 * Returns: 2574 * - 0, on success; 2575 * - <0, on error. 2576 */ 2577 int btf__add_enum64_value(struct btf *btf, const char *name, __u64 value) 2578 { 2579 struct btf_enum64 *v; 2580 struct btf_type *t; 2581 int sz, name_off; 2582 2583 /* last type should be BTF_KIND_ENUM64 */ 2584 if (btf->nr_types == 0) 2585 return libbpf_err(-EINVAL); 2586 t = btf_last_type(btf); 2587 if (!btf_is_enum64(t)) 2588 return libbpf_err(-EINVAL); 2589 2590 /* non-empty name */ 2591 if (str_is_empty(name)) 2592 return libbpf_err(-EINVAL); 2593 2594 /* decompose and invalidate raw data */ 2595 if (btf_ensure_modifiable(btf)) 2596 return libbpf_err(-ENOMEM); 2597 2598 sz = sizeof(struct btf_enum64); 2599 v = btf_add_type_mem(btf, sz); 2600 if (!v) 2601 return libbpf_err(-ENOMEM); 2602 2603 name_off = btf__add_str(btf, name); 2604 if (name_off < 0) 2605 return name_off; 2606 2607 v->name_off = name_off; 2608 v->val_lo32 = (__u32)value; 2609 v->val_hi32 = value >> 32; 2610 2611 /* update parent type's vlen */ 2612 t = btf_last_type(btf); 2613 btf_type_inc_vlen(t); 2614 2615 btf->hdr->type_len += sz; 2616 btf->hdr->str_off += sz; 2617 return 0; 2618 } 2619 2620 /* 2621 * Append new BTF_KIND_FWD type with: 2622 * - *name*, non-empty/non-NULL name; 2623 * - *fwd_kind*, kind of forward declaration, one of BTF_FWD_STRUCT, 2624 * BTF_FWD_UNION, or BTF_FWD_ENUM; 2625 * Returns: 2626 * - >0, type ID of newly added BTF type; 2627 * - <0, on error. 2628 */ 2629 int btf__add_fwd(struct btf *btf, const char *name, enum btf_fwd_kind fwd_kind) 2630 { 2631 if (str_is_empty(name)) 2632 return libbpf_err(-EINVAL); 2633 2634 switch (fwd_kind) { 2635 case BTF_FWD_STRUCT: 2636 case BTF_FWD_UNION: { 2637 struct btf_type *t; 2638 int id; 2639 2640 id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0, 0); 2641 if (id <= 0) 2642 return id; 2643 t = btf_type_by_id(btf, id); 2644 t->info = btf_type_info(BTF_KIND_FWD, 0, fwd_kind == BTF_FWD_UNION); 2645 return id; 2646 } 2647 case BTF_FWD_ENUM: 2648 /* enum forward in BTF currently is just an enum with no enum 2649 * values; we also assume a standard 4-byte size for it 2650 */ 2651 return btf__add_enum(btf, name, sizeof(int)); 2652 default: 2653 return libbpf_err(-EINVAL); 2654 } 2655 } 2656 2657 /* 2658 * Append new BTF_KING_TYPEDEF type with: 2659 * - *name*, non-empty/non-NULL name; 2660 * - *ref_type_id* - referenced type ID, it might not exist yet; 2661 * Returns: 2662 * - >0, type ID of newly added BTF type; 2663 * - <0, on error. 2664 */ 2665 int btf__add_typedef(struct btf *btf, const char *name, int ref_type_id) 2666 { 2667 if (str_is_empty(name)) 2668 return libbpf_err(-EINVAL); 2669 2670 return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id, 0); 2671 } 2672 2673 /* 2674 * Append new BTF_KIND_VOLATILE type with: 2675 * - *ref_type_id* - referenced type ID, it might not exist yet; 2676 * Returns: 2677 * - >0, type ID of newly added BTF type; 2678 * - <0, on error. 2679 */ 2680 int btf__add_volatile(struct btf *btf, int ref_type_id) 2681 { 2682 return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id, 0); 2683 } 2684 2685 /* 2686 * Append new BTF_KIND_CONST type with: 2687 * - *ref_type_id* - referenced type ID, it might not exist yet; 2688 * Returns: 2689 * - >0, type ID of newly added BTF type; 2690 * - <0, on error. 2691 */ 2692 int btf__add_const(struct btf *btf, int ref_type_id) 2693 { 2694 return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id, 0); 2695 } 2696 2697 /* 2698 * Append new BTF_KIND_RESTRICT type with: 2699 * - *ref_type_id* - referenced type ID, it might not exist yet; 2700 * Returns: 2701 * - >0, type ID of newly added BTF type; 2702 * - <0, on error. 2703 */ 2704 int btf__add_restrict(struct btf *btf, int ref_type_id) 2705 { 2706 return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id, 0); 2707 } 2708 2709 /* 2710 * Append new BTF_KIND_TYPE_TAG type with: 2711 * - *value*, non-empty/non-NULL tag value; 2712 * - *ref_type_id* - referenced type ID, it might not exist yet; 2713 * Returns: 2714 * - >0, type ID of newly added BTF type; 2715 * - <0, on error. 2716 */ 2717 int btf__add_type_tag(struct btf *btf, const char *value, int ref_type_id) 2718 { 2719 if (str_is_empty(value)) 2720 return libbpf_err(-EINVAL); 2721 2722 return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id, 0); 2723 } 2724 2725 /* 2726 * Append new BTF_KIND_TYPE_TAG type with: 2727 * - *value*, non-empty/non-NULL tag value; 2728 * - *ref_type_id* - referenced type ID, it might not exist yet; 2729 * Set info->kflag to 1, indicating this tag is an __attribute__ 2730 * Returns: 2731 * - >0, type ID of newly added BTF type; 2732 * - <0, on error. 2733 */ 2734 int btf__add_type_attr(struct btf *btf, const char *value, int ref_type_id) 2735 { 2736 if (str_is_empty(value)) 2737 return libbpf_err(-EINVAL); 2738 2739 return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id, 1); 2740 } 2741 2742 /* 2743 * Append new BTF_KIND_FUNC type with: 2744 * - *name*, non-empty/non-NULL name; 2745 * - *proto_type_id* - FUNC_PROTO's type ID, it might not exist yet; 2746 * Returns: 2747 * - >0, type ID of newly added BTF type; 2748 * - <0, on error. 2749 */ 2750 int btf__add_func(struct btf *btf, const char *name, 2751 enum btf_func_linkage linkage, int proto_type_id) 2752 { 2753 int id; 2754 2755 if (str_is_empty(name)) 2756 return libbpf_err(-EINVAL); 2757 if (linkage != BTF_FUNC_STATIC && linkage != BTF_FUNC_GLOBAL && 2758 linkage != BTF_FUNC_EXTERN) 2759 return libbpf_err(-EINVAL); 2760 2761 id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id, 0); 2762 if (id > 0) { 2763 struct btf_type *t = btf_type_by_id(btf, id); 2764 2765 t->info = btf_type_info(BTF_KIND_FUNC, linkage, 0); 2766 } 2767 return libbpf_err(id); 2768 } 2769 2770 /* 2771 * Append new BTF_KIND_FUNC_PROTO with: 2772 * - *ret_type_id* - type ID for return result of a function. 2773 * 2774 * Function prototype initially has no arguments, but they can be added by 2775 * btf__add_func_param() one by one, immediately after 2776 * btf__add_func_proto() succeeded. 2777 * 2778 * Returns: 2779 * - >0, type ID of newly added BTF type; 2780 * - <0, on error. 2781 */ 2782 int btf__add_func_proto(struct btf *btf, int ret_type_id) 2783 { 2784 struct btf_type *t; 2785 int sz; 2786 2787 if (validate_type_id(ret_type_id)) 2788 return libbpf_err(-EINVAL); 2789 2790 if (btf_ensure_modifiable(btf)) 2791 return libbpf_err(-ENOMEM); 2792 2793 sz = sizeof(struct btf_type); 2794 t = btf_add_type_mem(btf, sz); 2795 if (!t) 2796 return libbpf_err(-ENOMEM); 2797 2798 /* start out with vlen=0; this will be adjusted when adding enum 2799 * values, if necessary 2800 */ 2801 t->name_off = 0; 2802 t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0); 2803 t->type = ret_type_id; 2804 2805 return btf_commit_type(btf, sz); 2806 } 2807 2808 /* 2809 * Append new function parameter for current FUNC_PROTO type with: 2810 * - *name* - parameter name, can be NULL or empty; 2811 * - *type_id* - type ID describing the type of the parameter. 2812 * Returns: 2813 * - 0, on success; 2814 * - <0, on error. 2815 */ 2816 int btf__add_func_param(struct btf *btf, const char *name, int type_id) 2817 { 2818 struct btf_type *t; 2819 struct btf_param *p; 2820 int sz, name_off = 0; 2821 2822 if (validate_type_id(type_id)) 2823 return libbpf_err(-EINVAL); 2824 2825 /* last type should be BTF_KIND_FUNC_PROTO */ 2826 if (btf->nr_types == 0) 2827 return libbpf_err(-EINVAL); 2828 t = btf_last_type(btf); 2829 if (!btf_is_func_proto(t)) 2830 return libbpf_err(-EINVAL); 2831 2832 /* decompose and invalidate raw data */ 2833 if (btf_ensure_modifiable(btf)) 2834 return libbpf_err(-ENOMEM); 2835 2836 sz = sizeof(struct btf_param); 2837 p = btf_add_type_mem(btf, sz); 2838 if (!p) 2839 return libbpf_err(-ENOMEM); 2840 2841 if (!str_is_empty(name)) { 2842 name_off = btf__add_str(btf, name); 2843 if (name_off < 0) 2844 return name_off; 2845 } 2846 2847 p->name_off = name_off; 2848 p->type = type_id; 2849 2850 /* update parent type's vlen */ 2851 t = btf_last_type(btf); 2852 btf_type_inc_vlen(t); 2853 2854 btf->hdr->type_len += sz; 2855 btf->hdr->str_off += sz; 2856 return 0; 2857 } 2858 2859 /* 2860 * Append new BTF_KIND_VAR type with: 2861 * - *name* - non-empty/non-NULL name; 2862 * - *linkage* - variable linkage, one of BTF_VAR_STATIC, 2863 * BTF_VAR_GLOBAL_ALLOCATED, or BTF_VAR_GLOBAL_EXTERN; 2864 * - *type_id* - type ID of the type describing the type of the variable. 2865 * Returns: 2866 * - >0, type ID of newly added BTF type; 2867 * - <0, on error. 2868 */ 2869 int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id) 2870 { 2871 struct btf_type *t; 2872 struct btf_var *v; 2873 int sz, name_off; 2874 2875 /* non-empty name */ 2876 if (str_is_empty(name)) 2877 return libbpf_err(-EINVAL); 2878 if (linkage != BTF_VAR_STATIC && linkage != BTF_VAR_GLOBAL_ALLOCATED && 2879 linkage != BTF_VAR_GLOBAL_EXTERN) 2880 return libbpf_err(-EINVAL); 2881 if (validate_type_id(type_id)) 2882 return libbpf_err(-EINVAL); 2883 2884 /* deconstruct BTF, if necessary, and invalidate raw_data */ 2885 if (btf_ensure_modifiable(btf)) 2886 return libbpf_err(-ENOMEM); 2887 2888 sz = sizeof(struct btf_type) + sizeof(struct btf_var); 2889 t = btf_add_type_mem(btf, sz); 2890 if (!t) 2891 return libbpf_err(-ENOMEM); 2892 2893 name_off = btf__add_str(btf, name); 2894 if (name_off < 0) 2895 return name_off; 2896 2897 t->name_off = name_off; 2898 t->info = btf_type_info(BTF_KIND_VAR, 0, 0); 2899 t->type = type_id; 2900 2901 v = btf_var(t); 2902 v->linkage = linkage; 2903 2904 return btf_commit_type(btf, sz); 2905 } 2906 2907 /* 2908 * Append new BTF_KIND_DATASEC type with: 2909 * - *name* - non-empty/non-NULL name; 2910 * - *byte_sz* - data section size, in bytes. 2911 * 2912 * Data section is initially empty. Variables info can be added with 2913 * btf__add_datasec_var_info() calls, after btf__add_datasec() succeeds. 2914 * 2915 * Returns: 2916 * - >0, type ID of newly added BTF type; 2917 * - <0, on error. 2918 */ 2919 int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz) 2920 { 2921 struct btf_type *t; 2922 int sz, name_off; 2923 2924 /* non-empty name */ 2925 if (str_is_empty(name)) 2926 return libbpf_err(-EINVAL); 2927 2928 if (btf_ensure_modifiable(btf)) 2929 return libbpf_err(-ENOMEM); 2930 2931 sz = sizeof(struct btf_type); 2932 t = btf_add_type_mem(btf, sz); 2933 if (!t) 2934 return libbpf_err(-ENOMEM); 2935 2936 name_off = btf__add_str(btf, name); 2937 if (name_off < 0) 2938 return name_off; 2939 2940 /* start with vlen=0, which will be update as var_secinfos are added */ 2941 t->name_off = name_off; 2942 t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0); 2943 t->size = byte_sz; 2944 2945 return btf_commit_type(btf, sz); 2946 } 2947 2948 /* 2949 * Append new data section variable information entry for current DATASEC type: 2950 * - *var_type_id* - type ID, describing type of the variable; 2951 * - *offset* - variable offset within data section, in bytes; 2952 * - *byte_sz* - variable size, in bytes. 2953 * 2954 * Returns: 2955 * - 0, on success; 2956 * - <0, on error. 2957 */ 2958 int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __u32 byte_sz) 2959 { 2960 struct btf_type *t; 2961 struct btf_var_secinfo *v; 2962 int sz; 2963 2964 /* last type should be BTF_KIND_DATASEC */ 2965 if (btf->nr_types == 0) 2966 return libbpf_err(-EINVAL); 2967 t = btf_last_type(btf); 2968 if (!btf_is_datasec(t)) 2969 return libbpf_err(-EINVAL); 2970 2971 if (validate_type_id(var_type_id)) 2972 return libbpf_err(-EINVAL); 2973 2974 /* decompose and invalidate raw data */ 2975 if (btf_ensure_modifiable(btf)) 2976 return libbpf_err(-ENOMEM); 2977 2978 sz = sizeof(struct btf_var_secinfo); 2979 v = btf_add_type_mem(btf, sz); 2980 if (!v) 2981 return libbpf_err(-ENOMEM); 2982 2983 v->type = var_type_id; 2984 v->offset = offset; 2985 v->size = byte_sz; 2986 2987 /* update parent type's vlen */ 2988 t = btf_last_type(btf); 2989 btf_type_inc_vlen(t); 2990 2991 btf->hdr->type_len += sz; 2992 btf->hdr->str_off += sz; 2993 return 0; 2994 } 2995 2996 static int btf_add_decl_tag(struct btf *btf, const char *value, int ref_type_id, 2997 int component_idx, int kflag) 2998 { 2999 struct btf_type *t; 3000 int sz, value_off; 3001 3002 if (str_is_empty(value) || component_idx < -1) 3003 return libbpf_err(-EINVAL); 3004 3005 if (validate_type_id(ref_type_id)) 3006 return libbpf_err(-EINVAL); 3007 3008 if (btf_ensure_modifiable(btf)) 3009 return libbpf_err(-ENOMEM); 3010 3011 sz = sizeof(struct btf_type) + sizeof(struct btf_decl_tag); 3012 t = btf_add_type_mem(btf, sz); 3013 if (!t) 3014 return libbpf_err(-ENOMEM); 3015 3016 value_off = btf__add_str(btf, value); 3017 if (value_off < 0) 3018 return value_off; 3019 3020 t->name_off = value_off; 3021 t->info = btf_type_info(BTF_KIND_DECL_TAG, 0, kflag); 3022 t->type = ref_type_id; 3023 btf_decl_tag(t)->component_idx = component_idx; 3024 3025 return btf_commit_type(btf, sz); 3026 } 3027 3028 /* 3029 * Append new BTF_KIND_DECL_TAG type with: 3030 * - *value* - non-empty/non-NULL string; 3031 * - *ref_type_id* - referenced type ID, it might not exist yet; 3032 * - *component_idx* - -1 for tagging reference type, otherwise struct/union 3033 * member or function argument index; 3034 * Returns: 3035 * - >0, type ID of newly added BTF type; 3036 * - <0, on error. 3037 */ 3038 int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id, 3039 int component_idx) 3040 { 3041 return btf_add_decl_tag(btf, value, ref_type_id, component_idx, 0); 3042 } 3043 3044 /* 3045 * Append new BTF_KIND_DECL_TAG type with: 3046 * - *value* - non-empty/non-NULL string; 3047 * - *ref_type_id* - referenced type ID, it might not exist yet; 3048 * - *component_idx* - -1 for tagging reference type, otherwise struct/union 3049 * member or function argument index; 3050 * Set info->kflag to 1, indicating this tag is an __attribute__ 3051 * Returns: 3052 * - >0, type ID of newly added BTF type; 3053 * - <0, on error. 3054 */ 3055 int btf__add_decl_attr(struct btf *btf, const char *value, int ref_type_id, 3056 int component_idx) 3057 { 3058 return btf_add_decl_tag(btf, value, ref_type_id, component_idx, 1); 3059 } 3060 3061 struct btf_ext_sec_info_param { 3062 __u32 off; 3063 __u32 len; 3064 __u32 min_rec_size; 3065 struct btf_ext_info *ext_info; 3066 const char *desc; 3067 }; 3068 3069 /* 3070 * Parse a single info subsection of the BTF.ext info data: 3071 * - validate subsection structure and elements 3072 * - save info subsection start and sizing details in struct btf_ext 3073 * - endian-independent operation, for calling before byte-swapping 3074 */ 3075 static int btf_ext_parse_sec_info(struct btf_ext *btf_ext, 3076 struct btf_ext_sec_info_param *ext_sec, 3077 bool is_native) 3078 { 3079 const struct btf_ext_info_sec *sinfo; 3080 struct btf_ext_info *ext_info; 3081 __u32 info_left, record_size; 3082 size_t sec_cnt = 0; 3083 void *info; 3084 3085 if (ext_sec->len == 0) 3086 return 0; 3087 3088 if (ext_sec->off & 0x03) { 3089 pr_debug(".BTF.ext %s section is not aligned to 4 bytes\n", 3090 ext_sec->desc); 3091 return -EINVAL; 3092 } 3093 3094 /* The start of the info sec (including the __u32 record_size). */ 3095 info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off; 3096 info_left = ext_sec->len; 3097 3098 if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) { 3099 pr_debug("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n", 3100 ext_sec->desc, ext_sec->off, ext_sec->len); 3101 return -EINVAL; 3102 } 3103 3104 /* At least a record size */ 3105 if (info_left < sizeof(__u32)) { 3106 pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc); 3107 return -EINVAL; 3108 } 3109 3110 /* The record size needs to meet either the minimum standard or, when 3111 * handling non-native endianness data, the exact standard so as 3112 * to allow safe byte-swapping. 3113 */ 3114 record_size = is_native ? *(__u32 *)info : bswap_32(*(__u32 *)info); 3115 if (record_size < ext_sec->min_rec_size || 3116 (!is_native && record_size != ext_sec->min_rec_size) || 3117 record_size & 0x03) { 3118 pr_debug("%s section in .BTF.ext has invalid record size %u\n", 3119 ext_sec->desc, record_size); 3120 return -EINVAL; 3121 } 3122 3123 sinfo = info + sizeof(__u32); 3124 info_left -= sizeof(__u32); 3125 3126 /* If no records, return failure now so .BTF.ext won't be used. */ 3127 if (!info_left) { 3128 pr_debug("%s section in .BTF.ext has no records\n", ext_sec->desc); 3129 return -EINVAL; 3130 } 3131 3132 while (info_left) { 3133 unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec); 3134 __u64 total_record_size; 3135 __u32 num_records; 3136 3137 if (info_left < sec_hdrlen) { 3138 pr_debug("%s section header is not found in .BTF.ext\n", 3139 ext_sec->desc); 3140 return -EINVAL; 3141 } 3142 3143 num_records = is_native ? sinfo->num_info : bswap_32(sinfo->num_info); 3144 if (num_records == 0) { 3145 pr_debug("%s section has incorrect num_records in .BTF.ext\n", 3146 ext_sec->desc); 3147 return -EINVAL; 3148 } 3149 3150 total_record_size = sec_hdrlen + (__u64)num_records * record_size; 3151 if (info_left < total_record_size) { 3152 pr_debug("%s section has incorrect num_records in .BTF.ext\n", 3153 ext_sec->desc); 3154 return -EINVAL; 3155 } 3156 3157 info_left -= total_record_size; 3158 sinfo = (void *)sinfo + total_record_size; 3159 sec_cnt++; 3160 } 3161 3162 ext_info = ext_sec->ext_info; 3163 ext_info->len = ext_sec->len - sizeof(__u32); 3164 ext_info->rec_size = record_size; 3165 ext_info->info = info + sizeof(__u32); 3166 ext_info->sec_cnt = sec_cnt; 3167 3168 return 0; 3169 } 3170 3171 /* Parse all info secs in the BTF.ext info data */ 3172 static int btf_ext_parse_info(struct btf_ext *btf_ext, bool is_native) 3173 { 3174 struct btf_ext_sec_info_param func_info = { 3175 .off = btf_ext->hdr->func_info_off, 3176 .len = btf_ext->hdr->func_info_len, 3177 .min_rec_size = sizeof(struct bpf_func_info_min), 3178 .ext_info = &btf_ext->func_info, 3179 .desc = "func_info" 3180 }; 3181 struct btf_ext_sec_info_param line_info = { 3182 .off = btf_ext->hdr->line_info_off, 3183 .len = btf_ext->hdr->line_info_len, 3184 .min_rec_size = sizeof(struct bpf_line_info_min), 3185 .ext_info = &btf_ext->line_info, 3186 .desc = "line_info", 3187 }; 3188 struct btf_ext_sec_info_param core_relo = { 3189 .min_rec_size = sizeof(struct bpf_core_relo), 3190 .ext_info = &btf_ext->core_relo_info, 3191 .desc = "core_relo", 3192 }; 3193 int err; 3194 3195 err = btf_ext_parse_sec_info(btf_ext, &func_info, is_native); 3196 if (err) 3197 return err; 3198 3199 err = btf_ext_parse_sec_info(btf_ext, &line_info, is_native); 3200 if (err) 3201 return err; 3202 3203 if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) 3204 return 0; /* skip core relos parsing */ 3205 3206 core_relo.off = btf_ext->hdr->core_relo_off; 3207 core_relo.len = btf_ext->hdr->core_relo_len; 3208 err = btf_ext_parse_sec_info(btf_ext, &core_relo, is_native); 3209 if (err) 3210 return err; 3211 3212 return 0; 3213 } 3214 3215 /* Swap byte-order of BTF.ext header with any endianness */ 3216 static void btf_ext_bswap_hdr(struct btf_ext_header *h) 3217 { 3218 bool is_native = h->magic == BTF_MAGIC; 3219 __u32 hdr_len; 3220 3221 hdr_len = is_native ? h->hdr_len : bswap_32(h->hdr_len); 3222 3223 h->magic = bswap_16(h->magic); 3224 h->hdr_len = bswap_32(h->hdr_len); 3225 h->func_info_off = bswap_32(h->func_info_off); 3226 h->func_info_len = bswap_32(h->func_info_len); 3227 h->line_info_off = bswap_32(h->line_info_off); 3228 h->line_info_len = bswap_32(h->line_info_len); 3229 3230 if (hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) 3231 return; 3232 3233 h->core_relo_off = bswap_32(h->core_relo_off); 3234 h->core_relo_len = bswap_32(h->core_relo_len); 3235 } 3236 3237 /* Swap byte-order of generic info subsection */ 3238 static void btf_ext_bswap_info_sec(void *info, __u32 len, bool is_native, 3239 info_rec_bswap_fn bswap_fn) 3240 { 3241 struct btf_ext_info_sec *sec; 3242 __u32 info_left, rec_size, *rs; 3243 3244 if (len == 0) 3245 return; 3246 3247 rs = info; /* info record size */ 3248 rec_size = is_native ? *rs : bswap_32(*rs); 3249 *rs = bswap_32(*rs); 3250 3251 sec = info + sizeof(__u32); /* info sec #1 */ 3252 info_left = len - sizeof(__u32); 3253 while (info_left) { 3254 unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec); 3255 __u32 i, num_recs; 3256 void *p; 3257 3258 num_recs = is_native ? sec->num_info : bswap_32(sec->num_info); 3259 sec->sec_name_off = bswap_32(sec->sec_name_off); 3260 sec->num_info = bswap_32(sec->num_info); 3261 p = sec->data; /* info rec #1 */ 3262 for (i = 0; i < num_recs; i++, p += rec_size) 3263 bswap_fn(p); 3264 sec = p; 3265 info_left -= sec_hdrlen + (__u64)rec_size * num_recs; 3266 } 3267 } 3268 3269 /* 3270 * Swap byte-order of all info data in a BTF.ext section 3271 * - requires BTF.ext hdr in native endianness 3272 */ 3273 static void btf_ext_bswap_info(struct btf_ext *btf_ext, void *data) 3274 { 3275 const bool is_native = btf_ext->swapped_endian; 3276 const struct btf_ext_header *h = data; 3277 void *info; 3278 3279 /* Swap func_info subsection byte-order */ 3280 info = data + h->hdr_len + h->func_info_off; 3281 btf_ext_bswap_info_sec(info, h->func_info_len, is_native, 3282 (info_rec_bswap_fn)bpf_func_info_bswap); 3283 3284 /* Swap line_info subsection byte-order */ 3285 info = data + h->hdr_len + h->line_info_off; 3286 btf_ext_bswap_info_sec(info, h->line_info_len, is_native, 3287 (info_rec_bswap_fn)bpf_line_info_bswap); 3288 3289 /* Swap core_relo subsection byte-order (if present) */ 3290 if (h->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) 3291 return; 3292 3293 info = data + h->hdr_len + h->core_relo_off; 3294 btf_ext_bswap_info_sec(info, h->core_relo_len, is_native, 3295 (info_rec_bswap_fn)bpf_core_relo_bswap); 3296 } 3297 3298 /* Parse hdr data and info sections: check and convert to native endianness */ 3299 static int btf_ext_parse(struct btf_ext *btf_ext) 3300 { 3301 __u32 hdr_len, data_size = btf_ext->data_size; 3302 struct btf_ext_header *hdr = btf_ext->hdr; 3303 bool swapped_endian = false; 3304 int err; 3305 3306 if (data_size < offsetofend(struct btf_ext_header, hdr_len)) { 3307 pr_debug("BTF.ext header too short\n"); 3308 return -EINVAL; 3309 } 3310 3311 hdr_len = hdr->hdr_len; 3312 if (hdr->magic == bswap_16(BTF_MAGIC)) { 3313 swapped_endian = true; 3314 hdr_len = bswap_32(hdr_len); 3315 } else if (hdr->magic != BTF_MAGIC) { 3316 pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic); 3317 return -EINVAL; 3318 } 3319 3320 /* Ensure known version of structs, current BTF_VERSION == 1 */ 3321 if (hdr->version != 1) { 3322 pr_debug("Unsupported BTF.ext version:%u\n", hdr->version); 3323 return -ENOTSUP; 3324 } 3325 3326 if (hdr->flags) { 3327 pr_debug("Unsupported BTF.ext flags:%x\n", hdr->flags); 3328 return -ENOTSUP; 3329 } 3330 3331 if (data_size < hdr_len) { 3332 pr_debug("BTF.ext header not found\n"); 3333 return -EINVAL; 3334 } else if (data_size == hdr_len) { 3335 pr_debug("BTF.ext has no data\n"); 3336 return -EINVAL; 3337 } 3338 3339 /* Verify mandatory hdr info details present */ 3340 if (hdr_len < offsetofend(struct btf_ext_header, line_info_len)) { 3341 pr_warn("BTF.ext header missing func_info, line_info\n"); 3342 return -EINVAL; 3343 } 3344 3345 /* Keep hdr native byte-order in memory for introspection */ 3346 if (swapped_endian) 3347 btf_ext_bswap_hdr(btf_ext->hdr); 3348 3349 /* Validate info subsections and cache key metadata */ 3350 err = btf_ext_parse_info(btf_ext, !swapped_endian); 3351 if (err) 3352 return err; 3353 3354 /* Keep infos native byte-order in memory for introspection */ 3355 if (swapped_endian) 3356 btf_ext_bswap_info(btf_ext, btf_ext->data); 3357 3358 /* 3359 * Set btf_ext->swapped_endian only after all header and info data has 3360 * been swapped, helping bswap functions determine if their data are 3361 * in native byte-order when called. 3362 */ 3363 btf_ext->swapped_endian = swapped_endian; 3364 return 0; 3365 } 3366 3367 void btf_ext__free(struct btf_ext *btf_ext) 3368 { 3369 if (IS_ERR_OR_NULL(btf_ext)) 3370 return; 3371 free(btf_ext->func_info.sec_idxs); 3372 free(btf_ext->line_info.sec_idxs); 3373 free(btf_ext->core_relo_info.sec_idxs); 3374 free(btf_ext->data); 3375 free(btf_ext->data_swapped); 3376 free(btf_ext); 3377 } 3378 3379 struct btf_ext *btf_ext__new(const __u8 *data, __u32 size) 3380 { 3381 struct btf_ext *btf_ext; 3382 int err; 3383 3384 btf_ext = calloc(1, sizeof(struct btf_ext)); 3385 if (!btf_ext) 3386 return libbpf_err_ptr(-ENOMEM); 3387 3388 btf_ext->data_size = size; 3389 btf_ext->data = malloc(size); 3390 if (!btf_ext->data) { 3391 err = -ENOMEM; 3392 goto done; 3393 } 3394 memcpy(btf_ext->data, data, size); 3395 3396 err = btf_ext_parse(btf_ext); 3397 3398 done: 3399 if (err) { 3400 btf_ext__free(btf_ext); 3401 return libbpf_err_ptr(err); 3402 } 3403 3404 return btf_ext; 3405 } 3406 3407 static void *btf_ext_raw_data(const struct btf_ext *btf_ext_ro, bool swap_endian) 3408 { 3409 struct btf_ext *btf_ext = (struct btf_ext *)btf_ext_ro; 3410 const __u32 data_sz = btf_ext->data_size; 3411 void *data; 3412 3413 /* Return native data (always present) or swapped data if present */ 3414 if (!swap_endian) 3415 return btf_ext->data; 3416 else if (btf_ext->data_swapped) 3417 return btf_ext->data_swapped; 3418 3419 /* Recreate missing swapped data, then cache and return */ 3420 data = calloc(1, data_sz); 3421 if (!data) 3422 return NULL; 3423 memcpy(data, btf_ext->data, data_sz); 3424 3425 btf_ext_bswap_info(btf_ext, data); 3426 btf_ext_bswap_hdr(data); 3427 btf_ext->data_swapped = data; 3428 return data; 3429 } 3430 3431 const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size) 3432 { 3433 void *data; 3434 3435 data = btf_ext_raw_data(btf_ext, btf_ext->swapped_endian); 3436 if (!data) 3437 return errno = ENOMEM, NULL; 3438 3439 *size = btf_ext->data_size; 3440 return data; 3441 } 3442 3443 __attribute__((alias("btf_ext__raw_data"))) 3444 const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size); 3445 3446 enum btf_endianness btf_ext__endianness(const struct btf_ext *btf_ext) 3447 { 3448 if (is_host_big_endian()) 3449 return btf_ext->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN; 3450 else 3451 return btf_ext->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN; 3452 } 3453 3454 int btf_ext__set_endianness(struct btf_ext *btf_ext, enum btf_endianness endian) 3455 { 3456 if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN) 3457 return libbpf_err(-EINVAL); 3458 3459 btf_ext->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN); 3460 3461 if (!btf_ext->swapped_endian) { 3462 free(btf_ext->data_swapped); 3463 btf_ext->data_swapped = NULL; 3464 } 3465 return 0; 3466 } 3467 3468 struct btf_dedup; 3469 3470 static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts); 3471 static void btf_dedup_free(struct btf_dedup *d); 3472 static int btf_dedup_prep(struct btf_dedup *d); 3473 static int btf_dedup_strings(struct btf_dedup *d); 3474 static int btf_dedup_prim_types(struct btf_dedup *d); 3475 static int btf_dedup_struct_types(struct btf_dedup *d); 3476 static int btf_dedup_ref_types(struct btf_dedup *d); 3477 static int btf_dedup_resolve_fwds(struct btf_dedup *d); 3478 static int btf_dedup_compact_types(struct btf_dedup *d); 3479 static int btf_dedup_remap_types(struct btf_dedup *d); 3480 3481 /* 3482 * Deduplicate BTF types and strings. 3483 * 3484 * BTF dedup algorithm takes as an input `struct btf` representing `.BTF` ELF 3485 * section with all BTF type descriptors and string data. It overwrites that 3486 * memory in-place with deduplicated types and strings without any loss of 3487 * information. If optional `struct btf_ext` representing '.BTF.ext' ELF section 3488 * is provided, all the strings referenced from .BTF.ext section are honored 3489 * and updated to point to the right offsets after deduplication. 3490 * 3491 * If function returns with error, type/string data might be garbled and should 3492 * be discarded. 3493 * 3494 * More verbose and detailed description of both problem btf_dedup is solving, 3495 * as well as solution could be found at: 3496 * https://facebookmicrosites.github.io/bpf/blog/2018/11/14/btf-enhancement.html 3497 * 3498 * Problem description and justification 3499 * ===================================== 3500 * 3501 * BTF type information is typically emitted either as a result of conversion 3502 * from DWARF to BTF or directly by compiler. In both cases, each compilation 3503 * unit contains information about a subset of all the types that are used 3504 * in an application. These subsets are frequently overlapping and contain a lot 3505 * of duplicated information when later concatenated together into a single 3506 * binary. This algorithm ensures that each unique type is represented by single 3507 * BTF type descriptor, greatly reducing resulting size of BTF data. 3508 * 3509 * Compilation unit isolation and subsequent duplication of data is not the only 3510 * problem. The same type hierarchy (e.g., struct and all the type that struct 3511 * references) in different compilation units can be represented in BTF to 3512 * various degrees of completeness (or, rather, incompleteness) due to 3513 * struct/union forward declarations. 3514 * 3515 * Let's take a look at an example, that we'll use to better understand the 3516 * problem (and solution). Suppose we have two compilation units, each using 3517 * same `struct S`, but each of them having incomplete type information about 3518 * struct's fields: 3519 * 3520 * // CU #1: 3521 * struct S; 3522 * struct A { 3523 * int a; 3524 * struct A* self; 3525 * struct S* parent; 3526 * }; 3527 * struct B; 3528 * struct S { 3529 * struct A* a_ptr; 3530 * struct B* b_ptr; 3531 * }; 3532 * 3533 * // CU #2: 3534 * struct S; 3535 * struct A; 3536 * struct B { 3537 * int b; 3538 * struct B* self; 3539 * struct S* parent; 3540 * }; 3541 * struct S { 3542 * struct A* a_ptr; 3543 * struct B* b_ptr; 3544 * }; 3545 * 3546 * In case of CU #1, BTF data will know only that `struct B` exist (but no 3547 * more), but will know the complete type information about `struct A`. While 3548 * for CU #2, it will know full type information about `struct B`, but will 3549 * only know about forward declaration of `struct A` (in BTF terms, it will 3550 * have `BTF_KIND_FWD` type descriptor with name `B`). 3551 * 3552 * This compilation unit isolation means that it's possible that there is no 3553 * single CU with complete type information describing structs `S`, `A`, and 3554 * `B`. Also, we might get tons of duplicated and redundant type information. 3555 * 3556 * Additional complication we need to keep in mind comes from the fact that 3557 * types, in general, can form graphs containing cycles, not just DAGs. 3558 * 3559 * While algorithm does deduplication, it also merges and resolves type 3560 * information (unless disabled throught `struct btf_opts`), whenever possible. 3561 * E.g., in the example above with two compilation units having partial type 3562 * information for structs `A` and `B`, the output of algorithm will emit 3563 * a single copy of each BTF type that describes structs `A`, `B`, and `S` 3564 * (as well as type information for `int` and pointers), as if they were defined 3565 * in a single compilation unit as: 3566 * 3567 * struct A { 3568 * int a; 3569 * struct A* self; 3570 * struct S* parent; 3571 * }; 3572 * struct B { 3573 * int b; 3574 * struct B* self; 3575 * struct S* parent; 3576 * }; 3577 * struct S { 3578 * struct A* a_ptr; 3579 * struct B* b_ptr; 3580 * }; 3581 * 3582 * Algorithm summary 3583 * ================= 3584 * 3585 * Algorithm completes its work in 7 separate passes: 3586 * 3587 * 1. Strings deduplication. 3588 * 2. Primitive types deduplication (int, enum, fwd). 3589 * 3. Struct/union types deduplication. 3590 * 4. Resolve unambiguous forward declarations. 3591 * 5. Reference types deduplication (pointers, typedefs, arrays, funcs, func 3592 * protos, and const/volatile/restrict modifiers). 3593 * 6. Types compaction. 3594 * 7. Types remapping. 3595 * 3596 * Algorithm determines canonical type descriptor, which is a single 3597 * representative type for each truly unique type. This canonical type is the 3598 * one that will go into final deduplicated BTF type information. For 3599 * struct/unions, it is also the type that algorithm will merge additional type 3600 * information into (while resolving FWDs), as it discovers it from data in 3601 * other CUs. Each input BTF type eventually gets either mapped to itself, if 3602 * that type is canonical, or to some other type, if that type is equivalent 3603 * and was chosen as canonical representative. This mapping is stored in 3604 * `btf_dedup->map` array. This map is also used to record STRUCT/UNION that 3605 * FWD type got resolved to. 3606 * 3607 * To facilitate fast discovery of canonical types, we also maintain canonical 3608 * index (`btf_dedup->dedup_table`), which maps type descriptor's signature hash 3609 * (i.e., hashed kind, name, size, fields, etc) into a list of canonical types 3610 * that match that signature. With sufficiently good choice of type signature 3611 * hashing function, we can limit number of canonical types for each unique type 3612 * signature to a very small number, allowing to find canonical type for any 3613 * duplicated type very quickly. 3614 * 3615 * Struct/union deduplication is the most critical part and algorithm for 3616 * deduplicating structs/unions is described in greater details in comments for 3617 * `btf_dedup_is_equiv` function. 3618 */ 3619 int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts) 3620 { 3621 struct btf_dedup *d; 3622 int err; 3623 3624 if (!OPTS_VALID(opts, btf_dedup_opts)) 3625 return libbpf_err(-EINVAL); 3626 3627 d = btf_dedup_new(btf, opts); 3628 if (IS_ERR(d)) { 3629 pr_debug("btf_dedup_new failed: %ld\n", PTR_ERR(d)); 3630 return libbpf_err(-EINVAL); 3631 } 3632 3633 if (btf_ensure_modifiable(btf)) { 3634 err = -ENOMEM; 3635 goto done; 3636 } 3637 3638 err = btf_dedup_prep(d); 3639 if (err) { 3640 pr_debug("btf_dedup_prep failed: %s\n", errstr(err)); 3641 goto done; 3642 } 3643 err = btf_dedup_strings(d); 3644 if (err < 0) { 3645 pr_debug("btf_dedup_strings failed: %s\n", errstr(err)); 3646 goto done; 3647 } 3648 err = btf_dedup_prim_types(d); 3649 if (err < 0) { 3650 pr_debug("btf_dedup_prim_types failed: %s\n", errstr(err)); 3651 goto done; 3652 } 3653 err = btf_dedup_struct_types(d); 3654 if (err < 0) { 3655 pr_debug("btf_dedup_struct_types failed: %s\n", errstr(err)); 3656 goto done; 3657 } 3658 err = btf_dedup_resolve_fwds(d); 3659 if (err < 0) { 3660 pr_debug("btf_dedup_resolve_fwds failed: %s\n", errstr(err)); 3661 goto done; 3662 } 3663 err = btf_dedup_ref_types(d); 3664 if (err < 0) { 3665 pr_debug("btf_dedup_ref_types failed: %s\n", errstr(err)); 3666 goto done; 3667 } 3668 err = btf_dedup_compact_types(d); 3669 if (err < 0) { 3670 pr_debug("btf_dedup_compact_types failed: %s\n", errstr(err)); 3671 goto done; 3672 } 3673 err = btf_dedup_remap_types(d); 3674 if (err < 0) { 3675 pr_debug("btf_dedup_remap_types failed: %s\n", errstr(err)); 3676 goto done; 3677 } 3678 3679 done: 3680 btf_dedup_free(d); 3681 return libbpf_err(err); 3682 } 3683 3684 #define BTF_UNPROCESSED_ID ((__u32)-1) 3685 #define BTF_IN_PROGRESS_ID ((__u32)-2) 3686 3687 struct btf_dedup { 3688 /* .BTF section to be deduped in-place */ 3689 struct btf *btf; 3690 /* 3691 * Optional .BTF.ext section. When provided, any strings referenced 3692 * from it will be taken into account when deduping strings 3693 */ 3694 struct btf_ext *btf_ext; 3695 /* 3696 * This is a map from any type's signature hash to a list of possible 3697 * canonical representative type candidates. Hash collisions are 3698 * ignored, so even types of various kinds can share same list of 3699 * candidates, which is fine because we rely on subsequent 3700 * btf_xxx_equal() checks to authoritatively verify type equality. 3701 */ 3702 struct hashmap *dedup_table; 3703 /* Canonical types map */ 3704 __u32 *map; 3705 /* Hypothetical mapping, used during type graph equivalence checks */ 3706 __u32 *hypot_map; 3707 __u32 *hypot_list; 3708 size_t hypot_cnt; 3709 size_t hypot_cap; 3710 /* Whether hypothetical mapping, if successful, would need to adjust 3711 * already canonicalized types (due to a new forward declaration to 3712 * concrete type resolution). In such case, during split BTF dedup 3713 * candidate type would still be considered as different, because base 3714 * BTF is considered to be immutable. 3715 */ 3716 bool hypot_adjust_canon; 3717 /* Various option modifying behavior of algorithm */ 3718 struct btf_dedup_opts opts; 3719 /* temporary strings deduplication state */ 3720 struct strset *strs_set; 3721 }; 3722 3723 static unsigned long hash_combine(unsigned long h, unsigned long value) 3724 { 3725 return h * 31 + value; 3726 } 3727 3728 #define for_each_dedup_cand(d, node, hash) \ 3729 hashmap__for_each_key_entry(d->dedup_table, node, hash) 3730 3731 static int btf_dedup_table_add(struct btf_dedup *d, long hash, __u32 type_id) 3732 { 3733 return hashmap__append(d->dedup_table, hash, type_id); 3734 } 3735 3736 static int btf_dedup_hypot_map_add(struct btf_dedup *d, 3737 __u32 from_id, __u32 to_id) 3738 { 3739 if (d->hypot_cnt == d->hypot_cap) { 3740 __u32 *new_list; 3741 3742 d->hypot_cap += max((size_t)16, d->hypot_cap / 2); 3743 new_list = libbpf_reallocarray(d->hypot_list, d->hypot_cap, sizeof(__u32)); 3744 if (!new_list) 3745 return -ENOMEM; 3746 d->hypot_list = new_list; 3747 } 3748 d->hypot_list[d->hypot_cnt++] = from_id; 3749 d->hypot_map[from_id] = to_id; 3750 return 0; 3751 } 3752 3753 static void btf_dedup_clear_hypot_map(struct btf_dedup *d) 3754 { 3755 int i; 3756 3757 for (i = 0; i < d->hypot_cnt; i++) 3758 d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID; 3759 d->hypot_cnt = 0; 3760 d->hypot_adjust_canon = false; 3761 } 3762 3763 static void btf_dedup_free(struct btf_dedup *d) 3764 { 3765 hashmap__free(d->dedup_table); 3766 d->dedup_table = NULL; 3767 3768 free(d->map); 3769 d->map = NULL; 3770 3771 free(d->hypot_map); 3772 d->hypot_map = NULL; 3773 3774 free(d->hypot_list); 3775 d->hypot_list = NULL; 3776 3777 free(d); 3778 } 3779 3780 static size_t btf_dedup_identity_hash_fn(long key, void *ctx) 3781 { 3782 return key; 3783 } 3784 3785 static size_t btf_dedup_collision_hash_fn(long key, void *ctx) 3786 { 3787 return 0; 3788 } 3789 3790 static bool btf_dedup_equal_fn(long k1, long k2, void *ctx) 3791 { 3792 return k1 == k2; 3793 } 3794 3795 static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts) 3796 { 3797 struct btf_dedup *d = calloc(1, sizeof(struct btf_dedup)); 3798 hashmap_hash_fn hash_fn = btf_dedup_identity_hash_fn; 3799 int i, err = 0, type_cnt; 3800 3801 if (!d) 3802 return ERR_PTR(-ENOMEM); 3803 3804 if (OPTS_GET(opts, force_collisions, false)) 3805 hash_fn = btf_dedup_collision_hash_fn; 3806 3807 d->btf = btf; 3808 d->btf_ext = OPTS_GET(opts, btf_ext, NULL); 3809 3810 d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL); 3811 if (IS_ERR(d->dedup_table)) { 3812 err = PTR_ERR(d->dedup_table); 3813 d->dedup_table = NULL; 3814 goto done; 3815 } 3816 3817 type_cnt = btf__type_cnt(btf); 3818 d->map = malloc(sizeof(__u32) * type_cnt); 3819 if (!d->map) { 3820 err = -ENOMEM; 3821 goto done; 3822 } 3823 /* special BTF "void" type is made canonical immediately */ 3824 d->map[0] = 0; 3825 for (i = 1; i < type_cnt; i++) { 3826 struct btf_type *t = btf_type_by_id(d->btf, i); 3827 3828 /* VAR and DATASEC are never deduped and are self-canonical */ 3829 if (btf_is_var(t) || btf_is_datasec(t)) 3830 d->map[i] = i; 3831 else 3832 d->map[i] = BTF_UNPROCESSED_ID; 3833 } 3834 3835 d->hypot_map = malloc(sizeof(__u32) * type_cnt); 3836 if (!d->hypot_map) { 3837 err = -ENOMEM; 3838 goto done; 3839 } 3840 for (i = 0; i < type_cnt; i++) 3841 d->hypot_map[i] = BTF_UNPROCESSED_ID; 3842 3843 done: 3844 if (err) { 3845 btf_dedup_free(d); 3846 return ERR_PTR(err); 3847 } 3848 3849 return d; 3850 } 3851 3852 /* 3853 * Iterate over all possible places in .BTF and .BTF.ext that can reference 3854 * string and pass pointer to it to a provided callback `fn`. 3855 */ 3856 static int btf_for_each_str_off(struct btf_dedup *d, str_off_visit_fn fn, void *ctx) 3857 { 3858 int i, r; 3859 3860 for (i = 0; i < d->btf->nr_types; i++) { 3861 struct btf_field_iter it; 3862 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i); 3863 __u32 *str_off; 3864 3865 r = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS); 3866 if (r) 3867 return r; 3868 3869 while ((str_off = btf_field_iter_next(&it))) { 3870 r = fn(str_off, ctx); 3871 if (r) 3872 return r; 3873 } 3874 } 3875 3876 if (!d->btf_ext) 3877 return 0; 3878 3879 r = btf_ext_visit_str_offs(d->btf_ext, fn, ctx); 3880 if (r) 3881 return r; 3882 3883 return 0; 3884 } 3885 3886 static int strs_dedup_remap_str_off(__u32 *str_off_ptr, void *ctx) 3887 { 3888 struct btf_dedup *d = ctx; 3889 __u32 str_off = *str_off_ptr; 3890 const char *s; 3891 int off, err; 3892 3893 /* don't touch empty string or string in main BTF */ 3894 if (str_off == 0 || str_off < d->btf->start_str_off) 3895 return 0; 3896 3897 s = btf__str_by_offset(d->btf, str_off); 3898 if (d->btf->base_btf) { 3899 err = btf__find_str(d->btf->base_btf, s); 3900 if (err >= 0) { 3901 *str_off_ptr = err; 3902 return 0; 3903 } 3904 if (err != -ENOENT) 3905 return err; 3906 } 3907 3908 off = strset__add_str(d->strs_set, s); 3909 if (off < 0) 3910 return off; 3911 3912 *str_off_ptr = d->btf->start_str_off + off; 3913 return 0; 3914 } 3915 3916 /* 3917 * Dedup string and filter out those that are not referenced from either .BTF 3918 * or .BTF.ext (if provided) sections. 3919 * 3920 * This is done by building index of all strings in BTF's string section, 3921 * then iterating over all entities that can reference strings (e.g., type 3922 * names, struct field names, .BTF.ext line info, etc) and marking corresponding 3923 * strings as used. After that all used strings are deduped and compacted into 3924 * sequential blob of memory and new offsets are calculated. Then all the string 3925 * references are iterated again and rewritten using new offsets. 3926 */ 3927 static int btf_dedup_strings(struct btf_dedup *d) 3928 { 3929 int err; 3930 3931 if (d->btf->strs_deduped) 3932 return 0; 3933 3934 d->strs_set = strset__new(BTF_MAX_STR_OFFSET, NULL, 0); 3935 if (IS_ERR(d->strs_set)) { 3936 err = PTR_ERR(d->strs_set); 3937 goto err_out; 3938 } 3939 3940 if (!d->btf->base_btf) { 3941 /* insert empty string; we won't be looking it up during strings 3942 * dedup, but it's good to have it for generic BTF string lookups 3943 */ 3944 err = strset__add_str(d->strs_set, ""); 3945 if (err < 0) 3946 goto err_out; 3947 } 3948 3949 /* remap string offsets */ 3950 err = btf_for_each_str_off(d, strs_dedup_remap_str_off, d); 3951 if (err) 3952 goto err_out; 3953 3954 /* replace BTF string data and hash with deduped ones */ 3955 strset__free(d->btf->strs_set); 3956 d->btf->hdr->str_len = strset__data_size(d->strs_set); 3957 d->btf->strs_set = d->strs_set; 3958 d->strs_set = NULL; 3959 d->btf->strs_deduped = true; 3960 return 0; 3961 3962 err_out: 3963 strset__free(d->strs_set); 3964 d->strs_set = NULL; 3965 3966 return err; 3967 } 3968 3969 /* 3970 * Calculate type signature hash of TYPEDEF, ignoring referenced type IDs, 3971 * as referenced type IDs equivalence is established separately during type 3972 * graph equivalence check algorithm. 3973 */ 3974 static long btf_hash_typedef(struct btf_type *t) 3975 { 3976 long h; 3977 3978 h = hash_combine(0, t->name_off); 3979 h = hash_combine(h, t->info); 3980 return h; 3981 } 3982 3983 static long btf_hash_common(struct btf_type *t) 3984 { 3985 long h; 3986 3987 h = hash_combine(0, t->name_off); 3988 h = hash_combine(h, t->info); 3989 h = hash_combine(h, t->size); 3990 return h; 3991 } 3992 3993 static bool btf_equal_common(struct btf_type *t1, struct btf_type *t2) 3994 { 3995 return t1->name_off == t2->name_off && 3996 t1->info == t2->info && 3997 t1->size == t2->size; 3998 } 3999 4000 /* Check structural compatibility of two TYPEDEF. */ 4001 static bool btf_equal_typedef(struct btf_type *t1, struct btf_type *t2) 4002 { 4003 return t1->name_off == t2->name_off && 4004 t1->info == t2->info; 4005 } 4006 4007 /* Calculate type signature hash of INT or TAG. */ 4008 static long btf_hash_int_decl_tag(struct btf_type *t) 4009 { 4010 __u32 info = *(__u32 *)(t + 1); 4011 long h; 4012 4013 h = btf_hash_common(t); 4014 h = hash_combine(h, info); 4015 return h; 4016 } 4017 4018 /* Check structural equality of two INTs or TAGs. */ 4019 static bool btf_equal_int_tag(struct btf_type *t1, struct btf_type *t2) 4020 { 4021 __u32 info1, info2; 4022 4023 if (!btf_equal_common(t1, t2)) 4024 return false; 4025 info1 = *(__u32 *)(t1 + 1); 4026 info2 = *(__u32 *)(t2 + 1); 4027 return info1 == info2; 4028 } 4029 4030 /* Calculate type signature hash of ENUM/ENUM64. */ 4031 static long btf_hash_enum(struct btf_type *t) 4032 { 4033 long h; 4034 4035 /* don't hash vlen, enum members and size to support enum fwd resolving */ 4036 h = hash_combine(0, t->name_off); 4037 return h; 4038 } 4039 4040 static bool btf_equal_enum_members(struct btf_type *t1, struct btf_type *t2) 4041 { 4042 const struct btf_enum *m1, *m2; 4043 __u16 vlen; 4044 int i; 4045 4046 vlen = btf_vlen(t1); 4047 m1 = btf_enum(t1); 4048 m2 = btf_enum(t2); 4049 for (i = 0; i < vlen; i++) { 4050 if (m1->name_off != m2->name_off || m1->val != m2->val) 4051 return false; 4052 m1++; 4053 m2++; 4054 } 4055 return true; 4056 } 4057 4058 static bool btf_equal_enum64_members(struct btf_type *t1, struct btf_type *t2) 4059 { 4060 const struct btf_enum64 *m1, *m2; 4061 __u16 vlen; 4062 int i; 4063 4064 vlen = btf_vlen(t1); 4065 m1 = btf_enum64(t1); 4066 m2 = btf_enum64(t2); 4067 for (i = 0; i < vlen; i++) { 4068 if (m1->name_off != m2->name_off || m1->val_lo32 != m2->val_lo32 || 4069 m1->val_hi32 != m2->val_hi32) 4070 return false; 4071 m1++; 4072 m2++; 4073 } 4074 return true; 4075 } 4076 4077 /* Check structural equality of two ENUMs or ENUM64s. */ 4078 static bool btf_equal_enum(struct btf_type *t1, struct btf_type *t2) 4079 { 4080 if (!btf_equal_common(t1, t2)) 4081 return false; 4082 4083 /* t1 & t2 kinds are identical because of btf_equal_common */ 4084 if (btf_kind(t1) == BTF_KIND_ENUM) 4085 return btf_equal_enum_members(t1, t2); 4086 else 4087 return btf_equal_enum64_members(t1, t2); 4088 } 4089 4090 static inline bool btf_is_enum_fwd(struct btf_type *t) 4091 { 4092 return btf_is_any_enum(t) && btf_vlen(t) == 0; 4093 } 4094 4095 static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2) 4096 { 4097 if (!btf_is_enum_fwd(t1) && !btf_is_enum_fwd(t2)) 4098 return btf_equal_enum(t1, t2); 4099 /* At this point either t1 or t2 or both are forward declarations, thus: 4100 * - skip comparing vlen because it is zero for forward declarations; 4101 * - skip comparing size to allow enum forward declarations 4102 * to be compatible with enum64 full declarations; 4103 * - skip comparing kind for the same reason. 4104 */ 4105 return t1->name_off == t2->name_off && 4106 btf_is_any_enum(t1) && btf_is_any_enum(t2); 4107 } 4108 4109 /* 4110 * Calculate type signature hash of STRUCT/UNION, ignoring referenced type IDs, 4111 * as referenced type IDs equivalence is established separately during type 4112 * graph equivalence check algorithm. 4113 */ 4114 static long btf_hash_struct(struct btf_type *t) 4115 { 4116 const struct btf_member *member = btf_members(t); 4117 __u32 vlen = btf_vlen(t); 4118 long h = btf_hash_common(t); 4119 int i; 4120 4121 for (i = 0; i < vlen; i++) { 4122 h = hash_combine(h, member->name_off); 4123 h = hash_combine(h, member->offset); 4124 /* no hashing of referenced type ID, it can be unresolved yet */ 4125 member++; 4126 } 4127 return h; 4128 } 4129 4130 /* 4131 * Check structural compatibility of two STRUCTs/UNIONs, ignoring referenced 4132 * type IDs. This check is performed during type graph equivalence check and 4133 * referenced types equivalence is checked separately. 4134 */ 4135 static bool btf_shallow_equal_struct(struct btf_type *t1, struct btf_type *t2) 4136 { 4137 const struct btf_member *m1, *m2; 4138 __u16 vlen; 4139 int i; 4140 4141 if (!btf_equal_common(t1, t2)) 4142 return false; 4143 4144 vlen = btf_vlen(t1); 4145 m1 = btf_members(t1); 4146 m2 = btf_members(t2); 4147 for (i = 0; i < vlen; i++) { 4148 if (m1->name_off != m2->name_off || m1->offset != m2->offset) 4149 return false; 4150 m1++; 4151 m2++; 4152 } 4153 return true; 4154 } 4155 4156 /* 4157 * Calculate type signature hash of ARRAY, including referenced type IDs, 4158 * under assumption that they were already resolved to canonical type IDs and 4159 * are not going to change. 4160 */ 4161 static long btf_hash_array(struct btf_type *t) 4162 { 4163 const struct btf_array *info = btf_array(t); 4164 long h = btf_hash_common(t); 4165 4166 h = hash_combine(h, info->type); 4167 h = hash_combine(h, info->index_type); 4168 h = hash_combine(h, info->nelems); 4169 return h; 4170 } 4171 4172 /* 4173 * Check exact equality of two ARRAYs, taking into account referenced 4174 * type IDs, under assumption that they were already resolved to canonical 4175 * type IDs and are not going to change. 4176 * This function is called during reference types deduplication to compare 4177 * ARRAY to potential canonical representative. 4178 */ 4179 static bool btf_equal_array(struct btf_type *t1, struct btf_type *t2) 4180 { 4181 const struct btf_array *info1, *info2; 4182 4183 if (!btf_equal_common(t1, t2)) 4184 return false; 4185 4186 info1 = btf_array(t1); 4187 info2 = btf_array(t2); 4188 return info1->type == info2->type && 4189 info1->index_type == info2->index_type && 4190 info1->nelems == info2->nelems; 4191 } 4192 4193 /* 4194 * Check structural compatibility of two ARRAYs, ignoring referenced type 4195 * IDs. This check is performed during type graph equivalence check and 4196 * referenced types equivalence is checked separately. 4197 */ 4198 static bool btf_compat_array(struct btf_type *t1, struct btf_type *t2) 4199 { 4200 if (!btf_equal_common(t1, t2)) 4201 return false; 4202 4203 return btf_array(t1)->nelems == btf_array(t2)->nelems; 4204 } 4205 4206 /* 4207 * Calculate type signature hash of FUNC_PROTO, including referenced type IDs, 4208 * under assumption that they were already resolved to canonical type IDs and 4209 * are not going to change. 4210 */ 4211 static long btf_hash_fnproto(struct btf_type *t) 4212 { 4213 const struct btf_param *member = btf_params(t); 4214 __u16 vlen = btf_vlen(t); 4215 long h = btf_hash_common(t); 4216 int i; 4217 4218 for (i = 0; i < vlen; i++) { 4219 h = hash_combine(h, member->name_off); 4220 h = hash_combine(h, member->type); 4221 member++; 4222 } 4223 return h; 4224 } 4225 4226 /* 4227 * Check exact equality of two FUNC_PROTOs, taking into account referenced 4228 * type IDs, under assumption that they were already resolved to canonical 4229 * type IDs and are not going to change. 4230 * This function is called during reference types deduplication to compare 4231 * FUNC_PROTO to potential canonical representative. 4232 */ 4233 static bool btf_equal_fnproto(struct btf_type *t1, struct btf_type *t2) 4234 { 4235 const struct btf_param *m1, *m2; 4236 __u16 vlen; 4237 int i; 4238 4239 if (!btf_equal_common(t1, t2)) 4240 return false; 4241 4242 vlen = btf_vlen(t1); 4243 m1 = btf_params(t1); 4244 m2 = btf_params(t2); 4245 for (i = 0; i < vlen; i++) { 4246 if (m1->name_off != m2->name_off || m1->type != m2->type) 4247 return false; 4248 m1++; 4249 m2++; 4250 } 4251 return true; 4252 } 4253 4254 /* 4255 * Check structural compatibility of two FUNC_PROTOs, ignoring referenced type 4256 * IDs. This check is performed during type graph equivalence check and 4257 * referenced types equivalence is checked separately. 4258 */ 4259 static bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2) 4260 { 4261 const struct btf_param *m1, *m2; 4262 __u16 vlen; 4263 int i; 4264 4265 /* skip return type ID */ 4266 if (t1->name_off != t2->name_off || t1->info != t2->info) 4267 return false; 4268 4269 vlen = btf_vlen(t1); 4270 m1 = btf_params(t1); 4271 m2 = btf_params(t2); 4272 for (i = 0; i < vlen; i++) { 4273 if (m1->name_off != m2->name_off) 4274 return false; 4275 m1++; 4276 m2++; 4277 } 4278 return true; 4279 } 4280 4281 /* Prepare split BTF for deduplication by calculating hashes of base BTF's 4282 * types and initializing the rest of the state (canonical type mapping) for 4283 * the fixed base BTF part. 4284 */ 4285 static int btf_dedup_prep(struct btf_dedup *d) 4286 { 4287 struct btf_type *t; 4288 int type_id; 4289 long h; 4290 4291 if (!d->btf->base_btf) 4292 return 0; 4293 4294 for (type_id = 1; type_id < d->btf->start_id; type_id++) { 4295 t = btf_type_by_id(d->btf, type_id); 4296 4297 /* all base BTF types are self-canonical by definition */ 4298 d->map[type_id] = type_id; 4299 4300 switch (btf_kind(t)) { 4301 case BTF_KIND_VAR: 4302 case BTF_KIND_DATASEC: 4303 /* VAR and DATASEC are never hash/deduplicated */ 4304 continue; 4305 case BTF_KIND_CONST: 4306 case BTF_KIND_VOLATILE: 4307 case BTF_KIND_RESTRICT: 4308 case BTF_KIND_PTR: 4309 case BTF_KIND_FWD: 4310 case BTF_KIND_TYPEDEF: 4311 case BTF_KIND_FUNC: 4312 case BTF_KIND_FLOAT: 4313 case BTF_KIND_TYPE_TAG: 4314 h = btf_hash_common(t); 4315 break; 4316 case BTF_KIND_INT: 4317 case BTF_KIND_DECL_TAG: 4318 h = btf_hash_int_decl_tag(t); 4319 break; 4320 case BTF_KIND_ENUM: 4321 case BTF_KIND_ENUM64: 4322 h = btf_hash_enum(t); 4323 break; 4324 case BTF_KIND_STRUCT: 4325 case BTF_KIND_UNION: 4326 h = btf_hash_struct(t); 4327 break; 4328 case BTF_KIND_ARRAY: 4329 h = btf_hash_array(t); 4330 break; 4331 case BTF_KIND_FUNC_PROTO: 4332 h = btf_hash_fnproto(t); 4333 break; 4334 default: 4335 pr_debug("unknown kind %d for type [%d]\n", btf_kind(t), type_id); 4336 return -EINVAL; 4337 } 4338 if (btf_dedup_table_add(d, h, type_id)) 4339 return -ENOMEM; 4340 } 4341 4342 return 0; 4343 } 4344 4345 /* 4346 * Deduplicate primitive types, that can't reference other types, by calculating 4347 * their type signature hash and comparing them with any possible canonical 4348 * candidate. If no canonical candidate matches, type itself is marked as 4349 * canonical and is added into `btf_dedup->dedup_table` as another candidate. 4350 */ 4351 static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) 4352 { 4353 struct btf_type *t = btf_type_by_id(d->btf, type_id); 4354 struct hashmap_entry *hash_entry; 4355 struct btf_type *cand; 4356 /* if we don't find equivalent type, then we are canonical */ 4357 __u32 new_id = type_id; 4358 __u32 cand_id; 4359 long h; 4360 4361 switch (btf_kind(t)) { 4362 case BTF_KIND_CONST: 4363 case BTF_KIND_VOLATILE: 4364 case BTF_KIND_RESTRICT: 4365 case BTF_KIND_PTR: 4366 case BTF_KIND_TYPEDEF: 4367 case BTF_KIND_ARRAY: 4368 case BTF_KIND_STRUCT: 4369 case BTF_KIND_UNION: 4370 case BTF_KIND_FUNC: 4371 case BTF_KIND_FUNC_PROTO: 4372 case BTF_KIND_VAR: 4373 case BTF_KIND_DATASEC: 4374 case BTF_KIND_DECL_TAG: 4375 case BTF_KIND_TYPE_TAG: 4376 return 0; 4377 4378 case BTF_KIND_INT: 4379 h = btf_hash_int_decl_tag(t); 4380 for_each_dedup_cand(d, hash_entry, h) { 4381 cand_id = hash_entry->value; 4382 cand = btf_type_by_id(d->btf, cand_id); 4383 if (btf_equal_int_tag(t, cand)) { 4384 new_id = cand_id; 4385 break; 4386 } 4387 } 4388 break; 4389 4390 case BTF_KIND_ENUM: 4391 case BTF_KIND_ENUM64: 4392 h = btf_hash_enum(t); 4393 for_each_dedup_cand(d, hash_entry, h) { 4394 cand_id = hash_entry->value; 4395 cand = btf_type_by_id(d->btf, cand_id); 4396 if (btf_equal_enum(t, cand)) { 4397 new_id = cand_id; 4398 break; 4399 } 4400 if (btf_compat_enum(t, cand)) { 4401 if (btf_is_enum_fwd(t)) { 4402 /* resolve fwd to full enum */ 4403 new_id = cand_id; 4404 break; 4405 } 4406 /* resolve canonical enum fwd to full enum */ 4407 d->map[cand_id] = type_id; 4408 } 4409 } 4410 break; 4411 4412 case BTF_KIND_FWD: 4413 case BTF_KIND_FLOAT: 4414 h = btf_hash_common(t); 4415 for_each_dedup_cand(d, hash_entry, h) { 4416 cand_id = hash_entry->value; 4417 cand = btf_type_by_id(d->btf, cand_id); 4418 if (btf_equal_common(t, cand)) { 4419 new_id = cand_id; 4420 break; 4421 } 4422 } 4423 break; 4424 4425 default: 4426 return -EINVAL; 4427 } 4428 4429 d->map[type_id] = new_id; 4430 if (type_id == new_id && btf_dedup_table_add(d, h, type_id)) 4431 return -ENOMEM; 4432 4433 return 0; 4434 } 4435 4436 static int btf_dedup_prim_types(struct btf_dedup *d) 4437 { 4438 int i, err; 4439 4440 for (i = 0; i < d->btf->nr_types; i++) { 4441 err = btf_dedup_prim_type(d, d->btf->start_id + i); 4442 if (err) 4443 return err; 4444 } 4445 return 0; 4446 } 4447 4448 /* 4449 * Check whether type is already mapped into canonical one (could be to itself). 4450 */ 4451 static inline bool is_type_mapped(struct btf_dedup *d, uint32_t type_id) 4452 { 4453 return d->map[type_id] <= BTF_MAX_NR_TYPES; 4454 } 4455 4456 /* 4457 * Resolve type ID into its canonical type ID, if any; otherwise return original 4458 * type ID. If type is FWD and is resolved into STRUCT/UNION already, follow 4459 * STRUCT/UNION link and resolve it into canonical type ID as well. 4460 */ 4461 static inline __u32 resolve_type_id(struct btf_dedup *d, __u32 type_id) 4462 { 4463 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) 4464 type_id = d->map[type_id]; 4465 return type_id; 4466 } 4467 4468 /* 4469 * Resolve FWD to underlying STRUCT/UNION, if any; otherwise return original 4470 * type ID. 4471 */ 4472 static uint32_t resolve_fwd_id(struct btf_dedup *d, uint32_t type_id) 4473 { 4474 __u32 orig_type_id = type_id; 4475 4476 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) 4477 return type_id; 4478 4479 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) 4480 type_id = d->map[type_id]; 4481 4482 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) 4483 return type_id; 4484 4485 return orig_type_id; 4486 } 4487 4488 4489 static inline __u16 btf_fwd_kind(struct btf_type *t) 4490 { 4491 return btf_kflag(t) ? BTF_KIND_UNION : BTF_KIND_STRUCT; 4492 } 4493 4494 static bool btf_dedup_identical_types(struct btf_dedup *d, __u32 id1, __u32 id2, int depth) 4495 { 4496 struct btf_type *t1, *t2; 4497 int k1, k2; 4498 recur: 4499 t1 = btf_type_by_id(d->btf, id1); 4500 t2 = btf_type_by_id(d->btf, id2); 4501 if (depth <= 0) { 4502 pr_debug("Reached depth limit for identical type comparison for '%s'/'%s'\n", 4503 btf__name_by_offset(d->btf, t1->name_off), 4504 btf__name_by_offset(d->btf, t2->name_off)); 4505 return false; 4506 } 4507 4508 k1 = btf_kind(t1); 4509 k2 = btf_kind(t2); 4510 if (k1 != k2) 4511 return false; 4512 4513 switch (k1) { 4514 case BTF_KIND_UNKN: /* VOID */ 4515 return true; 4516 case BTF_KIND_INT: 4517 return btf_equal_int_tag(t1, t2); 4518 case BTF_KIND_ENUM: 4519 case BTF_KIND_ENUM64: 4520 return btf_compat_enum(t1, t2); 4521 case BTF_KIND_FWD: 4522 case BTF_KIND_FLOAT: 4523 return btf_equal_common(t1, t2); 4524 case BTF_KIND_CONST: 4525 case BTF_KIND_VOLATILE: 4526 case BTF_KIND_RESTRICT: 4527 case BTF_KIND_PTR: 4528 case BTF_KIND_TYPEDEF: 4529 case BTF_KIND_FUNC: 4530 case BTF_KIND_TYPE_TAG: 4531 if (t1->info != t2->info || t1->name_off != t2->name_off) 4532 return false; 4533 id1 = t1->type; 4534 id2 = t2->type; 4535 goto recur; 4536 case BTF_KIND_ARRAY: { 4537 struct btf_array *a1, *a2; 4538 4539 if (!btf_compat_array(t1, t2)) 4540 return false; 4541 4542 a1 = btf_array(t1); 4543 a2 = btf_array(t1); 4544 4545 if (a1->index_type != a2->index_type && 4546 !btf_dedup_identical_types(d, a1->index_type, a2->index_type, depth - 1)) 4547 return false; 4548 4549 if (a1->type != a2->type && 4550 !btf_dedup_identical_types(d, a1->type, a2->type, depth - 1)) 4551 return false; 4552 4553 return true; 4554 } 4555 case BTF_KIND_STRUCT: 4556 case BTF_KIND_UNION: { 4557 const struct btf_member *m1, *m2; 4558 int i, n; 4559 4560 if (!btf_shallow_equal_struct(t1, t2)) 4561 return false; 4562 4563 m1 = btf_members(t1); 4564 m2 = btf_members(t2); 4565 for (i = 0, n = btf_vlen(t1); i < n; i++, m1++, m2++) { 4566 if (m1->type == m2->type) 4567 continue; 4568 if (!btf_dedup_identical_types(d, m1->type, m2->type, depth - 1)) { 4569 if (t1->name_off) { 4570 pr_debug("%s '%s' size=%d vlen=%d id1[%u] id2[%u] shallow-equal but not identical for field#%d '%s'\n", 4571 k1 == BTF_KIND_STRUCT ? "STRUCT" : "UNION", 4572 btf__name_by_offset(d->btf, t1->name_off), 4573 t1->size, btf_vlen(t1), id1, id2, i, 4574 btf__name_by_offset(d->btf, m1->name_off)); 4575 } 4576 return false; 4577 } 4578 } 4579 return true; 4580 } 4581 case BTF_KIND_FUNC_PROTO: { 4582 const struct btf_param *p1, *p2; 4583 int i, n; 4584 4585 if (!btf_compat_fnproto(t1, t2)) 4586 return false; 4587 4588 if (t1->type != t2->type && 4589 !btf_dedup_identical_types(d, t1->type, t2->type, depth - 1)) 4590 return false; 4591 4592 p1 = btf_params(t1); 4593 p2 = btf_params(t2); 4594 for (i = 0, n = btf_vlen(t1); i < n; i++, p1++, p2++) { 4595 if (p1->type == p2->type) 4596 continue; 4597 if (!btf_dedup_identical_types(d, p1->type, p2->type, depth - 1)) 4598 return false; 4599 } 4600 return true; 4601 } 4602 default: 4603 return false; 4604 } 4605 } 4606 4607 4608 /* 4609 * Check equivalence of BTF type graph formed by candidate struct/union (we'll 4610 * call it "candidate graph" in this description for brevity) to a type graph 4611 * formed by (potential) canonical struct/union ("canonical graph" for brevity 4612 * here, though keep in mind that not all types in canonical graph are 4613 * necessarily canonical representatives themselves, some of them might be 4614 * duplicates or its uniqueness might not have been established yet). 4615 * Returns: 4616 * - >0, if type graphs are equivalent; 4617 * - 0, if not equivalent; 4618 * - <0, on error. 4619 * 4620 * Algorithm performs side-by-side DFS traversal of both type graphs and checks 4621 * equivalence of BTF types at each step. If at any point BTF types in candidate 4622 * and canonical graphs are not compatible structurally, whole graphs are 4623 * incompatible. If types are structurally equivalent (i.e., all information 4624 * except referenced type IDs is exactly the same), a mapping from `canon_id` to 4625 * a `cand_id` is recoded in hypothetical mapping (`btf_dedup->hypot_map`). 4626 * If a type references other types, then those referenced types are checked 4627 * for equivalence recursively. 4628 * 4629 * During DFS traversal, if we find that for current `canon_id` type we 4630 * already have some mapping in hypothetical map, we check for two possible 4631 * situations: 4632 * - `canon_id` is mapped to exactly the same type as `cand_id`. This will 4633 * happen when type graphs have cycles. In this case we assume those two 4634 * types are equivalent. 4635 * - `canon_id` is mapped to different type. This is contradiction in our 4636 * hypothetical mapping, because same graph in canonical graph corresponds 4637 * to two different types in candidate graph, which for equivalent type 4638 * graphs shouldn't happen. This condition terminates equivalence check 4639 * with negative result. 4640 * 4641 * If type graphs traversal exhausts types to check and find no contradiction, 4642 * then type graphs are equivalent. 4643 * 4644 * When checking types for equivalence, there is one special case: FWD types. 4645 * If FWD type resolution is allowed and one of the types (either from canonical 4646 * or candidate graph) is FWD and other is STRUCT/UNION (depending on FWD's kind 4647 * flag) and their names match, hypothetical mapping is updated to point from 4648 * FWD to STRUCT/UNION. If graphs will be determined as equivalent successfully, 4649 * this mapping will be used to record FWD -> STRUCT/UNION mapping permanently. 4650 * 4651 * Technically, this could lead to incorrect FWD to STRUCT/UNION resolution, 4652 * if there are two exactly named (or anonymous) structs/unions that are 4653 * compatible structurally, one of which has FWD field, while other is concrete 4654 * STRUCT/UNION, but according to C sources they are different structs/unions 4655 * that are referencing different types with the same name. This is extremely 4656 * unlikely to happen, but btf_dedup API allows to disable FWD resolution if 4657 * this logic is causing problems. 4658 * 4659 * Doing FWD resolution means that both candidate and/or canonical graphs can 4660 * consists of portions of the graph that come from multiple compilation units. 4661 * This is due to the fact that types within single compilation unit are always 4662 * deduplicated and FWDs are already resolved, if referenced struct/union 4663 * definition is available. So, if we had unresolved FWD and found corresponding 4664 * STRUCT/UNION, they will be from different compilation units. This 4665 * consequently means that when we "link" FWD to corresponding STRUCT/UNION, 4666 * type graph will likely have at least two different BTF types that describe 4667 * same type (e.g., most probably there will be two different BTF types for the 4668 * same 'int' primitive type) and could even have "overlapping" parts of type 4669 * graph that describe same subset of types. 4670 * 4671 * This in turn means that our assumption that each type in canonical graph 4672 * must correspond to exactly one type in candidate graph might not hold 4673 * anymore and will make it harder to detect contradictions using hypothetical 4674 * map. To handle this problem, we allow to follow FWD -> STRUCT/UNION 4675 * resolution only in canonical graph. FWDs in candidate graphs are never 4676 * resolved. To see why it's OK, let's check all possible situations w.r.t. FWDs 4677 * that can occur: 4678 * - Both types in canonical and candidate graphs are FWDs. If they are 4679 * structurally equivalent, then they can either be both resolved to the 4680 * same STRUCT/UNION or not resolved at all. In both cases they are 4681 * equivalent and there is no need to resolve FWD on candidate side. 4682 * - Both types in canonical and candidate graphs are concrete STRUCT/UNION, 4683 * so nothing to resolve as well, algorithm will check equivalence anyway. 4684 * - Type in canonical graph is FWD, while type in candidate is concrete 4685 * STRUCT/UNION. In this case candidate graph comes from single compilation 4686 * unit, so there is exactly one BTF type for each unique C type. After 4687 * resolving FWD into STRUCT/UNION, there might be more than one BTF type 4688 * in canonical graph mapping to single BTF type in candidate graph, but 4689 * because hypothetical mapping maps from canonical to candidate types, it's 4690 * alright, and we still maintain the property of having single `canon_id` 4691 * mapping to single `cand_id` (there could be two different `canon_id` 4692 * mapped to the same `cand_id`, but it's not contradictory). 4693 * - Type in canonical graph is concrete STRUCT/UNION, while type in candidate 4694 * graph is FWD. In this case we are just going to check compatibility of 4695 * STRUCT/UNION and corresponding FWD, and if they are compatible, we'll 4696 * assume that whatever STRUCT/UNION FWD resolves to must be equivalent to 4697 * a concrete STRUCT/UNION from canonical graph. If the rest of type graphs 4698 * turn out equivalent, we'll re-resolve FWD to concrete STRUCT/UNION from 4699 * canonical graph. 4700 */ 4701 static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, 4702 __u32 canon_id) 4703 { 4704 struct btf_type *cand_type; 4705 struct btf_type *canon_type; 4706 __u32 hypot_type_id; 4707 __u16 cand_kind; 4708 __u16 canon_kind; 4709 int i, eq; 4710 4711 /* if both resolve to the same canonical, they must be equivalent */ 4712 if (resolve_type_id(d, cand_id) == resolve_type_id(d, canon_id)) 4713 return 1; 4714 4715 canon_id = resolve_fwd_id(d, canon_id); 4716 4717 hypot_type_id = d->hypot_map[canon_id]; 4718 if (hypot_type_id <= BTF_MAX_NR_TYPES) { 4719 if (hypot_type_id == cand_id) 4720 return 1; 4721 /* In some cases compiler will generate different DWARF types 4722 * for *identical* array type definitions and use them for 4723 * different fields within the *same* struct. This breaks type 4724 * equivalence check, which makes an assumption that candidate 4725 * types sub-graph has a consistent and deduped-by-compiler 4726 * types within a single CU. And similar situation can happen 4727 * with struct/union sometimes, and event with pointers. 4728 * So accommodate cases like this doing a structural 4729 * comparison recursively, but avoiding being stuck in endless 4730 * loops by limiting the depth up to which we check. 4731 */ 4732 if (btf_dedup_identical_types(d, hypot_type_id, cand_id, 16)) 4733 return 1; 4734 return 0; 4735 } 4736 4737 if (btf_dedup_hypot_map_add(d, canon_id, cand_id)) 4738 return -ENOMEM; 4739 4740 cand_type = btf_type_by_id(d->btf, cand_id); 4741 canon_type = btf_type_by_id(d->btf, canon_id); 4742 cand_kind = btf_kind(cand_type); 4743 canon_kind = btf_kind(canon_type); 4744 4745 if (cand_type->name_off != canon_type->name_off) 4746 return 0; 4747 4748 /* FWD <--> STRUCT/UNION equivalence check, if enabled */ 4749 if ((cand_kind == BTF_KIND_FWD || canon_kind == BTF_KIND_FWD) 4750 && cand_kind != canon_kind) { 4751 __u16 real_kind; 4752 __u16 fwd_kind; 4753 4754 if (cand_kind == BTF_KIND_FWD) { 4755 real_kind = canon_kind; 4756 fwd_kind = btf_fwd_kind(cand_type); 4757 } else { 4758 real_kind = cand_kind; 4759 fwd_kind = btf_fwd_kind(canon_type); 4760 /* we'd need to resolve base FWD to STRUCT/UNION */ 4761 if (fwd_kind == real_kind && canon_id < d->btf->start_id) 4762 d->hypot_adjust_canon = true; 4763 } 4764 return fwd_kind == real_kind; 4765 } 4766 4767 if (cand_kind != canon_kind) 4768 return 0; 4769 4770 switch (cand_kind) { 4771 case BTF_KIND_INT: 4772 return btf_equal_int_tag(cand_type, canon_type); 4773 4774 case BTF_KIND_ENUM: 4775 case BTF_KIND_ENUM64: 4776 return btf_compat_enum(cand_type, canon_type); 4777 4778 case BTF_KIND_FWD: 4779 case BTF_KIND_FLOAT: 4780 return btf_equal_common(cand_type, canon_type); 4781 4782 case BTF_KIND_CONST: 4783 case BTF_KIND_VOLATILE: 4784 case BTF_KIND_RESTRICT: 4785 case BTF_KIND_PTR: 4786 case BTF_KIND_TYPEDEF: 4787 case BTF_KIND_FUNC: 4788 case BTF_KIND_TYPE_TAG: 4789 if (cand_type->info != canon_type->info) 4790 return 0; 4791 return btf_dedup_is_equiv(d, cand_type->type, canon_type->type); 4792 4793 case BTF_KIND_ARRAY: { 4794 const struct btf_array *cand_arr, *canon_arr; 4795 4796 if (!btf_compat_array(cand_type, canon_type)) 4797 return 0; 4798 cand_arr = btf_array(cand_type); 4799 canon_arr = btf_array(canon_type); 4800 eq = btf_dedup_is_equiv(d, cand_arr->index_type, canon_arr->index_type); 4801 if (eq <= 0) 4802 return eq; 4803 return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type); 4804 } 4805 4806 case BTF_KIND_STRUCT: 4807 case BTF_KIND_UNION: { 4808 const struct btf_member *cand_m, *canon_m; 4809 __u16 vlen; 4810 4811 if (!btf_shallow_equal_struct(cand_type, canon_type)) 4812 return 0; 4813 vlen = btf_vlen(cand_type); 4814 cand_m = btf_members(cand_type); 4815 canon_m = btf_members(canon_type); 4816 for (i = 0; i < vlen; i++) { 4817 eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type); 4818 if (eq <= 0) { 4819 if (cand_type->name_off) { 4820 pr_debug("%s '%s' size=%d vlen=%d cand_id[%u] canon_id[%u] shallow-equal but not equiv for field#%d '%s': %d\n", 4821 cand_kind == BTF_KIND_STRUCT ? "STRUCT" : "UNION", 4822 btf__name_by_offset(d->btf, cand_type->name_off), 4823 cand_type->size, vlen, cand_id, canon_id, i, 4824 btf__name_by_offset(d->btf, cand_m->name_off), eq); 4825 } 4826 return eq; 4827 } 4828 cand_m++; 4829 canon_m++; 4830 } 4831 4832 return 1; 4833 } 4834 4835 case BTF_KIND_FUNC_PROTO: { 4836 const struct btf_param *cand_p, *canon_p; 4837 __u16 vlen; 4838 4839 if (!btf_compat_fnproto(cand_type, canon_type)) 4840 return 0; 4841 eq = btf_dedup_is_equiv(d, cand_type->type, canon_type->type); 4842 if (eq <= 0) 4843 return eq; 4844 vlen = btf_vlen(cand_type); 4845 cand_p = btf_params(cand_type); 4846 canon_p = btf_params(canon_type); 4847 for (i = 0; i < vlen; i++) { 4848 eq = btf_dedup_is_equiv(d, cand_p->type, canon_p->type); 4849 if (eq <= 0) 4850 return eq; 4851 cand_p++; 4852 canon_p++; 4853 } 4854 return 1; 4855 } 4856 4857 default: 4858 return -EINVAL; 4859 } 4860 return 0; 4861 } 4862 4863 /* 4864 * Use hypothetical mapping, produced by successful type graph equivalence 4865 * check, to augment existing struct/union canonical mapping, where possible. 4866 * 4867 * If BTF_KIND_FWD resolution is allowed, this mapping is also used to record 4868 * FWD -> STRUCT/UNION correspondence as well. FWD resolution is bidirectional: 4869 * it doesn't matter if FWD type was part of canonical graph or candidate one, 4870 * we are recording the mapping anyway. As opposed to carefulness required 4871 * for struct/union correspondence mapping (described below), for FWD resolution 4872 * it's not important, as by the time that FWD type (reference type) will be 4873 * deduplicated all structs/unions will be deduped already anyway. 4874 * 4875 * Recording STRUCT/UNION mapping is purely a performance optimization and is 4876 * not required for correctness. It needs to be done carefully to ensure that 4877 * struct/union from candidate's type graph is not mapped into corresponding 4878 * struct/union from canonical type graph that itself hasn't been resolved into 4879 * canonical representative. The only guarantee we have is that canonical 4880 * struct/union was determined as canonical and that won't change. But any 4881 * types referenced through that struct/union fields could have been not yet 4882 * resolved, so in case like that it's too early to establish any kind of 4883 * correspondence between structs/unions. 4884 * 4885 * No canonical correspondence is derived for primitive types (they are already 4886 * deduplicated completely already anyway) or reference types (they rely on 4887 * stability of struct/union canonical relationship for equivalence checks). 4888 */ 4889 static void btf_dedup_merge_hypot_map(struct btf_dedup *d) 4890 { 4891 __u32 canon_type_id, targ_type_id; 4892 __u16 t_kind, c_kind; 4893 __u32 t_id, c_id; 4894 int i; 4895 4896 for (i = 0; i < d->hypot_cnt; i++) { 4897 canon_type_id = d->hypot_list[i]; 4898 targ_type_id = d->hypot_map[canon_type_id]; 4899 t_id = resolve_type_id(d, targ_type_id); 4900 c_id = resolve_type_id(d, canon_type_id); 4901 t_kind = btf_kind(btf__type_by_id(d->btf, t_id)); 4902 c_kind = btf_kind(btf__type_by_id(d->btf, c_id)); 4903 /* 4904 * Resolve FWD into STRUCT/UNION. 4905 * It's ok to resolve FWD into STRUCT/UNION that's not yet 4906 * mapped to canonical representative (as opposed to 4907 * STRUCT/UNION <--> STRUCT/UNION mapping logic below), because 4908 * eventually that struct is going to be mapped and all resolved 4909 * FWDs will automatically resolve to correct canonical 4910 * representative. This will happen before ref type deduping, 4911 * which critically depends on stability of these mapping. This 4912 * stability is not a requirement for STRUCT/UNION equivalence 4913 * checks, though. 4914 */ 4915 4916 /* if it's the split BTF case, we still need to point base FWD 4917 * to STRUCT/UNION in a split BTF, because FWDs from split BTF 4918 * will be resolved against base FWD. If we don't point base 4919 * canonical FWD to the resolved STRUCT/UNION, then all the 4920 * FWDs in split BTF won't be correctly resolved to a proper 4921 * STRUCT/UNION. 4922 */ 4923 if (t_kind != BTF_KIND_FWD && c_kind == BTF_KIND_FWD) 4924 d->map[c_id] = t_id; 4925 4926 /* if graph equivalence determined that we'd need to adjust 4927 * base canonical types, then we need to only point base FWDs 4928 * to STRUCTs/UNIONs and do no more modifications. For all 4929 * other purposes the type graphs were not equivalent. 4930 */ 4931 if (d->hypot_adjust_canon) 4932 continue; 4933 4934 if (t_kind == BTF_KIND_FWD && c_kind != BTF_KIND_FWD) 4935 d->map[t_id] = c_id; 4936 4937 if ((t_kind == BTF_KIND_STRUCT || t_kind == BTF_KIND_UNION) && 4938 c_kind != BTF_KIND_FWD && 4939 is_type_mapped(d, c_id) && 4940 !is_type_mapped(d, t_id)) { 4941 /* 4942 * as a perf optimization, we can map struct/union 4943 * that's part of type graph we just verified for 4944 * equivalence. We can do that for struct/union that has 4945 * canonical representative only, though. 4946 */ 4947 d->map[t_id] = c_id; 4948 } 4949 } 4950 } 4951 4952 static inline long btf_hash_by_kind(struct btf_type *t, __u16 kind) 4953 { 4954 if (kind == BTF_KIND_TYPEDEF) 4955 return btf_hash_typedef(t); 4956 else 4957 return btf_hash_struct(t); 4958 } 4959 4960 static inline bool btf_equal_by_kind(struct btf_type *t1, struct btf_type *t2, __u16 kind) 4961 { 4962 if (kind == BTF_KIND_TYPEDEF) 4963 return btf_equal_typedef(t1, t2); 4964 else 4965 return btf_shallow_equal_struct(t1, t2); 4966 } 4967 4968 /* 4969 * Deduplicate struct/union and typedef types. 4970 * 4971 * For each struct/union type its type signature hash is calculated, taking 4972 * into account type's name, size, number, order and names of fields, but 4973 * ignoring type ID's referenced from fields, because they might not be deduped 4974 * completely until after reference types deduplication phase. For each typedef 4975 * type, the hash is computed based on the type’s name and size. This type hash 4976 * is used to iterate over all potential canonical types, sharing same hash. 4977 * For each canonical candidate we check whether type graphs that they form 4978 * (through referenced types in fields and so on) are equivalent using algorithm 4979 * implemented in `btf_dedup_is_equiv`. If such equivalence is found and 4980 * BTF_KIND_FWD resolution is allowed, then hypothetical mapping 4981 * (btf_dedup->hypot_map) produced by aforementioned type graph equivalence 4982 * algorithm is used to record FWD -> STRUCT/UNION mapping. It's also used to 4983 * potentially map other structs/unions to their canonical representatives, 4984 * if such relationship hasn't yet been established. This speeds up algorithm 4985 * by eliminating some of the duplicate work. 4986 * 4987 * If no matching canonical representative was found, struct/union is marked 4988 * as canonical for itself and is added into btf_dedup->dedup_table hash map 4989 * for further look ups. 4990 */ 4991 static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id) 4992 { 4993 struct btf_type *cand_type, *t; 4994 struct hashmap_entry *hash_entry; 4995 /* if we don't find equivalent type, then we are canonical */ 4996 __u32 new_id = type_id; 4997 __u16 kind; 4998 long h; 4999 5000 /* already deduped or is in process of deduping (loop detected) */ 5001 if (d->map[type_id] <= BTF_MAX_NR_TYPES) 5002 return 0; 5003 5004 t = btf_type_by_id(d->btf, type_id); 5005 kind = btf_kind(t); 5006 5007 if (kind != BTF_KIND_STRUCT && 5008 kind != BTF_KIND_UNION && 5009 kind != BTF_KIND_TYPEDEF) 5010 return 0; 5011 5012 h = btf_hash_by_kind(t, kind); 5013 for_each_dedup_cand(d, hash_entry, h) { 5014 __u32 cand_id = hash_entry->value; 5015 int eq; 5016 5017 /* 5018 * Even though btf_dedup_is_equiv() checks for 5019 * btf_equal_by_kind() internally when checking two 5020 * structs (unions) or typedefs for equivalence, we need to guard here 5021 * from picking matching FWD type as a dedup candidate. 5022 * This can happen due to hash collision. In such case just 5023 * relying on btf_dedup_is_equiv() would lead to potentially 5024 * creating a loop (FWD -> STRUCT and STRUCT -> FWD), because 5025 * FWD and compatible STRUCT/UNION are considered equivalent. 5026 */ 5027 cand_type = btf_type_by_id(d->btf, cand_id); 5028 if (!btf_equal_by_kind(t, cand_type, kind)) 5029 continue; 5030 5031 btf_dedup_clear_hypot_map(d); 5032 eq = btf_dedup_is_equiv(d, type_id, cand_id); 5033 if (eq < 0) 5034 return eq; 5035 if (!eq) 5036 continue; 5037 btf_dedup_merge_hypot_map(d); 5038 if (d->hypot_adjust_canon) /* not really equivalent */ 5039 continue; 5040 new_id = cand_id; 5041 break; 5042 } 5043 5044 d->map[type_id] = new_id; 5045 if (type_id == new_id && btf_dedup_table_add(d, h, type_id)) 5046 return -ENOMEM; 5047 5048 return 0; 5049 } 5050 5051 static int btf_dedup_struct_types(struct btf_dedup *d) 5052 { 5053 int i, err; 5054 5055 for (i = 0; i < d->btf->nr_types; i++) { 5056 err = btf_dedup_struct_type(d, d->btf->start_id + i); 5057 if (err) 5058 return err; 5059 } 5060 return 0; 5061 } 5062 5063 /* 5064 * Deduplicate reference type. 5065 * 5066 * Once all primitive, struct/union and typedef types got deduplicated, we can easily 5067 * deduplicate all other (reference) BTF types. This is done in two steps: 5068 * 5069 * 1. Resolve all referenced type IDs into their canonical type IDs. This 5070 * resolution can be done either immediately for primitive, struct/union, and typedef 5071 * types (because they were deduped in previous two phases) or recursively for 5072 * reference types. Recursion will always terminate at either primitive or 5073 * struct/union and typedef types, at which point we can "unwind" chain of reference 5074 * types one by one. There is no danger of encountering cycles in C, as the only way to 5075 * form a type cycle is through struct or union types. Go can form such cycles through 5076 * typedef. Thus, any chain of reference types, even those taking part in a type cycle, 5077 * will inevitably reach a struct/union or typedef type at some point. 5078 * 5079 * 2. Once all referenced type IDs are resolved into canonical ones, BTF type 5080 * becomes "stable", in the sense that no further deduplication will cause 5081 * any changes to it. With that, it's now possible to calculate type's signature 5082 * hash (this time taking into account referenced type IDs) and loop over all 5083 * potential canonical representatives. If no match was found, current type 5084 * will become canonical representative of itself and will be added into 5085 * btf_dedup->dedup_table as another possible canonical representative. 5086 */ 5087 static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) 5088 { 5089 struct hashmap_entry *hash_entry; 5090 __u32 new_id = type_id, cand_id; 5091 struct btf_type *t, *cand; 5092 /* if we don't find equivalent type, then we are representative type */ 5093 int ref_type_id; 5094 long h; 5095 5096 if (d->map[type_id] == BTF_IN_PROGRESS_ID) 5097 return -ELOOP; 5098 if (d->map[type_id] <= BTF_MAX_NR_TYPES) 5099 return resolve_type_id(d, type_id); 5100 5101 t = btf_type_by_id(d->btf, type_id); 5102 d->map[type_id] = BTF_IN_PROGRESS_ID; 5103 5104 switch (btf_kind(t)) { 5105 case BTF_KIND_CONST: 5106 case BTF_KIND_VOLATILE: 5107 case BTF_KIND_RESTRICT: 5108 case BTF_KIND_PTR: 5109 case BTF_KIND_FUNC: 5110 case BTF_KIND_TYPE_TAG: 5111 ref_type_id = btf_dedup_ref_type(d, t->type); 5112 if (ref_type_id < 0) 5113 return ref_type_id; 5114 t->type = ref_type_id; 5115 5116 h = btf_hash_common(t); 5117 for_each_dedup_cand(d, hash_entry, h) { 5118 cand_id = hash_entry->value; 5119 cand = btf_type_by_id(d->btf, cand_id); 5120 if (btf_equal_common(t, cand)) { 5121 new_id = cand_id; 5122 break; 5123 } 5124 } 5125 break; 5126 5127 case BTF_KIND_DECL_TAG: 5128 ref_type_id = btf_dedup_ref_type(d, t->type); 5129 if (ref_type_id < 0) 5130 return ref_type_id; 5131 t->type = ref_type_id; 5132 5133 h = btf_hash_int_decl_tag(t); 5134 for_each_dedup_cand(d, hash_entry, h) { 5135 cand_id = hash_entry->value; 5136 cand = btf_type_by_id(d->btf, cand_id); 5137 if (btf_equal_int_tag(t, cand)) { 5138 new_id = cand_id; 5139 break; 5140 } 5141 } 5142 break; 5143 5144 case BTF_KIND_ARRAY: { 5145 struct btf_array *info = btf_array(t); 5146 5147 ref_type_id = btf_dedup_ref_type(d, info->type); 5148 if (ref_type_id < 0) 5149 return ref_type_id; 5150 info->type = ref_type_id; 5151 5152 ref_type_id = btf_dedup_ref_type(d, info->index_type); 5153 if (ref_type_id < 0) 5154 return ref_type_id; 5155 info->index_type = ref_type_id; 5156 5157 h = btf_hash_array(t); 5158 for_each_dedup_cand(d, hash_entry, h) { 5159 cand_id = hash_entry->value; 5160 cand = btf_type_by_id(d->btf, cand_id); 5161 if (btf_equal_array(t, cand)) { 5162 new_id = cand_id; 5163 break; 5164 } 5165 } 5166 break; 5167 } 5168 5169 case BTF_KIND_FUNC_PROTO: { 5170 struct btf_param *param; 5171 __u16 vlen; 5172 int i; 5173 5174 ref_type_id = btf_dedup_ref_type(d, t->type); 5175 if (ref_type_id < 0) 5176 return ref_type_id; 5177 t->type = ref_type_id; 5178 5179 vlen = btf_vlen(t); 5180 param = btf_params(t); 5181 for (i = 0; i < vlen; i++) { 5182 ref_type_id = btf_dedup_ref_type(d, param->type); 5183 if (ref_type_id < 0) 5184 return ref_type_id; 5185 param->type = ref_type_id; 5186 param++; 5187 } 5188 5189 h = btf_hash_fnproto(t); 5190 for_each_dedup_cand(d, hash_entry, h) { 5191 cand_id = hash_entry->value; 5192 cand = btf_type_by_id(d->btf, cand_id); 5193 if (btf_equal_fnproto(t, cand)) { 5194 new_id = cand_id; 5195 break; 5196 } 5197 } 5198 break; 5199 } 5200 5201 default: 5202 return -EINVAL; 5203 } 5204 5205 d->map[type_id] = new_id; 5206 if (type_id == new_id && btf_dedup_table_add(d, h, type_id)) 5207 return -ENOMEM; 5208 5209 return new_id; 5210 } 5211 5212 static int btf_dedup_ref_types(struct btf_dedup *d) 5213 { 5214 int i, err; 5215 5216 for (i = 0; i < d->btf->nr_types; i++) { 5217 err = btf_dedup_ref_type(d, d->btf->start_id + i); 5218 if (err < 0) 5219 return err; 5220 } 5221 /* we won't need d->dedup_table anymore */ 5222 hashmap__free(d->dedup_table); 5223 d->dedup_table = NULL; 5224 return 0; 5225 } 5226 5227 /* 5228 * Collect a map from type names to type ids for all canonical structs 5229 * and unions. If the same name is shared by several canonical types 5230 * use a special value 0 to indicate this fact. 5231 */ 5232 static int btf_dedup_fill_unique_names_map(struct btf_dedup *d, struct hashmap *names_map) 5233 { 5234 __u32 nr_types = btf__type_cnt(d->btf); 5235 struct btf_type *t; 5236 __u32 type_id; 5237 __u16 kind; 5238 int err; 5239 5240 /* 5241 * Iterate over base and split module ids in order to get all 5242 * available structs in the map. 5243 */ 5244 for (type_id = 1; type_id < nr_types; ++type_id) { 5245 t = btf_type_by_id(d->btf, type_id); 5246 kind = btf_kind(t); 5247 5248 if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION) 5249 continue; 5250 5251 /* Skip non-canonical types */ 5252 if (type_id != d->map[type_id]) 5253 continue; 5254 5255 err = hashmap__add(names_map, t->name_off, type_id); 5256 if (err == -EEXIST) 5257 err = hashmap__set(names_map, t->name_off, 0, NULL, NULL); 5258 5259 if (err) 5260 return err; 5261 } 5262 5263 return 0; 5264 } 5265 5266 static int btf_dedup_resolve_fwd(struct btf_dedup *d, struct hashmap *names_map, __u32 type_id) 5267 { 5268 struct btf_type *t = btf_type_by_id(d->btf, type_id); 5269 enum btf_fwd_kind fwd_kind = btf_kflag(t); 5270 __u16 cand_kind, kind = btf_kind(t); 5271 struct btf_type *cand_t; 5272 uintptr_t cand_id; 5273 5274 if (kind != BTF_KIND_FWD) 5275 return 0; 5276 5277 /* Skip if this FWD already has a mapping */ 5278 if (type_id != d->map[type_id]) 5279 return 0; 5280 5281 if (!hashmap__find(names_map, t->name_off, &cand_id)) 5282 return 0; 5283 5284 /* Zero is a special value indicating that name is not unique */ 5285 if (!cand_id) 5286 return 0; 5287 5288 cand_t = btf_type_by_id(d->btf, cand_id); 5289 cand_kind = btf_kind(cand_t); 5290 if ((cand_kind == BTF_KIND_STRUCT && fwd_kind != BTF_FWD_STRUCT) || 5291 (cand_kind == BTF_KIND_UNION && fwd_kind != BTF_FWD_UNION)) 5292 return 0; 5293 5294 d->map[type_id] = cand_id; 5295 5296 return 0; 5297 } 5298 5299 /* 5300 * Resolve unambiguous forward declarations. 5301 * 5302 * The lion's share of all FWD declarations is resolved during 5303 * `btf_dedup_struct_types` phase when different type graphs are 5304 * compared against each other. However, if in some compilation unit a 5305 * FWD declaration is not a part of a type graph compared against 5306 * another type graph that declaration's canonical type would not be 5307 * changed. Example: 5308 * 5309 * CU #1: 5310 * 5311 * struct foo; 5312 * struct foo *some_global; 5313 * 5314 * CU #2: 5315 * 5316 * struct foo { int u; }; 5317 * struct foo *another_global; 5318 * 5319 * After `btf_dedup_struct_types` the BTF looks as follows: 5320 * 5321 * [1] STRUCT 'foo' size=4 vlen=1 ... 5322 * [2] INT 'int' size=4 ... 5323 * [3] PTR '(anon)' type_id=1 5324 * [4] FWD 'foo' fwd_kind=struct 5325 * [5] PTR '(anon)' type_id=4 5326 * 5327 * This pass assumes that such FWD declarations should be mapped to 5328 * structs or unions with identical name in case if the name is not 5329 * ambiguous. 5330 */ 5331 static int btf_dedup_resolve_fwds(struct btf_dedup *d) 5332 { 5333 int i, err; 5334 struct hashmap *names_map; 5335 5336 names_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL); 5337 if (IS_ERR(names_map)) 5338 return PTR_ERR(names_map); 5339 5340 err = btf_dedup_fill_unique_names_map(d, names_map); 5341 if (err < 0) 5342 goto exit; 5343 5344 for (i = 0; i < d->btf->nr_types; i++) { 5345 err = btf_dedup_resolve_fwd(d, names_map, d->btf->start_id + i); 5346 if (err < 0) 5347 break; 5348 } 5349 5350 exit: 5351 hashmap__free(names_map); 5352 return err; 5353 } 5354 5355 /* 5356 * Compact types. 5357 * 5358 * After we established for each type its corresponding canonical representative 5359 * type, we now can eliminate types that are not canonical and leave only 5360 * canonical ones layed out sequentially in memory by copying them over 5361 * duplicates. During compaction btf_dedup->hypot_map array is reused to store 5362 * a map from original type ID to a new compacted type ID, which will be used 5363 * during next phase to "fix up" type IDs, referenced from struct/union and 5364 * reference types. 5365 */ 5366 static int btf_dedup_compact_types(struct btf_dedup *d) 5367 { 5368 __u32 *new_offs; 5369 __u32 next_type_id = d->btf->start_id; 5370 const struct btf_type *t; 5371 void *p; 5372 int i, id, len; 5373 5374 /* we are going to reuse hypot_map to store compaction remapping */ 5375 d->hypot_map[0] = 0; 5376 /* base BTF types are not renumbered */ 5377 for (id = 1; id < d->btf->start_id; id++) 5378 d->hypot_map[id] = id; 5379 for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) 5380 d->hypot_map[id] = BTF_UNPROCESSED_ID; 5381 5382 p = d->btf->types_data; 5383 5384 for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) { 5385 if (d->map[id] != id) 5386 continue; 5387 5388 t = btf__type_by_id(d->btf, id); 5389 len = btf_type_size(t); 5390 if (len < 0) 5391 return len; 5392 5393 memmove(p, t, len); 5394 d->hypot_map[id] = next_type_id; 5395 d->btf->type_offs[next_type_id - d->btf->start_id] = p - d->btf->types_data; 5396 p += len; 5397 next_type_id++; 5398 } 5399 5400 /* shrink struct btf's internal types index and update btf_header */ 5401 d->btf->nr_types = next_type_id - d->btf->start_id; 5402 d->btf->type_offs_cap = d->btf->nr_types; 5403 d->btf->hdr->type_len = p - d->btf->types_data; 5404 new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap, 5405 sizeof(*new_offs)); 5406 if (d->btf->type_offs_cap && !new_offs) 5407 return -ENOMEM; 5408 d->btf->type_offs = new_offs; 5409 d->btf->hdr->str_off = d->btf->hdr->type_len; 5410 d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len; 5411 return 0; 5412 } 5413 5414 /* 5415 * Figure out final (deduplicated and compacted) type ID for provided original 5416 * `type_id` by first resolving it into corresponding canonical type ID and 5417 * then mapping it to a deduplicated type ID, stored in btf_dedup->hypot_map, 5418 * which is populated during compaction phase. 5419 */ 5420 static int btf_dedup_remap_type_id(__u32 *type_id, void *ctx) 5421 { 5422 struct btf_dedup *d = ctx; 5423 __u32 resolved_type_id, new_type_id; 5424 5425 resolved_type_id = resolve_type_id(d, *type_id); 5426 new_type_id = d->hypot_map[resolved_type_id]; 5427 if (new_type_id > BTF_MAX_NR_TYPES) 5428 return -EINVAL; 5429 5430 *type_id = new_type_id; 5431 return 0; 5432 } 5433 5434 /* 5435 * Remap referenced type IDs into deduped type IDs. 5436 * 5437 * After BTF types are deduplicated and compacted, their final type IDs may 5438 * differ from original ones. The map from original to a corresponding 5439 * deduped type ID is stored in btf_dedup->hypot_map and is populated during 5440 * compaction phase. During remapping phase we are rewriting all type IDs 5441 * referenced from any BTF type (e.g., struct fields, func proto args, etc) to 5442 * their final deduped type IDs. 5443 */ 5444 static int btf_dedup_remap_types(struct btf_dedup *d) 5445 { 5446 int i, r; 5447 5448 for (i = 0; i < d->btf->nr_types; i++) { 5449 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i); 5450 struct btf_field_iter it; 5451 __u32 *type_id; 5452 5453 r = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS); 5454 if (r) 5455 return r; 5456 5457 while ((type_id = btf_field_iter_next(&it))) { 5458 __u32 resolved_id, new_id; 5459 5460 resolved_id = resolve_type_id(d, *type_id); 5461 new_id = d->hypot_map[resolved_id]; 5462 if (new_id > BTF_MAX_NR_TYPES) 5463 return -EINVAL; 5464 5465 *type_id = new_id; 5466 } 5467 } 5468 5469 if (!d->btf_ext) 5470 return 0; 5471 5472 r = btf_ext_visit_type_ids(d->btf_ext, btf_dedup_remap_type_id, d); 5473 if (r) 5474 return r; 5475 5476 return 0; 5477 } 5478 5479 /* 5480 * Probe few well-known locations for vmlinux kernel image and try to load BTF 5481 * data out of it to use for target BTF. 5482 */ 5483 struct btf *btf__load_vmlinux_btf(void) 5484 { 5485 const char *sysfs_btf_path = "/sys/kernel/btf/vmlinux"; 5486 /* fall back locations, trying to find vmlinux on disk */ 5487 const char *locations[] = { 5488 "/boot/vmlinux-%1$s", 5489 "/lib/modules/%1$s/vmlinux-%1$s", 5490 "/lib/modules/%1$s/build/vmlinux", 5491 "/usr/lib/modules/%1$s/kernel/vmlinux", 5492 "/usr/lib/debug/boot/vmlinux-%1$s", 5493 "/usr/lib/debug/boot/vmlinux-%1$s.debug", 5494 "/usr/lib/debug/lib/modules/%1$s/vmlinux", 5495 }; 5496 char path[PATH_MAX + 1]; 5497 struct utsname buf; 5498 struct btf *btf; 5499 int i, err; 5500 5501 /* is canonical sysfs location accessible? */ 5502 if (faccessat(AT_FDCWD, sysfs_btf_path, F_OK, AT_EACCESS) < 0) { 5503 pr_warn("kernel BTF is missing at '%s', was CONFIG_DEBUG_INFO_BTF enabled?\n", 5504 sysfs_btf_path); 5505 } else { 5506 btf = btf_parse_raw_mmap(sysfs_btf_path, NULL); 5507 if (IS_ERR(btf)) 5508 btf = btf__parse(sysfs_btf_path, NULL); 5509 5510 if (!btf) { 5511 err = -errno; 5512 pr_warn("failed to read kernel BTF from '%s': %s\n", 5513 sysfs_btf_path, errstr(err)); 5514 return libbpf_err_ptr(err); 5515 } 5516 pr_debug("loaded kernel BTF from '%s'\n", sysfs_btf_path); 5517 return btf; 5518 } 5519 5520 /* try fallback locations */ 5521 uname(&buf); 5522 for (i = 0; i < ARRAY_SIZE(locations); i++) { 5523 snprintf(path, PATH_MAX, locations[i], buf.release); 5524 5525 if (faccessat(AT_FDCWD, path, R_OK, AT_EACCESS)) 5526 continue; 5527 5528 btf = btf__parse(path, NULL); 5529 err = libbpf_get_error(btf); 5530 pr_debug("loading kernel BTF '%s': %s\n", path, errstr(err)); 5531 if (err) 5532 continue; 5533 5534 return btf; 5535 } 5536 5537 pr_warn("failed to find valid kernel BTF\n"); 5538 return libbpf_err_ptr(-ESRCH); 5539 } 5540 5541 struct btf *libbpf_find_kernel_btf(void) __attribute__((alias("btf__load_vmlinux_btf"))); 5542 5543 struct btf *btf__load_module_btf(const char *module_name, struct btf *vmlinux_btf) 5544 { 5545 char path[80]; 5546 5547 snprintf(path, sizeof(path), "/sys/kernel/btf/%s", module_name); 5548 return btf__parse_split(path, vmlinux_btf); 5549 } 5550 5551 int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx) 5552 { 5553 const struct btf_ext_info *seg; 5554 struct btf_ext_info_sec *sec; 5555 int i, err; 5556 5557 seg = &btf_ext->func_info; 5558 for_each_btf_ext_sec(seg, sec) { 5559 struct bpf_func_info_min *rec; 5560 5561 for_each_btf_ext_rec(seg, sec, i, rec) { 5562 err = visit(&rec->type_id, ctx); 5563 if (err < 0) 5564 return err; 5565 } 5566 } 5567 5568 seg = &btf_ext->core_relo_info; 5569 for_each_btf_ext_sec(seg, sec) { 5570 struct bpf_core_relo *rec; 5571 5572 for_each_btf_ext_rec(seg, sec, i, rec) { 5573 err = visit(&rec->type_id, ctx); 5574 if (err < 0) 5575 return err; 5576 } 5577 } 5578 5579 return 0; 5580 } 5581 5582 int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx) 5583 { 5584 const struct btf_ext_info *seg; 5585 struct btf_ext_info_sec *sec; 5586 int i, err; 5587 5588 seg = &btf_ext->func_info; 5589 for_each_btf_ext_sec(seg, sec) { 5590 err = visit(&sec->sec_name_off, ctx); 5591 if (err) 5592 return err; 5593 } 5594 5595 seg = &btf_ext->line_info; 5596 for_each_btf_ext_sec(seg, sec) { 5597 struct bpf_line_info_min *rec; 5598 5599 err = visit(&sec->sec_name_off, ctx); 5600 if (err) 5601 return err; 5602 5603 for_each_btf_ext_rec(seg, sec, i, rec) { 5604 err = visit(&rec->file_name_off, ctx); 5605 if (err) 5606 return err; 5607 err = visit(&rec->line_off, ctx); 5608 if (err) 5609 return err; 5610 } 5611 } 5612 5613 seg = &btf_ext->core_relo_info; 5614 for_each_btf_ext_sec(seg, sec) { 5615 struct bpf_core_relo *rec; 5616 5617 err = visit(&sec->sec_name_off, ctx); 5618 if (err) 5619 return err; 5620 5621 for_each_btf_ext_rec(seg, sec, i, rec) { 5622 err = visit(&rec->access_str_off, ctx); 5623 if (err) 5624 return err; 5625 } 5626 } 5627 5628 return 0; 5629 } 5630 5631 struct btf_distill { 5632 struct btf_pipe pipe; 5633 int *id_map; 5634 unsigned int split_start_id; 5635 unsigned int split_start_str; 5636 int diff_id; 5637 }; 5638 5639 static int btf_add_distilled_type_ids(struct btf_distill *dist, __u32 i) 5640 { 5641 struct btf_type *split_t = btf_type_by_id(dist->pipe.src, i); 5642 struct btf_field_iter it; 5643 __u32 *id; 5644 int err; 5645 5646 err = btf_field_iter_init(&it, split_t, BTF_FIELD_ITER_IDS); 5647 if (err) 5648 return err; 5649 while ((id = btf_field_iter_next(&it))) { 5650 struct btf_type *base_t; 5651 5652 if (!*id) 5653 continue; 5654 /* split BTF id, not needed */ 5655 if (*id >= dist->split_start_id) 5656 continue; 5657 /* already added ? */ 5658 if (dist->id_map[*id] > 0) 5659 continue; 5660 5661 /* only a subset of base BTF types should be referenced from 5662 * split BTF; ensure nothing unexpected is referenced. 5663 */ 5664 base_t = btf_type_by_id(dist->pipe.src, *id); 5665 switch (btf_kind(base_t)) { 5666 case BTF_KIND_INT: 5667 case BTF_KIND_FLOAT: 5668 case BTF_KIND_FWD: 5669 case BTF_KIND_ARRAY: 5670 case BTF_KIND_STRUCT: 5671 case BTF_KIND_UNION: 5672 case BTF_KIND_TYPEDEF: 5673 case BTF_KIND_ENUM: 5674 case BTF_KIND_ENUM64: 5675 case BTF_KIND_PTR: 5676 case BTF_KIND_CONST: 5677 case BTF_KIND_RESTRICT: 5678 case BTF_KIND_VOLATILE: 5679 case BTF_KIND_FUNC_PROTO: 5680 case BTF_KIND_TYPE_TAG: 5681 dist->id_map[*id] = *id; 5682 break; 5683 default: 5684 pr_warn("unexpected reference to base type[%u] of kind [%u] when creating distilled base BTF.\n", 5685 *id, btf_kind(base_t)); 5686 return -EINVAL; 5687 } 5688 /* If a base type is used, ensure types it refers to are 5689 * marked as used also; so for example if we find a PTR to INT 5690 * we need both the PTR and INT. 5691 * 5692 * The only exception is named struct/unions, since distilled 5693 * base BTF composite types have no members. 5694 */ 5695 if (btf_is_composite(base_t) && base_t->name_off) 5696 continue; 5697 err = btf_add_distilled_type_ids(dist, *id); 5698 if (err) 5699 return err; 5700 } 5701 return 0; 5702 } 5703 5704 static int btf_add_distilled_types(struct btf_distill *dist) 5705 { 5706 bool adding_to_base = dist->pipe.dst->start_id == 1; 5707 int id = btf__type_cnt(dist->pipe.dst); 5708 struct btf_type *t; 5709 int i, err = 0; 5710 5711 5712 /* Add types for each of the required references to either distilled 5713 * base or split BTF, depending on type characteristics. 5714 */ 5715 for (i = 1; i < dist->split_start_id; i++) { 5716 const char *name; 5717 int kind; 5718 5719 if (!dist->id_map[i]) 5720 continue; 5721 t = btf_type_by_id(dist->pipe.src, i); 5722 kind = btf_kind(t); 5723 name = btf__name_by_offset(dist->pipe.src, t->name_off); 5724 5725 switch (kind) { 5726 case BTF_KIND_INT: 5727 case BTF_KIND_FLOAT: 5728 case BTF_KIND_FWD: 5729 /* Named int, float, fwd are added to base. */ 5730 if (!adding_to_base) 5731 continue; 5732 err = btf_add_type(&dist->pipe, t); 5733 break; 5734 case BTF_KIND_STRUCT: 5735 case BTF_KIND_UNION: 5736 /* Named struct/union are added to base as 0-vlen 5737 * struct/union of same size. Anonymous struct/unions 5738 * are added to split BTF as-is. 5739 */ 5740 if (adding_to_base) { 5741 if (!t->name_off) 5742 continue; 5743 err = btf_add_composite(dist->pipe.dst, kind, name, t->size); 5744 } else { 5745 if (t->name_off) 5746 continue; 5747 err = btf_add_type(&dist->pipe, t); 5748 } 5749 break; 5750 case BTF_KIND_ENUM: 5751 case BTF_KIND_ENUM64: 5752 /* Named enum[64]s are added to base as a sized 5753 * enum; relocation will match with appropriately-named 5754 * and sized enum or enum64. 5755 * 5756 * Anonymous enums are added to split BTF as-is. 5757 */ 5758 if (adding_to_base) { 5759 if (!t->name_off) 5760 continue; 5761 err = btf__add_enum(dist->pipe.dst, name, t->size); 5762 } else { 5763 if (t->name_off) 5764 continue; 5765 err = btf_add_type(&dist->pipe, t); 5766 } 5767 break; 5768 case BTF_KIND_ARRAY: 5769 case BTF_KIND_TYPEDEF: 5770 case BTF_KIND_PTR: 5771 case BTF_KIND_CONST: 5772 case BTF_KIND_RESTRICT: 5773 case BTF_KIND_VOLATILE: 5774 case BTF_KIND_FUNC_PROTO: 5775 case BTF_KIND_TYPE_TAG: 5776 /* All other types are added to split BTF. */ 5777 if (adding_to_base) 5778 continue; 5779 err = btf_add_type(&dist->pipe, t); 5780 break; 5781 default: 5782 pr_warn("unexpected kind when adding base type '%s'[%u] of kind [%u] to distilled base BTF.\n", 5783 name, i, kind); 5784 return -EINVAL; 5785 5786 } 5787 if (err < 0) 5788 break; 5789 dist->id_map[i] = id++; 5790 } 5791 return err; 5792 } 5793 5794 /* Split BTF ids without a mapping will be shifted downwards since distilled 5795 * base BTF is smaller than the original base BTF. For those that have a 5796 * mapping (either to base or updated split BTF), update the id based on 5797 * that mapping. 5798 */ 5799 static int btf_update_distilled_type_ids(struct btf_distill *dist, __u32 i) 5800 { 5801 struct btf_type *t = btf_type_by_id(dist->pipe.dst, i); 5802 struct btf_field_iter it; 5803 __u32 *id; 5804 int err; 5805 5806 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS); 5807 if (err) 5808 return err; 5809 while ((id = btf_field_iter_next(&it))) { 5810 if (dist->id_map[*id]) 5811 *id = dist->id_map[*id]; 5812 else if (*id >= dist->split_start_id) 5813 *id -= dist->diff_id; 5814 } 5815 return 0; 5816 } 5817 5818 /* Create updated split BTF with distilled base BTF; distilled base BTF 5819 * consists of BTF information required to clarify the types that split 5820 * BTF refers to, omitting unneeded details. Specifically it will contain 5821 * base types and memberless definitions of named structs, unions and enumerated 5822 * types. Associated reference types like pointers, arrays and anonymous 5823 * structs, unions and enumerated types will be added to split BTF. 5824 * Size is recorded for named struct/unions to help guide matching to the 5825 * target base BTF during later relocation. 5826 * 5827 * The only case where structs, unions or enumerated types are fully represented 5828 * is when they are anonymous; in such cases, the anonymous type is added to 5829 * split BTF in full. 5830 * 5831 * We return newly-created split BTF where the split BTF refers to a newly-created 5832 * distilled base BTF. Both must be freed separately by the caller. 5833 */ 5834 int btf__distill_base(const struct btf *src_btf, struct btf **new_base_btf, 5835 struct btf **new_split_btf) 5836 { 5837 struct btf *new_base = NULL, *new_split = NULL; 5838 const struct btf *old_base; 5839 unsigned int n = btf__type_cnt(src_btf); 5840 struct btf_distill dist = {}; 5841 struct btf_type *t; 5842 int i, err = 0; 5843 5844 /* src BTF must be split BTF. */ 5845 old_base = btf__base_btf(src_btf); 5846 if (!new_base_btf || !new_split_btf || !old_base) 5847 return libbpf_err(-EINVAL); 5848 5849 new_base = btf__new_empty(); 5850 if (!new_base) 5851 return libbpf_err(-ENOMEM); 5852 5853 btf__set_endianness(new_base, btf__endianness(src_btf)); 5854 5855 dist.id_map = calloc(n, sizeof(*dist.id_map)); 5856 if (!dist.id_map) { 5857 err = -ENOMEM; 5858 goto done; 5859 } 5860 dist.pipe.src = src_btf; 5861 dist.pipe.dst = new_base; 5862 dist.pipe.str_off_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL); 5863 if (IS_ERR(dist.pipe.str_off_map)) { 5864 err = -ENOMEM; 5865 goto done; 5866 } 5867 dist.split_start_id = btf__type_cnt(old_base); 5868 dist.split_start_str = old_base->hdr->str_len; 5869 5870 /* Pass over src split BTF; generate the list of base BTF type ids it 5871 * references; these will constitute our distilled BTF set to be 5872 * distributed over base and split BTF as appropriate. 5873 */ 5874 for (i = src_btf->start_id; i < n; i++) { 5875 err = btf_add_distilled_type_ids(&dist, i); 5876 if (err < 0) 5877 goto done; 5878 } 5879 /* Next add types for each of the required references to base BTF and split BTF 5880 * in turn. 5881 */ 5882 err = btf_add_distilled_types(&dist); 5883 if (err < 0) 5884 goto done; 5885 5886 /* Create new split BTF with distilled base BTF as its base; the final 5887 * state is split BTF with distilled base BTF that represents enough 5888 * about its base references to allow it to be relocated with the base 5889 * BTF available. 5890 */ 5891 new_split = btf__new_empty_split(new_base); 5892 if (!new_split) { 5893 err = -errno; 5894 goto done; 5895 } 5896 dist.pipe.dst = new_split; 5897 /* First add all split types */ 5898 for (i = src_btf->start_id; i < n; i++) { 5899 t = btf_type_by_id(src_btf, i); 5900 err = btf_add_type(&dist.pipe, t); 5901 if (err < 0) 5902 goto done; 5903 } 5904 /* Now add distilled types to split BTF that are not added to base. */ 5905 err = btf_add_distilled_types(&dist); 5906 if (err < 0) 5907 goto done; 5908 5909 /* All split BTF ids will be shifted downwards since there are less base 5910 * BTF ids in distilled base BTF. 5911 */ 5912 dist.diff_id = dist.split_start_id - btf__type_cnt(new_base); 5913 5914 n = btf__type_cnt(new_split); 5915 /* Now update base/split BTF ids. */ 5916 for (i = 1; i < n; i++) { 5917 err = btf_update_distilled_type_ids(&dist, i); 5918 if (err < 0) 5919 break; 5920 } 5921 done: 5922 free(dist.id_map); 5923 hashmap__free(dist.pipe.str_off_map); 5924 if (err) { 5925 btf__free(new_split); 5926 btf__free(new_base); 5927 return libbpf_err(err); 5928 } 5929 *new_base_btf = new_base; 5930 *new_split_btf = new_split; 5931 5932 return 0; 5933 } 5934 5935 const struct btf_header *btf_header(const struct btf *btf) 5936 { 5937 return btf->hdr; 5938 } 5939 5940 void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) 5941 { 5942 btf->base_btf = (struct btf *)base_btf; 5943 btf->start_id = btf__type_cnt(base_btf); 5944 btf->start_str_off = base_btf->hdr->str_len + base_btf->start_str_off; 5945 } 5946 5947 int btf__relocate(struct btf *btf, const struct btf *base_btf) 5948 { 5949 int err = btf_relocate(btf, base_btf, NULL); 5950 5951 if (!err) 5952 btf->owns_base = false; 5953 return libbpf_err(err); 5954 } 5955 5956 struct btf_permute { 5957 struct btf *btf; 5958 __u32 *id_map; 5959 __u32 start_offs; 5960 }; 5961 5962 /* Callback function to remap individual type ID references */ 5963 static int btf_permute_remap_type_id(__u32 *type_id, void *ctx) 5964 { 5965 struct btf_permute *p = ctx; 5966 __u32 new_id = *type_id; 5967 5968 /* refer to the base BTF or VOID type */ 5969 if (new_id < p->btf->start_id) 5970 return 0; 5971 5972 if (new_id >= btf__type_cnt(p->btf)) 5973 return -EINVAL; 5974 5975 *type_id = p->id_map[new_id - p->btf->start_id + p->start_offs]; 5976 return 0; 5977 } 5978 5979 int btf__permute(struct btf *btf, __u32 *id_map, __u32 id_map_cnt, 5980 const struct btf_permute_opts *opts) 5981 { 5982 struct btf_permute p; 5983 struct btf_ext *btf_ext; 5984 void *nt, *new_types = NULL; 5985 __u32 *order_map = NULL; 5986 int err = 0, i; 5987 __u32 n, id, start_offs = 0; 5988 5989 if (!OPTS_VALID(opts, btf_permute_opts)) 5990 return libbpf_err(-EINVAL); 5991 5992 if (btf__base_btf(btf)) { 5993 n = btf->nr_types; 5994 } else { 5995 if (id_map[0] != 0) 5996 return libbpf_err(-EINVAL); 5997 n = btf__type_cnt(btf); 5998 start_offs = 1; 5999 } 6000 6001 if (id_map_cnt != n) 6002 return libbpf_err(-EINVAL); 6003 6004 /* record the sequence of types */ 6005 order_map = calloc(id_map_cnt, sizeof(*id_map)); 6006 if (!order_map) { 6007 err = -ENOMEM; 6008 goto done; 6009 } 6010 6011 new_types = calloc(btf->hdr->type_len, 1); 6012 if (!new_types) { 6013 err = -ENOMEM; 6014 goto done; 6015 } 6016 6017 if (btf_ensure_modifiable(btf)) { 6018 err = -ENOMEM; 6019 goto done; 6020 } 6021 6022 for (i = start_offs; i < id_map_cnt; i++) { 6023 id = id_map[i]; 6024 if (id < btf->start_id || id >= btf__type_cnt(btf)) { 6025 err = -EINVAL; 6026 goto done; 6027 } 6028 id -= btf->start_id - start_offs; 6029 /* cannot be mapped to the same ID */ 6030 if (order_map[id]) { 6031 err = -EINVAL; 6032 goto done; 6033 } 6034 order_map[id] = i + btf->start_id - start_offs; 6035 } 6036 6037 p.btf = btf; 6038 p.id_map = id_map; 6039 p.start_offs = start_offs; 6040 nt = new_types; 6041 for (i = start_offs; i < id_map_cnt; i++) { 6042 struct btf_field_iter it; 6043 const struct btf_type *t; 6044 __u32 *type_id; 6045 int type_size; 6046 6047 id = order_map[i]; 6048 t = btf__type_by_id(btf, id); 6049 type_size = btf_type_size(t); 6050 memcpy(nt, t, type_size); 6051 6052 /* fix up referenced IDs for BTF */ 6053 err = btf_field_iter_init(&it, nt, BTF_FIELD_ITER_IDS); 6054 if (err) 6055 goto done; 6056 while ((type_id = btf_field_iter_next(&it))) { 6057 err = btf_permute_remap_type_id(type_id, &p); 6058 if (err) 6059 goto done; 6060 } 6061 6062 nt += type_size; 6063 } 6064 6065 /* fix up referenced IDs for btf_ext */ 6066 btf_ext = OPTS_GET(opts, btf_ext, NULL); 6067 if (btf_ext) { 6068 err = btf_ext_visit_type_ids(btf_ext, btf_permute_remap_type_id, &p); 6069 if (err) 6070 goto done; 6071 } 6072 6073 for (nt = new_types, i = 0; i < id_map_cnt - start_offs; i++) { 6074 btf->type_offs[i] = nt - new_types; 6075 nt += btf_type_size(nt); 6076 } 6077 6078 free(order_map); 6079 free(btf->types_data); 6080 btf->types_data = new_types; 6081 return 0; 6082 6083 done: 6084 free(order_map); 6085 free(new_types); 6086 return libbpf_err(err); 6087 } 6088