1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* Copyright (c) 2018 Facebook */ 3 4 #include <byteswap.h> 5 #include <endian.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <fcntl.h> 10 #include <unistd.h> 11 #include <errno.h> 12 #include <sys/utsname.h> 13 #include <sys/param.h> 14 #include <sys/stat.h> 15 #include <linux/kernel.h> 16 #include <linux/err.h> 17 #include <linux/btf.h> 18 #include <gelf.h> 19 #include "btf.h" 20 #include "bpf.h" 21 #include "libbpf.h" 22 #include "libbpf_internal.h" 23 #include "hashmap.h" 24 #include "strset.h" 25 #include "str_error.h" 26 27 #define BTF_MAX_NR_TYPES 0x7fffffffU 28 #define BTF_MAX_STR_OFFSET 0x7fffffffU 29 30 static struct btf_type btf_void; 31 32 struct btf { 33 /* raw BTF data in native endianness */ 34 void *raw_data; 35 /* raw BTF data in non-native endianness */ 36 void *raw_data_swapped; 37 __u32 raw_size; 38 /* whether target endianness differs from the native one */ 39 bool swapped_endian; 40 41 /* 42 * When BTF is loaded from an ELF or raw memory it is stored 43 * in a contiguous memory block. The hdr, type_data, and, strs_data 44 * point inside that memory region to their respective parts of BTF 45 * representation: 46 * 47 * +--------------------------------+ 48 * | Header | Types | Strings | 49 * +--------------------------------+ 50 * ^ ^ ^ 51 * | | | 52 * hdr | | 53 * types_data-+ | 54 * strs_data------------+ 55 * 56 * If BTF data is later modified, e.g., due to types added or 57 * removed, BTF deduplication performed, etc, this contiguous 58 * representation is broken up into three independently allocated 59 * memory regions to be able to modify them independently. 60 * raw_data is nulled out at that point, but can be later allocated 61 * and cached again if user calls btf__raw_data(), at which point 62 * raw_data will contain a contiguous copy of header, types, and 63 * strings: 64 * 65 * +----------+ +---------+ +-----------+ 66 * | Header | | Types | | Strings | 67 * +----------+ +---------+ +-----------+ 68 * ^ ^ ^ 69 * | | | 70 * hdr | | 71 * types_data----+ | 72 * strset__data(strs_set)-----+ 73 * 74 * +----------+---------+-----------+ 75 * | Header | Types | Strings | 76 * raw_data----->+----------+---------+-----------+ 77 */ 78 struct btf_header *hdr; 79 80 void *types_data; 81 size_t types_data_cap; /* used size stored in hdr->type_len */ 82 83 /* type ID to `struct btf_type *` lookup index 84 * type_offs[0] corresponds to the first non-VOID type: 85 * - for base BTF it's type [1]; 86 * - for split BTF it's the first non-base BTF type. 87 */ 88 __u32 *type_offs; 89 size_t type_offs_cap; 90 /* number of types in this BTF instance: 91 * - doesn't include special [0] void type; 92 * - for split BTF counts number of types added on top of base BTF. 93 */ 94 __u32 nr_types; 95 /* if not NULL, points to the base BTF on top of which the current 96 * split BTF is based 97 */ 98 struct btf *base_btf; 99 /* BTF type ID of the first type in this BTF instance: 100 * - for base BTF it's equal to 1; 101 * - for split BTF it's equal to biggest type ID of base BTF plus 1. 102 */ 103 int start_id; 104 /* logical string offset of this BTF instance: 105 * - for base BTF it's equal to 0; 106 * - for split BTF it's equal to total size of base BTF's string section size. 107 */ 108 int start_str_off; 109 110 /* only one of strs_data or strs_set can be non-NULL, depending on 111 * whether BTF is in a modifiable state (strs_set is used) or not 112 * (strs_data points inside raw_data) 113 */ 114 void *strs_data; 115 /* a set of unique strings */ 116 struct strset *strs_set; 117 /* whether strings are already deduplicated */ 118 bool strs_deduped; 119 120 /* whether base_btf should be freed in btf_free for this instance */ 121 bool owns_base; 122 123 /* BTF object FD, if loaded into kernel */ 124 int fd; 125 126 /* Pointer size (in bytes) for a target architecture of this BTF */ 127 int ptr_sz; 128 }; 129 130 static inline __u64 ptr_to_u64(const void *ptr) 131 { 132 return (__u64) (unsigned long) ptr; 133 } 134 135 /* Ensure given dynamically allocated memory region pointed to by *data* with 136 * capacity of *cap_cnt* elements each taking *elem_sz* bytes has enough 137 * memory to accommodate *add_cnt* new elements, assuming *cur_cnt* elements 138 * are already used. At most *max_cnt* elements can be ever allocated. 139 * If necessary, memory is reallocated and all existing data is copied over, 140 * new pointer to the memory region is stored at *data, new memory region 141 * capacity (in number of elements) is stored in *cap. 142 * On success, memory pointer to the beginning of unused memory is returned. 143 * On error, NULL is returned. 144 */ 145 void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, 146 size_t cur_cnt, size_t max_cnt, size_t add_cnt) 147 { 148 size_t new_cnt; 149 void *new_data; 150 151 if (cur_cnt + add_cnt <= *cap_cnt) 152 return *data + cur_cnt * elem_sz; 153 154 /* requested more than the set limit */ 155 if (cur_cnt + add_cnt > max_cnt) 156 return NULL; 157 158 new_cnt = *cap_cnt; 159 new_cnt += new_cnt / 4; /* expand by 25% */ 160 if (new_cnt < 16) /* but at least 16 elements */ 161 new_cnt = 16; 162 if (new_cnt > max_cnt) /* but not exceeding a set limit */ 163 new_cnt = max_cnt; 164 if (new_cnt < cur_cnt + add_cnt) /* also ensure we have enough memory */ 165 new_cnt = cur_cnt + add_cnt; 166 167 new_data = libbpf_reallocarray(*data, new_cnt, elem_sz); 168 if (!new_data) 169 return NULL; 170 171 /* zero out newly allocated portion of memory */ 172 memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz); 173 174 *data = new_data; 175 *cap_cnt = new_cnt; 176 return new_data + cur_cnt * elem_sz; 177 } 178 179 /* Ensure given dynamically allocated memory region has enough allocated space 180 * to accommodate *need_cnt* elements of size *elem_sz* bytes each 181 */ 182 int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt) 183 { 184 void *p; 185 186 if (need_cnt <= *cap_cnt) 187 return 0; 188 189 p = libbpf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt); 190 if (!p) 191 return -ENOMEM; 192 193 return 0; 194 } 195 196 static void *btf_add_type_offs_mem(struct btf *btf, size_t add_cnt) 197 { 198 return libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32), 199 btf->nr_types, BTF_MAX_NR_TYPES, add_cnt); 200 } 201 202 static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off) 203 { 204 __u32 *p; 205 206 p = btf_add_type_offs_mem(btf, 1); 207 if (!p) 208 return -ENOMEM; 209 210 *p = type_off; 211 return 0; 212 } 213 214 static void btf_bswap_hdr(struct btf_header *h) 215 { 216 h->magic = bswap_16(h->magic); 217 h->hdr_len = bswap_32(h->hdr_len); 218 h->type_off = bswap_32(h->type_off); 219 h->type_len = bswap_32(h->type_len); 220 h->str_off = bswap_32(h->str_off); 221 h->str_len = bswap_32(h->str_len); 222 } 223 224 static int btf_parse_hdr(struct btf *btf) 225 { 226 struct btf_header *hdr = btf->hdr; 227 __u32 meta_left; 228 229 if (btf->raw_size < sizeof(struct btf_header)) { 230 pr_debug("BTF header not found\n"); 231 return -EINVAL; 232 } 233 234 if (hdr->magic == bswap_16(BTF_MAGIC)) { 235 btf->swapped_endian = true; 236 if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) { 237 pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n", 238 bswap_32(hdr->hdr_len)); 239 return -ENOTSUP; 240 } 241 btf_bswap_hdr(hdr); 242 } else if (hdr->magic != BTF_MAGIC) { 243 pr_debug("Invalid BTF magic: %x\n", hdr->magic); 244 return -EINVAL; 245 } 246 247 if (btf->raw_size < hdr->hdr_len) { 248 pr_debug("BTF header len %u larger than data size %u\n", 249 hdr->hdr_len, btf->raw_size); 250 return -EINVAL; 251 } 252 253 meta_left = btf->raw_size - hdr->hdr_len; 254 if (meta_left < (long long)hdr->str_off + hdr->str_len) { 255 pr_debug("Invalid BTF total size: %u\n", btf->raw_size); 256 return -EINVAL; 257 } 258 259 if ((long long)hdr->type_off + hdr->type_len > hdr->str_off) { 260 pr_debug("Invalid BTF data sections layout: type data at %u + %u, strings data at %u + %u\n", 261 hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len); 262 return -EINVAL; 263 } 264 265 if (hdr->type_off % 4) { 266 pr_debug("BTF type section is not aligned to 4 bytes\n"); 267 return -EINVAL; 268 } 269 270 return 0; 271 } 272 273 static int btf_parse_str_sec(struct btf *btf) 274 { 275 const struct btf_header *hdr = btf->hdr; 276 const char *start = btf->strs_data; 277 const char *end = start + btf->hdr->str_len; 278 279 if (btf->base_btf && hdr->str_len == 0) 280 return 0; 281 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) { 282 pr_debug("Invalid BTF string section\n"); 283 return -EINVAL; 284 } 285 if (!btf->base_btf && start[0]) { 286 pr_debug("Malformed BTF string section, did you forget to provide base BTF?\n"); 287 return -EINVAL; 288 } 289 return 0; 290 } 291 292 static int btf_type_size(const struct btf_type *t) 293 { 294 const int base_size = sizeof(struct btf_type); 295 __u16 vlen = btf_vlen(t); 296 297 switch (btf_kind(t)) { 298 case BTF_KIND_FWD: 299 case BTF_KIND_CONST: 300 case BTF_KIND_VOLATILE: 301 case BTF_KIND_RESTRICT: 302 case BTF_KIND_PTR: 303 case BTF_KIND_TYPEDEF: 304 case BTF_KIND_FUNC: 305 case BTF_KIND_FLOAT: 306 case BTF_KIND_TYPE_TAG: 307 return base_size; 308 case BTF_KIND_INT: 309 return base_size + sizeof(__u32); 310 case BTF_KIND_ENUM: 311 return base_size + vlen * sizeof(struct btf_enum); 312 case BTF_KIND_ENUM64: 313 return base_size + vlen * sizeof(struct btf_enum64); 314 case BTF_KIND_ARRAY: 315 return base_size + sizeof(struct btf_array); 316 case BTF_KIND_STRUCT: 317 case BTF_KIND_UNION: 318 return base_size + vlen * sizeof(struct btf_member); 319 case BTF_KIND_FUNC_PROTO: 320 return base_size + vlen * sizeof(struct btf_param); 321 case BTF_KIND_VAR: 322 return base_size + sizeof(struct btf_var); 323 case BTF_KIND_DATASEC: 324 return base_size + vlen * sizeof(struct btf_var_secinfo); 325 case BTF_KIND_DECL_TAG: 326 return base_size + sizeof(struct btf_decl_tag); 327 default: 328 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t)); 329 return -EINVAL; 330 } 331 } 332 333 static void btf_bswap_type_base(struct btf_type *t) 334 { 335 t->name_off = bswap_32(t->name_off); 336 t->info = bswap_32(t->info); 337 t->type = bswap_32(t->type); 338 } 339 340 static int btf_bswap_type_rest(struct btf_type *t) 341 { 342 struct btf_var_secinfo *v; 343 struct btf_enum64 *e64; 344 struct btf_member *m; 345 struct btf_array *a; 346 struct btf_param *p; 347 struct btf_enum *e; 348 __u16 vlen = btf_vlen(t); 349 int i; 350 351 switch (btf_kind(t)) { 352 case BTF_KIND_FWD: 353 case BTF_KIND_CONST: 354 case BTF_KIND_VOLATILE: 355 case BTF_KIND_RESTRICT: 356 case BTF_KIND_PTR: 357 case BTF_KIND_TYPEDEF: 358 case BTF_KIND_FUNC: 359 case BTF_KIND_FLOAT: 360 case BTF_KIND_TYPE_TAG: 361 return 0; 362 case BTF_KIND_INT: 363 *(__u32 *)(t + 1) = bswap_32(*(__u32 *)(t + 1)); 364 return 0; 365 case BTF_KIND_ENUM: 366 for (i = 0, e = btf_enum(t); i < vlen; i++, e++) { 367 e->name_off = bswap_32(e->name_off); 368 e->val = bswap_32(e->val); 369 } 370 return 0; 371 case BTF_KIND_ENUM64: 372 for (i = 0, e64 = btf_enum64(t); i < vlen; i++, e64++) { 373 e64->name_off = bswap_32(e64->name_off); 374 e64->val_lo32 = bswap_32(e64->val_lo32); 375 e64->val_hi32 = bswap_32(e64->val_hi32); 376 } 377 return 0; 378 case BTF_KIND_ARRAY: 379 a = btf_array(t); 380 a->type = bswap_32(a->type); 381 a->index_type = bswap_32(a->index_type); 382 a->nelems = bswap_32(a->nelems); 383 return 0; 384 case BTF_KIND_STRUCT: 385 case BTF_KIND_UNION: 386 for (i = 0, m = btf_members(t); i < vlen; i++, m++) { 387 m->name_off = bswap_32(m->name_off); 388 m->type = bswap_32(m->type); 389 m->offset = bswap_32(m->offset); 390 } 391 return 0; 392 case BTF_KIND_FUNC_PROTO: 393 for (i = 0, p = btf_params(t); i < vlen; i++, p++) { 394 p->name_off = bswap_32(p->name_off); 395 p->type = bswap_32(p->type); 396 } 397 return 0; 398 case BTF_KIND_VAR: 399 btf_var(t)->linkage = bswap_32(btf_var(t)->linkage); 400 return 0; 401 case BTF_KIND_DATASEC: 402 for (i = 0, v = btf_var_secinfos(t); i < vlen; i++, v++) { 403 v->type = bswap_32(v->type); 404 v->offset = bswap_32(v->offset); 405 v->size = bswap_32(v->size); 406 } 407 return 0; 408 case BTF_KIND_DECL_TAG: 409 btf_decl_tag(t)->component_idx = bswap_32(btf_decl_tag(t)->component_idx); 410 return 0; 411 default: 412 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t)); 413 return -EINVAL; 414 } 415 } 416 417 static int btf_parse_type_sec(struct btf *btf) 418 { 419 struct btf_header *hdr = btf->hdr; 420 void *next_type = btf->types_data; 421 void *end_type = next_type + hdr->type_len; 422 int err, type_size; 423 424 while (next_type + sizeof(struct btf_type) <= end_type) { 425 if (btf->swapped_endian) 426 btf_bswap_type_base(next_type); 427 428 type_size = btf_type_size(next_type); 429 if (type_size < 0) 430 return type_size; 431 if (next_type + type_size > end_type) { 432 pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types); 433 return -EINVAL; 434 } 435 436 if (btf->swapped_endian && btf_bswap_type_rest(next_type)) 437 return -EINVAL; 438 439 err = btf_add_type_idx_entry(btf, next_type - btf->types_data); 440 if (err) 441 return err; 442 443 next_type += type_size; 444 btf->nr_types++; 445 } 446 447 if (next_type != end_type) { 448 pr_warn("BTF types data is malformed\n"); 449 return -EINVAL; 450 } 451 452 return 0; 453 } 454 455 static int btf_validate_str(const struct btf *btf, __u32 str_off, const char *what, __u32 type_id) 456 { 457 const char *s; 458 459 s = btf__str_by_offset(btf, str_off); 460 if (!s) { 461 pr_warn("btf: type [%u]: invalid %s (string offset %u)\n", type_id, what, str_off); 462 return -EINVAL; 463 } 464 465 return 0; 466 } 467 468 static int btf_validate_id(const struct btf *btf, __u32 id, __u32 ctx_id) 469 { 470 const struct btf_type *t; 471 472 t = btf__type_by_id(btf, id); 473 if (!t) { 474 pr_warn("btf: type [%u]: invalid referenced type ID %u\n", ctx_id, id); 475 return -EINVAL; 476 } 477 478 return 0; 479 } 480 481 static int btf_validate_type(const struct btf *btf, const struct btf_type *t, __u32 id) 482 { 483 __u32 kind = btf_kind(t); 484 int err, i, n; 485 486 err = btf_validate_str(btf, t->name_off, "type name", id); 487 if (err) 488 return err; 489 490 switch (kind) { 491 case BTF_KIND_UNKN: 492 case BTF_KIND_INT: 493 case BTF_KIND_FWD: 494 case BTF_KIND_FLOAT: 495 break; 496 case BTF_KIND_PTR: 497 case BTF_KIND_TYPEDEF: 498 case BTF_KIND_VOLATILE: 499 case BTF_KIND_CONST: 500 case BTF_KIND_RESTRICT: 501 case BTF_KIND_VAR: 502 case BTF_KIND_DECL_TAG: 503 case BTF_KIND_TYPE_TAG: 504 err = btf_validate_id(btf, t->type, id); 505 if (err) 506 return err; 507 break; 508 case BTF_KIND_ARRAY: { 509 const struct btf_array *a = btf_array(t); 510 511 err = btf_validate_id(btf, a->type, id); 512 err = err ?: btf_validate_id(btf, a->index_type, id); 513 if (err) 514 return err; 515 break; 516 } 517 case BTF_KIND_STRUCT: 518 case BTF_KIND_UNION: { 519 const struct btf_member *m = btf_members(t); 520 521 n = btf_vlen(t); 522 for (i = 0; i < n; i++, m++) { 523 err = btf_validate_str(btf, m->name_off, "field name", id); 524 err = err ?: btf_validate_id(btf, m->type, id); 525 if (err) 526 return err; 527 } 528 break; 529 } 530 case BTF_KIND_ENUM: { 531 const struct btf_enum *m = btf_enum(t); 532 533 n = btf_vlen(t); 534 for (i = 0; i < n; i++, m++) { 535 err = btf_validate_str(btf, m->name_off, "enum name", id); 536 if (err) 537 return err; 538 } 539 break; 540 } 541 case BTF_KIND_ENUM64: { 542 const struct btf_enum64 *m = btf_enum64(t); 543 544 n = btf_vlen(t); 545 for (i = 0; i < n; i++, m++) { 546 err = btf_validate_str(btf, m->name_off, "enum name", id); 547 if (err) 548 return err; 549 } 550 break; 551 } 552 case BTF_KIND_FUNC: { 553 const struct btf_type *ft; 554 555 err = btf_validate_id(btf, t->type, id); 556 if (err) 557 return err; 558 ft = btf__type_by_id(btf, t->type); 559 if (btf_kind(ft) != BTF_KIND_FUNC_PROTO) { 560 pr_warn("btf: type [%u]: referenced type [%u] is not FUNC_PROTO\n", id, t->type); 561 return -EINVAL; 562 } 563 break; 564 } 565 case BTF_KIND_FUNC_PROTO: { 566 const struct btf_param *m = btf_params(t); 567 568 n = btf_vlen(t); 569 for (i = 0; i < n; i++, m++) { 570 err = btf_validate_str(btf, m->name_off, "param name", id); 571 err = err ?: btf_validate_id(btf, m->type, id); 572 if (err) 573 return err; 574 } 575 break; 576 } 577 case BTF_KIND_DATASEC: { 578 const struct btf_var_secinfo *m = btf_var_secinfos(t); 579 580 n = btf_vlen(t); 581 for (i = 0; i < n; i++, m++) { 582 err = btf_validate_id(btf, m->type, id); 583 if (err) 584 return err; 585 } 586 break; 587 } 588 default: 589 pr_warn("btf: type [%u]: unrecognized kind %u\n", id, kind); 590 return -EINVAL; 591 } 592 return 0; 593 } 594 595 /* Validate basic sanity of BTF. It's intentionally less thorough than 596 * kernel's validation and validates only properties of BTF that libbpf relies 597 * on to be correct (e.g., valid type IDs, valid string offsets, etc) 598 */ 599 static int btf_sanity_check(const struct btf *btf) 600 { 601 const struct btf_type *t; 602 __u32 i, n = btf__type_cnt(btf); 603 int err; 604 605 for (i = btf->start_id; i < n; i++) { 606 t = btf_type_by_id(btf, i); 607 err = btf_validate_type(btf, t, i); 608 if (err) 609 return err; 610 } 611 return 0; 612 } 613 614 __u32 btf__type_cnt(const struct btf *btf) 615 { 616 return btf->start_id + btf->nr_types; 617 } 618 619 const struct btf *btf__base_btf(const struct btf *btf) 620 { 621 return btf->base_btf; 622 } 623 624 /* internal helper returning non-const pointer to a type */ 625 struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id) 626 { 627 if (type_id == 0) 628 return &btf_void; 629 if (type_id < btf->start_id) 630 return btf_type_by_id(btf->base_btf, type_id); 631 return btf->types_data + btf->type_offs[type_id - btf->start_id]; 632 } 633 634 const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id) 635 { 636 if (type_id >= btf->start_id + btf->nr_types) 637 return errno = EINVAL, NULL; 638 return btf_type_by_id((struct btf *)btf, type_id); 639 } 640 641 static int determine_ptr_size(const struct btf *btf) 642 { 643 static const char * const long_aliases[] = { 644 "long", 645 "long int", 646 "int long", 647 "unsigned long", 648 "long unsigned", 649 "unsigned long int", 650 "unsigned int long", 651 "long unsigned int", 652 "long int unsigned", 653 "int unsigned long", 654 "int long unsigned", 655 }; 656 const struct btf_type *t; 657 const char *name; 658 int i, j, n; 659 660 if (btf->base_btf && btf->base_btf->ptr_sz > 0) 661 return btf->base_btf->ptr_sz; 662 663 n = btf__type_cnt(btf); 664 for (i = 1; i < n; i++) { 665 t = btf__type_by_id(btf, i); 666 if (!btf_is_int(t)) 667 continue; 668 669 if (t->size != 4 && t->size != 8) 670 continue; 671 672 name = btf__name_by_offset(btf, t->name_off); 673 if (!name) 674 continue; 675 676 for (j = 0; j < ARRAY_SIZE(long_aliases); j++) { 677 if (strcmp(name, long_aliases[j]) == 0) 678 return t->size; 679 } 680 } 681 682 return -1; 683 } 684 685 static size_t btf_ptr_sz(const struct btf *btf) 686 { 687 if (!btf->ptr_sz) 688 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); 689 return btf->ptr_sz < 0 ? sizeof(void *) : btf->ptr_sz; 690 } 691 692 /* Return pointer size this BTF instance assumes. The size is heuristically 693 * determined by looking for 'long' or 'unsigned long' integer type and 694 * recording its size in bytes. If BTF type information doesn't have any such 695 * type, this function returns 0. In the latter case, native architecture's 696 * pointer size is assumed, so will be either 4 or 8, depending on 697 * architecture that libbpf was compiled for. It's possible to override 698 * guessed value by using btf__set_pointer_size() API. 699 */ 700 size_t btf__pointer_size(const struct btf *btf) 701 { 702 if (!btf->ptr_sz) 703 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); 704 705 if (btf->ptr_sz < 0) 706 /* not enough BTF type info to guess */ 707 return 0; 708 709 return btf->ptr_sz; 710 } 711 712 /* Override or set pointer size in bytes. Only values of 4 and 8 are 713 * supported. 714 */ 715 int btf__set_pointer_size(struct btf *btf, size_t ptr_sz) 716 { 717 if (ptr_sz != 4 && ptr_sz != 8) 718 return libbpf_err(-EINVAL); 719 btf->ptr_sz = ptr_sz; 720 return 0; 721 } 722 723 static bool is_host_big_endian(void) 724 { 725 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 726 return false; 727 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 728 return true; 729 #else 730 # error "Unrecognized __BYTE_ORDER__" 731 #endif 732 } 733 734 enum btf_endianness btf__endianness(const struct btf *btf) 735 { 736 if (is_host_big_endian()) 737 return btf->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN; 738 else 739 return btf->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN; 740 } 741 742 int btf__set_endianness(struct btf *btf, enum btf_endianness endian) 743 { 744 if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN) 745 return libbpf_err(-EINVAL); 746 747 btf->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN); 748 if (!btf->swapped_endian) { 749 free(btf->raw_data_swapped); 750 btf->raw_data_swapped = NULL; 751 } 752 return 0; 753 } 754 755 static bool btf_type_is_void(const struct btf_type *t) 756 { 757 return t == &btf_void || btf_is_fwd(t); 758 } 759 760 static bool btf_type_is_void_or_null(const struct btf_type *t) 761 { 762 return !t || btf_type_is_void(t); 763 } 764 765 #define MAX_RESOLVE_DEPTH 32 766 767 __s64 btf__resolve_size(const struct btf *btf, __u32 type_id) 768 { 769 const struct btf_array *array; 770 const struct btf_type *t; 771 __u32 nelems = 1; 772 __s64 size = -1; 773 int i; 774 775 t = btf__type_by_id(btf, type_id); 776 for (i = 0; i < MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null(t); i++) { 777 switch (btf_kind(t)) { 778 case BTF_KIND_INT: 779 case BTF_KIND_STRUCT: 780 case BTF_KIND_UNION: 781 case BTF_KIND_ENUM: 782 case BTF_KIND_ENUM64: 783 case BTF_KIND_DATASEC: 784 case BTF_KIND_FLOAT: 785 size = t->size; 786 goto done; 787 case BTF_KIND_PTR: 788 size = btf_ptr_sz(btf); 789 goto done; 790 case BTF_KIND_TYPEDEF: 791 case BTF_KIND_VOLATILE: 792 case BTF_KIND_CONST: 793 case BTF_KIND_RESTRICT: 794 case BTF_KIND_VAR: 795 case BTF_KIND_DECL_TAG: 796 case BTF_KIND_TYPE_TAG: 797 type_id = t->type; 798 break; 799 case BTF_KIND_ARRAY: 800 array = btf_array(t); 801 if (nelems && array->nelems > UINT32_MAX / nelems) 802 return libbpf_err(-E2BIG); 803 nelems *= array->nelems; 804 type_id = array->type; 805 break; 806 default: 807 return libbpf_err(-EINVAL); 808 } 809 810 t = btf__type_by_id(btf, type_id); 811 } 812 813 done: 814 if (size < 0) 815 return libbpf_err(-EINVAL); 816 if (nelems && size > UINT32_MAX / nelems) 817 return libbpf_err(-E2BIG); 818 819 return nelems * size; 820 } 821 822 int btf__align_of(const struct btf *btf, __u32 id) 823 { 824 const struct btf_type *t = btf__type_by_id(btf, id); 825 __u16 kind = btf_kind(t); 826 827 switch (kind) { 828 case BTF_KIND_INT: 829 case BTF_KIND_ENUM: 830 case BTF_KIND_ENUM64: 831 case BTF_KIND_FLOAT: 832 return min(btf_ptr_sz(btf), (size_t)t->size); 833 case BTF_KIND_PTR: 834 return btf_ptr_sz(btf); 835 case BTF_KIND_TYPEDEF: 836 case BTF_KIND_VOLATILE: 837 case BTF_KIND_CONST: 838 case BTF_KIND_RESTRICT: 839 case BTF_KIND_TYPE_TAG: 840 return btf__align_of(btf, t->type); 841 case BTF_KIND_ARRAY: 842 return btf__align_of(btf, btf_array(t)->type); 843 case BTF_KIND_STRUCT: 844 case BTF_KIND_UNION: { 845 const struct btf_member *m = btf_members(t); 846 __u16 vlen = btf_vlen(t); 847 int i, max_align = 1, align; 848 849 for (i = 0; i < vlen; i++, m++) { 850 align = btf__align_of(btf, m->type); 851 if (align <= 0) 852 return libbpf_err(align); 853 max_align = max(max_align, align); 854 855 /* if field offset isn't aligned according to field 856 * type's alignment, then struct must be packed 857 */ 858 if (btf_member_bitfield_size(t, i) == 0 && 859 (m->offset % (8 * align)) != 0) 860 return 1; 861 } 862 863 /* if struct/union size isn't a multiple of its alignment, 864 * then struct must be packed 865 */ 866 if ((t->size % max_align) != 0) 867 return 1; 868 869 return max_align; 870 } 871 default: 872 pr_warn("unsupported BTF_KIND:%u\n", btf_kind(t)); 873 return errno = EINVAL, 0; 874 } 875 } 876 877 int btf__resolve_type(const struct btf *btf, __u32 type_id) 878 { 879 const struct btf_type *t; 880 int depth = 0; 881 882 t = btf__type_by_id(btf, type_id); 883 while (depth < MAX_RESOLVE_DEPTH && 884 !btf_type_is_void_or_null(t) && 885 (btf_is_mod(t) || btf_is_typedef(t) || btf_is_var(t))) { 886 type_id = t->type; 887 t = btf__type_by_id(btf, type_id); 888 depth++; 889 } 890 891 if (depth == MAX_RESOLVE_DEPTH || btf_type_is_void_or_null(t)) 892 return libbpf_err(-EINVAL); 893 894 return type_id; 895 } 896 897 __s32 btf__find_by_name(const struct btf *btf, const char *type_name) 898 { 899 __u32 i, nr_types = btf__type_cnt(btf); 900 901 if (!strcmp(type_name, "void")) 902 return 0; 903 904 for (i = 1; i < nr_types; i++) { 905 const struct btf_type *t = btf__type_by_id(btf, i); 906 const char *name = btf__name_by_offset(btf, t->name_off); 907 908 if (name && !strcmp(type_name, name)) 909 return i; 910 } 911 912 return libbpf_err(-ENOENT); 913 } 914 915 static __s32 btf_find_by_name_kind(const struct btf *btf, int start_id, 916 const char *type_name, __u32 kind) 917 { 918 __u32 i, nr_types = btf__type_cnt(btf); 919 920 if (kind == BTF_KIND_UNKN || !strcmp(type_name, "void")) 921 return 0; 922 923 for (i = start_id; i < nr_types; i++) { 924 const struct btf_type *t = btf__type_by_id(btf, i); 925 const char *name; 926 927 if (btf_kind(t) != kind) 928 continue; 929 name = btf__name_by_offset(btf, t->name_off); 930 if (name && !strcmp(type_name, name)) 931 return i; 932 } 933 934 return libbpf_err(-ENOENT); 935 } 936 937 __s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name, 938 __u32 kind) 939 { 940 return btf_find_by_name_kind(btf, btf->start_id, type_name, kind); 941 } 942 943 __s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name, 944 __u32 kind) 945 { 946 return btf_find_by_name_kind(btf, 1, type_name, kind); 947 } 948 949 static bool btf_is_modifiable(const struct btf *btf) 950 { 951 return (void *)btf->hdr != btf->raw_data; 952 } 953 954 void btf__free(struct btf *btf) 955 { 956 if (IS_ERR_OR_NULL(btf)) 957 return; 958 959 if (btf->fd >= 0) 960 close(btf->fd); 961 962 if (btf_is_modifiable(btf)) { 963 /* if BTF was modified after loading, it will have a split 964 * in-memory representation for header, types, and strings 965 * sections, so we need to free all of them individually. It 966 * might still have a cached contiguous raw data present, 967 * which will be unconditionally freed below. 968 */ 969 free(btf->hdr); 970 free(btf->types_data); 971 strset__free(btf->strs_set); 972 } 973 free(btf->raw_data); 974 free(btf->raw_data_swapped); 975 free(btf->type_offs); 976 if (btf->owns_base) 977 btf__free(btf->base_btf); 978 free(btf); 979 } 980 981 static struct btf *btf_new_empty(struct btf *base_btf) 982 { 983 struct btf *btf; 984 985 btf = calloc(1, sizeof(*btf)); 986 if (!btf) 987 return ERR_PTR(-ENOMEM); 988 989 btf->nr_types = 0; 990 btf->start_id = 1; 991 btf->start_str_off = 0; 992 btf->fd = -1; 993 btf->ptr_sz = sizeof(void *); 994 btf->swapped_endian = false; 995 996 if (base_btf) { 997 btf->base_btf = base_btf; 998 btf->start_id = btf__type_cnt(base_btf); 999 btf->start_str_off = base_btf->hdr->str_len; 1000 btf->swapped_endian = base_btf->swapped_endian; 1001 } 1002 1003 /* +1 for empty string at offset 0 */ 1004 btf->raw_size = sizeof(struct btf_header) + (base_btf ? 0 : 1); 1005 btf->raw_data = calloc(1, btf->raw_size); 1006 if (!btf->raw_data) { 1007 free(btf); 1008 return ERR_PTR(-ENOMEM); 1009 } 1010 1011 btf->hdr = btf->raw_data; 1012 btf->hdr->hdr_len = sizeof(struct btf_header); 1013 btf->hdr->magic = BTF_MAGIC; 1014 btf->hdr->version = BTF_VERSION; 1015 1016 btf->types_data = btf->raw_data + btf->hdr->hdr_len; 1017 btf->strs_data = btf->raw_data + btf->hdr->hdr_len; 1018 btf->hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */ 1019 1020 return btf; 1021 } 1022 1023 struct btf *btf__new_empty(void) 1024 { 1025 return libbpf_ptr(btf_new_empty(NULL)); 1026 } 1027 1028 struct btf *btf__new_empty_split(struct btf *base_btf) 1029 { 1030 return libbpf_ptr(btf_new_empty(base_btf)); 1031 } 1032 1033 static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf) 1034 { 1035 struct btf *btf; 1036 int err; 1037 1038 btf = calloc(1, sizeof(struct btf)); 1039 if (!btf) 1040 return ERR_PTR(-ENOMEM); 1041 1042 btf->nr_types = 0; 1043 btf->start_id = 1; 1044 btf->start_str_off = 0; 1045 btf->fd = -1; 1046 1047 if (base_btf) { 1048 btf->base_btf = base_btf; 1049 btf->start_id = btf__type_cnt(base_btf); 1050 btf->start_str_off = base_btf->hdr->str_len; 1051 } 1052 1053 btf->raw_data = malloc(size); 1054 if (!btf->raw_data) { 1055 err = -ENOMEM; 1056 goto done; 1057 } 1058 memcpy(btf->raw_data, data, size); 1059 btf->raw_size = size; 1060 1061 btf->hdr = btf->raw_data; 1062 err = btf_parse_hdr(btf); 1063 if (err) 1064 goto done; 1065 1066 btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off; 1067 btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off; 1068 1069 err = btf_parse_str_sec(btf); 1070 err = err ?: btf_parse_type_sec(btf); 1071 err = err ?: btf_sanity_check(btf); 1072 if (err) 1073 goto done; 1074 1075 done: 1076 if (err) { 1077 btf__free(btf); 1078 return ERR_PTR(err); 1079 } 1080 1081 return btf; 1082 } 1083 1084 struct btf *btf__new(const void *data, __u32 size) 1085 { 1086 return libbpf_ptr(btf_new(data, size, NULL)); 1087 } 1088 1089 struct btf *btf__new_split(const void *data, __u32 size, struct btf *base_btf) 1090 { 1091 return libbpf_ptr(btf_new(data, size, base_btf)); 1092 } 1093 1094 struct btf_elf_secs { 1095 Elf_Data *btf_data; 1096 Elf_Data *btf_ext_data; 1097 Elf_Data *btf_base_data; 1098 }; 1099 1100 static int btf_find_elf_sections(Elf *elf, const char *path, struct btf_elf_secs *secs) 1101 { 1102 Elf_Scn *scn = NULL; 1103 Elf_Data *data; 1104 GElf_Ehdr ehdr; 1105 size_t shstrndx; 1106 int idx = 0; 1107 1108 if (!gelf_getehdr(elf, &ehdr)) { 1109 pr_warn("failed to get EHDR from %s\n", path); 1110 goto err; 1111 } 1112 1113 if (elf_getshdrstrndx(elf, &shstrndx)) { 1114 pr_warn("failed to get section names section index for %s\n", 1115 path); 1116 goto err; 1117 } 1118 1119 if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL)) { 1120 pr_warn("failed to get e_shstrndx from %s\n", path); 1121 goto err; 1122 } 1123 1124 while ((scn = elf_nextscn(elf, scn)) != NULL) { 1125 Elf_Data **field; 1126 GElf_Shdr sh; 1127 char *name; 1128 1129 idx++; 1130 if (gelf_getshdr(scn, &sh) != &sh) { 1131 pr_warn("failed to get section(%d) header from %s\n", 1132 idx, path); 1133 goto err; 1134 } 1135 name = elf_strptr(elf, shstrndx, sh.sh_name); 1136 if (!name) { 1137 pr_warn("failed to get section(%d) name from %s\n", 1138 idx, path); 1139 goto err; 1140 } 1141 1142 if (strcmp(name, BTF_ELF_SEC) == 0) 1143 field = &secs->btf_data; 1144 else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) 1145 field = &secs->btf_ext_data; 1146 else if (strcmp(name, BTF_BASE_ELF_SEC) == 0) 1147 field = &secs->btf_base_data; 1148 else 1149 continue; 1150 1151 data = elf_getdata(scn, 0); 1152 if (!data) { 1153 pr_warn("failed to get section(%d, %s) data from %s\n", 1154 idx, name, path); 1155 goto err; 1156 } 1157 *field = data; 1158 } 1159 1160 return 0; 1161 1162 err: 1163 return -LIBBPF_ERRNO__FORMAT; 1164 } 1165 1166 static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, 1167 struct btf_ext **btf_ext) 1168 { 1169 struct btf_elf_secs secs = {}; 1170 struct btf *dist_base_btf = NULL; 1171 struct btf *btf = NULL; 1172 int err = 0, fd = -1; 1173 Elf *elf = NULL; 1174 1175 if (elf_version(EV_CURRENT) == EV_NONE) { 1176 pr_warn("failed to init libelf for %s\n", path); 1177 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 1178 } 1179 1180 fd = open(path, O_RDONLY | O_CLOEXEC); 1181 if (fd < 0) { 1182 err = -errno; 1183 pr_warn("failed to open %s: %s\n", path, errstr(err)); 1184 return ERR_PTR(err); 1185 } 1186 1187 elf = elf_begin(fd, ELF_C_READ, NULL); 1188 if (!elf) { 1189 err = -LIBBPF_ERRNO__FORMAT; 1190 pr_warn("failed to open %s as ELF file\n", path); 1191 goto done; 1192 } 1193 1194 err = btf_find_elf_sections(elf, path, &secs); 1195 if (err) 1196 goto done; 1197 1198 if (!secs.btf_data) { 1199 pr_warn("failed to find '%s' ELF section in %s\n", BTF_ELF_SEC, path); 1200 err = -ENODATA; 1201 goto done; 1202 } 1203 1204 if (secs.btf_base_data) { 1205 dist_base_btf = btf_new(secs.btf_base_data->d_buf, secs.btf_base_data->d_size, 1206 NULL); 1207 if (IS_ERR(dist_base_btf)) { 1208 err = PTR_ERR(dist_base_btf); 1209 dist_base_btf = NULL; 1210 goto done; 1211 } 1212 } 1213 1214 btf = btf_new(secs.btf_data->d_buf, secs.btf_data->d_size, 1215 dist_base_btf ?: base_btf); 1216 if (IS_ERR(btf)) { 1217 err = PTR_ERR(btf); 1218 goto done; 1219 } 1220 if (dist_base_btf && base_btf) { 1221 err = btf__relocate(btf, base_btf); 1222 if (err) 1223 goto done; 1224 btf__free(dist_base_btf); 1225 dist_base_btf = NULL; 1226 } 1227 1228 if (dist_base_btf) 1229 btf->owns_base = true; 1230 1231 switch (gelf_getclass(elf)) { 1232 case ELFCLASS32: 1233 btf__set_pointer_size(btf, 4); 1234 break; 1235 case ELFCLASS64: 1236 btf__set_pointer_size(btf, 8); 1237 break; 1238 default: 1239 pr_warn("failed to get ELF class (bitness) for %s\n", path); 1240 break; 1241 } 1242 1243 if (btf_ext && secs.btf_ext_data) { 1244 *btf_ext = btf_ext__new(secs.btf_ext_data->d_buf, secs.btf_ext_data->d_size); 1245 if (IS_ERR(*btf_ext)) { 1246 err = PTR_ERR(*btf_ext); 1247 goto done; 1248 } 1249 } else if (btf_ext) { 1250 *btf_ext = NULL; 1251 } 1252 done: 1253 if (elf) 1254 elf_end(elf); 1255 close(fd); 1256 1257 if (!err) 1258 return btf; 1259 1260 if (btf_ext) 1261 btf_ext__free(*btf_ext); 1262 btf__free(dist_base_btf); 1263 btf__free(btf); 1264 1265 return ERR_PTR(err); 1266 } 1267 1268 struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext) 1269 { 1270 return libbpf_ptr(btf_parse_elf(path, NULL, btf_ext)); 1271 } 1272 1273 struct btf *btf__parse_elf_split(const char *path, struct btf *base_btf) 1274 { 1275 return libbpf_ptr(btf_parse_elf(path, base_btf, NULL)); 1276 } 1277 1278 static struct btf *btf_parse_raw(const char *path, struct btf *base_btf) 1279 { 1280 struct btf *btf = NULL; 1281 void *data = NULL; 1282 FILE *f = NULL; 1283 __u16 magic; 1284 int err = 0; 1285 long sz; 1286 1287 f = fopen(path, "rbe"); 1288 if (!f) { 1289 err = -errno; 1290 goto err_out; 1291 } 1292 1293 /* check BTF magic */ 1294 if (fread(&magic, 1, sizeof(magic), f) < sizeof(magic)) { 1295 err = -EIO; 1296 goto err_out; 1297 } 1298 if (magic != BTF_MAGIC && magic != bswap_16(BTF_MAGIC)) { 1299 /* definitely not a raw BTF */ 1300 err = -EPROTO; 1301 goto err_out; 1302 } 1303 1304 /* get file size */ 1305 if (fseek(f, 0, SEEK_END)) { 1306 err = -errno; 1307 goto err_out; 1308 } 1309 sz = ftell(f); 1310 if (sz < 0) { 1311 err = -errno; 1312 goto err_out; 1313 } 1314 /* rewind to the start */ 1315 if (fseek(f, 0, SEEK_SET)) { 1316 err = -errno; 1317 goto err_out; 1318 } 1319 1320 /* pre-alloc memory and read all of BTF data */ 1321 data = malloc(sz); 1322 if (!data) { 1323 err = -ENOMEM; 1324 goto err_out; 1325 } 1326 if (fread(data, 1, sz, f) < sz) { 1327 err = -EIO; 1328 goto err_out; 1329 } 1330 1331 /* finally parse BTF data */ 1332 btf = btf_new(data, sz, base_btf); 1333 1334 err_out: 1335 free(data); 1336 if (f) 1337 fclose(f); 1338 return err ? ERR_PTR(err) : btf; 1339 } 1340 1341 struct btf *btf__parse_raw(const char *path) 1342 { 1343 return libbpf_ptr(btf_parse_raw(path, NULL)); 1344 } 1345 1346 struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf) 1347 { 1348 return libbpf_ptr(btf_parse_raw(path, base_btf)); 1349 } 1350 1351 static struct btf *btf_parse(const char *path, struct btf *base_btf, struct btf_ext **btf_ext) 1352 { 1353 struct btf *btf; 1354 int err; 1355 1356 if (btf_ext) 1357 *btf_ext = NULL; 1358 1359 btf = btf_parse_raw(path, base_btf); 1360 err = libbpf_get_error(btf); 1361 if (!err) 1362 return btf; 1363 if (err != -EPROTO) 1364 return ERR_PTR(err); 1365 return btf_parse_elf(path, base_btf, btf_ext); 1366 } 1367 1368 struct btf *btf__parse(const char *path, struct btf_ext **btf_ext) 1369 { 1370 return libbpf_ptr(btf_parse(path, NULL, btf_ext)); 1371 } 1372 1373 struct btf *btf__parse_split(const char *path, struct btf *base_btf) 1374 { 1375 return libbpf_ptr(btf_parse(path, base_btf, NULL)); 1376 } 1377 1378 static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian); 1379 1380 int btf_load_into_kernel(struct btf *btf, 1381 char *log_buf, size_t log_sz, __u32 log_level, 1382 int token_fd) 1383 { 1384 LIBBPF_OPTS(bpf_btf_load_opts, opts); 1385 __u32 buf_sz = 0, raw_size; 1386 char *buf = NULL, *tmp; 1387 void *raw_data; 1388 int err = 0; 1389 1390 if (btf->fd >= 0) 1391 return libbpf_err(-EEXIST); 1392 if (log_sz && !log_buf) 1393 return libbpf_err(-EINVAL); 1394 1395 /* cache native raw data representation */ 1396 raw_data = btf_get_raw_data(btf, &raw_size, false); 1397 if (!raw_data) { 1398 err = -ENOMEM; 1399 goto done; 1400 } 1401 btf->raw_size = raw_size; 1402 btf->raw_data = raw_data; 1403 1404 retry_load: 1405 /* if log_level is 0, we won't provide log_buf/log_size to the kernel, 1406 * initially. Only if BTF loading fails, we bump log_level to 1 and 1407 * retry, using either auto-allocated or custom log_buf. This way 1408 * non-NULL custom log_buf provides a buffer just in case, but hopes 1409 * for successful load and no need for log_buf. 1410 */ 1411 if (log_level) { 1412 /* if caller didn't provide custom log_buf, we'll keep 1413 * allocating our own progressively bigger buffers for BTF 1414 * verification log 1415 */ 1416 if (!log_buf) { 1417 buf_sz = max((__u32)BPF_LOG_BUF_SIZE, buf_sz * 2); 1418 tmp = realloc(buf, buf_sz); 1419 if (!tmp) { 1420 err = -ENOMEM; 1421 goto done; 1422 } 1423 buf = tmp; 1424 buf[0] = '\0'; 1425 } 1426 1427 opts.log_buf = log_buf ? log_buf : buf; 1428 opts.log_size = log_buf ? log_sz : buf_sz; 1429 opts.log_level = log_level; 1430 } 1431 1432 opts.token_fd = token_fd; 1433 if (token_fd) 1434 opts.btf_flags |= BPF_F_TOKEN_FD; 1435 1436 btf->fd = bpf_btf_load(raw_data, raw_size, &opts); 1437 if (btf->fd < 0) { 1438 /* time to turn on verbose mode and try again */ 1439 if (log_level == 0) { 1440 log_level = 1; 1441 goto retry_load; 1442 } 1443 /* only retry if caller didn't provide custom log_buf, but 1444 * make sure we can never overflow buf_sz 1445 */ 1446 if (!log_buf && errno == ENOSPC && buf_sz <= UINT_MAX / 2) 1447 goto retry_load; 1448 1449 err = -errno; 1450 pr_warn("BTF loading error: %s\n", errstr(err)); 1451 /* don't print out contents of custom log_buf */ 1452 if (!log_buf && buf[0]) 1453 pr_warn("-- BEGIN BTF LOAD LOG ---\n%s\n-- END BTF LOAD LOG --\n", buf); 1454 } 1455 1456 done: 1457 free(buf); 1458 return libbpf_err(err); 1459 } 1460 1461 int btf__load_into_kernel(struct btf *btf) 1462 { 1463 return btf_load_into_kernel(btf, NULL, 0, 0, 0); 1464 } 1465 1466 int btf__fd(const struct btf *btf) 1467 { 1468 return btf->fd; 1469 } 1470 1471 void btf__set_fd(struct btf *btf, int fd) 1472 { 1473 btf->fd = fd; 1474 } 1475 1476 static const void *btf_strs_data(const struct btf *btf) 1477 { 1478 return btf->strs_data ? btf->strs_data : strset__data(btf->strs_set); 1479 } 1480 1481 static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian) 1482 { 1483 struct btf_header *hdr = btf->hdr; 1484 struct btf_type *t; 1485 void *data, *p; 1486 __u32 data_sz; 1487 int i; 1488 1489 data = swap_endian ? btf->raw_data_swapped : btf->raw_data; 1490 if (data) { 1491 *size = btf->raw_size; 1492 return data; 1493 } 1494 1495 data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len; 1496 data = calloc(1, data_sz); 1497 if (!data) 1498 return NULL; 1499 p = data; 1500 1501 memcpy(p, hdr, hdr->hdr_len); 1502 if (swap_endian) 1503 btf_bswap_hdr(p); 1504 p += hdr->hdr_len; 1505 1506 memcpy(p, btf->types_data, hdr->type_len); 1507 if (swap_endian) { 1508 for (i = 0; i < btf->nr_types; i++) { 1509 t = p + btf->type_offs[i]; 1510 /* btf_bswap_type_rest() relies on native t->info, so 1511 * we swap base type info after we swapped all the 1512 * additional information 1513 */ 1514 if (btf_bswap_type_rest(t)) 1515 goto err_out; 1516 btf_bswap_type_base(t); 1517 } 1518 } 1519 p += hdr->type_len; 1520 1521 memcpy(p, btf_strs_data(btf), hdr->str_len); 1522 p += hdr->str_len; 1523 1524 *size = data_sz; 1525 return data; 1526 err_out: 1527 free(data); 1528 return NULL; 1529 } 1530 1531 const void *btf__raw_data(const struct btf *btf_ro, __u32 *size) 1532 { 1533 struct btf *btf = (struct btf *)btf_ro; 1534 __u32 data_sz; 1535 void *data; 1536 1537 data = btf_get_raw_data(btf, &data_sz, btf->swapped_endian); 1538 if (!data) 1539 return errno = ENOMEM, NULL; 1540 1541 btf->raw_size = data_sz; 1542 if (btf->swapped_endian) 1543 btf->raw_data_swapped = data; 1544 else 1545 btf->raw_data = data; 1546 *size = data_sz; 1547 return data; 1548 } 1549 1550 __attribute__((alias("btf__raw_data"))) 1551 const void *btf__get_raw_data(const struct btf *btf, __u32 *size); 1552 1553 const char *btf__str_by_offset(const struct btf *btf, __u32 offset) 1554 { 1555 if (offset < btf->start_str_off) 1556 return btf__str_by_offset(btf->base_btf, offset); 1557 else if (offset - btf->start_str_off < btf->hdr->str_len) 1558 return btf_strs_data(btf) + (offset - btf->start_str_off); 1559 else 1560 return errno = EINVAL, NULL; 1561 } 1562 1563 const char *btf__name_by_offset(const struct btf *btf, __u32 offset) 1564 { 1565 return btf__str_by_offset(btf, offset); 1566 } 1567 1568 struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf) 1569 { 1570 struct bpf_btf_info btf_info; 1571 __u32 len = sizeof(btf_info); 1572 __u32 last_size; 1573 struct btf *btf; 1574 void *ptr; 1575 int err; 1576 1577 /* we won't know btf_size until we call bpf_btf_get_info_by_fd(). so 1578 * let's start with a sane default - 4KiB here - and resize it only if 1579 * bpf_btf_get_info_by_fd() needs a bigger buffer. 1580 */ 1581 last_size = 4096; 1582 ptr = malloc(last_size); 1583 if (!ptr) 1584 return ERR_PTR(-ENOMEM); 1585 1586 memset(&btf_info, 0, sizeof(btf_info)); 1587 btf_info.btf = ptr_to_u64(ptr); 1588 btf_info.btf_size = last_size; 1589 err = bpf_btf_get_info_by_fd(btf_fd, &btf_info, &len); 1590 1591 if (!err && btf_info.btf_size > last_size) { 1592 void *temp_ptr; 1593 1594 last_size = btf_info.btf_size; 1595 temp_ptr = realloc(ptr, last_size); 1596 if (!temp_ptr) { 1597 btf = ERR_PTR(-ENOMEM); 1598 goto exit_free; 1599 } 1600 ptr = temp_ptr; 1601 1602 len = sizeof(btf_info); 1603 memset(&btf_info, 0, sizeof(btf_info)); 1604 btf_info.btf = ptr_to_u64(ptr); 1605 btf_info.btf_size = last_size; 1606 1607 err = bpf_btf_get_info_by_fd(btf_fd, &btf_info, &len); 1608 } 1609 1610 if (err || btf_info.btf_size > last_size) { 1611 btf = err ? ERR_PTR(-errno) : ERR_PTR(-E2BIG); 1612 goto exit_free; 1613 } 1614 1615 btf = btf_new(ptr, btf_info.btf_size, base_btf); 1616 1617 exit_free: 1618 free(ptr); 1619 return btf; 1620 } 1621 1622 struct btf *btf_load_from_kernel(__u32 id, struct btf *base_btf, int token_fd) 1623 { 1624 struct btf *btf; 1625 int btf_fd; 1626 LIBBPF_OPTS(bpf_get_fd_by_id_opts, opts); 1627 1628 if (token_fd) { 1629 opts.open_flags |= BPF_F_TOKEN_FD; 1630 opts.token_fd = token_fd; 1631 } 1632 1633 btf_fd = bpf_btf_get_fd_by_id_opts(id, &opts); 1634 if (btf_fd < 0) 1635 return libbpf_err_ptr(-errno); 1636 1637 btf = btf_get_from_fd(btf_fd, base_btf); 1638 close(btf_fd); 1639 1640 return libbpf_ptr(btf); 1641 } 1642 1643 struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf) 1644 { 1645 return btf_load_from_kernel(id, base_btf, 0); 1646 } 1647 1648 struct btf *btf__load_from_kernel_by_id(__u32 id) 1649 { 1650 return btf__load_from_kernel_by_id_split(id, NULL); 1651 } 1652 1653 static void btf_invalidate_raw_data(struct btf *btf) 1654 { 1655 if (btf->raw_data) { 1656 free(btf->raw_data); 1657 btf->raw_data = NULL; 1658 } 1659 if (btf->raw_data_swapped) { 1660 free(btf->raw_data_swapped); 1661 btf->raw_data_swapped = NULL; 1662 } 1663 } 1664 1665 /* Ensure BTF is ready to be modified (by splitting into a three memory 1666 * regions for header, types, and strings). Also invalidate cached 1667 * raw_data, if any. 1668 */ 1669 static int btf_ensure_modifiable(struct btf *btf) 1670 { 1671 void *hdr, *types; 1672 struct strset *set = NULL; 1673 int err = -ENOMEM; 1674 1675 if (btf_is_modifiable(btf)) { 1676 /* any BTF modification invalidates raw_data */ 1677 btf_invalidate_raw_data(btf); 1678 return 0; 1679 } 1680 1681 /* split raw data into three memory regions */ 1682 hdr = malloc(btf->hdr->hdr_len); 1683 types = malloc(btf->hdr->type_len); 1684 if (!hdr || !types) 1685 goto err_out; 1686 1687 memcpy(hdr, btf->hdr, btf->hdr->hdr_len); 1688 memcpy(types, btf->types_data, btf->hdr->type_len); 1689 1690 /* build lookup index for all strings */ 1691 set = strset__new(BTF_MAX_STR_OFFSET, btf->strs_data, btf->hdr->str_len); 1692 if (IS_ERR(set)) { 1693 err = PTR_ERR(set); 1694 goto err_out; 1695 } 1696 1697 /* only when everything was successful, update internal state */ 1698 btf->hdr = hdr; 1699 btf->types_data = types; 1700 btf->types_data_cap = btf->hdr->type_len; 1701 btf->strs_data = NULL; 1702 btf->strs_set = set; 1703 /* if BTF was created from scratch, all strings are guaranteed to be 1704 * unique and deduplicated 1705 */ 1706 if (btf->hdr->str_len == 0) 1707 btf->strs_deduped = true; 1708 if (!btf->base_btf && btf->hdr->str_len == 1) 1709 btf->strs_deduped = true; 1710 1711 /* invalidate raw_data representation */ 1712 btf_invalidate_raw_data(btf); 1713 1714 return 0; 1715 1716 err_out: 1717 strset__free(set); 1718 free(hdr); 1719 free(types); 1720 return err; 1721 } 1722 1723 /* Find an offset in BTF string section that corresponds to a given string *s*. 1724 * Returns: 1725 * - >0 offset into string section, if string is found; 1726 * - -ENOENT, if string is not in the string section; 1727 * - <0, on any other error. 1728 */ 1729 int btf__find_str(struct btf *btf, const char *s) 1730 { 1731 int off; 1732 1733 if (btf->base_btf) { 1734 off = btf__find_str(btf->base_btf, s); 1735 if (off != -ENOENT) 1736 return off; 1737 } 1738 1739 /* BTF needs to be in a modifiable state to build string lookup index */ 1740 if (btf_ensure_modifiable(btf)) 1741 return libbpf_err(-ENOMEM); 1742 1743 off = strset__find_str(btf->strs_set, s); 1744 if (off < 0) 1745 return libbpf_err(off); 1746 1747 return btf->start_str_off + off; 1748 } 1749 1750 /* Add a string s to the BTF string section. 1751 * Returns: 1752 * - > 0 offset into string section, on success; 1753 * - < 0, on error. 1754 */ 1755 int btf__add_str(struct btf *btf, const char *s) 1756 { 1757 int off; 1758 1759 if (btf->base_btf) { 1760 off = btf__find_str(btf->base_btf, s); 1761 if (off != -ENOENT) 1762 return off; 1763 } 1764 1765 if (btf_ensure_modifiable(btf)) 1766 return libbpf_err(-ENOMEM); 1767 1768 off = strset__add_str(btf->strs_set, s); 1769 if (off < 0) 1770 return libbpf_err(off); 1771 1772 btf->hdr->str_len = strset__data_size(btf->strs_set); 1773 1774 return btf->start_str_off + off; 1775 } 1776 1777 static void *btf_add_type_mem(struct btf *btf, size_t add_sz) 1778 { 1779 return libbpf_add_mem(&btf->types_data, &btf->types_data_cap, 1, 1780 btf->hdr->type_len, UINT_MAX, add_sz); 1781 } 1782 1783 static void btf_type_inc_vlen(struct btf_type *t) 1784 { 1785 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t)); 1786 } 1787 1788 static int btf_commit_type(struct btf *btf, int data_sz) 1789 { 1790 int err; 1791 1792 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); 1793 if (err) 1794 return libbpf_err(err); 1795 1796 btf->hdr->type_len += data_sz; 1797 btf->hdr->str_off += data_sz; 1798 btf->nr_types++; 1799 return btf->start_id + btf->nr_types - 1; 1800 } 1801 1802 struct btf_pipe { 1803 const struct btf *src; 1804 struct btf *dst; 1805 struct hashmap *str_off_map; /* map string offsets from src to dst */ 1806 }; 1807 1808 static int btf_rewrite_str(struct btf_pipe *p, __u32 *str_off) 1809 { 1810 long mapped_off; 1811 int off, err; 1812 1813 if (!*str_off) /* nothing to do for empty strings */ 1814 return 0; 1815 1816 if (p->str_off_map && 1817 hashmap__find(p->str_off_map, *str_off, &mapped_off)) { 1818 *str_off = mapped_off; 1819 return 0; 1820 } 1821 1822 off = btf__add_str(p->dst, btf__str_by_offset(p->src, *str_off)); 1823 if (off < 0) 1824 return off; 1825 1826 /* Remember string mapping from src to dst. It avoids 1827 * performing expensive string comparisons. 1828 */ 1829 if (p->str_off_map) { 1830 err = hashmap__append(p->str_off_map, *str_off, off); 1831 if (err) 1832 return err; 1833 } 1834 1835 *str_off = off; 1836 return 0; 1837 } 1838 1839 static int btf_add_type(struct btf_pipe *p, const struct btf_type *src_type) 1840 { 1841 struct btf_field_iter it; 1842 struct btf_type *t; 1843 __u32 *str_off; 1844 int sz, err; 1845 1846 sz = btf_type_size(src_type); 1847 if (sz < 0) 1848 return libbpf_err(sz); 1849 1850 /* deconstruct BTF, if necessary, and invalidate raw_data */ 1851 if (btf_ensure_modifiable(p->dst)) 1852 return libbpf_err(-ENOMEM); 1853 1854 t = btf_add_type_mem(p->dst, sz); 1855 if (!t) 1856 return libbpf_err(-ENOMEM); 1857 1858 memcpy(t, src_type, sz); 1859 1860 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS); 1861 if (err) 1862 return libbpf_err(err); 1863 1864 while ((str_off = btf_field_iter_next(&it))) { 1865 err = btf_rewrite_str(p, str_off); 1866 if (err) 1867 return libbpf_err(err); 1868 } 1869 1870 return btf_commit_type(p->dst, sz); 1871 } 1872 1873 int btf__add_type(struct btf *btf, const struct btf *src_btf, const struct btf_type *src_type) 1874 { 1875 struct btf_pipe p = { .src = src_btf, .dst = btf }; 1876 1877 return btf_add_type(&p, src_type); 1878 } 1879 1880 static size_t btf_dedup_identity_hash_fn(long key, void *ctx); 1881 static bool btf_dedup_equal_fn(long k1, long k2, void *ctx); 1882 1883 int btf__add_btf(struct btf *btf, const struct btf *src_btf) 1884 { 1885 struct btf_pipe p = { .src = src_btf, .dst = btf }; 1886 int data_sz, sz, cnt, i, err, old_strs_len; 1887 __u32 *off; 1888 void *t; 1889 1890 /* appending split BTF isn't supported yet */ 1891 if (src_btf->base_btf) 1892 return libbpf_err(-ENOTSUP); 1893 1894 /* deconstruct BTF, if necessary, and invalidate raw_data */ 1895 if (btf_ensure_modifiable(btf)) 1896 return libbpf_err(-ENOMEM); 1897 1898 /* remember original strings section size if we have to roll back 1899 * partial strings section changes 1900 */ 1901 old_strs_len = btf->hdr->str_len; 1902 1903 data_sz = src_btf->hdr->type_len; 1904 cnt = btf__type_cnt(src_btf) - 1; 1905 1906 /* pre-allocate enough memory for new types */ 1907 t = btf_add_type_mem(btf, data_sz); 1908 if (!t) 1909 return libbpf_err(-ENOMEM); 1910 1911 /* pre-allocate enough memory for type offset index for new types */ 1912 off = btf_add_type_offs_mem(btf, cnt); 1913 if (!off) 1914 return libbpf_err(-ENOMEM); 1915 1916 /* Map the string offsets from src_btf to the offsets from btf to improve performance */ 1917 p.str_off_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL); 1918 if (IS_ERR(p.str_off_map)) 1919 return libbpf_err(-ENOMEM); 1920 1921 /* bulk copy types data for all types from src_btf */ 1922 memcpy(t, src_btf->types_data, data_sz); 1923 1924 for (i = 0; i < cnt; i++) { 1925 struct btf_field_iter it; 1926 __u32 *type_id, *str_off; 1927 1928 sz = btf_type_size(t); 1929 if (sz < 0) { 1930 /* unlikely, has to be corrupted src_btf */ 1931 err = sz; 1932 goto err_out; 1933 } 1934 1935 /* fill out type ID to type offset mapping for lookups by type ID */ 1936 *off = t - btf->types_data; 1937 1938 /* add, dedup, and remap strings referenced by this BTF type */ 1939 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS); 1940 if (err) 1941 goto err_out; 1942 while ((str_off = btf_field_iter_next(&it))) { 1943 err = btf_rewrite_str(&p, str_off); 1944 if (err) 1945 goto err_out; 1946 } 1947 1948 /* remap all type IDs referenced from this BTF type */ 1949 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS); 1950 if (err) 1951 goto err_out; 1952 1953 while ((type_id = btf_field_iter_next(&it))) { 1954 if (!*type_id) /* nothing to do for VOID references */ 1955 continue; 1956 1957 /* we haven't updated btf's type count yet, so 1958 * btf->start_id + btf->nr_types - 1 is the type ID offset we should 1959 * add to all newly added BTF types 1960 */ 1961 *type_id += btf->start_id + btf->nr_types - 1; 1962 } 1963 1964 /* go to next type data and type offset index entry */ 1965 t += sz; 1966 off++; 1967 } 1968 1969 /* Up until now any of the copied type data was effectively invisible, 1970 * so if we exited early before this point due to error, BTF would be 1971 * effectively unmodified. There would be extra internal memory 1972 * pre-allocated, but it would not be available for querying. But now 1973 * that we've copied and rewritten all the data successfully, we can 1974 * update type count and various internal offsets and sizes to 1975 * "commit" the changes and made them visible to the outside world. 1976 */ 1977 btf->hdr->type_len += data_sz; 1978 btf->hdr->str_off += data_sz; 1979 btf->nr_types += cnt; 1980 1981 hashmap__free(p.str_off_map); 1982 1983 /* return type ID of the first added BTF type */ 1984 return btf->start_id + btf->nr_types - cnt; 1985 err_out: 1986 /* zero out preallocated memory as if it was just allocated with 1987 * libbpf_add_mem() 1988 */ 1989 memset(btf->types_data + btf->hdr->type_len, 0, data_sz); 1990 memset(btf->strs_data + old_strs_len, 0, btf->hdr->str_len - old_strs_len); 1991 1992 /* and now restore original strings section size; types data size 1993 * wasn't modified, so doesn't need restoring, see big comment above 1994 */ 1995 btf->hdr->str_len = old_strs_len; 1996 1997 hashmap__free(p.str_off_map); 1998 1999 return libbpf_err(err); 2000 } 2001 2002 /* 2003 * Append new BTF_KIND_INT type with: 2004 * - *name* - non-empty, non-NULL type name; 2005 * - *sz* - power-of-2 (1, 2, 4, ..) size of the type, in bytes; 2006 * - encoding is a combination of BTF_INT_SIGNED, BTF_INT_CHAR, BTF_INT_BOOL. 2007 * Returns: 2008 * - >0, type ID of newly added BTF type; 2009 * - <0, on error. 2010 */ 2011 int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding) 2012 { 2013 struct btf_type *t; 2014 int sz, name_off; 2015 2016 /* non-empty name */ 2017 if (!name || !name[0]) 2018 return libbpf_err(-EINVAL); 2019 /* byte_sz must be power of 2 */ 2020 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16) 2021 return libbpf_err(-EINVAL); 2022 if (encoding & ~(BTF_INT_SIGNED | BTF_INT_CHAR | BTF_INT_BOOL)) 2023 return libbpf_err(-EINVAL); 2024 2025 /* deconstruct BTF, if necessary, and invalidate raw_data */ 2026 if (btf_ensure_modifiable(btf)) 2027 return libbpf_err(-ENOMEM); 2028 2029 sz = sizeof(struct btf_type) + sizeof(int); 2030 t = btf_add_type_mem(btf, sz); 2031 if (!t) 2032 return libbpf_err(-ENOMEM); 2033 2034 /* if something goes wrong later, we might end up with an extra string, 2035 * but that shouldn't be a problem, because BTF can't be constructed 2036 * completely anyway and will most probably be just discarded 2037 */ 2038 name_off = btf__add_str(btf, name); 2039 if (name_off < 0) 2040 return name_off; 2041 2042 t->name_off = name_off; 2043 t->info = btf_type_info(BTF_KIND_INT, 0, 0); 2044 t->size = byte_sz; 2045 /* set INT info, we don't allow setting legacy bit offset/size */ 2046 *(__u32 *)(t + 1) = (encoding << 24) | (byte_sz * 8); 2047 2048 return btf_commit_type(btf, sz); 2049 } 2050 2051 /* 2052 * Append new BTF_KIND_FLOAT type with: 2053 * - *name* - non-empty, non-NULL type name; 2054 * - *sz* - size of the type, in bytes; 2055 * Returns: 2056 * - >0, type ID of newly added BTF type; 2057 * - <0, on error. 2058 */ 2059 int btf__add_float(struct btf *btf, const char *name, size_t byte_sz) 2060 { 2061 struct btf_type *t; 2062 int sz, name_off; 2063 2064 /* non-empty name */ 2065 if (!name || !name[0]) 2066 return libbpf_err(-EINVAL); 2067 2068 /* byte_sz must be one of the explicitly allowed values */ 2069 if (byte_sz != 2 && byte_sz != 4 && byte_sz != 8 && byte_sz != 12 && 2070 byte_sz != 16) 2071 return libbpf_err(-EINVAL); 2072 2073 if (btf_ensure_modifiable(btf)) 2074 return libbpf_err(-ENOMEM); 2075 2076 sz = sizeof(struct btf_type); 2077 t = btf_add_type_mem(btf, sz); 2078 if (!t) 2079 return libbpf_err(-ENOMEM); 2080 2081 name_off = btf__add_str(btf, name); 2082 if (name_off < 0) 2083 return name_off; 2084 2085 t->name_off = name_off; 2086 t->info = btf_type_info(BTF_KIND_FLOAT, 0, 0); 2087 t->size = byte_sz; 2088 2089 return btf_commit_type(btf, sz); 2090 } 2091 2092 /* it's completely legal to append BTF types with type IDs pointing forward to 2093 * types that haven't been appended yet, so we only make sure that id looks 2094 * sane, we can't guarantee that ID will always be valid 2095 */ 2096 static int validate_type_id(int id) 2097 { 2098 if (id < 0 || id > BTF_MAX_NR_TYPES) 2099 return -EINVAL; 2100 return 0; 2101 } 2102 2103 /* generic append function for PTR, TYPEDEF, CONST/VOLATILE/RESTRICT */ 2104 static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref_type_id, int kflag) 2105 { 2106 struct btf_type *t; 2107 int sz, name_off = 0; 2108 2109 if (validate_type_id(ref_type_id)) 2110 return libbpf_err(-EINVAL); 2111 2112 if (btf_ensure_modifiable(btf)) 2113 return libbpf_err(-ENOMEM); 2114 2115 sz = sizeof(struct btf_type); 2116 t = btf_add_type_mem(btf, sz); 2117 if (!t) 2118 return libbpf_err(-ENOMEM); 2119 2120 if (name && name[0]) { 2121 name_off = btf__add_str(btf, name); 2122 if (name_off < 0) 2123 return name_off; 2124 } 2125 2126 t->name_off = name_off; 2127 t->info = btf_type_info(kind, 0, kflag); 2128 t->type = ref_type_id; 2129 2130 return btf_commit_type(btf, sz); 2131 } 2132 2133 /* 2134 * Append new BTF_KIND_PTR type with: 2135 * - *ref_type_id* - referenced type ID, it might not exist yet; 2136 * Returns: 2137 * - >0, type ID of newly added BTF type; 2138 * - <0, on error. 2139 */ 2140 int btf__add_ptr(struct btf *btf, int ref_type_id) 2141 { 2142 return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id, 0); 2143 } 2144 2145 /* 2146 * Append new BTF_KIND_ARRAY type with: 2147 * - *index_type_id* - type ID of the type describing array index; 2148 * - *elem_type_id* - type ID of the type describing array element; 2149 * - *nr_elems* - the size of the array; 2150 * Returns: 2151 * - >0, type ID of newly added BTF type; 2152 * - <0, on error. 2153 */ 2154 int btf__add_array(struct btf *btf, int index_type_id, int elem_type_id, __u32 nr_elems) 2155 { 2156 struct btf_type *t; 2157 struct btf_array *a; 2158 int sz; 2159 2160 if (validate_type_id(index_type_id) || validate_type_id(elem_type_id)) 2161 return libbpf_err(-EINVAL); 2162 2163 if (btf_ensure_modifiable(btf)) 2164 return libbpf_err(-ENOMEM); 2165 2166 sz = sizeof(struct btf_type) + sizeof(struct btf_array); 2167 t = btf_add_type_mem(btf, sz); 2168 if (!t) 2169 return libbpf_err(-ENOMEM); 2170 2171 t->name_off = 0; 2172 t->info = btf_type_info(BTF_KIND_ARRAY, 0, 0); 2173 t->size = 0; 2174 2175 a = btf_array(t); 2176 a->type = elem_type_id; 2177 a->index_type = index_type_id; 2178 a->nelems = nr_elems; 2179 2180 return btf_commit_type(btf, sz); 2181 } 2182 2183 /* generic STRUCT/UNION append function */ 2184 static int btf_add_composite(struct btf *btf, int kind, const char *name, __u32 bytes_sz) 2185 { 2186 struct btf_type *t; 2187 int sz, name_off = 0; 2188 2189 if (btf_ensure_modifiable(btf)) 2190 return libbpf_err(-ENOMEM); 2191 2192 sz = sizeof(struct btf_type); 2193 t = btf_add_type_mem(btf, sz); 2194 if (!t) 2195 return libbpf_err(-ENOMEM); 2196 2197 if (name && name[0]) { 2198 name_off = btf__add_str(btf, name); 2199 if (name_off < 0) 2200 return name_off; 2201 } 2202 2203 /* start out with vlen=0 and no kflag; this will be adjusted when 2204 * adding each member 2205 */ 2206 t->name_off = name_off; 2207 t->info = btf_type_info(kind, 0, 0); 2208 t->size = bytes_sz; 2209 2210 return btf_commit_type(btf, sz); 2211 } 2212 2213 /* 2214 * Append new BTF_KIND_STRUCT type with: 2215 * - *name* - name of the struct, can be NULL or empty for anonymous structs; 2216 * - *byte_sz* - size of the struct, in bytes; 2217 * 2218 * Struct initially has no fields in it. Fields can be added by 2219 * btf__add_field() right after btf__add_struct() succeeds. 2220 * 2221 * Returns: 2222 * - >0, type ID of newly added BTF type; 2223 * - <0, on error. 2224 */ 2225 int btf__add_struct(struct btf *btf, const char *name, __u32 byte_sz) 2226 { 2227 return btf_add_composite(btf, BTF_KIND_STRUCT, name, byte_sz); 2228 } 2229 2230 /* 2231 * Append new BTF_KIND_UNION type with: 2232 * - *name* - name of the union, can be NULL or empty for anonymous union; 2233 * - *byte_sz* - size of the union, in bytes; 2234 * 2235 * Union initially has no fields in it. Fields can be added by 2236 * btf__add_field() right after btf__add_union() succeeds. All fields 2237 * should have *bit_offset* of 0. 2238 * 2239 * Returns: 2240 * - >0, type ID of newly added BTF type; 2241 * - <0, on error. 2242 */ 2243 int btf__add_union(struct btf *btf, const char *name, __u32 byte_sz) 2244 { 2245 return btf_add_composite(btf, BTF_KIND_UNION, name, byte_sz); 2246 } 2247 2248 static struct btf_type *btf_last_type(struct btf *btf) 2249 { 2250 return btf_type_by_id(btf, btf__type_cnt(btf) - 1); 2251 } 2252 2253 /* 2254 * Append new field for the current STRUCT/UNION type with: 2255 * - *name* - name of the field, can be NULL or empty for anonymous field; 2256 * - *type_id* - type ID for the type describing field type; 2257 * - *bit_offset* - bit offset of the start of the field within struct/union; 2258 * - *bit_size* - bit size of a bitfield, 0 for non-bitfield fields; 2259 * Returns: 2260 * - 0, on success; 2261 * - <0, on error. 2262 */ 2263 int btf__add_field(struct btf *btf, const char *name, int type_id, 2264 __u32 bit_offset, __u32 bit_size) 2265 { 2266 struct btf_type *t; 2267 struct btf_member *m; 2268 bool is_bitfield; 2269 int sz, name_off = 0; 2270 2271 /* last type should be union/struct */ 2272 if (btf->nr_types == 0) 2273 return libbpf_err(-EINVAL); 2274 t = btf_last_type(btf); 2275 if (!btf_is_composite(t)) 2276 return libbpf_err(-EINVAL); 2277 2278 if (validate_type_id(type_id)) 2279 return libbpf_err(-EINVAL); 2280 /* best-effort bit field offset/size enforcement */ 2281 is_bitfield = bit_size || (bit_offset % 8 != 0); 2282 if (is_bitfield && (bit_size == 0 || bit_size > 255 || bit_offset > 0xffffff)) 2283 return libbpf_err(-EINVAL); 2284 2285 /* only offset 0 is allowed for unions */ 2286 if (btf_is_union(t) && bit_offset) 2287 return libbpf_err(-EINVAL); 2288 2289 /* decompose and invalidate raw data */ 2290 if (btf_ensure_modifiable(btf)) 2291 return libbpf_err(-ENOMEM); 2292 2293 sz = sizeof(struct btf_member); 2294 m = btf_add_type_mem(btf, sz); 2295 if (!m) 2296 return libbpf_err(-ENOMEM); 2297 2298 if (name && name[0]) { 2299 name_off = btf__add_str(btf, name); 2300 if (name_off < 0) 2301 return name_off; 2302 } 2303 2304 m->name_off = name_off; 2305 m->type = type_id; 2306 m->offset = bit_offset | (bit_size << 24); 2307 2308 /* btf_add_type_mem can invalidate t pointer */ 2309 t = btf_last_type(btf); 2310 /* update parent type's vlen and kflag */ 2311 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t)); 2312 2313 btf->hdr->type_len += sz; 2314 btf->hdr->str_off += sz; 2315 return 0; 2316 } 2317 2318 static int btf_add_enum_common(struct btf *btf, const char *name, __u32 byte_sz, 2319 bool is_signed, __u8 kind) 2320 { 2321 struct btf_type *t; 2322 int sz, name_off = 0; 2323 2324 /* byte_sz must be power of 2 */ 2325 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8) 2326 return libbpf_err(-EINVAL); 2327 2328 if (btf_ensure_modifiable(btf)) 2329 return libbpf_err(-ENOMEM); 2330 2331 sz = sizeof(struct btf_type); 2332 t = btf_add_type_mem(btf, sz); 2333 if (!t) 2334 return libbpf_err(-ENOMEM); 2335 2336 if (name && name[0]) { 2337 name_off = btf__add_str(btf, name); 2338 if (name_off < 0) 2339 return name_off; 2340 } 2341 2342 /* start out with vlen=0; it will be adjusted when adding enum values */ 2343 t->name_off = name_off; 2344 t->info = btf_type_info(kind, 0, is_signed); 2345 t->size = byte_sz; 2346 2347 return btf_commit_type(btf, sz); 2348 } 2349 2350 /* 2351 * Append new BTF_KIND_ENUM type with: 2352 * - *name* - name of the enum, can be NULL or empty for anonymous enums; 2353 * - *byte_sz* - size of the enum, in bytes. 2354 * 2355 * Enum initially has no enum values in it (and corresponds to enum forward 2356 * declaration). Enumerator values can be added by btf__add_enum_value() 2357 * immediately after btf__add_enum() succeeds. 2358 * 2359 * Returns: 2360 * - >0, type ID of newly added BTF type; 2361 * - <0, on error. 2362 */ 2363 int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz) 2364 { 2365 /* 2366 * set the signedness to be unsigned, it will change to signed 2367 * if any later enumerator is negative. 2368 */ 2369 return btf_add_enum_common(btf, name, byte_sz, false, BTF_KIND_ENUM); 2370 } 2371 2372 /* 2373 * Append new enum value for the current ENUM type with: 2374 * - *name* - name of the enumerator value, can't be NULL or empty; 2375 * - *value* - integer value corresponding to enum value *name*; 2376 * Returns: 2377 * - 0, on success; 2378 * - <0, on error. 2379 */ 2380 int btf__add_enum_value(struct btf *btf, const char *name, __s64 value) 2381 { 2382 struct btf_type *t; 2383 struct btf_enum *v; 2384 int sz, name_off; 2385 2386 /* last type should be BTF_KIND_ENUM */ 2387 if (btf->nr_types == 0) 2388 return libbpf_err(-EINVAL); 2389 t = btf_last_type(btf); 2390 if (!btf_is_enum(t)) 2391 return libbpf_err(-EINVAL); 2392 2393 /* non-empty name */ 2394 if (!name || !name[0]) 2395 return libbpf_err(-EINVAL); 2396 if (value < INT_MIN || value > UINT_MAX) 2397 return libbpf_err(-E2BIG); 2398 2399 /* decompose and invalidate raw data */ 2400 if (btf_ensure_modifiable(btf)) 2401 return libbpf_err(-ENOMEM); 2402 2403 sz = sizeof(struct btf_enum); 2404 v = btf_add_type_mem(btf, sz); 2405 if (!v) 2406 return libbpf_err(-ENOMEM); 2407 2408 name_off = btf__add_str(btf, name); 2409 if (name_off < 0) 2410 return name_off; 2411 2412 v->name_off = name_off; 2413 v->val = value; 2414 2415 /* update parent type's vlen */ 2416 t = btf_last_type(btf); 2417 btf_type_inc_vlen(t); 2418 2419 /* if negative value, set signedness to signed */ 2420 if (value < 0) 2421 t->info = btf_type_info(btf_kind(t), btf_vlen(t), true); 2422 2423 btf->hdr->type_len += sz; 2424 btf->hdr->str_off += sz; 2425 return 0; 2426 } 2427 2428 /* 2429 * Append new BTF_KIND_ENUM64 type with: 2430 * - *name* - name of the enum, can be NULL or empty for anonymous enums; 2431 * - *byte_sz* - size of the enum, in bytes. 2432 * - *is_signed* - whether the enum values are signed or not; 2433 * 2434 * Enum initially has no enum values in it (and corresponds to enum forward 2435 * declaration). Enumerator values can be added by btf__add_enum64_value() 2436 * immediately after btf__add_enum64() succeeds. 2437 * 2438 * Returns: 2439 * - >0, type ID of newly added BTF type; 2440 * - <0, on error. 2441 */ 2442 int btf__add_enum64(struct btf *btf, const char *name, __u32 byte_sz, 2443 bool is_signed) 2444 { 2445 return btf_add_enum_common(btf, name, byte_sz, is_signed, 2446 BTF_KIND_ENUM64); 2447 } 2448 2449 /* 2450 * Append new enum value for the current ENUM64 type with: 2451 * - *name* - name of the enumerator value, can't be NULL or empty; 2452 * - *value* - integer value corresponding to enum value *name*; 2453 * Returns: 2454 * - 0, on success; 2455 * - <0, on error. 2456 */ 2457 int btf__add_enum64_value(struct btf *btf, const char *name, __u64 value) 2458 { 2459 struct btf_enum64 *v; 2460 struct btf_type *t; 2461 int sz, name_off; 2462 2463 /* last type should be BTF_KIND_ENUM64 */ 2464 if (btf->nr_types == 0) 2465 return libbpf_err(-EINVAL); 2466 t = btf_last_type(btf); 2467 if (!btf_is_enum64(t)) 2468 return libbpf_err(-EINVAL); 2469 2470 /* non-empty name */ 2471 if (!name || !name[0]) 2472 return libbpf_err(-EINVAL); 2473 2474 /* decompose and invalidate raw data */ 2475 if (btf_ensure_modifiable(btf)) 2476 return libbpf_err(-ENOMEM); 2477 2478 sz = sizeof(struct btf_enum64); 2479 v = btf_add_type_mem(btf, sz); 2480 if (!v) 2481 return libbpf_err(-ENOMEM); 2482 2483 name_off = btf__add_str(btf, name); 2484 if (name_off < 0) 2485 return name_off; 2486 2487 v->name_off = name_off; 2488 v->val_lo32 = (__u32)value; 2489 v->val_hi32 = value >> 32; 2490 2491 /* update parent type's vlen */ 2492 t = btf_last_type(btf); 2493 btf_type_inc_vlen(t); 2494 2495 btf->hdr->type_len += sz; 2496 btf->hdr->str_off += sz; 2497 return 0; 2498 } 2499 2500 /* 2501 * Append new BTF_KIND_FWD type with: 2502 * - *name*, non-empty/non-NULL name; 2503 * - *fwd_kind*, kind of forward declaration, one of BTF_FWD_STRUCT, 2504 * BTF_FWD_UNION, or BTF_FWD_ENUM; 2505 * Returns: 2506 * - >0, type ID of newly added BTF type; 2507 * - <0, on error. 2508 */ 2509 int btf__add_fwd(struct btf *btf, const char *name, enum btf_fwd_kind fwd_kind) 2510 { 2511 if (!name || !name[0]) 2512 return libbpf_err(-EINVAL); 2513 2514 switch (fwd_kind) { 2515 case BTF_FWD_STRUCT: 2516 case BTF_FWD_UNION: { 2517 struct btf_type *t; 2518 int id; 2519 2520 id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0, 0); 2521 if (id <= 0) 2522 return id; 2523 t = btf_type_by_id(btf, id); 2524 t->info = btf_type_info(BTF_KIND_FWD, 0, fwd_kind == BTF_FWD_UNION); 2525 return id; 2526 } 2527 case BTF_FWD_ENUM: 2528 /* enum forward in BTF currently is just an enum with no enum 2529 * values; we also assume a standard 4-byte size for it 2530 */ 2531 return btf__add_enum(btf, name, sizeof(int)); 2532 default: 2533 return libbpf_err(-EINVAL); 2534 } 2535 } 2536 2537 /* 2538 * Append new BTF_KING_TYPEDEF type with: 2539 * - *name*, non-empty/non-NULL name; 2540 * - *ref_type_id* - referenced type ID, it might not exist yet; 2541 * Returns: 2542 * - >0, type ID of newly added BTF type; 2543 * - <0, on error. 2544 */ 2545 int btf__add_typedef(struct btf *btf, const char *name, int ref_type_id) 2546 { 2547 if (!name || !name[0]) 2548 return libbpf_err(-EINVAL); 2549 2550 return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id, 0); 2551 } 2552 2553 /* 2554 * Append new BTF_KIND_VOLATILE type with: 2555 * - *ref_type_id* - referenced type ID, it might not exist yet; 2556 * Returns: 2557 * - >0, type ID of newly added BTF type; 2558 * - <0, on error. 2559 */ 2560 int btf__add_volatile(struct btf *btf, int ref_type_id) 2561 { 2562 return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id, 0); 2563 } 2564 2565 /* 2566 * Append new BTF_KIND_CONST type with: 2567 * - *ref_type_id* - referenced type ID, it might not exist yet; 2568 * Returns: 2569 * - >0, type ID of newly added BTF type; 2570 * - <0, on error. 2571 */ 2572 int btf__add_const(struct btf *btf, int ref_type_id) 2573 { 2574 return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id, 0); 2575 } 2576 2577 /* 2578 * Append new BTF_KIND_RESTRICT type with: 2579 * - *ref_type_id* - referenced type ID, it might not exist yet; 2580 * Returns: 2581 * - >0, type ID of newly added BTF type; 2582 * - <0, on error. 2583 */ 2584 int btf__add_restrict(struct btf *btf, int ref_type_id) 2585 { 2586 return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id, 0); 2587 } 2588 2589 /* 2590 * Append new BTF_KIND_TYPE_TAG type with: 2591 * - *value*, non-empty/non-NULL tag value; 2592 * - *ref_type_id* - referenced type ID, it might not exist yet; 2593 * Returns: 2594 * - >0, type ID of newly added BTF type; 2595 * - <0, on error. 2596 */ 2597 int btf__add_type_tag(struct btf *btf, const char *value, int ref_type_id) 2598 { 2599 if (!value || !value[0]) 2600 return libbpf_err(-EINVAL); 2601 2602 return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id, 0); 2603 } 2604 2605 /* 2606 * Append new BTF_KIND_TYPE_TAG type with: 2607 * - *value*, non-empty/non-NULL tag value; 2608 * - *ref_type_id* - referenced type ID, it might not exist yet; 2609 * Set info->kflag to 1, indicating this tag is an __attribute__ 2610 * Returns: 2611 * - >0, type ID of newly added BTF type; 2612 * - <0, on error. 2613 */ 2614 int btf__add_type_attr(struct btf *btf, const char *value, int ref_type_id) 2615 { 2616 if (!value || !value[0]) 2617 return libbpf_err(-EINVAL); 2618 2619 return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id, 1); 2620 } 2621 2622 /* 2623 * Append new BTF_KIND_FUNC type with: 2624 * - *name*, non-empty/non-NULL name; 2625 * - *proto_type_id* - FUNC_PROTO's type ID, it might not exist yet; 2626 * Returns: 2627 * - >0, type ID of newly added BTF type; 2628 * - <0, on error. 2629 */ 2630 int btf__add_func(struct btf *btf, const char *name, 2631 enum btf_func_linkage linkage, int proto_type_id) 2632 { 2633 int id; 2634 2635 if (!name || !name[0]) 2636 return libbpf_err(-EINVAL); 2637 if (linkage != BTF_FUNC_STATIC && linkage != BTF_FUNC_GLOBAL && 2638 linkage != BTF_FUNC_EXTERN) 2639 return libbpf_err(-EINVAL); 2640 2641 id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id, 0); 2642 if (id > 0) { 2643 struct btf_type *t = btf_type_by_id(btf, id); 2644 2645 t->info = btf_type_info(BTF_KIND_FUNC, linkage, 0); 2646 } 2647 return libbpf_err(id); 2648 } 2649 2650 /* 2651 * Append new BTF_KIND_FUNC_PROTO with: 2652 * - *ret_type_id* - type ID for return result of a function. 2653 * 2654 * Function prototype initially has no arguments, but they can be added by 2655 * btf__add_func_param() one by one, immediately after 2656 * btf__add_func_proto() succeeded. 2657 * 2658 * Returns: 2659 * - >0, type ID of newly added BTF type; 2660 * - <0, on error. 2661 */ 2662 int btf__add_func_proto(struct btf *btf, int ret_type_id) 2663 { 2664 struct btf_type *t; 2665 int sz; 2666 2667 if (validate_type_id(ret_type_id)) 2668 return libbpf_err(-EINVAL); 2669 2670 if (btf_ensure_modifiable(btf)) 2671 return libbpf_err(-ENOMEM); 2672 2673 sz = sizeof(struct btf_type); 2674 t = btf_add_type_mem(btf, sz); 2675 if (!t) 2676 return libbpf_err(-ENOMEM); 2677 2678 /* start out with vlen=0; this will be adjusted when adding enum 2679 * values, if necessary 2680 */ 2681 t->name_off = 0; 2682 t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0); 2683 t->type = ret_type_id; 2684 2685 return btf_commit_type(btf, sz); 2686 } 2687 2688 /* 2689 * Append new function parameter for current FUNC_PROTO type with: 2690 * - *name* - parameter name, can be NULL or empty; 2691 * - *type_id* - type ID describing the type of the parameter. 2692 * Returns: 2693 * - 0, on success; 2694 * - <0, on error. 2695 */ 2696 int btf__add_func_param(struct btf *btf, const char *name, int type_id) 2697 { 2698 struct btf_type *t; 2699 struct btf_param *p; 2700 int sz, name_off = 0; 2701 2702 if (validate_type_id(type_id)) 2703 return libbpf_err(-EINVAL); 2704 2705 /* last type should be BTF_KIND_FUNC_PROTO */ 2706 if (btf->nr_types == 0) 2707 return libbpf_err(-EINVAL); 2708 t = btf_last_type(btf); 2709 if (!btf_is_func_proto(t)) 2710 return libbpf_err(-EINVAL); 2711 2712 /* decompose and invalidate raw data */ 2713 if (btf_ensure_modifiable(btf)) 2714 return libbpf_err(-ENOMEM); 2715 2716 sz = sizeof(struct btf_param); 2717 p = btf_add_type_mem(btf, sz); 2718 if (!p) 2719 return libbpf_err(-ENOMEM); 2720 2721 if (name && name[0]) { 2722 name_off = btf__add_str(btf, name); 2723 if (name_off < 0) 2724 return name_off; 2725 } 2726 2727 p->name_off = name_off; 2728 p->type = type_id; 2729 2730 /* update parent type's vlen */ 2731 t = btf_last_type(btf); 2732 btf_type_inc_vlen(t); 2733 2734 btf->hdr->type_len += sz; 2735 btf->hdr->str_off += sz; 2736 return 0; 2737 } 2738 2739 /* 2740 * Append new BTF_KIND_VAR type with: 2741 * - *name* - non-empty/non-NULL name; 2742 * - *linkage* - variable linkage, one of BTF_VAR_STATIC, 2743 * BTF_VAR_GLOBAL_ALLOCATED, or BTF_VAR_GLOBAL_EXTERN; 2744 * - *type_id* - type ID of the type describing the type of the variable. 2745 * Returns: 2746 * - >0, type ID of newly added BTF type; 2747 * - <0, on error. 2748 */ 2749 int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id) 2750 { 2751 struct btf_type *t; 2752 struct btf_var *v; 2753 int sz, name_off; 2754 2755 /* non-empty name */ 2756 if (!name || !name[0]) 2757 return libbpf_err(-EINVAL); 2758 if (linkage != BTF_VAR_STATIC && linkage != BTF_VAR_GLOBAL_ALLOCATED && 2759 linkage != BTF_VAR_GLOBAL_EXTERN) 2760 return libbpf_err(-EINVAL); 2761 if (validate_type_id(type_id)) 2762 return libbpf_err(-EINVAL); 2763 2764 /* deconstruct BTF, if necessary, and invalidate raw_data */ 2765 if (btf_ensure_modifiable(btf)) 2766 return libbpf_err(-ENOMEM); 2767 2768 sz = sizeof(struct btf_type) + sizeof(struct btf_var); 2769 t = btf_add_type_mem(btf, sz); 2770 if (!t) 2771 return libbpf_err(-ENOMEM); 2772 2773 name_off = btf__add_str(btf, name); 2774 if (name_off < 0) 2775 return name_off; 2776 2777 t->name_off = name_off; 2778 t->info = btf_type_info(BTF_KIND_VAR, 0, 0); 2779 t->type = type_id; 2780 2781 v = btf_var(t); 2782 v->linkage = linkage; 2783 2784 return btf_commit_type(btf, sz); 2785 } 2786 2787 /* 2788 * Append new BTF_KIND_DATASEC type with: 2789 * - *name* - non-empty/non-NULL name; 2790 * - *byte_sz* - data section size, in bytes. 2791 * 2792 * Data section is initially empty. Variables info can be added with 2793 * btf__add_datasec_var_info() calls, after btf__add_datasec() succeeds. 2794 * 2795 * Returns: 2796 * - >0, type ID of newly added BTF type; 2797 * - <0, on error. 2798 */ 2799 int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz) 2800 { 2801 struct btf_type *t; 2802 int sz, name_off; 2803 2804 /* non-empty name */ 2805 if (!name || !name[0]) 2806 return libbpf_err(-EINVAL); 2807 2808 if (btf_ensure_modifiable(btf)) 2809 return libbpf_err(-ENOMEM); 2810 2811 sz = sizeof(struct btf_type); 2812 t = btf_add_type_mem(btf, sz); 2813 if (!t) 2814 return libbpf_err(-ENOMEM); 2815 2816 name_off = btf__add_str(btf, name); 2817 if (name_off < 0) 2818 return name_off; 2819 2820 /* start with vlen=0, which will be update as var_secinfos are added */ 2821 t->name_off = name_off; 2822 t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0); 2823 t->size = byte_sz; 2824 2825 return btf_commit_type(btf, sz); 2826 } 2827 2828 /* 2829 * Append new data section variable information entry for current DATASEC type: 2830 * - *var_type_id* - type ID, describing type of the variable; 2831 * - *offset* - variable offset within data section, in bytes; 2832 * - *byte_sz* - variable size, in bytes. 2833 * 2834 * Returns: 2835 * - 0, on success; 2836 * - <0, on error. 2837 */ 2838 int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __u32 byte_sz) 2839 { 2840 struct btf_type *t; 2841 struct btf_var_secinfo *v; 2842 int sz; 2843 2844 /* last type should be BTF_KIND_DATASEC */ 2845 if (btf->nr_types == 0) 2846 return libbpf_err(-EINVAL); 2847 t = btf_last_type(btf); 2848 if (!btf_is_datasec(t)) 2849 return libbpf_err(-EINVAL); 2850 2851 if (validate_type_id(var_type_id)) 2852 return libbpf_err(-EINVAL); 2853 2854 /* decompose and invalidate raw data */ 2855 if (btf_ensure_modifiable(btf)) 2856 return libbpf_err(-ENOMEM); 2857 2858 sz = sizeof(struct btf_var_secinfo); 2859 v = btf_add_type_mem(btf, sz); 2860 if (!v) 2861 return libbpf_err(-ENOMEM); 2862 2863 v->type = var_type_id; 2864 v->offset = offset; 2865 v->size = byte_sz; 2866 2867 /* update parent type's vlen */ 2868 t = btf_last_type(btf); 2869 btf_type_inc_vlen(t); 2870 2871 btf->hdr->type_len += sz; 2872 btf->hdr->str_off += sz; 2873 return 0; 2874 } 2875 2876 static int btf_add_decl_tag(struct btf *btf, const char *value, int ref_type_id, 2877 int component_idx, int kflag) 2878 { 2879 struct btf_type *t; 2880 int sz, value_off; 2881 2882 if (!value || !value[0] || component_idx < -1) 2883 return libbpf_err(-EINVAL); 2884 2885 if (validate_type_id(ref_type_id)) 2886 return libbpf_err(-EINVAL); 2887 2888 if (btf_ensure_modifiable(btf)) 2889 return libbpf_err(-ENOMEM); 2890 2891 sz = sizeof(struct btf_type) + sizeof(struct btf_decl_tag); 2892 t = btf_add_type_mem(btf, sz); 2893 if (!t) 2894 return libbpf_err(-ENOMEM); 2895 2896 value_off = btf__add_str(btf, value); 2897 if (value_off < 0) 2898 return value_off; 2899 2900 t->name_off = value_off; 2901 t->info = btf_type_info(BTF_KIND_DECL_TAG, 0, kflag); 2902 t->type = ref_type_id; 2903 btf_decl_tag(t)->component_idx = component_idx; 2904 2905 return btf_commit_type(btf, sz); 2906 } 2907 2908 /* 2909 * Append new BTF_KIND_DECL_TAG type with: 2910 * - *value* - non-empty/non-NULL string; 2911 * - *ref_type_id* - referenced type ID, it might not exist yet; 2912 * - *component_idx* - -1 for tagging reference type, otherwise struct/union 2913 * member or function argument index; 2914 * Returns: 2915 * - >0, type ID of newly added BTF type; 2916 * - <0, on error. 2917 */ 2918 int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id, 2919 int component_idx) 2920 { 2921 return btf_add_decl_tag(btf, value, ref_type_id, component_idx, 0); 2922 } 2923 2924 /* 2925 * Append new BTF_KIND_DECL_TAG type with: 2926 * - *value* - non-empty/non-NULL string; 2927 * - *ref_type_id* - referenced type ID, it might not exist yet; 2928 * - *component_idx* - -1 for tagging reference type, otherwise struct/union 2929 * member or function argument index; 2930 * Set info->kflag to 1, indicating this tag is an __attribute__ 2931 * Returns: 2932 * - >0, type ID of newly added BTF type; 2933 * - <0, on error. 2934 */ 2935 int btf__add_decl_attr(struct btf *btf, const char *value, int ref_type_id, 2936 int component_idx) 2937 { 2938 return btf_add_decl_tag(btf, value, ref_type_id, component_idx, 1); 2939 } 2940 2941 struct btf_ext_sec_info_param { 2942 __u32 off; 2943 __u32 len; 2944 __u32 min_rec_size; 2945 struct btf_ext_info *ext_info; 2946 const char *desc; 2947 }; 2948 2949 /* 2950 * Parse a single info subsection of the BTF.ext info data: 2951 * - validate subsection structure and elements 2952 * - save info subsection start and sizing details in struct btf_ext 2953 * - endian-independent operation, for calling before byte-swapping 2954 */ 2955 static int btf_ext_parse_sec_info(struct btf_ext *btf_ext, 2956 struct btf_ext_sec_info_param *ext_sec, 2957 bool is_native) 2958 { 2959 const struct btf_ext_info_sec *sinfo; 2960 struct btf_ext_info *ext_info; 2961 __u32 info_left, record_size; 2962 size_t sec_cnt = 0; 2963 void *info; 2964 2965 if (ext_sec->len == 0) 2966 return 0; 2967 2968 if (ext_sec->off & 0x03) { 2969 pr_debug(".BTF.ext %s section is not aligned to 4 bytes\n", 2970 ext_sec->desc); 2971 return -EINVAL; 2972 } 2973 2974 /* The start of the info sec (including the __u32 record_size). */ 2975 info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off; 2976 info_left = ext_sec->len; 2977 2978 if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) { 2979 pr_debug("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n", 2980 ext_sec->desc, ext_sec->off, ext_sec->len); 2981 return -EINVAL; 2982 } 2983 2984 /* At least a record size */ 2985 if (info_left < sizeof(__u32)) { 2986 pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc); 2987 return -EINVAL; 2988 } 2989 2990 /* The record size needs to meet either the minimum standard or, when 2991 * handling non-native endianness data, the exact standard so as 2992 * to allow safe byte-swapping. 2993 */ 2994 record_size = is_native ? *(__u32 *)info : bswap_32(*(__u32 *)info); 2995 if (record_size < ext_sec->min_rec_size || 2996 (!is_native && record_size != ext_sec->min_rec_size) || 2997 record_size & 0x03) { 2998 pr_debug("%s section in .BTF.ext has invalid record size %u\n", 2999 ext_sec->desc, record_size); 3000 return -EINVAL; 3001 } 3002 3003 sinfo = info + sizeof(__u32); 3004 info_left -= sizeof(__u32); 3005 3006 /* If no records, return failure now so .BTF.ext won't be used. */ 3007 if (!info_left) { 3008 pr_debug("%s section in .BTF.ext has no records\n", ext_sec->desc); 3009 return -EINVAL; 3010 } 3011 3012 while (info_left) { 3013 unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec); 3014 __u64 total_record_size; 3015 __u32 num_records; 3016 3017 if (info_left < sec_hdrlen) { 3018 pr_debug("%s section header is not found in .BTF.ext\n", 3019 ext_sec->desc); 3020 return -EINVAL; 3021 } 3022 3023 num_records = is_native ? sinfo->num_info : bswap_32(sinfo->num_info); 3024 if (num_records == 0) { 3025 pr_debug("%s section has incorrect num_records in .BTF.ext\n", 3026 ext_sec->desc); 3027 return -EINVAL; 3028 } 3029 3030 total_record_size = sec_hdrlen + (__u64)num_records * record_size; 3031 if (info_left < total_record_size) { 3032 pr_debug("%s section has incorrect num_records in .BTF.ext\n", 3033 ext_sec->desc); 3034 return -EINVAL; 3035 } 3036 3037 info_left -= total_record_size; 3038 sinfo = (void *)sinfo + total_record_size; 3039 sec_cnt++; 3040 } 3041 3042 ext_info = ext_sec->ext_info; 3043 ext_info->len = ext_sec->len - sizeof(__u32); 3044 ext_info->rec_size = record_size; 3045 ext_info->info = info + sizeof(__u32); 3046 ext_info->sec_cnt = sec_cnt; 3047 3048 return 0; 3049 } 3050 3051 /* Parse all info secs in the BTF.ext info data */ 3052 static int btf_ext_parse_info(struct btf_ext *btf_ext, bool is_native) 3053 { 3054 struct btf_ext_sec_info_param func_info = { 3055 .off = btf_ext->hdr->func_info_off, 3056 .len = btf_ext->hdr->func_info_len, 3057 .min_rec_size = sizeof(struct bpf_func_info_min), 3058 .ext_info = &btf_ext->func_info, 3059 .desc = "func_info" 3060 }; 3061 struct btf_ext_sec_info_param line_info = { 3062 .off = btf_ext->hdr->line_info_off, 3063 .len = btf_ext->hdr->line_info_len, 3064 .min_rec_size = sizeof(struct bpf_line_info_min), 3065 .ext_info = &btf_ext->line_info, 3066 .desc = "line_info", 3067 }; 3068 struct btf_ext_sec_info_param core_relo = { 3069 .min_rec_size = sizeof(struct bpf_core_relo), 3070 .ext_info = &btf_ext->core_relo_info, 3071 .desc = "core_relo", 3072 }; 3073 int err; 3074 3075 err = btf_ext_parse_sec_info(btf_ext, &func_info, is_native); 3076 if (err) 3077 return err; 3078 3079 err = btf_ext_parse_sec_info(btf_ext, &line_info, is_native); 3080 if (err) 3081 return err; 3082 3083 if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) 3084 return 0; /* skip core relos parsing */ 3085 3086 core_relo.off = btf_ext->hdr->core_relo_off; 3087 core_relo.len = btf_ext->hdr->core_relo_len; 3088 err = btf_ext_parse_sec_info(btf_ext, &core_relo, is_native); 3089 if (err) 3090 return err; 3091 3092 return 0; 3093 } 3094 3095 /* Swap byte-order of BTF.ext header with any endianness */ 3096 static void btf_ext_bswap_hdr(struct btf_ext_header *h) 3097 { 3098 bool is_native = h->magic == BTF_MAGIC; 3099 __u32 hdr_len; 3100 3101 hdr_len = is_native ? h->hdr_len : bswap_32(h->hdr_len); 3102 3103 h->magic = bswap_16(h->magic); 3104 h->hdr_len = bswap_32(h->hdr_len); 3105 h->func_info_off = bswap_32(h->func_info_off); 3106 h->func_info_len = bswap_32(h->func_info_len); 3107 h->line_info_off = bswap_32(h->line_info_off); 3108 h->line_info_len = bswap_32(h->line_info_len); 3109 3110 if (hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) 3111 return; 3112 3113 h->core_relo_off = bswap_32(h->core_relo_off); 3114 h->core_relo_len = bswap_32(h->core_relo_len); 3115 } 3116 3117 /* Swap byte-order of generic info subsection */ 3118 static void btf_ext_bswap_info_sec(void *info, __u32 len, bool is_native, 3119 info_rec_bswap_fn bswap_fn) 3120 { 3121 struct btf_ext_info_sec *sec; 3122 __u32 info_left, rec_size, *rs; 3123 3124 if (len == 0) 3125 return; 3126 3127 rs = info; /* info record size */ 3128 rec_size = is_native ? *rs : bswap_32(*rs); 3129 *rs = bswap_32(*rs); 3130 3131 sec = info + sizeof(__u32); /* info sec #1 */ 3132 info_left = len - sizeof(__u32); 3133 while (info_left) { 3134 unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec); 3135 __u32 i, num_recs; 3136 void *p; 3137 3138 num_recs = is_native ? sec->num_info : bswap_32(sec->num_info); 3139 sec->sec_name_off = bswap_32(sec->sec_name_off); 3140 sec->num_info = bswap_32(sec->num_info); 3141 p = sec->data; /* info rec #1 */ 3142 for (i = 0; i < num_recs; i++, p += rec_size) 3143 bswap_fn(p); 3144 sec = p; 3145 info_left -= sec_hdrlen + (__u64)rec_size * num_recs; 3146 } 3147 } 3148 3149 /* 3150 * Swap byte-order of all info data in a BTF.ext section 3151 * - requires BTF.ext hdr in native endianness 3152 */ 3153 static void btf_ext_bswap_info(struct btf_ext *btf_ext, void *data) 3154 { 3155 const bool is_native = btf_ext->swapped_endian; 3156 const struct btf_ext_header *h = data; 3157 void *info; 3158 3159 /* Swap func_info subsection byte-order */ 3160 info = data + h->hdr_len + h->func_info_off; 3161 btf_ext_bswap_info_sec(info, h->func_info_len, is_native, 3162 (info_rec_bswap_fn)bpf_func_info_bswap); 3163 3164 /* Swap line_info subsection byte-order */ 3165 info = data + h->hdr_len + h->line_info_off; 3166 btf_ext_bswap_info_sec(info, h->line_info_len, is_native, 3167 (info_rec_bswap_fn)bpf_line_info_bswap); 3168 3169 /* Swap core_relo subsection byte-order (if present) */ 3170 if (h->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) 3171 return; 3172 3173 info = data + h->hdr_len + h->core_relo_off; 3174 btf_ext_bswap_info_sec(info, h->core_relo_len, is_native, 3175 (info_rec_bswap_fn)bpf_core_relo_bswap); 3176 } 3177 3178 /* Parse hdr data and info sections: check and convert to native endianness */ 3179 static int btf_ext_parse(struct btf_ext *btf_ext) 3180 { 3181 __u32 hdr_len, data_size = btf_ext->data_size; 3182 struct btf_ext_header *hdr = btf_ext->hdr; 3183 bool swapped_endian = false; 3184 int err; 3185 3186 if (data_size < offsetofend(struct btf_ext_header, hdr_len)) { 3187 pr_debug("BTF.ext header too short\n"); 3188 return -EINVAL; 3189 } 3190 3191 hdr_len = hdr->hdr_len; 3192 if (hdr->magic == bswap_16(BTF_MAGIC)) { 3193 swapped_endian = true; 3194 hdr_len = bswap_32(hdr_len); 3195 } else if (hdr->magic != BTF_MAGIC) { 3196 pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic); 3197 return -EINVAL; 3198 } 3199 3200 /* Ensure known version of structs, current BTF_VERSION == 1 */ 3201 if (hdr->version != 1) { 3202 pr_debug("Unsupported BTF.ext version:%u\n", hdr->version); 3203 return -ENOTSUP; 3204 } 3205 3206 if (hdr->flags) { 3207 pr_debug("Unsupported BTF.ext flags:%x\n", hdr->flags); 3208 return -ENOTSUP; 3209 } 3210 3211 if (data_size < hdr_len) { 3212 pr_debug("BTF.ext header not found\n"); 3213 return -EINVAL; 3214 } else if (data_size == hdr_len) { 3215 pr_debug("BTF.ext has no data\n"); 3216 return -EINVAL; 3217 } 3218 3219 /* Verify mandatory hdr info details present */ 3220 if (hdr_len < offsetofend(struct btf_ext_header, line_info_len)) { 3221 pr_warn("BTF.ext header missing func_info, line_info\n"); 3222 return -EINVAL; 3223 } 3224 3225 /* Keep hdr native byte-order in memory for introspection */ 3226 if (swapped_endian) 3227 btf_ext_bswap_hdr(btf_ext->hdr); 3228 3229 /* Validate info subsections and cache key metadata */ 3230 err = btf_ext_parse_info(btf_ext, !swapped_endian); 3231 if (err) 3232 return err; 3233 3234 /* Keep infos native byte-order in memory for introspection */ 3235 if (swapped_endian) 3236 btf_ext_bswap_info(btf_ext, btf_ext->data); 3237 3238 /* 3239 * Set btf_ext->swapped_endian only after all header and info data has 3240 * been swapped, helping bswap functions determine if their data are 3241 * in native byte-order when called. 3242 */ 3243 btf_ext->swapped_endian = swapped_endian; 3244 return 0; 3245 } 3246 3247 void btf_ext__free(struct btf_ext *btf_ext) 3248 { 3249 if (IS_ERR_OR_NULL(btf_ext)) 3250 return; 3251 free(btf_ext->func_info.sec_idxs); 3252 free(btf_ext->line_info.sec_idxs); 3253 free(btf_ext->core_relo_info.sec_idxs); 3254 free(btf_ext->data); 3255 free(btf_ext->data_swapped); 3256 free(btf_ext); 3257 } 3258 3259 struct btf_ext *btf_ext__new(const __u8 *data, __u32 size) 3260 { 3261 struct btf_ext *btf_ext; 3262 int err; 3263 3264 btf_ext = calloc(1, sizeof(struct btf_ext)); 3265 if (!btf_ext) 3266 return libbpf_err_ptr(-ENOMEM); 3267 3268 btf_ext->data_size = size; 3269 btf_ext->data = malloc(size); 3270 if (!btf_ext->data) { 3271 err = -ENOMEM; 3272 goto done; 3273 } 3274 memcpy(btf_ext->data, data, size); 3275 3276 err = btf_ext_parse(btf_ext); 3277 3278 done: 3279 if (err) { 3280 btf_ext__free(btf_ext); 3281 return libbpf_err_ptr(err); 3282 } 3283 3284 return btf_ext; 3285 } 3286 3287 static void *btf_ext_raw_data(const struct btf_ext *btf_ext_ro, bool swap_endian) 3288 { 3289 struct btf_ext *btf_ext = (struct btf_ext *)btf_ext_ro; 3290 const __u32 data_sz = btf_ext->data_size; 3291 void *data; 3292 3293 /* Return native data (always present) or swapped data if present */ 3294 if (!swap_endian) 3295 return btf_ext->data; 3296 else if (btf_ext->data_swapped) 3297 return btf_ext->data_swapped; 3298 3299 /* Recreate missing swapped data, then cache and return */ 3300 data = calloc(1, data_sz); 3301 if (!data) 3302 return NULL; 3303 memcpy(data, btf_ext->data, data_sz); 3304 3305 btf_ext_bswap_info(btf_ext, data); 3306 btf_ext_bswap_hdr(data); 3307 btf_ext->data_swapped = data; 3308 return data; 3309 } 3310 3311 const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size) 3312 { 3313 void *data; 3314 3315 data = btf_ext_raw_data(btf_ext, btf_ext->swapped_endian); 3316 if (!data) 3317 return errno = ENOMEM, NULL; 3318 3319 *size = btf_ext->data_size; 3320 return data; 3321 } 3322 3323 __attribute__((alias("btf_ext__raw_data"))) 3324 const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size); 3325 3326 enum btf_endianness btf_ext__endianness(const struct btf_ext *btf_ext) 3327 { 3328 if (is_host_big_endian()) 3329 return btf_ext->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN; 3330 else 3331 return btf_ext->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN; 3332 } 3333 3334 int btf_ext__set_endianness(struct btf_ext *btf_ext, enum btf_endianness endian) 3335 { 3336 if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN) 3337 return libbpf_err(-EINVAL); 3338 3339 btf_ext->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN); 3340 3341 if (!btf_ext->swapped_endian) { 3342 free(btf_ext->data_swapped); 3343 btf_ext->data_swapped = NULL; 3344 } 3345 return 0; 3346 } 3347 3348 struct btf_dedup; 3349 3350 static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts); 3351 static void btf_dedup_free(struct btf_dedup *d); 3352 static int btf_dedup_prep(struct btf_dedup *d); 3353 static int btf_dedup_strings(struct btf_dedup *d); 3354 static int btf_dedup_prim_types(struct btf_dedup *d); 3355 static int btf_dedup_struct_types(struct btf_dedup *d); 3356 static int btf_dedup_ref_types(struct btf_dedup *d); 3357 static int btf_dedup_resolve_fwds(struct btf_dedup *d); 3358 static int btf_dedup_compact_types(struct btf_dedup *d); 3359 static int btf_dedup_remap_types(struct btf_dedup *d); 3360 3361 /* 3362 * Deduplicate BTF types and strings. 3363 * 3364 * BTF dedup algorithm takes as an input `struct btf` representing `.BTF` ELF 3365 * section with all BTF type descriptors and string data. It overwrites that 3366 * memory in-place with deduplicated types and strings without any loss of 3367 * information. If optional `struct btf_ext` representing '.BTF.ext' ELF section 3368 * is provided, all the strings referenced from .BTF.ext section are honored 3369 * and updated to point to the right offsets after deduplication. 3370 * 3371 * If function returns with error, type/string data might be garbled and should 3372 * be discarded. 3373 * 3374 * More verbose and detailed description of both problem btf_dedup is solving, 3375 * as well as solution could be found at: 3376 * https://facebookmicrosites.github.io/bpf/blog/2018/11/14/btf-enhancement.html 3377 * 3378 * Problem description and justification 3379 * ===================================== 3380 * 3381 * BTF type information is typically emitted either as a result of conversion 3382 * from DWARF to BTF or directly by compiler. In both cases, each compilation 3383 * unit contains information about a subset of all the types that are used 3384 * in an application. These subsets are frequently overlapping and contain a lot 3385 * of duplicated information when later concatenated together into a single 3386 * binary. This algorithm ensures that each unique type is represented by single 3387 * BTF type descriptor, greatly reducing resulting size of BTF data. 3388 * 3389 * Compilation unit isolation and subsequent duplication of data is not the only 3390 * problem. The same type hierarchy (e.g., struct and all the type that struct 3391 * references) in different compilation units can be represented in BTF to 3392 * various degrees of completeness (or, rather, incompleteness) due to 3393 * struct/union forward declarations. 3394 * 3395 * Let's take a look at an example, that we'll use to better understand the 3396 * problem (and solution). Suppose we have two compilation units, each using 3397 * same `struct S`, but each of them having incomplete type information about 3398 * struct's fields: 3399 * 3400 * // CU #1: 3401 * struct S; 3402 * struct A { 3403 * int a; 3404 * struct A* self; 3405 * struct S* parent; 3406 * }; 3407 * struct B; 3408 * struct S { 3409 * struct A* a_ptr; 3410 * struct B* b_ptr; 3411 * }; 3412 * 3413 * // CU #2: 3414 * struct S; 3415 * struct A; 3416 * struct B { 3417 * int b; 3418 * struct B* self; 3419 * struct S* parent; 3420 * }; 3421 * struct S { 3422 * struct A* a_ptr; 3423 * struct B* b_ptr; 3424 * }; 3425 * 3426 * In case of CU #1, BTF data will know only that `struct B` exist (but no 3427 * more), but will know the complete type information about `struct A`. While 3428 * for CU #2, it will know full type information about `struct B`, but will 3429 * only know about forward declaration of `struct A` (in BTF terms, it will 3430 * have `BTF_KIND_FWD` type descriptor with name `B`). 3431 * 3432 * This compilation unit isolation means that it's possible that there is no 3433 * single CU with complete type information describing structs `S`, `A`, and 3434 * `B`. Also, we might get tons of duplicated and redundant type information. 3435 * 3436 * Additional complication we need to keep in mind comes from the fact that 3437 * types, in general, can form graphs containing cycles, not just DAGs. 3438 * 3439 * While algorithm does deduplication, it also merges and resolves type 3440 * information (unless disabled throught `struct btf_opts`), whenever possible. 3441 * E.g., in the example above with two compilation units having partial type 3442 * information for structs `A` and `B`, the output of algorithm will emit 3443 * a single copy of each BTF type that describes structs `A`, `B`, and `S` 3444 * (as well as type information for `int` and pointers), as if they were defined 3445 * in a single compilation unit as: 3446 * 3447 * struct A { 3448 * int a; 3449 * struct A* self; 3450 * struct S* parent; 3451 * }; 3452 * struct B { 3453 * int b; 3454 * struct B* self; 3455 * struct S* parent; 3456 * }; 3457 * struct S { 3458 * struct A* a_ptr; 3459 * struct B* b_ptr; 3460 * }; 3461 * 3462 * Algorithm summary 3463 * ================= 3464 * 3465 * Algorithm completes its work in 7 separate passes: 3466 * 3467 * 1. Strings deduplication. 3468 * 2. Primitive types deduplication (int, enum, fwd). 3469 * 3. Struct/union types deduplication. 3470 * 4. Resolve unambiguous forward declarations. 3471 * 5. Reference types deduplication (pointers, typedefs, arrays, funcs, func 3472 * protos, and const/volatile/restrict modifiers). 3473 * 6. Types compaction. 3474 * 7. Types remapping. 3475 * 3476 * Algorithm determines canonical type descriptor, which is a single 3477 * representative type for each truly unique type. This canonical type is the 3478 * one that will go into final deduplicated BTF type information. For 3479 * struct/unions, it is also the type that algorithm will merge additional type 3480 * information into (while resolving FWDs), as it discovers it from data in 3481 * other CUs. Each input BTF type eventually gets either mapped to itself, if 3482 * that type is canonical, or to some other type, if that type is equivalent 3483 * and was chosen as canonical representative. This mapping is stored in 3484 * `btf_dedup->map` array. This map is also used to record STRUCT/UNION that 3485 * FWD type got resolved to. 3486 * 3487 * To facilitate fast discovery of canonical types, we also maintain canonical 3488 * index (`btf_dedup->dedup_table`), which maps type descriptor's signature hash 3489 * (i.e., hashed kind, name, size, fields, etc) into a list of canonical types 3490 * that match that signature. With sufficiently good choice of type signature 3491 * hashing function, we can limit number of canonical types for each unique type 3492 * signature to a very small number, allowing to find canonical type for any 3493 * duplicated type very quickly. 3494 * 3495 * Struct/union deduplication is the most critical part and algorithm for 3496 * deduplicating structs/unions is described in greater details in comments for 3497 * `btf_dedup_is_equiv` function. 3498 */ 3499 int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts) 3500 { 3501 struct btf_dedup *d; 3502 int err; 3503 3504 if (!OPTS_VALID(opts, btf_dedup_opts)) 3505 return libbpf_err(-EINVAL); 3506 3507 d = btf_dedup_new(btf, opts); 3508 if (IS_ERR(d)) { 3509 pr_debug("btf_dedup_new failed: %ld\n", PTR_ERR(d)); 3510 return libbpf_err(-EINVAL); 3511 } 3512 3513 if (btf_ensure_modifiable(btf)) { 3514 err = -ENOMEM; 3515 goto done; 3516 } 3517 3518 err = btf_dedup_prep(d); 3519 if (err) { 3520 pr_debug("btf_dedup_prep failed: %s\n", errstr(err)); 3521 goto done; 3522 } 3523 err = btf_dedup_strings(d); 3524 if (err < 0) { 3525 pr_debug("btf_dedup_strings failed: %s\n", errstr(err)); 3526 goto done; 3527 } 3528 err = btf_dedup_prim_types(d); 3529 if (err < 0) { 3530 pr_debug("btf_dedup_prim_types failed: %s\n", errstr(err)); 3531 goto done; 3532 } 3533 err = btf_dedup_struct_types(d); 3534 if (err < 0) { 3535 pr_debug("btf_dedup_struct_types failed: %s\n", errstr(err)); 3536 goto done; 3537 } 3538 err = btf_dedup_resolve_fwds(d); 3539 if (err < 0) { 3540 pr_debug("btf_dedup_resolve_fwds failed: %s\n", errstr(err)); 3541 goto done; 3542 } 3543 err = btf_dedup_ref_types(d); 3544 if (err < 0) { 3545 pr_debug("btf_dedup_ref_types failed: %s\n", errstr(err)); 3546 goto done; 3547 } 3548 err = btf_dedup_compact_types(d); 3549 if (err < 0) { 3550 pr_debug("btf_dedup_compact_types failed: %s\n", errstr(err)); 3551 goto done; 3552 } 3553 err = btf_dedup_remap_types(d); 3554 if (err < 0) { 3555 pr_debug("btf_dedup_remap_types failed: %s\n", errstr(err)); 3556 goto done; 3557 } 3558 3559 done: 3560 btf_dedup_free(d); 3561 return libbpf_err(err); 3562 } 3563 3564 #define BTF_UNPROCESSED_ID ((__u32)-1) 3565 #define BTF_IN_PROGRESS_ID ((__u32)-2) 3566 3567 struct btf_dedup { 3568 /* .BTF section to be deduped in-place */ 3569 struct btf *btf; 3570 /* 3571 * Optional .BTF.ext section. When provided, any strings referenced 3572 * from it will be taken into account when deduping strings 3573 */ 3574 struct btf_ext *btf_ext; 3575 /* 3576 * This is a map from any type's signature hash to a list of possible 3577 * canonical representative type candidates. Hash collisions are 3578 * ignored, so even types of various kinds can share same list of 3579 * candidates, which is fine because we rely on subsequent 3580 * btf_xxx_equal() checks to authoritatively verify type equality. 3581 */ 3582 struct hashmap *dedup_table; 3583 /* Canonical types map */ 3584 __u32 *map; 3585 /* Hypothetical mapping, used during type graph equivalence checks */ 3586 __u32 *hypot_map; 3587 __u32 *hypot_list; 3588 size_t hypot_cnt; 3589 size_t hypot_cap; 3590 /* Whether hypothetical mapping, if successful, would need to adjust 3591 * already canonicalized types (due to a new forward declaration to 3592 * concrete type resolution). In such case, during split BTF dedup 3593 * candidate type would still be considered as different, because base 3594 * BTF is considered to be immutable. 3595 */ 3596 bool hypot_adjust_canon; 3597 /* Various option modifying behavior of algorithm */ 3598 struct btf_dedup_opts opts; 3599 /* temporary strings deduplication state */ 3600 struct strset *strs_set; 3601 }; 3602 3603 static unsigned long hash_combine(unsigned long h, unsigned long value) 3604 { 3605 return h * 31 + value; 3606 } 3607 3608 #define for_each_dedup_cand(d, node, hash) \ 3609 hashmap__for_each_key_entry(d->dedup_table, node, hash) 3610 3611 static int btf_dedup_table_add(struct btf_dedup *d, long hash, __u32 type_id) 3612 { 3613 return hashmap__append(d->dedup_table, hash, type_id); 3614 } 3615 3616 static int btf_dedup_hypot_map_add(struct btf_dedup *d, 3617 __u32 from_id, __u32 to_id) 3618 { 3619 if (d->hypot_cnt == d->hypot_cap) { 3620 __u32 *new_list; 3621 3622 d->hypot_cap += max((size_t)16, d->hypot_cap / 2); 3623 new_list = libbpf_reallocarray(d->hypot_list, d->hypot_cap, sizeof(__u32)); 3624 if (!new_list) 3625 return -ENOMEM; 3626 d->hypot_list = new_list; 3627 } 3628 d->hypot_list[d->hypot_cnt++] = from_id; 3629 d->hypot_map[from_id] = to_id; 3630 return 0; 3631 } 3632 3633 static void btf_dedup_clear_hypot_map(struct btf_dedup *d) 3634 { 3635 int i; 3636 3637 for (i = 0; i < d->hypot_cnt; i++) 3638 d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID; 3639 d->hypot_cnt = 0; 3640 d->hypot_adjust_canon = false; 3641 } 3642 3643 static void btf_dedup_free(struct btf_dedup *d) 3644 { 3645 hashmap__free(d->dedup_table); 3646 d->dedup_table = NULL; 3647 3648 free(d->map); 3649 d->map = NULL; 3650 3651 free(d->hypot_map); 3652 d->hypot_map = NULL; 3653 3654 free(d->hypot_list); 3655 d->hypot_list = NULL; 3656 3657 free(d); 3658 } 3659 3660 static size_t btf_dedup_identity_hash_fn(long key, void *ctx) 3661 { 3662 return key; 3663 } 3664 3665 static size_t btf_dedup_collision_hash_fn(long key, void *ctx) 3666 { 3667 return 0; 3668 } 3669 3670 static bool btf_dedup_equal_fn(long k1, long k2, void *ctx) 3671 { 3672 return k1 == k2; 3673 } 3674 3675 static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts) 3676 { 3677 struct btf_dedup *d = calloc(1, sizeof(struct btf_dedup)); 3678 hashmap_hash_fn hash_fn = btf_dedup_identity_hash_fn; 3679 int i, err = 0, type_cnt; 3680 3681 if (!d) 3682 return ERR_PTR(-ENOMEM); 3683 3684 if (OPTS_GET(opts, force_collisions, false)) 3685 hash_fn = btf_dedup_collision_hash_fn; 3686 3687 d->btf = btf; 3688 d->btf_ext = OPTS_GET(opts, btf_ext, NULL); 3689 3690 d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL); 3691 if (IS_ERR(d->dedup_table)) { 3692 err = PTR_ERR(d->dedup_table); 3693 d->dedup_table = NULL; 3694 goto done; 3695 } 3696 3697 type_cnt = btf__type_cnt(btf); 3698 d->map = malloc(sizeof(__u32) * type_cnt); 3699 if (!d->map) { 3700 err = -ENOMEM; 3701 goto done; 3702 } 3703 /* special BTF "void" type is made canonical immediately */ 3704 d->map[0] = 0; 3705 for (i = 1; i < type_cnt; i++) { 3706 struct btf_type *t = btf_type_by_id(d->btf, i); 3707 3708 /* VAR and DATASEC are never deduped and are self-canonical */ 3709 if (btf_is_var(t) || btf_is_datasec(t)) 3710 d->map[i] = i; 3711 else 3712 d->map[i] = BTF_UNPROCESSED_ID; 3713 } 3714 3715 d->hypot_map = malloc(sizeof(__u32) * type_cnt); 3716 if (!d->hypot_map) { 3717 err = -ENOMEM; 3718 goto done; 3719 } 3720 for (i = 0; i < type_cnt; i++) 3721 d->hypot_map[i] = BTF_UNPROCESSED_ID; 3722 3723 done: 3724 if (err) { 3725 btf_dedup_free(d); 3726 return ERR_PTR(err); 3727 } 3728 3729 return d; 3730 } 3731 3732 /* 3733 * Iterate over all possible places in .BTF and .BTF.ext that can reference 3734 * string and pass pointer to it to a provided callback `fn`. 3735 */ 3736 static int btf_for_each_str_off(struct btf_dedup *d, str_off_visit_fn fn, void *ctx) 3737 { 3738 int i, r; 3739 3740 for (i = 0; i < d->btf->nr_types; i++) { 3741 struct btf_field_iter it; 3742 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i); 3743 __u32 *str_off; 3744 3745 r = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS); 3746 if (r) 3747 return r; 3748 3749 while ((str_off = btf_field_iter_next(&it))) { 3750 r = fn(str_off, ctx); 3751 if (r) 3752 return r; 3753 } 3754 } 3755 3756 if (!d->btf_ext) 3757 return 0; 3758 3759 r = btf_ext_visit_str_offs(d->btf_ext, fn, ctx); 3760 if (r) 3761 return r; 3762 3763 return 0; 3764 } 3765 3766 static int strs_dedup_remap_str_off(__u32 *str_off_ptr, void *ctx) 3767 { 3768 struct btf_dedup *d = ctx; 3769 __u32 str_off = *str_off_ptr; 3770 const char *s; 3771 int off, err; 3772 3773 /* don't touch empty string or string in main BTF */ 3774 if (str_off == 0 || str_off < d->btf->start_str_off) 3775 return 0; 3776 3777 s = btf__str_by_offset(d->btf, str_off); 3778 if (d->btf->base_btf) { 3779 err = btf__find_str(d->btf->base_btf, s); 3780 if (err >= 0) { 3781 *str_off_ptr = err; 3782 return 0; 3783 } 3784 if (err != -ENOENT) 3785 return err; 3786 } 3787 3788 off = strset__add_str(d->strs_set, s); 3789 if (off < 0) 3790 return off; 3791 3792 *str_off_ptr = d->btf->start_str_off + off; 3793 return 0; 3794 } 3795 3796 /* 3797 * Dedup string and filter out those that are not referenced from either .BTF 3798 * or .BTF.ext (if provided) sections. 3799 * 3800 * This is done by building index of all strings in BTF's string section, 3801 * then iterating over all entities that can reference strings (e.g., type 3802 * names, struct field names, .BTF.ext line info, etc) and marking corresponding 3803 * strings as used. After that all used strings are deduped and compacted into 3804 * sequential blob of memory and new offsets are calculated. Then all the string 3805 * references are iterated again and rewritten using new offsets. 3806 */ 3807 static int btf_dedup_strings(struct btf_dedup *d) 3808 { 3809 int err; 3810 3811 if (d->btf->strs_deduped) 3812 return 0; 3813 3814 d->strs_set = strset__new(BTF_MAX_STR_OFFSET, NULL, 0); 3815 if (IS_ERR(d->strs_set)) { 3816 err = PTR_ERR(d->strs_set); 3817 goto err_out; 3818 } 3819 3820 if (!d->btf->base_btf) { 3821 /* insert empty string; we won't be looking it up during strings 3822 * dedup, but it's good to have it for generic BTF string lookups 3823 */ 3824 err = strset__add_str(d->strs_set, ""); 3825 if (err < 0) 3826 goto err_out; 3827 } 3828 3829 /* remap string offsets */ 3830 err = btf_for_each_str_off(d, strs_dedup_remap_str_off, d); 3831 if (err) 3832 goto err_out; 3833 3834 /* replace BTF string data and hash with deduped ones */ 3835 strset__free(d->btf->strs_set); 3836 d->btf->hdr->str_len = strset__data_size(d->strs_set); 3837 d->btf->strs_set = d->strs_set; 3838 d->strs_set = NULL; 3839 d->btf->strs_deduped = true; 3840 return 0; 3841 3842 err_out: 3843 strset__free(d->strs_set); 3844 d->strs_set = NULL; 3845 3846 return err; 3847 } 3848 3849 static long btf_hash_common(struct btf_type *t) 3850 { 3851 long h; 3852 3853 h = hash_combine(0, t->name_off); 3854 h = hash_combine(h, t->info); 3855 h = hash_combine(h, t->size); 3856 return h; 3857 } 3858 3859 static bool btf_equal_common(struct btf_type *t1, struct btf_type *t2) 3860 { 3861 return t1->name_off == t2->name_off && 3862 t1->info == t2->info && 3863 t1->size == t2->size; 3864 } 3865 3866 /* Calculate type signature hash of INT or TAG. */ 3867 static long btf_hash_int_decl_tag(struct btf_type *t) 3868 { 3869 __u32 info = *(__u32 *)(t + 1); 3870 long h; 3871 3872 h = btf_hash_common(t); 3873 h = hash_combine(h, info); 3874 return h; 3875 } 3876 3877 /* Check structural equality of two INTs or TAGs. */ 3878 static bool btf_equal_int_tag(struct btf_type *t1, struct btf_type *t2) 3879 { 3880 __u32 info1, info2; 3881 3882 if (!btf_equal_common(t1, t2)) 3883 return false; 3884 info1 = *(__u32 *)(t1 + 1); 3885 info2 = *(__u32 *)(t2 + 1); 3886 return info1 == info2; 3887 } 3888 3889 /* Calculate type signature hash of ENUM/ENUM64. */ 3890 static long btf_hash_enum(struct btf_type *t) 3891 { 3892 long h; 3893 3894 /* don't hash vlen, enum members and size to support enum fwd resolving */ 3895 h = hash_combine(0, t->name_off); 3896 return h; 3897 } 3898 3899 static bool btf_equal_enum_members(struct btf_type *t1, struct btf_type *t2) 3900 { 3901 const struct btf_enum *m1, *m2; 3902 __u16 vlen; 3903 int i; 3904 3905 vlen = btf_vlen(t1); 3906 m1 = btf_enum(t1); 3907 m2 = btf_enum(t2); 3908 for (i = 0; i < vlen; i++) { 3909 if (m1->name_off != m2->name_off || m1->val != m2->val) 3910 return false; 3911 m1++; 3912 m2++; 3913 } 3914 return true; 3915 } 3916 3917 static bool btf_equal_enum64_members(struct btf_type *t1, struct btf_type *t2) 3918 { 3919 const struct btf_enum64 *m1, *m2; 3920 __u16 vlen; 3921 int i; 3922 3923 vlen = btf_vlen(t1); 3924 m1 = btf_enum64(t1); 3925 m2 = btf_enum64(t2); 3926 for (i = 0; i < vlen; i++) { 3927 if (m1->name_off != m2->name_off || m1->val_lo32 != m2->val_lo32 || 3928 m1->val_hi32 != m2->val_hi32) 3929 return false; 3930 m1++; 3931 m2++; 3932 } 3933 return true; 3934 } 3935 3936 /* Check structural equality of two ENUMs or ENUM64s. */ 3937 static bool btf_equal_enum(struct btf_type *t1, struct btf_type *t2) 3938 { 3939 if (!btf_equal_common(t1, t2)) 3940 return false; 3941 3942 /* t1 & t2 kinds are identical because of btf_equal_common */ 3943 if (btf_kind(t1) == BTF_KIND_ENUM) 3944 return btf_equal_enum_members(t1, t2); 3945 else 3946 return btf_equal_enum64_members(t1, t2); 3947 } 3948 3949 static inline bool btf_is_enum_fwd(struct btf_type *t) 3950 { 3951 return btf_is_any_enum(t) && btf_vlen(t) == 0; 3952 } 3953 3954 static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2) 3955 { 3956 if (!btf_is_enum_fwd(t1) && !btf_is_enum_fwd(t2)) 3957 return btf_equal_enum(t1, t2); 3958 /* At this point either t1 or t2 or both are forward declarations, thus: 3959 * - skip comparing vlen because it is zero for forward declarations; 3960 * - skip comparing size to allow enum forward declarations 3961 * to be compatible with enum64 full declarations; 3962 * - skip comparing kind for the same reason. 3963 */ 3964 return t1->name_off == t2->name_off && 3965 btf_is_any_enum(t1) && btf_is_any_enum(t2); 3966 } 3967 3968 /* 3969 * Calculate type signature hash of STRUCT/UNION, ignoring referenced type IDs, 3970 * as referenced type IDs equivalence is established separately during type 3971 * graph equivalence check algorithm. 3972 */ 3973 static long btf_hash_struct(struct btf_type *t) 3974 { 3975 const struct btf_member *member = btf_members(t); 3976 __u32 vlen = btf_vlen(t); 3977 long h = btf_hash_common(t); 3978 int i; 3979 3980 for (i = 0; i < vlen; i++) { 3981 h = hash_combine(h, member->name_off); 3982 h = hash_combine(h, member->offset); 3983 /* no hashing of referenced type ID, it can be unresolved yet */ 3984 member++; 3985 } 3986 return h; 3987 } 3988 3989 /* 3990 * Check structural compatibility of two STRUCTs/UNIONs, ignoring referenced 3991 * type IDs. This check is performed during type graph equivalence check and 3992 * referenced types equivalence is checked separately. 3993 */ 3994 static bool btf_shallow_equal_struct(struct btf_type *t1, struct btf_type *t2) 3995 { 3996 const struct btf_member *m1, *m2; 3997 __u16 vlen; 3998 int i; 3999 4000 if (!btf_equal_common(t1, t2)) 4001 return false; 4002 4003 vlen = btf_vlen(t1); 4004 m1 = btf_members(t1); 4005 m2 = btf_members(t2); 4006 for (i = 0; i < vlen; i++) { 4007 if (m1->name_off != m2->name_off || m1->offset != m2->offset) 4008 return false; 4009 m1++; 4010 m2++; 4011 } 4012 return true; 4013 } 4014 4015 /* 4016 * Calculate type signature hash of ARRAY, including referenced type IDs, 4017 * under assumption that they were already resolved to canonical type IDs and 4018 * are not going to change. 4019 */ 4020 static long btf_hash_array(struct btf_type *t) 4021 { 4022 const struct btf_array *info = btf_array(t); 4023 long h = btf_hash_common(t); 4024 4025 h = hash_combine(h, info->type); 4026 h = hash_combine(h, info->index_type); 4027 h = hash_combine(h, info->nelems); 4028 return h; 4029 } 4030 4031 /* 4032 * Check exact equality of two ARRAYs, taking into account referenced 4033 * type IDs, under assumption that they were already resolved to canonical 4034 * type IDs and are not going to change. 4035 * This function is called during reference types deduplication to compare 4036 * ARRAY to potential canonical representative. 4037 */ 4038 static bool btf_equal_array(struct btf_type *t1, struct btf_type *t2) 4039 { 4040 const struct btf_array *info1, *info2; 4041 4042 if (!btf_equal_common(t1, t2)) 4043 return false; 4044 4045 info1 = btf_array(t1); 4046 info2 = btf_array(t2); 4047 return info1->type == info2->type && 4048 info1->index_type == info2->index_type && 4049 info1->nelems == info2->nelems; 4050 } 4051 4052 /* 4053 * Check structural compatibility of two ARRAYs, ignoring referenced type 4054 * IDs. This check is performed during type graph equivalence check and 4055 * referenced types equivalence is checked separately. 4056 */ 4057 static bool btf_compat_array(struct btf_type *t1, struct btf_type *t2) 4058 { 4059 if (!btf_equal_common(t1, t2)) 4060 return false; 4061 4062 return btf_array(t1)->nelems == btf_array(t2)->nelems; 4063 } 4064 4065 /* 4066 * Calculate type signature hash of FUNC_PROTO, including referenced type IDs, 4067 * under assumption that they were already resolved to canonical type IDs and 4068 * are not going to change. 4069 */ 4070 static long btf_hash_fnproto(struct btf_type *t) 4071 { 4072 const struct btf_param *member = btf_params(t); 4073 __u16 vlen = btf_vlen(t); 4074 long h = btf_hash_common(t); 4075 int i; 4076 4077 for (i = 0; i < vlen; i++) { 4078 h = hash_combine(h, member->name_off); 4079 h = hash_combine(h, member->type); 4080 member++; 4081 } 4082 return h; 4083 } 4084 4085 /* 4086 * Check exact equality of two FUNC_PROTOs, taking into account referenced 4087 * type IDs, under assumption that they were already resolved to canonical 4088 * type IDs and are not going to change. 4089 * This function is called during reference types deduplication to compare 4090 * FUNC_PROTO to potential canonical representative. 4091 */ 4092 static bool btf_equal_fnproto(struct btf_type *t1, struct btf_type *t2) 4093 { 4094 const struct btf_param *m1, *m2; 4095 __u16 vlen; 4096 int i; 4097 4098 if (!btf_equal_common(t1, t2)) 4099 return false; 4100 4101 vlen = btf_vlen(t1); 4102 m1 = btf_params(t1); 4103 m2 = btf_params(t2); 4104 for (i = 0; i < vlen; i++) { 4105 if (m1->name_off != m2->name_off || m1->type != m2->type) 4106 return false; 4107 m1++; 4108 m2++; 4109 } 4110 return true; 4111 } 4112 4113 /* 4114 * Check structural compatibility of two FUNC_PROTOs, ignoring referenced type 4115 * IDs. This check is performed during type graph equivalence check and 4116 * referenced types equivalence is checked separately. 4117 */ 4118 static bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2) 4119 { 4120 const struct btf_param *m1, *m2; 4121 __u16 vlen; 4122 int i; 4123 4124 /* skip return type ID */ 4125 if (t1->name_off != t2->name_off || t1->info != t2->info) 4126 return false; 4127 4128 vlen = btf_vlen(t1); 4129 m1 = btf_params(t1); 4130 m2 = btf_params(t2); 4131 for (i = 0; i < vlen; i++) { 4132 if (m1->name_off != m2->name_off) 4133 return false; 4134 m1++; 4135 m2++; 4136 } 4137 return true; 4138 } 4139 4140 /* Prepare split BTF for deduplication by calculating hashes of base BTF's 4141 * types and initializing the rest of the state (canonical type mapping) for 4142 * the fixed base BTF part. 4143 */ 4144 static int btf_dedup_prep(struct btf_dedup *d) 4145 { 4146 struct btf_type *t; 4147 int type_id; 4148 long h; 4149 4150 if (!d->btf->base_btf) 4151 return 0; 4152 4153 for (type_id = 1; type_id < d->btf->start_id; type_id++) { 4154 t = btf_type_by_id(d->btf, type_id); 4155 4156 /* all base BTF types are self-canonical by definition */ 4157 d->map[type_id] = type_id; 4158 4159 switch (btf_kind(t)) { 4160 case BTF_KIND_VAR: 4161 case BTF_KIND_DATASEC: 4162 /* VAR and DATASEC are never hash/deduplicated */ 4163 continue; 4164 case BTF_KIND_CONST: 4165 case BTF_KIND_VOLATILE: 4166 case BTF_KIND_RESTRICT: 4167 case BTF_KIND_PTR: 4168 case BTF_KIND_FWD: 4169 case BTF_KIND_TYPEDEF: 4170 case BTF_KIND_FUNC: 4171 case BTF_KIND_FLOAT: 4172 case BTF_KIND_TYPE_TAG: 4173 h = btf_hash_common(t); 4174 break; 4175 case BTF_KIND_INT: 4176 case BTF_KIND_DECL_TAG: 4177 h = btf_hash_int_decl_tag(t); 4178 break; 4179 case BTF_KIND_ENUM: 4180 case BTF_KIND_ENUM64: 4181 h = btf_hash_enum(t); 4182 break; 4183 case BTF_KIND_STRUCT: 4184 case BTF_KIND_UNION: 4185 h = btf_hash_struct(t); 4186 break; 4187 case BTF_KIND_ARRAY: 4188 h = btf_hash_array(t); 4189 break; 4190 case BTF_KIND_FUNC_PROTO: 4191 h = btf_hash_fnproto(t); 4192 break; 4193 default: 4194 pr_debug("unknown kind %d for type [%d]\n", btf_kind(t), type_id); 4195 return -EINVAL; 4196 } 4197 if (btf_dedup_table_add(d, h, type_id)) 4198 return -ENOMEM; 4199 } 4200 4201 return 0; 4202 } 4203 4204 /* 4205 * Deduplicate primitive types, that can't reference other types, by calculating 4206 * their type signature hash and comparing them with any possible canonical 4207 * candidate. If no canonical candidate matches, type itself is marked as 4208 * canonical and is added into `btf_dedup->dedup_table` as another candidate. 4209 */ 4210 static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) 4211 { 4212 struct btf_type *t = btf_type_by_id(d->btf, type_id); 4213 struct hashmap_entry *hash_entry; 4214 struct btf_type *cand; 4215 /* if we don't find equivalent type, then we are canonical */ 4216 __u32 new_id = type_id; 4217 __u32 cand_id; 4218 long h; 4219 4220 switch (btf_kind(t)) { 4221 case BTF_KIND_CONST: 4222 case BTF_KIND_VOLATILE: 4223 case BTF_KIND_RESTRICT: 4224 case BTF_KIND_PTR: 4225 case BTF_KIND_TYPEDEF: 4226 case BTF_KIND_ARRAY: 4227 case BTF_KIND_STRUCT: 4228 case BTF_KIND_UNION: 4229 case BTF_KIND_FUNC: 4230 case BTF_KIND_FUNC_PROTO: 4231 case BTF_KIND_VAR: 4232 case BTF_KIND_DATASEC: 4233 case BTF_KIND_DECL_TAG: 4234 case BTF_KIND_TYPE_TAG: 4235 return 0; 4236 4237 case BTF_KIND_INT: 4238 h = btf_hash_int_decl_tag(t); 4239 for_each_dedup_cand(d, hash_entry, h) { 4240 cand_id = hash_entry->value; 4241 cand = btf_type_by_id(d->btf, cand_id); 4242 if (btf_equal_int_tag(t, cand)) { 4243 new_id = cand_id; 4244 break; 4245 } 4246 } 4247 break; 4248 4249 case BTF_KIND_ENUM: 4250 case BTF_KIND_ENUM64: 4251 h = btf_hash_enum(t); 4252 for_each_dedup_cand(d, hash_entry, h) { 4253 cand_id = hash_entry->value; 4254 cand = btf_type_by_id(d->btf, cand_id); 4255 if (btf_equal_enum(t, cand)) { 4256 new_id = cand_id; 4257 break; 4258 } 4259 if (btf_compat_enum(t, cand)) { 4260 if (btf_is_enum_fwd(t)) { 4261 /* resolve fwd to full enum */ 4262 new_id = cand_id; 4263 break; 4264 } 4265 /* resolve canonical enum fwd to full enum */ 4266 d->map[cand_id] = type_id; 4267 } 4268 } 4269 break; 4270 4271 case BTF_KIND_FWD: 4272 case BTF_KIND_FLOAT: 4273 h = btf_hash_common(t); 4274 for_each_dedup_cand(d, hash_entry, h) { 4275 cand_id = hash_entry->value; 4276 cand = btf_type_by_id(d->btf, cand_id); 4277 if (btf_equal_common(t, cand)) { 4278 new_id = cand_id; 4279 break; 4280 } 4281 } 4282 break; 4283 4284 default: 4285 return -EINVAL; 4286 } 4287 4288 d->map[type_id] = new_id; 4289 if (type_id == new_id && btf_dedup_table_add(d, h, type_id)) 4290 return -ENOMEM; 4291 4292 return 0; 4293 } 4294 4295 static int btf_dedup_prim_types(struct btf_dedup *d) 4296 { 4297 int i, err; 4298 4299 for (i = 0; i < d->btf->nr_types; i++) { 4300 err = btf_dedup_prim_type(d, d->btf->start_id + i); 4301 if (err) 4302 return err; 4303 } 4304 return 0; 4305 } 4306 4307 /* 4308 * Check whether type is already mapped into canonical one (could be to itself). 4309 */ 4310 static inline bool is_type_mapped(struct btf_dedup *d, uint32_t type_id) 4311 { 4312 return d->map[type_id] <= BTF_MAX_NR_TYPES; 4313 } 4314 4315 /* 4316 * Resolve type ID into its canonical type ID, if any; otherwise return original 4317 * type ID. If type is FWD and is resolved into STRUCT/UNION already, follow 4318 * STRUCT/UNION link and resolve it into canonical type ID as well. 4319 */ 4320 static inline __u32 resolve_type_id(struct btf_dedup *d, __u32 type_id) 4321 { 4322 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) 4323 type_id = d->map[type_id]; 4324 return type_id; 4325 } 4326 4327 /* 4328 * Resolve FWD to underlying STRUCT/UNION, if any; otherwise return original 4329 * type ID. 4330 */ 4331 static uint32_t resolve_fwd_id(struct btf_dedup *d, uint32_t type_id) 4332 { 4333 __u32 orig_type_id = type_id; 4334 4335 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) 4336 return type_id; 4337 4338 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) 4339 type_id = d->map[type_id]; 4340 4341 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) 4342 return type_id; 4343 4344 return orig_type_id; 4345 } 4346 4347 4348 static inline __u16 btf_fwd_kind(struct btf_type *t) 4349 { 4350 return btf_kflag(t) ? BTF_KIND_UNION : BTF_KIND_STRUCT; 4351 } 4352 4353 /* Check if given two types are identical ARRAY definitions */ 4354 static bool btf_dedup_identical_arrays(struct btf_dedup *d, __u32 id1, __u32 id2) 4355 { 4356 struct btf_type *t1, *t2; 4357 4358 t1 = btf_type_by_id(d->btf, id1); 4359 t2 = btf_type_by_id(d->btf, id2); 4360 if (!btf_is_array(t1) || !btf_is_array(t2)) 4361 return false; 4362 4363 return btf_equal_array(t1, t2); 4364 } 4365 4366 /* Check if given two types are identical STRUCT/UNION definitions */ 4367 static bool btf_dedup_identical_structs(struct btf_dedup *d, __u32 id1, __u32 id2) 4368 { 4369 const struct btf_member *m1, *m2; 4370 struct btf_type *t1, *t2; 4371 int n, i; 4372 4373 t1 = btf_type_by_id(d->btf, id1); 4374 t2 = btf_type_by_id(d->btf, id2); 4375 4376 if (!btf_is_composite(t1) || btf_kind(t1) != btf_kind(t2)) 4377 return false; 4378 4379 if (!btf_shallow_equal_struct(t1, t2)) 4380 return false; 4381 4382 m1 = btf_members(t1); 4383 m2 = btf_members(t2); 4384 for (i = 0, n = btf_vlen(t1); i < n; i++, m1++, m2++) { 4385 if (m1->type != m2->type && 4386 !btf_dedup_identical_arrays(d, m1->type, m2->type) && 4387 !btf_dedup_identical_structs(d, m1->type, m2->type)) 4388 return false; 4389 } 4390 return true; 4391 } 4392 4393 /* 4394 * Check equivalence of BTF type graph formed by candidate struct/union (we'll 4395 * call it "candidate graph" in this description for brevity) to a type graph 4396 * formed by (potential) canonical struct/union ("canonical graph" for brevity 4397 * here, though keep in mind that not all types in canonical graph are 4398 * necessarily canonical representatives themselves, some of them might be 4399 * duplicates or its uniqueness might not have been established yet). 4400 * Returns: 4401 * - >0, if type graphs are equivalent; 4402 * - 0, if not equivalent; 4403 * - <0, on error. 4404 * 4405 * Algorithm performs side-by-side DFS traversal of both type graphs and checks 4406 * equivalence of BTF types at each step. If at any point BTF types in candidate 4407 * and canonical graphs are not compatible structurally, whole graphs are 4408 * incompatible. If types are structurally equivalent (i.e., all information 4409 * except referenced type IDs is exactly the same), a mapping from `canon_id` to 4410 * a `cand_id` is recoded in hypothetical mapping (`btf_dedup->hypot_map`). 4411 * If a type references other types, then those referenced types are checked 4412 * for equivalence recursively. 4413 * 4414 * During DFS traversal, if we find that for current `canon_id` type we 4415 * already have some mapping in hypothetical map, we check for two possible 4416 * situations: 4417 * - `canon_id` is mapped to exactly the same type as `cand_id`. This will 4418 * happen when type graphs have cycles. In this case we assume those two 4419 * types are equivalent. 4420 * - `canon_id` is mapped to different type. This is contradiction in our 4421 * hypothetical mapping, because same graph in canonical graph corresponds 4422 * to two different types in candidate graph, which for equivalent type 4423 * graphs shouldn't happen. This condition terminates equivalence check 4424 * with negative result. 4425 * 4426 * If type graphs traversal exhausts types to check and find no contradiction, 4427 * then type graphs are equivalent. 4428 * 4429 * When checking types for equivalence, there is one special case: FWD types. 4430 * If FWD type resolution is allowed and one of the types (either from canonical 4431 * or candidate graph) is FWD and other is STRUCT/UNION (depending on FWD's kind 4432 * flag) and their names match, hypothetical mapping is updated to point from 4433 * FWD to STRUCT/UNION. If graphs will be determined as equivalent successfully, 4434 * this mapping will be used to record FWD -> STRUCT/UNION mapping permanently. 4435 * 4436 * Technically, this could lead to incorrect FWD to STRUCT/UNION resolution, 4437 * if there are two exactly named (or anonymous) structs/unions that are 4438 * compatible structurally, one of which has FWD field, while other is concrete 4439 * STRUCT/UNION, but according to C sources they are different structs/unions 4440 * that are referencing different types with the same name. This is extremely 4441 * unlikely to happen, but btf_dedup API allows to disable FWD resolution if 4442 * this logic is causing problems. 4443 * 4444 * Doing FWD resolution means that both candidate and/or canonical graphs can 4445 * consists of portions of the graph that come from multiple compilation units. 4446 * This is due to the fact that types within single compilation unit are always 4447 * deduplicated and FWDs are already resolved, if referenced struct/union 4448 * definition is available. So, if we had unresolved FWD and found corresponding 4449 * STRUCT/UNION, they will be from different compilation units. This 4450 * consequently means that when we "link" FWD to corresponding STRUCT/UNION, 4451 * type graph will likely have at least two different BTF types that describe 4452 * same type (e.g., most probably there will be two different BTF types for the 4453 * same 'int' primitive type) and could even have "overlapping" parts of type 4454 * graph that describe same subset of types. 4455 * 4456 * This in turn means that our assumption that each type in canonical graph 4457 * must correspond to exactly one type in candidate graph might not hold 4458 * anymore and will make it harder to detect contradictions using hypothetical 4459 * map. To handle this problem, we allow to follow FWD -> STRUCT/UNION 4460 * resolution only in canonical graph. FWDs in candidate graphs are never 4461 * resolved. To see why it's OK, let's check all possible situations w.r.t. FWDs 4462 * that can occur: 4463 * - Both types in canonical and candidate graphs are FWDs. If they are 4464 * structurally equivalent, then they can either be both resolved to the 4465 * same STRUCT/UNION or not resolved at all. In both cases they are 4466 * equivalent and there is no need to resolve FWD on candidate side. 4467 * - Both types in canonical and candidate graphs are concrete STRUCT/UNION, 4468 * so nothing to resolve as well, algorithm will check equivalence anyway. 4469 * - Type in canonical graph is FWD, while type in candidate is concrete 4470 * STRUCT/UNION. In this case candidate graph comes from single compilation 4471 * unit, so there is exactly one BTF type for each unique C type. After 4472 * resolving FWD into STRUCT/UNION, there might be more than one BTF type 4473 * in canonical graph mapping to single BTF type in candidate graph, but 4474 * because hypothetical mapping maps from canonical to candidate types, it's 4475 * alright, and we still maintain the property of having single `canon_id` 4476 * mapping to single `cand_id` (there could be two different `canon_id` 4477 * mapped to the same `cand_id`, but it's not contradictory). 4478 * - Type in canonical graph is concrete STRUCT/UNION, while type in candidate 4479 * graph is FWD. In this case we are just going to check compatibility of 4480 * STRUCT/UNION and corresponding FWD, and if they are compatible, we'll 4481 * assume that whatever STRUCT/UNION FWD resolves to must be equivalent to 4482 * a concrete STRUCT/UNION from canonical graph. If the rest of type graphs 4483 * turn out equivalent, we'll re-resolve FWD to concrete STRUCT/UNION from 4484 * canonical graph. 4485 */ 4486 static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, 4487 __u32 canon_id) 4488 { 4489 struct btf_type *cand_type; 4490 struct btf_type *canon_type; 4491 __u32 hypot_type_id; 4492 __u16 cand_kind; 4493 __u16 canon_kind; 4494 int i, eq; 4495 4496 /* if both resolve to the same canonical, they must be equivalent */ 4497 if (resolve_type_id(d, cand_id) == resolve_type_id(d, canon_id)) 4498 return 1; 4499 4500 canon_id = resolve_fwd_id(d, canon_id); 4501 4502 hypot_type_id = d->hypot_map[canon_id]; 4503 if (hypot_type_id <= BTF_MAX_NR_TYPES) { 4504 if (hypot_type_id == cand_id) 4505 return 1; 4506 /* In some cases compiler will generate different DWARF types 4507 * for *identical* array type definitions and use them for 4508 * different fields within the *same* struct. This breaks type 4509 * equivalence check, which makes an assumption that candidate 4510 * types sub-graph has a consistent and deduped-by-compiler 4511 * types within a single CU. So work around that by explicitly 4512 * allowing identical array types here. 4513 */ 4514 if (btf_dedup_identical_arrays(d, hypot_type_id, cand_id)) 4515 return 1; 4516 /* It turns out that similar situation can happen with 4517 * struct/union sometimes, sigh... Handle the case where 4518 * structs/unions are exactly the same, down to the referenced 4519 * type IDs. Anything more complicated (e.g., if referenced 4520 * types are different, but equivalent) is *way more* 4521 * complicated and requires a many-to-many equivalence mapping. 4522 */ 4523 if (btf_dedup_identical_structs(d, hypot_type_id, cand_id)) 4524 return 1; 4525 return 0; 4526 } 4527 4528 if (btf_dedup_hypot_map_add(d, canon_id, cand_id)) 4529 return -ENOMEM; 4530 4531 cand_type = btf_type_by_id(d->btf, cand_id); 4532 canon_type = btf_type_by_id(d->btf, canon_id); 4533 cand_kind = btf_kind(cand_type); 4534 canon_kind = btf_kind(canon_type); 4535 4536 if (cand_type->name_off != canon_type->name_off) 4537 return 0; 4538 4539 /* FWD <--> STRUCT/UNION equivalence check, if enabled */ 4540 if ((cand_kind == BTF_KIND_FWD || canon_kind == BTF_KIND_FWD) 4541 && cand_kind != canon_kind) { 4542 __u16 real_kind; 4543 __u16 fwd_kind; 4544 4545 if (cand_kind == BTF_KIND_FWD) { 4546 real_kind = canon_kind; 4547 fwd_kind = btf_fwd_kind(cand_type); 4548 } else { 4549 real_kind = cand_kind; 4550 fwd_kind = btf_fwd_kind(canon_type); 4551 /* we'd need to resolve base FWD to STRUCT/UNION */ 4552 if (fwd_kind == real_kind && canon_id < d->btf->start_id) 4553 d->hypot_adjust_canon = true; 4554 } 4555 return fwd_kind == real_kind; 4556 } 4557 4558 if (cand_kind != canon_kind) 4559 return 0; 4560 4561 switch (cand_kind) { 4562 case BTF_KIND_INT: 4563 return btf_equal_int_tag(cand_type, canon_type); 4564 4565 case BTF_KIND_ENUM: 4566 case BTF_KIND_ENUM64: 4567 return btf_compat_enum(cand_type, canon_type); 4568 4569 case BTF_KIND_FWD: 4570 case BTF_KIND_FLOAT: 4571 return btf_equal_common(cand_type, canon_type); 4572 4573 case BTF_KIND_CONST: 4574 case BTF_KIND_VOLATILE: 4575 case BTF_KIND_RESTRICT: 4576 case BTF_KIND_PTR: 4577 case BTF_KIND_TYPEDEF: 4578 case BTF_KIND_FUNC: 4579 case BTF_KIND_TYPE_TAG: 4580 if (cand_type->info != canon_type->info) 4581 return 0; 4582 return btf_dedup_is_equiv(d, cand_type->type, canon_type->type); 4583 4584 case BTF_KIND_ARRAY: { 4585 const struct btf_array *cand_arr, *canon_arr; 4586 4587 if (!btf_compat_array(cand_type, canon_type)) 4588 return 0; 4589 cand_arr = btf_array(cand_type); 4590 canon_arr = btf_array(canon_type); 4591 eq = btf_dedup_is_equiv(d, cand_arr->index_type, canon_arr->index_type); 4592 if (eq <= 0) 4593 return eq; 4594 return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type); 4595 } 4596 4597 case BTF_KIND_STRUCT: 4598 case BTF_KIND_UNION: { 4599 const struct btf_member *cand_m, *canon_m; 4600 __u16 vlen; 4601 4602 if (!btf_shallow_equal_struct(cand_type, canon_type)) 4603 return 0; 4604 vlen = btf_vlen(cand_type); 4605 cand_m = btf_members(cand_type); 4606 canon_m = btf_members(canon_type); 4607 for (i = 0; i < vlen; i++) { 4608 eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type); 4609 if (eq <= 0) 4610 return eq; 4611 cand_m++; 4612 canon_m++; 4613 } 4614 4615 return 1; 4616 } 4617 4618 case BTF_KIND_FUNC_PROTO: { 4619 const struct btf_param *cand_p, *canon_p; 4620 __u16 vlen; 4621 4622 if (!btf_compat_fnproto(cand_type, canon_type)) 4623 return 0; 4624 eq = btf_dedup_is_equiv(d, cand_type->type, canon_type->type); 4625 if (eq <= 0) 4626 return eq; 4627 vlen = btf_vlen(cand_type); 4628 cand_p = btf_params(cand_type); 4629 canon_p = btf_params(canon_type); 4630 for (i = 0; i < vlen; i++) { 4631 eq = btf_dedup_is_equiv(d, cand_p->type, canon_p->type); 4632 if (eq <= 0) 4633 return eq; 4634 cand_p++; 4635 canon_p++; 4636 } 4637 return 1; 4638 } 4639 4640 default: 4641 return -EINVAL; 4642 } 4643 return 0; 4644 } 4645 4646 /* 4647 * Use hypothetical mapping, produced by successful type graph equivalence 4648 * check, to augment existing struct/union canonical mapping, where possible. 4649 * 4650 * If BTF_KIND_FWD resolution is allowed, this mapping is also used to record 4651 * FWD -> STRUCT/UNION correspondence as well. FWD resolution is bidirectional: 4652 * it doesn't matter if FWD type was part of canonical graph or candidate one, 4653 * we are recording the mapping anyway. As opposed to carefulness required 4654 * for struct/union correspondence mapping (described below), for FWD resolution 4655 * it's not important, as by the time that FWD type (reference type) will be 4656 * deduplicated all structs/unions will be deduped already anyway. 4657 * 4658 * Recording STRUCT/UNION mapping is purely a performance optimization and is 4659 * not required for correctness. It needs to be done carefully to ensure that 4660 * struct/union from candidate's type graph is not mapped into corresponding 4661 * struct/union from canonical type graph that itself hasn't been resolved into 4662 * canonical representative. The only guarantee we have is that canonical 4663 * struct/union was determined as canonical and that won't change. But any 4664 * types referenced through that struct/union fields could have been not yet 4665 * resolved, so in case like that it's too early to establish any kind of 4666 * correspondence between structs/unions. 4667 * 4668 * No canonical correspondence is derived for primitive types (they are already 4669 * deduplicated completely already anyway) or reference types (they rely on 4670 * stability of struct/union canonical relationship for equivalence checks). 4671 */ 4672 static void btf_dedup_merge_hypot_map(struct btf_dedup *d) 4673 { 4674 __u32 canon_type_id, targ_type_id; 4675 __u16 t_kind, c_kind; 4676 __u32 t_id, c_id; 4677 int i; 4678 4679 for (i = 0; i < d->hypot_cnt; i++) { 4680 canon_type_id = d->hypot_list[i]; 4681 targ_type_id = d->hypot_map[canon_type_id]; 4682 t_id = resolve_type_id(d, targ_type_id); 4683 c_id = resolve_type_id(d, canon_type_id); 4684 t_kind = btf_kind(btf__type_by_id(d->btf, t_id)); 4685 c_kind = btf_kind(btf__type_by_id(d->btf, c_id)); 4686 /* 4687 * Resolve FWD into STRUCT/UNION. 4688 * It's ok to resolve FWD into STRUCT/UNION that's not yet 4689 * mapped to canonical representative (as opposed to 4690 * STRUCT/UNION <--> STRUCT/UNION mapping logic below), because 4691 * eventually that struct is going to be mapped and all resolved 4692 * FWDs will automatically resolve to correct canonical 4693 * representative. This will happen before ref type deduping, 4694 * which critically depends on stability of these mapping. This 4695 * stability is not a requirement for STRUCT/UNION equivalence 4696 * checks, though. 4697 */ 4698 4699 /* if it's the split BTF case, we still need to point base FWD 4700 * to STRUCT/UNION in a split BTF, because FWDs from split BTF 4701 * will be resolved against base FWD. If we don't point base 4702 * canonical FWD to the resolved STRUCT/UNION, then all the 4703 * FWDs in split BTF won't be correctly resolved to a proper 4704 * STRUCT/UNION. 4705 */ 4706 if (t_kind != BTF_KIND_FWD && c_kind == BTF_KIND_FWD) 4707 d->map[c_id] = t_id; 4708 4709 /* if graph equivalence determined that we'd need to adjust 4710 * base canonical types, then we need to only point base FWDs 4711 * to STRUCTs/UNIONs and do no more modifications. For all 4712 * other purposes the type graphs were not equivalent. 4713 */ 4714 if (d->hypot_adjust_canon) 4715 continue; 4716 4717 if (t_kind == BTF_KIND_FWD && c_kind != BTF_KIND_FWD) 4718 d->map[t_id] = c_id; 4719 4720 if ((t_kind == BTF_KIND_STRUCT || t_kind == BTF_KIND_UNION) && 4721 c_kind != BTF_KIND_FWD && 4722 is_type_mapped(d, c_id) && 4723 !is_type_mapped(d, t_id)) { 4724 /* 4725 * as a perf optimization, we can map struct/union 4726 * that's part of type graph we just verified for 4727 * equivalence. We can do that for struct/union that has 4728 * canonical representative only, though. 4729 */ 4730 d->map[t_id] = c_id; 4731 } 4732 } 4733 } 4734 4735 /* 4736 * Deduplicate struct/union types. 4737 * 4738 * For each struct/union type its type signature hash is calculated, taking 4739 * into account type's name, size, number, order and names of fields, but 4740 * ignoring type ID's referenced from fields, because they might not be deduped 4741 * completely until after reference types deduplication phase. This type hash 4742 * is used to iterate over all potential canonical types, sharing same hash. 4743 * For each canonical candidate we check whether type graphs that they form 4744 * (through referenced types in fields and so on) are equivalent using algorithm 4745 * implemented in `btf_dedup_is_equiv`. If such equivalence is found and 4746 * BTF_KIND_FWD resolution is allowed, then hypothetical mapping 4747 * (btf_dedup->hypot_map) produced by aforementioned type graph equivalence 4748 * algorithm is used to record FWD -> STRUCT/UNION mapping. It's also used to 4749 * potentially map other structs/unions to their canonical representatives, 4750 * if such relationship hasn't yet been established. This speeds up algorithm 4751 * by eliminating some of the duplicate work. 4752 * 4753 * If no matching canonical representative was found, struct/union is marked 4754 * as canonical for itself and is added into btf_dedup->dedup_table hash map 4755 * for further look ups. 4756 */ 4757 static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id) 4758 { 4759 struct btf_type *cand_type, *t; 4760 struct hashmap_entry *hash_entry; 4761 /* if we don't find equivalent type, then we are canonical */ 4762 __u32 new_id = type_id; 4763 __u16 kind; 4764 long h; 4765 4766 /* already deduped or is in process of deduping (loop detected) */ 4767 if (d->map[type_id] <= BTF_MAX_NR_TYPES) 4768 return 0; 4769 4770 t = btf_type_by_id(d->btf, type_id); 4771 kind = btf_kind(t); 4772 4773 if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION) 4774 return 0; 4775 4776 h = btf_hash_struct(t); 4777 for_each_dedup_cand(d, hash_entry, h) { 4778 __u32 cand_id = hash_entry->value; 4779 int eq; 4780 4781 /* 4782 * Even though btf_dedup_is_equiv() checks for 4783 * btf_shallow_equal_struct() internally when checking two 4784 * structs (unions) for equivalence, we need to guard here 4785 * from picking matching FWD type as a dedup candidate. 4786 * This can happen due to hash collision. In such case just 4787 * relying on btf_dedup_is_equiv() would lead to potentially 4788 * creating a loop (FWD -> STRUCT and STRUCT -> FWD), because 4789 * FWD and compatible STRUCT/UNION are considered equivalent. 4790 */ 4791 cand_type = btf_type_by_id(d->btf, cand_id); 4792 if (!btf_shallow_equal_struct(t, cand_type)) 4793 continue; 4794 4795 btf_dedup_clear_hypot_map(d); 4796 eq = btf_dedup_is_equiv(d, type_id, cand_id); 4797 if (eq < 0) 4798 return eq; 4799 if (!eq) 4800 continue; 4801 btf_dedup_merge_hypot_map(d); 4802 if (d->hypot_adjust_canon) /* not really equivalent */ 4803 continue; 4804 new_id = cand_id; 4805 break; 4806 } 4807 4808 d->map[type_id] = new_id; 4809 if (type_id == new_id && btf_dedup_table_add(d, h, type_id)) 4810 return -ENOMEM; 4811 4812 return 0; 4813 } 4814 4815 static int btf_dedup_struct_types(struct btf_dedup *d) 4816 { 4817 int i, err; 4818 4819 for (i = 0; i < d->btf->nr_types; i++) { 4820 err = btf_dedup_struct_type(d, d->btf->start_id + i); 4821 if (err) 4822 return err; 4823 } 4824 return 0; 4825 } 4826 4827 /* 4828 * Deduplicate reference type. 4829 * 4830 * Once all primitive and struct/union types got deduplicated, we can easily 4831 * deduplicate all other (reference) BTF types. This is done in two steps: 4832 * 4833 * 1. Resolve all referenced type IDs into their canonical type IDs. This 4834 * resolution can be done either immediately for primitive or struct/union types 4835 * (because they were deduped in previous two phases) or recursively for 4836 * reference types. Recursion will always terminate at either primitive or 4837 * struct/union type, at which point we can "unwind" chain of reference types 4838 * one by one. There is no danger of encountering cycles because in C type 4839 * system the only way to form type cycle is through struct/union, so any chain 4840 * of reference types, even those taking part in a type cycle, will inevitably 4841 * reach struct/union at some point. 4842 * 4843 * 2. Once all referenced type IDs are resolved into canonical ones, BTF type 4844 * becomes "stable", in the sense that no further deduplication will cause 4845 * any changes to it. With that, it's now possible to calculate type's signature 4846 * hash (this time taking into account referenced type IDs) and loop over all 4847 * potential canonical representatives. If no match was found, current type 4848 * will become canonical representative of itself and will be added into 4849 * btf_dedup->dedup_table as another possible canonical representative. 4850 */ 4851 static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) 4852 { 4853 struct hashmap_entry *hash_entry; 4854 __u32 new_id = type_id, cand_id; 4855 struct btf_type *t, *cand; 4856 /* if we don't find equivalent type, then we are representative type */ 4857 int ref_type_id; 4858 long h; 4859 4860 if (d->map[type_id] == BTF_IN_PROGRESS_ID) 4861 return -ELOOP; 4862 if (d->map[type_id] <= BTF_MAX_NR_TYPES) 4863 return resolve_type_id(d, type_id); 4864 4865 t = btf_type_by_id(d->btf, type_id); 4866 d->map[type_id] = BTF_IN_PROGRESS_ID; 4867 4868 switch (btf_kind(t)) { 4869 case BTF_KIND_CONST: 4870 case BTF_KIND_VOLATILE: 4871 case BTF_KIND_RESTRICT: 4872 case BTF_KIND_PTR: 4873 case BTF_KIND_TYPEDEF: 4874 case BTF_KIND_FUNC: 4875 case BTF_KIND_TYPE_TAG: 4876 ref_type_id = btf_dedup_ref_type(d, t->type); 4877 if (ref_type_id < 0) 4878 return ref_type_id; 4879 t->type = ref_type_id; 4880 4881 h = btf_hash_common(t); 4882 for_each_dedup_cand(d, hash_entry, h) { 4883 cand_id = hash_entry->value; 4884 cand = btf_type_by_id(d->btf, cand_id); 4885 if (btf_equal_common(t, cand)) { 4886 new_id = cand_id; 4887 break; 4888 } 4889 } 4890 break; 4891 4892 case BTF_KIND_DECL_TAG: 4893 ref_type_id = btf_dedup_ref_type(d, t->type); 4894 if (ref_type_id < 0) 4895 return ref_type_id; 4896 t->type = ref_type_id; 4897 4898 h = btf_hash_int_decl_tag(t); 4899 for_each_dedup_cand(d, hash_entry, h) { 4900 cand_id = hash_entry->value; 4901 cand = btf_type_by_id(d->btf, cand_id); 4902 if (btf_equal_int_tag(t, cand)) { 4903 new_id = cand_id; 4904 break; 4905 } 4906 } 4907 break; 4908 4909 case BTF_KIND_ARRAY: { 4910 struct btf_array *info = btf_array(t); 4911 4912 ref_type_id = btf_dedup_ref_type(d, info->type); 4913 if (ref_type_id < 0) 4914 return ref_type_id; 4915 info->type = ref_type_id; 4916 4917 ref_type_id = btf_dedup_ref_type(d, info->index_type); 4918 if (ref_type_id < 0) 4919 return ref_type_id; 4920 info->index_type = ref_type_id; 4921 4922 h = btf_hash_array(t); 4923 for_each_dedup_cand(d, hash_entry, h) { 4924 cand_id = hash_entry->value; 4925 cand = btf_type_by_id(d->btf, cand_id); 4926 if (btf_equal_array(t, cand)) { 4927 new_id = cand_id; 4928 break; 4929 } 4930 } 4931 break; 4932 } 4933 4934 case BTF_KIND_FUNC_PROTO: { 4935 struct btf_param *param; 4936 __u16 vlen; 4937 int i; 4938 4939 ref_type_id = btf_dedup_ref_type(d, t->type); 4940 if (ref_type_id < 0) 4941 return ref_type_id; 4942 t->type = ref_type_id; 4943 4944 vlen = btf_vlen(t); 4945 param = btf_params(t); 4946 for (i = 0; i < vlen; i++) { 4947 ref_type_id = btf_dedup_ref_type(d, param->type); 4948 if (ref_type_id < 0) 4949 return ref_type_id; 4950 param->type = ref_type_id; 4951 param++; 4952 } 4953 4954 h = btf_hash_fnproto(t); 4955 for_each_dedup_cand(d, hash_entry, h) { 4956 cand_id = hash_entry->value; 4957 cand = btf_type_by_id(d->btf, cand_id); 4958 if (btf_equal_fnproto(t, cand)) { 4959 new_id = cand_id; 4960 break; 4961 } 4962 } 4963 break; 4964 } 4965 4966 default: 4967 return -EINVAL; 4968 } 4969 4970 d->map[type_id] = new_id; 4971 if (type_id == new_id && btf_dedup_table_add(d, h, type_id)) 4972 return -ENOMEM; 4973 4974 return new_id; 4975 } 4976 4977 static int btf_dedup_ref_types(struct btf_dedup *d) 4978 { 4979 int i, err; 4980 4981 for (i = 0; i < d->btf->nr_types; i++) { 4982 err = btf_dedup_ref_type(d, d->btf->start_id + i); 4983 if (err < 0) 4984 return err; 4985 } 4986 /* we won't need d->dedup_table anymore */ 4987 hashmap__free(d->dedup_table); 4988 d->dedup_table = NULL; 4989 return 0; 4990 } 4991 4992 /* 4993 * Collect a map from type names to type ids for all canonical structs 4994 * and unions. If the same name is shared by several canonical types 4995 * use a special value 0 to indicate this fact. 4996 */ 4997 static int btf_dedup_fill_unique_names_map(struct btf_dedup *d, struct hashmap *names_map) 4998 { 4999 __u32 nr_types = btf__type_cnt(d->btf); 5000 struct btf_type *t; 5001 __u32 type_id; 5002 __u16 kind; 5003 int err; 5004 5005 /* 5006 * Iterate over base and split module ids in order to get all 5007 * available structs in the map. 5008 */ 5009 for (type_id = 1; type_id < nr_types; ++type_id) { 5010 t = btf_type_by_id(d->btf, type_id); 5011 kind = btf_kind(t); 5012 5013 if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION) 5014 continue; 5015 5016 /* Skip non-canonical types */ 5017 if (type_id != d->map[type_id]) 5018 continue; 5019 5020 err = hashmap__add(names_map, t->name_off, type_id); 5021 if (err == -EEXIST) 5022 err = hashmap__set(names_map, t->name_off, 0, NULL, NULL); 5023 5024 if (err) 5025 return err; 5026 } 5027 5028 return 0; 5029 } 5030 5031 static int btf_dedup_resolve_fwd(struct btf_dedup *d, struct hashmap *names_map, __u32 type_id) 5032 { 5033 struct btf_type *t = btf_type_by_id(d->btf, type_id); 5034 enum btf_fwd_kind fwd_kind = btf_kflag(t); 5035 __u16 cand_kind, kind = btf_kind(t); 5036 struct btf_type *cand_t; 5037 uintptr_t cand_id; 5038 5039 if (kind != BTF_KIND_FWD) 5040 return 0; 5041 5042 /* Skip if this FWD already has a mapping */ 5043 if (type_id != d->map[type_id]) 5044 return 0; 5045 5046 if (!hashmap__find(names_map, t->name_off, &cand_id)) 5047 return 0; 5048 5049 /* Zero is a special value indicating that name is not unique */ 5050 if (!cand_id) 5051 return 0; 5052 5053 cand_t = btf_type_by_id(d->btf, cand_id); 5054 cand_kind = btf_kind(cand_t); 5055 if ((cand_kind == BTF_KIND_STRUCT && fwd_kind != BTF_FWD_STRUCT) || 5056 (cand_kind == BTF_KIND_UNION && fwd_kind != BTF_FWD_UNION)) 5057 return 0; 5058 5059 d->map[type_id] = cand_id; 5060 5061 return 0; 5062 } 5063 5064 /* 5065 * Resolve unambiguous forward declarations. 5066 * 5067 * The lion's share of all FWD declarations is resolved during 5068 * `btf_dedup_struct_types` phase when different type graphs are 5069 * compared against each other. However, if in some compilation unit a 5070 * FWD declaration is not a part of a type graph compared against 5071 * another type graph that declaration's canonical type would not be 5072 * changed. Example: 5073 * 5074 * CU #1: 5075 * 5076 * struct foo; 5077 * struct foo *some_global; 5078 * 5079 * CU #2: 5080 * 5081 * struct foo { int u; }; 5082 * struct foo *another_global; 5083 * 5084 * After `btf_dedup_struct_types` the BTF looks as follows: 5085 * 5086 * [1] STRUCT 'foo' size=4 vlen=1 ... 5087 * [2] INT 'int' size=4 ... 5088 * [3] PTR '(anon)' type_id=1 5089 * [4] FWD 'foo' fwd_kind=struct 5090 * [5] PTR '(anon)' type_id=4 5091 * 5092 * This pass assumes that such FWD declarations should be mapped to 5093 * structs or unions with identical name in case if the name is not 5094 * ambiguous. 5095 */ 5096 static int btf_dedup_resolve_fwds(struct btf_dedup *d) 5097 { 5098 int i, err; 5099 struct hashmap *names_map; 5100 5101 names_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL); 5102 if (IS_ERR(names_map)) 5103 return PTR_ERR(names_map); 5104 5105 err = btf_dedup_fill_unique_names_map(d, names_map); 5106 if (err < 0) 5107 goto exit; 5108 5109 for (i = 0; i < d->btf->nr_types; i++) { 5110 err = btf_dedup_resolve_fwd(d, names_map, d->btf->start_id + i); 5111 if (err < 0) 5112 break; 5113 } 5114 5115 exit: 5116 hashmap__free(names_map); 5117 return err; 5118 } 5119 5120 /* 5121 * Compact types. 5122 * 5123 * After we established for each type its corresponding canonical representative 5124 * type, we now can eliminate types that are not canonical and leave only 5125 * canonical ones layed out sequentially in memory by copying them over 5126 * duplicates. During compaction btf_dedup->hypot_map array is reused to store 5127 * a map from original type ID to a new compacted type ID, which will be used 5128 * during next phase to "fix up" type IDs, referenced from struct/union and 5129 * reference types. 5130 */ 5131 static int btf_dedup_compact_types(struct btf_dedup *d) 5132 { 5133 __u32 *new_offs; 5134 __u32 next_type_id = d->btf->start_id; 5135 const struct btf_type *t; 5136 void *p; 5137 int i, id, len; 5138 5139 /* we are going to reuse hypot_map to store compaction remapping */ 5140 d->hypot_map[0] = 0; 5141 /* base BTF types are not renumbered */ 5142 for (id = 1; id < d->btf->start_id; id++) 5143 d->hypot_map[id] = id; 5144 for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) 5145 d->hypot_map[id] = BTF_UNPROCESSED_ID; 5146 5147 p = d->btf->types_data; 5148 5149 for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) { 5150 if (d->map[id] != id) 5151 continue; 5152 5153 t = btf__type_by_id(d->btf, id); 5154 len = btf_type_size(t); 5155 if (len < 0) 5156 return len; 5157 5158 memmove(p, t, len); 5159 d->hypot_map[id] = next_type_id; 5160 d->btf->type_offs[next_type_id - d->btf->start_id] = p - d->btf->types_data; 5161 p += len; 5162 next_type_id++; 5163 } 5164 5165 /* shrink struct btf's internal types index and update btf_header */ 5166 d->btf->nr_types = next_type_id - d->btf->start_id; 5167 d->btf->type_offs_cap = d->btf->nr_types; 5168 d->btf->hdr->type_len = p - d->btf->types_data; 5169 new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap, 5170 sizeof(*new_offs)); 5171 if (d->btf->type_offs_cap && !new_offs) 5172 return -ENOMEM; 5173 d->btf->type_offs = new_offs; 5174 d->btf->hdr->str_off = d->btf->hdr->type_len; 5175 d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len; 5176 return 0; 5177 } 5178 5179 /* 5180 * Figure out final (deduplicated and compacted) type ID for provided original 5181 * `type_id` by first resolving it into corresponding canonical type ID and 5182 * then mapping it to a deduplicated type ID, stored in btf_dedup->hypot_map, 5183 * which is populated during compaction phase. 5184 */ 5185 static int btf_dedup_remap_type_id(__u32 *type_id, void *ctx) 5186 { 5187 struct btf_dedup *d = ctx; 5188 __u32 resolved_type_id, new_type_id; 5189 5190 resolved_type_id = resolve_type_id(d, *type_id); 5191 new_type_id = d->hypot_map[resolved_type_id]; 5192 if (new_type_id > BTF_MAX_NR_TYPES) 5193 return -EINVAL; 5194 5195 *type_id = new_type_id; 5196 return 0; 5197 } 5198 5199 /* 5200 * Remap referenced type IDs into deduped type IDs. 5201 * 5202 * After BTF types are deduplicated and compacted, their final type IDs may 5203 * differ from original ones. The map from original to a corresponding 5204 * deduped type ID is stored in btf_dedup->hypot_map and is populated during 5205 * compaction phase. During remapping phase we are rewriting all type IDs 5206 * referenced from any BTF type (e.g., struct fields, func proto args, etc) to 5207 * their final deduped type IDs. 5208 */ 5209 static int btf_dedup_remap_types(struct btf_dedup *d) 5210 { 5211 int i, r; 5212 5213 for (i = 0; i < d->btf->nr_types; i++) { 5214 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i); 5215 struct btf_field_iter it; 5216 __u32 *type_id; 5217 5218 r = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS); 5219 if (r) 5220 return r; 5221 5222 while ((type_id = btf_field_iter_next(&it))) { 5223 __u32 resolved_id, new_id; 5224 5225 resolved_id = resolve_type_id(d, *type_id); 5226 new_id = d->hypot_map[resolved_id]; 5227 if (new_id > BTF_MAX_NR_TYPES) 5228 return -EINVAL; 5229 5230 *type_id = new_id; 5231 } 5232 } 5233 5234 if (!d->btf_ext) 5235 return 0; 5236 5237 r = btf_ext_visit_type_ids(d->btf_ext, btf_dedup_remap_type_id, d); 5238 if (r) 5239 return r; 5240 5241 return 0; 5242 } 5243 5244 /* 5245 * Probe few well-known locations for vmlinux kernel image and try to load BTF 5246 * data out of it to use for target BTF. 5247 */ 5248 struct btf *btf__load_vmlinux_btf(void) 5249 { 5250 const char *sysfs_btf_path = "/sys/kernel/btf/vmlinux"; 5251 /* fall back locations, trying to find vmlinux on disk */ 5252 const char *locations[] = { 5253 "/boot/vmlinux-%1$s", 5254 "/lib/modules/%1$s/vmlinux-%1$s", 5255 "/lib/modules/%1$s/build/vmlinux", 5256 "/usr/lib/modules/%1$s/kernel/vmlinux", 5257 "/usr/lib/debug/boot/vmlinux-%1$s", 5258 "/usr/lib/debug/boot/vmlinux-%1$s.debug", 5259 "/usr/lib/debug/lib/modules/%1$s/vmlinux", 5260 }; 5261 char path[PATH_MAX + 1]; 5262 struct utsname buf; 5263 struct btf *btf; 5264 int i, err; 5265 5266 /* is canonical sysfs location accessible? */ 5267 if (faccessat(AT_FDCWD, sysfs_btf_path, F_OK, AT_EACCESS) < 0) { 5268 pr_warn("kernel BTF is missing at '%s', was CONFIG_DEBUG_INFO_BTF enabled?\n", 5269 sysfs_btf_path); 5270 } else { 5271 btf = btf__parse(sysfs_btf_path, NULL); 5272 if (!btf) { 5273 err = -errno; 5274 pr_warn("failed to read kernel BTF from '%s': %s\n", 5275 sysfs_btf_path, errstr(err)); 5276 return libbpf_err_ptr(err); 5277 } 5278 pr_debug("loaded kernel BTF from '%s'\n", sysfs_btf_path); 5279 return btf; 5280 } 5281 5282 /* try fallback locations */ 5283 uname(&buf); 5284 for (i = 0; i < ARRAY_SIZE(locations); i++) { 5285 snprintf(path, PATH_MAX, locations[i], buf.release); 5286 5287 if (faccessat(AT_FDCWD, path, R_OK, AT_EACCESS)) 5288 continue; 5289 5290 btf = btf__parse(path, NULL); 5291 err = libbpf_get_error(btf); 5292 pr_debug("loading kernel BTF '%s': %s\n", path, errstr(err)); 5293 if (err) 5294 continue; 5295 5296 return btf; 5297 } 5298 5299 pr_warn("failed to find valid kernel BTF\n"); 5300 return libbpf_err_ptr(-ESRCH); 5301 } 5302 5303 struct btf *libbpf_find_kernel_btf(void) __attribute__((alias("btf__load_vmlinux_btf"))); 5304 5305 struct btf *btf__load_module_btf(const char *module_name, struct btf *vmlinux_btf) 5306 { 5307 char path[80]; 5308 5309 snprintf(path, sizeof(path), "/sys/kernel/btf/%s", module_name); 5310 return btf__parse_split(path, vmlinux_btf); 5311 } 5312 5313 int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx) 5314 { 5315 const struct btf_ext_info *seg; 5316 struct btf_ext_info_sec *sec; 5317 int i, err; 5318 5319 seg = &btf_ext->func_info; 5320 for_each_btf_ext_sec(seg, sec) { 5321 struct bpf_func_info_min *rec; 5322 5323 for_each_btf_ext_rec(seg, sec, i, rec) { 5324 err = visit(&rec->type_id, ctx); 5325 if (err < 0) 5326 return err; 5327 } 5328 } 5329 5330 seg = &btf_ext->core_relo_info; 5331 for_each_btf_ext_sec(seg, sec) { 5332 struct bpf_core_relo *rec; 5333 5334 for_each_btf_ext_rec(seg, sec, i, rec) { 5335 err = visit(&rec->type_id, ctx); 5336 if (err < 0) 5337 return err; 5338 } 5339 } 5340 5341 return 0; 5342 } 5343 5344 int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx) 5345 { 5346 const struct btf_ext_info *seg; 5347 struct btf_ext_info_sec *sec; 5348 int i, err; 5349 5350 seg = &btf_ext->func_info; 5351 for_each_btf_ext_sec(seg, sec) { 5352 err = visit(&sec->sec_name_off, ctx); 5353 if (err) 5354 return err; 5355 } 5356 5357 seg = &btf_ext->line_info; 5358 for_each_btf_ext_sec(seg, sec) { 5359 struct bpf_line_info_min *rec; 5360 5361 err = visit(&sec->sec_name_off, ctx); 5362 if (err) 5363 return err; 5364 5365 for_each_btf_ext_rec(seg, sec, i, rec) { 5366 err = visit(&rec->file_name_off, ctx); 5367 if (err) 5368 return err; 5369 err = visit(&rec->line_off, ctx); 5370 if (err) 5371 return err; 5372 } 5373 } 5374 5375 seg = &btf_ext->core_relo_info; 5376 for_each_btf_ext_sec(seg, sec) { 5377 struct bpf_core_relo *rec; 5378 5379 err = visit(&sec->sec_name_off, ctx); 5380 if (err) 5381 return err; 5382 5383 for_each_btf_ext_rec(seg, sec, i, rec) { 5384 err = visit(&rec->access_str_off, ctx); 5385 if (err) 5386 return err; 5387 } 5388 } 5389 5390 return 0; 5391 } 5392 5393 struct btf_distill { 5394 struct btf_pipe pipe; 5395 int *id_map; 5396 unsigned int split_start_id; 5397 unsigned int split_start_str; 5398 int diff_id; 5399 }; 5400 5401 static int btf_add_distilled_type_ids(struct btf_distill *dist, __u32 i) 5402 { 5403 struct btf_type *split_t = btf_type_by_id(dist->pipe.src, i); 5404 struct btf_field_iter it; 5405 __u32 *id; 5406 int err; 5407 5408 err = btf_field_iter_init(&it, split_t, BTF_FIELD_ITER_IDS); 5409 if (err) 5410 return err; 5411 while ((id = btf_field_iter_next(&it))) { 5412 struct btf_type *base_t; 5413 5414 if (!*id) 5415 continue; 5416 /* split BTF id, not needed */ 5417 if (*id >= dist->split_start_id) 5418 continue; 5419 /* already added ? */ 5420 if (dist->id_map[*id] > 0) 5421 continue; 5422 5423 /* only a subset of base BTF types should be referenced from 5424 * split BTF; ensure nothing unexpected is referenced. 5425 */ 5426 base_t = btf_type_by_id(dist->pipe.src, *id); 5427 switch (btf_kind(base_t)) { 5428 case BTF_KIND_INT: 5429 case BTF_KIND_FLOAT: 5430 case BTF_KIND_FWD: 5431 case BTF_KIND_ARRAY: 5432 case BTF_KIND_STRUCT: 5433 case BTF_KIND_UNION: 5434 case BTF_KIND_TYPEDEF: 5435 case BTF_KIND_ENUM: 5436 case BTF_KIND_ENUM64: 5437 case BTF_KIND_PTR: 5438 case BTF_KIND_CONST: 5439 case BTF_KIND_RESTRICT: 5440 case BTF_KIND_VOLATILE: 5441 case BTF_KIND_FUNC_PROTO: 5442 case BTF_KIND_TYPE_TAG: 5443 dist->id_map[*id] = *id; 5444 break; 5445 default: 5446 pr_warn("unexpected reference to base type[%u] of kind [%u] when creating distilled base BTF.\n", 5447 *id, btf_kind(base_t)); 5448 return -EINVAL; 5449 } 5450 /* If a base type is used, ensure types it refers to are 5451 * marked as used also; so for example if we find a PTR to INT 5452 * we need both the PTR and INT. 5453 * 5454 * The only exception is named struct/unions, since distilled 5455 * base BTF composite types have no members. 5456 */ 5457 if (btf_is_composite(base_t) && base_t->name_off) 5458 continue; 5459 err = btf_add_distilled_type_ids(dist, *id); 5460 if (err) 5461 return err; 5462 } 5463 return 0; 5464 } 5465 5466 static int btf_add_distilled_types(struct btf_distill *dist) 5467 { 5468 bool adding_to_base = dist->pipe.dst->start_id == 1; 5469 int id = btf__type_cnt(dist->pipe.dst); 5470 struct btf_type *t; 5471 int i, err = 0; 5472 5473 5474 /* Add types for each of the required references to either distilled 5475 * base or split BTF, depending on type characteristics. 5476 */ 5477 for (i = 1; i < dist->split_start_id; i++) { 5478 const char *name; 5479 int kind; 5480 5481 if (!dist->id_map[i]) 5482 continue; 5483 t = btf_type_by_id(dist->pipe.src, i); 5484 kind = btf_kind(t); 5485 name = btf__name_by_offset(dist->pipe.src, t->name_off); 5486 5487 switch (kind) { 5488 case BTF_KIND_INT: 5489 case BTF_KIND_FLOAT: 5490 case BTF_KIND_FWD: 5491 /* Named int, float, fwd are added to base. */ 5492 if (!adding_to_base) 5493 continue; 5494 err = btf_add_type(&dist->pipe, t); 5495 break; 5496 case BTF_KIND_STRUCT: 5497 case BTF_KIND_UNION: 5498 /* Named struct/union are added to base as 0-vlen 5499 * struct/union of same size. Anonymous struct/unions 5500 * are added to split BTF as-is. 5501 */ 5502 if (adding_to_base) { 5503 if (!t->name_off) 5504 continue; 5505 err = btf_add_composite(dist->pipe.dst, kind, name, t->size); 5506 } else { 5507 if (t->name_off) 5508 continue; 5509 err = btf_add_type(&dist->pipe, t); 5510 } 5511 break; 5512 case BTF_KIND_ENUM: 5513 case BTF_KIND_ENUM64: 5514 /* Named enum[64]s are added to base as a sized 5515 * enum; relocation will match with appropriately-named 5516 * and sized enum or enum64. 5517 * 5518 * Anonymous enums are added to split BTF as-is. 5519 */ 5520 if (adding_to_base) { 5521 if (!t->name_off) 5522 continue; 5523 err = btf__add_enum(dist->pipe.dst, name, t->size); 5524 } else { 5525 if (t->name_off) 5526 continue; 5527 err = btf_add_type(&dist->pipe, t); 5528 } 5529 break; 5530 case BTF_KIND_ARRAY: 5531 case BTF_KIND_TYPEDEF: 5532 case BTF_KIND_PTR: 5533 case BTF_KIND_CONST: 5534 case BTF_KIND_RESTRICT: 5535 case BTF_KIND_VOLATILE: 5536 case BTF_KIND_FUNC_PROTO: 5537 case BTF_KIND_TYPE_TAG: 5538 /* All other types are added to split BTF. */ 5539 if (adding_to_base) 5540 continue; 5541 err = btf_add_type(&dist->pipe, t); 5542 break; 5543 default: 5544 pr_warn("unexpected kind when adding base type '%s'[%u] of kind [%u] to distilled base BTF.\n", 5545 name, i, kind); 5546 return -EINVAL; 5547 5548 } 5549 if (err < 0) 5550 break; 5551 dist->id_map[i] = id++; 5552 } 5553 return err; 5554 } 5555 5556 /* Split BTF ids without a mapping will be shifted downwards since distilled 5557 * base BTF is smaller than the original base BTF. For those that have a 5558 * mapping (either to base or updated split BTF), update the id based on 5559 * that mapping. 5560 */ 5561 static int btf_update_distilled_type_ids(struct btf_distill *dist, __u32 i) 5562 { 5563 struct btf_type *t = btf_type_by_id(dist->pipe.dst, i); 5564 struct btf_field_iter it; 5565 __u32 *id; 5566 int err; 5567 5568 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS); 5569 if (err) 5570 return err; 5571 while ((id = btf_field_iter_next(&it))) { 5572 if (dist->id_map[*id]) 5573 *id = dist->id_map[*id]; 5574 else if (*id >= dist->split_start_id) 5575 *id -= dist->diff_id; 5576 } 5577 return 0; 5578 } 5579 5580 /* Create updated split BTF with distilled base BTF; distilled base BTF 5581 * consists of BTF information required to clarify the types that split 5582 * BTF refers to, omitting unneeded details. Specifically it will contain 5583 * base types and memberless definitions of named structs, unions and enumerated 5584 * types. Associated reference types like pointers, arrays and anonymous 5585 * structs, unions and enumerated types will be added to split BTF. 5586 * Size is recorded for named struct/unions to help guide matching to the 5587 * target base BTF during later relocation. 5588 * 5589 * The only case where structs, unions or enumerated types are fully represented 5590 * is when they are anonymous; in such cases, the anonymous type is added to 5591 * split BTF in full. 5592 * 5593 * We return newly-created split BTF where the split BTF refers to a newly-created 5594 * distilled base BTF. Both must be freed separately by the caller. 5595 */ 5596 int btf__distill_base(const struct btf *src_btf, struct btf **new_base_btf, 5597 struct btf **new_split_btf) 5598 { 5599 struct btf *new_base = NULL, *new_split = NULL; 5600 const struct btf *old_base; 5601 unsigned int n = btf__type_cnt(src_btf); 5602 struct btf_distill dist = {}; 5603 struct btf_type *t; 5604 int i, err = 0; 5605 5606 /* src BTF must be split BTF. */ 5607 old_base = btf__base_btf(src_btf); 5608 if (!new_base_btf || !new_split_btf || !old_base) 5609 return libbpf_err(-EINVAL); 5610 5611 new_base = btf__new_empty(); 5612 if (!new_base) 5613 return libbpf_err(-ENOMEM); 5614 5615 btf__set_endianness(new_base, btf__endianness(src_btf)); 5616 5617 dist.id_map = calloc(n, sizeof(*dist.id_map)); 5618 if (!dist.id_map) { 5619 err = -ENOMEM; 5620 goto done; 5621 } 5622 dist.pipe.src = src_btf; 5623 dist.pipe.dst = new_base; 5624 dist.pipe.str_off_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL); 5625 if (IS_ERR(dist.pipe.str_off_map)) { 5626 err = -ENOMEM; 5627 goto done; 5628 } 5629 dist.split_start_id = btf__type_cnt(old_base); 5630 dist.split_start_str = old_base->hdr->str_len; 5631 5632 /* Pass over src split BTF; generate the list of base BTF type ids it 5633 * references; these will constitute our distilled BTF set to be 5634 * distributed over base and split BTF as appropriate. 5635 */ 5636 for (i = src_btf->start_id; i < n; i++) { 5637 err = btf_add_distilled_type_ids(&dist, i); 5638 if (err < 0) 5639 goto done; 5640 } 5641 /* Next add types for each of the required references to base BTF and split BTF 5642 * in turn. 5643 */ 5644 err = btf_add_distilled_types(&dist); 5645 if (err < 0) 5646 goto done; 5647 5648 /* Create new split BTF with distilled base BTF as its base; the final 5649 * state is split BTF with distilled base BTF that represents enough 5650 * about its base references to allow it to be relocated with the base 5651 * BTF available. 5652 */ 5653 new_split = btf__new_empty_split(new_base); 5654 if (!new_split) { 5655 err = -errno; 5656 goto done; 5657 } 5658 dist.pipe.dst = new_split; 5659 /* First add all split types */ 5660 for (i = src_btf->start_id; i < n; i++) { 5661 t = btf_type_by_id(src_btf, i); 5662 err = btf_add_type(&dist.pipe, t); 5663 if (err < 0) 5664 goto done; 5665 } 5666 /* Now add distilled types to split BTF that are not added to base. */ 5667 err = btf_add_distilled_types(&dist); 5668 if (err < 0) 5669 goto done; 5670 5671 /* All split BTF ids will be shifted downwards since there are less base 5672 * BTF ids in distilled base BTF. 5673 */ 5674 dist.diff_id = dist.split_start_id - btf__type_cnt(new_base); 5675 5676 n = btf__type_cnt(new_split); 5677 /* Now update base/split BTF ids. */ 5678 for (i = 1; i < n; i++) { 5679 err = btf_update_distilled_type_ids(&dist, i); 5680 if (err < 0) 5681 break; 5682 } 5683 done: 5684 free(dist.id_map); 5685 hashmap__free(dist.pipe.str_off_map); 5686 if (err) { 5687 btf__free(new_split); 5688 btf__free(new_base); 5689 return libbpf_err(err); 5690 } 5691 *new_base_btf = new_base; 5692 *new_split_btf = new_split; 5693 5694 return 0; 5695 } 5696 5697 const struct btf_header *btf_header(const struct btf *btf) 5698 { 5699 return btf->hdr; 5700 } 5701 5702 void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) 5703 { 5704 btf->base_btf = (struct btf *)base_btf; 5705 btf->start_id = btf__type_cnt(base_btf); 5706 btf->start_str_off = base_btf->hdr->str_len; 5707 } 5708 5709 int btf__relocate(struct btf *btf, const struct btf *base_btf) 5710 { 5711 int err = btf_relocate(btf, base_btf, NULL); 5712 5713 if (!err) 5714 btf->owns_base = false; 5715 return libbpf_err(err); 5716 } 5717