1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* Copyright (c) 2018 Facebook */ 3 4 #include <byteswap.h> 5 #include <endian.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <fcntl.h> 10 #include <unistd.h> 11 #include <errno.h> 12 #include <sys/utsname.h> 13 #include <sys/param.h> 14 #include <sys/stat.h> 15 #include <linux/kernel.h> 16 #include <linux/err.h> 17 #include <linux/btf.h> 18 #include <gelf.h> 19 #include "btf.h" 20 #include "bpf.h" 21 #include "libbpf.h" 22 #include "libbpf_internal.h" 23 #include "hashmap.h" 24 #include "strset.h" 25 #include "str_error.h" 26 27 #define BTF_MAX_NR_TYPES 0x7fffffffU 28 #define BTF_MAX_STR_OFFSET 0x7fffffffU 29 30 static struct btf_type btf_void; 31 32 struct btf { 33 /* raw BTF data in native endianness */ 34 void *raw_data; 35 /* raw BTF data in non-native endianness */ 36 void *raw_data_swapped; 37 __u32 raw_size; 38 /* whether target endianness differs from the native one */ 39 bool swapped_endian; 40 41 /* 42 * When BTF is loaded from an ELF or raw memory it is stored 43 * in a contiguous memory block. The hdr, type_data, and, strs_data 44 * point inside that memory region to their respective parts of BTF 45 * representation: 46 * 47 * +--------------------------------+ 48 * | Header | Types | Strings | 49 * +--------------------------------+ 50 * ^ ^ ^ 51 * | | | 52 * hdr | | 53 * types_data-+ | 54 * strs_data------------+ 55 * 56 * If BTF data is later modified, e.g., due to types added or 57 * removed, BTF deduplication performed, etc, this contiguous 58 * representation is broken up into three independently allocated 59 * memory regions to be able to modify them independently. 60 * raw_data is nulled out at that point, but can be later allocated 61 * and cached again if user calls btf__raw_data(), at which point 62 * raw_data will contain a contiguous copy of header, types, and 63 * strings: 64 * 65 * +----------+ +---------+ +-----------+ 66 * | Header | | Types | | Strings | 67 * +----------+ +---------+ +-----------+ 68 * ^ ^ ^ 69 * | | | 70 * hdr | | 71 * types_data----+ | 72 * strset__data(strs_set)-----+ 73 * 74 * +----------+---------+-----------+ 75 * | Header | Types | Strings | 76 * raw_data----->+----------+---------+-----------+ 77 */ 78 struct btf_header *hdr; 79 80 void *types_data; 81 size_t types_data_cap; /* used size stored in hdr->type_len */ 82 83 /* type ID to `struct btf_type *` lookup index 84 * type_offs[0] corresponds to the first non-VOID type: 85 * - for base BTF it's type [1]; 86 * - for split BTF it's the first non-base BTF type. 87 */ 88 __u32 *type_offs; 89 size_t type_offs_cap; 90 /* number of types in this BTF instance: 91 * - doesn't include special [0] void type; 92 * - for split BTF counts number of types added on top of base BTF. 93 */ 94 __u32 nr_types; 95 /* if not NULL, points to the base BTF on top of which the current 96 * split BTF is based 97 */ 98 struct btf *base_btf; 99 /* BTF type ID of the first type in this BTF instance: 100 * - for base BTF it's equal to 1; 101 * - for split BTF it's equal to biggest type ID of base BTF plus 1. 102 */ 103 int start_id; 104 /* logical string offset of this BTF instance: 105 * - for base BTF it's equal to 0; 106 * - for split BTF it's equal to total size of base BTF's string section size. 107 */ 108 int start_str_off; 109 110 /* only one of strs_data or strs_set can be non-NULL, depending on 111 * whether BTF is in a modifiable state (strs_set is used) or not 112 * (strs_data points inside raw_data) 113 */ 114 void *strs_data; 115 /* a set of unique strings */ 116 struct strset *strs_set; 117 /* whether strings are already deduplicated */ 118 bool strs_deduped; 119 120 /* whether base_btf should be freed in btf_free for this instance */ 121 bool owns_base; 122 123 /* BTF object FD, if loaded into kernel */ 124 int fd; 125 126 /* Pointer size (in bytes) for a target architecture of this BTF */ 127 int ptr_sz; 128 }; 129 130 static inline __u64 ptr_to_u64(const void *ptr) 131 { 132 return (__u64) (unsigned long) ptr; 133 } 134 135 /* Ensure given dynamically allocated memory region pointed to by *data* with 136 * capacity of *cap_cnt* elements each taking *elem_sz* bytes has enough 137 * memory to accommodate *add_cnt* new elements, assuming *cur_cnt* elements 138 * are already used. At most *max_cnt* elements can be ever allocated. 139 * If necessary, memory is reallocated and all existing data is copied over, 140 * new pointer to the memory region is stored at *data, new memory region 141 * capacity (in number of elements) is stored in *cap. 142 * On success, memory pointer to the beginning of unused memory is returned. 143 * On error, NULL is returned. 144 */ 145 void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, 146 size_t cur_cnt, size_t max_cnt, size_t add_cnt) 147 { 148 size_t new_cnt; 149 void *new_data; 150 151 if (cur_cnt + add_cnt <= *cap_cnt) 152 return *data + cur_cnt * elem_sz; 153 154 /* requested more than the set limit */ 155 if (cur_cnt + add_cnt > max_cnt) 156 return NULL; 157 158 new_cnt = *cap_cnt; 159 new_cnt += new_cnt / 4; /* expand by 25% */ 160 if (new_cnt < 16) /* but at least 16 elements */ 161 new_cnt = 16; 162 if (new_cnt > max_cnt) /* but not exceeding a set limit */ 163 new_cnt = max_cnt; 164 if (new_cnt < cur_cnt + add_cnt) /* also ensure we have enough memory */ 165 new_cnt = cur_cnt + add_cnt; 166 167 new_data = libbpf_reallocarray(*data, new_cnt, elem_sz); 168 if (!new_data) 169 return NULL; 170 171 /* zero out newly allocated portion of memory */ 172 memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz); 173 174 *data = new_data; 175 *cap_cnt = new_cnt; 176 return new_data + cur_cnt * elem_sz; 177 } 178 179 /* Ensure given dynamically allocated memory region has enough allocated space 180 * to accommodate *need_cnt* elements of size *elem_sz* bytes each 181 */ 182 int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt) 183 { 184 void *p; 185 186 if (need_cnt <= *cap_cnt) 187 return 0; 188 189 p = libbpf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt); 190 if (!p) 191 return -ENOMEM; 192 193 return 0; 194 } 195 196 static void *btf_add_type_offs_mem(struct btf *btf, size_t add_cnt) 197 { 198 return libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32), 199 btf->nr_types, BTF_MAX_NR_TYPES, add_cnt); 200 } 201 202 static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off) 203 { 204 __u32 *p; 205 206 p = btf_add_type_offs_mem(btf, 1); 207 if (!p) 208 return -ENOMEM; 209 210 *p = type_off; 211 return 0; 212 } 213 214 static void btf_bswap_hdr(struct btf_header *h) 215 { 216 h->magic = bswap_16(h->magic); 217 h->hdr_len = bswap_32(h->hdr_len); 218 h->type_off = bswap_32(h->type_off); 219 h->type_len = bswap_32(h->type_len); 220 h->str_off = bswap_32(h->str_off); 221 h->str_len = bswap_32(h->str_len); 222 } 223 224 static int btf_parse_hdr(struct btf *btf) 225 { 226 struct btf_header *hdr = btf->hdr; 227 __u32 meta_left; 228 229 if (btf->raw_size < sizeof(struct btf_header)) { 230 pr_debug("BTF header not found\n"); 231 return -EINVAL; 232 } 233 234 if (hdr->magic == bswap_16(BTF_MAGIC)) { 235 btf->swapped_endian = true; 236 if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) { 237 pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n", 238 bswap_32(hdr->hdr_len)); 239 return -ENOTSUP; 240 } 241 btf_bswap_hdr(hdr); 242 } else if (hdr->magic != BTF_MAGIC) { 243 pr_debug("Invalid BTF magic: %x\n", hdr->magic); 244 return -EINVAL; 245 } 246 247 if (btf->raw_size < hdr->hdr_len) { 248 pr_debug("BTF header len %u larger than data size %u\n", 249 hdr->hdr_len, btf->raw_size); 250 return -EINVAL; 251 } 252 253 meta_left = btf->raw_size - hdr->hdr_len; 254 if (meta_left < (long long)hdr->str_off + hdr->str_len) { 255 pr_debug("Invalid BTF total size: %u\n", btf->raw_size); 256 return -EINVAL; 257 } 258 259 if ((long long)hdr->type_off + hdr->type_len > hdr->str_off) { 260 pr_debug("Invalid BTF data sections layout: type data at %u + %u, strings data at %u + %u\n", 261 hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len); 262 return -EINVAL; 263 } 264 265 if (hdr->type_off % 4) { 266 pr_debug("BTF type section is not aligned to 4 bytes\n"); 267 return -EINVAL; 268 } 269 270 return 0; 271 } 272 273 static int btf_parse_str_sec(struct btf *btf) 274 { 275 const struct btf_header *hdr = btf->hdr; 276 const char *start = btf->strs_data; 277 const char *end = start + btf->hdr->str_len; 278 279 if (btf->base_btf && hdr->str_len == 0) 280 return 0; 281 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) { 282 pr_debug("Invalid BTF string section\n"); 283 return -EINVAL; 284 } 285 if (!btf->base_btf && start[0]) { 286 pr_debug("Malformed BTF string section, did you forget to provide base BTF?\n"); 287 return -EINVAL; 288 } 289 return 0; 290 } 291 292 static int btf_type_size(const struct btf_type *t) 293 { 294 const int base_size = sizeof(struct btf_type); 295 __u16 vlen = btf_vlen(t); 296 297 switch (btf_kind(t)) { 298 case BTF_KIND_FWD: 299 case BTF_KIND_CONST: 300 case BTF_KIND_VOLATILE: 301 case BTF_KIND_RESTRICT: 302 case BTF_KIND_PTR: 303 case BTF_KIND_TYPEDEF: 304 case BTF_KIND_FUNC: 305 case BTF_KIND_FLOAT: 306 case BTF_KIND_TYPE_TAG: 307 return base_size; 308 case BTF_KIND_INT: 309 return base_size + sizeof(__u32); 310 case BTF_KIND_ENUM: 311 return base_size + vlen * sizeof(struct btf_enum); 312 case BTF_KIND_ENUM64: 313 return base_size + vlen * sizeof(struct btf_enum64); 314 case BTF_KIND_ARRAY: 315 return base_size + sizeof(struct btf_array); 316 case BTF_KIND_STRUCT: 317 case BTF_KIND_UNION: 318 return base_size + vlen * sizeof(struct btf_member); 319 case BTF_KIND_FUNC_PROTO: 320 return base_size + vlen * sizeof(struct btf_param); 321 case BTF_KIND_VAR: 322 return base_size + sizeof(struct btf_var); 323 case BTF_KIND_DATASEC: 324 return base_size + vlen * sizeof(struct btf_var_secinfo); 325 case BTF_KIND_DECL_TAG: 326 return base_size + sizeof(struct btf_decl_tag); 327 default: 328 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t)); 329 return -EINVAL; 330 } 331 } 332 333 static void btf_bswap_type_base(struct btf_type *t) 334 { 335 t->name_off = bswap_32(t->name_off); 336 t->info = bswap_32(t->info); 337 t->type = bswap_32(t->type); 338 } 339 340 static int btf_bswap_type_rest(struct btf_type *t) 341 { 342 struct btf_var_secinfo *v; 343 struct btf_enum64 *e64; 344 struct btf_member *m; 345 struct btf_array *a; 346 struct btf_param *p; 347 struct btf_enum *e; 348 __u16 vlen = btf_vlen(t); 349 int i; 350 351 switch (btf_kind(t)) { 352 case BTF_KIND_FWD: 353 case BTF_KIND_CONST: 354 case BTF_KIND_VOLATILE: 355 case BTF_KIND_RESTRICT: 356 case BTF_KIND_PTR: 357 case BTF_KIND_TYPEDEF: 358 case BTF_KIND_FUNC: 359 case BTF_KIND_FLOAT: 360 case BTF_KIND_TYPE_TAG: 361 return 0; 362 case BTF_KIND_INT: 363 *(__u32 *)(t + 1) = bswap_32(*(__u32 *)(t + 1)); 364 return 0; 365 case BTF_KIND_ENUM: 366 for (i = 0, e = btf_enum(t); i < vlen; i++, e++) { 367 e->name_off = bswap_32(e->name_off); 368 e->val = bswap_32(e->val); 369 } 370 return 0; 371 case BTF_KIND_ENUM64: 372 for (i = 0, e64 = btf_enum64(t); i < vlen; i++, e64++) { 373 e64->name_off = bswap_32(e64->name_off); 374 e64->val_lo32 = bswap_32(e64->val_lo32); 375 e64->val_hi32 = bswap_32(e64->val_hi32); 376 } 377 return 0; 378 case BTF_KIND_ARRAY: 379 a = btf_array(t); 380 a->type = bswap_32(a->type); 381 a->index_type = bswap_32(a->index_type); 382 a->nelems = bswap_32(a->nelems); 383 return 0; 384 case BTF_KIND_STRUCT: 385 case BTF_KIND_UNION: 386 for (i = 0, m = btf_members(t); i < vlen; i++, m++) { 387 m->name_off = bswap_32(m->name_off); 388 m->type = bswap_32(m->type); 389 m->offset = bswap_32(m->offset); 390 } 391 return 0; 392 case BTF_KIND_FUNC_PROTO: 393 for (i = 0, p = btf_params(t); i < vlen; i++, p++) { 394 p->name_off = bswap_32(p->name_off); 395 p->type = bswap_32(p->type); 396 } 397 return 0; 398 case BTF_KIND_VAR: 399 btf_var(t)->linkage = bswap_32(btf_var(t)->linkage); 400 return 0; 401 case BTF_KIND_DATASEC: 402 for (i = 0, v = btf_var_secinfos(t); i < vlen; i++, v++) { 403 v->type = bswap_32(v->type); 404 v->offset = bswap_32(v->offset); 405 v->size = bswap_32(v->size); 406 } 407 return 0; 408 case BTF_KIND_DECL_TAG: 409 btf_decl_tag(t)->component_idx = bswap_32(btf_decl_tag(t)->component_idx); 410 return 0; 411 default: 412 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t)); 413 return -EINVAL; 414 } 415 } 416 417 static int btf_parse_type_sec(struct btf *btf) 418 { 419 struct btf_header *hdr = btf->hdr; 420 void *next_type = btf->types_data; 421 void *end_type = next_type + hdr->type_len; 422 int err, type_size; 423 424 while (next_type + sizeof(struct btf_type) <= end_type) { 425 if (btf->swapped_endian) 426 btf_bswap_type_base(next_type); 427 428 type_size = btf_type_size(next_type); 429 if (type_size < 0) 430 return type_size; 431 if (next_type + type_size > end_type) { 432 pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types); 433 return -EINVAL; 434 } 435 436 if (btf->swapped_endian && btf_bswap_type_rest(next_type)) 437 return -EINVAL; 438 439 err = btf_add_type_idx_entry(btf, next_type - btf->types_data); 440 if (err) 441 return err; 442 443 next_type += type_size; 444 btf->nr_types++; 445 } 446 447 if (next_type != end_type) { 448 pr_warn("BTF types data is malformed\n"); 449 return -EINVAL; 450 } 451 452 return 0; 453 } 454 455 static int btf_validate_str(const struct btf *btf, __u32 str_off, const char *what, __u32 type_id) 456 { 457 const char *s; 458 459 s = btf__str_by_offset(btf, str_off); 460 if (!s) { 461 pr_warn("btf: type [%u]: invalid %s (string offset %u)\n", type_id, what, str_off); 462 return -EINVAL; 463 } 464 465 return 0; 466 } 467 468 static int btf_validate_id(const struct btf *btf, __u32 id, __u32 ctx_id) 469 { 470 const struct btf_type *t; 471 472 t = btf__type_by_id(btf, id); 473 if (!t) { 474 pr_warn("btf: type [%u]: invalid referenced type ID %u\n", ctx_id, id); 475 return -EINVAL; 476 } 477 478 return 0; 479 } 480 481 static int btf_validate_type(const struct btf *btf, const struct btf_type *t, __u32 id) 482 { 483 __u32 kind = btf_kind(t); 484 int err, i, n; 485 486 err = btf_validate_str(btf, t->name_off, "type name", id); 487 if (err) 488 return err; 489 490 switch (kind) { 491 case BTF_KIND_UNKN: 492 case BTF_KIND_INT: 493 case BTF_KIND_FWD: 494 case BTF_KIND_FLOAT: 495 break; 496 case BTF_KIND_PTR: 497 case BTF_KIND_TYPEDEF: 498 case BTF_KIND_VOLATILE: 499 case BTF_KIND_CONST: 500 case BTF_KIND_RESTRICT: 501 case BTF_KIND_VAR: 502 case BTF_KIND_DECL_TAG: 503 case BTF_KIND_TYPE_TAG: 504 err = btf_validate_id(btf, t->type, id); 505 if (err) 506 return err; 507 break; 508 case BTF_KIND_ARRAY: { 509 const struct btf_array *a = btf_array(t); 510 511 err = btf_validate_id(btf, a->type, id); 512 err = err ?: btf_validate_id(btf, a->index_type, id); 513 if (err) 514 return err; 515 break; 516 } 517 case BTF_KIND_STRUCT: 518 case BTF_KIND_UNION: { 519 const struct btf_member *m = btf_members(t); 520 521 n = btf_vlen(t); 522 for (i = 0; i < n; i++, m++) { 523 err = btf_validate_str(btf, m->name_off, "field name", id); 524 err = err ?: btf_validate_id(btf, m->type, id); 525 if (err) 526 return err; 527 } 528 break; 529 } 530 case BTF_KIND_ENUM: { 531 const struct btf_enum *m = btf_enum(t); 532 533 n = btf_vlen(t); 534 for (i = 0; i < n; i++, m++) { 535 err = btf_validate_str(btf, m->name_off, "enum name", id); 536 if (err) 537 return err; 538 } 539 break; 540 } 541 case BTF_KIND_ENUM64: { 542 const struct btf_enum64 *m = btf_enum64(t); 543 544 n = btf_vlen(t); 545 for (i = 0; i < n; i++, m++) { 546 err = btf_validate_str(btf, m->name_off, "enum name", id); 547 if (err) 548 return err; 549 } 550 break; 551 } 552 case BTF_KIND_FUNC: { 553 const struct btf_type *ft; 554 555 err = btf_validate_id(btf, t->type, id); 556 if (err) 557 return err; 558 ft = btf__type_by_id(btf, t->type); 559 if (btf_kind(ft) != BTF_KIND_FUNC_PROTO) { 560 pr_warn("btf: type [%u]: referenced type [%u] is not FUNC_PROTO\n", id, t->type); 561 return -EINVAL; 562 } 563 break; 564 } 565 case BTF_KIND_FUNC_PROTO: { 566 const struct btf_param *m = btf_params(t); 567 568 n = btf_vlen(t); 569 for (i = 0; i < n; i++, m++) { 570 err = btf_validate_str(btf, m->name_off, "param name", id); 571 err = err ?: btf_validate_id(btf, m->type, id); 572 if (err) 573 return err; 574 } 575 break; 576 } 577 case BTF_KIND_DATASEC: { 578 const struct btf_var_secinfo *m = btf_var_secinfos(t); 579 580 n = btf_vlen(t); 581 for (i = 0; i < n; i++, m++) { 582 err = btf_validate_id(btf, m->type, id); 583 if (err) 584 return err; 585 } 586 break; 587 } 588 default: 589 pr_warn("btf: type [%u]: unrecognized kind %u\n", id, kind); 590 return -EINVAL; 591 } 592 return 0; 593 } 594 595 /* Validate basic sanity of BTF. It's intentionally less thorough than 596 * kernel's validation and validates only properties of BTF that libbpf relies 597 * on to be correct (e.g., valid type IDs, valid string offsets, etc) 598 */ 599 static int btf_sanity_check(const struct btf *btf) 600 { 601 const struct btf_type *t; 602 __u32 i, n = btf__type_cnt(btf); 603 int err; 604 605 for (i = btf->start_id; i < n; i++) { 606 t = btf_type_by_id(btf, i); 607 err = btf_validate_type(btf, t, i); 608 if (err) 609 return err; 610 } 611 return 0; 612 } 613 614 __u32 btf__type_cnt(const struct btf *btf) 615 { 616 return btf->start_id + btf->nr_types; 617 } 618 619 const struct btf *btf__base_btf(const struct btf *btf) 620 { 621 return btf->base_btf; 622 } 623 624 /* internal helper returning non-const pointer to a type */ 625 struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id) 626 { 627 if (type_id == 0) 628 return &btf_void; 629 if (type_id < btf->start_id) 630 return btf_type_by_id(btf->base_btf, type_id); 631 return btf->types_data + btf->type_offs[type_id - btf->start_id]; 632 } 633 634 const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id) 635 { 636 if (type_id >= btf->start_id + btf->nr_types) 637 return errno = EINVAL, NULL; 638 return btf_type_by_id((struct btf *)btf, type_id); 639 } 640 641 static int determine_ptr_size(const struct btf *btf) 642 { 643 static const char * const long_aliases[] = { 644 "long", 645 "long int", 646 "int long", 647 "unsigned long", 648 "long unsigned", 649 "unsigned long int", 650 "unsigned int long", 651 "long unsigned int", 652 "long int unsigned", 653 "int unsigned long", 654 "int long unsigned", 655 }; 656 const struct btf_type *t; 657 const char *name; 658 int i, j, n; 659 660 if (btf->base_btf && btf->base_btf->ptr_sz > 0) 661 return btf->base_btf->ptr_sz; 662 663 n = btf__type_cnt(btf); 664 for (i = 1; i < n; i++) { 665 t = btf__type_by_id(btf, i); 666 if (!btf_is_int(t)) 667 continue; 668 669 if (t->size != 4 && t->size != 8) 670 continue; 671 672 name = btf__name_by_offset(btf, t->name_off); 673 if (!name) 674 continue; 675 676 for (j = 0; j < ARRAY_SIZE(long_aliases); j++) { 677 if (strcmp(name, long_aliases[j]) == 0) 678 return t->size; 679 } 680 } 681 682 return -1; 683 } 684 685 static size_t btf_ptr_sz(const struct btf *btf) 686 { 687 if (!btf->ptr_sz) 688 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); 689 return btf->ptr_sz < 0 ? sizeof(void *) : btf->ptr_sz; 690 } 691 692 /* Return pointer size this BTF instance assumes. The size is heuristically 693 * determined by looking for 'long' or 'unsigned long' integer type and 694 * recording its size in bytes. If BTF type information doesn't have any such 695 * type, this function returns 0. In the latter case, native architecture's 696 * pointer size is assumed, so will be either 4 or 8, depending on 697 * architecture that libbpf was compiled for. It's possible to override 698 * guessed value by using btf__set_pointer_size() API. 699 */ 700 size_t btf__pointer_size(const struct btf *btf) 701 { 702 if (!btf->ptr_sz) 703 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); 704 705 if (btf->ptr_sz < 0) 706 /* not enough BTF type info to guess */ 707 return 0; 708 709 return btf->ptr_sz; 710 } 711 712 /* Override or set pointer size in bytes. Only values of 4 and 8 are 713 * supported. 714 */ 715 int btf__set_pointer_size(struct btf *btf, size_t ptr_sz) 716 { 717 if (ptr_sz != 4 && ptr_sz != 8) 718 return libbpf_err(-EINVAL); 719 btf->ptr_sz = ptr_sz; 720 return 0; 721 } 722 723 static bool is_host_big_endian(void) 724 { 725 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 726 return false; 727 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 728 return true; 729 #else 730 # error "Unrecognized __BYTE_ORDER__" 731 #endif 732 } 733 734 enum btf_endianness btf__endianness(const struct btf *btf) 735 { 736 if (is_host_big_endian()) 737 return btf->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN; 738 else 739 return btf->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN; 740 } 741 742 int btf__set_endianness(struct btf *btf, enum btf_endianness endian) 743 { 744 if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN) 745 return libbpf_err(-EINVAL); 746 747 btf->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN); 748 if (!btf->swapped_endian) { 749 free(btf->raw_data_swapped); 750 btf->raw_data_swapped = NULL; 751 } 752 return 0; 753 } 754 755 static bool btf_type_is_void(const struct btf_type *t) 756 { 757 return t == &btf_void || btf_is_fwd(t); 758 } 759 760 static bool btf_type_is_void_or_null(const struct btf_type *t) 761 { 762 return !t || btf_type_is_void(t); 763 } 764 765 #define MAX_RESOLVE_DEPTH 32 766 767 __s64 btf__resolve_size(const struct btf *btf, __u32 type_id) 768 { 769 const struct btf_array *array; 770 const struct btf_type *t; 771 __u32 nelems = 1; 772 __s64 size = -1; 773 int i; 774 775 t = btf__type_by_id(btf, type_id); 776 for (i = 0; i < MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null(t); i++) { 777 switch (btf_kind(t)) { 778 case BTF_KIND_INT: 779 case BTF_KIND_STRUCT: 780 case BTF_KIND_UNION: 781 case BTF_KIND_ENUM: 782 case BTF_KIND_ENUM64: 783 case BTF_KIND_DATASEC: 784 case BTF_KIND_FLOAT: 785 size = t->size; 786 goto done; 787 case BTF_KIND_PTR: 788 size = btf_ptr_sz(btf); 789 goto done; 790 case BTF_KIND_TYPEDEF: 791 case BTF_KIND_VOLATILE: 792 case BTF_KIND_CONST: 793 case BTF_KIND_RESTRICT: 794 case BTF_KIND_VAR: 795 case BTF_KIND_DECL_TAG: 796 case BTF_KIND_TYPE_TAG: 797 type_id = t->type; 798 break; 799 case BTF_KIND_ARRAY: 800 array = btf_array(t); 801 if (nelems && array->nelems > UINT32_MAX / nelems) 802 return libbpf_err(-E2BIG); 803 nelems *= array->nelems; 804 type_id = array->type; 805 break; 806 default: 807 return libbpf_err(-EINVAL); 808 } 809 810 t = btf__type_by_id(btf, type_id); 811 } 812 813 done: 814 if (size < 0) 815 return libbpf_err(-EINVAL); 816 if (nelems && size > UINT32_MAX / nelems) 817 return libbpf_err(-E2BIG); 818 819 return nelems * size; 820 } 821 822 int btf__align_of(const struct btf *btf, __u32 id) 823 { 824 const struct btf_type *t = btf__type_by_id(btf, id); 825 __u16 kind = btf_kind(t); 826 827 switch (kind) { 828 case BTF_KIND_INT: 829 case BTF_KIND_ENUM: 830 case BTF_KIND_ENUM64: 831 case BTF_KIND_FLOAT: 832 return min(btf_ptr_sz(btf), (size_t)t->size); 833 case BTF_KIND_PTR: 834 return btf_ptr_sz(btf); 835 case BTF_KIND_TYPEDEF: 836 case BTF_KIND_VOLATILE: 837 case BTF_KIND_CONST: 838 case BTF_KIND_RESTRICT: 839 case BTF_KIND_TYPE_TAG: 840 return btf__align_of(btf, t->type); 841 case BTF_KIND_ARRAY: 842 return btf__align_of(btf, btf_array(t)->type); 843 case BTF_KIND_STRUCT: 844 case BTF_KIND_UNION: { 845 const struct btf_member *m = btf_members(t); 846 __u16 vlen = btf_vlen(t); 847 int i, max_align = 1, align; 848 849 for (i = 0; i < vlen; i++, m++) { 850 align = btf__align_of(btf, m->type); 851 if (align <= 0) 852 return libbpf_err(align); 853 max_align = max(max_align, align); 854 855 /* if field offset isn't aligned according to field 856 * type's alignment, then struct must be packed 857 */ 858 if (btf_member_bitfield_size(t, i) == 0 && 859 (m->offset % (8 * align)) != 0) 860 return 1; 861 } 862 863 /* if struct/union size isn't a multiple of its alignment, 864 * then struct must be packed 865 */ 866 if ((t->size % max_align) != 0) 867 return 1; 868 869 return max_align; 870 } 871 default: 872 pr_warn("unsupported BTF_KIND:%u\n", btf_kind(t)); 873 return errno = EINVAL, 0; 874 } 875 } 876 877 int btf__resolve_type(const struct btf *btf, __u32 type_id) 878 { 879 const struct btf_type *t; 880 int depth = 0; 881 882 t = btf__type_by_id(btf, type_id); 883 while (depth < MAX_RESOLVE_DEPTH && 884 !btf_type_is_void_or_null(t) && 885 (btf_is_mod(t) || btf_is_typedef(t) || btf_is_var(t))) { 886 type_id = t->type; 887 t = btf__type_by_id(btf, type_id); 888 depth++; 889 } 890 891 if (depth == MAX_RESOLVE_DEPTH || btf_type_is_void_or_null(t)) 892 return libbpf_err(-EINVAL); 893 894 return type_id; 895 } 896 897 __s32 btf__find_by_name(const struct btf *btf, const char *type_name) 898 { 899 __u32 i, nr_types = btf__type_cnt(btf); 900 901 if (!strcmp(type_name, "void")) 902 return 0; 903 904 for (i = 1; i < nr_types; i++) { 905 const struct btf_type *t = btf__type_by_id(btf, i); 906 const char *name = btf__name_by_offset(btf, t->name_off); 907 908 if (name && !strcmp(type_name, name)) 909 return i; 910 } 911 912 return libbpf_err(-ENOENT); 913 } 914 915 static __s32 btf_find_by_name_kind(const struct btf *btf, int start_id, 916 const char *type_name, __u32 kind) 917 { 918 __u32 i, nr_types = btf__type_cnt(btf); 919 920 if (kind == BTF_KIND_UNKN || !strcmp(type_name, "void")) 921 return 0; 922 923 for (i = start_id; i < nr_types; i++) { 924 const struct btf_type *t = btf__type_by_id(btf, i); 925 const char *name; 926 927 if (btf_kind(t) != kind) 928 continue; 929 name = btf__name_by_offset(btf, t->name_off); 930 if (name && !strcmp(type_name, name)) 931 return i; 932 } 933 934 return libbpf_err(-ENOENT); 935 } 936 937 __s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name, 938 __u32 kind) 939 { 940 return btf_find_by_name_kind(btf, btf->start_id, type_name, kind); 941 } 942 943 __s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name, 944 __u32 kind) 945 { 946 return btf_find_by_name_kind(btf, 1, type_name, kind); 947 } 948 949 static bool btf_is_modifiable(const struct btf *btf) 950 { 951 return (void *)btf->hdr != btf->raw_data; 952 } 953 954 void btf__free(struct btf *btf) 955 { 956 if (IS_ERR_OR_NULL(btf)) 957 return; 958 959 if (btf->fd >= 0) 960 close(btf->fd); 961 962 if (btf_is_modifiable(btf)) { 963 /* if BTF was modified after loading, it will have a split 964 * in-memory representation for header, types, and strings 965 * sections, so we need to free all of them individually. It 966 * might still have a cached contiguous raw data present, 967 * which will be unconditionally freed below. 968 */ 969 free(btf->hdr); 970 free(btf->types_data); 971 strset__free(btf->strs_set); 972 } 973 free(btf->raw_data); 974 free(btf->raw_data_swapped); 975 free(btf->type_offs); 976 if (btf->owns_base) 977 btf__free(btf->base_btf); 978 free(btf); 979 } 980 981 static struct btf *btf_new_empty(struct btf *base_btf) 982 { 983 struct btf *btf; 984 985 btf = calloc(1, sizeof(*btf)); 986 if (!btf) 987 return ERR_PTR(-ENOMEM); 988 989 btf->nr_types = 0; 990 btf->start_id = 1; 991 btf->start_str_off = 0; 992 btf->fd = -1; 993 btf->ptr_sz = sizeof(void *); 994 btf->swapped_endian = false; 995 996 if (base_btf) { 997 btf->base_btf = base_btf; 998 btf->start_id = btf__type_cnt(base_btf); 999 btf->start_str_off = base_btf->hdr->str_len; 1000 btf->swapped_endian = base_btf->swapped_endian; 1001 } 1002 1003 /* +1 for empty string at offset 0 */ 1004 btf->raw_size = sizeof(struct btf_header) + (base_btf ? 0 : 1); 1005 btf->raw_data = calloc(1, btf->raw_size); 1006 if (!btf->raw_data) { 1007 free(btf); 1008 return ERR_PTR(-ENOMEM); 1009 } 1010 1011 btf->hdr = btf->raw_data; 1012 btf->hdr->hdr_len = sizeof(struct btf_header); 1013 btf->hdr->magic = BTF_MAGIC; 1014 btf->hdr->version = BTF_VERSION; 1015 1016 btf->types_data = btf->raw_data + btf->hdr->hdr_len; 1017 btf->strs_data = btf->raw_data + btf->hdr->hdr_len; 1018 btf->hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */ 1019 1020 return btf; 1021 } 1022 1023 struct btf *btf__new_empty(void) 1024 { 1025 return libbpf_ptr(btf_new_empty(NULL)); 1026 } 1027 1028 struct btf *btf__new_empty_split(struct btf *base_btf) 1029 { 1030 return libbpf_ptr(btf_new_empty(base_btf)); 1031 } 1032 1033 static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf) 1034 { 1035 struct btf *btf; 1036 int err; 1037 1038 btf = calloc(1, sizeof(struct btf)); 1039 if (!btf) 1040 return ERR_PTR(-ENOMEM); 1041 1042 btf->nr_types = 0; 1043 btf->start_id = 1; 1044 btf->start_str_off = 0; 1045 btf->fd = -1; 1046 1047 if (base_btf) { 1048 btf->base_btf = base_btf; 1049 btf->start_id = btf__type_cnt(base_btf); 1050 btf->start_str_off = base_btf->hdr->str_len; 1051 } 1052 1053 btf->raw_data = malloc(size); 1054 if (!btf->raw_data) { 1055 err = -ENOMEM; 1056 goto done; 1057 } 1058 memcpy(btf->raw_data, data, size); 1059 btf->raw_size = size; 1060 1061 btf->hdr = btf->raw_data; 1062 err = btf_parse_hdr(btf); 1063 if (err) 1064 goto done; 1065 1066 btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off; 1067 btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off; 1068 1069 err = btf_parse_str_sec(btf); 1070 err = err ?: btf_parse_type_sec(btf); 1071 err = err ?: btf_sanity_check(btf); 1072 if (err) 1073 goto done; 1074 1075 done: 1076 if (err) { 1077 btf__free(btf); 1078 return ERR_PTR(err); 1079 } 1080 1081 return btf; 1082 } 1083 1084 struct btf *btf__new(const void *data, __u32 size) 1085 { 1086 return libbpf_ptr(btf_new(data, size, NULL)); 1087 } 1088 1089 struct btf *btf__new_split(const void *data, __u32 size, struct btf *base_btf) 1090 { 1091 return libbpf_ptr(btf_new(data, size, base_btf)); 1092 } 1093 1094 struct btf_elf_secs { 1095 Elf_Data *btf_data; 1096 Elf_Data *btf_ext_data; 1097 Elf_Data *btf_base_data; 1098 }; 1099 1100 static int btf_find_elf_sections(Elf *elf, const char *path, struct btf_elf_secs *secs) 1101 { 1102 Elf_Scn *scn = NULL; 1103 Elf_Data *data; 1104 GElf_Ehdr ehdr; 1105 size_t shstrndx; 1106 int idx = 0; 1107 1108 if (!gelf_getehdr(elf, &ehdr)) { 1109 pr_warn("failed to get EHDR from %s\n", path); 1110 goto err; 1111 } 1112 1113 if (elf_getshdrstrndx(elf, &shstrndx)) { 1114 pr_warn("failed to get section names section index for %s\n", 1115 path); 1116 goto err; 1117 } 1118 1119 if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL)) { 1120 pr_warn("failed to get e_shstrndx from %s\n", path); 1121 goto err; 1122 } 1123 1124 while ((scn = elf_nextscn(elf, scn)) != NULL) { 1125 Elf_Data **field; 1126 GElf_Shdr sh; 1127 char *name; 1128 1129 idx++; 1130 if (gelf_getshdr(scn, &sh) != &sh) { 1131 pr_warn("failed to get section(%d) header from %s\n", 1132 idx, path); 1133 goto err; 1134 } 1135 name = elf_strptr(elf, shstrndx, sh.sh_name); 1136 if (!name) { 1137 pr_warn("failed to get section(%d) name from %s\n", 1138 idx, path); 1139 goto err; 1140 } 1141 1142 if (strcmp(name, BTF_ELF_SEC) == 0) 1143 field = &secs->btf_data; 1144 else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) 1145 field = &secs->btf_ext_data; 1146 else if (strcmp(name, BTF_BASE_ELF_SEC) == 0) 1147 field = &secs->btf_base_data; 1148 else 1149 continue; 1150 1151 data = elf_getdata(scn, 0); 1152 if (!data) { 1153 pr_warn("failed to get section(%d, %s) data from %s\n", 1154 idx, name, path); 1155 goto err; 1156 } 1157 *field = data; 1158 } 1159 1160 return 0; 1161 1162 err: 1163 return -LIBBPF_ERRNO__FORMAT; 1164 } 1165 1166 static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, 1167 struct btf_ext **btf_ext) 1168 { 1169 struct btf_elf_secs secs = {}; 1170 struct btf *dist_base_btf = NULL; 1171 struct btf *btf = NULL; 1172 int err = 0, fd = -1; 1173 Elf *elf = NULL; 1174 1175 if (elf_version(EV_CURRENT) == EV_NONE) { 1176 pr_warn("failed to init libelf for %s\n", path); 1177 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 1178 } 1179 1180 fd = open(path, O_RDONLY | O_CLOEXEC); 1181 if (fd < 0) { 1182 err = -errno; 1183 pr_warn("failed to open %s: %s\n", path, errstr(err)); 1184 return ERR_PTR(err); 1185 } 1186 1187 elf = elf_begin(fd, ELF_C_READ, NULL); 1188 if (!elf) { 1189 err = -LIBBPF_ERRNO__FORMAT; 1190 pr_warn("failed to open %s as ELF file\n", path); 1191 goto done; 1192 } 1193 1194 err = btf_find_elf_sections(elf, path, &secs); 1195 if (err) 1196 goto done; 1197 1198 if (!secs.btf_data) { 1199 pr_warn("failed to find '%s' ELF section in %s\n", BTF_ELF_SEC, path); 1200 err = -ENODATA; 1201 goto done; 1202 } 1203 1204 if (secs.btf_base_data) { 1205 dist_base_btf = btf_new(secs.btf_base_data->d_buf, secs.btf_base_data->d_size, 1206 NULL); 1207 if (IS_ERR(dist_base_btf)) { 1208 err = PTR_ERR(dist_base_btf); 1209 dist_base_btf = NULL; 1210 goto done; 1211 } 1212 } 1213 1214 btf = btf_new(secs.btf_data->d_buf, secs.btf_data->d_size, 1215 dist_base_btf ?: base_btf); 1216 if (IS_ERR(btf)) { 1217 err = PTR_ERR(btf); 1218 goto done; 1219 } 1220 if (dist_base_btf && base_btf) { 1221 err = btf__relocate(btf, base_btf); 1222 if (err) 1223 goto done; 1224 btf__free(dist_base_btf); 1225 dist_base_btf = NULL; 1226 } 1227 1228 if (dist_base_btf) 1229 btf->owns_base = true; 1230 1231 switch (gelf_getclass(elf)) { 1232 case ELFCLASS32: 1233 btf__set_pointer_size(btf, 4); 1234 break; 1235 case ELFCLASS64: 1236 btf__set_pointer_size(btf, 8); 1237 break; 1238 default: 1239 pr_warn("failed to get ELF class (bitness) for %s\n", path); 1240 break; 1241 } 1242 1243 if (btf_ext && secs.btf_ext_data) { 1244 *btf_ext = btf_ext__new(secs.btf_ext_data->d_buf, secs.btf_ext_data->d_size); 1245 if (IS_ERR(*btf_ext)) { 1246 err = PTR_ERR(*btf_ext); 1247 goto done; 1248 } 1249 } else if (btf_ext) { 1250 *btf_ext = NULL; 1251 } 1252 done: 1253 if (elf) 1254 elf_end(elf); 1255 close(fd); 1256 1257 if (!err) 1258 return btf; 1259 1260 if (btf_ext) 1261 btf_ext__free(*btf_ext); 1262 btf__free(dist_base_btf); 1263 btf__free(btf); 1264 1265 return ERR_PTR(err); 1266 } 1267 1268 struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext) 1269 { 1270 return libbpf_ptr(btf_parse_elf(path, NULL, btf_ext)); 1271 } 1272 1273 struct btf *btf__parse_elf_split(const char *path, struct btf *base_btf) 1274 { 1275 return libbpf_ptr(btf_parse_elf(path, base_btf, NULL)); 1276 } 1277 1278 static struct btf *btf_parse_raw(const char *path, struct btf *base_btf) 1279 { 1280 struct btf *btf = NULL; 1281 void *data = NULL; 1282 FILE *f = NULL; 1283 __u16 magic; 1284 int err = 0; 1285 long sz; 1286 1287 f = fopen(path, "rbe"); 1288 if (!f) { 1289 err = -errno; 1290 goto err_out; 1291 } 1292 1293 /* check BTF magic */ 1294 if (fread(&magic, 1, sizeof(magic), f) < sizeof(magic)) { 1295 err = -EIO; 1296 goto err_out; 1297 } 1298 if (magic != BTF_MAGIC && magic != bswap_16(BTF_MAGIC)) { 1299 /* definitely not a raw BTF */ 1300 err = -EPROTO; 1301 goto err_out; 1302 } 1303 1304 /* get file size */ 1305 if (fseek(f, 0, SEEK_END)) { 1306 err = -errno; 1307 goto err_out; 1308 } 1309 sz = ftell(f); 1310 if (sz < 0) { 1311 err = -errno; 1312 goto err_out; 1313 } 1314 /* rewind to the start */ 1315 if (fseek(f, 0, SEEK_SET)) { 1316 err = -errno; 1317 goto err_out; 1318 } 1319 1320 /* pre-alloc memory and read all of BTF data */ 1321 data = malloc(sz); 1322 if (!data) { 1323 err = -ENOMEM; 1324 goto err_out; 1325 } 1326 if (fread(data, 1, sz, f) < sz) { 1327 err = -EIO; 1328 goto err_out; 1329 } 1330 1331 /* finally parse BTF data */ 1332 btf = btf_new(data, sz, base_btf); 1333 1334 err_out: 1335 free(data); 1336 if (f) 1337 fclose(f); 1338 return err ? ERR_PTR(err) : btf; 1339 } 1340 1341 struct btf *btf__parse_raw(const char *path) 1342 { 1343 return libbpf_ptr(btf_parse_raw(path, NULL)); 1344 } 1345 1346 struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf) 1347 { 1348 return libbpf_ptr(btf_parse_raw(path, base_btf)); 1349 } 1350 1351 static struct btf *btf_parse(const char *path, struct btf *base_btf, struct btf_ext **btf_ext) 1352 { 1353 struct btf *btf; 1354 int err; 1355 1356 if (btf_ext) 1357 *btf_ext = NULL; 1358 1359 btf = btf_parse_raw(path, base_btf); 1360 err = libbpf_get_error(btf); 1361 if (!err) 1362 return btf; 1363 if (err != -EPROTO) 1364 return ERR_PTR(err); 1365 return btf_parse_elf(path, base_btf, btf_ext); 1366 } 1367 1368 struct btf *btf__parse(const char *path, struct btf_ext **btf_ext) 1369 { 1370 return libbpf_ptr(btf_parse(path, NULL, btf_ext)); 1371 } 1372 1373 struct btf *btf__parse_split(const char *path, struct btf *base_btf) 1374 { 1375 return libbpf_ptr(btf_parse(path, base_btf, NULL)); 1376 } 1377 1378 static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian); 1379 1380 int btf_load_into_kernel(struct btf *btf, 1381 char *log_buf, size_t log_sz, __u32 log_level, 1382 int token_fd) 1383 { 1384 LIBBPF_OPTS(bpf_btf_load_opts, opts); 1385 __u32 buf_sz = 0, raw_size; 1386 char *buf = NULL, *tmp; 1387 void *raw_data; 1388 int err = 0; 1389 1390 if (btf->fd >= 0) 1391 return libbpf_err(-EEXIST); 1392 if (log_sz && !log_buf) 1393 return libbpf_err(-EINVAL); 1394 1395 /* cache native raw data representation */ 1396 raw_data = btf_get_raw_data(btf, &raw_size, false); 1397 if (!raw_data) { 1398 err = -ENOMEM; 1399 goto done; 1400 } 1401 btf->raw_size = raw_size; 1402 btf->raw_data = raw_data; 1403 1404 retry_load: 1405 /* if log_level is 0, we won't provide log_buf/log_size to the kernel, 1406 * initially. Only if BTF loading fails, we bump log_level to 1 and 1407 * retry, using either auto-allocated or custom log_buf. This way 1408 * non-NULL custom log_buf provides a buffer just in case, but hopes 1409 * for successful load and no need for log_buf. 1410 */ 1411 if (log_level) { 1412 /* if caller didn't provide custom log_buf, we'll keep 1413 * allocating our own progressively bigger buffers for BTF 1414 * verification log 1415 */ 1416 if (!log_buf) { 1417 buf_sz = max((__u32)BPF_LOG_BUF_SIZE, buf_sz * 2); 1418 tmp = realloc(buf, buf_sz); 1419 if (!tmp) { 1420 err = -ENOMEM; 1421 goto done; 1422 } 1423 buf = tmp; 1424 buf[0] = '\0'; 1425 } 1426 1427 opts.log_buf = log_buf ? log_buf : buf; 1428 opts.log_size = log_buf ? log_sz : buf_sz; 1429 opts.log_level = log_level; 1430 } 1431 1432 opts.token_fd = token_fd; 1433 if (token_fd) 1434 opts.btf_flags |= BPF_F_TOKEN_FD; 1435 1436 btf->fd = bpf_btf_load(raw_data, raw_size, &opts); 1437 if (btf->fd < 0) { 1438 /* time to turn on verbose mode and try again */ 1439 if (log_level == 0) { 1440 log_level = 1; 1441 goto retry_load; 1442 } 1443 /* only retry if caller didn't provide custom log_buf, but 1444 * make sure we can never overflow buf_sz 1445 */ 1446 if (!log_buf && errno == ENOSPC && buf_sz <= UINT_MAX / 2) 1447 goto retry_load; 1448 1449 err = -errno; 1450 pr_warn("BTF loading error: %s\n", errstr(err)); 1451 /* don't print out contents of custom log_buf */ 1452 if (!log_buf && buf[0]) 1453 pr_warn("-- BEGIN BTF LOAD LOG ---\n%s\n-- END BTF LOAD LOG --\n", buf); 1454 } 1455 1456 done: 1457 free(buf); 1458 return libbpf_err(err); 1459 } 1460 1461 int btf__load_into_kernel(struct btf *btf) 1462 { 1463 return btf_load_into_kernel(btf, NULL, 0, 0, 0); 1464 } 1465 1466 int btf__fd(const struct btf *btf) 1467 { 1468 return btf->fd; 1469 } 1470 1471 void btf__set_fd(struct btf *btf, int fd) 1472 { 1473 btf->fd = fd; 1474 } 1475 1476 static const void *btf_strs_data(const struct btf *btf) 1477 { 1478 return btf->strs_data ? btf->strs_data : strset__data(btf->strs_set); 1479 } 1480 1481 static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian) 1482 { 1483 struct btf_header *hdr = btf->hdr; 1484 struct btf_type *t; 1485 void *data, *p; 1486 __u32 data_sz; 1487 int i; 1488 1489 data = swap_endian ? btf->raw_data_swapped : btf->raw_data; 1490 if (data) { 1491 *size = btf->raw_size; 1492 return data; 1493 } 1494 1495 data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len; 1496 data = calloc(1, data_sz); 1497 if (!data) 1498 return NULL; 1499 p = data; 1500 1501 memcpy(p, hdr, hdr->hdr_len); 1502 if (swap_endian) 1503 btf_bswap_hdr(p); 1504 p += hdr->hdr_len; 1505 1506 memcpy(p, btf->types_data, hdr->type_len); 1507 if (swap_endian) { 1508 for (i = 0; i < btf->nr_types; i++) { 1509 t = p + btf->type_offs[i]; 1510 /* btf_bswap_type_rest() relies on native t->info, so 1511 * we swap base type info after we swapped all the 1512 * additional information 1513 */ 1514 if (btf_bswap_type_rest(t)) 1515 goto err_out; 1516 btf_bswap_type_base(t); 1517 } 1518 } 1519 p += hdr->type_len; 1520 1521 memcpy(p, btf_strs_data(btf), hdr->str_len); 1522 p += hdr->str_len; 1523 1524 *size = data_sz; 1525 return data; 1526 err_out: 1527 free(data); 1528 return NULL; 1529 } 1530 1531 const void *btf__raw_data(const struct btf *btf_ro, __u32 *size) 1532 { 1533 struct btf *btf = (struct btf *)btf_ro; 1534 __u32 data_sz; 1535 void *data; 1536 1537 data = btf_get_raw_data(btf, &data_sz, btf->swapped_endian); 1538 if (!data) 1539 return errno = ENOMEM, NULL; 1540 1541 btf->raw_size = data_sz; 1542 if (btf->swapped_endian) 1543 btf->raw_data_swapped = data; 1544 else 1545 btf->raw_data = data; 1546 *size = data_sz; 1547 return data; 1548 } 1549 1550 __attribute__((alias("btf__raw_data"))) 1551 const void *btf__get_raw_data(const struct btf *btf, __u32 *size); 1552 1553 const char *btf__str_by_offset(const struct btf *btf, __u32 offset) 1554 { 1555 if (offset < btf->start_str_off) 1556 return btf__str_by_offset(btf->base_btf, offset); 1557 else if (offset - btf->start_str_off < btf->hdr->str_len) 1558 return btf_strs_data(btf) + (offset - btf->start_str_off); 1559 else 1560 return errno = EINVAL, NULL; 1561 } 1562 1563 const char *btf__name_by_offset(const struct btf *btf, __u32 offset) 1564 { 1565 return btf__str_by_offset(btf, offset); 1566 } 1567 1568 struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf) 1569 { 1570 struct bpf_btf_info btf_info; 1571 __u32 len = sizeof(btf_info); 1572 __u32 last_size; 1573 struct btf *btf; 1574 void *ptr; 1575 int err; 1576 1577 /* we won't know btf_size until we call bpf_btf_get_info_by_fd(). so 1578 * let's start with a sane default - 4KiB here - and resize it only if 1579 * bpf_btf_get_info_by_fd() needs a bigger buffer. 1580 */ 1581 last_size = 4096; 1582 ptr = malloc(last_size); 1583 if (!ptr) 1584 return ERR_PTR(-ENOMEM); 1585 1586 memset(&btf_info, 0, sizeof(btf_info)); 1587 btf_info.btf = ptr_to_u64(ptr); 1588 btf_info.btf_size = last_size; 1589 err = bpf_btf_get_info_by_fd(btf_fd, &btf_info, &len); 1590 1591 if (!err && btf_info.btf_size > last_size) { 1592 void *temp_ptr; 1593 1594 last_size = btf_info.btf_size; 1595 temp_ptr = realloc(ptr, last_size); 1596 if (!temp_ptr) { 1597 btf = ERR_PTR(-ENOMEM); 1598 goto exit_free; 1599 } 1600 ptr = temp_ptr; 1601 1602 len = sizeof(btf_info); 1603 memset(&btf_info, 0, sizeof(btf_info)); 1604 btf_info.btf = ptr_to_u64(ptr); 1605 btf_info.btf_size = last_size; 1606 1607 err = bpf_btf_get_info_by_fd(btf_fd, &btf_info, &len); 1608 } 1609 1610 if (err || btf_info.btf_size > last_size) { 1611 btf = err ? ERR_PTR(-errno) : ERR_PTR(-E2BIG); 1612 goto exit_free; 1613 } 1614 1615 btf = btf_new(ptr, btf_info.btf_size, base_btf); 1616 1617 exit_free: 1618 free(ptr); 1619 return btf; 1620 } 1621 1622 struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf) 1623 { 1624 struct btf *btf; 1625 int btf_fd; 1626 1627 btf_fd = bpf_btf_get_fd_by_id(id); 1628 if (btf_fd < 0) 1629 return libbpf_err_ptr(-errno); 1630 1631 btf = btf_get_from_fd(btf_fd, base_btf); 1632 close(btf_fd); 1633 1634 return libbpf_ptr(btf); 1635 } 1636 1637 struct btf *btf__load_from_kernel_by_id(__u32 id) 1638 { 1639 return btf__load_from_kernel_by_id_split(id, NULL); 1640 } 1641 1642 static void btf_invalidate_raw_data(struct btf *btf) 1643 { 1644 if (btf->raw_data) { 1645 free(btf->raw_data); 1646 btf->raw_data = NULL; 1647 } 1648 if (btf->raw_data_swapped) { 1649 free(btf->raw_data_swapped); 1650 btf->raw_data_swapped = NULL; 1651 } 1652 } 1653 1654 /* Ensure BTF is ready to be modified (by splitting into a three memory 1655 * regions for header, types, and strings). Also invalidate cached 1656 * raw_data, if any. 1657 */ 1658 static int btf_ensure_modifiable(struct btf *btf) 1659 { 1660 void *hdr, *types; 1661 struct strset *set = NULL; 1662 int err = -ENOMEM; 1663 1664 if (btf_is_modifiable(btf)) { 1665 /* any BTF modification invalidates raw_data */ 1666 btf_invalidate_raw_data(btf); 1667 return 0; 1668 } 1669 1670 /* split raw data into three memory regions */ 1671 hdr = malloc(btf->hdr->hdr_len); 1672 types = malloc(btf->hdr->type_len); 1673 if (!hdr || !types) 1674 goto err_out; 1675 1676 memcpy(hdr, btf->hdr, btf->hdr->hdr_len); 1677 memcpy(types, btf->types_data, btf->hdr->type_len); 1678 1679 /* build lookup index for all strings */ 1680 set = strset__new(BTF_MAX_STR_OFFSET, btf->strs_data, btf->hdr->str_len); 1681 if (IS_ERR(set)) { 1682 err = PTR_ERR(set); 1683 goto err_out; 1684 } 1685 1686 /* only when everything was successful, update internal state */ 1687 btf->hdr = hdr; 1688 btf->types_data = types; 1689 btf->types_data_cap = btf->hdr->type_len; 1690 btf->strs_data = NULL; 1691 btf->strs_set = set; 1692 /* if BTF was created from scratch, all strings are guaranteed to be 1693 * unique and deduplicated 1694 */ 1695 if (btf->hdr->str_len == 0) 1696 btf->strs_deduped = true; 1697 if (!btf->base_btf && btf->hdr->str_len == 1) 1698 btf->strs_deduped = true; 1699 1700 /* invalidate raw_data representation */ 1701 btf_invalidate_raw_data(btf); 1702 1703 return 0; 1704 1705 err_out: 1706 strset__free(set); 1707 free(hdr); 1708 free(types); 1709 return err; 1710 } 1711 1712 /* Find an offset in BTF string section that corresponds to a given string *s*. 1713 * Returns: 1714 * - >0 offset into string section, if string is found; 1715 * - -ENOENT, if string is not in the string section; 1716 * - <0, on any other error. 1717 */ 1718 int btf__find_str(struct btf *btf, const char *s) 1719 { 1720 int off; 1721 1722 if (btf->base_btf) { 1723 off = btf__find_str(btf->base_btf, s); 1724 if (off != -ENOENT) 1725 return off; 1726 } 1727 1728 /* BTF needs to be in a modifiable state to build string lookup index */ 1729 if (btf_ensure_modifiable(btf)) 1730 return libbpf_err(-ENOMEM); 1731 1732 off = strset__find_str(btf->strs_set, s); 1733 if (off < 0) 1734 return libbpf_err(off); 1735 1736 return btf->start_str_off + off; 1737 } 1738 1739 /* Add a string s to the BTF string section. 1740 * Returns: 1741 * - > 0 offset into string section, on success; 1742 * - < 0, on error. 1743 */ 1744 int btf__add_str(struct btf *btf, const char *s) 1745 { 1746 int off; 1747 1748 if (btf->base_btf) { 1749 off = btf__find_str(btf->base_btf, s); 1750 if (off != -ENOENT) 1751 return off; 1752 } 1753 1754 if (btf_ensure_modifiable(btf)) 1755 return libbpf_err(-ENOMEM); 1756 1757 off = strset__add_str(btf->strs_set, s); 1758 if (off < 0) 1759 return libbpf_err(off); 1760 1761 btf->hdr->str_len = strset__data_size(btf->strs_set); 1762 1763 return btf->start_str_off + off; 1764 } 1765 1766 static void *btf_add_type_mem(struct btf *btf, size_t add_sz) 1767 { 1768 return libbpf_add_mem(&btf->types_data, &btf->types_data_cap, 1, 1769 btf->hdr->type_len, UINT_MAX, add_sz); 1770 } 1771 1772 static void btf_type_inc_vlen(struct btf_type *t) 1773 { 1774 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t)); 1775 } 1776 1777 static int btf_commit_type(struct btf *btf, int data_sz) 1778 { 1779 int err; 1780 1781 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); 1782 if (err) 1783 return libbpf_err(err); 1784 1785 btf->hdr->type_len += data_sz; 1786 btf->hdr->str_off += data_sz; 1787 btf->nr_types++; 1788 return btf->start_id + btf->nr_types - 1; 1789 } 1790 1791 struct btf_pipe { 1792 const struct btf *src; 1793 struct btf *dst; 1794 struct hashmap *str_off_map; /* map string offsets from src to dst */ 1795 }; 1796 1797 static int btf_rewrite_str(struct btf_pipe *p, __u32 *str_off) 1798 { 1799 long mapped_off; 1800 int off, err; 1801 1802 if (!*str_off) /* nothing to do for empty strings */ 1803 return 0; 1804 1805 if (p->str_off_map && 1806 hashmap__find(p->str_off_map, *str_off, &mapped_off)) { 1807 *str_off = mapped_off; 1808 return 0; 1809 } 1810 1811 off = btf__add_str(p->dst, btf__str_by_offset(p->src, *str_off)); 1812 if (off < 0) 1813 return off; 1814 1815 /* Remember string mapping from src to dst. It avoids 1816 * performing expensive string comparisons. 1817 */ 1818 if (p->str_off_map) { 1819 err = hashmap__append(p->str_off_map, *str_off, off); 1820 if (err) 1821 return err; 1822 } 1823 1824 *str_off = off; 1825 return 0; 1826 } 1827 1828 static int btf_add_type(struct btf_pipe *p, const struct btf_type *src_type) 1829 { 1830 struct btf_field_iter it; 1831 struct btf_type *t; 1832 __u32 *str_off; 1833 int sz, err; 1834 1835 sz = btf_type_size(src_type); 1836 if (sz < 0) 1837 return libbpf_err(sz); 1838 1839 /* deconstruct BTF, if necessary, and invalidate raw_data */ 1840 if (btf_ensure_modifiable(p->dst)) 1841 return libbpf_err(-ENOMEM); 1842 1843 t = btf_add_type_mem(p->dst, sz); 1844 if (!t) 1845 return libbpf_err(-ENOMEM); 1846 1847 memcpy(t, src_type, sz); 1848 1849 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS); 1850 if (err) 1851 return libbpf_err(err); 1852 1853 while ((str_off = btf_field_iter_next(&it))) { 1854 err = btf_rewrite_str(p, str_off); 1855 if (err) 1856 return libbpf_err(err); 1857 } 1858 1859 return btf_commit_type(p->dst, sz); 1860 } 1861 1862 int btf__add_type(struct btf *btf, const struct btf *src_btf, const struct btf_type *src_type) 1863 { 1864 struct btf_pipe p = { .src = src_btf, .dst = btf }; 1865 1866 return btf_add_type(&p, src_type); 1867 } 1868 1869 static size_t btf_dedup_identity_hash_fn(long key, void *ctx); 1870 static bool btf_dedup_equal_fn(long k1, long k2, void *ctx); 1871 1872 int btf__add_btf(struct btf *btf, const struct btf *src_btf) 1873 { 1874 struct btf_pipe p = { .src = src_btf, .dst = btf }; 1875 int data_sz, sz, cnt, i, err, old_strs_len; 1876 __u32 *off; 1877 void *t; 1878 1879 /* appending split BTF isn't supported yet */ 1880 if (src_btf->base_btf) 1881 return libbpf_err(-ENOTSUP); 1882 1883 /* deconstruct BTF, if necessary, and invalidate raw_data */ 1884 if (btf_ensure_modifiable(btf)) 1885 return libbpf_err(-ENOMEM); 1886 1887 /* remember original strings section size if we have to roll back 1888 * partial strings section changes 1889 */ 1890 old_strs_len = btf->hdr->str_len; 1891 1892 data_sz = src_btf->hdr->type_len; 1893 cnt = btf__type_cnt(src_btf) - 1; 1894 1895 /* pre-allocate enough memory for new types */ 1896 t = btf_add_type_mem(btf, data_sz); 1897 if (!t) 1898 return libbpf_err(-ENOMEM); 1899 1900 /* pre-allocate enough memory for type offset index for new types */ 1901 off = btf_add_type_offs_mem(btf, cnt); 1902 if (!off) 1903 return libbpf_err(-ENOMEM); 1904 1905 /* Map the string offsets from src_btf to the offsets from btf to improve performance */ 1906 p.str_off_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL); 1907 if (IS_ERR(p.str_off_map)) 1908 return libbpf_err(-ENOMEM); 1909 1910 /* bulk copy types data for all types from src_btf */ 1911 memcpy(t, src_btf->types_data, data_sz); 1912 1913 for (i = 0; i < cnt; i++) { 1914 struct btf_field_iter it; 1915 __u32 *type_id, *str_off; 1916 1917 sz = btf_type_size(t); 1918 if (sz < 0) { 1919 /* unlikely, has to be corrupted src_btf */ 1920 err = sz; 1921 goto err_out; 1922 } 1923 1924 /* fill out type ID to type offset mapping for lookups by type ID */ 1925 *off = t - btf->types_data; 1926 1927 /* add, dedup, and remap strings referenced by this BTF type */ 1928 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS); 1929 if (err) 1930 goto err_out; 1931 while ((str_off = btf_field_iter_next(&it))) { 1932 err = btf_rewrite_str(&p, str_off); 1933 if (err) 1934 goto err_out; 1935 } 1936 1937 /* remap all type IDs referenced from this BTF type */ 1938 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS); 1939 if (err) 1940 goto err_out; 1941 1942 while ((type_id = btf_field_iter_next(&it))) { 1943 if (!*type_id) /* nothing to do for VOID references */ 1944 continue; 1945 1946 /* we haven't updated btf's type count yet, so 1947 * btf->start_id + btf->nr_types - 1 is the type ID offset we should 1948 * add to all newly added BTF types 1949 */ 1950 *type_id += btf->start_id + btf->nr_types - 1; 1951 } 1952 1953 /* go to next type data and type offset index entry */ 1954 t += sz; 1955 off++; 1956 } 1957 1958 /* Up until now any of the copied type data was effectively invisible, 1959 * so if we exited early before this point due to error, BTF would be 1960 * effectively unmodified. There would be extra internal memory 1961 * pre-allocated, but it would not be available for querying. But now 1962 * that we've copied and rewritten all the data successfully, we can 1963 * update type count and various internal offsets and sizes to 1964 * "commit" the changes and made them visible to the outside world. 1965 */ 1966 btf->hdr->type_len += data_sz; 1967 btf->hdr->str_off += data_sz; 1968 btf->nr_types += cnt; 1969 1970 hashmap__free(p.str_off_map); 1971 1972 /* return type ID of the first added BTF type */ 1973 return btf->start_id + btf->nr_types - cnt; 1974 err_out: 1975 /* zero out preallocated memory as if it was just allocated with 1976 * libbpf_add_mem() 1977 */ 1978 memset(btf->types_data + btf->hdr->type_len, 0, data_sz); 1979 memset(btf->strs_data + old_strs_len, 0, btf->hdr->str_len - old_strs_len); 1980 1981 /* and now restore original strings section size; types data size 1982 * wasn't modified, so doesn't need restoring, see big comment above 1983 */ 1984 btf->hdr->str_len = old_strs_len; 1985 1986 hashmap__free(p.str_off_map); 1987 1988 return libbpf_err(err); 1989 } 1990 1991 /* 1992 * Append new BTF_KIND_INT type with: 1993 * - *name* - non-empty, non-NULL type name; 1994 * - *sz* - power-of-2 (1, 2, 4, ..) size of the type, in bytes; 1995 * - encoding is a combination of BTF_INT_SIGNED, BTF_INT_CHAR, BTF_INT_BOOL. 1996 * Returns: 1997 * - >0, type ID of newly added BTF type; 1998 * - <0, on error. 1999 */ 2000 int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding) 2001 { 2002 struct btf_type *t; 2003 int sz, name_off; 2004 2005 /* non-empty name */ 2006 if (!name || !name[0]) 2007 return libbpf_err(-EINVAL); 2008 /* byte_sz must be power of 2 */ 2009 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16) 2010 return libbpf_err(-EINVAL); 2011 if (encoding & ~(BTF_INT_SIGNED | BTF_INT_CHAR | BTF_INT_BOOL)) 2012 return libbpf_err(-EINVAL); 2013 2014 /* deconstruct BTF, if necessary, and invalidate raw_data */ 2015 if (btf_ensure_modifiable(btf)) 2016 return libbpf_err(-ENOMEM); 2017 2018 sz = sizeof(struct btf_type) + sizeof(int); 2019 t = btf_add_type_mem(btf, sz); 2020 if (!t) 2021 return libbpf_err(-ENOMEM); 2022 2023 /* if something goes wrong later, we might end up with an extra string, 2024 * but that shouldn't be a problem, because BTF can't be constructed 2025 * completely anyway and will most probably be just discarded 2026 */ 2027 name_off = btf__add_str(btf, name); 2028 if (name_off < 0) 2029 return name_off; 2030 2031 t->name_off = name_off; 2032 t->info = btf_type_info(BTF_KIND_INT, 0, 0); 2033 t->size = byte_sz; 2034 /* set INT info, we don't allow setting legacy bit offset/size */ 2035 *(__u32 *)(t + 1) = (encoding << 24) | (byte_sz * 8); 2036 2037 return btf_commit_type(btf, sz); 2038 } 2039 2040 /* 2041 * Append new BTF_KIND_FLOAT type with: 2042 * - *name* - non-empty, non-NULL type name; 2043 * - *sz* - size of the type, in bytes; 2044 * Returns: 2045 * - >0, type ID of newly added BTF type; 2046 * - <0, on error. 2047 */ 2048 int btf__add_float(struct btf *btf, const char *name, size_t byte_sz) 2049 { 2050 struct btf_type *t; 2051 int sz, name_off; 2052 2053 /* non-empty name */ 2054 if (!name || !name[0]) 2055 return libbpf_err(-EINVAL); 2056 2057 /* byte_sz must be one of the explicitly allowed values */ 2058 if (byte_sz != 2 && byte_sz != 4 && byte_sz != 8 && byte_sz != 12 && 2059 byte_sz != 16) 2060 return libbpf_err(-EINVAL); 2061 2062 if (btf_ensure_modifiable(btf)) 2063 return libbpf_err(-ENOMEM); 2064 2065 sz = sizeof(struct btf_type); 2066 t = btf_add_type_mem(btf, sz); 2067 if (!t) 2068 return libbpf_err(-ENOMEM); 2069 2070 name_off = btf__add_str(btf, name); 2071 if (name_off < 0) 2072 return name_off; 2073 2074 t->name_off = name_off; 2075 t->info = btf_type_info(BTF_KIND_FLOAT, 0, 0); 2076 t->size = byte_sz; 2077 2078 return btf_commit_type(btf, sz); 2079 } 2080 2081 /* it's completely legal to append BTF types with type IDs pointing forward to 2082 * types that haven't been appended yet, so we only make sure that id looks 2083 * sane, we can't guarantee that ID will always be valid 2084 */ 2085 static int validate_type_id(int id) 2086 { 2087 if (id < 0 || id > BTF_MAX_NR_TYPES) 2088 return -EINVAL; 2089 return 0; 2090 } 2091 2092 /* generic append function for PTR, TYPEDEF, CONST/VOLATILE/RESTRICT */ 2093 static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref_type_id, int kflag) 2094 { 2095 struct btf_type *t; 2096 int sz, name_off = 0; 2097 2098 if (validate_type_id(ref_type_id)) 2099 return libbpf_err(-EINVAL); 2100 2101 if (btf_ensure_modifiable(btf)) 2102 return libbpf_err(-ENOMEM); 2103 2104 sz = sizeof(struct btf_type); 2105 t = btf_add_type_mem(btf, sz); 2106 if (!t) 2107 return libbpf_err(-ENOMEM); 2108 2109 if (name && name[0]) { 2110 name_off = btf__add_str(btf, name); 2111 if (name_off < 0) 2112 return name_off; 2113 } 2114 2115 t->name_off = name_off; 2116 t->info = btf_type_info(kind, 0, kflag); 2117 t->type = ref_type_id; 2118 2119 return btf_commit_type(btf, sz); 2120 } 2121 2122 /* 2123 * Append new BTF_KIND_PTR type with: 2124 * - *ref_type_id* - referenced type ID, it might not exist yet; 2125 * Returns: 2126 * - >0, type ID of newly added BTF type; 2127 * - <0, on error. 2128 */ 2129 int btf__add_ptr(struct btf *btf, int ref_type_id) 2130 { 2131 return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id, 0); 2132 } 2133 2134 /* 2135 * Append new BTF_KIND_ARRAY type with: 2136 * - *index_type_id* - type ID of the type describing array index; 2137 * - *elem_type_id* - type ID of the type describing array element; 2138 * - *nr_elems* - the size of the array; 2139 * Returns: 2140 * - >0, type ID of newly added BTF type; 2141 * - <0, on error. 2142 */ 2143 int btf__add_array(struct btf *btf, int index_type_id, int elem_type_id, __u32 nr_elems) 2144 { 2145 struct btf_type *t; 2146 struct btf_array *a; 2147 int sz; 2148 2149 if (validate_type_id(index_type_id) || validate_type_id(elem_type_id)) 2150 return libbpf_err(-EINVAL); 2151 2152 if (btf_ensure_modifiable(btf)) 2153 return libbpf_err(-ENOMEM); 2154 2155 sz = sizeof(struct btf_type) + sizeof(struct btf_array); 2156 t = btf_add_type_mem(btf, sz); 2157 if (!t) 2158 return libbpf_err(-ENOMEM); 2159 2160 t->name_off = 0; 2161 t->info = btf_type_info(BTF_KIND_ARRAY, 0, 0); 2162 t->size = 0; 2163 2164 a = btf_array(t); 2165 a->type = elem_type_id; 2166 a->index_type = index_type_id; 2167 a->nelems = nr_elems; 2168 2169 return btf_commit_type(btf, sz); 2170 } 2171 2172 /* generic STRUCT/UNION append function */ 2173 static int btf_add_composite(struct btf *btf, int kind, const char *name, __u32 bytes_sz) 2174 { 2175 struct btf_type *t; 2176 int sz, name_off = 0; 2177 2178 if (btf_ensure_modifiable(btf)) 2179 return libbpf_err(-ENOMEM); 2180 2181 sz = sizeof(struct btf_type); 2182 t = btf_add_type_mem(btf, sz); 2183 if (!t) 2184 return libbpf_err(-ENOMEM); 2185 2186 if (name && name[0]) { 2187 name_off = btf__add_str(btf, name); 2188 if (name_off < 0) 2189 return name_off; 2190 } 2191 2192 /* start out with vlen=0 and no kflag; this will be adjusted when 2193 * adding each member 2194 */ 2195 t->name_off = name_off; 2196 t->info = btf_type_info(kind, 0, 0); 2197 t->size = bytes_sz; 2198 2199 return btf_commit_type(btf, sz); 2200 } 2201 2202 /* 2203 * Append new BTF_KIND_STRUCT type with: 2204 * - *name* - name of the struct, can be NULL or empty for anonymous structs; 2205 * - *byte_sz* - size of the struct, in bytes; 2206 * 2207 * Struct initially has no fields in it. Fields can be added by 2208 * btf__add_field() right after btf__add_struct() succeeds. 2209 * 2210 * Returns: 2211 * - >0, type ID of newly added BTF type; 2212 * - <0, on error. 2213 */ 2214 int btf__add_struct(struct btf *btf, const char *name, __u32 byte_sz) 2215 { 2216 return btf_add_composite(btf, BTF_KIND_STRUCT, name, byte_sz); 2217 } 2218 2219 /* 2220 * Append new BTF_KIND_UNION type with: 2221 * - *name* - name of the union, can be NULL or empty for anonymous union; 2222 * - *byte_sz* - size of the union, in bytes; 2223 * 2224 * Union initially has no fields in it. Fields can be added by 2225 * btf__add_field() right after btf__add_union() succeeds. All fields 2226 * should have *bit_offset* of 0. 2227 * 2228 * Returns: 2229 * - >0, type ID of newly added BTF type; 2230 * - <0, on error. 2231 */ 2232 int btf__add_union(struct btf *btf, const char *name, __u32 byte_sz) 2233 { 2234 return btf_add_composite(btf, BTF_KIND_UNION, name, byte_sz); 2235 } 2236 2237 static struct btf_type *btf_last_type(struct btf *btf) 2238 { 2239 return btf_type_by_id(btf, btf__type_cnt(btf) - 1); 2240 } 2241 2242 /* 2243 * Append new field for the current STRUCT/UNION type with: 2244 * - *name* - name of the field, can be NULL or empty for anonymous field; 2245 * - *type_id* - type ID for the type describing field type; 2246 * - *bit_offset* - bit offset of the start of the field within struct/union; 2247 * - *bit_size* - bit size of a bitfield, 0 for non-bitfield fields; 2248 * Returns: 2249 * - 0, on success; 2250 * - <0, on error. 2251 */ 2252 int btf__add_field(struct btf *btf, const char *name, int type_id, 2253 __u32 bit_offset, __u32 bit_size) 2254 { 2255 struct btf_type *t; 2256 struct btf_member *m; 2257 bool is_bitfield; 2258 int sz, name_off = 0; 2259 2260 /* last type should be union/struct */ 2261 if (btf->nr_types == 0) 2262 return libbpf_err(-EINVAL); 2263 t = btf_last_type(btf); 2264 if (!btf_is_composite(t)) 2265 return libbpf_err(-EINVAL); 2266 2267 if (validate_type_id(type_id)) 2268 return libbpf_err(-EINVAL); 2269 /* best-effort bit field offset/size enforcement */ 2270 is_bitfield = bit_size || (bit_offset % 8 != 0); 2271 if (is_bitfield && (bit_size == 0 || bit_size > 255 || bit_offset > 0xffffff)) 2272 return libbpf_err(-EINVAL); 2273 2274 /* only offset 0 is allowed for unions */ 2275 if (btf_is_union(t) && bit_offset) 2276 return libbpf_err(-EINVAL); 2277 2278 /* decompose and invalidate raw data */ 2279 if (btf_ensure_modifiable(btf)) 2280 return libbpf_err(-ENOMEM); 2281 2282 sz = sizeof(struct btf_member); 2283 m = btf_add_type_mem(btf, sz); 2284 if (!m) 2285 return libbpf_err(-ENOMEM); 2286 2287 if (name && name[0]) { 2288 name_off = btf__add_str(btf, name); 2289 if (name_off < 0) 2290 return name_off; 2291 } 2292 2293 m->name_off = name_off; 2294 m->type = type_id; 2295 m->offset = bit_offset | (bit_size << 24); 2296 2297 /* btf_add_type_mem can invalidate t pointer */ 2298 t = btf_last_type(btf); 2299 /* update parent type's vlen and kflag */ 2300 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t)); 2301 2302 btf->hdr->type_len += sz; 2303 btf->hdr->str_off += sz; 2304 return 0; 2305 } 2306 2307 static int btf_add_enum_common(struct btf *btf, const char *name, __u32 byte_sz, 2308 bool is_signed, __u8 kind) 2309 { 2310 struct btf_type *t; 2311 int sz, name_off = 0; 2312 2313 /* byte_sz must be power of 2 */ 2314 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8) 2315 return libbpf_err(-EINVAL); 2316 2317 if (btf_ensure_modifiable(btf)) 2318 return libbpf_err(-ENOMEM); 2319 2320 sz = sizeof(struct btf_type); 2321 t = btf_add_type_mem(btf, sz); 2322 if (!t) 2323 return libbpf_err(-ENOMEM); 2324 2325 if (name && name[0]) { 2326 name_off = btf__add_str(btf, name); 2327 if (name_off < 0) 2328 return name_off; 2329 } 2330 2331 /* start out with vlen=0; it will be adjusted when adding enum values */ 2332 t->name_off = name_off; 2333 t->info = btf_type_info(kind, 0, is_signed); 2334 t->size = byte_sz; 2335 2336 return btf_commit_type(btf, sz); 2337 } 2338 2339 /* 2340 * Append new BTF_KIND_ENUM type with: 2341 * - *name* - name of the enum, can be NULL or empty for anonymous enums; 2342 * - *byte_sz* - size of the enum, in bytes. 2343 * 2344 * Enum initially has no enum values in it (and corresponds to enum forward 2345 * declaration). Enumerator values can be added by btf__add_enum_value() 2346 * immediately after btf__add_enum() succeeds. 2347 * 2348 * Returns: 2349 * - >0, type ID of newly added BTF type; 2350 * - <0, on error. 2351 */ 2352 int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz) 2353 { 2354 /* 2355 * set the signedness to be unsigned, it will change to signed 2356 * if any later enumerator is negative. 2357 */ 2358 return btf_add_enum_common(btf, name, byte_sz, false, BTF_KIND_ENUM); 2359 } 2360 2361 /* 2362 * Append new enum value for the current ENUM type with: 2363 * - *name* - name of the enumerator value, can't be NULL or empty; 2364 * - *value* - integer value corresponding to enum value *name*; 2365 * Returns: 2366 * - 0, on success; 2367 * - <0, on error. 2368 */ 2369 int btf__add_enum_value(struct btf *btf, const char *name, __s64 value) 2370 { 2371 struct btf_type *t; 2372 struct btf_enum *v; 2373 int sz, name_off; 2374 2375 /* last type should be BTF_KIND_ENUM */ 2376 if (btf->nr_types == 0) 2377 return libbpf_err(-EINVAL); 2378 t = btf_last_type(btf); 2379 if (!btf_is_enum(t)) 2380 return libbpf_err(-EINVAL); 2381 2382 /* non-empty name */ 2383 if (!name || !name[0]) 2384 return libbpf_err(-EINVAL); 2385 if (value < INT_MIN || value > UINT_MAX) 2386 return libbpf_err(-E2BIG); 2387 2388 /* decompose and invalidate raw data */ 2389 if (btf_ensure_modifiable(btf)) 2390 return libbpf_err(-ENOMEM); 2391 2392 sz = sizeof(struct btf_enum); 2393 v = btf_add_type_mem(btf, sz); 2394 if (!v) 2395 return libbpf_err(-ENOMEM); 2396 2397 name_off = btf__add_str(btf, name); 2398 if (name_off < 0) 2399 return name_off; 2400 2401 v->name_off = name_off; 2402 v->val = value; 2403 2404 /* update parent type's vlen */ 2405 t = btf_last_type(btf); 2406 btf_type_inc_vlen(t); 2407 2408 /* if negative value, set signedness to signed */ 2409 if (value < 0) 2410 t->info = btf_type_info(btf_kind(t), btf_vlen(t), true); 2411 2412 btf->hdr->type_len += sz; 2413 btf->hdr->str_off += sz; 2414 return 0; 2415 } 2416 2417 /* 2418 * Append new BTF_KIND_ENUM64 type with: 2419 * - *name* - name of the enum, can be NULL or empty for anonymous enums; 2420 * - *byte_sz* - size of the enum, in bytes. 2421 * - *is_signed* - whether the enum values are signed or not; 2422 * 2423 * Enum initially has no enum values in it (and corresponds to enum forward 2424 * declaration). Enumerator values can be added by btf__add_enum64_value() 2425 * immediately after btf__add_enum64() succeeds. 2426 * 2427 * Returns: 2428 * - >0, type ID of newly added BTF type; 2429 * - <0, on error. 2430 */ 2431 int btf__add_enum64(struct btf *btf, const char *name, __u32 byte_sz, 2432 bool is_signed) 2433 { 2434 return btf_add_enum_common(btf, name, byte_sz, is_signed, 2435 BTF_KIND_ENUM64); 2436 } 2437 2438 /* 2439 * Append new enum value for the current ENUM64 type with: 2440 * - *name* - name of the enumerator value, can't be NULL or empty; 2441 * - *value* - integer value corresponding to enum value *name*; 2442 * Returns: 2443 * - 0, on success; 2444 * - <0, on error. 2445 */ 2446 int btf__add_enum64_value(struct btf *btf, const char *name, __u64 value) 2447 { 2448 struct btf_enum64 *v; 2449 struct btf_type *t; 2450 int sz, name_off; 2451 2452 /* last type should be BTF_KIND_ENUM64 */ 2453 if (btf->nr_types == 0) 2454 return libbpf_err(-EINVAL); 2455 t = btf_last_type(btf); 2456 if (!btf_is_enum64(t)) 2457 return libbpf_err(-EINVAL); 2458 2459 /* non-empty name */ 2460 if (!name || !name[0]) 2461 return libbpf_err(-EINVAL); 2462 2463 /* decompose and invalidate raw data */ 2464 if (btf_ensure_modifiable(btf)) 2465 return libbpf_err(-ENOMEM); 2466 2467 sz = sizeof(struct btf_enum64); 2468 v = btf_add_type_mem(btf, sz); 2469 if (!v) 2470 return libbpf_err(-ENOMEM); 2471 2472 name_off = btf__add_str(btf, name); 2473 if (name_off < 0) 2474 return name_off; 2475 2476 v->name_off = name_off; 2477 v->val_lo32 = (__u32)value; 2478 v->val_hi32 = value >> 32; 2479 2480 /* update parent type's vlen */ 2481 t = btf_last_type(btf); 2482 btf_type_inc_vlen(t); 2483 2484 btf->hdr->type_len += sz; 2485 btf->hdr->str_off += sz; 2486 return 0; 2487 } 2488 2489 /* 2490 * Append new BTF_KIND_FWD type with: 2491 * - *name*, non-empty/non-NULL name; 2492 * - *fwd_kind*, kind of forward declaration, one of BTF_FWD_STRUCT, 2493 * BTF_FWD_UNION, or BTF_FWD_ENUM; 2494 * Returns: 2495 * - >0, type ID of newly added BTF type; 2496 * - <0, on error. 2497 */ 2498 int btf__add_fwd(struct btf *btf, const char *name, enum btf_fwd_kind fwd_kind) 2499 { 2500 if (!name || !name[0]) 2501 return libbpf_err(-EINVAL); 2502 2503 switch (fwd_kind) { 2504 case BTF_FWD_STRUCT: 2505 case BTF_FWD_UNION: { 2506 struct btf_type *t; 2507 int id; 2508 2509 id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0, 0); 2510 if (id <= 0) 2511 return id; 2512 t = btf_type_by_id(btf, id); 2513 t->info = btf_type_info(BTF_KIND_FWD, 0, fwd_kind == BTF_FWD_UNION); 2514 return id; 2515 } 2516 case BTF_FWD_ENUM: 2517 /* enum forward in BTF currently is just an enum with no enum 2518 * values; we also assume a standard 4-byte size for it 2519 */ 2520 return btf__add_enum(btf, name, sizeof(int)); 2521 default: 2522 return libbpf_err(-EINVAL); 2523 } 2524 } 2525 2526 /* 2527 * Append new BTF_KING_TYPEDEF type with: 2528 * - *name*, non-empty/non-NULL name; 2529 * - *ref_type_id* - referenced type ID, it might not exist yet; 2530 * Returns: 2531 * - >0, type ID of newly added BTF type; 2532 * - <0, on error. 2533 */ 2534 int btf__add_typedef(struct btf *btf, const char *name, int ref_type_id) 2535 { 2536 if (!name || !name[0]) 2537 return libbpf_err(-EINVAL); 2538 2539 return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id, 0); 2540 } 2541 2542 /* 2543 * Append new BTF_KIND_VOLATILE type with: 2544 * - *ref_type_id* - referenced type ID, it might not exist yet; 2545 * Returns: 2546 * - >0, type ID of newly added BTF type; 2547 * - <0, on error. 2548 */ 2549 int btf__add_volatile(struct btf *btf, int ref_type_id) 2550 { 2551 return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id, 0); 2552 } 2553 2554 /* 2555 * Append new BTF_KIND_CONST type with: 2556 * - *ref_type_id* - referenced type ID, it might not exist yet; 2557 * Returns: 2558 * - >0, type ID of newly added BTF type; 2559 * - <0, on error. 2560 */ 2561 int btf__add_const(struct btf *btf, int ref_type_id) 2562 { 2563 return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id, 0); 2564 } 2565 2566 /* 2567 * Append new BTF_KIND_RESTRICT type with: 2568 * - *ref_type_id* - referenced type ID, it might not exist yet; 2569 * Returns: 2570 * - >0, type ID of newly added BTF type; 2571 * - <0, on error. 2572 */ 2573 int btf__add_restrict(struct btf *btf, int ref_type_id) 2574 { 2575 return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id, 0); 2576 } 2577 2578 /* 2579 * Append new BTF_KIND_TYPE_TAG type with: 2580 * - *value*, non-empty/non-NULL tag value; 2581 * - *ref_type_id* - referenced type ID, it might not exist yet; 2582 * Returns: 2583 * - >0, type ID of newly added BTF type; 2584 * - <0, on error. 2585 */ 2586 int btf__add_type_tag(struct btf *btf, const char *value, int ref_type_id) 2587 { 2588 if (!value || !value[0]) 2589 return libbpf_err(-EINVAL); 2590 2591 return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id, 0); 2592 } 2593 2594 /* 2595 * Append new BTF_KIND_TYPE_TAG type with: 2596 * - *value*, non-empty/non-NULL tag value; 2597 * - *ref_type_id* - referenced type ID, it might not exist yet; 2598 * Set info->kflag to 1, indicating this tag is an __attribute__ 2599 * Returns: 2600 * - >0, type ID of newly added BTF type; 2601 * - <0, on error. 2602 */ 2603 int btf__add_type_attr(struct btf *btf, const char *value, int ref_type_id) 2604 { 2605 if (!value || !value[0]) 2606 return libbpf_err(-EINVAL); 2607 2608 return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id, 1); 2609 } 2610 2611 /* 2612 * Append new BTF_KIND_FUNC type with: 2613 * - *name*, non-empty/non-NULL name; 2614 * - *proto_type_id* - FUNC_PROTO's type ID, it might not exist yet; 2615 * Returns: 2616 * - >0, type ID of newly added BTF type; 2617 * - <0, on error. 2618 */ 2619 int btf__add_func(struct btf *btf, const char *name, 2620 enum btf_func_linkage linkage, int proto_type_id) 2621 { 2622 int id; 2623 2624 if (!name || !name[0]) 2625 return libbpf_err(-EINVAL); 2626 if (linkage != BTF_FUNC_STATIC && linkage != BTF_FUNC_GLOBAL && 2627 linkage != BTF_FUNC_EXTERN) 2628 return libbpf_err(-EINVAL); 2629 2630 id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id, 0); 2631 if (id > 0) { 2632 struct btf_type *t = btf_type_by_id(btf, id); 2633 2634 t->info = btf_type_info(BTF_KIND_FUNC, linkage, 0); 2635 } 2636 return libbpf_err(id); 2637 } 2638 2639 /* 2640 * Append new BTF_KIND_FUNC_PROTO with: 2641 * - *ret_type_id* - type ID for return result of a function. 2642 * 2643 * Function prototype initially has no arguments, but they can be added by 2644 * btf__add_func_param() one by one, immediately after 2645 * btf__add_func_proto() succeeded. 2646 * 2647 * Returns: 2648 * - >0, type ID of newly added BTF type; 2649 * - <0, on error. 2650 */ 2651 int btf__add_func_proto(struct btf *btf, int ret_type_id) 2652 { 2653 struct btf_type *t; 2654 int sz; 2655 2656 if (validate_type_id(ret_type_id)) 2657 return libbpf_err(-EINVAL); 2658 2659 if (btf_ensure_modifiable(btf)) 2660 return libbpf_err(-ENOMEM); 2661 2662 sz = sizeof(struct btf_type); 2663 t = btf_add_type_mem(btf, sz); 2664 if (!t) 2665 return libbpf_err(-ENOMEM); 2666 2667 /* start out with vlen=0; this will be adjusted when adding enum 2668 * values, if necessary 2669 */ 2670 t->name_off = 0; 2671 t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0); 2672 t->type = ret_type_id; 2673 2674 return btf_commit_type(btf, sz); 2675 } 2676 2677 /* 2678 * Append new function parameter for current FUNC_PROTO type with: 2679 * - *name* - parameter name, can be NULL or empty; 2680 * - *type_id* - type ID describing the type of the parameter. 2681 * Returns: 2682 * - 0, on success; 2683 * - <0, on error. 2684 */ 2685 int btf__add_func_param(struct btf *btf, const char *name, int type_id) 2686 { 2687 struct btf_type *t; 2688 struct btf_param *p; 2689 int sz, name_off = 0; 2690 2691 if (validate_type_id(type_id)) 2692 return libbpf_err(-EINVAL); 2693 2694 /* last type should be BTF_KIND_FUNC_PROTO */ 2695 if (btf->nr_types == 0) 2696 return libbpf_err(-EINVAL); 2697 t = btf_last_type(btf); 2698 if (!btf_is_func_proto(t)) 2699 return libbpf_err(-EINVAL); 2700 2701 /* decompose and invalidate raw data */ 2702 if (btf_ensure_modifiable(btf)) 2703 return libbpf_err(-ENOMEM); 2704 2705 sz = sizeof(struct btf_param); 2706 p = btf_add_type_mem(btf, sz); 2707 if (!p) 2708 return libbpf_err(-ENOMEM); 2709 2710 if (name && name[0]) { 2711 name_off = btf__add_str(btf, name); 2712 if (name_off < 0) 2713 return name_off; 2714 } 2715 2716 p->name_off = name_off; 2717 p->type = type_id; 2718 2719 /* update parent type's vlen */ 2720 t = btf_last_type(btf); 2721 btf_type_inc_vlen(t); 2722 2723 btf->hdr->type_len += sz; 2724 btf->hdr->str_off += sz; 2725 return 0; 2726 } 2727 2728 /* 2729 * Append new BTF_KIND_VAR type with: 2730 * - *name* - non-empty/non-NULL name; 2731 * - *linkage* - variable linkage, one of BTF_VAR_STATIC, 2732 * BTF_VAR_GLOBAL_ALLOCATED, or BTF_VAR_GLOBAL_EXTERN; 2733 * - *type_id* - type ID of the type describing the type of the variable. 2734 * Returns: 2735 * - >0, type ID of newly added BTF type; 2736 * - <0, on error. 2737 */ 2738 int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id) 2739 { 2740 struct btf_type *t; 2741 struct btf_var *v; 2742 int sz, name_off; 2743 2744 /* non-empty name */ 2745 if (!name || !name[0]) 2746 return libbpf_err(-EINVAL); 2747 if (linkage != BTF_VAR_STATIC && linkage != BTF_VAR_GLOBAL_ALLOCATED && 2748 linkage != BTF_VAR_GLOBAL_EXTERN) 2749 return libbpf_err(-EINVAL); 2750 if (validate_type_id(type_id)) 2751 return libbpf_err(-EINVAL); 2752 2753 /* deconstruct BTF, if necessary, and invalidate raw_data */ 2754 if (btf_ensure_modifiable(btf)) 2755 return libbpf_err(-ENOMEM); 2756 2757 sz = sizeof(struct btf_type) + sizeof(struct btf_var); 2758 t = btf_add_type_mem(btf, sz); 2759 if (!t) 2760 return libbpf_err(-ENOMEM); 2761 2762 name_off = btf__add_str(btf, name); 2763 if (name_off < 0) 2764 return name_off; 2765 2766 t->name_off = name_off; 2767 t->info = btf_type_info(BTF_KIND_VAR, 0, 0); 2768 t->type = type_id; 2769 2770 v = btf_var(t); 2771 v->linkage = linkage; 2772 2773 return btf_commit_type(btf, sz); 2774 } 2775 2776 /* 2777 * Append new BTF_KIND_DATASEC type with: 2778 * - *name* - non-empty/non-NULL name; 2779 * - *byte_sz* - data section size, in bytes. 2780 * 2781 * Data section is initially empty. Variables info can be added with 2782 * btf__add_datasec_var_info() calls, after btf__add_datasec() succeeds. 2783 * 2784 * Returns: 2785 * - >0, type ID of newly added BTF type; 2786 * - <0, on error. 2787 */ 2788 int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz) 2789 { 2790 struct btf_type *t; 2791 int sz, name_off; 2792 2793 /* non-empty name */ 2794 if (!name || !name[0]) 2795 return libbpf_err(-EINVAL); 2796 2797 if (btf_ensure_modifiable(btf)) 2798 return libbpf_err(-ENOMEM); 2799 2800 sz = sizeof(struct btf_type); 2801 t = btf_add_type_mem(btf, sz); 2802 if (!t) 2803 return libbpf_err(-ENOMEM); 2804 2805 name_off = btf__add_str(btf, name); 2806 if (name_off < 0) 2807 return name_off; 2808 2809 /* start with vlen=0, which will be update as var_secinfos are added */ 2810 t->name_off = name_off; 2811 t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0); 2812 t->size = byte_sz; 2813 2814 return btf_commit_type(btf, sz); 2815 } 2816 2817 /* 2818 * Append new data section variable information entry for current DATASEC type: 2819 * - *var_type_id* - type ID, describing type of the variable; 2820 * - *offset* - variable offset within data section, in bytes; 2821 * - *byte_sz* - variable size, in bytes. 2822 * 2823 * Returns: 2824 * - 0, on success; 2825 * - <0, on error. 2826 */ 2827 int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __u32 byte_sz) 2828 { 2829 struct btf_type *t; 2830 struct btf_var_secinfo *v; 2831 int sz; 2832 2833 /* last type should be BTF_KIND_DATASEC */ 2834 if (btf->nr_types == 0) 2835 return libbpf_err(-EINVAL); 2836 t = btf_last_type(btf); 2837 if (!btf_is_datasec(t)) 2838 return libbpf_err(-EINVAL); 2839 2840 if (validate_type_id(var_type_id)) 2841 return libbpf_err(-EINVAL); 2842 2843 /* decompose and invalidate raw data */ 2844 if (btf_ensure_modifiable(btf)) 2845 return libbpf_err(-ENOMEM); 2846 2847 sz = sizeof(struct btf_var_secinfo); 2848 v = btf_add_type_mem(btf, sz); 2849 if (!v) 2850 return libbpf_err(-ENOMEM); 2851 2852 v->type = var_type_id; 2853 v->offset = offset; 2854 v->size = byte_sz; 2855 2856 /* update parent type's vlen */ 2857 t = btf_last_type(btf); 2858 btf_type_inc_vlen(t); 2859 2860 btf->hdr->type_len += sz; 2861 btf->hdr->str_off += sz; 2862 return 0; 2863 } 2864 2865 static int btf_add_decl_tag(struct btf *btf, const char *value, int ref_type_id, 2866 int component_idx, int kflag) 2867 { 2868 struct btf_type *t; 2869 int sz, value_off; 2870 2871 if (!value || !value[0] || component_idx < -1) 2872 return libbpf_err(-EINVAL); 2873 2874 if (validate_type_id(ref_type_id)) 2875 return libbpf_err(-EINVAL); 2876 2877 if (btf_ensure_modifiable(btf)) 2878 return libbpf_err(-ENOMEM); 2879 2880 sz = sizeof(struct btf_type) + sizeof(struct btf_decl_tag); 2881 t = btf_add_type_mem(btf, sz); 2882 if (!t) 2883 return libbpf_err(-ENOMEM); 2884 2885 value_off = btf__add_str(btf, value); 2886 if (value_off < 0) 2887 return value_off; 2888 2889 t->name_off = value_off; 2890 t->info = btf_type_info(BTF_KIND_DECL_TAG, 0, kflag); 2891 t->type = ref_type_id; 2892 btf_decl_tag(t)->component_idx = component_idx; 2893 2894 return btf_commit_type(btf, sz); 2895 } 2896 2897 /* 2898 * Append new BTF_KIND_DECL_TAG type with: 2899 * - *value* - non-empty/non-NULL string; 2900 * - *ref_type_id* - referenced type ID, it might not exist yet; 2901 * - *component_idx* - -1 for tagging reference type, otherwise struct/union 2902 * member or function argument index; 2903 * Returns: 2904 * - >0, type ID of newly added BTF type; 2905 * - <0, on error. 2906 */ 2907 int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id, 2908 int component_idx) 2909 { 2910 return btf_add_decl_tag(btf, value, ref_type_id, component_idx, 0); 2911 } 2912 2913 /* 2914 * Append new BTF_KIND_DECL_TAG type with: 2915 * - *value* - non-empty/non-NULL string; 2916 * - *ref_type_id* - referenced type ID, it might not exist yet; 2917 * - *component_idx* - -1 for tagging reference type, otherwise struct/union 2918 * member or function argument index; 2919 * Set info->kflag to 1, indicating this tag is an __attribute__ 2920 * Returns: 2921 * - >0, type ID of newly added BTF type; 2922 * - <0, on error. 2923 */ 2924 int btf__add_decl_attr(struct btf *btf, const char *value, int ref_type_id, 2925 int component_idx) 2926 { 2927 return btf_add_decl_tag(btf, value, ref_type_id, component_idx, 1); 2928 } 2929 2930 struct btf_ext_sec_info_param { 2931 __u32 off; 2932 __u32 len; 2933 __u32 min_rec_size; 2934 struct btf_ext_info *ext_info; 2935 const char *desc; 2936 }; 2937 2938 /* 2939 * Parse a single info subsection of the BTF.ext info data: 2940 * - validate subsection structure and elements 2941 * - save info subsection start and sizing details in struct btf_ext 2942 * - endian-independent operation, for calling before byte-swapping 2943 */ 2944 static int btf_ext_parse_sec_info(struct btf_ext *btf_ext, 2945 struct btf_ext_sec_info_param *ext_sec, 2946 bool is_native) 2947 { 2948 const struct btf_ext_info_sec *sinfo; 2949 struct btf_ext_info *ext_info; 2950 __u32 info_left, record_size; 2951 size_t sec_cnt = 0; 2952 void *info; 2953 2954 if (ext_sec->len == 0) 2955 return 0; 2956 2957 if (ext_sec->off & 0x03) { 2958 pr_debug(".BTF.ext %s section is not aligned to 4 bytes\n", 2959 ext_sec->desc); 2960 return -EINVAL; 2961 } 2962 2963 /* The start of the info sec (including the __u32 record_size). */ 2964 info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off; 2965 info_left = ext_sec->len; 2966 2967 if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) { 2968 pr_debug("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n", 2969 ext_sec->desc, ext_sec->off, ext_sec->len); 2970 return -EINVAL; 2971 } 2972 2973 /* At least a record size */ 2974 if (info_left < sizeof(__u32)) { 2975 pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc); 2976 return -EINVAL; 2977 } 2978 2979 /* The record size needs to meet either the minimum standard or, when 2980 * handling non-native endianness data, the exact standard so as 2981 * to allow safe byte-swapping. 2982 */ 2983 record_size = is_native ? *(__u32 *)info : bswap_32(*(__u32 *)info); 2984 if (record_size < ext_sec->min_rec_size || 2985 (!is_native && record_size != ext_sec->min_rec_size) || 2986 record_size & 0x03) { 2987 pr_debug("%s section in .BTF.ext has invalid record size %u\n", 2988 ext_sec->desc, record_size); 2989 return -EINVAL; 2990 } 2991 2992 sinfo = info + sizeof(__u32); 2993 info_left -= sizeof(__u32); 2994 2995 /* If no records, return failure now so .BTF.ext won't be used. */ 2996 if (!info_left) { 2997 pr_debug("%s section in .BTF.ext has no records\n", ext_sec->desc); 2998 return -EINVAL; 2999 } 3000 3001 while (info_left) { 3002 unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec); 3003 __u64 total_record_size; 3004 __u32 num_records; 3005 3006 if (info_left < sec_hdrlen) { 3007 pr_debug("%s section header is not found in .BTF.ext\n", 3008 ext_sec->desc); 3009 return -EINVAL; 3010 } 3011 3012 num_records = is_native ? sinfo->num_info : bswap_32(sinfo->num_info); 3013 if (num_records == 0) { 3014 pr_debug("%s section has incorrect num_records in .BTF.ext\n", 3015 ext_sec->desc); 3016 return -EINVAL; 3017 } 3018 3019 total_record_size = sec_hdrlen + (__u64)num_records * record_size; 3020 if (info_left < total_record_size) { 3021 pr_debug("%s section has incorrect num_records in .BTF.ext\n", 3022 ext_sec->desc); 3023 return -EINVAL; 3024 } 3025 3026 info_left -= total_record_size; 3027 sinfo = (void *)sinfo + total_record_size; 3028 sec_cnt++; 3029 } 3030 3031 ext_info = ext_sec->ext_info; 3032 ext_info->len = ext_sec->len - sizeof(__u32); 3033 ext_info->rec_size = record_size; 3034 ext_info->info = info + sizeof(__u32); 3035 ext_info->sec_cnt = sec_cnt; 3036 3037 return 0; 3038 } 3039 3040 /* Parse all info secs in the BTF.ext info data */ 3041 static int btf_ext_parse_info(struct btf_ext *btf_ext, bool is_native) 3042 { 3043 struct btf_ext_sec_info_param func_info = { 3044 .off = btf_ext->hdr->func_info_off, 3045 .len = btf_ext->hdr->func_info_len, 3046 .min_rec_size = sizeof(struct bpf_func_info_min), 3047 .ext_info = &btf_ext->func_info, 3048 .desc = "func_info" 3049 }; 3050 struct btf_ext_sec_info_param line_info = { 3051 .off = btf_ext->hdr->line_info_off, 3052 .len = btf_ext->hdr->line_info_len, 3053 .min_rec_size = sizeof(struct bpf_line_info_min), 3054 .ext_info = &btf_ext->line_info, 3055 .desc = "line_info", 3056 }; 3057 struct btf_ext_sec_info_param core_relo = { 3058 .min_rec_size = sizeof(struct bpf_core_relo), 3059 .ext_info = &btf_ext->core_relo_info, 3060 .desc = "core_relo", 3061 }; 3062 int err; 3063 3064 err = btf_ext_parse_sec_info(btf_ext, &func_info, is_native); 3065 if (err) 3066 return err; 3067 3068 err = btf_ext_parse_sec_info(btf_ext, &line_info, is_native); 3069 if (err) 3070 return err; 3071 3072 if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) 3073 return 0; /* skip core relos parsing */ 3074 3075 core_relo.off = btf_ext->hdr->core_relo_off; 3076 core_relo.len = btf_ext->hdr->core_relo_len; 3077 err = btf_ext_parse_sec_info(btf_ext, &core_relo, is_native); 3078 if (err) 3079 return err; 3080 3081 return 0; 3082 } 3083 3084 /* Swap byte-order of BTF.ext header with any endianness */ 3085 static void btf_ext_bswap_hdr(struct btf_ext_header *h) 3086 { 3087 bool is_native = h->magic == BTF_MAGIC; 3088 __u32 hdr_len; 3089 3090 hdr_len = is_native ? h->hdr_len : bswap_32(h->hdr_len); 3091 3092 h->magic = bswap_16(h->magic); 3093 h->hdr_len = bswap_32(h->hdr_len); 3094 h->func_info_off = bswap_32(h->func_info_off); 3095 h->func_info_len = bswap_32(h->func_info_len); 3096 h->line_info_off = bswap_32(h->line_info_off); 3097 h->line_info_len = bswap_32(h->line_info_len); 3098 3099 if (hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) 3100 return; 3101 3102 h->core_relo_off = bswap_32(h->core_relo_off); 3103 h->core_relo_len = bswap_32(h->core_relo_len); 3104 } 3105 3106 /* Swap byte-order of generic info subsection */ 3107 static void btf_ext_bswap_info_sec(void *info, __u32 len, bool is_native, 3108 info_rec_bswap_fn bswap_fn) 3109 { 3110 struct btf_ext_info_sec *sec; 3111 __u32 info_left, rec_size, *rs; 3112 3113 if (len == 0) 3114 return; 3115 3116 rs = info; /* info record size */ 3117 rec_size = is_native ? *rs : bswap_32(*rs); 3118 *rs = bswap_32(*rs); 3119 3120 sec = info + sizeof(__u32); /* info sec #1 */ 3121 info_left = len - sizeof(__u32); 3122 while (info_left) { 3123 unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec); 3124 __u32 i, num_recs; 3125 void *p; 3126 3127 num_recs = is_native ? sec->num_info : bswap_32(sec->num_info); 3128 sec->sec_name_off = bswap_32(sec->sec_name_off); 3129 sec->num_info = bswap_32(sec->num_info); 3130 p = sec->data; /* info rec #1 */ 3131 for (i = 0; i < num_recs; i++, p += rec_size) 3132 bswap_fn(p); 3133 sec = p; 3134 info_left -= sec_hdrlen + (__u64)rec_size * num_recs; 3135 } 3136 } 3137 3138 /* 3139 * Swap byte-order of all info data in a BTF.ext section 3140 * - requires BTF.ext hdr in native endianness 3141 */ 3142 static void btf_ext_bswap_info(struct btf_ext *btf_ext, void *data) 3143 { 3144 const bool is_native = btf_ext->swapped_endian; 3145 const struct btf_ext_header *h = data; 3146 void *info; 3147 3148 /* Swap func_info subsection byte-order */ 3149 info = data + h->hdr_len + h->func_info_off; 3150 btf_ext_bswap_info_sec(info, h->func_info_len, is_native, 3151 (info_rec_bswap_fn)bpf_func_info_bswap); 3152 3153 /* Swap line_info subsection byte-order */ 3154 info = data + h->hdr_len + h->line_info_off; 3155 btf_ext_bswap_info_sec(info, h->line_info_len, is_native, 3156 (info_rec_bswap_fn)bpf_line_info_bswap); 3157 3158 /* Swap core_relo subsection byte-order (if present) */ 3159 if (h->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) 3160 return; 3161 3162 info = data + h->hdr_len + h->core_relo_off; 3163 btf_ext_bswap_info_sec(info, h->core_relo_len, is_native, 3164 (info_rec_bswap_fn)bpf_core_relo_bswap); 3165 } 3166 3167 /* Parse hdr data and info sections: check and convert to native endianness */ 3168 static int btf_ext_parse(struct btf_ext *btf_ext) 3169 { 3170 __u32 hdr_len, data_size = btf_ext->data_size; 3171 struct btf_ext_header *hdr = btf_ext->hdr; 3172 bool swapped_endian = false; 3173 int err; 3174 3175 if (data_size < offsetofend(struct btf_ext_header, hdr_len)) { 3176 pr_debug("BTF.ext header too short\n"); 3177 return -EINVAL; 3178 } 3179 3180 hdr_len = hdr->hdr_len; 3181 if (hdr->magic == bswap_16(BTF_MAGIC)) { 3182 swapped_endian = true; 3183 hdr_len = bswap_32(hdr_len); 3184 } else if (hdr->magic != BTF_MAGIC) { 3185 pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic); 3186 return -EINVAL; 3187 } 3188 3189 /* Ensure known version of structs, current BTF_VERSION == 1 */ 3190 if (hdr->version != 1) { 3191 pr_debug("Unsupported BTF.ext version:%u\n", hdr->version); 3192 return -ENOTSUP; 3193 } 3194 3195 if (hdr->flags) { 3196 pr_debug("Unsupported BTF.ext flags:%x\n", hdr->flags); 3197 return -ENOTSUP; 3198 } 3199 3200 if (data_size < hdr_len) { 3201 pr_debug("BTF.ext header not found\n"); 3202 return -EINVAL; 3203 } else if (data_size == hdr_len) { 3204 pr_debug("BTF.ext has no data\n"); 3205 return -EINVAL; 3206 } 3207 3208 /* Verify mandatory hdr info details present */ 3209 if (hdr_len < offsetofend(struct btf_ext_header, line_info_len)) { 3210 pr_warn("BTF.ext header missing func_info, line_info\n"); 3211 return -EINVAL; 3212 } 3213 3214 /* Keep hdr native byte-order in memory for introspection */ 3215 if (swapped_endian) 3216 btf_ext_bswap_hdr(btf_ext->hdr); 3217 3218 /* Validate info subsections and cache key metadata */ 3219 err = btf_ext_parse_info(btf_ext, !swapped_endian); 3220 if (err) 3221 return err; 3222 3223 /* Keep infos native byte-order in memory for introspection */ 3224 if (swapped_endian) 3225 btf_ext_bswap_info(btf_ext, btf_ext->data); 3226 3227 /* 3228 * Set btf_ext->swapped_endian only after all header and info data has 3229 * been swapped, helping bswap functions determine if their data are 3230 * in native byte-order when called. 3231 */ 3232 btf_ext->swapped_endian = swapped_endian; 3233 return 0; 3234 } 3235 3236 void btf_ext__free(struct btf_ext *btf_ext) 3237 { 3238 if (IS_ERR_OR_NULL(btf_ext)) 3239 return; 3240 free(btf_ext->func_info.sec_idxs); 3241 free(btf_ext->line_info.sec_idxs); 3242 free(btf_ext->core_relo_info.sec_idxs); 3243 free(btf_ext->data); 3244 free(btf_ext->data_swapped); 3245 free(btf_ext); 3246 } 3247 3248 struct btf_ext *btf_ext__new(const __u8 *data, __u32 size) 3249 { 3250 struct btf_ext *btf_ext; 3251 int err; 3252 3253 btf_ext = calloc(1, sizeof(struct btf_ext)); 3254 if (!btf_ext) 3255 return libbpf_err_ptr(-ENOMEM); 3256 3257 btf_ext->data_size = size; 3258 btf_ext->data = malloc(size); 3259 if (!btf_ext->data) { 3260 err = -ENOMEM; 3261 goto done; 3262 } 3263 memcpy(btf_ext->data, data, size); 3264 3265 err = btf_ext_parse(btf_ext); 3266 3267 done: 3268 if (err) { 3269 btf_ext__free(btf_ext); 3270 return libbpf_err_ptr(err); 3271 } 3272 3273 return btf_ext; 3274 } 3275 3276 static void *btf_ext_raw_data(const struct btf_ext *btf_ext_ro, bool swap_endian) 3277 { 3278 struct btf_ext *btf_ext = (struct btf_ext *)btf_ext_ro; 3279 const __u32 data_sz = btf_ext->data_size; 3280 void *data; 3281 3282 /* Return native data (always present) or swapped data if present */ 3283 if (!swap_endian) 3284 return btf_ext->data; 3285 else if (btf_ext->data_swapped) 3286 return btf_ext->data_swapped; 3287 3288 /* Recreate missing swapped data, then cache and return */ 3289 data = calloc(1, data_sz); 3290 if (!data) 3291 return NULL; 3292 memcpy(data, btf_ext->data, data_sz); 3293 3294 btf_ext_bswap_info(btf_ext, data); 3295 btf_ext_bswap_hdr(data); 3296 btf_ext->data_swapped = data; 3297 return data; 3298 } 3299 3300 const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size) 3301 { 3302 void *data; 3303 3304 data = btf_ext_raw_data(btf_ext, btf_ext->swapped_endian); 3305 if (!data) 3306 return errno = ENOMEM, NULL; 3307 3308 *size = btf_ext->data_size; 3309 return data; 3310 } 3311 3312 __attribute__((alias("btf_ext__raw_data"))) 3313 const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size); 3314 3315 enum btf_endianness btf_ext__endianness(const struct btf_ext *btf_ext) 3316 { 3317 if (is_host_big_endian()) 3318 return btf_ext->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN; 3319 else 3320 return btf_ext->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN; 3321 } 3322 3323 int btf_ext__set_endianness(struct btf_ext *btf_ext, enum btf_endianness endian) 3324 { 3325 if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN) 3326 return libbpf_err(-EINVAL); 3327 3328 btf_ext->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN); 3329 3330 if (!btf_ext->swapped_endian) { 3331 free(btf_ext->data_swapped); 3332 btf_ext->data_swapped = NULL; 3333 } 3334 return 0; 3335 } 3336 3337 struct btf_dedup; 3338 3339 static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts); 3340 static void btf_dedup_free(struct btf_dedup *d); 3341 static int btf_dedup_prep(struct btf_dedup *d); 3342 static int btf_dedup_strings(struct btf_dedup *d); 3343 static int btf_dedup_prim_types(struct btf_dedup *d); 3344 static int btf_dedup_struct_types(struct btf_dedup *d); 3345 static int btf_dedup_ref_types(struct btf_dedup *d); 3346 static int btf_dedup_resolve_fwds(struct btf_dedup *d); 3347 static int btf_dedup_compact_types(struct btf_dedup *d); 3348 static int btf_dedup_remap_types(struct btf_dedup *d); 3349 3350 /* 3351 * Deduplicate BTF types and strings. 3352 * 3353 * BTF dedup algorithm takes as an input `struct btf` representing `.BTF` ELF 3354 * section with all BTF type descriptors and string data. It overwrites that 3355 * memory in-place with deduplicated types and strings without any loss of 3356 * information. If optional `struct btf_ext` representing '.BTF.ext' ELF section 3357 * is provided, all the strings referenced from .BTF.ext section are honored 3358 * and updated to point to the right offsets after deduplication. 3359 * 3360 * If function returns with error, type/string data might be garbled and should 3361 * be discarded. 3362 * 3363 * More verbose and detailed description of both problem btf_dedup is solving, 3364 * as well as solution could be found at: 3365 * https://facebookmicrosites.github.io/bpf/blog/2018/11/14/btf-enhancement.html 3366 * 3367 * Problem description and justification 3368 * ===================================== 3369 * 3370 * BTF type information is typically emitted either as a result of conversion 3371 * from DWARF to BTF or directly by compiler. In both cases, each compilation 3372 * unit contains information about a subset of all the types that are used 3373 * in an application. These subsets are frequently overlapping and contain a lot 3374 * of duplicated information when later concatenated together into a single 3375 * binary. This algorithm ensures that each unique type is represented by single 3376 * BTF type descriptor, greatly reducing resulting size of BTF data. 3377 * 3378 * Compilation unit isolation and subsequent duplication of data is not the only 3379 * problem. The same type hierarchy (e.g., struct and all the type that struct 3380 * references) in different compilation units can be represented in BTF to 3381 * various degrees of completeness (or, rather, incompleteness) due to 3382 * struct/union forward declarations. 3383 * 3384 * Let's take a look at an example, that we'll use to better understand the 3385 * problem (and solution). Suppose we have two compilation units, each using 3386 * same `struct S`, but each of them having incomplete type information about 3387 * struct's fields: 3388 * 3389 * // CU #1: 3390 * struct S; 3391 * struct A { 3392 * int a; 3393 * struct A* self; 3394 * struct S* parent; 3395 * }; 3396 * struct B; 3397 * struct S { 3398 * struct A* a_ptr; 3399 * struct B* b_ptr; 3400 * }; 3401 * 3402 * // CU #2: 3403 * struct S; 3404 * struct A; 3405 * struct B { 3406 * int b; 3407 * struct B* self; 3408 * struct S* parent; 3409 * }; 3410 * struct S { 3411 * struct A* a_ptr; 3412 * struct B* b_ptr; 3413 * }; 3414 * 3415 * In case of CU #1, BTF data will know only that `struct B` exist (but no 3416 * more), but will know the complete type information about `struct A`. While 3417 * for CU #2, it will know full type information about `struct B`, but will 3418 * only know about forward declaration of `struct A` (in BTF terms, it will 3419 * have `BTF_KIND_FWD` type descriptor with name `B`). 3420 * 3421 * This compilation unit isolation means that it's possible that there is no 3422 * single CU with complete type information describing structs `S`, `A`, and 3423 * `B`. Also, we might get tons of duplicated and redundant type information. 3424 * 3425 * Additional complication we need to keep in mind comes from the fact that 3426 * types, in general, can form graphs containing cycles, not just DAGs. 3427 * 3428 * While algorithm does deduplication, it also merges and resolves type 3429 * information (unless disabled throught `struct btf_opts`), whenever possible. 3430 * E.g., in the example above with two compilation units having partial type 3431 * information for structs `A` and `B`, the output of algorithm will emit 3432 * a single copy of each BTF type that describes structs `A`, `B`, and `S` 3433 * (as well as type information for `int` and pointers), as if they were defined 3434 * in a single compilation unit as: 3435 * 3436 * struct A { 3437 * int a; 3438 * struct A* self; 3439 * struct S* parent; 3440 * }; 3441 * struct B { 3442 * int b; 3443 * struct B* self; 3444 * struct S* parent; 3445 * }; 3446 * struct S { 3447 * struct A* a_ptr; 3448 * struct B* b_ptr; 3449 * }; 3450 * 3451 * Algorithm summary 3452 * ================= 3453 * 3454 * Algorithm completes its work in 7 separate passes: 3455 * 3456 * 1. Strings deduplication. 3457 * 2. Primitive types deduplication (int, enum, fwd). 3458 * 3. Struct/union types deduplication. 3459 * 4. Resolve unambiguous forward declarations. 3460 * 5. Reference types deduplication (pointers, typedefs, arrays, funcs, func 3461 * protos, and const/volatile/restrict modifiers). 3462 * 6. Types compaction. 3463 * 7. Types remapping. 3464 * 3465 * Algorithm determines canonical type descriptor, which is a single 3466 * representative type for each truly unique type. This canonical type is the 3467 * one that will go into final deduplicated BTF type information. For 3468 * struct/unions, it is also the type that algorithm will merge additional type 3469 * information into (while resolving FWDs), as it discovers it from data in 3470 * other CUs. Each input BTF type eventually gets either mapped to itself, if 3471 * that type is canonical, or to some other type, if that type is equivalent 3472 * and was chosen as canonical representative. This mapping is stored in 3473 * `btf_dedup->map` array. This map is also used to record STRUCT/UNION that 3474 * FWD type got resolved to. 3475 * 3476 * To facilitate fast discovery of canonical types, we also maintain canonical 3477 * index (`btf_dedup->dedup_table`), which maps type descriptor's signature hash 3478 * (i.e., hashed kind, name, size, fields, etc) into a list of canonical types 3479 * that match that signature. With sufficiently good choice of type signature 3480 * hashing function, we can limit number of canonical types for each unique type 3481 * signature to a very small number, allowing to find canonical type for any 3482 * duplicated type very quickly. 3483 * 3484 * Struct/union deduplication is the most critical part and algorithm for 3485 * deduplicating structs/unions is described in greater details in comments for 3486 * `btf_dedup_is_equiv` function. 3487 */ 3488 int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts) 3489 { 3490 struct btf_dedup *d; 3491 int err; 3492 3493 if (!OPTS_VALID(opts, btf_dedup_opts)) 3494 return libbpf_err(-EINVAL); 3495 3496 d = btf_dedup_new(btf, opts); 3497 if (IS_ERR(d)) { 3498 pr_debug("btf_dedup_new failed: %ld\n", PTR_ERR(d)); 3499 return libbpf_err(-EINVAL); 3500 } 3501 3502 if (btf_ensure_modifiable(btf)) { 3503 err = -ENOMEM; 3504 goto done; 3505 } 3506 3507 err = btf_dedup_prep(d); 3508 if (err) { 3509 pr_debug("btf_dedup_prep failed: %s\n", errstr(err)); 3510 goto done; 3511 } 3512 err = btf_dedup_strings(d); 3513 if (err < 0) { 3514 pr_debug("btf_dedup_strings failed: %s\n", errstr(err)); 3515 goto done; 3516 } 3517 err = btf_dedup_prim_types(d); 3518 if (err < 0) { 3519 pr_debug("btf_dedup_prim_types failed: %s\n", errstr(err)); 3520 goto done; 3521 } 3522 err = btf_dedup_struct_types(d); 3523 if (err < 0) { 3524 pr_debug("btf_dedup_struct_types failed: %s\n", errstr(err)); 3525 goto done; 3526 } 3527 err = btf_dedup_resolve_fwds(d); 3528 if (err < 0) { 3529 pr_debug("btf_dedup_resolve_fwds failed: %s\n", errstr(err)); 3530 goto done; 3531 } 3532 err = btf_dedup_ref_types(d); 3533 if (err < 0) { 3534 pr_debug("btf_dedup_ref_types failed: %s\n", errstr(err)); 3535 goto done; 3536 } 3537 err = btf_dedup_compact_types(d); 3538 if (err < 0) { 3539 pr_debug("btf_dedup_compact_types failed: %s\n", errstr(err)); 3540 goto done; 3541 } 3542 err = btf_dedup_remap_types(d); 3543 if (err < 0) { 3544 pr_debug("btf_dedup_remap_types failed: %s\n", errstr(err)); 3545 goto done; 3546 } 3547 3548 done: 3549 btf_dedup_free(d); 3550 return libbpf_err(err); 3551 } 3552 3553 #define BTF_UNPROCESSED_ID ((__u32)-1) 3554 #define BTF_IN_PROGRESS_ID ((__u32)-2) 3555 3556 struct btf_dedup { 3557 /* .BTF section to be deduped in-place */ 3558 struct btf *btf; 3559 /* 3560 * Optional .BTF.ext section. When provided, any strings referenced 3561 * from it will be taken into account when deduping strings 3562 */ 3563 struct btf_ext *btf_ext; 3564 /* 3565 * This is a map from any type's signature hash to a list of possible 3566 * canonical representative type candidates. Hash collisions are 3567 * ignored, so even types of various kinds can share same list of 3568 * candidates, which is fine because we rely on subsequent 3569 * btf_xxx_equal() checks to authoritatively verify type equality. 3570 */ 3571 struct hashmap *dedup_table; 3572 /* Canonical types map */ 3573 __u32 *map; 3574 /* Hypothetical mapping, used during type graph equivalence checks */ 3575 __u32 *hypot_map; 3576 __u32 *hypot_list; 3577 size_t hypot_cnt; 3578 size_t hypot_cap; 3579 /* Whether hypothetical mapping, if successful, would need to adjust 3580 * already canonicalized types (due to a new forward declaration to 3581 * concrete type resolution). In such case, during split BTF dedup 3582 * candidate type would still be considered as different, because base 3583 * BTF is considered to be immutable. 3584 */ 3585 bool hypot_adjust_canon; 3586 /* Various option modifying behavior of algorithm */ 3587 struct btf_dedup_opts opts; 3588 /* temporary strings deduplication state */ 3589 struct strset *strs_set; 3590 }; 3591 3592 static unsigned long hash_combine(unsigned long h, unsigned long value) 3593 { 3594 return h * 31 + value; 3595 } 3596 3597 #define for_each_dedup_cand(d, node, hash) \ 3598 hashmap__for_each_key_entry(d->dedup_table, node, hash) 3599 3600 static int btf_dedup_table_add(struct btf_dedup *d, long hash, __u32 type_id) 3601 { 3602 return hashmap__append(d->dedup_table, hash, type_id); 3603 } 3604 3605 static int btf_dedup_hypot_map_add(struct btf_dedup *d, 3606 __u32 from_id, __u32 to_id) 3607 { 3608 if (d->hypot_cnt == d->hypot_cap) { 3609 __u32 *new_list; 3610 3611 d->hypot_cap += max((size_t)16, d->hypot_cap / 2); 3612 new_list = libbpf_reallocarray(d->hypot_list, d->hypot_cap, sizeof(__u32)); 3613 if (!new_list) 3614 return -ENOMEM; 3615 d->hypot_list = new_list; 3616 } 3617 d->hypot_list[d->hypot_cnt++] = from_id; 3618 d->hypot_map[from_id] = to_id; 3619 return 0; 3620 } 3621 3622 static void btf_dedup_clear_hypot_map(struct btf_dedup *d) 3623 { 3624 int i; 3625 3626 for (i = 0; i < d->hypot_cnt; i++) 3627 d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID; 3628 d->hypot_cnt = 0; 3629 d->hypot_adjust_canon = false; 3630 } 3631 3632 static void btf_dedup_free(struct btf_dedup *d) 3633 { 3634 hashmap__free(d->dedup_table); 3635 d->dedup_table = NULL; 3636 3637 free(d->map); 3638 d->map = NULL; 3639 3640 free(d->hypot_map); 3641 d->hypot_map = NULL; 3642 3643 free(d->hypot_list); 3644 d->hypot_list = NULL; 3645 3646 free(d); 3647 } 3648 3649 static size_t btf_dedup_identity_hash_fn(long key, void *ctx) 3650 { 3651 return key; 3652 } 3653 3654 static size_t btf_dedup_collision_hash_fn(long key, void *ctx) 3655 { 3656 return 0; 3657 } 3658 3659 static bool btf_dedup_equal_fn(long k1, long k2, void *ctx) 3660 { 3661 return k1 == k2; 3662 } 3663 3664 static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts) 3665 { 3666 struct btf_dedup *d = calloc(1, sizeof(struct btf_dedup)); 3667 hashmap_hash_fn hash_fn = btf_dedup_identity_hash_fn; 3668 int i, err = 0, type_cnt; 3669 3670 if (!d) 3671 return ERR_PTR(-ENOMEM); 3672 3673 if (OPTS_GET(opts, force_collisions, false)) 3674 hash_fn = btf_dedup_collision_hash_fn; 3675 3676 d->btf = btf; 3677 d->btf_ext = OPTS_GET(opts, btf_ext, NULL); 3678 3679 d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL); 3680 if (IS_ERR(d->dedup_table)) { 3681 err = PTR_ERR(d->dedup_table); 3682 d->dedup_table = NULL; 3683 goto done; 3684 } 3685 3686 type_cnt = btf__type_cnt(btf); 3687 d->map = malloc(sizeof(__u32) * type_cnt); 3688 if (!d->map) { 3689 err = -ENOMEM; 3690 goto done; 3691 } 3692 /* special BTF "void" type is made canonical immediately */ 3693 d->map[0] = 0; 3694 for (i = 1; i < type_cnt; i++) { 3695 struct btf_type *t = btf_type_by_id(d->btf, i); 3696 3697 /* VAR and DATASEC are never deduped and are self-canonical */ 3698 if (btf_is_var(t) || btf_is_datasec(t)) 3699 d->map[i] = i; 3700 else 3701 d->map[i] = BTF_UNPROCESSED_ID; 3702 } 3703 3704 d->hypot_map = malloc(sizeof(__u32) * type_cnt); 3705 if (!d->hypot_map) { 3706 err = -ENOMEM; 3707 goto done; 3708 } 3709 for (i = 0; i < type_cnt; i++) 3710 d->hypot_map[i] = BTF_UNPROCESSED_ID; 3711 3712 done: 3713 if (err) { 3714 btf_dedup_free(d); 3715 return ERR_PTR(err); 3716 } 3717 3718 return d; 3719 } 3720 3721 /* 3722 * Iterate over all possible places in .BTF and .BTF.ext that can reference 3723 * string and pass pointer to it to a provided callback `fn`. 3724 */ 3725 static int btf_for_each_str_off(struct btf_dedup *d, str_off_visit_fn fn, void *ctx) 3726 { 3727 int i, r; 3728 3729 for (i = 0; i < d->btf->nr_types; i++) { 3730 struct btf_field_iter it; 3731 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i); 3732 __u32 *str_off; 3733 3734 r = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS); 3735 if (r) 3736 return r; 3737 3738 while ((str_off = btf_field_iter_next(&it))) { 3739 r = fn(str_off, ctx); 3740 if (r) 3741 return r; 3742 } 3743 } 3744 3745 if (!d->btf_ext) 3746 return 0; 3747 3748 r = btf_ext_visit_str_offs(d->btf_ext, fn, ctx); 3749 if (r) 3750 return r; 3751 3752 return 0; 3753 } 3754 3755 static int strs_dedup_remap_str_off(__u32 *str_off_ptr, void *ctx) 3756 { 3757 struct btf_dedup *d = ctx; 3758 __u32 str_off = *str_off_ptr; 3759 const char *s; 3760 int off, err; 3761 3762 /* don't touch empty string or string in main BTF */ 3763 if (str_off == 0 || str_off < d->btf->start_str_off) 3764 return 0; 3765 3766 s = btf__str_by_offset(d->btf, str_off); 3767 if (d->btf->base_btf) { 3768 err = btf__find_str(d->btf->base_btf, s); 3769 if (err >= 0) { 3770 *str_off_ptr = err; 3771 return 0; 3772 } 3773 if (err != -ENOENT) 3774 return err; 3775 } 3776 3777 off = strset__add_str(d->strs_set, s); 3778 if (off < 0) 3779 return off; 3780 3781 *str_off_ptr = d->btf->start_str_off + off; 3782 return 0; 3783 } 3784 3785 /* 3786 * Dedup string and filter out those that are not referenced from either .BTF 3787 * or .BTF.ext (if provided) sections. 3788 * 3789 * This is done by building index of all strings in BTF's string section, 3790 * then iterating over all entities that can reference strings (e.g., type 3791 * names, struct field names, .BTF.ext line info, etc) and marking corresponding 3792 * strings as used. After that all used strings are deduped and compacted into 3793 * sequential blob of memory and new offsets are calculated. Then all the string 3794 * references are iterated again and rewritten using new offsets. 3795 */ 3796 static int btf_dedup_strings(struct btf_dedup *d) 3797 { 3798 int err; 3799 3800 if (d->btf->strs_deduped) 3801 return 0; 3802 3803 d->strs_set = strset__new(BTF_MAX_STR_OFFSET, NULL, 0); 3804 if (IS_ERR(d->strs_set)) { 3805 err = PTR_ERR(d->strs_set); 3806 goto err_out; 3807 } 3808 3809 if (!d->btf->base_btf) { 3810 /* insert empty string; we won't be looking it up during strings 3811 * dedup, but it's good to have it for generic BTF string lookups 3812 */ 3813 err = strset__add_str(d->strs_set, ""); 3814 if (err < 0) 3815 goto err_out; 3816 } 3817 3818 /* remap string offsets */ 3819 err = btf_for_each_str_off(d, strs_dedup_remap_str_off, d); 3820 if (err) 3821 goto err_out; 3822 3823 /* replace BTF string data and hash with deduped ones */ 3824 strset__free(d->btf->strs_set); 3825 d->btf->hdr->str_len = strset__data_size(d->strs_set); 3826 d->btf->strs_set = d->strs_set; 3827 d->strs_set = NULL; 3828 d->btf->strs_deduped = true; 3829 return 0; 3830 3831 err_out: 3832 strset__free(d->strs_set); 3833 d->strs_set = NULL; 3834 3835 return err; 3836 } 3837 3838 static long btf_hash_common(struct btf_type *t) 3839 { 3840 long h; 3841 3842 h = hash_combine(0, t->name_off); 3843 h = hash_combine(h, t->info); 3844 h = hash_combine(h, t->size); 3845 return h; 3846 } 3847 3848 static bool btf_equal_common(struct btf_type *t1, struct btf_type *t2) 3849 { 3850 return t1->name_off == t2->name_off && 3851 t1->info == t2->info && 3852 t1->size == t2->size; 3853 } 3854 3855 /* Calculate type signature hash of INT or TAG. */ 3856 static long btf_hash_int_decl_tag(struct btf_type *t) 3857 { 3858 __u32 info = *(__u32 *)(t + 1); 3859 long h; 3860 3861 h = btf_hash_common(t); 3862 h = hash_combine(h, info); 3863 return h; 3864 } 3865 3866 /* Check structural equality of two INTs or TAGs. */ 3867 static bool btf_equal_int_tag(struct btf_type *t1, struct btf_type *t2) 3868 { 3869 __u32 info1, info2; 3870 3871 if (!btf_equal_common(t1, t2)) 3872 return false; 3873 info1 = *(__u32 *)(t1 + 1); 3874 info2 = *(__u32 *)(t2 + 1); 3875 return info1 == info2; 3876 } 3877 3878 /* Calculate type signature hash of ENUM/ENUM64. */ 3879 static long btf_hash_enum(struct btf_type *t) 3880 { 3881 long h; 3882 3883 /* don't hash vlen, enum members and size to support enum fwd resolving */ 3884 h = hash_combine(0, t->name_off); 3885 return h; 3886 } 3887 3888 static bool btf_equal_enum_members(struct btf_type *t1, struct btf_type *t2) 3889 { 3890 const struct btf_enum *m1, *m2; 3891 __u16 vlen; 3892 int i; 3893 3894 vlen = btf_vlen(t1); 3895 m1 = btf_enum(t1); 3896 m2 = btf_enum(t2); 3897 for (i = 0; i < vlen; i++) { 3898 if (m1->name_off != m2->name_off || m1->val != m2->val) 3899 return false; 3900 m1++; 3901 m2++; 3902 } 3903 return true; 3904 } 3905 3906 static bool btf_equal_enum64_members(struct btf_type *t1, struct btf_type *t2) 3907 { 3908 const struct btf_enum64 *m1, *m2; 3909 __u16 vlen; 3910 int i; 3911 3912 vlen = btf_vlen(t1); 3913 m1 = btf_enum64(t1); 3914 m2 = btf_enum64(t2); 3915 for (i = 0; i < vlen; i++) { 3916 if (m1->name_off != m2->name_off || m1->val_lo32 != m2->val_lo32 || 3917 m1->val_hi32 != m2->val_hi32) 3918 return false; 3919 m1++; 3920 m2++; 3921 } 3922 return true; 3923 } 3924 3925 /* Check structural equality of two ENUMs or ENUM64s. */ 3926 static bool btf_equal_enum(struct btf_type *t1, struct btf_type *t2) 3927 { 3928 if (!btf_equal_common(t1, t2)) 3929 return false; 3930 3931 /* t1 & t2 kinds are identical because of btf_equal_common */ 3932 if (btf_kind(t1) == BTF_KIND_ENUM) 3933 return btf_equal_enum_members(t1, t2); 3934 else 3935 return btf_equal_enum64_members(t1, t2); 3936 } 3937 3938 static inline bool btf_is_enum_fwd(struct btf_type *t) 3939 { 3940 return btf_is_any_enum(t) && btf_vlen(t) == 0; 3941 } 3942 3943 static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2) 3944 { 3945 if (!btf_is_enum_fwd(t1) && !btf_is_enum_fwd(t2)) 3946 return btf_equal_enum(t1, t2); 3947 /* At this point either t1 or t2 or both are forward declarations, thus: 3948 * - skip comparing vlen because it is zero for forward declarations; 3949 * - skip comparing size to allow enum forward declarations 3950 * to be compatible with enum64 full declarations; 3951 * - skip comparing kind for the same reason. 3952 */ 3953 return t1->name_off == t2->name_off && 3954 btf_is_any_enum(t1) && btf_is_any_enum(t2); 3955 } 3956 3957 /* 3958 * Calculate type signature hash of STRUCT/UNION, ignoring referenced type IDs, 3959 * as referenced type IDs equivalence is established separately during type 3960 * graph equivalence check algorithm. 3961 */ 3962 static long btf_hash_struct(struct btf_type *t) 3963 { 3964 const struct btf_member *member = btf_members(t); 3965 __u32 vlen = btf_vlen(t); 3966 long h = btf_hash_common(t); 3967 int i; 3968 3969 for (i = 0; i < vlen; i++) { 3970 h = hash_combine(h, member->name_off); 3971 h = hash_combine(h, member->offset); 3972 /* no hashing of referenced type ID, it can be unresolved yet */ 3973 member++; 3974 } 3975 return h; 3976 } 3977 3978 /* 3979 * Check structural compatibility of two STRUCTs/UNIONs, ignoring referenced 3980 * type IDs. This check is performed during type graph equivalence check and 3981 * referenced types equivalence is checked separately. 3982 */ 3983 static bool btf_shallow_equal_struct(struct btf_type *t1, struct btf_type *t2) 3984 { 3985 const struct btf_member *m1, *m2; 3986 __u16 vlen; 3987 int i; 3988 3989 if (!btf_equal_common(t1, t2)) 3990 return false; 3991 3992 vlen = btf_vlen(t1); 3993 m1 = btf_members(t1); 3994 m2 = btf_members(t2); 3995 for (i = 0; i < vlen; i++) { 3996 if (m1->name_off != m2->name_off || m1->offset != m2->offset) 3997 return false; 3998 m1++; 3999 m2++; 4000 } 4001 return true; 4002 } 4003 4004 /* 4005 * Calculate type signature hash of ARRAY, including referenced type IDs, 4006 * under assumption that they were already resolved to canonical type IDs and 4007 * are not going to change. 4008 */ 4009 static long btf_hash_array(struct btf_type *t) 4010 { 4011 const struct btf_array *info = btf_array(t); 4012 long h = btf_hash_common(t); 4013 4014 h = hash_combine(h, info->type); 4015 h = hash_combine(h, info->index_type); 4016 h = hash_combine(h, info->nelems); 4017 return h; 4018 } 4019 4020 /* 4021 * Check exact equality of two ARRAYs, taking into account referenced 4022 * type IDs, under assumption that they were already resolved to canonical 4023 * type IDs and are not going to change. 4024 * This function is called during reference types deduplication to compare 4025 * ARRAY to potential canonical representative. 4026 */ 4027 static bool btf_equal_array(struct btf_type *t1, struct btf_type *t2) 4028 { 4029 const struct btf_array *info1, *info2; 4030 4031 if (!btf_equal_common(t1, t2)) 4032 return false; 4033 4034 info1 = btf_array(t1); 4035 info2 = btf_array(t2); 4036 return info1->type == info2->type && 4037 info1->index_type == info2->index_type && 4038 info1->nelems == info2->nelems; 4039 } 4040 4041 /* 4042 * Check structural compatibility of two ARRAYs, ignoring referenced type 4043 * IDs. This check is performed during type graph equivalence check and 4044 * referenced types equivalence is checked separately. 4045 */ 4046 static bool btf_compat_array(struct btf_type *t1, struct btf_type *t2) 4047 { 4048 if (!btf_equal_common(t1, t2)) 4049 return false; 4050 4051 return btf_array(t1)->nelems == btf_array(t2)->nelems; 4052 } 4053 4054 /* 4055 * Calculate type signature hash of FUNC_PROTO, including referenced type IDs, 4056 * under assumption that they were already resolved to canonical type IDs and 4057 * are not going to change. 4058 */ 4059 static long btf_hash_fnproto(struct btf_type *t) 4060 { 4061 const struct btf_param *member = btf_params(t); 4062 __u16 vlen = btf_vlen(t); 4063 long h = btf_hash_common(t); 4064 int i; 4065 4066 for (i = 0; i < vlen; i++) { 4067 h = hash_combine(h, member->name_off); 4068 h = hash_combine(h, member->type); 4069 member++; 4070 } 4071 return h; 4072 } 4073 4074 /* 4075 * Check exact equality of two FUNC_PROTOs, taking into account referenced 4076 * type IDs, under assumption that they were already resolved to canonical 4077 * type IDs and are not going to change. 4078 * This function is called during reference types deduplication to compare 4079 * FUNC_PROTO to potential canonical representative. 4080 */ 4081 static bool btf_equal_fnproto(struct btf_type *t1, struct btf_type *t2) 4082 { 4083 const struct btf_param *m1, *m2; 4084 __u16 vlen; 4085 int i; 4086 4087 if (!btf_equal_common(t1, t2)) 4088 return false; 4089 4090 vlen = btf_vlen(t1); 4091 m1 = btf_params(t1); 4092 m2 = btf_params(t2); 4093 for (i = 0; i < vlen; i++) { 4094 if (m1->name_off != m2->name_off || m1->type != m2->type) 4095 return false; 4096 m1++; 4097 m2++; 4098 } 4099 return true; 4100 } 4101 4102 /* 4103 * Check structural compatibility of two FUNC_PROTOs, ignoring referenced type 4104 * IDs. This check is performed during type graph equivalence check and 4105 * referenced types equivalence is checked separately. 4106 */ 4107 static bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2) 4108 { 4109 const struct btf_param *m1, *m2; 4110 __u16 vlen; 4111 int i; 4112 4113 /* skip return type ID */ 4114 if (t1->name_off != t2->name_off || t1->info != t2->info) 4115 return false; 4116 4117 vlen = btf_vlen(t1); 4118 m1 = btf_params(t1); 4119 m2 = btf_params(t2); 4120 for (i = 0; i < vlen; i++) { 4121 if (m1->name_off != m2->name_off) 4122 return false; 4123 m1++; 4124 m2++; 4125 } 4126 return true; 4127 } 4128 4129 /* Prepare split BTF for deduplication by calculating hashes of base BTF's 4130 * types and initializing the rest of the state (canonical type mapping) for 4131 * the fixed base BTF part. 4132 */ 4133 static int btf_dedup_prep(struct btf_dedup *d) 4134 { 4135 struct btf_type *t; 4136 int type_id; 4137 long h; 4138 4139 if (!d->btf->base_btf) 4140 return 0; 4141 4142 for (type_id = 1; type_id < d->btf->start_id; type_id++) { 4143 t = btf_type_by_id(d->btf, type_id); 4144 4145 /* all base BTF types are self-canonical by definition */ 4146 d->map[type_id] = type_id; 4147 4148 switch (btf_kind(t)) { 4149 case BTF_KIND_VAR: 4150 case BTF_KIND_DATASEC: 4151 /* VAR and DATASEC are never hash/deduplicated */ 4152 continue; 4153 case BTF_KIND_CONST: 4154 case BTF_KIND_VOLATILE: 4155 case BTF_KIND_RESTRICT: 4156 case BTF_KIND_PTR: 4157 case BTF_KIND_FWD: 4158 case BTF_KIND_TYPEDEF: 4159 case BTF_KIND_FUNC: 4160 case BTF_KIND_FLOAT: 4161 case BTF_KIND_TYPE_TAG: 4162 h = btf_hash_common(t); 4163 break; 4164 case BTF_KIND_INT: 4165 case BTF_KIND_DECL_TAG: 4166 h = btf_hash_int_decl_tag(t); 4167 break; 4168 case BTF_KIND_ENUM: 4169 case BTF_KIND_ENUM64: 4170 h = btf_hash_enum(t); 4171 break; 4172 case BTF_KIND_STRUCT: 4173 case BTF_KIND_UNION: 4174 h = btf_hash_struct(t); 4175 break; 4176 case BTF_KIND_ARRAY: 4177 h = btf_hash_array(t); 4178 break; 4179 case BTF_KIND_FUNC_PROTO: 4180 h = btf_hash_fnproto(t); 4181 break; 4182 default: 4183 pr_debug("unknown kind %d for type [%d]\n", btf_kind(t), type_id); 4184 return -EINVAL; 4185 } 4186 if (btf_dedup_table_add(d, h, type_id)) 4187 return -ENOMEM; 4188 } 4189 4190 return 0; 4191 } 4192 4193 /* 4194 * Deduplicate primitive types, that can't reference other types, by calculating 4195 * their type signature hash and comparing them with any possible canonical 4196 * candidate. If no canonical candidate matches, type itself is marked as 4197 * canonical and is added into `btf_dedup->dedup_table` as another candidate. 4198 */ 4199 static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) 4200 { 4201 struct btf_type *t = btf_type_by_id(d->btf, type_id); 4202 struct hashmap_entry *hash_entry; 4203 struct btf_type *cand; 4204 /* if we don't find equivalent type, then we are canonical */ 4205 __u32 new_id = type_id; 4206 __u32 cand_id; 4207 long h; 4208 4209 switch (btf_kind(t)) { 4210 case BTF_KIND_CONST: 4211 case BTF_KIND_VOLATILE: 4212 case BTF_KIND_RESTRICT: 4213 case BTF_KIND_PTR: 4214 case BTF_KIND_TYPEDEF: 4215 case BTF_KIND_ARRAY: 4216 case BTF_KIND_STRUCT: 4217 case BTF_KIND_UNION: 4218 case BTF_KIND_FUNC: 4219 case BTF_KIND_FUNC_PROTO: 4220 case BTF_KIND_VAR: 4221 case BTF_KIND_DATASEC: 4222 case BTF_KIND_DECL_TAG: 4223 case BTF_KIND_TYPE_TAG: 4224 return 0; 4225 4226 case BTF_KIND_INT: 4227 h = btf_hash_int_decl_tag(t); 4228 for_each_dedup_cand(d, hash_entry, h) { 4229 cand_id = hash_entry->value; 4230 cand = btf_type_by_id(d->btf, cand_id); 4231 if (btf_equal_int_tag(t, cand)) { 4232 new_id = cand_id; 4233 break; 4234 } 4235 } 4236 break; 4237 4238 case BTF_KIND_ENUM: 4239 case BTF_KIND_ENUM64: 4240 h = btf_hash_enum(t); 4241 for_each_dedup_cand(d, hash_entry, h) { 4242 cand_id = hash_entry->value; 4243 cand = btf_type_by_id(d->btf, cand_id); 4244 if (btf_equal_enum(t, cand)) { 4245 new_id = cand_id; 4246 break; 4247 } 4248 if (btf_compat_enum(t, cand)) { 4249 if (btf_is_enum_fwd(t)) { 4250 /* resolve fwd to full enum */ 4251 new_id = cand_id; 4252 break; 4253 } 4254 /* resolve canonical enum fwd to full enum */ 4255 d->map[cand_id] = type_id; 4256 } 4257 } 4258 break; 4259 4260 case BTF_KIND_FWD: 4261 case BTF_KIND_FLOAT: 4262 h = btf_hash_common(t); 4263 for_each_dedup_cand(d, hash_entry, h) { 4264 cand_id = hash_entry->value; 4265 cand = btf_type_by_id(d->btf, cand_id); 4266 if (btf_equal_common(t, cand)) { 4267 new_id = cand_id; 4268 break; 4269 } 4270 } 4271 break; 4272 4273 default: 4274 return -EINVAL; 4275 } 4276 4277 d->map[type_id] = new_id; 4278 if (type_id == new_id && btf_dedup_table_add(d, h, type_id)) 4279 return -ENOMEM; 4280 4281 return 0; 4282 } 4283 4284 static int btf_dedup_prim_types(struct btf_dedup *d) 4285 { 4286 int i, err; 4287 4288 for (i = 0; i < d->btf->nr_types; i++) { 4289 err = btf_dedup_prim_type(d, d->btf->start_id + i); 4290 if (err) 4291 return err; 4292 } 4293 return 0; 4294 } 4295 4296 /* 4297 * Check whether type is already mapped into canonical one (could be to itself). 4298 */ 4299 static inline bool is_type_mapped(struct btf_dedup *d, uint32_t type_id) 4300 { 4301 return d->map[type_id] <= BTF_MAX_NR_TYPES; 4302 } 4303 4304 /* 4305 * Resolve type ID into its canonical type ID, if any; otherwise return original 4306 * type ID. If type is FWD and is resolved into STRUCT/UNION already, follow 4307 * STRUCT/UNION link and resolve it into canonical type ID as well. 4308 */ 4309 static inline __u32 resolve_type_id(struct btf_dedup *d, __u32 type_id) 4310 { 4311 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) 4312 type_id = d->map[type_id]; 4313 return type_id; 4314 } 4315 4316 /* 4317 * Resolve FWD to underlying STRUCT/UNION, if any; otherwise return original 4318 * type ID. 4319 */ 4320 static uint32_t resolve_fwd_id(struct btf_dedup *d, uint32_t type_id) 4321 { 4322 __u32 orig_type_id = type_id; 4323 4324 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) 4325 return type_id; 4326 4327 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) 4328 type_id = d->map[type_id]; 4329 4330 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) 4331 return type_id; 4332 4333 return orig_type_id; 4334 } 4335 4336 4337 static inline __u16 btf_fwd_kind(struct btf_type *t) 4338 { 4339 return btf_kflag(t) ? BTF_KIND_UNION : BTF_KIND_STRUCT; 4340 } 4341 4342 /* Check if given two types are identical ARRAY definitions */ 4343 static bool btf_dedup_identical_arrays(struct btf_dedup *d, __u32 id1, __u32 id2) 4344 { 4345 struct btf_type *t1, *t2; 4346 4347 t1 = btf_type_by_id(d->btf, id1); 4348 t2 = btf_type_by_id(d->btf, id2); 4349 if (!btf_is_array(t1) || !btf_is_array(t2)) 4350 return false; 4351 4352 return btf_equal_array(t1, t2); 4353 } 4354 4355 /* Check if given two types are identical STRUCT/UNION definitions */ 4356 static bool btf_dedup_identical_structs(struct btf_dedup *d, __u32 id1, __u32 id2) 4357 { 4358 const struct btf_member *m1, *m2; 4359 struct btf_type *t1, *t2; 4360 int n, i; 4361 4362 t1 = btf_type_by_id(d->btf, id1); 4363 t2 = btf_type_by_id(d->btf, id2); 4364 4365 if (!btf_is_composite(t1) || btf_kind(t1) != btf_kind(t2)) 4366 return false; 4367 4368 if (!btf_shallow_equal_struct(t1, t2)) 4369 return false; 4370 4371 m1 = btf_members(t1); 4372 m2 = btf_members(t2); 4373 for (i = 0, n = btf_vlen(t1); i < n; i++, m1++, m2++) { 4374 if (m1->type != m2->type && 4375 !btf_dedup_identical_arrays(d, m1->type, m2->type) && 4376 !btf_dedup_identical_structs(d, m1->type, m2->type)) 4377 return false; 4378 } 4379 return true; 4380 } 4381 4382 /* 4383 * Check equivalence of BTF type graph formed by candidate struct/union (we'll 4384 * call it "candidate graph" in this description for brevity) to a type graph 4385 * formed by (potential) canonical struct/union ("canonical graph" for brevity 4386 * here, though keep in mind that not all types in canonical graph are 4387 * necessarily canonical representatives themselves, some of them might be 4388 * duplicates or its uniqueness might not have been established yet). 4389 * Returns: 4390 * - >0, if type graphs are equivalent; 4391 * - 0, if not equivalent; 4392 * - <0, on error. 4393 * 4394 * Algorithm performs side-by-side DFS traversal of both type graphs and checks 4395 * equivalence of BTF types at each step. If at any point BTF types in candidate 4396 * and canonical graphs are not compatible structurally, whole graphs are 4397 * incompatible. If types are structurally equivalent (i.e., all information 4398 * except referenced type IDs is exactly the same), a mapping from `canon_id` to 4399 * a `cand_id` is recoded in hypothetical mapping (`btf_dedup->hypot_map`). 4400 * If a type references other types, then those referenced types are checked 4401 * for equivalence recursively. 4402 * 4403 * During DFS traversal, if we find that for current `canon_id` type we 4404 * already have some mapping in hypothetical map, we check for two possible 4405 * situations: 4406 * - `canon_id` is mapped to exactly the same type as `cand_id`. This will 4407 * happen when type graphs have cycles. In this case we assume those two 4408 * types are equivalent. 4409 * - `canon_id` is mapped to different type. This is contradiction in our 4410 * hypothetical mapping, because same graph in canonical graph corresponds 4411 * to two different types in candidate graph, which for equivalent type 4412 * graphs shouldn't happen. This condition terminates equivalence check 4413 * with negative result. 4414 * 4415 * If type graphs traversal exhausts types to check and find no contradiction, 4416 * then type graphs are equivalent. 4417 * 4418 * When checking types for equivalence, there is one special case: FWD types. 4419 * If FWD type resolution is allowed and one of the types (either from canonical 4420 * or candidate graph) is FWD and other is STRUCT/UNION (depending on FWD's kind 4421 * flag) and their names match, hypothetical mapping is updated to point from 4422 * FWD to STRUCT/UNION. If graphs will be determined as equivalent successfully, 4423 * this mapping will be used to record FWD -> STRUCT/UNION mapping permanently. 4424 * 4425 * Technically, this could lead to incorrect FWD to STRUCT/UNION resolution, 4426 * if there are two exactly named (or anonymous) structs/unions that are 4427 * compatible structurally, one of which has FWD field, while other is concrete 4428 * STRUCT/UNION, but according to C sources they are different structs/unions 4429 * that are referencing different types with the same name. This is extremely 4430 * unlikely to happen, but btf_dedup API allows to disable FWD resolution if 4431 * this logic is causing problems. 4432 * 4433 * Doing FWD resolution means that both candidate and/or canonical graphs can 4434 * consists of portions of the graph that come from multiple compilation units. 4435 * This is due to the fact that types within single compilation unit are always 4436 * deduplicated and FWDs are already resolved, if referenced struct/union 4437 * definition is available. So, if we had unresolved FWD and found corresponding 4438 * STRUCT/UNION, they will be from different compilation units. This 4439 * consequently means that when we "link" FWD to corresponding STRUCT/UNION, 4440 * type graph will likely have at least two different BTF types that describe 4441 * same type (e.g., most probably there will be two different BTF types for the 4442 * same 'int' primitive type) and could even have "overlapping" parts of type 4443 * graph that describe same subset of types. 4444 * 4445 * This in turn means that our assumption that each type in canonical graph 4446 * must correspond to exactly one type in candidate graph might not hold 4447 * anymore and will make it harder to detect contradictions using hypothetical 4448 * map. To handle this problem, we allow to follow FWD -> STRUCT/UNION 4449 * resolution only in canonical graph. FWDs in candidate graphs are never 4450 * resolved. To see why it's OK, let's check all possible situations w.r.t. FWDs 4451 * that can occur: 4452 * - Both types in canonical and candidate graphs are FWDs. If they are 4453 * structurally equivalent, then they can either be both resolved to the 4454 * same STRUCT/UNION or not resolved at all. In both cases they are 4455 * equivalent and there is no need to resolve FWD on candidate side. 4456 * - Both types in canonical and candidate graphs are concrete STRUCT/UNION, 4457 * so nothing to resolve as well, algorithm will check equivalence anyway. 4458 * - Type in canonical graph is FWD, while type in candidate is concrete 4459 * STRUCT/UNION. In this case candidate graph comes from single compilation 4460 * unit, so there is exactly one BTF type for each unique C type. After 4461 * resolving FWD into STRUCT/UNION, there might be more than one BTF type 4462 * in canonical graph mapping to single BTF type in candidate graph, but 4463 * because hypothetical mapping maps from canonical to candidate types, it's 4464 * alright, and we still maintain the property of having single `canon_id` 4465 * mapping to single `cand_id` (there could be two different `canon_id` 4466 * mapped to the same `cand_id`, but it's not contradictory). 4467 * - Type in canonical graph is concrete STRUCT/UNION, while type in candidate 4468 * graph is FWD. In this case we are just going to check compatibility of 4469 * STRUCT/UNION and corresponding FWD, and if they are compatible, we'll 4470 * assume that whatever STRUCT/UNION FWD resolves to must be equivalent to 4471 * a concrete STRUCT/UNION from canonical graph. If the rest of type graphs 4472 * turn out equivalent, we'll re-resolve FWD to concrete STRUCT/UNION from 4473 * canonical graph. 4474 */ 4475 static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, 4476 __u32 canon_id) 4477 { 4478 struct btf_type *cand_type; 4479 struct btf_type *canon_type; 4480 __u32 hypot_type_id; 4481 __u16 cand_kind; 4482 __u16 canon_kind; 4483 int i, eq; 4484 4485 /* if both resolve to the same canonical, they must be equivalent */ 4486 if (resolve_type_id(d, cand_id) == resolve_type_id(d, canon_id)) 4487 return 1; 4488 4489 canon_id = resolve_fwd_id(d, canon_id); 4490 4491 hypot_type_id = d->hypot_map[canon_id]; 4492 if (hypot_type_id <= BTF_MAX_NR_TYPES) { 4493 if (hypot_type_id == cand_id) 4494 return 1; 4495 /* In some cases compiler will generate different DWARF types 4496 * for *identical* array type definitions and use them for 4497 * different fields within the *same* struct. This breaks type 4498 * equivalence check, which makes an assumption that candidate 4499 * types sub-graph has a consistent and deduped-by-compiler 4500 * types within a single CU. So work around that by explicitly 4501 * allowing identical array types here. 4502 */ 4503 if (btf_dedup_identical_arrays(d, hypot_type_id, cand_id)) 4504 return 1; 4505 /* It turns out that similar situation can happen with 4506 * struct/union sometimes, sigh... Handle the case where 4507 * structs/unions are exactly the same, down to the referenced 4508 * type IDs. Anything more complicated (e.g., if referenced 4509 * types are different, but equivalent) is *way more* 4510 * complicated and requires a many-to-many equivalence mapping. 4511 */ 4512 if (btf_dedup_identical_structs(d, hypot_type_id, cand_id)) 4513 return 1; 4514 return 0; 4515 } 4516 4517 if (btf_dedup_hypot_map_add(d, canon_id, cand_id)) 4518 return -ENOMEM; 4519 4520 cand_type = btf_type_by_id(d->btf, cand_id); 4521 canon_type = btf_type_by_id(d->btf, canon_id); 4522 cand_kind = btf_kind(cand_type); 4523 canon_kind = btf_kind(canon_type); 4524 4525 if (cand_type->name_off != canon_type->name_off) 4526 return 0; 4527 4528 /* FWD <--> STRUCT/UNION equivalence check, if enabled */ 4529 if ((cand_kind == BTF_KIND_FWD || canon_kind == BTF_KIND_FWD) 4530 && cand_kind != canon_kind) { 4531 __u16 real_kind; 4532 __u16 fwd_kind; 4533 4534 if (cand_kind == BTF_KIND_FWD) { 4535 real_kind = canon_kind; 4536 fwd_kind = btf_fwd_kind(cand_type); 4537 } else { 4538 real_kind = cand_kind; 4539 fwd_kind = btf_fwd_kind(canon_type); 4540 /* we'd need to resolve base FWD to STRUCT/UNION */ 4541 if (fwd_kind == real_kind && canon_id < d->btf->start_id) 4542 d->hypot_adjust_canon = true; 4543 } 4544 return fwd_kind == real_kind; 4545 } 4546 4547 if (cand_kind != canon_kind) 4548 return 0; 4549 4550 switch (cand_kind) { 4551 case BTF_KIND_INT: 4552 return btf_equal_int_tag(cand_type, canon_type); 4553 4554 case BTF_KIND_ENUM: 4555 case BTF_KIND_ENUM64: 4556 return btf_compat_enum(cand_type, canon_type); 4557 4558 case BTF_KIND_FWD: 4559 case BTF_KIND_FLOAT: 4560 return btf_equal_common(cand_type, canon_type); 4561 4562 case BTF_KIND_CONST: 4563 case BTF_KIND_VOLATILE: 4564 case BTF_KIND_RESTRICT: 4565 case BTF_KIND_PTR: 4566 case BTF_KIND_TYPEDEF: 4567 case BTF_KIND_FUNC: 4568 case BTF_KIND_TYPE_TAG: 4569 if (cand_type->info != canon_type->info) 4570 return 0; 4571 return btf_dedup_is_equiv(d, cand_type->type, canon_type->type); 4572 4573 case BTF_KIND_ARRAY: { 4574 const struct btf_array *cand_arr, *canon_arr; 4575 4576 if (!btf_compat_array(cand_type, canon_type)) 4577 return 0; 4578 cand_arr = btf_array(cand_type); 4579 canon_arr = btf_array(canon_type); 4580 eq = btf_dedup_is_equiv(d, cand_arr->index_type, canon_arr->index_type); 4581 if (eq <= 0) 4582 return eq; 4583 return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type); 4584 } 4585 4586 case BTF_KIND_STRUCT: 4587 case BTF_KIND_UNION: { 4588 const struct btf_member *cand_m, *canon_m; 4589 __u16 vlen; 4590 4591 if (!btf_shallow_equal_struct(cand_type, canon_type)) 4592 return 0; 4593 vlen = btf_vlen(cand_type); 4594 cand_m = btf_members(cand_type); 4595 canon_m = btf_members(canon_type); 4596 for (i = 0; i < vlen; i++) { 4597 eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type); 4598 if (eq <= 0) 4599 return eq; 4600 cand_m++; 4601 canon_m++; 4602 } 4603 4604 return 1; 4605 } 4606 4607 case BTF_KIND_FUNC_PROTO: { 4608 const struct btf_param *cand_p, *canon_p; 4609 __u16 vlen; 4610 4611 if (!btf_compat_fnproto(cand_type, canon_type)) 4612 return 0; 4613 eq = btf_dedup_is_equiv(d, cand_type->type, canon_type->type); 4614 if (eq <= 0) 4615 return eq; 4616 vlen = btf_vlen(cand_type); 4617 cand_p = btf_params(cand_type); 4618 canon_p = btf_params(canon_type); 4619 for (i = 0; i < vlen; i++) { 4620 eq = btf_dedup_is_equiv(d, cand_p->type, canon_p->type); 4621 if (eq <= 0) 4622 return eq; 4623 cand_p++; 4624 canon_p++; 4625 } 4626 return 1; 4627 } 4628 4629 default: 4630 return -EINVAL; 4631 } 4632 return 0; 4633 } 4634 4635 /* 4636 * Use hypothetical mapping, produced by successful type graph equivalence 4637 * check, to augment existing struct/union canonical mapping, where possible. 4638 * 4639 * If BTF_KIND_FWD resolution is allowed, this mapping is also used to record 4640 * FWD -> STRUCT/UNION correspondence as well. FWD resolution is bidirectional: 4641 * it doesn't matter if FWD type was part of canonical graph or candidate one, 4642 * we are recording the mapping anyway. As opposed to carefulness required 4643 * for struct/union correspondence mapping (described below), for FWD resolution 4644 * it's not important, as by the time that FWD type (reference type) will be 4645 * deduplicated all structs/unions will be deduped already anyway. 4646 * 4647 * Recording STRUCT/UNION mapping is purely a performance optimization and is 4648 * not required for correctness. It needs to be done carefully to ensure that 4649 * struct/union from candidate's type graph is not mapped into corresponding 4650 * struct/union from canonical type graph that itself hasn't been resolved into 4651 * canonical representative. The only guarantee we have is that canonical 4652 * struct/union was determined as canonical and that won't change. But any 4653 * types referenced through that struct/union fields could have been not yet 4654 * resolved, so in case like that it's too early to establish any kind of 4655 * correspondence between structs/unions. 4656 * 4657 * No canonical correspondence is derived for primitive types (they are already 4658 * deduplicated completely already anyway) or reference types (they rely on 4659 * stability of struct/union canonical relationship for equivalence checks). 4660 */ 4661 static void btf_dedup_merge_hypot_map(struct btf_dedup *d) 4662 { 4663 __u32 canon_type_id, targ_type_id; 4664 __u16 t_kind, c_kind; 4665 __u32 t_id, c_id; 4666 int i; 4667 4668 for (i = 0; i < d->hypot_cnt; i++) { 4669 canon_type_id = d->hypot_list[i]; 4670 targ_type_id = d->hypot_map[canon_type_id]; 4671 t_id = resolve_type_id(d, targ_type_id); 4672 c_id = resolve_type_id(d, canon_type_id); 4673 t_kind = btf_kind(btf__type_by_id(d->btf, t_id)); 4674 c_kind = btf_kind(btf__type_by_id(d->btf, c_id)); 4675 /* 4676 * Resolve FWD into STRUCT/UNION. 4677 * It's ok to resolve FWD into STRUCT/UNION that's not yet 4678 * mapped to canonical representative (as opposed to 4679 * STRUCT/UNION <--> STRUCT/UNION mapping logic below), because 4680 * eventually that struct is going to be mapped and all resolved 4681 * FWDs will automatically resolve to correct canonical 4682 * representative. This will happen before ref type deduping, 4683 * which critically depends on stability of these mapping. This 4684 * stability is not a requirement for STRUCT/UNION equivalence 4685 * checks, though. 4686 */ 4687 4688 /* if it's the split BTF case, we still need to point base FWD 4689 * to STRUCT/UNION in a split BTF, because FWDs from split BTF 4690 * will be resolved against base FWD. If we don't point base 4691 * canonical FWD to the resolved STRUCT/UNION, then all the 4692 * FWDs in split BTF won't be correctly resolved to a proper 4693 * STRUCT/UNION. 4694 */ 4695 if (t_kind != BTF_KIND_FWD && c_kind == BTF_KIND_FWD) 4696 d->map[c_id] = t_id; 4697 4698 /* if graph equivalence determined that we'd need to adjust 4699 * base canonical types, then we need to only point base FWDs 4700 * to STRUCTs/UNIONs and do no more modifications. For all 4701 * other purposes the type graphs were not equivalent. 4702 */ 4703 if (d->hypot_adjust_canon) 4704 continue; 4705 4706 if (t_kind == BTF_KIND_FWD && c_kind != BTF_KIND_FWD) 4707 d->map[t_id] = c_id; 4708 4709 if ((t_kind == BTF_KIND_STRUCT || t_kind == BTF_KIND_UNION) && 4710 c_kind != BTF_KIND_FWD && 4711 is_type_mapped(d, c_id) && 4712 !is_type_mapped(d, t_id)) { 4713 /* 4714 * as a perf optimization, we can map struct/union 4715 * that's part of type graph we just verified for 4716 * equivalence. We can do that for struct/union that has 4717 * canonical representative only, though. 4718 */ 4719 d->map[t_id] = c_id; 4720 } 4721 } 4722 } 4723 4724 /* 4725 * Deduplicate struct/union types. 4726 * 4727 * For each struct/union type its type signature hash is calculated, taking 4728 * into account type's name, size, number, order and names of fields, but 4729 * ignoring type ID's referenced from fields, because they might not be deduped 4730 * completely until after reference types deduplication phase. This type hash 4731 * is used to iterate over all potential canonical types, sharing same hash. 4732 * For each canonical candidate we check whether type graphs that they form 4733 * (through referenced types in fields and so on) are equivalent using algorithm 4734 * implemented in `btf_dedup_is_equiv`. If such equivalence is found and 4735 * BTF_KIND_FWD resolution is allowed, then hypothetical mapping 4736 * (btf_dedup->hypot_map) produced by aforementioned type graph equivalence 4737 * algorithm is used to record FWD -> STRUCT/UNION mapping. It's also used to 4738 * potentially map other structs/unions to their canonical representatives, 4739 * if such relationship hasn't yet been established. This speeds up algorithm 4740 * by eliminating some of the duplicate work. 4741 * 4742 * If no matching canonical representative was found, struct/union is marked 4743 * as canonical for itself and is added into btf_dedup->dedup_table hash map 4744 * for further look ups. 4745 */ 4746 static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id) 4747 { 4748 struct btf_type *cand_type, *t; 4749 struct hashmap_entry *hash_entry; 4750 /* if we don't find equivalent type, then we are canonical */ 4751 __u32 new_id = type_id; 4752 __u16 kind; 4753 long h; 4754 4755 /* already deduped or is in process of deduping (loop detected) */ 4756 if (d->map[type_id] <= BTF_MAX_NR_TYPES) 4757 return 0; 4758 4759 t = btf_type_by_id(d->btf, type_id); 4760 kind = btf_kind(t); 4761 4762 if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION) 4763 return 0; 4764 4765 h = btf_hash_struct(t); 4766 for_each_dedup_cand(d, hash_entry, h) { 4767 __u32 cand_id = hash_entry->value; 4768 int eq; 4769 4770 /* 4771 * Even though btf_dedup_is_equiv() checks for 4772 * btf_shallow_equal_struct() internally when checking two 4773 * structs (unions) for equivalence, we need to guard here 4774 * from picking matching FWD type as a dedup candidate. 4775 * This can happen due to hash collision. In such case just 4776 * relying on btf_dedup_is_equiv() would lead to potentially 4777 * creating a loop (FWD -> STRUCT and STRUCT -> FWD), because 4778 * FWD and compatible STRUCT/UNION are considered equivalent. 4779 */ 4780 cand_type = btf_type_by_id(d->btf, cand_id); 4781 if (!btf_shallow_equal_struct(t, cand_type)) 4782 continue; 4783 4784 btf_dedup_clear_hypot_map(d); 4785 eq = btf_dedup_is_equiv(d, type_id, cand_id); 4786 if (eq < 0) 4787 return eq; 4788 if (!eq) 4789 continue; 4790 btf_dedup_merge_hypot_map(d); 4791 if (d->hypot_adjust_canon) /* not really equivalent */ 4792 continue; 4793 new_id = cand_id; 4794 break; 4795 } 4796 4797 d->map[type_id] = new_id; 4798 if (type_id == new_id && btf_dedup_table_add(d, h, type_id)) 4799 return -ENOMEM; 4800 4801 return 0; 4802 } 4803 4804 static int btf_dedup_struct_types(struct btf_dedup *d) 4805 { 4806 int i, err; 4807 4808 for (i = 0; i < d->btf->nr_types; i++) { 4809 err = btf_dedup_struct_type(d, d->btf->start_id + i); 4810 if (err) 4811 return err; 4812 } 4813 return 0; 4814 } 4815 4816 /* 4817 * Deduplicate reference type. 4818 * 4819 * Once all primitive and struct/union types got deduplicated, we can easily 4820 * deduplicate all other (reference) BTF types. This is done in two steps: 4821 * 4822 * 1. Resolve all referenced type IDs into their canonical type IDs. This 4823 * resolution can be done either immediately for primitive or struct/union types 4824 * (because they were deduped in previous two phases) or recursively for 4825 * reference types. Recursion will always terminate at either primitive or 4826 * struct/union type, at which point we can "unwind" chain of reference types 4827 * one by one. There is no danger of encountering cycles because in C type 4828 * system the only way to form type cycle is through struct/union, so any chain 4829 * of reference types, even those taking part in a type cycle, will inevitably 4830 * reach struct/union at some point. 4831 * 4832 * 2. Once all referenced type IDs are resolved into canonical ones, BTF type 4833 * becomes "stable", in the sense that no further deduplication will cause 4834 * any changes to it. With that, it's now possible to calculate type's signature 4835 * hash (this time taking into account referenced type IDs) and loop over all 4836 * potential canonical representatives. If no match was found, current type 4837 * will become canonical representative of itself and will be added into 4838 * btf_dedup->dedup_table as another possible canonical representative. 4839 */ 4840 static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) 4841 { 4842 struct hashmap_entry *hash_entry; 4843 __u32 new_id = type_id, cand_id; 4844 struct btf_type *t, *cand; 4845 /* if we don't find equivalent type, then we are representative type */ 4846 int ref_type_id; 4847 long h; 4848 4849 if (d->map[type_id] == BTF_IN_PROGRESS_ID) 4850 return -ELOOP; 4851 if (d->map[type_id] <= BTF_MAX_NR_TYPES) 4852 return resolve_type_id(d, type_id); 4853 4854 t = btf_type_by_id(d->btf, type_id); 4855 d->map[type_id] = BTF_IN_PROGRESS_ID; 4856 4857 switch (btf_kind(t)) { 4858 case BTF_KIND_CONST: 4859 case BTF_KIND_VOLATILE: 4860 case BTF_KIND_RESTRICT: 4861 case BTF_KIND_PTR: 4862 case BTF_KIND_TYPEDEF: 4863 case BTF_KIND_FUNC: 4864 case BTF_KIND_TYPE_TAG: 4865 ref_type_id = btf_dedup_ref_type(d, t->type); 4866 if (ref_type_id < 0) 4867 return ref_type_id; 4868 t->type = ref_type_id; 4869 4870 h = btf_hash_common(t); 4871 for_each_dedup_cand(d, hash_entry, h) { 4872 cand_id = hash_entry->value; 4873 cand = btf_type_by_id(d->btf, cand_id); 4874 if (btf_equal_common(t, cand)) { 4875 new_id = cand_id; 4876 break; 4877 } 4878 } 4879 break; 4880 4881 case BTF_KIND_DECL_TAG: 4882 ref_type_id = btf_dedup_ref_type(d, t->type); 4883 if (ref_type_id < 0) 4884 return ref_type_id; 4885 t->type = ref_type_id; 4886 4887 h = btf_hash_int_decl_tag(t); 4888 for_each_dedup_cand(d, hash_entry, h) { 4889 cand_id = hash_entry->value; 4890 cand = btf_type_by_id(d->btf, cand_id); 4891 if (btf_equal_int_tag(t, cand)) { 4892 new_id = cand_id; 4893 break; 4894 } 4895 } 4896 break; 4897 4898 case BTF_KIND_ARRAY: { 4899 struct btf_array *info = btf_array(t); 4900 4901 ref_type_id = btf_dedup_ref_type(d, info->type); 4902 if (ref_type_id < 0) 4903 return ref_type_id; 4904 info->type = ref_type_id; 4905 4906 ref_type_id = btf_dedup_ref_type(d, info->index_type); 4907 if (ref_type_id < 0) 4908 return ref_type_id; 4909 info->index_type = ref_type_id; 4910 4911 h = btf_hash_array(t); 4912 for_each_dedup_cand(d, hash_entry, h) { 4913 cand_id = hash_entry->value; 4914 cand = btf_type_by_id(d->btf, cand_id); 4915 if (btf_equal_array(t, cand)) { 4916 new_id = cand_id; 4917 break; 4918 } 4919 } 4920 break; 4921 } 4922 4923 case BTF_KIND_FUNC_PROTO: { 4924 struct btf_param *param; 4925 __u16 vlen; 4926 int i; 4927 4928 ref_type_id = btf_dedup_ref_type(d, t->type); 4929 if (ref_type_id < 0) 4930 return ref_type_id; 4931 t->type = ref_type_id; 4932 4933 vlen = btf_vlen(t); 4934 param = btf_params(t); 4935 for (i = 0; i < vlen; i++) { 4936 ref_type_id = btf_dedup_ref_type(d, param->type); 4937 if (ref_type_id < 0) 4938 return ref_type_id; 4939 param->type = ref_type_id; 4940 param++; 4941 } 4942 4943 h = btf_hash_fnproto(t); 4944 for_each_dedup_cand(d, hash_entry, h) { 4945 cand_id = hash_entry->value; 4946 cand = btf_type_by_id(d->btf, cand_id); 4947 if (btf_equal_fnproto(t, cand)) { 4948 new_id = cand_id; 4949 break; 4950 } 4951 } 4952 break; 4953 } 4954 4955 default: 4956 return -EINVAL; 4957 } 4958 4959 d->map[type_id] = new_id; 4960 if (type_id == new_id && btf_dedup_table_add(d, h, type_id)) 4961 return -ENOMEM; 4962 4963 return new_id; 4964 } 4965 4966 static int btf_dedup_ref_types(struct btf_dedup *d) 4967 { 4968 int i, err; 4969 4970 for (i = 0; i < d->btf->nr_types; i++) { 4971 err = btf_dedup_ref_type(d, d->btf->start_id + i); 4972 if (err < 0) 4973 return err; 4974 } 4975 /* we won't need d->dedup_table anymore */ 4976 hashmap__free(d->dedup_table); 4977 d->dedup_table = NULL; 4978 return 0; 4979 } 4980 4981 /* 4982 * Collect a map from type names to type ids for all canonical structs 4983 * and unions. If the same name is shared by several canonical types 4984 * use a special value 0 to indicate this fact. 4985 */ 4986 static int btf_dedup_fill_unique_names_map(struct btf_dedup *d, struct hashmap *names_map) 4987 { 4988 __u32 nr_types = btf__type_cnt(d->btf); 4989 struct btf_type *t; 4990 __u32 type_id; 4991 __u16 kind; 4992 int err; 4993 4994 /* 4995 * Iterate over base and split module ids in order to get all 4996 * available structs in the map. 4997 */ 4998 for (type_id = 1; type_id < nr_types; ++type_id) { 4999 t = btf_type_by_id(d->btf, type_id); 5000 kind = btf_kind(t); 5001 5002 if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION) 5003 continue; 5004 5005 /* Skip non-canonical types */ 5006 if (type_id != d->map[type_id]) 5007 continue; 5008 5009 err = hashmap__add(names_map, t->name_off, type_id); 5010 if (err == -EEXIST) 5011 err = hashmap__set(names_map, t->name_off, 0, NULL, NULL); 5012 5013 if (err) 5014 return err; 5015 } 5016 5017 return 0; 5018 } 5019 5020 static int btf_dedup_resolve_fwd(struct btf_dedup *d, struct hashmap *names_map, __u32 type_id) 5021 { 5022 struct btf_type *t = btf_type_by_id(d->btf, type_id); 5023 enum btf_fwd_kind fwd_kind = btf_kflag(t); 5024 __u16 cand_kind, kind = btf_kind(t); 5025 struct btf_type *cand_t; 5026 uintptr_t cand_id; 5027 5028 if (kind != BTF_KIND_FWD) 5029 return 0; 5030 5031 /* Skip if this FWD already has a mapping */ 5032 if (type_id != d->map[type_id]) 5033 return 0; 5034 5035 if (!hashmap__find(names_map, t->name_off, &cand_id)) 5036 return 0; 5037 5038 /* Zero is a special value indicating that name is not unique */ 5039 if (!cand_id) 5040 return 0; 5041 5042 cand_t = btf_type_by_id(d->btf, cand_id); 5043 cand_kind = btf_kind(cand_t); 5044 if ((cand_kind == BTF_KIND_STRUCT && fwd_kind != BTF_FWD_STRUCT) || 5045 (cand_kind == BTF_KIND_UNION && fwd_kind != BTF_FWD_UNION)) 5046 return 0; 5047 5048 d->map[type_id] = cand_id; 5049 5050 return 0; 5051 } 5052 5053 /* 5054 * Resolve unambiguous forward declarations. 5055 * 5056 * The lion's share of all FWD declarations is resolved during 5057 * `btf_dedup_struct_types` phase when different type graphs are 5058 * compared against each other. However, if in some compilation unit a 5059 * FWD declaration is not a part of a type graph compared against 5060 * another type graph that declaration's canonical type would not be 5061 * changed. Example: 5062 * 5063 * CU #1: 5064 * 5065 * struct foo; 5066 * struct foo *some_global; 5067 * 5068 * CU #2: 5069 * 5070 * struct foo { int u; }; 5071 * struct foo *another_global; 5072 * 5073 * After `btf_dedup_struct_types` the BTF looks as follows: 5074 * 5075 * [1] STRUCT 'foo' size=4 vlen=1 ... 5076 * [2] INT 'int' size=4 ... 5077 * [3] PTR '(anon)' type_id=1 5078 * [4] FWD 'foo' fwd_kind=struct 5079 * [5] PTR '(anon)' type_id=4 5080 * 5081 * This pass assumes that such FWD declarations should be mapped to 5082 * structs or unions with identical name in case if the name is not 5083 * ambiguous. 5084 */ 5085 static int btf_dedup_resolve_fwds(struct btf_dedup *d) 5086 { 5087 int i, err; 5088 struct hashmap *names_map; 5089 5090 names_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL); 5091 if (IS_ERR(names_map)) 5092 return PTR_ERR(names_map); 5093 5094 err = btf_dedup_fill_unique_names_map(d, names_map); 5095 if (err < 0) 5096 goto exit; 5097 5098 for (i = 0; i < d->btf->nr_types; i++) { 5099 err = btf_dedup_resolve_fwd(d, names_map, d->btf->start_id + i); 5100 if (err < 0) 5101 break; 5102 } 5103 5104 exit: 5105 hashmap__free(names_map); 5106 return err; 5107 } 5108 5109 /* 5110 * Compact types. 5111 * 5112 * After we established for each type its corresponding canonical representative 5113 * type, we now can eliminate types that are not canonical and leave only 5114 * canonical ones layed out sequentially in memory by copying them over 5115 * duplicates. During compaction btf_dedup->hypot_map array is reused to store 5116 * a map from original type ID to a new compacted type ID, which will be used 5117 * during next phase to "fix up" type IDs, referenced from struct/union and 5118 * reference types. 5119 */ 5120 static int btf_dedup_compact_types(struct btf_dedup *d) 5121 { 5122 __u32 *new_offs; 5123 __u32 next_type_id = d->btf->start_id; 5124 const struct btf_type *t; 5125 void *p; 5126 int i, id, len; 5127 5128 /* we are going to reuse hypot_map to store compaction remapping */ 5129 d->hypot_map[0] = 0; 5130 /* base BTF types are not renumbered */ 5131 for (id = 1; id < d->btf->start_id; id++) 5132 d->hypot_map[id] = id; 5133 for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) 5134 d->hypot_map[id] = BTF_UNPROCESSED_ID; 5135 5136 p = d->btf->types_data; 5137 5138 for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) { 5139 if (d->map[id] != id) 5140 continue; 5141 5142 t = btf__type_by_id(d->btf, id); 5143 len = btf_type_size(t); 5144 if (len < 0) 5145 return len; 5146 5147 memmove(p, t, len); 5148 d->hypot_map[id] = next_type_id; 5149 d->btf->type_offs[next_type_id - d->btf->start_id] = p - d->btf->types_data; 5150 p += len; 5151 next_type_id++; 5152 } 5153 5154 /* shrink struct btf's internal types index and update btf_header */ 5155 d->btf->nr_types = next_type_id - d->btf->start_id; 5156 d->btf->type_offs_cap = d->btf->nr_types; 5157 d->btf->hdr->type_len = p - d->btf->types_data; 5158 new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap, 5159 sizeof(*new_offs)); 5160 if (d->btf->type_offs_cap && !new_offs) 5161 return -ENOMEM; 5162 d->btf->type_offs = new_offs; 5163 d->btf->hdr->str_off = d->btf->hdr->type_len; 5164 d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len; 5165 return 0; 5166 } 5167 5168 /* 5169 * Figure out final (deduplicated and compacted) type ID for provided original 5170 * `type_id` by first resolving it into corresponding canonical type ID and 5171 * then mapping it to a deduplicated type ID, stored in btf_dedup->hypot_map, 5172 * which is populated during compaction phase. 5173 */ 5174 static int btf_dedup_remap_type_id(__u32 *type_id, void *ctx) 5175 { 5176 struct btf_dedup *d = ctx; 5177 __u32 resolved_type_id, new_type_id; 5178 5179 resolved_type_id = resolve_type_id(d, *type_id); 5180 new_type_id = d->hypot_map[resolved_type_id]; 5181 if (new_type_id > BTF_MAX_NR_TYPES) 5182 return -EINVAL; 5183 5184 *type_id = new_type_id; 5185 return 0; 5186 } 5187 5188 /* 5189 * Remap referenced type IDs into deduped type IDs. 5190 * 5191 * After BTF types are deduplicated and compacted, their final type IDs may 5192 * differ from original ones. The map from original to a corresponding 5193 * deduped type ID is stored in btf_dedup->hypot_map and is populated during 5194 * compaction phase. During remapping phase we are rewriting all type IDs 5195 * referenced from any BTF type (e.g., struct fields, func proto args, etc) to 5196 * their final deduped type IDs. 5197 */ 5198 static int btf_dedup_remap_types(struct btf_dedup *d) 5199 { 5200 int i, r; 5201 5202 for (i = 0; i < d->btf->nr_types; i++) { 5203 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i); 5204 struct btf_field_iter it; 5205 __u32 *type_id; 5206 5207 r = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS); 5208 if (r) 5209 return r; 5210 5211 while ((type_id = btf_field_iter_next(&it))) { 5212 __u32 resolved_id, new_id; 5213 5214 resolved_id = resolve_type_id(d, *type_id); 5215 new_id = d->hypot_map[resolved_id]; 5216 if (new_id > BTF_MAX_NR_TYPES) 5217 return -EINVAL; 5218 5219 *type_id = new_id; 5220 } 5221 } 5222 5223 if (!d->btf_ext) 5224 return 0; 5225 5226 r = btf_ext_visit_type_ids(d->btf_ext, btf_dedup_remap_type_id, d); 5227 if (r) 5228 return r; 5229 5230 return 0; 5231 } 5232 5233 /* 5234 * Probe few well-known locations for vmlinux kernel image and try to load BTF 5235 * data out of it to use for target BTF. 5236 */ 5237 struct btf *btf__load_vmlinux_btf(void) 5238 { 5239 const char *sysfs_btf_path = "/sys/kernel/btf/vmlinux"; 5240 /* fall back locations, trying to find vmlinux on disk */ 5241 const char *locations[] = { 5242 "/boot/vmlinux-%1$s", 5243 "/lib/modules/%1$s/vmlinux-%1$s", 5244 "/lib/modules/%1$s/build/vmlinux", 5245 "/usr/lib/modules/%1$s/kernel/vmlinux", 5246 "/usr/lib/debug/boot/vmlinux-%1$s", 5247 "/usr/lib/debug/boot/vmlinux-%1$s.debug", 5248 "/usr/lib/debug/lib/modules/%1$s/vmlinux", 5249 }; 5250 char path[PATH_MAX + 1]; 5251 struct utsname buf; 5252 struct btf *btf; 5253 int i, err; 5254 5255 /* is canonical sysfs location accessible? */ 5256 if (faccessat(AT_FDCWD, sysfs_btf_path, F_OK, AT_EACCESS) < 0) { 5257 pr_warn("kernel BTF is missing at '%s', was CONFIG_DEBUG_INFO_BTF enabled?\n", 5258 sysfs_btf_path); 5259 } else { 5260 btf = btf__parse(sysfs_btf_path, NULL); 5261 if (!btf) { 5262 err = -errno; 5263 pr_warn("failed to read kernel BTF from '%s': %s\n", 5264 sysfs_btf_path, errstr(err)); 5265 return libbpf_err_ptr(err); 5266 } 5267 pr_debug("loaded kernel BTF from '%s'\n", sysfs_btf_path); 5268 return btf; 5269 } 5270 5271 /* try fallback locations */ 5272 uname(&buf); 5273 for (i = 0; i < ARRAY_SIZE(locations); i++) { 5274 snprintf(path, PATH_MAX, locations[i], buf.release); 5275 5276 if (faccessat(AT_FDCWD, path, R_OK, AT_EACCESS)) 5277 continue; 5278 5279 btf = btf__parse(path, NULL); 5280 err = libbpf_get_error(btf); 5281 pr_debug("loading kernel BTF '%s': %s\n", path, errstr(err)); 5282 if (err) 5283 continue; 5284 5285 return btf; 5286 } 5287 5288 pr_warn("failed to find valid kernel BTF\n"); 5289 return libbpf_err_ptr(-ESRCH); 5290 } 5291 5292 struct btf *libbpf_find_kernel_btf(void) __attribute__((alias("btf__load_vmlinux_btf"))); 5293 5294 struct btf *btf__load_module_btf(const char *module_name, struct btf *vmlinux_btf) 5295 { 5296 char path[80]; 5297 5298 snprintf(path, sizeof(path), "/sys/kernel/btf/%s", module_name); 5299 return btf__parse_split(path, vmlinux_btf); 5300 } 5301 5302 int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx) 5303 { 5304 const struct btf_ext_info *seg; 5305 struct btf_ext_info_sec *sec; 5306 int i, err; 5307 5308 seg = &btf_ext->func_info; 5309 for_each_btf_ext_sec(seg, sec) { 5310 struct bpf_func_info_min *rec; 5311 5312 for_each_btf_ext_rec(seg, sec, i, rec) { 5313 err = visit(&rec->type_id, ctx); 5314 if (err < 0) 5315 return err; 5316 } 5317 } 5318 5319 seg = &btf_ext->core_relo_info; 5320 for_each_btf_ext_sec(seg, sec) { 5321 struct bpf_core_relo *rec; 5322 5323 for_each_btf_ext_rec(seg, sec, i, rec) { 5324 err = visit(&rec->type_id, ctx); 5325 if (err < 0) 5326 return err; 5327 } 5328 } 5329 5330 return 0; 5331 } 5332 5333 int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx) 5334 { 5335 const struct btf_ext_info *seg; 5336 struct btf_ext_info_sec *sec; 5337 int i, err; 5338 5339 seg = &btf_ext->func_info; 5340 for_each_btf_ext_sec(seg, sec) { 5341 err = visit(&sec->sec_name_off, ctx); 5342 if (err) 5343 return err; 5344 } 5345 5346 seg = &btf_ext->line_info; 5347 for_each_btf_ext_sec(seg, sec) { 5348 struct bpf_line_info_min *rec; 5349 5350 err = visit(&sec->sec_name_off, ctx); 5351 if (err) 5352 return err; 5353 5354 for_each_btf_ext_rec(seg, sec, i, rec) { 5355 err = visit(&rec->file_name_off, ctx); 5356 if (err) 5357 return err; 5358 err = visit(&rec->line_off, ctx); 5359 if (err) 5360 return err; 5361 } 5362 } 5363 5364 seg = &btf_ext->core_relo_info; 5365 for_each_btf_ext_sec(seg, sec) { 5366 struct bpf_core_relo *rec; 5367 5368 err = visit(&sec->sec_name_off, ctx); 5369 if (err) 5370 return err; 5371 5372 for_each_btf_ext_rec(seg, sec, i, rec) { 5373 err = visit(&rec->access_str_off, ctx); 5374 if (err) 5375 return err; 5376 } 5377 } 5378 5379 return 0; 5380 } 5381 5382 struct btf_distill { 5383 struct btf_pipe pipe; 5384 int *id_map; 5385 unsigned int split_start_id; 5386 unsigned int split_start_str; 5387 int diff_id; 5388 }; 5389 5390 static int btf_add_distilled_type_ids(struct btf_distill *dist, __u32 i) 5391 { 5392 struct btf_type *split_t = btf_type_by_id(dist->pipe.src, i); 5393 struct btf_field_iter it; 5394 __u32 *id; 5395 int err; 5396 5397 err = btf_field_iter_init(&it, split_t, BTF_FIELD_ITER_IDS); 5398 if (err) 5399 return err; 5400 while ((id = btf_field_iter_next(&it))) { 5401 struct btf_type *base_t; 5402 5403 if (!*id) 5404 continue; 5405 /* split BTF id, not needed */ 5406 if (*id >= dist->split_start_id) 5407 continue; 5408 /* already added ? */ 5409 if (dist->id_map[*id] > 0) 5410 continue; 5411 5412 /* only a subset of base BTF types should be referenced from 5413 * split BTF; ensure nothing unexpected is referenced. 5414 */ 5415 base_t = btf_type_by_id(dist->pipe.src, *id); 5416 switch (btf_kind(base_t)) { 5417 case BTF_KIND_INT: 5418 case BTF_KIND_FLOAT: 5419 case BTF_KIND_FWD: 5420 case BTF_KIND_ARRAY: 5421 case BTF_KIND_STRUCT: 5422 case BTF_KIND_UNION: 5423 case BTF_KIND_TYPEDEF: 5424 case BTF_KIND_ENUM: 5425 case BTF_KIND_ENUM64: 5426 case BTF_KIND_PTR: 5427 case BTF_KIND_CONST: 5428 case BTF_KIND_RESTRICT: 5429 case BTF_KIND_VOLATILE: 5430 case BTF_KIND_FUNC_PROTO: 5431 case BTF_KIND_TYPE_TAG: 5432 dist->id_map[*id] = *id; 5433 break; 5434 default: 5435 pr_warn("unexpected reference to base type[%u] of kind [%u] when creating distilled base BTF.\n", 5436 *id, btf_kind(base_t)); 5437 return -EINVAL; 5438 } 5439 /* If a base type is used, ensure types it refers to are 5440 * marked as used also; so for example if we find a PTR to INT 5441 * we need both the PTR and INT. 5442 * 5443 * The only exception is named struct/unions, since distilled 5444 * base BTF composite types have no members. 5445 */ 5446 if (btf_is_composite(base_t) && base_t->name_off) 5447 continue; 5448 err = btf_add_distilled_type_ids(dist, *id); 5449 if (err) 5450 return err; 5451 } 5452 return 0; 5453 } 5454 5455 static int btf_add_distilled_types(struct btf_distill *dist) 5456 { 5457 bool adding_to_base = dist->pipe.dst->start_id == 1; 5458 int id = btf__type_cnt(dist->pipe.dst); 5459 struct btf_type *t; 5460 int i, err = 0; 5461 5462 5463 /* Add types for each of the required references to either distilled 5464 * base or split BTF, depending on type characteristics. 5465 */ 5466 for (i = 1; i < dist->split_start_id; i++) { 5467 const char *name; 5468 int kind; 5469 5470 if (!dist->id_map[i]) 5471 continue; 5472 t = btf_type_by_id(dist->pipe.src, i); 5473 kind = btf_kind(t); 5474 name = btf__name_by_offset(dist->pipe.src, t->name_off); 5475 5476 switch (kind) { 5477 case BTF_KIND_INT: 5478 case BTF_KIND_FLOAT: 5479 case BTF_KIND_FWD: 5480 /* Named int, float, fwd are added to base. */ 5481 if (!adding_to_base) 5482 continue; 5483 err = btf_add_type(&dist->pipe, t); 5484 break; 5485 case BTF_KIND_STRUCT: 5486 case BTF_KIND_UNION: 5487 /* Named struct/union are added to base as 0-vlen 5488 * struct/union of same size. Anonymous struct/unions 5489 * are added to split BTF as-is. 5490 */ 5491 if (adding_to_base) { 5492 if (!t->name_off) 5493 continue; 5494 err = btf_add_composite(dist->pipe.dst, kind, name, t->size); 5495 } else { 5496 if (t->name_off) 5497 continue; 5498 err = btf_add_type(&dist->pipe, t); 5499 } 5500 break; 5501 case BTF_KIND_ENUM: 5502 case BTF_KIND_ENUM64: 5503 /* Named enum[64]s are added to base as a sized 5504 * enum; relocation will match with appropriately-named 5505 * and sized enum or enum64. 5506 * 5507 * Anonymous enums are added to split BTF as-is. 5508 */ 5509 if (adding_to_base) { 5510 if (!t->name_off) 5511 continue; 5512 err = btf__add_enum(dist->pipe.dst, name, t->size); 5513 } else { 5514 if (t->name_off) 5515 continue; 5516 err = btf_add_type(&dist->pipe, t); 5517 } 5518 break; 5519 case BTF_KIND_ARRAY: 5520 case BTF_KIND_TYPEDEF: 5521 case BTF_KIND_PTR: 5522 case BTF_KIND_CONST: 5523 case BTF_KIND_RESTRICT: 5524 case BTF_KIND_VOLATILE: 5525 case BTF_KIND_FUNC_PROTO: 5526 case BTF_KIND_TYPE_TAG: 5527 /* All other types are added to split BTF. */ 5528 if (adding_to_base) 5529 continue; 5530 err = btf_add_type(&dist->pipe, t); 5531 break; 5532 default: 5533 pr_warn("unexpected kind when adding base type '%s'[%u] of kind [%u] to distilled base BTF.\n", 5534 name, i, kind); 5535 return -EINVAL; 5536 5537 } 5538 if (err < 0) 5539 break; 5540 dist->id_map[i] = id++; 5541 } 5542 return err; 5543 } 5544 5545 /* Split BTF ids without a mapping will be shifted downwards since distilled 5546 * base BTF is smaller than the original base BTF. For those that have a 5547 * mapping (either to base or updated split BTF), update the id based on 5548 * that mapping. 5549 */ 5550 static int btf_update_distilled_type_ids(struct btf_distill *dist, __u32 i) 5551 { 5552 struct btf_type *t = btf_type_by_id(dist->pipe.dst, i); 5553 struct btf_field_iter it; 5554 __u32 *id; 5555 int err; 5556 5557 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS); 5558 if (err) 5559 return err; 5560 while ((id = btf_field_iter_next(&it))) { 5561 if (dist->id_map[*id]) 5562 *id = dist->id_map[*id]; 5563 else if (*id >= dist->split_start_id) 5564 *id -= dist->diff_id; 5565 } 5566 return 0; 5567 } 5568 5569 /* Create updated split BTF with distilled base BTF; distilled base BTF 5570 * consists of BTF information required to clarify the types that split 5571 * BTF refers to, omitting unneeded details. Specifically it will contain 5572 * base types and memberless definitions of named structs, unions and enumerated 5573 * types. Associated reference types like pointers, arrays and anonymous 5574 * structs, unions and enumerated types will be added to split BTF. 5575 * Size is recorded for named struct/unions to help guide matching to the 5576 * target base BTF during later relocation. 5577 * 5578 * The only case where structs, unions or enumerated types are fully represented 5579 * is when they are anonymous; in such cases, the anonymous type is added to 5580 * split BTF in full. 5581 * 5582 * We return newly-created split BTF where the split BTF refers to a newly-created 5583 * distilled base BTF. Both must be freed separately by the caller. 5584 */ 5585 int btf__distill_base(const struct btf *src_btf, struct btf **new_base_btf, 5586 struct btf **new_split_btf) 5587 { 5588 struct btf *new_base = NULL, *new_split = NULL; 5589 const struct btf *old_base; 5590 unsigned int n = btf__type_cnt(src_btf); 5591 struct btf_distill dist = {}; 5592 struct btf_type *t; 5593 int i, err = 0; 5594 5595 /* src BTF must be split BTF. */ 5596 old_base = btf__base_btf(src_btf); 5597 if (!new_base_btf || !new_split_btf || !old_base) 5598 return libbpf_err(-EINVAL); 5599 5600 new_base = btf__new_empty(); 5601 if (!new_base) 5602 return libbpf_err(-ENOMEM); 5603 5604 btf__set_endianness(new_base, btf__endianness(src_btf)); 5605 5606 dist.id_map = calloc(n, sizeof(*dist.id_map)); 5607 if (!dist.id_map) { 5608 err = -ENOMEM; 5609 goto done; 5610 } 5611 dist.pipe.src = src_btf; 5612 dist.pipe.dst = new_base; 5613 dist.pipe.str_off_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL); 5614 if (IS_ERR(dist.pipe.str_off_map)) { 5615 err = -ENOMEM; 5616 goto done; 5617 } 5618 dist.split_start_id = btf__type_cnt(old_base); 5619 dist.split_start_str = old_base->hdr->str_len; 5620 5621 /* Pass over src split BTF; generate the list of base BTF type ids it 5622 * references; these will constitute our distilled BTF set to be 5623 * distributed over base and split BTF as appropriate. 5624 */ 5625 for (i = src_btf->start_id; i < n; i++) { 5626 err = btf_add_distilled_type_ids(&dist, i); 5627 if (err < 0) 5628 goto done; 5629 } 5630 /* Next add types for each of the required references to base BTF and split BTF 5631 * in turn. 5632 */ 5633 err = btf_add_distilled_types(&dist); 5634 if (err < 0) 5635 goto done; 5636 5637 /* Create new split BTF with distilled base BTF as its base; the final 5638 * state is split BTF with distilled base BTF that represents enough 5639 * about its base references to allow it to be relocated with the base 5640 * BTF available. 5641 */ 5642 new_split = btf__new_empty_split(new_base); 5643 if (!new_split) { 5644 err = -errno; 5645 goto done; 5646 } 5647 dist.pipe.dst = new_split; 5648 /* First add all split types */ 5649 for (i = src_btf->start_id; i < n; i++) { 5650 t = btf_type_by_id(src_btf, i); 5651 err = btf_add_type(&dist.pipe, t); 5652 if (err < 0) 5653 goto done; 5654 } 5655 /* Now add distilled types to split BTF that are not added to base. */ 5656 err = btf_add_distilled_types(&dist); 5657 if (err < 0) 5658 goto done; 5659 5660 /* All split BTF ids will be shifted downwards since there are less base 5661 * BTF ids in distilled base BTF. 5662 */ 5663 dist.diff_id = dist.split_start_id - btf__type_cnt(new_base); 5664 5665 n = btf__type_cnt(new_split); 5666 /* Now update base/split BTF ids. */ 5667 for (i = 1; i < n; i++) { 5668 err = btf_update_distilled_type_ids(&dist, i); 5669 if (err < 0) 5670 break; 5671 } 5672 done: 5673 free(dist.id_map); 5674 hashmap__free(dist.pipe.str_off_map); 5675 if (err) { 5676 btf__free(new_split); 5677 btf__free(new_base); 5678 return libbpf_err(err); 5679 } 5680 *new_base_btf = new_base; 5681 *new_split_btf = new_split; 5682 5683 return 0; 5684 } 5685 5686 const struct btf_header *btf_header(const struct btf *btf) 5687 { 5688 return btf->hdr; 5689 } 5690 5691 void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) 5692 { 5693 btf->base_btf = (struct btf *)base_btf; 5694 btf->start_id = btf__type_cnt(base_btf); 5695 btf->start_str_off = base_btf->hdr->str_len; 5696 } 5697 5698 int btf__relocate(struct btf *btf, const struct btf *base_btf) 5699 { 5700 int err = btf_relocate(btf, base_btf, NULL); 5701 5702 if (!err) 5703 btf->owns_base = false; 5704 return libbpf_err(err); 5705 } 5706