1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* Copyright (c) 2019 Facebook */ 3 4 #ifdef __KERNEL__ 5 #include <linux/bpf.h> 6 #include <linux/btf.h> 7 #include <linux/string.h> 8 #include <linux/bpf_verifier.h> 9 #include "relo_core.h" 10 11 static const char *btf_kind_str(const struct btf_type *t) 12 { 13 return btf_type_str(t); 14 } 15 16 static bool is_ldimm64_insn(struct bpf_insn *insn) 17 { 18 return insn->code == (BPF_LD | BPF_IMM | BPF_DW); 19 } 20 21 static const struct btf_type * 22 skip_mods_and_typedefs(const struct btf *btf, u32 id, u32 *res_id) 23 { 24 return btf_type_skip_modifiers(btf, id, res_id); 25 } 26 27 static const char *btf__name_by_offset(const struct btf *btf, u32 offset) 28 { 29 return btf_name_by_offset(btf, offset); 30 } 31 32 static s64 btf__resolve_size(const struct btf *btf, u32 type_id) 33 { 34 const struct btf_type *t; 35 int size; 36 37 t = btf_type_by_id(btf, type_id); 38 t = btf_resolve_size(btf, t, &size); 39 if (IS_ERR(t)) 40 return PTR_ERR(t); 41 return size; 42 } 43 44 enum libbpf_print_level { 45 LIBBPF_WARN, 46 LIBBPF_INFO, 47 LIBBPF_DEBUG, 48 }; 49 50 #undef pr_warn 51 #undef pr_info 52 #undef pr_debug 53 #define pr_warn(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__) 54 #define pr_info(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__) 55 #define pr_debug(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__) 56 #define libbpf_print(level, fmt, ...) bpf_log((void *)prog_name, fmt, ##__VA_ARGS__) 57 #else 58 #include <stdio.h> 59 #include <string.h> 60 #include <errno.h> 61 #include <ctype.h> 62 #include <linux/err.h> 63 64 #include "libbpf.h" 65 #include "bpf.h" 66 #include "btf.h" 67 #include "libbpf_internal.h" 68 #endif 69 70 static bool is_flex_arr(const struct btf *btf, 71 const struct bpf_core_accessor *acc, 72 const struct btf_array *arr) 73 { 74 const struct btf_type *t; 75 76 /* not a flexible array, if not inside a struct or has non-zero size */ 77 if (!acc->name || arr->nelems > 0) 78 return false; 79 80 /* has to be the last member of enclosing struct */ 81 t = btf_type_by_id(btf, acc->type_id); 82 return acc->idx == btf_vlen(t) - 1; 83 } 84 85 static const char *core_relo_kind_str(enum bpf_core_relo_kind kind) 86 { 87 switch (kind) { 88 case BPF_CORE_FIELD_BYTE_OFFSET: return "byte_off"; 89 case BPF_CORE_FIELD_BYTE_SIZE: return "byte_sz"; 90 case BPF_CORE_FIELD_EXISTS: return "field_exists"; 91 case BPF_CORE_FIELD_SIGNED: return "signed"; 92 case BPF_CORE_FIELD_LSHIFT_U64: return "lshift_u64"; 93 case BPF_CORE_FIELD_RSHIFT_U64: return "rshift_u64"; 94 case BPF_CORE_TYPE_ID_LOCAL: return "local_type_id"; 95 case BPF_CORE_TYPE_ID_TARGET: return "target_type_id"; 96 case BPF_CORE_TYPE_EXISTS: return "type_exists"; 97 case BPF_CORE_TYPE_MATCHES: return "type_matches"; 98 case BPF_CORE_TYPE_SIZE: return "type_size"; 99 case BPF_CORE_ENUMVAL_EXISTS: return "enumval_exists"; 100 case BPF_CORE_ENUMVAL_VALUE: return "enumval_value"; 101 default: return "unknown"; 102 } 103 } 104 105 static bool core_relo_is_field_based(enum bpf_core_relo_kind kind) 106 { 107 switch (kind) { 108 case BPF_CORE_FIELD_BYTE_OFFSET: 109 case BPF_CORE_FIELD_BYTE_SIZE: 110 case BPF_CORE_FIELD_EXISTS: 111 case BPF_CORE_FIELD_SIGNED: 112 case BPF_CORE_FIELD_LSHIFT_U64: 113 case BPF_CORE_FIELD_RSHIFT_U64: 114 return true; 115 default: 116 return false; 117 } 118 } 119 120 static bool core_relo_is_type_based(enum bpf_core_relo_kind kind) 121 { 122 switch (kind) { 123 case BPF_CORE_TYPE_ID_LOCAL: 124 case BPF_CORE_TYPE_ID_TARGET: 125 case BPF_CORE_TYPE_EXISTS: 126 case BPF_CORE_TYPE_MATCHES: 127 case BPF_CORE_TYPE_SIZE: 128 return true; 129 default: 130 return false; 131 } 132 } 133 134 static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind) 135 { 136 switch (kind) { 137 case BPF_CORE_ENUMVAL_EXISTS: 138 case BPF_CORE_ENUMVAL_VALUE: 139 return true; 140 default: 141 return false; 142 } 143 } 144 145 int __bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, 146 const struct btf *targ_btf, __u32 targ_id, int level) 147 { 148 const struct btf_type *local_type, *targ_type; 149 int depth = 32; /* max recursion depth */ 150 151 /* caller made sure that names match (ignoring flavor suffix) */ 152 local_type = btf_type_by_id(local_btf, local_id); 153 targ_type = btf_type_by_id(targ_btf, targ_id); 154 if (!btf_kind_core_compat(local_type, targ_type)) 155 return 0; 156 157 recur: 158 depth--; 159 if (depth < 0) 160 return -EINVAL; 161 162 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id); 163 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); 164 if (!local_type || !targ_type) 165 return -EINVAL; 166 167 if (!btf_kind_core_compat(local_type, targ_type)) 168 return 0; 169 170 switch (btf_kind(local_type)) { 171 case BTF_KIND_UNKN: 172 case BTF_KIND_STRUCT: 173 case BTF_KIND_UNION: 174 case BTF_KIND_ENUM: 175 case BTF_KIND_FWD: 176 case BTF_KIND_ENUM64: 177 return 1; 178 case BTF_KIND_INT: 179 /* just reject deprecated bitfield-like integers; all other 180 * integers are by default compatible between each other 181 */ 182 return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0; 183 case BTF_KIND_PTR: 184 local_id = local_type->type; 185 targ_id = targ_type->type; 186 goto recur; 187 case BTF_KIND_ARRAY: 188 local_id = btf_array(local_type)->type; 189 targ_id = btf_array(targ_type)->type; 190 goto recur; 191 case BTF_KIND_FUNC_PROTO: { 192 struct btf_param *local_p = btf_params(local_type); 193 struct btf_param *targ_p = btf_params(targ_type); 194 __u16 local_vlen = btf_vlen(local_type); 195 __u16 targ_vlen = btf_vlen(targ_type); 196 int i, err; 197 198 if (local_vlen != targ_vlen) 199 return 0; 200 201 for (i = 0; i < local_vlen; i++, local_p++, targ_p++) { 202 if (level <= 0) 203 return -EINVAL; 204 205 skip_mods_and_typedefs(local_btf, local_p->type, &local_id); 206 skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id); 207 err = __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 208 level - 1); 209 if (err <= 0) 210 return err; 211 } 212 213 /* tail recurse for return type check */ 214 skip_mods_and_typedefs(local_btf, local_type->type, &local_id); 215 skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id); 216 goto recur; 217 } 218 default: 219 pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", 220 btf_kind_str(local_type), local_id, targ_id); 221 return 0; 222 } 223 } 224 225 /* 226 * Turn bpf_core_relo into a low- and high-level spec representation, 227 * validating correctness along the way, as well as calculating resulting 228 * field bit offset, specified by accessor string. Low-level spec captures 229 * every single level of nestedness, including traversing anonymous 230 * struct/union members. High-level one only captures semantically meaningful 231 * "turning points": named fields and array indicies. 232 * E.g., for this case: 233 * 234 * struct sample { 235 * int __unimportant; 236 * struct { 237 * int __1; 238 * int __2; 239 * int a[7]; 240 * }; 241 * }; 242 * 243 * struct sample *s = ...; 244 * 245 * int x = &s->a[3]; // access string = '0:1:2:3' 246 * 247 * Low-level spec has 1:1 mapping with each element of access string (it's 248 * just a parsed access string representation): [0, 1, 2, 3]. 249 * 250 * High-level spec will capture only 3 points: 251 * - initial zero-index access by pointer (&s->... is the same as &s[0]...); 252 * - field 'a' access (corresponds to '2' in low-level spec); 253 * - array element #3 access (corresponds to '3' in low-level spec). 254 * 255 * Type-based relocations (TYPE_EXISTS/TYPE_MATCHES/TYPE_SIZE, 256 * TYPE_ID_LOCAL/TYPE_ID_TARGET) don't capture any field information. Their 257 * spec and raw_spec are kept empty. 258 * 259 * Enum value-based relocations (ENUMVAL_EXISTS/ENUMVAL_VALUE) use access 260 * string to specify enumerator's value index that need to be relocated. 261 */ 262 int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, 263 const struct bpf_core_relo *relo, 264 struct bpf_core_spec *spec) 265 { 266 int access_idx, parsed_len, i; 267 struct bpf_core_accessor *acc; 268 const struct btf_type *t; 269 const char *name, *spec_str; 270 __u32 id, name_off; 271 __s64 sz; 272 273 spec_str = btf__name_by_offset(btf, relo->access_str_off); 274 if (str_is_empty(spec_str) || *spec_str == ':') 275 return -EINVAL; 276 277 memset(spec, 0, sizeof(*spec)); 278 spec->btf = btf; 279 spec->root_type_id = relo->type_id; 280 spec->relo_kind = relo->kind; 281 282 /* type-based relocations don't have a field access string */ 283 if (core_relo_is_type_based(relo->kind)) { 284 if (strcmp(spec_str, "0")) 285 return -EINVAL; 286 return 0; 287 } 288 289 /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */ 290 while (*spec_str) { 291 if (*spec_str == ':') 292 ++spec_str; 293 if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1) 294 return -EINVAL; 295 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) 296 return -E2BIG; 297 spec_str += parsed_len; 298 spec->raw_spec[spec->raw_len++] = access_idx; 299 } 300 301 if (spec->raw_len == 0) 302 return -EINVAL; 303 304 t = skip_mods_and_typedefs(btf, relo->type_id, &id); 305 if (!t) 306 return -EINVAL; 307 308 access_idx = spec->raw_spec[0]; 309 acc = &spec->spec[0]; 310 acc->type_id = id; 311 acc->idx = access_idx; 312 spec->len++; 313 314 if (core_relo_is_enumval_based(relo->kind)) { 315 if (!btf_is_any_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t)) 316 return -EINVAL; 317 318 /* record enumerator name in a first accessor */ 319 name_off = btf_is_enum(t) ? btf_enum(t)[access_idx].name_off 320 : btf_enum64(t)[access_idx].name_off; 321 acc->name = btf__name_by_offset(btf, name_off); 322 return 0; 323 } 324 325 if (!core_relo_is_field_based(relo->kind)) 326 return -EINVAL; 327 328 sz = btf__resolve_size(btf, id); 329 if (sz < 0) 330 return sz; 331 spec->bit_offset = access_idx * sz * 8; 332 333 for (i = 1; i < spec->raw_len; i++) { 334 t = skip_mods_and_typedefs(btf, id, &id); 335 if (!t) 336 return -EINVAL; 337 338 access_idx = spec->raw_spec[i]; 339 acc = &spec->spec[spec->len]; 340 341 if (btf_is_composite(t)) { 342 const struct btf_member *m; 343 __u32 bit_offset; 344 345 if (access_idx >= btf_vlen(t)) 346 return -EINVAL; 347 348 bit_offset = btf_member_bit_offset(t, access_idx); 349 spec->bit_offset += bit_offset; 350 351 m = btf_members(t) + access_idx; 352 if (m->name_off) { 353 name = btf__name_by_offset(btf, m->name_off); 354 if (str_is_empty(name)) 355 return -EINVAL; 356 357 acc->type_id = id; 358 acc->idx = access_idx; 359 acc->name = name; 360 spec->len++; 361 } 362 363 id = m->type; 364 } else if (btf_is_array(t)) { 365 const struct btf_array *a = btf_array(t); 366 bool flex; 367 368 t = skip_mods_and_typedefs(btf, a->type, &id); 369 if (!t) 370 return -EINVAL; 371 372 flex = is_flex_arr(btf, acc - 1, a); 373 if (!flex && access_idx >= a->nelems) 374 return -EINVAL; 375 376 spec->spec[spec->len].type_id = id; 377 spec->spec[spec->len].idx = access_idx; 378 spec->len++; 379 380 sz = btf__resolve_size(btf, id); 381 if (sz < 0) 382 return sz; 383 spec->bit_offset += access_idx * sz * 8; 384 } else { 385 pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n", 386 prog_name, relo->type_id, spec_str, i, id, btf_kind_str(t)); 387 return -EINVAL; 388 } 389 } 390 391 return 0; 392 } 393 394 /* Check two types for compatibility for the purpose of field access 395 * relocation. const/volatile/restrict and typedefs are skipped to ensure we 396 * are relocating semantically compatible entities: 397 * - any two STRUCTs/UNIONs are compatible and can be mixed; 398 * - any two FWDs are compatible, if their names match (modulo flavor suffix); 399 * - any two PTRs are always compatible; 400 * - for ENUMs, names should be the same (ignoring flavor suffix) or at 401 * least one of enums should be anonymous; 402 * - for ENUMs, check sizes, names are ignored; 403 * - for INT, size and signedness are ignored; 404 * - any two FLOATs are always compatible; 405 * - for ARRAY, dimensionality is ignored, element types are checked for 406 * compatibility recursively; 407 * - everything else shouldn't be ever a target of relocation. 408 * These rules are not set in stone and probably will be adjusted as we get 409 * more experience with using BPF CO-RE relocations. 410 */ 411 static int bpf_core_fields_are_compat(const struct btf *local_btf, 412 __u32 local_id, 413 const struct btf *targ_btf, 414 __u32 targ_id) 415 { 416 const struct btf_type *local_type, *targ_type; 417 418 recur: 419 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id); 420 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); 421 if (!local_type || !targ_type) 422 return -EINVAL; 423 424 if (btf_is_composite(local_type) && btf_is_composite(targ_type)) 425 return 1; 426 if (!btf_kind_core_compat(local_type, targ_type)) 427 return 0; 428 429 switch (btf_kind(local_type)) { 430 case BTF_KIND_PTR: 431 case BTF_KIND_FLOAT: 432 return 1; 433 case BTF_KIND_FWD: 434 case BTF_KIND_ENUM64: 435 case BTF_KIND_ENUM: { 436 const char *local_name, *targ_name; 437 size_t local_len, targ_len; 438 439 local_name = btf__name_by_offset(local_btf, 440 local_type->name_off); 441 targ_name = btf__name_by_offset(targ_btf, targ_type->name_off); 442 local_len = bpf_core_essential_name_len(local_name); 443 targ_len = bpf_core_essential_name_len(targ_name); 444 /* one of them is anonymous or both w/ same flavor-less names */ 445 return local_len == 0 || targ_len == 0 || 446 (local_len == targ_len && 447 strncmp(local_name, targ_name, local_len) == 0); 448 } 449 case BTF_KIND_INT: 450 /* just reject deprecated bitfield-like integers; all other 451 * integers are by default compatible between each other 452 */ 453 return btf_int_offset(local_type) == 0 && 454 btf_int_offset(targ_type) == 0; 455 case BTF_KIND_ARRAY: 456 local_id = btf_array(local_type)->type; 457 targ_id = btf_array(targ_type)->type; 458 goto recur; 459 default: 460 return 0; 461 } 462 } 463 464 /* 465 * Given single high-level named field accessor in local type, find 466 * corresponding high-level accessor for a target type. Along the way, 467 * maintain low-level spec for target as well. Also keep updating target 468 * bit offset. 469 * 470 * Searching is performed through recursive exhaustive enumeration of all 471 * fields of a struct/union. If there are any anonymous (embedded) 472 * structs/unions, they are recursively searched as well. If field with 473 * desired name is found, check compatibility between local and target types, 474 * before returning result. 475 * 476 * 1 is returned, if field is found. 477 * 0 is returned if no compatible field is found. 478 * <0 is returned on error. 479 */ 480 static int bpf_core_match_member(const struct btf *local_btf, 481 const struct bpf_core_accessor *local_acc, 482 const struct btf *targ_btf, 483 __u32 targ_id, 484 struct bpf_core_spec *spec, 485 __u32 *next_targ_id) 486 { 487 const struct btf_type *local_type, *targ_type; 488 const struct btf_member *local_member, *m; 489 const char *local_name, *targ_name; 490 __u32 local_id; 491 int i, n, found; 492 493 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); 494 if (!targ_type) 495 return -EINVAL; 496 if (!btf_is_composite(targ_type)) 497 return 0; 498 499 local_id = local_acc->type_id; 500 local_type = btf_type_by_id(local_btf, local_id); 501 local_member = btf_members(local_type) + local_acc->idx; 502 local_name = btf__name_by_offset(local_btf, local_member->name_off); 503 504 n = btf_vlen(targ_type); 505 m = btf_members(targ_type); 506 for (i = 0; i < n; i++, m++) { 507 __u32 bit_offset; 508 509 bit_offset = btf_member_bit_offset(targ_type, i); 510 511 /* too deep struct/union/array nesting */ 512 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) 513 return -E2BIG; 514 515 /* speculate this member will be the good one */ 516 spec->bit_offset += bit_offset; 517 spec->raw_spec[spec->raw_len++] = i; 518 519 targ_name = btf__name_by_offset(targ_btf, m->name_off); 520 if (str_is_empty(targ_name)) { 521 /* embedded struct/union, we need to go deeper */ 522 found = bpf_core_match_member(local_btf, local_acc, 523 targ_btf, m->type, 524 spec, next_targ_id); 525 if (found) /* either found or error */ 526 return found; 527 } else if (strcmp(local_name, targ_name) == 0) { 528 /* matching named field */ 529 struct bpf_core_accessor *targ_acc; 530 531 targ_acc = &spec->spec[spec->len++]; 532 targ_acc->type_id = targ_id; 533 targ_acc->idx = i; 534 targ_acc->name = targ_name; 535 536 *next_targ_id = m->type; 537 found = bpf_core_fields_are_compat(local_btf, 538 local_member->type, 539 targ_btf, m->type); 540 if (!found) 541 spec->len--; /* pop accessor */ 542 return found; 543 } 544 /* member turned out not to be what we looked for */ 545 spec->bit_offset -= bit_offset; 546 spec->raw_len--; 547 } 548 549 return 0; 550 } 551 552 /* 553 * Try to match local spec to a target type and, if successful, produce full 554 * target spec (high-level, low-level + bit offset). 555 */ 556 static int bpf_core_spec_match(struct bpf_core_spec *local_spec, 557 const struct btf *targ_btf, __u32 targ_id, 558 struct bpf_core_spec *targ_spec) 559 { 560 const struct btf_type *targ_type; 561 const struct bpf_core_accessor *local_acc; 562 struct bpf_core_accessor *targ_acc; 563 int i, sz, matched; 564 __u32 name_off; 565 566 memset(targ_spec, 0, sizeof(*targ_spec)); 567 targ_spec->btf = targ_btf; 568 targ_spec->root_type_id = targ_id; 569 targ_spec->relo_kind = local_spec->relo_kind; 570 571 if (core_relo_is_type_based(local_spec->relo_kind)) { 572 if (local_spec->relo_kind == BPF_CORE_TYPE_MATCHES) 573 return bpf_core_types_match(local_spec->btf, 574 local_spec->root_type_id, 575 targ_btf, targ_id); 576 else 577 return bpf_core_types_are_compat(local_spec->btf, 578 local_spec->root_type_id, 579 targ_btf, targ_id); 580 } 581 582 local_acc = &local_spec->spec[0]; 583 targ_acc = &targ_spec->spec[0]; 584 585 if (core_relo_is_enumval_based(local_spec->relo_kind)) { 586 size_t local_essent_len, targ_essent_len; 587 const char *targ_name; 588 589 /* has to resolve to an enum */ 590 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, &targ_id); 591 if (!btf_is_any_enum(targ_type)) 592 return 0; 593 594 local_essent_len = bpf_core_essential_name_len(local_acc->name); 595 596 for (i = 0; i < btf_vlen(targ_type); i++) { 597 if (btf_is_enum(targ_type)) 598 name_off = btf_enum(targ_type)[i].name_off; 599 else 600 name_off = btf_enum64(targ_type)[i].name_off; 601 602 targ_name = btf__name_by_offset(targ_spec->btf, name_off); 603 targ_essent_len = bpf_core_essential_name_len(targ_name); 604 if (targ_essent_len != local_essent_len) 605 continue; 606 if (strncmp(local_acc->name, targ_name, local_essent_len) == 0) { 607 targ_acc->type_id = targ_id; 608 targ_acc->idx = i; 609 targ_acc->name = targ_name; 610 targ_spec->len++; 611 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx; 612 targ_spec->raw_len++; 613 return 1; 614 } 615 } 616 return 0; 617 } 618 619 if (!core_relo_is_field_based(local_spec->relo_kind)) 620 return -EINVAL; 621 622 for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) { 623 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, 624 &targ_id); 625 if (!targ_type) 626 return -EINVAL; 627 628 if (local_acc->name) { 629 matched = bpf_core_match_member(local_spec->btf, 630 local_acc, 631 targ_btf, targ_id, 632 targ_spec, &targ_id); 633 if (matched <= 0) 634 return matched; 635 } else { 636 /* for i=0, targ_id is already treated as array element 637 * type (because it's the original struct), for others 638 * we should find array element type first 639 */ 640 if (i > 0) { 641 const struct btf_array *a; 642 bool flex; 643 644 if (!btf_is_array(targ_type)) 645 return 0; 646 647 a = btf_array(targ_type); 648 flex = is_flex_arr(targ_btf, targ_acc - 1, a); 649 if (!flex && local_acc->idx >= a->nelems) 650 return 0; 651 if (!skip_mods_and_typedefs(targ_btf, a->type, 652 &targ_id)) 653 return -EINVAL; 654 } 655 656 /* too deep struct/union/array nesting */ 657 if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN) 658 return -E2BIG; 659 660 targ_acc->type_id = targ_id; 661 targ_acc->idx = local_acc->idx; 662 targ_acc->name = NULL; 663 targ_spec->len++; 664 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx; 665 targ_spec->raw_len++; 666 667 sz = btf__resolve_size(targ_btf, targ_id); 668 if (sz < 0) 669 return sz; 670 targ_spec->bit_offset += local_acc->idx * sz * 8; 671 } 672 } 673 674 return 1; 675 } 676 677 static int bpf_core_calc_field_relo(const char *prog_name, 678 const struct bpf_core_relo *relo, 679 const struct bpf_core_spec *spec, 680 __u64 *val, __u32 *field_sz, __u32 *type_id, 681 bool *validate) 682 { 683 const struct bpf_core_accessor *acc; 684 const struct btf_type *t; 685 __u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id, elem_id; 686 const struct btf_member *m; 687 const struct btf_type *mt; 688 bool bitfield; 689 __s64 sz; 690 691 *field_sz = 0; 692 693 if (relo->kind == BPF_CORE_FIELD_EXISTS) { 694 *val = spec ? 1 : 0; 695 return 0; 696 } 697 698 if (!spec) 699 return -EUCLEAN; /* request instruction poisoning */ 700 701 acc = &spec->spec[spec->len - 1]; 702 t = btf_type_by_id(spec->btf, acc->type_id); 703 704 /* a[n] accessor needs special handling */ 705 if (!acc->name) { 706 if (relo->kind == BPF_CORE_FIELD_BYTE_OFFSET) { 707 *val = spec->bit_offset / 8; 708 /* remember field size for load/store mem size; 709 * note, for arrays we care about individual element 710 * sizes, not the overall array size 711 */ 712 t = skip_mods_and_typedefs(spec->btf, acc->type_id, &elem_id); 713 while (btf_is_array(t)) 714 t = skip_mods_and_typedefs(spec->btf, btf_array(t)->type, &elem_id); 715 sz = btf__resolve_size(spec->btf, elem_id); 716 if (sz < 0) 717 return -EINVAL; 718 *field_sz = sz; 719 *type_id = acc->type_id; 720 } else if (relo->kind == BPF_CORE_FIELD_BYTE_SIZE) { 721 sz = btf__resolve_size(spec->btf, acc->type_id); 722 if (sz < 0) 723 return -EINVAL; 724 *val = sz; 725 } else { 726 pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n", 727 prog_name, relo->kind, relo->insn_off / 8); 728 return -EINVAL; 729 } 730 if (validate) 731 *validate = true; 732 return 0; 733 } 734 735 m = btf_members(t) + acc->idx; 736 mt = skip_mods_and_typedefs(spec->btf, m->type, &field_type_id); 737 bit_off = spec->bit_offset; 738 bit_sz = btf_member_bitfield_size(t, acc->idx); 739 740 bitfield = bit_sz > 0; 741 if (bitfield) { 742 byte_sz = mt->size; 743 byte_off = bit_off / 8 / byte_sz * byte_sz; 744 /* figure out smallest int size necessary for bitfield load */ 745 while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) { 746 if (byte_sz >= 8) { 747 /* bitfield can't be read with 64-bit read */ 748 pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n", 749 prog_name, relo->kind, relo->insn_off / 8); 750 return -E2BIG; 751 } 752 byte_sz *= 2; 753 byte_off = bit_off / 8 / byte_sz * byte_sz; 754 } 755 } else { 756 sz = btf__resolve_size(spec->btf, field_type_id); 757 if (sz < 0) 758 return -EINVAL; 759 byte_sz = sz; 760 byte_off = spec->bit_offset / 8; 761 bit_sz = byte_sz * 8; 762 } 763 764 /* for bitfields, all the relocatable aspects are ambiguous and we 765 * might disagree with compiler, so turn off validation of expected 766 * value, except for signedness 767 */ 768 if (validate) 769 *validate = !bitfield; 770 771 switch (relo->kind) { 772 case BPF_CORE_FIELD_BYTE_OFFSET: 773 *val = byte_off; 774 if (!bitfield) { 775 /* remember field size for load/store mem size; 776 * note, for arrays we care about individual element 777 * sizes, not the overall array size 778 */ 779 t = skip_mods_and_typedefs(spec->btf, field_type_id, &elem_id); 780 while (btf_is_array(t)) 781 t = skip_mods_and_typedefs(spec->btf, btf_array(t)->type, &elem_id); 782 sz = btf__resolve_size(spec->btf, elem_id); 783 if (sz < 0) 784 return -EINVAL; 785 *field_sz = sz; 786 *type_id = field_type_id; 787 } 788 break; 789 case BPF_CORE_FIELD_BYTE_SIZE: 790 *val = byte_sz; 791 break; 792 case BPF_CORE_FIELD_SIGNED: 793 *val = (btf_is_any_enum(mt) && BTF_INFO_KFLAG(mt->info)) || 794 (btf_is_int(mt) && (btf_int_encoding(mt) & BTF_INT_SIGNED)); 795 if (validate) 796 *validate = true; /* signedness is never ambiguous */ 797 break; 798 case BPF_CORE_FIELD_LSHIFT_U64: 799 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 800 *val = 64 - (bit_off + bit_sz - byte_off * 8); 801 #else 802 *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8); 803 #endif 804 break; 805 case BPF_CORE_FIELD_RSHIFT_U64: 806 *val = 64 - bit_sz; 807 if (validate) 808 *validate = true; /* right shift is never ambiguous */ 809 break; 810 case BPF_CORE_FIELD_EXISTS: 811 default: 812 return -EOPNOTSUPP; 813 } 814 815 return 0; 816 } 817 818 static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo, 819 const struct bpf_core_spec *spec, 820 __u64 *val, bool *validate) 821 { 822 __s64 sz; 823 824 /* by default, always check expected value in bpf_insn */ 825 if (validate) 826 *validate = true; 827 828 /* type-based relos return zero when target type is not found */ 829 if (!spec) { 830 *val = 0; 831 return 0; 832 } 833 834 switch (relo->kind) { 835 case BPF_CORE_TYPE_ID_TARGET: 836 *val = spec->root_type_id; 837 /* type ID, embedded in bpf_insn, might change during linking, 838 * so enforcing it is pointless 839 */ 840 if (validate) 841 *validate = false; 842 break; 843 case BPF_CORE_TYPE_EXISTS: 844 case BPF_CORE_TYPE_MATCHES: 845 *val = 1; 846 break; 847 case BPF_CORE_TYPE_SIZE: 848 sz = btf__resolve_size(spec->btf, spec->root_type_id); 849 if (sz < 0) 850 return -EINVAL; 851 *val = sz; 852 break; 853 case BPF_CORE_TYPE_ID_LOCAL: 854 /* BPF_CORE_TYPE_ID_LOCAL is handled specially and shouldn't get here */ 855 default: 856 return -EOPNOTSUPP; 857 } 858 859 return 0; 860 } 861 862 static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo, 863 const struct bpf_core_spec *spec, 864 __u64 *val) 865 { 866 const struct btf_type *t; 867 868 switch (relo->kind) { 869 case BPF_CORE_ENUMVAL_EXISTS: 870 *val = spec ? 1 : 0; 871 break; 872 case BPF_CORE_ENUMVAL_VALUE: 873 if (!spec) 874 return -EUCLEAN; /* request instruction poisoning */ 875 t = btf_type_by_id(spec->btf, spec->spec[0].type_id); 876 if (btf_is_enum(t)) 877 *val = btf_enum(t)[spec->spec[0].idx].val; 878 else 879 *val = btf_enum64_value(btf_enum64(t) + spec->spec[0].idx); 880 break; 881 default: 882 return -EOPNOTSUPP; 883 } 884 885 return 0; 886 } 887 888 /* Calculate original and target relocation values, given local and target 889 * specs and relocation kind. These values are calculated for each candidate. 890 * If there are multiple candidates, resulting values should all be consistent 891 * with each other. Otherwise, libbpf will refuse to proceed due to ambiguity. 892 * If instruction has to be poisoned, *poison will be set to true. 893 */ 894 static int bpf_core_calc_relo(const char *prog_name, 895 const struct bpf_core_relo *relo, 896 int relo_idx, 897 const struct bpf_core_spec *local_spec, 898 const struct bpf_core_spec *targ_spec, 899 struct bpf_core_relo_res *res) 900 { 901 int err = -EOPNOTSUPP; 902 903 res->orig_val = 0; 904 res->new_val = 0; 905 res->poison = false; 906 res->validate = true; 907 res->fail_memsz_adjust = false; 908 res->orig_sz = res->new_sz = 0; 909 res->orig_type_id = res->new_type_id = 0; 910 911 if (core_relo_is_field_based(relo->kind)) { 912 err = bpf_core_calc_field_relo(prog_name, relo, local_spec, 913 &res->orig_val, &res->orig_sz, 914 &res->orig_type_id, &res->validate); 915 err = err ?: bpf_core_calc_field_relo(prog_name, relo, targ_spec, 916 &res->new_val, &res->new_sz, 917 &res->new_type_id, NULL); 918 if (err) 919 goto done; 920 /* Validate if it's safe to adjust load/store memory size. 921 * Adjustments are performed only if original and new memory 922 * sizes differ. 923 */ 924 res->fail_memsz_adjust = false; 925 if (res->orig_sz != res->new_sz) { 926 const struct btf_type *orig_t, *new_t; 927 928 orig_t = btf_type_by_id(local_spec->btf, res->orig_type_id); 929 new_t = btf_type_by_id(targ_spec->btf, res->new_type_id); 930 931 /* There are two use cases in which it's safe to 932 * adjust load/store's mem size: 933 * - reading a 32-bit kernel pointer, while on BPF 934 * size pointers are always 64-bit; in this case 935 * it's safe to "downsize" instruction size due to 936 * pointer being treated as unsigned integer with 937 * zero-extended upper 32-bits; 938 * - reading unsigned integers, again due to 939 * zero-extension is preserving the value correctly. 940 * 941 * In all other cases it's incorrect to attempt to 942 * load/store field because read value will be 943 * incorrect, so we poison relocated instruction. 944 */ 945 if (btf_is_ptr(orig_t) && btf_is_ptr(new_t)) 946 goto done; 947 if (btf_is_int(orig_t) && btf_is_int(new_t) && 948 btf_int_encoding(orig_t) != BTF_INT_SIGNED && 949 btf_int_encoding(new_t) != BTF_INT_SIGNED) 950 goto done; 951 952 /* mark as invalid mem size adjustment, but this will 953 * only be checked for LDX/STX/ST insns 954 */ 955 res->fail_memsz_adjust = true; 956 } 957 } else if (core_relo_is_type_based(relo->kind)) { 958 err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val, &res->validate); 959 err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val, NULL); 960 } else if (core_relo_is_enumval_based(relo->kind)) { 961 err = bpf_core_calc_enumval_relo(relo, local_spec, &res->orig_val); 962 err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val); 963 } 964 965 done: 966 if (err == -EUCLEAN) { 967 /* EUCLEAN is used to signal instruction poisoning request */ 968 res->poison = true; 969 err = 0; 970 } else if (err == -EOPNOTSUPP) { 971 /* EOPNOTSUPP means unknown/unsupported relocation */ 972 pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%d) at insn #%d\n", 973 prog_name, relo_idx, core_relo_kind_str(relo->kind), 974 relo->kind, relo->insn_off / 8); 975 } 976 977 return err; 978 } 979 980 /* 981 * Turn instruction for which CO_RE relocation failed into invalid one with 982 * distinct signature. 983 */ 984 static void bpf_core_poison_insn(const char *prog_name, int relo_idx, 985 int insn_idx, struct bpf_insn *insn) 986 { 987 pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n", 988 prog_name, relo_idx, insn_idx); 989 insn->code = BPF_JMP | BPF_CALL; 990 insn->dst_reg = 0; 991 insn->src_reg = 0; 992 insn->off = 0; 993 /* if this instruction is reachable (not a dead code), 994 * verifier will complain with the following message: 995 * invalid func unknown#195896080 996 */ 997 insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */ 998 } 999 1000 static int insn_bpf_size_to_bytes(struct bpf_insn *insn) 1001 { 1002 switch (BPF_SIZE(insn->code)) { 1003 case BPF_DW: return 8; 1004 case BPF_W: return 4; 1005 case BPF_H: return 2; 1006 case BPF_B: return 1; 1007 default: return -1; 1008 } 1009 } 1010 1011 static int insn_bytes_to_bpf_size(__u32 sz) 1012 { 1013 switch (sz) { 1014 case 8: return BPF_DW; 1015 case 4: return BPF_W; 1016 case 2: return BPF_H; 1017 case 1: return BPF_B; 1018 default: return -1; 1019 } 1020 } 1021 1022 /* 1023 * Patch relocatable BPF instruction. 1024 * 1025 * Patched value is determined by relocation kind and target specification. 1026 * For existence relocations target spec will be NULL if field/type is not found. 1027 * Expected insn->imm value is determined using relocation kind and local 1028 * spec, and is checked before patching instruction. If actual insn->imm value 1029 * is wrong, bail out with error. 1030 * 1031 * Currently supported classes of BPF instruction are: 1032 * 1. rX = <imm> (assignment with immediate operand); 1033 * 2. rX += <imm> (arithmetic operations with immediate operand); 1034 * 3. rX = <imm64> (load with 64-bit immediate value); 1035 * 4. rX = *(T *)(rY + <off>), where T is one of {u8, u16, u32, u64}; 1036 * 5. *(T *)(rX + <off>) = rY, where T is one of {u8, u16, u32, u64}; 1037 * 6. *(T *)(rX + <off>) = <imm>, where T is one of {u8, u16, u32, u64}. 1038 */ 1039 int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, 1040 int insn_idx, const struct bpf_core_relo *relo, 1041 int relo_idx, const struct bpf_core_relo_res *res) 1042 { 1043 __u64 orig_val, new_val; 1044 __u8 class; 1045 1046 class = BPF_CLASS(insn->code); 1047 1048 if (res->poison) { 1049 poison: 1050 /* poison second part of ldimm64 to avoid confusing error from 1051 * verifier about "unknown opcode 00" 1052 */ 1053 if (is_ldimm64_insn(insn)) 1054 bpf_core_poison_insn(prog_name, relo_idx, insn_idx + 1, insn + 1); 1055 bpf_core_poison_insn(prog_name, relo_idx, insn_idx, insn); 1056 return 0; 1057 } 1058 1059 orig_val = res->orig_val; 1060 new_val = res->new_val; 1061 1062 switch (class) { 1063 case BPF_ALU: 1064 case BPF_ALU64: 1065 if (BPF_SRC(insn->code) != BPF_K) 1066 return -EINVAL; 1067 if (res->validate && insn->imm != orig_val) { 1068 pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %llu -> %llu\n", 1069 prog_name, relo_idx, 1070 insn_idx, insn->imm, (unsigned long long)orig_val, 1071 (unsigned long long)new_val); 1072 return -EINVAL; 1073 } 1074 orig_val = insn->imm; 1075 insn->imm = new_val; 1076 pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %llu -> %llu\n", 1077 prog_name, relo_idx, insn_idx, 1078 (unsigned long long)orig_val, (unsigned long long)new_val); 1079 break; 1080 case BPF_LDX: 1081 case BPF_ST: 1082 case BPF_STX: 1083 if (res->validate && insn->off != orig_val) { 1084 pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %llu -> %llu\n", 1085 prog_name, relo_idx, insn_idx, insn->off, (unsigned long long)orig_val, 1086 (unsigned long long)new_val); 1087 return -EINVAL; 1088 } 1089 if (new_val > SHRT_MAX) { 1090 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %llu\n", 1091 prog_name, relo_idx, insn_idx, (unsigned long long)new_val); 1092 return -ERANGE; 1093 } 1094 if (res->fail_memsz_adjust) { 1095 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) accesses field incorrectly. " 1096 "Make sure you are accessing pointers, unsigned integers, or fields of matching type and size.\n", 1097 prog_name, relo_idx, insn_idx); 1098 goto poison; 1099 } 1100 1101 orig_val = insn->off; 1102 insn->off = new_val; 1103 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %llu -> %llu\n", 1104 prog_name, relo_idx, insn_idx, (unsigned long long)orig_val, 1105 (unsigned long long)new_val); 1106 1107 if (res->new_sz != res->orig_sz) { 1108 int insn_bytes_sz, insn_bpf_sz; 1109 1110 insn_bytes_sz = insn_bpf_size_to_bytes(insn); 1111 if (insn_bytes_sz != res->orig_sz) { 1112 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) unexpected mem size: got %d, exp %u\n", 1113 prog_name, relo_idx, insn_idx, insn_bytes_sz, res->orig_sz); 1114 return -EINVAL; 1115 } 1116 1117 insn_bpf_sz = insn_bytes_to_bpf_size(res->new_sz); 1118 if (insn_bpf_sz < 0) { 1119 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) invalid new mem size: %u\n", 1120 prog_name, relo_idx, insn_idx, res->new_sz); 1121 return -EINVAL; 1122 } 1123 1124 insn->code = BPF_MODE(insn->code) | insn_bpf_sz | BPF_CLASS(insn->code); 1125 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) mem_sz %u -> %u\n", 1126 prog_name, relo_idx, insn_idx, res->orig_sz, res->new_sz); 1127 } 1128 break; 1129 case BPF_LD: { 1130 __u64 imm; 1131 1132 if (!is_ldimm64_insn(insn) || 1133 insn[0].src_reg != 0 || insn[0].off != 0 || 1134 insn[1].code != 0 || insn[1].dst_reg != 0 || 1135 insn[1].src_reg != 0 || insn[1].off != 0) { 1136 pr_warn("prog '%s': relo #%d: insn #%d (LDIMM64) has unexpected form\n", 1137 prog_name, relo_idx, insn_idx); 1138 return -EINVAL; 1139 } 1140 1141 imm = (__u32)insn[0].imm | ((__u64)insn[1].imm << 32); 1142 if (res->validate && imm != orig_val) { 1143 pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %llu -> %llu\n", 1144 prog_name, relo_idx, 1145 insn_idx, (unsigned long long)imm, 1146 (unsigned long long)orig_val, (unsigned long long)new_val); 1147 return -EINVAL; 1148 } 1149 1150 insn[0].imm = new_val; 1151 insn[1].imm = new_val >> 32; 1152 pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %llu\n", 1153 prog_name, relo_idx, insn_idx, 1154 (unsigned long long)imm, (unsigned long long)new_val); 1155 break; 1156 } 1157 default: 1158 pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n", 1159 prog_name, relo_idx, insn_idx, insn->code, 1160 insn->src_reg, insn->dst_reg, insn->off, insn->imm); 1161 return -EINVAL; 1162 } 1163 1164 return 0; 1165 } 1166 1167 /* Output spec definition in the format: 1168 * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>, 1169 * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b 1170 */ 1171 int bpf_core_format_spec(char *buf, size_t buf_sz, const struct bpf_core_spec *spec) 1172 { 1173 const struct btf_type *t; 1174 const char *s; 1175 __u32 type_id; 1176 int i, len = 0; 1177 1178 #define append_buf(fmt, args...) \ 1179 ({ \ 1180 int r; \ 1181 r = snprintf(buf, buf_sz, fmt, ##args); \ 1182 len += r; \ 1183 if (r >= buf_sz) \ 1184 r = buf_sz; \ 1185 buf += r; \ 1186 buf_sz -= r; \ 1187 }) 1188 1189 type_id = spec->root_type_id; 1190 t = btf_type_by_id(spec->btf, type_id); 1191 s = btf__name_by_offset(spec->btf, t->name_off); 1192 1193 append_buf("<%s> [%u] %s %s", 1194 core_relo_kind_str(spec->relo_kind), 1195 type_id, btf_kind_str(t), str_is_empty(s) ? "<anon>" : s); 1196 1197 if (core_relo_is_type_based(spec->relo_kind)) 1198 return len; 1199 1200 if (core_relo_is_enumval_based(spec->relo_kind)) { 1201 t = skip_mods_and_typedefs(spec->btf, type_id, NULL); 1202 if (btf_is_enum(t)) { 1203 const struct btf_enum *e; 1204 const char *fmt_str; 1205 1206 e = btf_enum(t) + spec->raw_spec[0]; 1207 s = btf__name_by_offset(spec->btf, e->name_off); 1208 fmt_str = BTF_INFO_KFLAG(t->info) ? "::%s = %d" : "::%s = %u"; 1209 append_buf(fmt_str, s, e->val); 1210 } else { 1211 const struct btf_enum64 *e; 1212 const char *fmt_str; 1213 1214 e = btf_enum64(t) + spec->raw_spec[0]; 1215 s = btf__name_by_offset(spec->btf, e->name_off); 1216 fmt_str = BTF_INFO_KFLAG(t->info) ? "::%s = %lld" : "::%s = %llu"; 1217 append_buf(fmt_str, s, (unsigned long long)btf_enum64_value(e)); 1218 } 1219 return len; 1220 } 1221 1222 if (core_relo_is_field_based(spec->relo_kind)) { 1223 for (i = 0; i < spec->len; i++) { 1224 if (spec->spec[i].name) 1225 append_buf(".%s", spec->spec[i].name); 1226 else if (i > 0 || spec->spec[i].idx > 0) 1227 append_buf("[%u]", spec->spec[i].idx); 1228 } 1229 1230 append_buf(" ("); 1231 for (i = 0; i < spec->raw_len; i++) 1232 append_buf("%s%d", i == 0 ? "" : ":", spec->raw_spec[i]); 1233 1234 if (spec->bit_offset % 8) 1235 append_buf(" @ offset %u.%u)", spec->bit_offset / 8, spec->bit_offset % 8); 1236 else 1237 append_buf(" @ offset %u)", spec->bit_offset / 8); 1238 return len; 1239 } 1240 1241 return len; 1242 #undef append_buf 1243 } 1244 1245 /* 1246 * Calculate CO-RE relocation target result. 1247 * 1248 * The outline and important points of the algorithm: 1249 * 1. For given local type, find corresponding candidate target types. 1250 * Candidate type is a type with the same "essential" name, ignoring 1251 * everything after last triple underscore (___). E.g., `sample`, 1252 * `sample___flavor_one`, `sample___flavor_another_one`, are all candidates 1253 * for each other. Names with triple underscore are referred to as 1254 * "flavors" and are useful, among other things, to allow to 1255 * specify/support incompatible variations of the same kernel struct, which 1256 * might differ between different kernel versions and/or build 1257 * configurations. 1258 * 1259 * N.B. Struct "flavors" could be generated by bpftool's BTF-to-C 1260 * converter, when deduplicated BTF of a kernel still contains more than 1261 * one different types with the same name. In that case, ___2, ___3, etc 1262 * are appended starting from second name conflict. But start flavors are 1263 * also useful to be defined "locally", in BPF program, to extract same 1264 * data from incompatible changes between different kernel 1265 * versions/configurations. For instance, to handle field renames between 1266 * kernel versions, one can use two flavors of the struct name with the 1267 * same common name and use conditional relocations to extract that field, 1268 * depending on target kernel version. 1269 * 2. For each candidate type, try to match local specification to this 1270 * candidate target type. Matching involves finding corresponding 1271 * high-level spec accessors, meaning that all named fields should match, 1272 * as well as all array accesses should be within the actual bounds. Also, 1273 * types should be compatible (see bpf_core_fields_are_compat for details). 1274 * 3. It is supported and expected that there might be multiple flavors 1275 * matching the spec. As long as all the specs resolve to the same set of 1276 * offsets across all candidates, there is no error. If there is any 1277 * ambiguity, CO-RE relocation will fail. This is necessary to accommodate 1278 * imperfection of BTF deduplication, which can cause slight duplication of 1279 * the same BTF type, if some directly or indirectly referenced (by 1280 * pointer) type gets resolved to different actual types in different 1281 * object files. If such a situation occurs, deduplicated BTF will end up 1282 * with two (or more) structurally identical types, which differ only in 1283 * types they refer to through pointer. This should be OK in most cases and 1284 * is not an error. 1285 * 4. Candidate types search is performed by linearly scanning through all 1286 * types in target BTF. It is anticipated that this is overall more 1287 * efficient memory-wise and not significantly worse (if not better) 1288 * CPU-wise compared to prebuilding a map from all local type names to 1289 * a list of candidate type names. It's also sped up by caching resolved 1290 * list of matching candidates per each local "root" type ID, that has at 1291 * least one bpf_core_relo associated with it. This list is shared 1292 * between multiple relocations for the same type ID and is updated as some 1293 * of the candidates are pruned due to structural incompatibility. 1294 */ 1295 int bpf_core_calc_relo_insn(const char *prog_name, 1296 const struct bpf_core_relo *relo, 1297 int relo_idx, 1298 const struct btf *local_btf, 1299 struct bpf_core_cand_list *cands, 1300 struct bpf_core_spec *specs_scratch, 1301 struct bpf_core_relo_res *targ_res) 1302 { 1303 struct bpf_core_spec *local_spec = &specs_scratch[0]; 1304 struct bpf_core_spec *cand_spec = &specs_scratch[1]; 1305 struct bpf_core_spec *targ_spec = &specs_scratch[2]; 1306 struct bpf_core_relo_res cand_res; 1307 const struct btf_type *local_type; 1308 const char *local_name; 1309 __u32 local_id; 1310 char spec_buf[256]; 1311 int i, j, err; 1312 1313 local_id = relo->type_id; 1314 local_type = btf_type_by_id(local_btf, local_id); 1315 local_name = btf__name_by_offset(local_btf, local_type->name_off); 1316 if (!local_name) 1317 return -EINVAL; 1318 1319 err = bpf_core_parse_spec(prog_name, local_btf, relo, local_spec); 1320 if (err) { 1321 const char *spec_str; 1322 1323 spec_str = btf__name_by_offset(local_btf, relo->access_str_off); 1324 pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n", 1325 prog_name, relo_idx, local_id, btf_kind_str(local_type), 1326 str_is_empty(local_name) ? "<anon>" : local_name, 1327 spec_str ?: "<?>", err); 1328 return -EINVAL; 1329 } 1330 1331 bpf_core_format_spec(spec_buf, sizeof(spec_buf), local_spec); 1332 pr_debug("prog '%s': relo #%d: %s\n", prog_name, relo_idx, spec_buf); 1333 1334 /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */ 1335 if (relo->kind == BPF_CORE_TYPE_ID_LOCAL) { 1336 /* bpf_insn's imm value could get out of sync during linking */ 1337 memset(targ_res, 0, sizeof(*targ_res)); 1338 targ_res->validate = false; 1339 targ_res->poison = false; 1340 targ_res->orig_val = local_spec->root_type_id; 1341 targ_res->new_val = local_spec->root_type_id; 1342 return 0; 1343 } 1344 1345 /* libbpf doesn't support candidate search for anonymous types */ 1346 if (str_is_empty(local_name)) { 1347 pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n", 1348 prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind); 1349 return -EOPNOTSUPP; 1350 } 1351 1352 for (i = 0, j = 0; i < cands->len; i++) { 1353 err = bpf_core_spec_match(local_spec, cands->cands[i].btf, 1354 cands->cands[i].id, cand_spec); 1355 if (err < 0) { 1356 bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec); 1357 pr_warn("prog '%s': relo #%d: error matching candidate #%d %s: %d\n", 1358 prog_name, relo_idx, i, spec_buf, err); 1359 return err; 1360 } 1361 1362 bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec); 1363 pr_debug("prog '%s': relo #%d: %s candidate #%d %s\n", prog_name, 1364 relo_idx, err == 0 ? "non-matching" : "matching", i, spec_buf); 1365 1366 if (err == 0) 1367 continue; 1368 1369 err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, cand_spec, &cand_res); 1370 if (err) 1371 return err; 1372 1373 if (j == 0) { 1374 *targ_res = cand_res; 1375 *targ_spec = *cand_spec; 1376 } else if (cand_spec->bit_offset != targ_spec->bit_offset) { 1377 /* if there are many field relo candidates, they 1378 * should all resolve to the same bit offset 1379 */ 1380 pr_warn("prog '%s': relo #%d: field offset ambiguity: %u != %u\n", 1381 prog_name, relo_idx, cand_spec->bit_offset, 1382 targ_spec->bit_offset); 1383 return -EINVAL; 1384 } else if (cand_res.poison != targ_res->poison || 1385 cand_res.new_val != targ_res->new_val) { 1386 /* all candidates should result in the same relocation 1387 * decision and value, otherwise it's dangerous to 1388 * proceed due to ambiguity 1389 */ 1390 pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %llu != %s %llu\n", 1391 prog_name, relo_idx, 1392 cand_res.poison ? "failure" : "success", 1393 (unsigned long long)cand_res.new_val, 1394 targ_res->poison ? "failure" : "success", 1395 (unsigned long long)targ_res->new_val); 1396 return -EINVAL; 1397 } 1398 1399 cands->cands[j++] = cands->cands[i]; 1400 } 1401 1402 /* 1403 * For BPF_CORE_FIELD_EXISTS relo or when used BPF program has field 1404 * existence checks or kernel version/config checks, it's expected 1405 * that we might not find any candidates. In this case, if field 1406 * wasn't found in any candidate, the list of candidates shouldn't 1407 * change at all, we'll just handle relocating appropriately, 1408 * depending on relo's kind. 1409 */ 1410 if (j > 0) 1411 cands->len = j; 1412 1413 /* 1414 * If no candidates were found, it might be both a programmer error, 1415 * as well as expected case, depending whether instruction w/ 1416 * relocation is guarded in some way that makes it unreachable (dead 1417 * code) if relocation can't be resolved. This is handled in 1418 * bpf_core_patch_insn() uniformly by replacing that instruction with 1419 * BPF helper call insn (using invalid helper ID). If that instruction 1420 * is indeed unreachable, then it will be ignored and eliminated by 1421 * verifier. If it was an error, then verifier will complain and point 1422 * to a specific instruction number in its log. 1423 */ 1424 if (j == 0) { 1425 pr_debug("prog '%s': relo #%d: no matching targets found\n", 1426 prog_name, relo_idx); 1427 1428 /* calculate single target relo result explicitly */ 1429 err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, NULL, targ_res); 1430 if (err) 1431 return err; 1432 } 1433 1434 return 0; 1435 } 1436 1437 static bool bpf_core_names_match(const struct btf *local_btf, size_t local_name_off, 1438 const struct btf *targ_btf, size_t targ_name_off) 1439 { 1440 const char *local_n, *targ_n; 1441 size_t local_len, targ_len; 1442 1443 local_n = btf__name_by_offset(local_btf, local_name_off); 1444 targ_n = btf__name_by_offset(targ_btf, targ_name_off); 1445 1446 if (str_is_empty(targ_n)) 1447 return str_is_empty(local_n); 1448 1449 targ_len = bpf_core_essential_name_len(targ_n); 1450 local_len = bpf_core_essential_name_len(local_n); 1451 1452 return targ_len == local_len && strncmp(local_n, targ_n, local_len) == 0; 1453 } 1454 1455 static int bpf_core_enums_match(const struct btf *local_btf, const struct btf_type *local_t, 1456 const struct btf *targ_btf, const struct btf_type *targ_t) 1457 { 1458 __u16 local_vlen = btf_vlen(local_t); 1459 __u16 targ_vlen = btf_vlen(targ_t); 1460 int i, j; 1461 1462 if (local_t->size != targ_t->size) 1463 return 0; 1464 1465 if (local_vlen > targ_vlen) 1466 return 0; 1467 1468 /* iterate over the local enum's variants and make sure each has 1469 * a symbolic name correspondent in the target 1470 */ 1471 for (i = 0; i < local_vlen; i++) { 1472 bool matched = false; 1473 __u32 local_n_off, targ_n_off; 1474 1475 local_n_off = btf_is_enum(local_t) ? btf_enum(local_t)[i].name_off : 1476 btf_enum64(local_t)[i].name_off; 1477 1478 for (j = 0; j < targ_vlen; j++) { 1479 targ_n_off = btf_is_enum(targ_t) ? btf_enum(targ_t)[j].name_off : 1480 btf_enum64(targ_t)[j].name_off; 1481 1482 if (bpf_core_names_match(local_btf, local_n_off, targ_btf, targ_n_off)) { 1483 matched = true; 1484 break; 1485 } 1486 } 1487 1488 if (!matched) 1489 return 0; 1490 } 1491 return 1; 1492 } 1493 1494 static int bpf_core_composites_match(const struct btf *local_btf, const struct btf_type *local_t, 1495 const struct btf *targ_btf, const struct btf_type *targ_t, 1496 bool behind_ptr, int level) 1497 { 1498 const struct btf_member *local_m = btf_members(local_t); 1499 __u16 local_vlen = btf_vlen(local_t); 1500 __u16 targ_vlen = btf_vlen(targ_t); 1501 int i, j, err; 1502 1503 if (local_vlen > targ_vlen) 1504 return 0; 1505 1506 /* check that all local members have a match in the target */ 1507 for (i = 0; i < local_vlen; i++, local_m++) { 1508 const struct btf_member *targ_m = btf_members(targ_t); 1509 bool matched = false; 1510 1511 for (j = 0; j < targ_vlen; j++, targ_m++) { 1512 if (!bpf_core_names_match(local_btf, local_m->name_off, 1513 targ_btf, targ_m->name_off)) 1514 continue; 1515 1516 err = __bpf_core_types_match(local_btf, local_m->type, targ_btf, 1517 targ_m->type, behind_ptr, level - 1); 1518 if (err < 0) 1519 return err; 1520 if (err > 0) { 1521 matched = true; 1522 break; 1523 } 1524 } 1525 1526 if (!matched) 1527 return 0; 1528 } 1529 return 1; 1530 } 1531 1532 /* Check that two types "match". This function assumes that root types were 1533 * already checked for name match. 1534 * 1535 * The matching relation is defined as follows: 1536 * - modifiers and typedefs are stripped (and, hence, effectively ignored) 1537 * - generally speaking types need to be of same kind (struct vs. struct, union 1538 * vs. union, etc.) 1539 * - exceptions are struct/union behind a pointer which could also match a 1540 * forward declaration of a struct or union, respectively, and enum vs. 1541 * enum64 (see below) 1542 * Then, depending on type: 1543 * - integers: 1544 * - match if size and signedness match 1545 * - arrays & pointers: 1546 * - target types are recursively matched 1547 * - structs & unions: 1548 * - local members need to exist in target with the same name 1549 * - for each member we recursively check match unless it is already behind a 1550 * pointer, in which case we only check matching names and compatible kind 1551 * - enums: 1552 * - local variants have to have a match in target by symbolic name (but not 1553 * numeric value) 1554 * - size has to match (but enum may match enum64 and vice versa) 1555 * - function pointers: 1556 * - number and position of arguments in local type has to match target 1557 * - for each argument and the return value we recursively check match 1558 */ 1559 int __bpf_core_types_match(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf, 1560 __u32 targ_id, bool behind_ptr, int level) 1561 { 1562 const struct btf_type *local_t, *targ_t; 1563 int depth = 32; /* max recursion depth */ 1564 __u16 local_k, targ_k; 1565 1566 if (level <= 0) 1567 return -EINVAL; 1568 1569 recur: 1570 depth--; 1571 if (depth < 0) 1572 return -EINVAL; 1573 1574 local_t = skip_mods_and_typedefs(local_btf, local_id, &local_id); 1575 targ_t = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); 1576 if (!local_t || !targ_t) 1577 return -EINVAL; 1578 1579 /* While the name check happens after typedefs are skipped, root-level 1580 * typedefs would still be name-matched as that's the contract with 1581 * callers. 1582 */ 1583 if (!bpf_core_names_match(local_btf, local_t->name_off, targ_btf, targ_t->name_off)) 1584 return 0; 1585 1586 local_k = btf_kind(local_t); 1587 targ_k = btf_kind(targ_t); 1588 1589 switch (local_k) { 1590 case BTF_KIND_UNKN: 1591 return local_k == targ_k; 1592 case BTF_KIND_FWD: { 1593 bool local_f = BTF_INFO_KFLAG(local_t->info); 1594 1595 if (behind_ptr) { 1596 if (local_k == targ_k) 1597 return local_f == BTF_INFO_KFLAG(targ_t->info); 1598 1599 /* for forward declarations kflag dictates whether the 1600 * target is a struct (0) or union (1) 1601 */ 1602 return (targ_k == BTF_KIND_STRUCT && !local_f) || 1603 (targ_k == BTF_KIND_UNION && local_f); 1604 } else { 1605 if (local_k != targ_k) 1606 return 0; 1607 1608 /* match if the forward declaration is for the same kind */ 1609 return local_f == BTF_INFO_KFLAG(targ_t->info); 1610 } 1611 } 1612 case BTF_KIND_ENUM: 1613 case BTF_KIND_ENUM64: 1614 if (!btf_is_any_enum(targ_t)) 1615 return 0; 1616 1617 return bpf_core_enums_match(local_btf, local_t, targ_btf, targ_t); 1618 case BTF_KIND_STRUCT: 1619 case BTF_KIND_UNION: 1620 if (behind_ptr) { 1621 bool targ_f = BTF_INFO_KFLAG(targ_t->info); 1622 1623 if (local_k == targ_k) 1624 return 1; 1625 1626 if (targ_k != BTF_KIND_FWD) 1627 return 0; 1628 1629 return (local_k == BTF_KIND_UNION) == targ_f; 1630 } else { 1631 if (local_k != targ_k) 1632 return 0; 1633 1634 return bpf_core_composites_match(local_btf, local_t, targ_btf, targ_t, 1635 behind_ptr, level); 1636 } 1637 case BTF_KIND_INT: { 1638 __u8 local_sgn; 1639 __u8 targ_sgn; 1640 1641 if (local_k != targ_k) 1642 return 0; 1643 1644 local_sgn = btf_int_encoding(local_t) & BTF_INT_SIGNED; 1645 targ_sgn = btf_int_encoding(targ_t) & BTF_INT_SIGNED; 1646 1647 return local_t->size == targ_t->size && local_sgn == targ_sgn; 1648 } 1649 case BTF_KIND_PTR: 1650 if (local_k != targ_k) 1651 return 0; 1652 1653 behind_ptr = true; 1654 1655 local_id = local_t->type; 1656 targ_id = targ_t->type; 1657 goto recur; 1658 case BTF_KIND_ARRAY: { 1659 const struct btf_array *local_array = btf_array(local_t); 1660 const struct btf_array *targ_array = btf_array(targ_t); 1661 1662 if (local_k != targ_k) 1663 return 0; 1664 1665 if (local_array->nelems != targ_array->nelems) 1666 return 0; 1667 1668 local_id = local_array->type; 1669 targ_id = targ_array->type; 1670 goto recur; 1671 } 1672 case BTF_KIND_FUNC_PROTO: { 1673 struct btf_param *local_p = btf_params(local_t); 1674 struct btf_param *targ_p = btf_params(targ_t); 1675 __u16 local_vlen = btf_vlen(local_t); 1676 __u16 targ_vlen = btf_vlen(targ_t); 1677 int i, err; 1678 1679 if (local_k != targ_k) 1680 return 0; 1681 1682 if (local_vlen != targ_vlen) 1683 return 0; 1684 1685 for (i = 0; i < local_vlen; i++, local_p++, targ_p++) { 1686 err = __bpf_core_types_match(local_btf, local_p->type, targ_btf, 1687 targ_p->type, behind_ptr, level - 1); 1688 if (err <= 0) 1689 return err; 1690 } 1691 1692 /* tail recurse for return type check */ 1693 local_id = local_t->type; 1694 targ_id = targ_t->type; 1695 goto recur; 1696 } 1697 default: 1698 pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", 1699 btf_kind_str(local_t), local_id, targ_id); 1700 return 0; 1701 } 1702 } 1703