1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Facebook */ 3 4 #include <uapi/linux/btf.h> 5 #include <uapi/linux/bpf.h> 6 #include <uapi/linux/bpf_perf_event.h> 7 #include <uapi/linux/types.h> 8 #include <linux/seq_file.h> 9 #include <linux/compiler.h> 10 #include <linux/ctype.h> 11 #include <linux/errno.h> 12 #include <linux/slab.h> 13 #include <linux/anon_inodes.h> 14 #include <linux/file.h> 15 #include <linux/uaccess.h> 16 #include <linux/kernel.h> 17 #include <linux/idr.h> 18 #include <linux/sort.h> 19 #include <linux/bpf_verifier.h> 20 #include <linux/btf.h> 21 #include <linux/btf_ids.h> 22 #include <linux/bpf.h> 23 #include <linux/bpf-cgroup.h> 24 #include <linux/bpf_lsm.h> 25 #include <linux/skmsg.h> 26 #include <linux/perf_event.h> 27 #include <linux/bsearch.h> 28 #include <linux/string.h> 29 #include <linux/overflow.h> 30 #include <linux/bitops.h> 31 #include <linux/unaligned.h> 32 #include <linux/vmalloc.h> 33 #include <crypto/sha2.h> 34 35 #include <net/netfilter/nf_bpf_link.h> 36 37 #include <net/sock.h> 38 #include <net/xdp.h> 39 #include "../tools/lib/bpf/relo_core.h" 40 41 /* BTF (BPF Type Format) is the meta data format which describes 42 * the data types of BPF program/map. Hence, it basically focus 43 * on the C programming language which the modern BPF is primary 44 * using. 45 * 46 * ELF Section: 47 * ~~~~~~~~~~~ 48 * The BTF data is stored under the ".BTF" ELF section 49 * 50 * struct btf_type: 51 * ~~~~~~~~~~~~~~~ 52 * Each 'struct btf_type' object describes a C data type. 53 * Depending on the type it is describing, a 'struct btf_type' 54 * object may be followed by more data. F.e. 55 * To describe an array, 'struct btf_type' is followed by 56 * 'struct btf_array'. 57 * 58 * 'struct btf_type' and any extra data following it are 59 * 4 bytes aligned. 60 * 61 * Type section: 62 * ~~~~~~~~~~~~~ 63 * The BTF type section contains a list of 'struct btf_type' objects. 64 * Each one describes a C type. Recall from the above section 65 * that a 'struct btf_type' object could be immediately followed by extra 66 * data in order to describe some particular C types. 67 * 68 * type_id: 69 * ~~~~~~~ 70 * Each btf_type object is identified by a type_id. The type_id 71 * is implicitly implied by the location of the btf_type object in 72 * the BTF type section. The first one has type_id 1. The second 73 * one has type_id 2...etc. Hence, an earlier btf_type has 74 * a smaller type_id. 75 * 76 * A btf_type object may refer to another btf_type object by using 77 * type_id (i.e. the "type" in the "struct btf_type"). 78 * 79 * NOTE that we cannot assume any reference-order. 80 * A btf_type object can refer to an earlier btf_type object 81 * but it can also refer to a later btf_type object. 82 * 83 * For example, to describe "const void *". A btf_type 84 * object describing "const" may refer to another btf_type 85 * object describing "void *". This type-reference is done 86 * by specifying type_id: 87 * 88 * [1] CONST (anon) type_id=2 89 * [2] PTR (anon) type_id=0 90 * 91 * The above is the btf_verifier debug log: 92 * - Each line started with "[?]" is a btf_type object 93 * - [?] is the type_id of the btf_type object. 94 * - CONST/PTR is the BTF_KIND_XXX 95 * - "(anon)" is the name of the type. It just 96 * happens that CONST and PTR has no name. 97 * - type_id=XXX is the 'u32 type' in btf_type 98 * 99 * NOTE: "void" has type_id 0 100 * 101 * String section: 102 * ~~~~~~~~~~~~~~ 103 * The BTF string section contains the names used by the type section. 104 * Each string is referred by an "offset" from the beginning of the 105 * string section. 106 * 107 * Each string is '\0' terminated. 108 * 109 * The first character in the string section must be '\0' 110 * which is used to mean 'anonymous'. Some btf_type may not 111 * have a name. 112 */ 113 114 /* BTF verification: 115 * 116 * To verify BTF data, two passes are needed. 117 * 118 * Pass #1 119 * ~~~~~~~ 120 * The first pass is to collect all btf_type objects to 121 * an array: "btf->types". 122 * 123 * Depending on the C type that a btf_type is describing, 124 * a btf_type may be followed by extra data. We don't know 125 * how many btf_type is there, and more importantly we don't 126 * know where each btf_type is located in the type section. 127 * 128 * Without knowing the location of each type_id, most verifications 129 * cannot be done. e.g. an earlier btf_type may refer to a later 130 * btf_type (recall the "const void *" above), so we cannot 131 * check this type-reference in the first pass. 132 * 133 * In the first pass, it still does some verifications (e.g. 134 * checking the name is a valid offset to the string section). 135 * 136 * Pass #2 137 * ~~~~~~~ 138 * The main focus is to resolve a btf_type that is referring 139 * to another type. 140 * 141 * We have to ensure the referring type: 142 * 1) does exist in the BTF (i.e. in btf->types[]) 143 * 2) does not cause a loop: 144 * struct A { 145 * struct B b; 146 * }; 147 * 148 * struct B { 149 * struct A a; 150 * }; 151 * 152 * btf_type_needs_resolve() decides if a btf_type needs 153 * to be resolved. 154 * 155 * The needs_resolve type implements the "resolve()" ops which 156 * essentially does a DFS and detects backedge. 157 * 158 * During resolve (or DFS), different C types have different 159 * "RESOLVED" conditions. 160 * 161 * When resolving a BTF_KIND_STRUCT, we need to resolve all its 162 * members because a member is always referring to another 163 * type. A struct's member can be treated as "RESOLVED" if 164 * it is referring to a BTF_KIND_PTR. Otherwise, the 165 * following valid C struct would be rejected: 166 * 167 * struct A { 168 * int m; 169 * struct A *a; 170 * }; 171 * 172 * When resolving a BTF_KIND_PTR, it needs to keep resolving if 173 * it is referring to another BTF_KIND_PTR. Otherwise, we cannot 174 * detect a pointer loop, e.g.: 175 * BTF_KIND_CONST -> BTF_KIND_PTR -> BTF_KIND_CONST -> BTF_KIND_PTR + 176 * ^ | 177 * +-----------------------------------------+ 178 * 179 */ 180 181 #define BITS_PER_U128 (sizeof(u64) * BITS_PER_BYTE * 2) 182 #define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1) 183 #define BITS_PER_BYTE_MASKED(bits) ((bits) & BITS_PER_BYTE_MASK) 184 #define BITS_ROUNDDOWN_BYTES(bits) ((bits) >> 3) 185 #define BITS_ROUNDUP_BYTES(bits) \ 186 (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits)) 187 188 #define BTF_INT_MASK 0x0fffffff 189 #define BTF_TYPE_ID_VALID(type_id) ((type_id) <= BTF_MAX_TYPE) 190 #define BTF_STR_OFFSET_VALID(name_off) ((name_off) <= BTF_MAX_NAME_OFFSET) 191 192 /* 16MB for 64k structs and each has 16 members and 193 * a few MB spaces for the string section. 194 * The hard limit is S32_MAX. 195 */ 196 #define BTF_MAX_SIZE (16 * 1024 * 1024) 197 198 #define for_each_member_from(i, from, struct_type, member) \ 199 for (i = from, member = btf_type_member(struct_type) + from; \ 200 i < btf_type_vlen(struct_type); \ 201 i++, member++) 202 203 #define for_each_vsi_from(i, from, struct_type, member) \ 204 for (i = from, member = btf_type_var_secinfo(struct_type) + from; \ 205 i < btf_type_vlen(struct_type); \ 206 i++, member++) 207 208 DEFINE_IDR(btf_idr); 209 DEFINE_SPINLOCK(btf_idr_lock); 210 211 enum btf_kfunc_hook { 212 BTF_KFUNC_HOOK_COMMON, 213 BTF_KFUNC_HOOK_XDP, 214 BTF_KFUNC_HOOK_TC, 215 BTF_KFUNC_HOOK_STRUCT_OPS, 216 BTF_KFUNC_HOOK_TRACING, 217 BTF_KFUNC_HOOK_SYSCALL, 218 BTF_KFUNC_HOOK_FMODRET, 219 BTF_KFUNC_HOOK_CGROUP, 220 BTF_KFUNC_HOOK_SCHED_ACT, 221 BTF_KFUNC_HOOK_SK_SKB, 222 BTF_KFUNC_HOOK_SOCKET_FILTER, 223 BTF_KFUNC_HOOK_LWT, 224 BTF_KFUNC_HOOK_NETFILTER, 225 BTF_KFUNC_HOOK_KPROBE, 226 BTF_KFUNC_HOOK_MAX, 227 }; 228 229 enum { 230 BTF_KFUNC_SET_MAX_CNT = 256, 231 BTF_DTOR_KFUNC_MAX_CNT = 256, 232 BTF_KFUNC_FILTER_MAX_CNT = 16, 233 }; 234 235 struct btf_kfunc_hook_filter { 236 btf_kfunc_filter_t filters[BTF_KFUNC_FILTER_MAX_CNT]; 237 u32 nr_filters; 238 }; 239 240 struct btf_kfunc_set_tab { 241 struct btf_id_set8 *sets[BTF_KFUNC_HOOK_MAX]; 242 struct btf_kfunc_hook_filter hook_filters[BTF_KFUNC_HOOK_MAX]; 243 }; 244 245 struct btf_id_dtor_kfunc_tab { 246 u32 cnt; 247 struct btf_id_dtor_kfunc dtors[]; 248 }; 249 250 struct btf_struct_ops_tab { 251 u32 cnt; 252 u32 capacity; 253 struct bpf_struct_ops_desc ops[]; 254 }; 255 256 struct btf { 257 void *data; 258 struct btf_type **types; 259 u32 *resolved_ids; 260 u32 *resolved_sizes; 261 const char *strings; 262 void *nohdr_data; 263 struct btf_header hdr; 264 u32 nr_types; /* includes VOID for base BTF */ 265 u32 named_start_id; 266 u32 types_size; 267 u32 data_size; 268 refcount_t refcnt; 269 u32 id; 270 struct rcu_head rcu; 271 struct btf_kfunc_set_tab *kfunc_set_tab; 272 struct btf_id_dtor_kfunc_tab *dtor_kfunc_tab; 273 struct btf_struct_metas *struct_meta_tab; 274 struct btf_struct_ops_tab *struct_ops_tab; 275 struct btf_layout *layout; 276 277 /* split BTF support */ 278 struct btf *base_btf; 279 u32 start_id; /* first type ID in this BTF (0 for base BTF) */ 280 u32 start_str_off; /* first string offset (0 for base BTF) */ 281 char name[MODULE_NAME_LEN]; 282 bool kernel_btf; 283 __u32 *base_id_map; /* map from distilled base BTF -> vmlinux BTF ids */ 284 }; 285 286 enum verifier_phase { 287 CHECK_META, 288 CHECK_TYPE, 289 }; 290 291 struct resolve_vertex { 292 const struct btf_type *t; 293 u32 type_id; 294 u32 next_member; 295 }; 296 297 enum visit_state { 298 NOT_VISITED, 299 VISITED, 300 RESOLVED, 301 }; 302 303 enum resolve_mode { 304 RESOLVE_TBD, /* To Be Determined */ 305 RESOLVE_PTR, /* Resolving for Pointer */ 306 RESOLVE_STRUCT_OR_ARRAY, /* Resolving for struct/union 307 * or array 308 */ 309 }; 310 311 #define MAX_RESOLVE_DEPTH 32 312 313 struct btf_sec_info { 314 u32 off; 315 u32 len; 316 }; 317 318 struct btf_verifier_env { 319 struct btf *btf; 320 u8 *visit_states; 321 struct resolve_vertex stack[MAX_RESOLVE_DEPTH]; 322 struct bpf_verifier_log log; 323 u32 log_type_id; 324 u32 top_stack; 325 enum verifier_phase phase; 326 enum resolve_mode resolve_mode; 327 }; 328 329 static const char * const btf_kind_str[NR_BTF_KINDS] = { 330 [BTF_KIND_UNKN] = "UNKNOWN", 331 [BTF_KIND_INT] = "INT", 332 [BTF_KIND_PTR] = "PTR", 333 [BTF_KIND_ARRAY] = "ARRAY", 334 [BTF_KIND_STRUCT] = "STRUCT", 335 [BTF_KIND_UNION] = "UNION", 336 [BTF_KIND_ENUM] = "ENUM", 337 [BTF_KIND_FWD] = "FWD", 338 [BTF_KIND_TYPEDEF] = "TYPEDEF", 339 [BTF_KIND_VOLATILE] = "VOLATILE", 340 [BTF_KIND_CONST] = "CONST", 341 [BTF_KIND_RESTRICT] = "RESTRICT", 342 [BTF_KIND_FUNC] = "FUNC", 343 [BTF_KIND_FUNC_PROTO] = "FUNC_PROTO", 344 [BTF_KIND_VAR] = "VAR", 345 [BTF_KIND_DATASEC] = "DATASEC", 346 [BTF_KIND_FLOAT] = "FLOAT", 347 [BTF_KIND_DECL_TAG] = "DECL_TAG", 348 [BTF_KIND_TYPE_TAG] = "TYPE_TAG", 349 [BTF_KIND_ENUM64] = "ENUM64", 350 [BTF_KIND_LOC_PARAM] = "LOC_PARAM", 351 [BTF_KIND_LOC_PROTO] = "LOC_PROTO", 352 [BTF_KIND_LOCSEC] = "LOCSEC", 353 }; 354 355 const char *btf_type_str(const struct btf_type *t) 356 { 357 return btf_kind_str[BTF_INFO_KIND(t->info)]; 358 } 359 360 /* Chunk size we use in safe copy of data to be shown. */ 361 #define BTF_SHOW_OBJ_SAFE_SIZE 32 362 363 /* 364 * This is the maximum size of a base type value (equivalent to a 365 * 128-bit int); if we are at the end of our safe buffer and have 366 * less than 16 bytes space we can't be assured of being able 367 * to copy the next type safely, so in such cases we will initiate 368 * a new copy. 369 */ 370 #define BTF_SHOW_OBJ_BASE_TYPE_SIZE 16 371 372 /* Type name size */ 373 #define BTF_SHOW_NAME_SIZE 80 374 375 /* 376 * The suffix of a type that indicates it cannot alias another type when 377 * comparing BTF IDs for kfunc invocations. 378 */ 379 #define NOCAST_ALIAS_SUFFIX "___init" 380 381 /* 382 * Common data to all BTF show operations. Private show functions can add 383 * their own data to a structure containing a struct btf_show and consult it 384 * in the show callback. See btf_type_show() below. 385 * 386 * One challenge with showing nested data is we want to skip 0-valued 387 * data, but in order to figure out whether a nested object is all zeros 388 * we need to walk through it. As a result, we need to make two passes 389 * when handling structs, unions and arrays; the first path simply looks 390 * for nonzero data, while the second actually does the display. The first 391 * pass is signalled by show->state.depth_check being set, and if we 392 * encounter a non-zero value we set show->state.depth_to_show to 393 * the depth at which we encountered it. When we have completed the 394 * first pass, we will know if anything needs to be displayed if 395 * depth_to_show > depth. See btf_[struct,array]_show() for the 396 * implementation of this. 397 * 398 * Another problem is we want to ensure the data for display is safe to 399 * access. To support this, the anonymous "struct {} obj" tracks the data 400 * object and our safe copy of it. We copy portions of the data needed 401 * to the object "copy" buffer, but because its size is limited to 402 * BTF_SHOW_OBJ_COPY_LEN bytes, multiple copies may be required as we 403 * traverse larger objects for display. 404 * 405 * The various data type show functions all start with a call to 406 * btf_show_start_type() which returns a pointer to the safe copy 407 * of the data needed (or if BTF_SHOW_UNSAFE is specified, to the 408 * raw data itself). btf_show_obj_safe() is responsible for 409 * using copy_from_kernel_nofault() to update the safe data if necessary 410 * as we traverse the object's data. skbuff-like semantics are 411 * used: 412 * 413 * - obj.head points to the start of the toplevel object for display 414 * - obj.size is the size of the toplevel object 415 * - obj.data points to the current point in the original data at 416 * which our safe data starts. obj.data will advance as we copy 417 * portions of the data. 418 * 419 * In most cases a single copy will suffice, but larger data structures 420 * such as "struct task_struct" will require many copies. The logic in 421 * btf_show_obj_safe() handles the logic that determines if a new 422 * copy_from_kernel_nofault() is needed. 423 */ 424 struct btf_show { 425 u64 flags; 426 void *target; /* target of show operation (seq file, buffer) */ 427 __printf(2, 0) void (*showfn)(struct btf_show *show, const char *fmt, va_list args); 428 const struct btf *btf; 429 /* below are used during iteration */ 430 struct { 431 u8 depth; 432 u8 depth_to_show; 433 u8 depth_check; 434 u8 array_member:1, 435 array_terminated:1; 436 u16 array_encoding; 437 u32 type_id; 438 int status; /* non-zero for error */ 439 const struct btf_type *type; 440 const struct btf_member *member; 441 char name[BTF_SHOW_NAME_SIZE]; /* space for member name/type */ 442 } state; 443 struct { 444 u32 size; 445 void *head; 446 void *data; 447 u8 safe[BTF_SHOW_OBJ_SAFE_SIZE]; 448 } obj; 449 }; 450 451 struct btf_kind_operations { 452 s32 (*check_meta)(struct btf_verifier_env *env, 453 const struct btf_type *t, 454 u32 meta_left); 455 int (*resolve)(struct btf_verifier_env *env, 456 const struct resolve_vertex *v); 457 int (*check_member)(struct btf_verifier_env *env, 458 const struct btf_type *struct_type, 459 const struct btf_member *member, 460 const struct btf_type *member_type); 461 int (*check_kflag_member)(struct btf_verifier_env *env, 462 const struct btf_type *struct_type, 463 const struct btf_member *member, 464 const struct btf_type *member_type); 465 void (*log_details)(struct btf_verifier_env *env, 466 const struct btf_type *t); 467 void (*show)(const struct btf *btf, const struct btf_type *t, 468 u32 type_id, void *data, u8 bits_offsets, 469 struct btf_show *show); 470 }; 471 472 static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS]; 473 static struct btf_type btf_void; 474 475 static int btf_resolve(struct btf_verifier_env *env, 476 const struct btf_type *t, u32 type_id); 477 478 static int btf_func_check(struct btf_verifier_env *env, 479 const struct btf_type *t); 480 481 static bool btf_type_is_modifier(const struct btf_type *t) 482 { 483 /* Some of them is not strictly a C modifier 484 * but they are grouped into the same bucket 485 * for BTF concern: 486 * A type (t) that refers to another 487 * type through t->type AND its size cannot 488 * be determined without following the t->type. 489 * 490 * ptr does not fall into this bucket 491 * because its size is always sizeof(void *). 492 */ 493 switch (BTF_INFO_KIND(t->info)) { 494 case BTF_KIND_TYPEDEF: 495 case BTF_KIND_VOLATILE: 496 case BTF_KIND_CONST: 497 case BTF_KIND_RESTRICT: 498 case BTF_KIND_TYPE_TAG: 499 return true; 500 } 501 502 return false; 503 } 504 505 static int btf_start_id(const struct btf *btf) 506 { 507 return btf->start_id + (btf->base_btf ? 0 : 1); 508 } 509 510 bool btf_type_is_void(const struct btf_type *t) 511 { 512 return t == &btf_void; 513 } 514 515 static bool btf_type_is_datasec(const struct btf_type *t) 516 { 517 return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC; 518 } 519 520 static bool btf_type_is_decl_tag(const struct btf_type *t) 521 { 522 return BTF_INFO_KIND(t->info) == BTF_KIND_DECL_TAG; 523 } 524 525 static bool btf_type_is_loc_param(const struct btf_type *t) 526 { 527 return BTF_INFO_KIND(t->info) == BTF_KIND_LOC_PARAM; 528 } 529 530 static bool btf_type_is_loc_proto(const struct btf_type *t) 531 { 532 return BTF_INFO_KIND(t->info) == BTF_KIND_LOC_PROTO; 533 } 534 535 static bool btf_type_is_locsec(const struct btf_type *t) 536 { 537 return BTF_INFO_KIND(t->info) == BTF_KIND_LOCSEC; 538 } 539 540 static bool btf_type_nosize(const struct btf_type *t) 541 { 542 return btf_type_is_void(t) || btf_type_is_fwd(t) || 543 btf_type_is_func(t) || btf_type_is_func_proto(t) || 544 btf_type_is_decl_tag(t) || btf_type_is_loc_param(t) || 545 btf_type_is_loc_proto(t) || btf_type_is_locsec(t); 546 } 547 548 static bool btf_type_nosize_or_null(const struct btf_type *t) 549 { 550 return !t || btf_type_nosize(t); 551 } 552 553 static bool btf_type_is_decl_tag_target(const struct btf_type *t) 554 { 555 return btf_type_is_func(t) || btf_type_is_struct(t) || 556 btf_type_is_var(t) || btf_type_is_typedef(t); 557 } 558 559 bool btf_is_vmlinux(const struct btf *btf) 560 { 561 return btf->kernel_btf && !btf->base_btf; 562 } 563 564 u32 btf_nr_types(const struct btf *btf) 565 { 566 u32 total = 0; 567 568 while (btf) { 569 total += btf->nr_types; 570 btf = btf->base_btf; 571 } 572 573 return total; 574 } 575 576 /* 577 * Note that vmlinux and kernel module BTFs are always sorted 578 * during the building phase. 579 */ 580 static void btf_check_sorted(struct btf *btf) 581 { 582 u32 i, n, named_start_id = 0; 583 584 n = btf_nr_types(btf); 585 if (btf_is_vmlinux(btf)) { 586 for (i = btf_start_id(btf); i < n; i++) { 587 const struct btf_type *t = btf_type_by_id(btf, i); 588 const char *n = btf_name_by_offset(btf, t->name_off); 589 590 if (n[0] != '\0') { 591 btf->named_start_id = i; 592 return; 593 } 594 } 595 return; 596 } 597 598 for (i = btf_start_id(btf) + 1; i < n; i++) { 599 const struct btf_type *ta = btf_type_by_id(btf, i - 1); 600 const struct btf_type *tb = btf_type_by_id(btf, i); 601 const char *na = btf_name_by_offset(btf, ta->name_off); 602 const char *nb = btf_name_by_offset(btf, tb->name_off); 603 604 if (strcmp(na, nb) > 0) 605 return; 606 607 if (named_start_id == 0 && na[0] != '\0') 608 named_start_id = i - 1; 609 if (named_start_id == 0 && nb[0] != '\0') 610 named_start_id = i; 611 } 612 613 if (named_start_id) 614 btf->named_start_id = named_start_id; 615 } 616 617 /* 618 * btf_named_start_id - Get the named starting ID for the BTF 619 * @btf: Pointer to the target BTF object 620 * @own: Flag indicating whether to query only the current BTF (true = current BTF only, 621 * false = recursively traverse the base BTF chain) 622 * 623 * Return value rules: 624 * 1. For a sorted btf, return its named_start_id 625 * 2. Else for a split BTF, return its start_id 626 * 3. Else for a base BTF, return 1 627 */ 628 u32 btf_named_start_id(const struct btf *btf, bool own) 629 { 630 const struct btf *base_btf = btf; 631 632 while (!own && base_btf->base_btf) 633 base_btf = base_btf->base_btf; 634 635 return base_btf->named_start_id ?: (base_btf->start_id ?: 1); 636 } 637 638 static s32 btf_find_by_name_kind_bsearch(const struct btf *btf, const char *name) 639 { 640 const struct btf_type *t; 641 const char *tname; 642 s32 l, r, m; 643 644 l = btf_named_start_id(btf, true); 645 r = btf_nr_types(btf) - 1; 646 while (l <= r) { 647 m = l + (r - l) / 2; 648 t = btf_type_by_id(btf, m); 649 tname = btf_name_by_offset(btf, t->name_off); 650 if (strcmp(tname, name) >= 0) { 651 if (l == r) 652 return r; 653 r = m; 654 } else { 655 l = m + 1; 656 } 657 } 658 659 return btf_nr_types(btf); 660 } 661 662 s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind) 663 { 664 const struct btf *base_btf = btf_base_btf(btf); 665 const struct btf_type *t; 666 const char *tname; 667 s32 id, total; 668 669 if (base_btf) { 670 id = btf_find_by_name_kind(base_btf, name, kind); 671 if (id > 0) 672 return id; 673 } 674 675 total = btf_nr_types(btf); 676 if (btf->named_start_id > 0 && name[0]) { 677 id = btf_find_by_name_kind_bsearch(btf, name); 678 for (; id < total; id++) { 679 t = btf_type_by_id(btf, id); 680 tname = btf_name_by_offset(btf, t->name_off); 681 if (strcmp(tname, name) != 0) 682 return -ENOENT; 683 if (BTF_INFO_KIND(t->info) == kind) 684 return id; 685 } 686 } else { 687 for (id = btf_start_id(btf); id < total; id++) { 688 t = btf_type_by_id(btf, id); 689 if (BTF_INFO_KIND(t->info) != kind) 690 continue; 691 tname = btf_name_by_offset(btf, t->name_off); 692 if (strcmp(tname, name) == 0) 693 return id; 694 } 695 } 696 697 return -ENOENT; 698 } 699 700 s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p) 701 { 702 struct btf *btf; 703 s32 ret; 704 int id; 705 706 btf = bpf_get_btf_vmlinux(); 707 if (IS_ERR(btf)) 708 return PTR_ERR(btf); 709 if (!btf) 710 return -EINVAL; 711 712 ret = btf_find_by_name_kind(btf, name, kind); 713 /* ret is never zero, since btf_find_by_name_kind returns 714 * positive btf_id or negative error. 715 */ 716 if (ret > 0) { 717 btf_get(btf); 718 *btf_p = btf; 719 return ret; 720 } 721 722 /* If name is not found in vmlinux's BTF then search in module's BTFs */ 723 spin_lock_bh(&btf_idr_lock); 724 idr_for_each_entry(&btf_idr, btf, id) { 725 if (!btf_is_module(btf)) 726 continue; 727 /* linear search could be slow hence unlock/lock 728 * the IDR to avoiding holding it for too long 729 */ 730 btf_get(btf); 731 spin_unlock_bh(&btf_idr_lock); 732 ret = btf_find_by_name_kind(btf, name, kind); 733 if (ret > 0) { 734 *btf_p = btf; 735 return ret; 736 } 737 btf_put(btf); 738 spin_lock_bh(&btf_idr_lock); 739 } 740 spin_unlock_bh(&btf_idr_lock); 741 return ret; 742 } 743 EXPORT_SYMBOL_GPL(bpf_find_btf_id); 744 745 const struct btf_type *btf_type_skip_modifiers(const struct btf *btf, 746 u32 id, u32 *res_id) 747 { 748 const struct btf_type *t = btf_type_by_id(btf, id); 749 750 while (btf_type_is_modifier(t)) { 751 id = t->type; 752 t = btf_type_by_id(btf, t->type); 753 } 754 755 if (res_id) 756 *res_id = id; 757 758 return t; 759 } 760 761 const struct btf_type *btf_type_resolve_ptr(const struct btf *btf, 762 u32 id, u32 *res_id) 763 { 764 const struct btf_type *t; 765 766 t = btf_type_skip_modifiers(btf, id, NULL); 767 if (!btf_type_is_ptr(t)) 768 return NULL; 769 770 return btf_type_skip_modifiers(btf, t->type, res_id); 771 } 772 773 const struct btf_type *btf_type_resolve_func_ptr(const struct btf *btf, 774 u32 id, u32 *res_id) 775 { 776 const struct btf_type *ptype; 777 778 ptype = btf_type_resolve_ptr(btf, id, res_id); 779 if (ptype && btf_type_is_func_proto(ptype)) 780 return ptype; 781 782 return NULL; 783 } 784 785 /* Types that act only as a source, not sink or intermediate 786 * type when resolving. 787 */ 788 static bool btf_type_is_resolve_source_only(const struct btf_type *t) 789 { 790 return btf_type_is_var(t) || 791 btf_type_is_decl_tag(t) || 792 btf_type_is_datasec(t) || 793 btf_type_is_loc_proto(t) || 794 btf_type_is_locsec(t); 795 } 796 797 /* What types need to be resolved? 798 * 799 * btf_type_is_modifier() is an obvious one. 800 * 801 * btf_type_is_struct() because its member refers to 802 * another type (through member->type). 803 * 804 * btf_type_is_var() because the variable refers to 805 * another type. btf_type_is_datasec() holds multiple 806 * btf_type_is_var() types that need resolving. 807 * 808 * btf_type_is_array() because its element (array->type) 809 * refers to another type. Array can be thought of a 810 * special case of struct while array just has the same 811 * member-type repeated by array->nelems of times. 812 */ 813 static bool btf_type_needs_resolve(const struct btf_type *t) 814 { 815 return btf_type_is_modifier(t) || 816 btf_type_is_ptr(t) || 817 btf_type_is_struct(t) || 818 btf_type_is_array(t) || 819 btf_type_is_var(t) || 820 btf_type_is_func(t) || 821 btf_type_is_decl_tag(t) || 822 btf_type_is_datasec(t) || 823 btf_type_is_loc_proto(t) || 824 btf_type_is_locsec(t); 825 } 826 827 /* t->size can be used */ 828 static bool btf_type_has_size(const struct btf_type *t) 829 { 830 switch (BTF_INFO_KIND(t->info)) { 831 case BTF_KIND_INT: 832 case BTF_KIND_STRUCT: 833 case BTF_KIND_UNION: 834 case BTF_KIND_ENUM: 835 case BTF_KIND_DATASEC: 836 case BTF_KIND_FLOAT: 837 case BTF_KIND_ENUM64: 838 return true; 839 } 840 841 return false; 842 } 843 844 static const char *btf_int_encoding_str(u8 encoding) 845 { 846 if (encoding == 0) 847 return "(none)"; 848 else if (encoding == BTF_INT_SIGNED) 849 return "SIGNED"; 850 else if (encoding == BTF_INT_CHAR) 851 return "CHAR"; 852 else if (encoding == BTF_INT_BOOL) 853 return "BOOL"; 854 else 855 return "UNKN"; 856 } 857 858 static u32 btf_type_int(const struct btf_type *t) 859 { 860 return *(u32 *)(t + 1); 861 } 862 863 static const struct btf_array *btf_type_array(const struct btf_type *t) 864 { 865 return (const struct btf_array *)(t + 1); 866 } 867 868 static const struct btf_enum *btf_type_enum(const struct btf_type *t) 869 { 870 return (const struct btf_enum *)(t + 1); 871 } 872 873 static const struct btf_var *btf_type_var(const struct btf_type *t) 874 { 875 return (const struct btf_var *)(t + 1); 876 } 877 878 static const struct btf_decl_tag *btf_type_decl_tag(const struct btf_type *t) 879 { 880 return (const struct btf_decl_tag *)(t + 1); 881 } 882 883 static const struct btf_enum64 *btf_type_enum64(const struct btf_type *t) 884 { 885 return (const struct btf_enum64 *)(t + 1); 886 } 887 888 static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t) 889 { 890 return kind_ops[BTF_INFO_KIND(t->info)]; 891 } 892 893 static bool btf_name_offset_valid(const struct btf *btf, u32 offset) 894 { 895 if (!BTF_STR_OFFSET_VALID(offset)) 896 return false; 897 898 while (offset < btf->start_str_off) 899 btf = btf->base_btf; 900 901 offset -= btf->start_str_off; 902 return offset < btf->hdr.str_len; 903 } 904 905 const char *btf_str_by_offset(const struct btf *btf, u32 offset) 906 { 907 while (offset < btf->start_str_off) 908 btf = btf->base_btf; 909 910 offset -= btf->start_str_off; 911 if (offset < btf->hdr.str_len) 912 return &btf->strings[offset]; 913 914 return NULL; 915 } 916 917 /* Names in BTF of Rust are not C identifiers. Allow any printable character */ 918 static bool btf_name_valid_identifier(const struct btf *btf, u32 offset) 919 { 920 /* offset must be valid */ 921 const char *src = btf_str_by_offset(btf, offset); 922 const char *src_limit; 923 924 if (!isprint(*src)) 925 return false; 926 927 /* set a limit on identifier length */ 928 src_limit = src + KSYM_NAME_LEN; 929 src++; 930 while (*src && src < src_limit) { 931 if (!isprint(*src)) 932 return false; 933 src++; 934 } 935 936 return !*src; 937 } 938 939 /* Allow any printable character in DATASEC names */ 940 static bool btf_name_valid_section(const struct btf *btf, u32 offset) 941 { 942 /* offset must be valid */ 943 const char *src = btf_str_by_offset(btf, offset); 944 const char *src_limit; 945 946 if (!*src) 947 return false; 948 949 /* set a limit on identifier length */ 950 src_limit = src + KSYM_NAME_LEN; 951 while (*src && src < src_limit) { 952 if (!isprint(*src)) 953 return false; 954 src++; 955 } 956 957 return !*src; 958 } 959 960 static const char *__btf_name_by_offset(const struct btf *btf, u32 offset) 961 { 962 const char *name; 963 964 if (!offset) 965 return "(anon)"; 966 967 name = btf_str_by_offset(btf, offset); 968 return name ?: "(invalid-name-offset)"; 969 } 970 971 const char *btf_name_by_offset(const struct btf *btf, u32 offset) 972 { 973 return btf_str_by_offset(btf, offset); 974 } 975 976 const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id) 977 { 978 while (type_id < btf->start_id) 979 btf = btf->base_btf; 980 981 type_id -= btf->start_id; 982 if (type_id >= btf->nr_types) 983 return NULL; 984 return btf->types[type_id]; 985 } 986 EXPORT_SYMBOL_GPL(btf_type_by_id); 987 988 /* 989 * Check that the type @t is a regular int. This means that @t is not 990 * a bit field and it has the same size as either of u8/u16/u32/u64 991 * or __int128. If @expected_size is not zero, then size of @t should 992 * be the same. A caller should already have checked that the type @t 993 * is an integer. 994 */ 995 static bool __btf_type_int_is_regular(const struct btf_type *t, size_t expected_size) 996 { 997 u32 int_data = btf_type_int(t); 998 u8 nr_bits = BTF_INT_BITS(int_data); 999 u8 nr_bytes = BITS_ROUNDUP_BYTES(nr_bits); 1000 1001 return BITS_PER_BYTE_MASKED(nr_bits) == 0 && 1002 BTF_INT_OFFSET(int_data) == 0 && 1003 (nr_bytes <= 16 && is_power_of_2(nr_bytes)) && 1004 (expected_size == 0 || nr_bytes == expected_size); 1005 } 1006 1007 static bool btf_type_int_is_regular(const struct btf_type *t) 1008 { 1009 return __btf_type_int_is_regular(t, 0); 1010 } 1011 1012 bool btf_type_is_i32(const struct btf_type *t) 1013 { 1014 return btf_type_is_int(t) && __btf_type_int_is_regular(t, 4); 1015 } 1016 1017 bool btf_type_is_i64(const struct btf_type *t) 1018 { 1019 return btf_type_is_int(t) && __btf_type_int_is_regular(t, 8); 1020 } 1021 1022 bool btf_type_is_primitive(const struct btf_type *t) 1023 { 1024 return (btf_type_is_int(t) && btf_type_int_is_regular(t)) || 1025 btf_is_any_enum(t); 1026 } 1027 1028 /* 1029 * Check that given struct member is a regular int with expected 1030 * offset and size. 1031 */ 1032 bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s, 1033 const struct btf_member *m, 1034 u32 expected_offset, u32 expected_size) 1035 { 1036 const struct btf_type *t; 1037 u32 id, int_data; 1038 u8 nr_bits; 1039 1040 id = m->type; 1041 t = btf_type_id_size(btf, &id, NULL); 1042 if (!t || !btf_type_is_int(t)) 1043 return false; 1044 1045 int_data = btf_type_int(t); 1046 nr_bits = BTF_INT_BITS(int_data); 1047 if (btf_type_kflag(s)) { 1048 u32 bitfield_size = BTF_MEMBER_BITFIELD_SIZE(m->offset); 1049 u32 bit_offset = BTF_MEMBER_BIT_OFFSET(m->offset); 1050 1051 /* if kflag set, int should be a regular int and 1052 * bit offset should be at byte boundary. 1053 */ 1054 return !bitfield_size && 1055 BITS_ROUNDUP_BYTES(bit_offset) == expected_offset && 1056 BITS_ROUNDUP_BYTES(nr_bits) == expected_size; 1057 } 1058 1059 if (BTF_INT_OFFSET(int_data) || 1060 BITS_PER_BYTE_MASKED(m->offset) || 1061 BITS_ROUNDUP_BYTES(m->offset) != expected_offset || 1062 BITS_PER_BYTE_MASKED(nr_bits) || 1063 BITS_ROUNDUP_BYTES(nr_bits) != expected_size) 1064 return false; 1065 1066 return true; 1067 } 1068 1069 /* Similar to btf_type_skip_modifiers() but does not skip typedefs. */ 1070 static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf, 1071 u32 id) 1072 { 1073 const struct btf_type *t = btf_type_by_id(btf, id); 1074 1075 while (btf_type_is_modifier(t) && 1076 BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF) { 1077 t = btf_type_by_id(btf, t->type); 1078 } 1079 1080 return t; 1081 } 1082 1083 #define BTF_SHOW_MAX_ITER 10 1084 /* nested structs and arrays shown; __btf_show_indent() has as many columns */ 1085 #define BTF_SHOW_MAX_DEPTH 32 1086 1087 #define BTF_KIND_BIT(kind) (1ULL << kind) 1088 1089 /* 1090 * Populate show->state.name with type name information. 1091 * Format of type name is 1092 * 1093 * [.member_name = ] (type_name) 1094 */ 1095 static const char *btf_show_name(struct btf_show *show) 1096 { 1097 /* BTF_MAX_ITER array suffixes "[]" */ 1098 const char *array_suffixes = "[][][][][][][][][][]"; 1099 const char *array_suffix = &array_suffixes[strlen(array_suffixes)]; 1100 /* BTF_MAX_ITER pointer suffixes "*" */ 1101 const char *ptr_suffixes = "**********"; 1102 const char *ptr_suffix = &ptr_suffixes[strlen(ptr_suffixes)]; 1103 const char *name = NULL, *prefix = "", *parens = ""; 1104 const struct btf_member *m = show->state.member; 1105 const struct btf_type *t; 1106 const struct btf_array *array; 1107 u32 id = show->state.type_id; 1108 const char *member = NULL; 1109 bool show_member = false; 1110 u64 kinds = 0; 1111 int i; 1112 1113 show->state.name[0] = '\0'; 1114 1115 /* 1116 * Don't show type name if we're showing an array member; 1117 * in that case we show the array type so don't need to repeat 1118 * ourselves for each member. 1119 */ 1120 if (show->state.array_member) 1121 return ""; 1122 1123 /* Retrieve member name, if any. */ 1124 if (m) { 1125 member = btf_name_by_offset(show->btf, m->name_off); 1126 show_member = strlen(member) > 0; 1127 id = m->type; 1128 } 1129 1130 /* 1131 * Start with type_id, as we have resolved the struct btf_type * 1132 * via btf_modifier_show() past the parent typedef to the child 1133 * struct, int etc it is defined as. In such cases, the type_id 1134 * still represents the starting type while the struct btf_type * 1135 * in our show->state points at the resolved type of the typedef. 1136 */ 1137 t = btf_type_by_id(show->btf, id); 1138 if (!t) 1139 return ""; 1140 1141 /* 1142 * The goal here is to build up the right number of pointer and 1143 * array suffixes while ensuring the type name for a typedef 1144 * is represented. Along the way we accumulate a list of 1145 * BTF kinds we have encountered, since these will inform later 1146 * display; for example, pointer types will not require an 1147 * opening "{" for struct, we will just display the pointer value. 1148 * 1149 * We also want to accumulate the right number of pointer or array 1150 * indices in the format string while iterating until we get to 1151 * the typedef/pointee/array member target type. 1152 * 1153 * We start by pointing at the end of pointer and array suffix 1154 * strings; as we accumulate pointers and arrays we move the pointer 1155 * or array string backwards so it will show the expected number of 1156 * '*' or '[]' for the type. BTF_SHOW_MAX_ITER of nesting of pointers 1157 * and/or arrays and typedefs are supported as a precaution. 1158 * 1159 * We also want to get typedef name while proceeding to resolve 1160 * type it points to so that we can add parentheses if it is a 1161 * "typedef struct" etc. 1162 */ 1163 for (i = 0; i < BTF_SHOW_MAX_ITER; i++) { 1164 1165 switch (BTF_INFO_KIND(t->info)) { 1166 case BTF_KIND_TYPEDEF: 1167 if (!name) 1168 name = btf_name_by_offset(show->btf, 1169 t->name_off); 1170 kinds |= BTF_KIND_BIT(BTF_KIND_TYPEDEF); 1171 id = t->type; 1172 break; 1173 case BTF_KIND_ARRAY: 1174 kinds |= BTF_KIND_BIT(BTF_KIND_ARRAY); 1175 parens = "["; 1176 if (!t) 1177 return ""; 1178 array = btf_type_array(t); 1179 if (array_suffix > array_suffixes) 1180 array_suffix -= 2; 1181 id = array->type; 1182 break; 1183 case BTF_KIND_PTR: 1184 kinds |= BTF_KIND_BIT(BTF_KIND_PTR); 1185 if (ptr_suffix > ptr_suffixes) 1186 ptr_suffix -= 1; 1187 id = t->type; 1188 break; 1189 default: 1190 goto resolved; 1191 } 1192 t = btf_type_skip_qualifiers(show->btf, id); 1193 if (!id) 1194 break; 1195 } 1196 /* We may not be able to represent this type; bail to be safe */ 1197 if (i == BTF_SHOW_MAX_ITER) 1198 return ""; 1199 1200 resolved: 1201 if (!name) 1202 name = btf_type_is_void(t) ? "void" : btf_name_by_offset(show->btf, t->name_off); 1203 1204 switch (BTF_INFO_KIND(t->info)) { 1205 case BTF_KIND_STRUCT: 1206 case BTF_KIND_UNION: 1207 prefix = BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT ? 1208 "struct" : "union"; 1209 /* if it's an array of struct/union, parens is already set */ 1210 if (!(kinds & (BTF_KIND_BIT(BTF_KIND_ARRAY)))) 1211 parens = "{"; 1212 break; 1213 case BTF_KIND_ENUM: 1214 case BTF_KIND_ENUM64: 1215 prefix = "enum"; 1216 break; 1217 default: 1218 break; 1219 } 1220 1221 /* pointer does not require parens */ 1222 if (kinds & BTF_KIND_BIT(BTF_KIND_PTR)) 1223 parens = ""; 1224 /* typedef does not require struct/union/enum prefix */ 1225 if (kinds & BTF_KIND_BIT(BTF_KIND_TYPEDEF)) 1226 prefix = ""; 1227 1228 if (!name) 1229 name = ""; 1230 1231 /* Even if we don't want type name info, we want parentheses etc */ 1232 if (show->flags & BTF_SHOW_NONAME) 1233 snprintf(show->state.name, sizeof(show->state.name), "%s", 1234 parens); 1235 else 1236 snprintf(show->state.name, sizeof(show->state.name), 1237 "%s%s%s(%s%s%s%s%s%s)%s", 1238 /* first 3 strings comprise ".member = " */ 1239 show_member ? "." : "", 1240 show_member ? member : "", 1241 show_member ? " = " : "", 1242 /* ...next is our prefix (struct, enum, etc) */ 1243 prefix, 1244 strlen(prefix) > 0 && strlen(name) > 0 ? " " : "", 1245 /* ...this is the type name itself */ 1246 name, 1247 /* ...suffixed by the appropriate '*', '[]' suffixes */ 1248 strlen(ptr_suffix) > 0 ? " " : "", ptr_suffix, 1249 array_suffix, parens); 1250 1251 return show->state.name; 1252 } 1253 1254 static const char *__btf_show_indent(struct btf_show *show) 1255 { 1256 const char *indents = " "; 1257 const char *indent = &indents[strlen(indents)]; 1258 1259 if ((indent - show->state.depth) >= indents) 1260 return indent - show->state.depth; 1261 return indents; 1262 } 1263 1264 static const char *btf_show_indent(struct btf_show *show) 1265 { 1266 return show->flags & BTF_SHOW_COMPACT ? "" : __btf_show_indent(show); 1267 } 1268 1269 static const char *btf_show_newline(struct btf_show *show) 1270 { 1271 return show->flags & BTF_SHOW_COMPACT ? "" : "\n"; 1272 } 1273 1274 static const char *btf_show_delim(struct btf_show *show) 1275 { 1276 if (show->state.depth == 0) 1277 return ""; 1278 1279 if ((show->flags & BTF_SHOW_COMPACT) && show->state.type && 1280 BTF_INFO_KIND(show->state.type->info) == BTF_KIND_UNION) 1281 return "|"; 1282 1283 return ","; 1284 } 1285 1286 __printf(2, 3) static void btf_show(struct btf_show *show, const char *fmt, ...) 1287 { 1288 va_list args; 1289 1290 if (!show->state.depth_check) { 1291 va_start(args, fmt); 1292 show->showfn(show, fmt, args); 1293 va_end(args); 1294 } 1295 } 1296 1297 /* Macros are used here as btf_show_type_value[s]() prepends and appends 1298 * format specifiers to the format specifier passed in; these do the work of 1299 * adding indentation, delimiters etc while the caller simply has to specify 1300 * the type value(s) in the format specifier + value(s). 1301 */ 1302 #define btf_show_type_value(show, fmt, value) \ 1303 do { \ 1304 if ((value) != (__typeof__(value))0 || \ 1305 (show->flags & BTF_SHOW_ZERO) || \ 1306 show->state.depth == 0) { \ 1307 btf_show(show, "%s%s" fmt "%s%s", \ 1308 btf_show_indent(show), \ 1309 btf_show_name(show), \ 1310 value, btf_show_delim(show), \ 1311 btf_show_newline(show)); \ 1312 if (show->state.depth > show->state.depth_to_show) \ 1313 show->state.depth_to_show = show->state.depth; \ 1314 } \ 1315 } while (0) 1316 1317 #define btf_show_type_values(show, fmt, ...) \ 1318 do { \ 1319 btf_show(show, "%s%s" fmt "%s%s", btf_show_indent(show), \ 1320 btf_show_name(show), \ 1321 __VA_ARGS__, btf_show_delim(show), \ 1322 btf_show_newline(show)); \ 1323 if (show->state.depth > show->state.depth_to_show) \ 1324 show->state.depth_to_show = show->state.depth; \ 1325 } while (0) 1326 1327 /* How much is left to copy to safe buffer after @data? */ 1328 static int btf_show_obj_size_left(struct btf_show *show, void *data) 1329 { 1330 return show->obj.head + show->obj.size - data; 1331 } 1332 1333 /* Is object pointed to by @data of @size already copied to our safe buffer? */ 1334 static bool btf_show_obj_is_safe(struct btf_show *show, void *data, int size) 1335 { 1336 return data >= show->obj.data && 1337 (data + size) < (show->obj.data + BTF_SHOW_OBJ_SAFE_SIZE); 1338 } 1339 1340 /* 1341 * If object pointed to by @data of @size falls within our safe buffer, return 1342 * the equivalent pointer to the same safe data. Assumes 1343 * copy_from_kernel_nofault() has already happened and our safe buffer is 1344 * populated. 1345 */ 1346 static void *__btf_show_obj_safe(struct btf_show *show, void *data, int size) 1347 { 1348 if (btf_show_obj_is_safe(show, data, size)) 1349 return show->obj.safe + (data - show->obj.data); 1350 return NULL; 1351 } 1352 1353 /* 1354 * Return a safe-to-access version of data pointed to by @data. 1355 * We do this by copying the relevant amount of information 1356 * to the struct btf_show obj.safe buffer using copy_from_kernel_nofault(). 1357 * 1358 * If BTF_SHOW_UNSAFE is specified, just return data as-is; no 1359 * safe copy is needed. 1360 * 1361 * Otherwise we need to determine if we have the required amount 1362 * of data (determined by the @data pointer and the size of the 1363 * largest base type we can encounter (represented by 1364 * BTF_SHOW_OBJ_BASE_TYPE_SIZE). Having that much data ensures 1365 * that we will be able to print some of the current object, 1366 * and if more is needed a copy will be triggered. 1367 * Some objects such as structs will not fit into the buffer; 1368 * in such cases additional copies when we iterate over their 1369 * members may be needed. 1370 * 1371 * btf_show_obj_safe() is used to return a safe buffer for 1372 * btf_show_start_type(); this ensures that as we recurse into 1373 * nested types we always have safe data for the given type. 1374 * This approach is somewhat wasteful; it's possible for example 1375 * that when iterating over a large union we'll end up copying the 1376 * same data repeatedly, but the goal is safety not performance. 1377 * We use stack data as opposed to per-CPU buffers because the 1378 * iteration over a type can take some time, and preemption handling 1379 * would greatly complicate use of the safe buffer. 1380 */ 1381 static void *btf_show_obj_safe(struct btf_show *show, 1382 const struct btf_type *t, 1383 void *data) 1384 { 1385 const struct btf_type *rt; 1386 int size_left, size; 1387 void *safe = NULL; 1388 1389 if (show->flags & BTF_SHOW_UNSAFE) 1390 return data; 1391 1392 rt = btf_resolve_size(show->btf, t, &size); 1393 if (IS_ERR(rt)) { 1394 show->state.status = PTR_ERR(rt); 1395 return NULL; 1396 } 1397 1398 /* 1399 * Is this toplevel object? If so, set total object size and 1400 * initialize pointers. Otherwise check if we still fall within 1401 * our safe object data. 1402 */ 1403 if (show->state.depth == 0) { 1404 show->obj.size = size; 1405 show->obj.head = data; 1406 } else { 1407 /* 1408 * If the size of the current object is > our remaining 1409 * safe buffer we _may_ need to do a new copy. However 1410 * consider the case of a nested struct; it's size pushes 1411 * us over the safe buffer limit, but showing any individual 1412 * struct members does not. In such cases, we don't need 1413 * to initiate a fresh copy yet; however we definitely need 1414 * at least BTF_SHOW_OBJ_BASE_TYPE_SIZE bytes left 1415 * in our buffer, regardless of the current object size. 1416 * The logic here is that as we resolve types we will 1417 * hit a base type at some point, and we need to be sure 1418 * the next chunk of data is safely available to display 1419 * that type info safely. We cannot rely on the size of 1420 * the current object here because it may be much larger 1421 * than our current buffer (e.g. task_struct is 8k). 1422 * All we want to do here is ensure that we can print the 1423 * next basic type, which we can if either 1424 * - the current type size is within the safe buffer; or 1425 * - at least BTF_SHOW_OBJ_BASE_TYPE_SIZE bytes are left in 1426 * the safe buffer. 1427 */ 1428 safe = __btf_show_obj_safe(show, data, 1429 min(size, 1430 BTF_SHOW_OBJ_BASE_TYPE_SIZE)); 1431 } 1432 1433 /* 1434 * We need a new copy to our safe object, either because we haven't 1435 * yet copied and are initializing safe data, or because the data 1436 * we want falls outside the boundaries of the safe object. 1437 */ 1438 if (!safe) { 1439 size_left = btf_show_obj_size_left(show, data); 1440 if (size_left > BTF_SHOW_OBJ_SAFE_SIZE) 1441 size_left = BTF_SHOW_OBJ_SAFE_SIZE; 1442 show->state.status = copy_from_kernel_nofault(show->obj.safe, 1443 data, size_left); 1444 if (!show->state.status) { 1445 show->obj.data = data; 1446 safe = show->obj.safe; 1447 } 1448 } 1449 1450 return safe; 1451 } 1452 1453 /* 1454 * Set the type we are starting to show and return a safe data pointer 1455 * to be used for showing the associated data. 1456 */ 1457 static void *btf_show_start_type(struct btf_show *show, 1458 const struct btf_type *t, 1459 u32 type_id, void *data) 1460 { 1461 show->state.type = t; 1462 show->state.type_id = type_id; 1463 show->state.name[0] = '\0'; 1464 1465 return btf_show_obj_safe(show, t, data); 1466 } 1467 1468 static void btf_show_end_type(struct btf_show *show) 1469 { 1470 show->state.type = NULL; 1471 show->state.type_id = 0; 1472 show->state.name[0] = '\0'; 1473 } 1474 1475 static void *btf_show_start_aggr_type(struct btf_show *show, 1476 const struct btf_type *t, 1477 u32 type_id, void *data) 1478 { 1479 void *safe_data; 1480 1481 /* 1482 * Structs and arrays are shown by recursion. BTF loading bounds the 1483 * depth of one resolve walk, not how deep resolved types may nest. 1484 */ 1485 if (show->state.depth >= BTF_SHOW_MAX_DEPTH) { 1486 show->state.status = -E2BIG; 1487 return NULL; 1488 } 1489 1490 safe_data = btf_show_start_type(show, t, type_id, data); 1491 if (!safe_data) 1492 return safe_data; 1493 1494 btf_show(show, "%s%s%s", btf_show_indent(show), 1495 btf_show_name(show), 1496 btf_show_newline(show)); 1497 show->state.depth++; 1498 return safe_data; 1499 } 1500 1501 static void btf_show_end_aggr_type(struct btf_show *show, 1502 const char *suffix) 1503 { 1504 show->state.depth--; 1505 btf_show(show, "%s%s%s%s", btf_show_indent(show), suffix, 1506 btf_show_delim(show), btf_show_newline(show)); 1507 btf_show_end_type(show); 1508 } 1509 1510 static void btf_show_start_member(struct btf_show *show, 1511 const struct btf_member *m) 1512 { 1513 show->state.member = m; 1514 } 1515 1516 static void btf_show_start_array_member(struct btf_show *show) 1517 { 1518 show->state.array_member = 1; 1519 btf_show_start_member(show, NULL); 1520 } 1521 1522 static void btf_show_end_member(struct btf_show *show) 1523 { 1524 show->state.member = NULL; 1525 } 1526 1527 static void btf_show_end_array_member(struct btf_show *show) 1528 { 1529 show->state.array_member = 0; 1530 btf_show_end_member(show); 1531 } 1532 1533 static void *btf_show_start_array_type(struct btf_show *show, 1534 const struct btf_type *t, 1535 u32 type_id, 1536 u16 array_encoding, 1537 void *data) 1538 { 1539 show->state.array_encoding = array_encoding; 1540 show->state.array_terminated = 0; 1541 return btf_show_start_aggr_type(show, t, type_id, data); 1542 } 1543 1544 static void btf_show_end_array_type(struct btf_show *show) 1545 { 1546 show->state.array_encoding = 0; 1547 show->state.array_terminated = 0; 1548 btf_show_end_aggr_type(show, "]"); 1549 } 1550 1551 static void *btf_show_start_struct_type(struct btf_show *show, 1552 const struct btf_type *t, 1553 u32 type_id, 1554 void *data) 1555 { 1556 return btf_show_start_aggr_type(show, t, type_id, data); 1557 } 1558 1559 static void btf_show_end_struct_type(struct btf_show *show) 1560 { 1561 btf_show_end_aggr_type(show, "}"); 1562 } 1563 1564 __printf(2, 3) static void __btf_verifier_log(struct bpf_verifier_log *log, 1565 const char *fmt, ...) 1566 { 1567 va_list args; 1568 1569 va_start(args, fmt); 1570 bpf_verifier_vlog(log, fmt, args); 1571 va_end(args); 1572 } 1573 1574 __printf(2, 3) static void btf_verifier_log(struct btf_verifier_env *env, 1575 const char *fmt, ...) 1576 { 1577 struct bpf_verifier_log *log = &env->log; 1578 va_list args; 1579 1580 if (!bpf_verifier_log_needed(log)) 1581 return; 1582 1583 va_start(args, fmt); 1584 bpf_verifier_vlog(log, fmt, args); 1585 va_end(args); 1586 } 1587 1588 __printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env, 1589 const struct btf_type *t, 1590 bool log_details, 1591 const char *fmt, ...) 1592 { 1593 struct bpf_verifier_log *log = &env->log; 1594 struct btf *btf = env->btf; 1595 va_list args; 1596 1597 if (!bpf_verifier_log_needed(log)) 1598 return; 1599 1600 if (log->level == BPF_LOG_KERNEL) { 1601 /* btf verifier prints all types it is processing via 1602 * btf_verifier_log_type(..., fmt = NULL). 1603 * Skip those prints for in-kernel BTF verification. 1604 */ 1605 if (!fmt) 1606 return; 1607 1608 /* Skip logging when loading module BTF with mismatches permitted */ 1609 if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) 1610 return; 1611 } 1612 1613 __btf_verifier_log(log, "[%u] %s %s%s", 1614 env->log_type_id, 1615 btf_type_str(t), 1616 __btf_name_by_offset(btf, t->name_off), 1617 log_details ? " " : ""); 1618 1619 if (log_details) 1620 btf_type_ops(t)->log_details(env, t); 1621 1622 if (fmt && *fmt) { 1623 __btf_verifier_log(log, " "); 1624 va_start(args, fmt); 1625 bpf_verifier_vlog(log, fmt, args); 1626 va_end(args); 1627 } 1628 1629 __btf_verifier_log(log, "\n"); 1630 } 1631 1632 #define btf_verifier_log_type(env, t, ...) \ 1633 __btf_verifier_log_type((env), (t), true, __VA_ARGS__) 1634 #define btf_verifier_log_basic(env, t, ...) \ 1635 __btf_verifier_log_type((env), (t), false, __VA_ARGS__) 1636 1637 __printf(4, 5) 1638 static void btf_verifier_log_member(struct btf_verifier_env *env, 1639 const struct btf_type *struct_type, 1640 const struct btf_member *member, 1641 const char *fmt, ...) 1642 { 1643 struct bpf_verifier_log *log = &env->log; 1644 struct btf *btf = env->btf; 1645 va_list args; 1646 1647 if (!bpf_verifier_log_needed(log)) 1648 return; 1649 1650 if (log->level == BPF_LOG_KERNEL) { 1651 if (!fmt) 1652 return; 1653 1654 /* Skip logging when loading module BTF with mismatches permitted */ 1655 if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) 1656 return; 1657 } 1658 1659 /* The CHECK_META phase already did a btf dump. 1660 * 1661 * If member is logged again, it must hit an error in 1662 * parsing this member. It is useful to print out which 1663 * struct this member belongs to. 1664 */ 1665 if (env->phase != CHECK_META) 1666 btf_verifier_log_type(env, struct_type, NULL); 1667 1668 if (btf_type_kflag(struct_type)) 1669 __btf_verifier_log(log, 1670 "\t%s type_id=%u bitfield_size=%u bits_offset=%u", 1671 __btf_name_by_offset(btf, member->name_off), 1672 member->type, 1673 BTF_MEMBER_BITFIELD_SIZE(member->offset), 1674 BTF_MEMBER_BIT_OFFSET(member->offset)); 1675 else 1676 __btf_verifier_log(log, "\t%s type_id=%u bits_offset=%u", 1677 __btf_name_by_offset(btf, member->name_off), 1678 member->type, member->offset); 1679 1680 if (fmt && *fmt) { 1681 __btf_verifier_log(log, " "); 1682 va_start(args, fmt); 1683 bpf_verifier_vlog(log, fmt, args); 1684 va_end(args); 1685 } 1686 1687 __btf_verifier_log(log, "\n"); 1688 } 1689 1690 __printf(4, 5) 1691 static void btf_verifier_log_vsi(struct btf_verifier_env *env, 1692 const struct btf_type *datasec_type, 1693 const struct btf_var_secinfo *vsi, 1694 const char *fmt, ...) 1695 { 1696 struct bpf_verifier_log *log = &env->log; 1697 va_list args; 1698 1699 if (!bpf_verifier_log_needed(log)) 1700 return; 1701 if (log->level == BPF_LOG_KERNEL && !fmt) 1702 return; 1703 if (env->phase != CHECK_META) 1704 btf_verifier_log_type(env, datasec_type, NULL); 1705 1706 __btf_verifier_log(log, "\t type_id=%u offset=%u size=%u", 1707 vsi->type, vsi->offset, vsi->size); 1708 if (fmt && *fmt) { 1709 __btf_verifier_log(log, " "); 1710 va_start(args, fmt); 1711 bpf_verifier_vlog(log, fmt, args); 1712 va_end(args); 1713 } 1714 1715 __btf_verifier_log(log, "\n"); 1716 } 1717 1718 static void btf_verifier_log_hdr(struct btf_verifier_env *env, 1719 u32 btf_data_size) 1720 { 1721 struct bpf_verifier_log *log = &env->log; 1722 const struct btf *btf = env->btf; 1723 const struct btf_header *hdr; 1724 1725 if (!bpf_verifier_log_needed(log)) 1726 return; 1727 1728 if (log->level == BPF_LOG_KERNEL) 1729 return; 1730 hdr = &btf->hdr; 1731 __btf_verifier_log(log, "magic: 0x%x\n", hdr->magic); 1732 __btf_verifier_log(log, "version: %u\n", hdr->version); 1733 __btf_verifier_log(log, "flags: 0x%x\n", hdr->flags); 1734 __btf_verifier_log(log, "hdr_len: %u\n", hdr->hdr_len); 1735 __btf_verifier_log(log, "type_off: %u\n", hdr->type_off); 1736 __btf_verifier_log(log, "type_len: %u\n", hdr->type_len); 1737 __btf_verifier_log(log, "str_off: %u\n", hdr->str_off); 1738 __btf_verifier_log(log, "str_len: %u\n", hdr->str_len); 1739 if (hdr->hdr_len >= sizeof(struct btf_header) && 1740 btf_data_size >= hdr->hdr_len) { 1741 __btf_verifier_log(log, "layout_off: %u\n", hdr->layout_off); 1742 __btf_verifier_log(log, "layout_len: %u\n", hdr->layout_len); 1743 } 1744 __btf_verifier_log(log, "btf_total_size: %u\n", btf_data_size); 1745 } 1746 1747 static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t) 1748 { 1749 struct btf *btf = env->btf; 1750 1751 if (btf->types_size == btf->nr_types) { 1752 /* Expand 'types' array */ 1753 1754 struct btf_type **new_types; 1755 u32 expand_by, new_size, max_size; 1756 1757 if (btf->start_id + btf->types_size >= BTF_MAX_TYPE) { 1758 btf_verifier_log(env, "Exceeded max num of types"); 1759 return -E2BIG; 1760 } 1761 1762 max_size = BTF_MAX_TYPE - btf->start_id; 1763 expand_by = max_t(u32, btf->types_size >> 2, 16); 1764 new_size = min_t(u32, max_size, 1765 btf->types_size + expand_by); 1766 1767 new_types = kvzalloc_objs(*new_types, new_size, 1768 GFP_KERNEL | __GFP_NOWARN); 1769 if (!new_types) 1770 return -ENOMEM; 1771 1772 if (btf->nr_types == 0) { 1773 if (!btf->base_btf) { 1774 /* lazily init VOID type */ 1775 new_types[0] = &btf_void; 1776 btf->nr_types++; 1777 } 1778 } else { 1779 memcpy(new_types, btf->types, 1780 sizeof(*btf->types) * btf->nr_types); 1781 } 1782 1783 kvfree(btf->types); 1784 btf->types = new_types; 1785 btf->types_size = new_size; 1786 } 1787 1788 btf->types[btf->nr_types++] = t; 1789 1790 return 0; 1791 } 1792 1793 static int btf_alloc_id(struct btf *btf) 1794 { 1795 int id; 1796 1797 idr_preload(GFP_KERNEL); 1798 spin_lock_bh(&btf_idr_lock); 1799 id = idr_alloc_cyclic(&btf_idr, btf, 1, INT_MAX, GFP_ATOMIC); 1800 if (id > 0) 1801 btf->id = id; 1802 spin_unlock_bh(&btf_idr_lock); 1803 idr_preload_end(); 1804 1805 if (WARN_ON_ONCE(!id)) 1806 return -ENOSPC; 1807 1808 return id > 0 ? 0 : id; 1809 } 1810 1811 static void btf_free_id(struct btf *btf) 1812 { 1813 unsigned long flags; 1814 1815 /* 1816 * In map-in-map, calling map_delete_elem() on outer 1817 * map will call bpf_map_put on the inner map. 1818 * It will then eventually call btf_free_id() 1819 * on the inner map. Some of the map_delete_elem() 1820 * implementation may have irq disabled, so 1821 * we need to use the _irqsave() version instead 1822 * of the _bh() version. 1823 */ 1824 spin_lock_irqsave(&btf_idr_lock, flags); 1825 if (btf->id) { 1826 idr_remove(&btf_idr, btf->id); 1827 /* 1828 * Clear the id here to make this function idempotent, since it will get 1829 * called a couple of times for module BTFs: on module unload, and then 1830 * the final btf_put(). btf_alloc_id() starts IDs with 1, so we can use 1831 * 0 as sentinel value. 1832 */ 1833 WRITE_ONCE(btf->id, 0); 1834 } 1835 spin_unlock_irqrestore(&btf_idr_lock, flags); 1836 } 1837 1838 static void btf_free_kfunc_set_tab(struct btf *btf) 1839 { 1840 struct btf_kfunc_set_tab *tab = btf->kfunc_set_tab; 1841 int hook; 1842 1843 if (!tab) 1844 return; 1845 for (hook = 0; hook < ARRAY_SIZE(tab->sets); hook++) 1846 kfree(tab->sets[hook]); 1847 kfree(tab); 1848 btf->kfunc_set_tab = NULL; 1849 } 1850 1851 static void btf_free_dtor_kfunc_tab(struct btf *btf) 1852 { 1853 struct btf_id_dtor_kfunc_tab *tab = btf->dtor_kfunc_tab; 1854 1855 if (!tab) 1856 return; 1857 kfree(tab); 1858 btf->dtor_kfunc_tab = NULL; 1859 } 1860 1861 static void btf_struct_metas_free(struct btf_struct_metas *tab) 1862 { 1863 int i; 1864 1865 if (!tab) 1866 return; 1867 for (i = 0; i < tab->cnt; i++) 1868 btf_record_free(tab->types[i].record); 1869 kfree(tab); 1870 } 1871 1872 static void btf_free_struct_meta_tab(struct btf *btf) 1873 { 1874 struct btf_struct_metas *tab = btf->struct_meta_tab; 1875 1876 btf_struct_metas_free(tab); 1877 btf->struct_meta_tab = NULL; 1878 } 1879 1880 static void btf_free_struct_ops_tab(struct btf *btf) 1881 { 1882 struct btf_struct_ops_tab *tab = btf->struct_ops_tab; 1883 u32 i; 1884 1885 if (!tab) 1886 return; 1887 1888 for (i = 0; i < tab->cnt; i++) 1889 bpf_struct_ops_desc_release(&tab->ops[i]); 1890 1891 kfree(tab); 1892 btf->struct_ops_tab = NULL; 1893 } 1894 1895 static void btf_free(struct btf *btf) 1896 { 1897 btf_free_struct_meta_tab(btf); 1898 btf_free_dtor_kfunc_tab(btf); 1899 btf_free_kfunc_set_tab(btf); 1900 btf_free_struct_ops_tab(btf); 1901 kvfree(btf->types); 1902 kvfree(btf->resolved_sizes); 1903 kvfree(btf->resolved_ids); 1904 /* vmlinux does not allocate btf->data, it simply points it at 1905 * __start_BTF. 1906 */ 1907 if (!btf_is_vmlinux(btf)) 1908 kvfree(btf->data); 1909 kvfree(btf->base_id_map); 1910 kfree(btf); 1911 } 1912 1913 static void btf_free_rcu(struct rcu_head *rcu) 1914 { 1915 struct btf *btf = container_of(rcu, struct btf, rcu); 1916 1917 btf_free(btf); 1918 } 1919 1920 const char *btf_get_name(const struct btf *btf) 1921 { 1922 return btf->name; 1923 } 1924 1925 void btf_get(struct btf *btf) 1926 { 1927 refcount_inc(&btf->refcnt); 1928 } 1929 1930 void btf_put(struct btf *btf) 1931 { 1932 if (btf && refcount_dec_and_test(&btf->refcnt)) { 1933 btf_free_id(btf); 1934 call_rcu(&btf->rcu, btf_free_rcu); 1935 } 1936 } 1937 1938 struct btf *btf_base_btf(const struct btf *btf) 1939 { 1940 return btf->base_btf; 1941 } 1942 1943 const struct btf_header *btf_header(const struct btf *btf) 1944 { 1945 return &btf->hdr; 1946 } 1947 1948 void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) 1949 { 1950 btf->base_btf = (struct btf *)base_btf; 1951 btf->start_id = btf_nr_types(base_btf); 1952 btf->start_str_off = base_btf->start_str_off + base_btf->hdr.str_len; 1953 } 1954 1955 static int env_resolve_init(struct btf_verifier_env *env) 1956 { 1957 struct btf *btf = env->btf; 1958 u32 nr_types = btf->nr_types; 1959 u32 *resolved_sizes = NULL; 1960 u32 *resolved_ids = NULL; 1961 u8 *visit_states = NULL; 1962 1963 resolved_sizes = kvzalloc_objs(*resolved_sizes, nr_types, 1964 GFP_KERNEL | __GFP_NOWARN); 1965 if (!resolved_sizes) 1966 goto nomem; 1967 1968 resolved_ids = kvzalloc_objs(*resolved_ids, nr_types, 1969 GFP_KERNEL | __GFP_NOWARN); 1970 if (!resolved_ids) 1971 goto nomem; 1972 1973 visit_states = kvcalloc(nr_types, sizeof(*visit_states), 1974 GFP_KERNEL | __GFP_NOWARN); 1975 if (!visit_states) 1976 goto nomem; 1977 1978 btf->resolved_sizes = resolved_sizes; 1979 btf->resolved_ids = resolved_ids; 1980 env->visit_states = visit_states; 1981 1982 return 0; 1983 1984 nomem: 1985 kvfree(resolved_sizes); 1986 kvfree(resolved_ids); 1987 kvfree(visit_states); 1988 return -ENOMEM; 1989 } 1990 1991 static void btf_verifier_env_free(struct btf_verifier_env *env) 1992 { 1993 kvfree(env->visit_states); 1994 kfree(env); 1995 } 1996 1997 static bool env_type_is_resolve_sink(const struct btf_verifier_env *env, 1998 const struct btf_type *next_type) 1999 { 2000 switch (env->resolve_mode) { 2001 case RESOLVE_TBD: 2002 /* int, enum or void is a sink */ 2003 return !btf_type_needs_resolve(next_type); 2004 case RESOLVE_PTR: 2005 /* int, enum, void, struct, array, func or func_proto is a sink 2006 * for ptr 2007 */ 2008 return !btf_type_is_modifier(next_type) && 2009 !btf_type_is_ptr(next_type); 2010 case RESOLVE_STRUCT_OR_ARRAY: 2011 /* int, enum, void, ptr, func or func_proto is a sink 2012 * for struct and array 2013 */ 2014 return !btf_type_is_modifier(next_type) && 2015 !btf_type_is_array(next_type) && 2016 !btf_type_is_struct(next_type); 2017 default: 2018 BUG(); 2019 } 2020 } 2021 2022 static bool env_type_is_resolved(const struct btf_verifier_env *env, 2023 u32 type_id) 2024 { 2025 /* base BTF types should be resolved by now */ 2026 if (type_id < env->btf->start_id) 2027 return true; 2028 2029 return env->visit_states[type_id - env->btf->start_id] == RESOLVED; 2030 } 2031 2032 static int env_stack_push(struct btf_verifier_env *env, 2033 const struct btf_type *t, u32 type_id) 2034 { 2035 const struct btf *btf = env->btf; 2036 struct resolve_vertex *v; 2037 2038 if (env->top_stack == MAX_RESOLVE_DEPTH) 2039 return -E2BIG; 2040 2041 if (type_id < btf->start_id 2042 || env->visit_states[type_id - btf->start_id] != NOT_VISITED) 2043 return -EEXIST; 2044 2045 env->visit_states[type_id - btf->start_id] = VISITED; 2046 2047 v = &env->stack[env->top_stack++]; 2048 v->t = t; 2049 v->type_id = type_id; 2050 v->next_member = 0; 2051 2052 if (env->resolve_mode == RESOLVE_TBD) { 2053 if (btf_type_is_ptr(t)) 2054 env->resolve_mode = RESOLVE_PTR; 2055 else if (btf_type_is_struct(t) || btf_type_is_array(t)) 2056 env->resolve_mode = RESOLVE_STRUCT_OR_ARRAY; 2057 } 2058 2059 return 0; 2060 } 2061 2062 static void env_stack_set_next_member(struct btf_verifier_env *env, 2063 u32 next_member) 2064 { 2065 env->stack[env->top_stack - 1].next_member = next_member; 2066 } 2067 2068 static void env_stack_pop_resolved(struct btf_verifier_env *env, 2069 u32 resolved_type_id, 2070 u32 resolved_size) 2071 { 2072 u32 type_id = env->stack[--(env->top_stack)].type_id; 2073 struct btf *btf = env->btf; 2074 2075 type_id -= btf->start_id; /* adjust to local type id */ 2076 btf->resolved_sizes[type_id] = resolved_size; 2077 btf->resolved_ids[type_id] = resolved_type_id; 2078 env->visit_states[type_id] = RESOLVED; 2079 } 2080 2081 static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env) 2082 { 2083 return env->top_stack ? &env->stack[env->top_stack - 1] : NULL; 2084 } 2085 2086 /* Resolve the size of a passed-in "type" 2087 * 2088 * type: is an array (e.g. u32 array[x][y]) 2089 * return type: type "u32[x][y]", i.e. BTF_KIND_ARRAY, 2090 * *type_size: (x * y * sizeof(u32)). Hence, *type_size always 2091 * corresponds to the return type. 2092 * *elem_type: u32 2093 * *elem_id: id of u32 2094 * *total_nelems: (x * y). Hence, individual elem size is 2095 * (*type_size / *total_nelems) 2096 * *type_id: ID of the returned type if leading modifiers were stripped; 2097 * otherwise left unchanged. 2098 * 2099 * type: is not an array (e.g. const struct X) 2100 * return type: type "struct X" 2101 * *type_size: sizeof(struct X) 2102 * *elem_type: same as return type ("struct X") 2103 * *elem_id: 0 2104 * *total_nelems: 1 2105 * *type_id: ID of the returned type if leading modifiers were stripped; 2106 * otherwise left unchanged. 2107 */ 2108 static const struct btf_type * 2109 __btf_resolve_size(const struct btf *btf, const struct btf_type *type, 2110 u32 *type_size, const struct btf_type **elem_type, 2111 u32 *elem_id, u32 *total_nelems, u32 *type_id) 2112 { 2113 const struct btf_type *array_type = NULL; 2114 const struct btf_array *array = NULL; 2115 u32 i, size, nelems = 1, id = 0; 2116 2117 for (i = 0; i < MAX_RESOLVE_DEPTH; i++) { 2118 switch (BTF_INFO_KIND(type->info)) { 2119 /* type->size can be used */ 2120 case BTF_KIND_INT: 2121 case BTF_KIND_STRUCT: 2122 case BTF_KIND_UNION: 2123 case BTF_KIND_ENUM: 2124 case BTF_KIND_FLOAT: 2125 case BTF_KIND_ENUM64: 2126 size = type->size; 2127 goto resolved; 2128 2129 case BTF_KIND_PTR: 2130 size = sizeof(void *); 2131 goto resolved; 2132 2133 /* Modifiers */ 2134 case BTF_KIND_TYPEDEF: 2135 case BTF_KIND_VOLATILE: 2136 case BTF_KIND_CONST: 2137 case BTF_KIND_RESTRICT: 2138 case BTF_KIND_TYPE_TAG: 2139 if (!array_type) 2140 id = type->type; 2141 type = btf_type_by_id(btf, type->type); 2142 break; 2143 2144 case BTF_KIND_ARRAY: 2145 if (!array_type) 2146 array_type = type; 2147 array = btf_type_array(type); 2148 if (nelems && array->nelems > U32_MAX / nelems) 2149 return ERR_PTR(-EINVAL); 2150 nelems *= array->nelems; 2151 type = btf_type_by_id(btf, array->type); 2152 break; 2153 2154 /* type without size */ 2155 default: 2156 return ERR_PTR(-EINVAL); 2157 } 2158 } 2159 2160 return ERR_PTR(-EINVAL); 2161 2162 resolved: 2163 if (nelems && size > U32_MAX / nelems) 2164 return ERR_PTR(-EINVAL); 2165 2166 *type_size = nelems * size; 2167 if (total_nelems) 2168 *total_nelems = nelems; 2169 if (elem_type) 2170 *elem_type = type; 2171 if (elem_id) 2172 *elem_id = array ? array->type : 0; 2173 if (type_id && id) 2174 *type_id = id; 2175 2176 return array_type ? : type; 2177 } 2178 2179 const struct btf_type * 2180 btf_resolve_size(const struct btf *btf, const struct btf_type *type, 2181 u32 *type_size) 2182 { 2183 return __btf_resolve_size(btf, type, type_size, NULL, NULL, NULL, NULL); 2184 } 2185 2186 static u32 btf_resolved_type_id(const struct btf *btf, u32 type_id) 2187 { 2188 while (type_id < btf->start_id) 2189 btf = btf->base_btf; 2190 2191 return btf->resolved_ids[type_id - btf->start_id]; 2192 } 2193 2194 /* The input param "type_id" must point to a needs_resolve type */ 2195 static const struct btf_type *btf_type_id_resolve(const struct btf *btf, 2196 u32 *type_id) 2197 { 2198 *type_id = btf_resolved_type_id(btf, *type_id); 2199 return btf_type_by_id(btf, *type_id); 2200 } 2201 2202 static u32 btf_resolved_type_size(const struct btf *btf, u32 type_id) 2203 { 2204 while (type_id < btf->start_id) 2205 btf = btf->base_btf; 2206 2207 return btf->resolved_sizes[type_id - btf->start_id]; 2208 } 2209 2210 const struct btf_type *btf_type_id_size(const struct btf *btf, 2211 u32 *type_id, u32 *ret_size) 2212 { 2213 const struct btf_type *size_type; 2214 u32 size_type_id = *type_id; 2215 u32 size = 0; 2216 2217 size_type = btf_type_by_id(btf, size_type_id); 2218 if (btf_type_nosize_or_null(size_type)) 2219 return NULL; 2220 2221 if (btf_type_has_size(size_type)) { 2222 size = size_type->size; 2223 } else if (btf_type_is_array(size_type)) { 2224 size = btf_resolved_type_size(btf, size_type_id); 2225 } else if (btf_type_is_ptr(size_type)) { 2226 size = sizeof(void *); 2227 } else { 2228 if (WARN_ON_ONCE(!btf_type_is_modifier(size_type) && 2229 !btf_type_is_var(size_type))) 2230 return NULL; 2231 2232 size_type_id = btf_resolved_type_id(btf, size_type_id); 2233 size_type = btf_type_by_id(btf, size_type_id); 2234 if (btf_type_nosize_or_null(size_type)) 2235 return NULL; 2236 else if (btf_type_has_size(size_type)) 2237 size = size_type->size; 2238 else if (btf_type_is_array(size_type)) 2239 size = btf_resolved_type_size(btf, size_type_id); 2240 else if (btf_type_is_ptr(size_type)) 2241 size = sizeof(void *); 2242 else 2243 return NULL; 2244 } 2245 2246 *type_id = size_type_id; 2247 if (ret_size) 2248 *ret_size = size; 2249 2250 return size_type; 2251 } 2252 2253 static int btf_df_check_member(struct btf_verifier_env *env, 2254 const struct btf_type *struct_type, 2255 const struct btf_member *member, 2256 const struct btf_type *member_type) 2257 { 2258 btf_verifier_log_basic(env, struct_type, 2259 "Unsupported check_member"); 2260 return -EINVAL; 2261 } 2262 2263 static int btf_df_check_kflag_member(struct btf_verifier_env *env, 2264 const struct btf_type *struct_type, 2265 const struct btf_member *member, 2266 const struct btf_type *member_type) 2267 { 2268 btf_verifier_log_basic(env, struct_type, 2269 "Unsupported check_kflag_member"); 2270 return -EINVAL; 2271 } 2272 2273 /* Used for ptr, array struct/union and float type members. 2274 * int, enum and modifier types have their specific callback functions. 2275 */ 2276 static int btf_generic_check_kflag_member(struct btf_verifier_env *env, 2277 const struct btf_type *struct_type, 2278 const struct btf_member *member, 2279 const struct btf_type *member_type) 2280 { 2281 if (BTF_MEMBER_BITFIELD_SIZE(member->offset)) { 2282 btf_verifier_log_member(env, struct_type, member, 2283 "Invalid member bitfield_size"); 2284 return -EINVAL; 2285 } 2286 2287 /* bitfield size is 0, so member->offset represents bit offset only. 2288 * It is safe to call non kflag check_member variants. 2289 */ 2290 return btf_type_ops(member_type)->check_member(env, struct_type, 2291 member, 2292 member_type); 2293 } 2294 2295 static int btf_df_resolve(struct btf_verifier_env *env, 2296 const struct resolve_vertex *v) 2297 { 2298 btf_verifier_log_basic(env, v->t, "Unsupported resolve"); 2299 return -EINVAL; 2300 } 2301 2302 static void btf_df_show(const struct btf *btf, const struct btf_type *t, 2303 u32 type_id, void *data, u8 bits_offsets, 2304 struct btf_show *show) 2305 { 2306 btf_show(show, "<unsupported kind:%u>", BTF_INFO_KIND(t->info)); 2307 } 2308 2309 static int btf_int_check_member(struct btf_verifier_env *env, 2310 const struct btf_type *struct_type, 2311 const struct btf_member *member, 2312 const struct btf_type *member_type) 2313 { 2314 u32 int_data = btf_type_int(member_type); 2315 u32 struct_bits_off = member->offset; 2316 u32 struct_size = struct_type->size; 2317 u32 nr_copy_bits; 2318 u32 bytes_offset; 2319 2320 if (U32_MAX - struct_bits_off < BTF_INT_OFFSET(int_data)) { 2321 btf_verifier_log_member(env, struct_type, member, 2322 "bits_offset exceeds U32_MAX"); 2323 return -EINVAL; 2324 } 2325 2326 struct_bits_off += BTF_INT_OFFSET(int_data); 2327 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); 2328 nr_copy_bits = BTF_INT_BITS(int_data) + 2329 BITS_PER_BYTE_MASKED(struct_bits_off); 2330 2331 if (nr_copy_bits > BITS_PER_U128) { 2332 btf_verifier_log_member(env, struct_type, member, 2333 "nr_copy_bits exceeds 128"); 2334 return -EINVAL; 2335 } 2336 2337 if (struct_size < bytes_offset || 2338 struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) { 2339 btf_verifier_log_member(env, struct_type, member, 2340 "Member exceeds struct_size"); 2341 return -EINVAL; 2342 } 2343 2344 return 0; 2345 } 2346 2347 static int btf_int_check_kflag_member(struct btf_verifier_env *env, 2348 const struct btf_type *struct_type, 2349 const struct btf_member *member, 2350 const struct btf_type *member_type) 2351 { 2352 u32 struct_bits_off, nr_bits, nr_int_data_bits, bytes_offset; 2353 u32 int_data = btf_type_int(member_type); 2354 u32 struct_size = struct_type->size; 2355 u32 nr_copy_bits; 2356 2357 /* a regular int type is required for the kflag int member */ 2358 if (!btf_type_int_is_regular(member_type)) { 2359 btf_verifier_log_member(env, struct_type, member, 2360 "Invalid member base type"); 2361 return -EINVAL; 2362 } 2363 2364 /* check sanity of bitfield size */ 2365 nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset); 2366 struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset); 2367 nr_int_data_bits = BTF_INT_BITS(int_data); 2368 if (!nr_bits) { 2369 /* Not a bitfield member, member offset must be at byte 2370 * boundary. 2371 */ 2372 if (BITS_PER_BYTE_MASKED(struct_bits_off)) { 2373 btf_verifier_log_member(env, struct_type, member, 2374 "Invalid member offset"); 2375 return -EINVAL; 2376 } 2377 2378 nr_bits = nr_int_data_bits; 2379 } else if (nr_bits > nr_int_data_bits) { 2380 btf_verifier_log_member(env, struct_type, member, 2381 "Invalid member bitfield_size"); 2382 return -EINVAL; 2383 } 2384 2385 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); 2386 nr_copy_bits = nr_bits + BITS_PER_BYTE_MASKED(struct_bits_off); 2387 if (nr_copy_bits > BITS_PER_U128) { 2388 btf_verifier_log_member(env, struct_type, member, 2389 "nr_copy_bits exceeds 128"); 2390 return -EINVAL; 2391 } 2392 2393 if (struct_size < bytes_offset || 2394 struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) { 2395 btf_verifier_log_member(env, struct_type, member, 2396 "Member exceeds struct_size"); 2397 return -EINVAL; 2398 } 2399 2400 return 0; 2401 } 2402 2403 static s32 btf_int_check_meta(struct btf_verifier_env *env, 2404 const struct btf_type *t, 2405 u32 meta_left) 2406 { 2407 u32 int_data, nr_bits, meta_needed = sizeof(int_data); 2408 u16 encoding; 2409 2410 if (meta_left < meta_needed) { 2411 btf_verifier_log_basic(env, t, 2412 "meta_left:%u meta_needed:%u", 2413 meta_left, meta_needed); 2414 return -EINVAL; 2415 } 2416 2417 if (btf_type_vlen(t)) { 2418 btf_verifier_log_type(env, t, "vlen != 0"); 2419 return -EINVAL; 2420 } 2421 2422 if (btf_type_kflag(t)) { 2423 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 2424 return -EINVAL; 2425 } 2426 2427 int_data = btf_type_int(t); 2428 if (int_data & ~BTF_INT_MASK) { 2429 btf_verifier_log_basic(env, t, "Invalid int_data:%x", 2430 int_data); 2431 return -EINVAL; 2432 } 2433 2434 nr_bits = BTF_INT_BITS(int_data) + BTF_INT_OFFSET(int_data); 2435 2436 if (nr_bits > BITS_PER_U128) { 2437 btf_verifier_log_type(env, t, "nr_bits exceeds %zu", 2438 BITS_PER_U128); 2439 return -EINVAL; 2440 } 2441 2442 if (BITS_ROUNDUP_BYTES(nr_bits) > t->size) { 2443 btf_verifier_log_type(env, t, "nr_bits exceeds type_size"); 2444 return -EINVAL; 2445 } 2446 2447 /* 2448 * Only one of the encoding bits is allowed and it 2449 * should be sufficient for the pretty print purpose (i.e. decoding). 2450 * Multiple bits can be allowed later if it is found 2451 * to be insufficient. 2452 */ 2453 encoding = BTF_INT_ENCODING(int_data); 2454 if (encoding && 2455 encoding != BTF_INT_SIGNED && 2456 encoding != BTF_INT_CHAR && 2457 encoding != BTF_INT_BOOL) { 2458 btf_verifier_log_type(env, t, "Unsupported encoding"); 2459 return -ENOTSUPP; 2460 } 2461 2462 btf_verifier_log_type(env, t, NULL); 2463 2464 return meta_needed; 2465 } 2466 2467 static void btf_int_log(struct btf_verifier_env *env, 2468 const struct btf_type *t) 2469 { 2470 int int_data = btf_type_int(t); 2471 2472 btf_verifier_log(env, 2473 "size=%u bits_offset=%u nr_bits=%u encoding=%s", 2474 t->size, BTF_INT_OFFSET(int_data), 2475 BTF_INT_BITS(int_data), 2476 btf_int_encoding_str(BTF_INT_ENCODING(int_data))); 2477 } 2478 2479 static void btf_int128_print(struct btf_show *show, void *data) 2480 { 2481 /* data points to a __int128 number. 2482 * Suppose 2483 * int128_num = *(__int128 *)data; 2484 * The below formulas shows what upper_num and lower_num represents: 2485 * upper_num = int128_num >> 64; 2486 * lower_num = int128_num & 0xffffffffFFFFFFFFULL; 2487 */ 2488 u64 upper_num, lower_num; 2489 2490 #ifdef __BIG_ENDIAN_BITFIELD 2491 upper_num = *(u64 *)data; 2492 lower_num = *(u64 *)(data + 8); 2493 #else 2494 upper_num = *(u64 *)(data + 8); 2495 lower_num = *(u64 *)data; 2496 #endif 2497 if (upper_num == 0) 2498 btf_show_type_value(show, "0x%llx", lower_num); 2499 else 2500 btf_show_type_values(show, "0x%llx%016llx", upper_num, 2501 lower_num); 2502 } 2503 2504 static void btf_int128_shift(u64 *print_num, u16 left_shift_bits, 2505 u16 right_shift_bits) 2506 { 2507 u64 upper_num, lower_num; 2508 2509 #ifdef __BIG_ENDIAN_BITFIELD 2510 upper_num = print_num[0]; 2511 lower_num = print_num[1]; 2512 #else 2513 upper_num = print_num[1]; 2514 lower_num = print_num[0]; 2515 #endif 2516 2517 /* shake out un-needed bits by shift/or operations */ 2518 if (left_shift_bits >= 64) { 2519 upper_num = lower_num << (left_shift_bits - 64); 2520 lower_num = 0; 2521 } else { 2522 upper_num = (upper_num << left_shift_bits) | 2523 (lower_num >> (64 - left_shift_bits)); 2524 lower_num = lower_num << left_shift_bits; 2525 } 2526 2527 if (right_shift_bits >= 64) { 2528 lower_num = upper_num >> (right_shift_bits - 64); 2529 upper_num = 0; 2530 } else { 2531 lower_num = (lower_num >> right_shift_bits) | 2532 (upper_num << (64 - right_shift_bits)); 2533 upper_num = upper_num >> right_shift_bits; 2534 } 2535 2536 #ifdef __BIG_ENDIAN_BITFIELD 2537 print_num[0] = upper_num; 2538 print_num[1] = lower_num; 2539 #else 2540 print_num[0] = lower_num; 2541 print_num[1] = upper_num; 2542 #endif 2543 } 2544 2545 static void btf_bitfield_show(void *data, u8 bits_offset, 2546 u8 nr_bits, struct btf_show *show) 2547 { 2548 u16 left_shift_bits, right_shift_bits; 2549 u8 nr_copy_bytes; 2550 u8 nr_copy_bits; 2551 u64 print_num[2] = {}; 2552 2553 nr_copy_bits = nr_bits + bits_offset; 2554 nr_copy_bytes = BITS_ROUNDUP_BYTES(nr_copy_bits); 2555 2556 memcpy(print_num, data, nr_copy_bytes); 2557 2558 #ifdef __BIG_ENDIAN_BITFIELD 2559 left_shift_bits = bits_offset; 2560 #else 2561 left_shift_bits = BITS_PER_U128 - nr_copy_bits; 2562 #endif 2563 right_shift_bits = BITS_PER_U128 - nr_bits; 2564 2565 btf_int128_shift(print_num, left_shift_bits, right_shift_bits); 2566 btf_int128_print(show, print_num); 2567 } 2568 2569 static void btf_int_bits_show(const struct btf *btf, 2570 const struct btf_type *t, 2571 void *data, u8 bits_offset, 2572 struct btf_show *show) 2573 { 2574 u32 int_data = btf_type_int(t); 2575 u8 nr_bits = BTF_INT_BITS(int_data); 2576 u8 total_bits_offset; 2577 2578 /* 2579 * bits_offset is at most 7. 2580 * BTF_INT_OFFSET() cannot exceed 128 bits. 2581 */ 2582 total_bits_offset = bits_offset + BTF_INT_OFFSET(int_data); 2583 data += BITS_ROUNDDOWN_BYTES(total_bits_offset); 2584 bits_offset = BITS_PER_BYTE_MASKED(total_bits_offset); 2585 btf_bitfield_show(data, bits_offset, nr_bits, show); 2586 } 2587 2588 static void btf_int_show(const struct btf *btf, const struct btf_type *t, 2589 u32 type_id, void *data, u8 bits_offset, 2590 struct btf_show *show) 2591 { 2592 u32 int_data = btf_type_int(t); 2593 u8 encoding = BTF_INT_ENCODING(int_data); 2594 bool sign = encoding & BTF_INT_SIGNED; 2595 u8 nr_bits = BTF_INT_BITS(int_data); 2596 void *safe_data; 2597 2598 safe_data = btf_show_start_type(show, t, type_id, data); 2599 if (!safe_data) 2600 return; 2601 2602 if (bits_offset || BTF_INT_OFFSET(int_data) || 2603 BITS_PER_BYTE_MASKED(nr_bits)) { 2604 btf_int_bits_show(btf, t, safe_data, bits_offset, show); 2605 goto out; 2606 } 2607 2608 switch (nr_bits) { 2609 case 128: 2610 btf_int128_print(show, safe_data); 2611 break; 2612 case 64: 2613 if (sign) 2614 btf_show_type_value(show, "%lld", *(s64 *)safe_data); 2615 else 2616 btf_show_type_value(show, "%llu", *(u64 *)safe_data); 2617 break; 2618 case 32: 2619 if (sign) 2620 btf_show_type_value(show, "%d", *(s32 *)safe_data); 2621 else 2622 btf_show_type_value(show, "%u", *(u32 *)safe_data); 2623 break; 2624 case 16: 2625 if (sign) 2626 btf_show_type_value(show, "%d", *(s16 *)safe_data); 2627 else 2628 btf_show_type_value(show, "%u", *(u16 *)safe_data); 2629 break; 2630 case 8: 2631 if (show->state.array_encoding == BTF_INT_CHAR) { 2632 /* check for null terminator */ 2633 if (show->state.array_terminated) 2634 break; 2635 if (*(char *)data == '\0') { 2636 show->state.array_terminated = 1; 2637 break; 2638 } 2639 if (isprint(*(char *)data)) { 2640 btf_show_type_value(show, "'%c'", 2641 *(char *)safe_data); 2642 break; 2643 } 2644 } 2645 if (sign) 2646 btf_show_type_value(show, "%d", *(s8 *)safe_data); 2647 else 2648 btf_show_type_value(show, "%u", *(u8 *)safe_data); 2649 break; 2650 default: 2651 btf_int_bits_show(btf, t, safe_data, bits_offset, show); 2652 break; 2653 } 2654 out: 2655 btf_show_end_type(show); 2656 } 2657 2658 static const struct btf_kind_operations int_ops = { 2659 .check_meta = btf_int_check_meta, 2660 .resolve = btf_df_resolve, 2661 .check_member = btf_int_check_member, 2662 .check_kflag_member = btf_int_check_kflag_member, 2663 .log_details = btf_int_log, 2664 .show = btf_int_show, 2665 }; 2666 2667 static int btf_modifier_check_member(struct btf_verifier_env *env, 2668 const struct btf_type *struct_type, 2669 const struct btf_member *member, 2670 const struct btf_type *member_type) 2671 { 2672 const struct btf_type *resolved_type; 2673 u32 resolved_type_id = member->type; 2674 struct btf_member resolved_member; 2675 struct btf *btf = env->btf; 2676 2677 resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL); 2678 if (!resolved_type) { 2679 btf_verifier_log_member(env, struct_type, member, 2680 "Invalid member"); 2681 return -EINVAL; 2682 } 2683 2684 resolved_member = *member; 2685 resolved_member.type = resolved_type_id; 2686 2687 return btf_type_ops(resolved_type)->check_member(env, struct_type, 2688 &resolved_member, 2689 resolved_type); 2690 } 2691 2692 static int btf_modifier_check_kflag_member(struct btf_verifier_env *env, 2693 const struct btf_type *struct_type, 2694 const struct btf_member *member, 2695 const struct btf_type *member_type) 2696 { 2697 const struct btf_type *resolved_type; 2698 u32 resolved_type_id = member->type; 2699 struct btf_member resolved_member; 2700 struct btf *btf = env->btf; 2701 2702 resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL); 2703 if (!resolved_type) { 2704 btf_verifier_log_member(env, struct_type, member, 2705 "Invalid member"); 2706 return -EINVAL; 2707 } 2708 2709 resolved_member = *member; 2710 resolved_member.type = resolved_type_id; 2711 2712 return btf_type_ops(resolved_type)->check_kflag_member(env, struct_type, 2713 &resolved_member, 2714 resolved_type); 2715 } 2716 2717 static int btf_ptr_check_member(struct btf_verifier_env *env, 2718 const struct btf_type *struct_type, 2719 const struct btf_member *member, 2720 const struct btf_type *member_type) 2721 { 2722 u32 struct_size, struct_bits_off, bytes_offset; 2723 2724 struct_size = struct_type->size; 2725 struct_bits_off = member->offset; 2726 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); 2727 2728 if (BITS_PER_BYTE_MASKED(struct_bits_off)) { 2729 btf_verifier_log_member(env, struct_type, member, 2730 "Member is not byte aligned"); 2731 return -EINVAL; 2732 } 2733 2734 if (struct_size - bytes_offset < sizeof(void *)) { 2735 btf_verifier_log_member(env, struct_type, member, 2736 "Member exceeds struct_size"); 2737 return -EINVAL; 2738 } 2739 2740 return 0; 2741 } 2742 2743 static int btf_ref_type_check_meta(struct btf_verifier_env *env, 2744 const struct btf_type *t, 2745 u32 meta_left) 2746 { 2747 const char *value; 2748 2749 if (btf_type_vlen(t)) { 2750 btf_verifier_log_type(env, t, "vlen != 0"); 2751 return -EINVAL; 2752 } 2753 2754 if (btf_type_kflag(t) && !btf_type_is_type_tag(t)) { 2755 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 2756 return -EINVAL; 2757 } 2758 2759 if (!BTF_TYPE_ID_VALID(t->type)) { 2760 btf_verifier_log_type(env, t, "Invalid type_id"); 2761 return -EINVAL; 2762 } 2763 2764 /* typedef/type_tag type must have a valid name, and other ref types, 2765 * volatile, const, restrict, should have a null name. 2766 */ 2767 if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF) { 2768 if (!t->name_off || 2769 !btf_name_valid_identifier(env->btf, t->name_off)) { 2770 btf_verifier_log_type(env, t, "Invalid name"); 2771 return -EINVAL; 2772 } 2773 } else if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPE_TAG) { 2774 value = btf_name_by_offset(env->btf, t->name_off); 2775 if (!value || !value[0]) { 2776 btf_verifier_log_type(env, t, "Invalid name"); 2777 return -EINVAL; 2778 } 2779 } else { 2780 if (t->name_off) { 2781 btf_verifier_log_type(env, t, "Invalid name"); 2782 return -EINVAL; 2783 } 2784 } 2785 2786 btf_verifier_log_type(env, t, NULL); 2787 2788 return 0; 2789 } 2790 2791 static int btf_modifier_resolve(struct btf_verifier_env *env, 2792 const struct resolve_vertex *v) 2793 { 2794 const struct btf_type *t = v->t; 2795 const struct btf_type *next_type; 2796 u32 next_type_id = t->type; 2797 struct btf *btf = env->btf; 2798 2799 next_type = btf_type_by_id(btf, next_type_id); 2800 if (!next_type || btf_type_is_resolve_source_only(next_type)) { 2801 btf_verifier_log_type(env, v->t, "Invalid type_id"); 2802 return -EINVAL; 2803 } 2804 2805 if (!env_type_is_resolve_sink(env, next_type) && 2806 !env_type_is_resolved(env, next_type_id)) 2807 return env_stack_push(env, next_type, next_type_id); 2808 2809 /* Figure out the resolved next_type_id with size. 2810 * They will be stored in the current modifier's 2811 * resolved_ids and resolved_sizes such that it can 2812 * save us a few type-following when we use it later (e.g. in 2813 * pretty print). 2814 */ 2815 if (!btf_type_id_size(btf, &next_type_id, NULL)) { 2816 if (env_type_is_resolved(env, next_type_id)) 2817 next_type = btf_type_id_resolve(btf, &next_type_id); 2818 2819 /* "typedef void new_void", "const void"...etc */ 2820 if (!btf_type_is_void(next_type) && 2821 !btf_type_is_fwd(next_type) && 2822 !btf_type_is_func_proto(next_type)) { 2823 btf_verifier_log_type(env, v->t, "Invalid type_id"); 2824 return -EINVAL; 2825 } 2826 } 2827 2828 env_stack_pop_resolved(env, next_type_id, 0); 2829 2830 return 0; 2831 } 2832 2833 static int btf_var_resolve(struct btf_verifier_env *env, 2834 const struct resolve_vertex *v) 2835 { 2836 const struct btf_type *next_type; 2837 const struct btf_type *t = v->t; 2838 u32 next_type_id = t->type; 2839 struct btf *btf = env->btf; 2840 2841 next_type = btf_type_by_id(btf, next_type_id); 2842 if (!next_type || btf_type_is_resolve_source_only(next_type)) { 2843 btf_verifier_log_type(env, v->t, "Invalid type_id"); 2844 return -EINVAL; 2845 } 2846 2847 if (!env_type_is_resolve_sink(env, next_type) && 2848 !env_type_is_resolved(env, next_type_id)) 2849 return env_stack_push(env, next_type, next_type_id); 2850 2851 if (btf_type_is_modifier(next_type)) { 2852 const struct btf_type *resolved_type; 2853 u32 resolved_type_id; 2854 2855 resolved_type_id = next_type_id; 2856 resolved_type = btf_type_id_resolve(btf, &resolved_type_id); 2857 2858 if (btf_type_is_ptr(resolved_type) && 2859 !env_type_is_resolve_sink(env, resolved_type) && 2860 !env_type_is_resolved(env, resolved_type_id)) 2861 return env_stack_push(env, resolved_type, 2862 resolved_type_id); 2863 } 2864 2865 /* We must resolve to something concrete at this point, no 2866 * forward types or similar that would resolve to size of 2867 * zero is allowed. 2868 */ 2869 if (!btf_type_id_size(btf, &next_type_id, NULL)) { 2870 btf_verifier_log_type(env, v->t, "Invalid type_id"); 2871 return -EINVAL; 2872 } 2873 2874 env_stack_pop_resolved(env, next_type_id, 0); 2875 2876 return 0; 2877 } 2878 2879 static int btf_ptr_resolve(struct btf_verifier_env *env, 2880 const struct resolve_vertex *v) 2881 { 2882 const struct btf_type *next_type; 2883 const struct btf_type *t = v->t; 2884 u32 next_type_id = t->type; 2885 struct btf *btf = env->btf; 2886 2887 next_type = btf_type_by_id(btf, next_type_id); 2888 if (!next_type || btf_type_is_resolve_source_only(next_type)) { 2889 btf_verifier_log_type(env, v->t, "Invalid type_id"); 2890 return -EINVAL; 2891 } 2892 2893 if (!env_type_is_resolve_sink(env, next_type) && 2894 !env_type_is_resolved(env, next_type_id)) 2895 return env_stack_push(env, next_type, next_type_id); 2896 2897 /* If the modifier was RESOLVED during RESOLVE_STRUCT_OR_ARRAY, 2898 * the modifier may have stopped resolving when it was resolved 2899 * to a ptr (last-resolved-ptr). 2900 * 2901 * We now need to continue from the last-resolved-ptr to 2902 * ensure the last-resolved-ptr will not referring back to 2903 * the current ptr (t). 2904 */ 2905 if (btf_type_is_modifier(next_type)) { 2906 const struct btf_type *resolved_type; 2907 u32 resolved_type_id; 2908 2909 resolved_type_id = next_type_id; 2910 resolved_type = btf_type_id_resolve(btf, &resolved_type_id); 2911 2912 if (btf_type_is_ptr(resolved_type) && 2913 !env_type_is_resolve_sink(env, resolved_type) && 2914 !env_type_is_resolved(env, resolved_type_id)) 2915 return env_stack_push(env, resolved_type, 2916 resolved_type_id); 2917 } 2918 2919 if (!btf_type_id_size(btf, &next_type_id, NULL)) { 2920 if (env_type_is_resolved(env, next_type_id)) 2921 next_type = btf_type_id_resolve(btf, &next_type_id); 2922 2923 if (!btf_type_is_void(next_type) && 2924 !btf_type_is_fwd(next_type) && 2925 !btf_type_is_func_proto(next_type)) { 2926 btf_verifier_log_type(env, v->t, "Invalid type_id"); 2927 return -EINVAL; 2928 } 2929 } 2930 2931 env_stack_pop_resolved(env, next_type_id, 0); 2932 2933 return 0; 2934 } 2935 2936 static void btf_modifier_show(const struct btf *btf, 2937 const struct btf_type *t, 2938 u32 type_id, void *data, 2939 u8 bits_offset, struct btf_show *show) 2940 { 2941 if (btf->resolved_ids) 2942 t = btf_type_id_resolve(btf, &type_id); 2943 else 2944 t = btf_type_skip_modifiers(btf, type_id, NULL); 2945 2946 /* 2947 * A modifier can resolve to void, which has no show op; print a 2948 * placeholder rather than dereferencing NULL. 2949 */ 2950 if (!btf_type_ops(t)) 2951 btf_df_show(btf, t, type_id, data, bits_offset, show); 2952 else 2953 btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); 2954 } 2955 2956 static void btf_var_show(const struct btf *btf, const struct btf_type *t, 2957 u32 type_id, void *data, u8 bits_offset, 2958 struct btf_show *show) 2959 { 2960 /* 2961 * btf_type_id_resolve() dereferences btf->resolved_ids, which is NULL 2962 * for a base BTF (e.g. the vmlinux BTF that bpf_snprintf_btf() uses). 2963 * Resolve the var's type directly in that case. 2964 */ 2965 if (btf->resolved_ids) 2966 t = btf_type_id_resolve(btf, &type_id); 2967 else 2968 t = btf_type_skip_modifiers(btf, t->type, &type_id); 2969 2970 btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); 2971 } 2972 2973 static void btf_ptr_show(const struct btf *btf, const struct btf_type *t, 2974 u32 type_id, void *data, u8 bits_offset, 2975 struct btf_show *show) 2976 { 2977 void *safe_data; 2978 2979 safe_data = btf_show_start_type(show, t, type_id, data); 2980 if (!safe_data) 2981 return; 2982 2983 /* It is a hashed value unless BTF_SHOW_PTR_RAW is specified */ 2984 if (show->flags & BTF_SHOW_PTR_RAW) 2985 btf_show_type_value(show, "0x%px", *(void **)safe_data); 2986 else 2987 btf_show_type_value(show, "0x%p", *(void **)safe_data); 2988 btf_show_end_type(show); 2989 } 2990 2991 static void btf_ref_type_log(struct btf_verifier_env *env, 2992 const struct btf_type *t) 2993 { 2994 btf_verifier_log(env, "type_id=%u", t->type); 2995 } 2996 2997 static const struct btf_kind_operations modifier_ops = { 2998 .check_meta = btf_ref_type_check_meta, 2999 .resolve = btf_modifier_resolve, 3000 .check_member = btf_modifier_check_member, 3001 .check_kflag_member = btf_modifier_check_kflag_member, 3002 .log_details = btf_ref_type_log, 3003 .show = btf_modifier_show, 3004 }; 3005 3006 static const struct btf_kind_operations ptr_ops = { 3007 .check_meta = btf_ref_type_check_meta, 3008 .resolve = btf_ptr_resolve, 3009 .check_member = btf_ptr_check_member, 3010 .check_kflag_member = btf_generic_check_kflag_member, 3011 .log_details = btf_ref_type_log, 3012 .show = btf_ptr_show, 3013 }; 3014 3015 static s32 btf_fwd_check_meta(struct btf_verifier_env *env, 3016 const struct btf_type *t, 3017 u32 meta_left) 3018 { 3019 if (btf_type_vlen(t)) { 3020 btf_verifier_log_type(env, t, "vlen != 0"); 3021 return -EINVAL; 3022 } 3023 3024 if (t->type) { 3025 btf_verifier_log_type(env, t, "type != 0"); 3026 return -EINVAL; 3027 } 3028 3029 /* fwd type must have a valid name */ 3030 if (!t->name_off || 3031 !btf_name_valid_identifier(env->btf, t->name_off)) { 3032 btf_verifier_log_type(env, t, "Invalid name"); 3033 return -EINVAL; 3034 } 3035 3036 btf_verifier_log_type(env, t, NULL); 3037 3038 return 0; 3039 } 3040 3041 static void btf_fwd_type_log(struct btf_verifier_env *env, 3042 const struct btf_type *t) 3043 { 3044 btf_verifier_log(env, "%s", btf_type_kflag(t) ? "union" : "struct"); 3045 } 3046 3047 static const struct btf_kind_operations fwd_ops = { 3048 .check_meta = btf_fwd_check_meta, 3049 .resolve = btf_df_resolve, 3050 .check_member = btf_df_check_member, 3051 .check_kflag_member = btf_df_check_kflag_member, 3052 .log_details = btf_fwd_type_log, 3053 .show = btf_df_show, 3054 }; 3055 3056 static int btf_array_check_member(struct btf_verifier_env *env, 3057 const struct btf_type *struct_type, 3058 const struct btf_member *member, 3059 const struct btf_type *member_type) 3060 { 3061 u32 struct_bits_off = member->offset; 3062 u32 struct_size, bytes_offset; 3063 u32 array_type_id, array_size; 3064 struct btf *btf = env->btf; 3065 3066 if (BITS_PER_BYTE_MASKED(struct_bits_off)) { 3067 btf_verifier_log_member(env, struct_type, member, 3068 "Member is not byte aligned"); 3069 return -EINVAL; 3070 } 3071 3072 array_type_id = member->type; 3073 btf_type_id_size(btf, &array_type_id, &array_size); 3074 struct_size = struct_type->size; 3075 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); 3076 if (struct_size - bytes_offset < array_size) { 3077 btf_verifier_log_member(env, struct_type, member, 3078 "Member exceeds struct_size"); 3079 return -EINVAL; 3080 } 3081 3082 return 0; 3083 } 3084 3085 static s32 btf_array_check_meta(struct btf_verifier_env *env, 3086 const struct btf_type *t, 3087 u32 meta_left) 3088 { 3089 const struct btf_array *array = btf_type_array(t); 3090 u32 meta_needed = sizeof(*array); 3091 3092 if (meta_left < meta_needed) { 3093 btf_verifier_log_basic(env, t, 3094 "meta_left:%u meta_needed:%u", 3095 meta_left, meta_needed); 3096 return -EINVAL; 3097 } 3098 3099 /* array type should not have a name */ 3100 if (t->name_off) { 3101 btf_verifier_log_type(env, t, "Invalid name"); 3102 return -EINVAL; 3103 } 3104 3105 if (btf_type_vlen(t)) { 3106 btf_verifier_log_type(env, t, "vlen != 0"); 3107 return -EINVAL; 3108 } 3109 3110 if (btf_type_kflag(t)) { 3111 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 3112 return -EINVAL; 3113 } 3114 3115 if (t->size) { 3116 btf_verifier_log_type(env, t, "size != 0"); 3117 return -EINVAL; 3118 } 3119 3120 /* Array elem type and index type cannot be in type void, 3121 * so !array->type and !array->index_type are not allowed. 3122 */ 3123 if (!array->type || !BTF_TYPE_ID_VALID(array->type)) { 3124 btf_verifier_log_type(env, t, "Invalid elem"); 3125 return -EINVAL; 3126 } 3127 3128 if (!array->index_type || !BTF_TYPE_ID_VALID(array->index_type)) { 3129 btf_verifier_log_type(env, t, "Invalid index"); 3130 return -EINVAL; 3131 } 3132 3133 btf_verifier_log_type(env, t, NULL); 3134 3135 return meta_needed; 3136 } 3137 3138 static int btf_array_resolve(struct btf_verifier_env *env, 3139 const struct resolve_vertex *v) 3140 { 3141 const struct btf_array *array = btf_type_array(v->t); 3142 const struct btf_type *elem_type, *index_type; 3143 u32 elem_type_id, index_type_id; 3144 struct btf *btf = env->btf; 3145 u32 elem_size; 3146 3147 /* Check array->index_type */ 3148 index_type_id = array->index_type; 3149 index_type = btf_type_by_id(btf, index_type_id); 3150 if (btf_type_nosize_or_null(index_type) || 3151 btf_type_is_resolve_source_only(index_type)) { 3152 btf_verifier_log_type(env, v->t, "Invalid index"); 3153 return -EINVAL; 3154 } 3155 3156 if (!env_type_is_resolve_sink(env, index_type) && 3157 !env_type_is_resolved(env, index_type_id)) 3158 return env_stack_push(env, index_type, index_type_id); 3159 3160 index_type = btf_type_id_size(btf, &index_type_id, NULL); 3161 if (!index_type || !btf_type_is_int(index_type) || 3162 !btf_type_int_is_regular(index_type)) { 3163 btf_verifier_log_type(env, v->t, "Invalid index"); 3164 return -EINVAL; 3165 } 3166 3167 /* Check array->type */ 3168 elem_type_id = array->type; 3169 elem_type = btf_type_by_id(btf, elem_type_id); 3170 if (btf_type_nosize_or_null(elem_type) || 3171 btf_type_is_resolve_source_only(elem_type)) { 3172 btf_verifier_log_type(env, v->t, 3173 "Invalid elem"); 3174 return -EINVAL; 3175 } 3176 3177 if (!env_type_is_resolve_sink(env, elem_type) && 3178 !env_type_is_resolved(env, elem_type_id)) 3179 return env_stack_push(env, elem_type, elem_type_id); 3180 3181 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size); 3182 if (!elem_type) { 3183 btf_verifier_log_type(env, v->t, "Invalid elem"); 3184 return -EINVAL; 3185 } 3186 3187 if (btf_type_is_int(elem_type) && !btf_type_int_is_regular(elem_type)) { 3188 btf_verifier_log_type(env, v->t, "Invalid array of int"); 3189 return -EINVAL; 3190 } 3191 3192 if (array->nelems && elem_size > U32_MAX / array->nelems) { 3193 btf_verifier_log_type(env, v->t, 3194 "Array size overflows U32_MAX"); 3195 return -EINVAL; 3196 } 3197 3198 env_stack_pop_resolved(env, elem_type_id, elem_size * array->nelems); 3199 3200 return 0; 3201 } 3202 3203 static void btf_array_log(struct btf_verifier_env *env, 3204 const struct btf_type *t) 3205 { 3206 const struct btf_array *array = btf_type_array(t); 3207 3208 btf_verifier_log(env, "type_id=%u index_type_id=%u nr_elems=%u", 3209 array->type, array->index_type, array->nelems); 3210 } 3211 3212 static void __btf_array_show(const struct btf *btf, const struct btf_type *t, 3213 u32 type_id, void *data, u8 bits_offset, 3214 struct btf_show *show) 3215 { 3216 const struct btf_array *array = btf_type_array(t); 3217 const struct btf_kind_operations *elem_ops; 3218 const struct btf_type *elem_type; 3219 u32 i, elem_size = 0, elem_type_id; 3220 u16 encoding = 0; 3221 3222 elem_type_id = array->type; 3223 elem_type = btf_type_skip_modifiers(btf, elem_type_id, NULL); 3224 if (elem_type && btf_type_has_size(elem_type)) 3225 elem_size = elem_type->size; 3226 3227 if (elem_type && btf_type_is_int(elem_type)) { 3228 u32 int_type = btf_type_int(elem_type); 3229 3230 encoding = BTF_INT_ENCODING(int_type); 3231 3232 /* 3233 * BTF_INT_CHAR encoding never seems to be set for 3234 * char arrays, so if size is 1 and element is 3235 * printable as a char, we'll do that. 3236 */ 3237 if (elem_size == 1) 3238 encoding = BTF_INT_CHAR; 3239 } 3240 3241 if (!btf_show_start_array_type(show, t, type_id, encoding, data)) 3242 return; 3243 3244 if (!elem_type) 3245 goto out; 3246 elem_ops = btf_type_ops(elem_type); 3247 3248 for (i = 0; i < array->nelems; i++) { 3249 3250 btf_show_start_array_member(show); 3251 3252 elem_ops->show(btf, elem_type, elem_type_id, data, 3253 bits_offset, show); 3254 data += elem_size; 3255 3256 btf_show_end_array_member(show); 3257 3258 if (show->state.array_terminated) 3259 break; 3260 } 3261 out: 3262 btf_show_end_array_type(show); 3263 } 3264 3265 static void btf_array_show(const struct btf *btf, const struct btf_type *t, 3266 u32 type_id, void *data, u8 bits_offset, 3267 struct btf_show *show) 3268 { 3269 const struct btf_member *m = show->state.member; 3270 3271 /* 3272 * First check if any members would be shown (are non-zero). 3273 * See comments above "struct btf_show" definition for more 3274 * details on how this works at a high-level. 3275 */ 3276 if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) { 3277 if (!show->state.depth_check) { 3278 show->state.depth_check = show->state.depth + 1; 3279 show->state.depth_to_show = 0; 3280 } 3281 __btf_array_show(btf, t, type_id, data, bits_offset, show); 3282 show->state.member = m; 3283 3284 if (show->state.depth_check != show->state.depth + 1) 3285 return; 3286 show->state.depth_check = 0; 3287 3288 if (show->state.depth_to_show <= show->state.depth) 3289 return; 3290 /* 3291 * Reaching here indicates we have recursed and found 3292 * non-zero array member(s). 3293 */ 3294 } 3295 __btf_array_show(btf, t, type_id, data, bits_offset, show); 3296 } 3297 3298 static const struct btf_kind_operations array_ops = { 3299 .check_meta = btf_array_check_meta, 3300 .resolve = btf_array_resolve, 3301 .check_member = btf_array_check_member, 3302 .check_kflag_member = btf_generic_check_kflag_member, 3303 .log_details = btf_array_log, 3304 .show = btf_array_show, 3305 }; 3306 3307 static int btf_struct_check_member(struct btf_verifier_env *env, 3308 const struct btf_type *struct_type, 3309 const struct btf_member *member, 3310 const struct btf_type *member_type) 3311 { 3312 u32 struct_bits_off = member->offset; 3313 u32 struct_size, bytes_offset; 3314 3315 if (BITS_PER_BYTE_MASKED(struct_bits_off)) { 3316 btf_verifier_log_member(env, struct_type, member, 3317 "Member is not byte aligned"); 3318 return -EINVAL; 3319 } 3320 3321 struct_size = struct_type->size; 3322 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); 3323 if (struct_size - bytes_offset < member_type->size) { 3324 btf_verifier_log_member(env, struct_type, member, 3325 "Member exceeds struct_size"); 3326 return -EINVAL; 3327 } 3328 3329 return 0; 3330 } 3331 3332 static s32 btf_struct_check_meta(struct btf_verifier_env *env, 3333 const struct btf_type *t, 3334 u32 meta_left) 3335 { 3336 bool is_union = BTF_INFO_KIND(t->info) == BTF_KIND_UNION; 3337 const struct btf_member *member; 3338 u32 meta_needed, last_offset; 3339 struct btf *btf = env->btf; 3340 u32 struct_size = t->size; 3341 u32 offset; 3342 u32 i; 3343 3344 meta_needed = btf_type_vlen(t) * sizeof(*member); 3345 if (meta_left < meta_needed) { 3346 btf_verifier_log_basic(env, t, 3347 "meta_left:%u meta_needed:%u", 3348 meta_left, meta_needed); 3349 return -EINVAL; 3350 } 3351 3352 /* struct type either no name or a valid one */ 3353 if (t->name_off && 3354 !btf_name_valid_identifier(env->btf, t->name_off)) { 3355 btf_verifier_log_type(env, t, "Invalid name"); 3356 return -EINVAL; 3357 } 3358 3359 btf_verifier_log_type(env, t, NULL); 3360 3361 last_offset = 0; 3362 for_each_member(i, t, member) { 3363 if (!btf_name_offset_valid(btf, member->name_off)) { 3364 btf_verifier_log_member(env, t, member, 3365 "Invalid member name_offset:%u", 3366 member->name_off); 3367 return -EINVAL; 3368 } 3369 3370 /* struct member either no name or a valid one */ 3371 if (member->name_off && 3372 !btf_name_valid_identifier(btf, member->name_off)) { 3373 btf_verifier_log_member(env, t, member, "Invalid name"); 3374 return -EINVAL; 3375 } 3376 /* A member cannot be in type void */ 3377 if (!member->type || !BTF_TYPE_ID_VALID(member->type)) { 3378 btf_verifier_log_member(env, t, member, 3379 "Invalid type_id"); 3380 return -EINVAL; 3381 } 3382 3383 offset = __btf_member_bit_offset(t, member); 3384 if (is_union && offset) { 3385 btf_verifier_log_member(env, t, member, 3386 "Invalid member bits_offset"); 3387 return -EINVAL; 3388 } 3389 3390 /* 3391 * ">" instead of ">=" because the last member could be 3392 * "char a[0];" 3393 */ 3394 if (last_offset > offset) { 3395 btf_verifier_log_member(env, t, member, 3396 "Invalid member bits_offset"); 3397 return -EINVAL; 3398 } 3399 3400 if (BITS_ROUNDUP_BYTES(offset) > struct_size) { 3401 btf_verifier_log_member(env, t, member, 3402 "Member bits_offset exceeds its struct size"); 3403 return -EINVAL; 3404 } 3405 3406 btf_verifier_log_member(env, t, member, NULL); 3407 last_offset = offset; 3408 } 3409 3410 return meta_needed; 3411 } 3412 3413 static int btf_struct_resolve(struct btf_verifier_env *env, 3414 const struct resolve_vertex *v) 3415 { 3416 const struct btf_member *member; 3417 int err; 3418 u32 i; 3419 3420 /* Before continue resolving the next_member, 3421 * ensure the last member is indeed resolved to a 3422 * type with size info. 3423 */ 3424 if (v->next_member) { 3425 const struct btf_type *last_member_type; 3426 const struct btf_member *last_member; 3427 u32 last_member_type_id; 3428 3429 last_member = btf_type_member(v->t) + v->next_member - 1; 3430 last_member_type_id = last_member->type; 3431 if (WARN_ON_ONCE(!env_type_is_resolved(env, 3432 last_member_type_id))) 3433 return -EINVAL; 3434 3435 last_member_type = btf_type_by_id(env->btf, 3436 last_member_type_id); 3437 if (btf_type_kflag(v->t)) 3438 err = btf_type_ops(last_member_type)->check_kflag_member(env, v->t, 3439 last_member, 3440 last_member_type); 3441 else 3442 err = btf_type_ops(last_member_type)->check_member(env, v->t, 3443 last_member, 3444 last_member_type); 3445 if (err) 3446 return err; 3447 } 3448 3449 for_each_member_from(i, v->next_member, v->t, member) { 3450 u32 member_type_id = member->type; 3451 const struct btf_type *member_type = btf_type_by_id(env->btf, 3452 member_type_id); 3453 3454 if (btf_type_nosize_or_null(member_type) || 3455 btf_type_is_resolve_source_only(member_type)) { 3456 btf_verifier_log_member(env, v->t, member, 3457 "Invalid member"); 3458 return -EINVAL; 3459 } 3460 3461 if (!env_type_is_resolve_sink(env, member_type) && 3462 !env_type_is_resolved(env, member_type_id)) { 3463 env_stack_set_next_member(env, i + 1); 3464 return env_stack_push(env, member_type, member_type_id); 3465 } 3466 3467 if (btf_type_kflag(v->t)) 3468 err = btf_type_ops(member_type)->check_kflag_member(env, v->t, 3469 member, 3470 member_type); 3471 else 3472 err = btf_type_ops(member_type)->check_member(env, v->t, 3473 member, 3474 member_type); 3475 if (err) 3476 return err; 3477 } 3478 3479 env_stack_pop_resolved(env, 0, 0); 3480 3481 return 0; 3482 } 3483 3484 static void btf_struct_log(struct btf_verifier_env *env, 3485 const struct btf_type *t) 3486 { 3487 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); 3488 } 3489 3490 enum { 3491 BTF_FIELD_IGNORE = 0, 3492 BTF_FIELD_FOUND = 1, 3493 }; 3494 3495 struct btf_field_info { 3496 enum btf_field_type type; 3497 u32 off; 3498 union { 3499 struct { 3500 u32 type_id; 3501 } kptr; 3502 struct { 3503 const char *node_name; 3504 u32 value_btf_id; 3505 } graph_root; 3506 }; 3507 }; 3508 3509 static int btf_find_struct(const struct btf *btf, const struct btf_type *t, 3510 u32 off, int sz, enum btf_field_type field_type, 3511 struct btf_field_info *info) 3512 { 3513 if (!__btf_type_is_struct(t)) 3514 return BTF_FIELD_IGNORE; 3515 if (t->size != sz) 3516 return BTF_FIELD_IGNORE; 3517 info->type = field_type; 3518 info->off = off; 3519 return BTF_FIELD_FOUND; 3520 } 3521 3522 struct btf_type_tag_match { 3523 const char *name; 3524 u32 flag; 3525 }; 3526 3527 struct btf_type_tag_walk_ctx { 3528 const struct btf_type *t; /* Input/Output */ 3529 u32 id; /* Output */ 3530 u32 res; /* Output */ 3531 }; 3532 3533 static int btf_type_tag_walk(const struct btf *btf, 3534 struct btf_type_tag_walk_ctx *ctx, 3535 const struct btf_type_tag_match *matches, 3536 u32 match_cnt) 3537 { 3538 const struct btf_type *t = ctx->t; 3539 u32 res = 0; 3540 const char *tag; 3541 u32 id, i; 3542 3543 do { 3544 id = t->type; 3545 t = btf_type_by_id(btf, id); 3546 3547 if (!btf_type_is_modifier(t)) 3548 break; 3549 3550 if (!btf_type_is_type_tag(t) || btf_type_kflag(t)) 3551 continue; 3552 3553 tag = __btf_name_by_offset(btf, t->name_off); 3554 for (i = 0; i < match_cnt; i++) { 3555 if (strcmp(tag, matches[i].name)) 3556 continue; 3557 res |= matches[i].flag; 3558 break; 3559 } 3560 } while (true); 3561 3562 /* We only support a single tag. */ 3563 if (hweight32(res) > 1) 3564 return -EINVAL; 3565 3566 ctx->t = t; 3567 ctx->id = id; 3568 ctx->res = res; 3569 3570 return 0; 3571 } 3572 3573 /* Whether the pointee of pointer type @t carries the type tag @name */ 3574 static bool btf_ptr_has_type_tag(const struct btf *btf, const struct btf_type *t, const char *name) 3575 { 3576 if (!btf_type_is_ptr(t)) 3577 return false; 3578 3579 for (t = btf_type_by_id(btf, t->type); btf_type_is_modifier(t); 3580 t = btf_type_by_id(btf, t->type)) { 3581 if (!btf_type_is_type_tag(t) || btf_type_kflag(t)) 3582 continue; 3583 if (!strcmp(__btf_name_by_offset(btf, t->name_off), name)) 3584 return true; 3585 } 3586 3587 return false; 3588 } 3589 3590 bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t) 3591 { 3592 return btf_ptr_has_type_tag(btf, t, "arena"); 3593 } 3594 3595 static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, 3596 u32 off, int sz, struct btf_field_info *info, u32 field_mask) 3597 { 3598 static const struct btf_type_tag_match kptr_type_tags[] = { 3599 { "kptr_untrusted", BPF_KPTR_UNREF }, 3600 { "kptr", BPF_KPTR_REF }, 3601 { "percpu_kptr", BPF_KPTR_PERCPU }, 3602 { "uptr", BPF_UPTR }, 3603 }; 3604 struct btf_type_tag_walk_ctx ctx; 3605 enum btf_field_type type = 0; 3606 int err; 3607 u32 res_id; 3608 3609 /* Permit modifiers on the pointer itself */ 3610 if (btf_type_is_volatile(t)) 3611 t = btf_type_by_id(btf, t->type); 3612 /* For PTR, sz is always == 8 */ 3613 if (!btf_type_is_ptr(t)) 3614 return BTF_FIELD_IGNORE; 3615 3616 ctx.t = t; 3617 err = btf_type_tag_walk(btf, &ctx, kptr_type_tags, 3618 ARRAY_SIZE(kptr_type_tags)); 3619 if (err) 3620 return err; 3621 3622 t = ctx.t; 3623 res_id = ctx.id; 3624 type = ctx.res; 3625 3626 if (!(type & field_mask)) 3627 return BTF_FIELD_IGNORE; 3628 3629 /* Only pointer to struct is allowed */ 3630 if (!__btf_type_is_struct(t)) 3631 return -EINVAL; 3632 3633 info->type = type; 3634 info->off = off; 3635 info->kptr.type_id = res_id; 3636 return BTF_FIELD_FOUND; 3637 } 3638 3639 int btf_find_next_decl_tag(const struct btf *btf, const struct btf_type *pt, 3640 int comp_idx, const char *tag_key, int last_id) 3641 { 3642 int len = strlen(tag_key); 3643 int i, n; 3644 3645 for (i = last_id + 1, n = btf_nr_types(btf); i < n; i++) { 3646 const struct btf_type *t = btf_type_by_id(btf, i); 3647 3648 if (!btf_type_is_decl_tag(t)) 3649 continue; 3650 if (pt != btf_type_by_id(btf, t->type)) 3651 continue; 3652 if (btf_type_decl_tag(t)->component_idx != comp_idx) 3653 continue; 3654 if (strncmp(__btf_name_by_offset(btf, t->name_off), tag_key, len)) 3655 continue; 3656 return i; 3657 } 3658 return -ENOENT; 3659 } 3660 3661 const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt, 3662 int comp_idx, const char *tag_key) 3663 { 3664 const char *value = NULL; 3665 const struct btf_type *t; 3666 int len, id; 3667 3668 id = btf_find_next_decl_tag(btf, pt, comp_idx, tag_key, 3669 btf_named_start_id(btf, false) - 1); 3670 if (id < 0) 3671 return ERR_PTR(id); 3672 3673 t = btf_type_by_id(btf, id); 3674 len = strlen(tag_key); 3675 value = __btf_name_by_offset(btf, t->name_off) + len; 3676 3677 /* Prevent duplicate entries for same type */ 3678 id = btf_find_next_decl_tag(btf, pt, comp_idx, tag_key, id); 3679 if (id >= 0) 3680 return ERR_PTR(-EEXIST); 3681 3682 return value; 3683 } 3684 3685 static int 3686 btf_find_graph_root(const struct btf *btf, const struct btf_type *pt, 3687 const struct btf_type *t, int comp_idx, u32 off, 3688 int sz, struct btf_field_info *info, 3689 enum btf_field_type head_type) 3690 { 3691 const char *node_field_name; 3692 const char *value_type; 3693 s32 id; 3694 3695 if (!__btf_type_is_struct(t)) 3696 return BTF_FIELD_IGNORE; 3697 if (t->size != sz) 3698 return BTF_FIELD_IGNORE; 3699 value_type = btf_find_decl_tag_value(btf, pt, comp_idx, "contains:"); 3700 if (IS_ERR(value_type)) 3701 return -EINVAL; 3702 node_field_name = strstr(value_type, ":"); 3703 if (!node_field_name) 3704 return -EINVAL; 3705 value_type = kstrndup(value_type, node_field_name - value_type, 3706 GFP_KERNEL_ACCOUNT | __GFP_NOWARN); 3707 if (!value_type) 3708 return -ENOMEM; 3709 id = btf_find_by_name_kind(btf, value_type, BTF_KIND_STRUCT); 3710 kfree(value_type); 3711 if (id < 0) 3712 return id; 3713 node_field_name++; 3714 if (str_is_empty(node_field_name)) 3715 return -EINVAL; 3716 info->type = head_type; 3717 info->off = off; 3718 info->graph_root.value_btf_id = id; 3719 info->graph_root.node_name = node_field_name; 3720 return BTF_FIELD_FOUND; 3721 } 3722 3723 static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_type, 3724 u32 field_mask, u32 *seen_mask, int *align, int *sz) 3725 { 3726 const struct { 3727 enum btf_field_type type; 3728 const char *const name; 3729 const bool is_unique; 3730 } field_types[] = { 3731 { BPF_SPIN_LOCK, "bpf_spin_lock", true }, 3732 { BPF_RES_SPIN_LOCK, "bpf_res_spin_lock", true }, 3733 { BPF_TIMER, "bpf_timer", true }, 3734 { BPF_WORKQUEUE, "bpf_wq", true }, 3735 { BPF_TASK_WORK, "bpf_task_work", true }, 3736 { BPF_RCU_HEAD, "bpf_rcu_head", true }, 3737 { BPF_LIST_HEAD, "bpf_list_head", false }, 3738 { BPF_LIST_NODE, "bpf_list_node", false }, 3739 { BPF_RB_ROOT, "bpf_rb_root", false }, 3740 { BPF_RB_NODE, "bpf_rb_node", false }, 3741 { BPF_REFCOUNT, "bpf_refcount", true }, 3742 }; 3743 int type = 0, i; 3744 const char *name = __btf_name_by_offset(btf, var_type->name_off); 3745 const char *field_type_name; 3746 enum btf_field_type field_type; 3747 bool is_unique; 3748 3749 for (i = 0; i < ARRAY_SIZE(field_types); ++i) { 3750 field_type = field_types[i].type; 3751 field_type_name = field_types[i].name; 3752 is_unique = field_types[i].is_unique; 3753 if (!(field_mask & field_type) || strcmp(name, field_type_name)) 3754 continue; 3755 if (is_unique) { 3756 if (*seen_mask & field_type) 3757 return -E2BIG; 3758 *seen_mask |= field_type; 3759 } 3760 type = field_type; 3761 goto end; 3762 } 3763 3764 /* Only return BPF_KPTR when all other types with matchable names fail */ 3765 if (field_mask & (BPF_KPTR | BPF_UPTR) && !__btf_type_is_struct(var_type)) { 3766 type = BPF_KPTR_REF; 3767 goto end; 3768 } 3769 return 0; 3770 end: 3771 *sz = btf_field_type_size(type); 3772 *align = btf_field_type_align(type); 3773 return type; 3774 } 3775 3776 /* Repeat a number of fields for a specified number of times. 3777 * 3778 * Copy the fields starting from the first field and repeat them for 3779 * repeat_cnt times. The fields are repeated by adding the offset of each 3780 * field with 3781 * (i + 1) * elem_size 3782 * where i is the repeat index and elem_size is the size of an element. 3783 */ 3784 static int btf_repeat_fields(struct btf_field_info *info, int info_cnt, 3785 u32 field_cnt, u32 repeat_cnt, u32 elem_size) 3786 { 3787 u32 i, j, total_cnt, total_repeats; 3788 u32 cur; 3789 3790 /* Ensure not repeating fields that should not be repeated. */ 3791 for (i = 0; i < field_cnt; i++) { 3792 switch (info[i].type) { 3793 case BPF_KPTR_UNREF: 3794 case BPF_KPTR_REF: 3795 case BPF_KPTR_PERCPU: 3796 case BPF_UPTR: 3797 case BPF_LIST_HEAD: 3798 case BPF_RB_ROOT: 3799 break; 3800 default: 3801 return -EINVAL; 3802 } 3803 } 3804 3805 if (check_add_overflow(repeat_cnt, 1, &total_repeats) || 3806 check_mul_overflow(field_cnt, total_repeats, &total_cnt) || 3807 total_cnt > (u32)info_cnt) 3808 return -E2BIG; 3809 3810 cur = field_cnt; 3811 for (i = 0; i < repeat_cnt; i++) { 3812 memcpy(&info[cur], &info[0], field_cnt * sizeof(info[0])); 3813 for (j = 0; j < field_cnt; j++) 3814 info[cur++].off += (i + 1) * elem_size; 3815 } 3816 3817 return 0; 3818 } 3819 3820 static int btf_find_struct_field(const struct btf *btf, 3821 const struct btf_type *t, u32 field_mask, 3822 struct btf_field_info *info, int info_cnt, 3823 u32 level, u32 *seen_mask); 3824 3825 /* Find special fields in the struct type of a field. 3826 * 3827 * This function is used to find fields of special types that is not a 3828 * global variable or a direct field of a struct type. It also handles the 3829 * repetition if it is the element type of an array. 3830 */ 3831 static int btf_find_nested_struct(const struct btf *btf, const struct btf_type *t, 3832 u32 off, u32 nelems, 3833 u32 field_mask, struct btf_field_info *info, 3834 int info_cnt, u32 level, u32 *seen_mask) 3835 { 3836 int ret, err, i; 3837 3838 level++; 3839 if (level >= MAX_RESOLVE_DEPTH) 3840 return -E2BIG; 3841 3842 ret = btf_find_struct_field(btf, t, field_mask, info, info_cnt, level, seen_mask); 3843 3844 if (ret <= 0) 3845 return ret; 3846 3847 /* Shift the offsets of the nested struct fields to the offsets 3848 * related to the container. 3849 */ 3850 for (i = 0; i < ret; i++) 3851 info[i].off += off; 3852 3853 if (nelems > 1) { 3854 err = btf_repeat_fields(info, info_cnt, ret, nelems - 1, t->size); 3855 if (err == 0) 3856 ret *= nelems; 3857 else 3858 ret = err; 3859 } 3860 3861 return ret; 3862 } 3863 3864 static int btf_find_field_one(const struct btf *btf, 3865 const struct btf_type *var, 3866 const struct btf_type *var_type, 3867 int var_idx, 3868 u32 off, u32 expected_size, 3869 u32 field_mask, u32 *seen_mask, 3870 struct btf_field_info *info, int info_cnt, 3871 u32 level) 3872 { 3873 int ret, align, sz, field_type; 3874 struct btf_field_info tmp; 3875 const struct btf_array *array; 3876 u32 i, nelems = 1; 3877 3878 /* Walk into array types to find the element type and the number of 3879 * elements in the (flattened) array. 3880 */ 3881 for (i = 0; i < MAX_RESOLVE_DEPTH && btf_type_is_array(var_type); i++) { 3882 array = btf_array(var_type); 3883 nelems *= array->nelems; 3884 var_type = btf_type_by_id(btf, array->type); 3885 } 3886 if (i == MAX_RESOLVE_DEPTH) 3887 return -E2BIG; 3888 if (nelems == 0) 3889 return 0; 3890 3891 field_type = btf_get_field_type(btf, var_type, 3892 field_mask, seen_mask, &align, &sz); 3893 /* Look into variables of struct types */ 3894 if (!field_type && __btf_type_is_struct(var_type)) { 3895 sz = var_type->size; 3896 if (expected_size && expected_size != sz * nelems) 3897 return 0; 3898 ret = btf_find_nested_struct(btf, var_type, off, nelems, field_mask, 3899 &info[0], info_cnt, level, seen_mask); 3900 return ret; 3901 } 3902 3903 if (field_type == 0) 3904 return 0; 3905 if (field_type < 0) 3906 return field_type; 3907 3908 if (expected_size && expected_size != sz * nelems) 3909 return 0; 3910 if (off % align) 3911 return 0; 3912 3913 switch (field_type) { 3914 case BPF_SPIN_LOCK: 3915 case BPF_RES_SPIN_LOCK: 3916 case BPF_TIMER: 3917 case BPF_WORKQUEUE: 3918 case BPF_LIST_NODE: 3919 case BPF_RB_NODE: 3920 case BPF_REFCOUNT: 3921 case BPF_TASK_WORK: 3922 case BPF_RCU_HEAD: 3923 ret = btf_find_struct(btf, var_type, off, sz, field_type, 3924 info_cnt ? &info[0] : &tmp); 3925 if (ret < 0) 3926 return ret; 3927 break; 3928 case BPF_KPTR_UNREF: 3929 case BPF_KPTR_REF: 3930 case BPF_KPTR_PERCPU: 3931 case BPF_UPTR: 3932 ret = btf_find_kptr(btf, var_type, off, sz, 3933 info_cnt ? &info[0] : &tmp, field_mask); 3934 if (ret < 0) 3935 return ret; 3936 break; 3937 case BPF_LIST_HEAD: 3938 case BPF_RB_ROOT: 3939 ret = btf_find_graph_root(btf, var, var_type, 3940 var_idx, off, sz, 3941 info_cnt ? &info[0] : &tmp, 3942 field_type); 3943 if (ret < 0) 3944 return ret; 3945 break; 3946 default: 3947 return -EFAULT; 3948 } 3949 3950 if (ret == BTF_FIELD_IGNORE) 3951 return 0; 3952 if (!info_cnt) 3953 return -E2BIG; 3954 if (nelems > 1) { 3955 ret = btf_repeat_fields(info, info_cnt, 1, nelems - 1, sz); 3956 if (ret < 0) 3957 return ret; 3958 } 3959 return nelems; 3960 } 3961 3962 static int btf_find_struct_field(const struct btf *btf, 3963 const struct btf_type *t, u32 field_mask, 3964 struct btf_field_info *info, int info_cnt, 3965 u32 level, u32 *seen_mask) 3966 { 3967 int ret, idx = 0; 3968 const struct btf_member *member; 3969 u32 i, off; 3970 3971 for_each_member(i, t, member) { 3972 const struct btf_type *member_type = btf_type_by_id(btf, 3973 member->type); 3974 3975 off = __btf_member_bit_offset(t, member); 3976 if (off % 8) 3977 /* valid C code cannot generate such BTF */ 3978 return -EINVAL; 3979 off /= 8; 3980 3981 ret = btf_find_field_one(btf, t, member_type, i, 3982 off, 0, 3983 field_mask, seen_mask, 3984 &info[idx], info_cnt - idx, level); 3985 if (ret < 0) 3986 return ret; 3987 idx += ret; 3988 } 3989 return idx; 3990 } 3991 3992 static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, 3993 u32 field_mask, struct btf_field_info *info, 3994 int info_cnt, u32 level, u32 *seen_mask) 3995 { 3996 int ret, idx = 0; 3997 const struct btf_var_secinfo *vsi; 3998 u32 i, off; 3999 4000 for_each_vsi(i, t, vsi) { 4001 const struct btf_type *var = btf_type_by_id(btf, vsi->type); 4002 const struct btf_type *var_type = btf_type_by_id(btf, var->type); 4003 4004 off = vsi->offset; 4005 ret = btf_find_field_one(btf, var, var_type, -1, off, vsi->size, 4006 field_mask, seen_mask, 4007 &info[idx], info_cnt - idx, 4008 level); 4009 if (ret < 0) 4010 return ret; 4011 idx += ret; 4012 } 4013 return idx; 4014 } 4015 4016 static int btf_find_field(const struct btf *btf, const struct btf_type *t, 4017 u32 field_mask, struct btf_field_info *info, 4018 int info_cnt) 4019 { 4020 u32 seen_mask = 0; 4021 4022 if (__btf_type_is_struct(t)) 4023 return btf_find_struct_field(btf, t, field_mask, info, info_cnt, 0, &seen_mask); 4024 else if (btf_type_is_datasec(t)) 4025 return btf_find_datasec_var(btf, t, field_mask, info, info_cnt, 0, &seen_mask); 4026 return -EINVAL; 4027 } 4028 4029 /* Callers have to ensure the life cycle of btf if it is program BTF */ 4030 static int btf_parse_kptr(const struct btf *btf, struct btf_field *field, 4031 struct btf_field_info *info) 4032 { 4033 struct module *mod = NULL; 4034 const struct btf_type *t; 4035 /* If a matching btf type is found in kernel or module BTFs, kptr_ref 4036 * is that BTF, otherwise it's program BTF 4037 */ 4038 struct btf *kptr_btf; 4039 int ret; 4040 s32 id; 4041 4042 /* Find type in map BTF, and use it to look up the matching type 4043 * in vmlinux or module BTFs, by name and kind. 4044 */ 4045 t = btf_type_by_id(btf, info->kptr.type_id); 4046 id = bpf_find_btf_id(__btf_name_by_offset(btf, t->name_off), BTF_INFO_KIND(t->info), 4047 &kptr_btf); 4048 if (id == -ENOENT) { 4049 /* btf_parse_kptr should only be called w/ btf = program BTF */ 4050 WARN_ON_ONCE(btf_is_kernel(btf)); 4051 4052 /* Type exists only in program BTF. Assume that it's a MEM_ALLOC 4053 * kptr allocated via bpf_obj_new 4054 */ 4055 field->kptr.dtor = NULL; 4056 id = info->kptr.type_id; 4057 kptr_btf = (struct btf *)btf; 4058 goto found_dtor; 4059 } 4060 if (id < 0) 4061 return id; 4062 4063 /* Find and stash the function pointer for the destruction function that 4064 * needs to be eventually invoked from the map free path. 4065 */ 4066 if (info->type == BPF_KPTR_REF) { 4067 const struct btf_type *dtor_func; 4068 const char *dtor_func_name; 4069 unsigned long addr; 4070 s32 dtor_btf_id; 4071 4072 /* This call also serves as a whitelist of allowed objects that 4073 * can be used as a referenced pointer and be stored in a map at 4074 * the same time. 4075 */ 4076 dtor_btf_id = btf_find_dtor_kfunc(kptr_btf, id); 4077 if (dtor_btf_id < 0) { 4078 ret = dtor_btf_id; 4079 goto end_btf; 4080 } 4081 4082 dtor_func = btf_type_by_id(kptr_btf, dtor_btf_id); 4083 if (!dtor_func) { 4084 ret = -ENOENT; 4085 goto end_btf; 4086 } 4087 4088 if (btf_is_module(kptr_btf)) { 4089 mod = btf_try_get_module(kptr_btf); 4090 if (!mod) { 4091 ret = -ENXIO; 4092 goto end_btf; 4093 } 4094 } 4095 4096 /* We already verified dtor_func to be btf_type_is_func 4097 * in register_btf_id_dtor_kfuncs. 4098 */ 4099 dtor_func_name = __btf_name_by_offset(kptr_btf, dtor_func->name_off); 4100 addr = kallsyms_lookup_name(dtor_func_name); 4101 if (!addr) { 4102 ret = -EINVAL; 4103 goto end_mod; 4104 } 4105 field->kptr.dtor = (void *)addr; 4106 } 4107 4108 found_dtor: 4109 field->kptr.btf_id = id; 4110 field->kptr.btf = kptr_btf; 4111 field->kptr.module = mod; 4112 return 0; 4113 end_mod: 4114 module_put(mod); 4115 end_btf: 4116 btf_put(kptr_btf); 4117 return ret; 4118 } 4119 4120 static int btf_parse_graph_root(const struct btf *btf, 4121 struct btf_field *field, 4122 struct btf_field_info *info, 4123 const char *node_type_name, 4124 size_t node_type_align) 4125 { 4126 const struct btf_type *t, *n = NULL; 4127 const struct btf_member *member; 4128 u32 offset; 4129 int i; 4130 4131 t = btf_type_by_id(btf, info->graph_root.value_btf_id); 4132 /* We've already checked that value_btf_id is a struct type. We 4133 * just need to figure out the offset of the list_node, and 4134 * verify its type. 4135 */ 4136 for_each_member(i, t, member) { 4137 if (strcmp(info->graph_root.node_name, 4138 __btf_name_by_offset(btf, member->name_off))) 4139 continue; 4140 /* Invalid BTF, two members with same name */ 4141 if (n) 4142 return -EINVAL; 4143 n = btf_type_by_id(btf, member->type); 4144 if (!__btf_type_is_struct(n)) 4145 return -EINVAL; 4146 if (strcmp(node_type_name, __btf_name_by_offset(btf, n->name_off))) 4147 return -EINVAL; 4148 offset = __btf_member_bit_offset(n, member); 4149 if (offset % 8) 4150 return -EINVAL; 4151 offset /= 8; 4152 if (offset % node_type_align) 4153 return -EINVAL; 4154 4155 field->graph_root.btf = (struct btf *)btf; 4156 field->graph_root.value_btf_id = info->graph_root.value_btf_id; 4157 field->graph_root.node_offset = offset; 4158 } 4159 if (!n) 4160 return -ENOENT; 4161 return 0; 4162 } 4163 4164 static int btf_parse_list_head(const struct btf *btf, struct btf_field *field, 4165 struct btf_field_info *info) 4166 { 4167 return btf_parse_graph_root(btf, field, info, "bpf_list_node", 4168 __alignof__(struct bpf_list_node)); 4169 } 4170 4171 static int btf_parse_rb_root(const struct btf *btf, struct btf_field *field, 4172 struct btf_field_info *info) 4173 { 4174 return btf_parse_graph_root(btf, field, info, "bpf_rb_node", 4175 __alignof__(struct bpf_rb_node)); 4176 } 4177 4178 static int btf_field_cmp(const void *_a, const void *_b, const void *priv) 4179 { 4180 const struct btf_field *a = (const struct btf_field *)_a; 4181 const struct btf_field *b = (const struct btf_field *)_b; 4182 4183 if (a->offset < b->offset) 4184 return -1; 4185 else if (a->offset > b->offset) 4186 return 1; 4187 return 0; 4188 } 4189 4190 struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type *t, 4191 u32 field_mask, u32 value_size) 4192 { 4193 struct btf_field_info info_arr[BTF_FIELDS_MAX]; 4194 u32 next_off = 0, field_type_size; 4195 struct btf_record *rec; 4196 int ret, i, cnt; 4197 4198 ret = btf_find_field(btf, t, field_mask, info_arr, ARRAY_SIZE(info_arr)); 4199 if (ret < 0) 4200 return ERR_PTR(ret); 4201 if (!ret) 4202 return NULL; 4203 4204 cnt = ret; 4205 /* This needs to be kzalloc to zero out padding and unused fields, see 4206 * comment in btf_record_equal. 4207 */ 4208 rec = kzalloc_flex(*rec, fields, cnt, GFP_KERNEL_ACCOUNT | __GFP_NOWARN); 4209 if (!rec) 4210 return ERR_PTR(-ENOMEM); 4211 4212 rec->spin_lock_off = -EINVAL; 4213 rec->res_spin_lock_off = -EINVAL; 4214 rec->timer_off = -EINVAL; 4215 rec->wq_off = -EINVAL; 4216 rec->refcount_off = -EINVAL; 4217 rec->task_work_off = -EINVAL; 4218 rec->rcu_head_off = -EINVAL; 4219 for (i = 0; i < cnt; i++) { 4220 field_type_size = btf_field_type_size(info_arr[i].type); 4221 if (info_arr[i].off + field_type_size > value_size) { 4222 WARN_ONCE(1, "verifier bug off %d size %d", info_arr[i].off, value_size); 4223 ret = -EFAULT; 4224 goto end; 4225 } 4226 if (info_arr[i].off < next_off) { 4227 ret = -EEXIST; 4228 goto end; 4229 } 4230 next_off = info_arr[i].off + field_type_size; 4231 4232 rec->field_mask |= info_arr[i].type; 4233 rec->fields[i].offset = info_arr[i].off; 4234 rec->fields[i].type = info_arr[i].type; 4235 rec->fields[i].size = field_type_size; 4236 4237 switch (info_arr[i].type) { 4238 case BPF_SPIN_LOCK: 4239 WARN_ON_ONCE(rec->spin_lock_off >= 0); 4240 /* Cache offset for faster lookup at runtime */ 4241 rec->spin_lock_off = rec->fields[i].offset; 4242 break; 4243 case BPF_RES_SPIN_LOCK: 4244 WARN_ON_ONCE(rec->res_spin_lock_off >= 0); 4245 /* Cache offset for faster lookup at runtime */ 4246 rec->res_spin_lock_off = rec->fields[i].offset; 4247 break; 4248 case BPF_TIMER: 4249 WARN_ON_ONCE(rec->timer_off >= 0); 4250 /* Cache offset for faster lookup at runtime */ 4251 rec->timer_off = rec->fields[i].offset; 4252 break; 4253 case BPF_WORKQUEUE: 4254 WARN_ON_ONCE(rec->wq_off >= 0); 4255 /* Cache offset for faster lookup at runtime */ 4256 rec->wq_off = rec->fields[i].offset; 4257 break; 4258 case BPF_TASK_WORK: 4259 WARN_ON_ONCE(rec->task_work_off >= 0); 4260 rec->task_work_off = rec->fields[i].offset; 4261 break; 4262 case BPF_RCU_HEAD: 4263 WARN_ON_ONCE(rec->rcu_head_off >= 0); 4264 rec->rcu_head_off = rec->fields[i].offset; 4265 break; 4266 case BPF_REFCOUNT: 4267 WARN_ON_ONCE(rec->refcount_off >= 0); 4268 /* Cache offset for faster lookup at runtime */ 4269 rec->refcount_off = rec->fields[i].offset; 4270 break; 4271 case BPF_KPTR_UNREF: 4272 case BPF_KPTR_REF: 4273 case BPF_KPTR_PERCPU: 4274 case BPF_UPTR: 4275 ret = btf_parse_kptr(btf, &rec->fields[i], &info_arr[i]); 4276 if (ret < 0) 4277 goto end; 4278 break; 4279 case BPF_LIST_HEAD: 4280 ret = btf_parse_list_head(btf, &rec->fields[i], &info_arr[i]); 4281 if (ret < 0) 4282 goto end; 4283 break; 4284 case BPF_RB_ROOT: 4285 ret = btf_parse_rb_root(btf, &rec->fields[i], &info_arr[i]); 4286 if (ret < 0) 4287 goto end; 4288 break; 4289 case BPF_LIST_NODE: 4290 case BPF_RB_NODE: 4291 break; 4292 default: 4293 ret = -EFAULT; 4294 goto end; 4295 } 4296 rec->cnt++; 4297 } 4298 4299 if (rec->spin_lock_off >= 0 && rec->res_spin_lock_off >= 0) { 4300 ret = -EINVAL; 4301 goto end; 4302 } 4303 4304 /* bpf_{list_head, rb_node} require bpf_spin_lock */ 4305 if ((btf_record_has_field(rec, BPF_LIST_HEAD) || 4306 btf_record_has_field(rec, BPF_RB_ROOT)) && 4307 (rec->spin_lock_off < 0 && rec->res_spin_lock_off < 0)) { 4308 ret = -EINVAL; 4309 goto end; 4310 } 4311 4312 if (rec->refcount_off < 0 && 4313 btf_record_has_field(rec, BPF_LIST_NODE) && 4314 btf_record_has_field(rec, BPF_RB_NODE)) { 4315 ret = -EINVAL; 4316 goto end; 4317 } 4318 4319 sort_r(rec->fields, rec->cnt, sizeof(struct btf_field), btf_field_cmp, 4320 NULL, rec); 4321 4322 return rec; 4323 end: 4324 btf_record_free(rec); 4325 return ERR_PTR(ret); 4326 } 4327 4328 int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) 4329 { 4330 int i; 4331 4332 /* 4333 * Check fields which require the complete BTF and initialize runtime 4334 * metadata. Ownership relationships are validated after every record has 4335 * been fixed up. 4336 */ 4337 if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & (BPF_GRAPH_ROOT | BPF_UPTR))) 4338 return 0; 4339 for (i = 0; i < rec->cnt; i++) { 4340 struct btf_struct_meta *meta; 4341 const struct btf_type *t; 4342 u32 btf_id; 4343 4344 if (rec->fields[i].type == BPF_UPTR) { 4345 /* The uptr only supports pinning one page and cannot 4346 * point to a kernel struct 4347 */ 4348 if (btf_is_kernel(rec->fields[i].kptr.btf)) 4349 return -EINVAL; 4350 t = btf_type_by_id(rec->fields[i].kptr.btf, 4351 rec->fields[i].kptr.btf_id); 4352 if (!t->size) 4353 return -EINVAL; 4354 if (t->size > PAGE_SIZE) 4355 return -E2BIG; 4356 continue; 4357 } 4358 4359 if (!(rec->fields[i].type & BPF_GRAPH_ROOT)) 4360 continue; 4361 btf_id = rec->fields[i].graph_root.value_btf_id; 4362 meta = btf_find_struct_meta(btf, btf_id); 4363 if (!meta) 4364 return -EFAULT; 4365 rec->fields[i].graph_root.value_rec = meta->record; 4366 } 4367 return 0; 4368 } 4369 4370 static int btf_owned_type_idx(const struct btf *btf, struct btf_struct_metas *tab, 4371 const struct btf_field *field) 4372 { 4373 struct btf_struct_meta *meta; 4374 u32 btf_id; 4375 4376 if (field->type & BPF_GRAPH_ROOT) { 4377 btf_id = field->graph_root.value_btf_id; 4378 } else if (field->type == BPF_KPTR_REF || field->type == BPF_KPTR_PERCPU) { 4379 if (btf_is_kernel(field->kptr.btf)) 4380 return -ENOENT; 4381 btf_id = field->kptr.btf_id; 4382 } else { 4383 return -ENOENT; 4384 } 4385 4386 meta = btf_find_struct_meta(btf, btf_id); 4387 if (!meta) 4388 return field->type & BPF_GRAPH_ROOT ? -EFAULT : -ENOENT; 4389 return meta - tab->types; 4390 } 4391 4392 /* 4393 * Each ownership edge adds kernel frames through bpf_obj_free_fields() and 4394 * __bpf_obj_drop_impl(). Keep the bound deliberately small because object 4395 * destruction can itself run below a BPF call chain. A final pointee without 4396 * special fields is not present in the struct metadata table and adds only a 4397 * non-recursing drop. 4398 */ 4399 #define BTF_MAX_OWNERSHIP_DEPTH 8 4400 4401 static int btf_ownership_depth(const struct btf *btf, 4402 struct btf_struct_metas *tab, u8 *depth, 4403 int idx, int depth_left) 4404 { 4405 const struct btf_record *rec = tab->types[idx].record; 4406 int i, ret, max_depth = 0; 4407 4408 if (!depth_left) 4409 return -ELOOP; 4410 if (depth[idx]) 4411 goto done; 4412 4413 for (i = 0; i < rec->cnt; i++) { 4414 ret = btf_owned_type_idx(btf, tab, &rec->fields[i]); 4415 if (ret == -ENOENT) 4416 continue; 4417 if (ret < 0) 4418 return ret; 4419 ret = btf_ownership_depth(btf, tab, depth, ret, depth_left - 1); 4420 if (ret < 0) 4421 return ret; 4422 max_depth = max(max_depth, ret); 4423 } 4424 depth[idx] = max_depth + 1; 4425 done: 4426 return depth[idx] > depth_left ? -ELOOP : depth[idx]; 4427 } 4428 4429 static int btf_check_ownership_depth(const struct btf *btf, 4430 struct btf_struct_metas *tab) 4431 { 4432 u8 *depth; 4433 int i, ret = 0; 4434 4435 depth = kvcalloc(tab->cnt, sizeof(*depth), GFP_KERNEL | __GFP_NOWARN); 4436 if (!depth) 4437 return -ENOMEM; 4438 4439 for (i = 0; i < tab->cnt; i++) { 4440 ret = btf_ownership_depth(btf, tab, depth, i, 4441 BTF_MAX_OWNERSHIP_DEPTH); 4442 if (ret < 0) 4443 break; 4444 ret = 0; 4445 } 4446 kvfree(depth); 4447 return ret; 4448 } 4449 4450 static void __btf_struct_show(const struct btf *btf, const struct btf_type *t, 4451 u32 type_id, void *data, u8 bits_offset, 4452 struct btf_show *show) 4453 { 4454 const struct btf_member *member; 4455 void *safe_data; 4456 u32 i; 4457 4458 safe_data = btf_show_start_struct_type(show, t, type_id, data); 4459 if (!safe_data) 4460 return; 4461 4462 for_each_member(i, t, member) { 4463 const struct btf_type *member_type = btf_type_by_id(btf, 4464 member->type); 4465 const struct btf_kind_operations *ops; 4466 u32 member_offset, bitfield_size; 4467 u32 bytes_offset; 4468 u8 bits8_offset; 4469 4470 btf_show_start_member(show, member); 4471 4472 member_offset = __btf_member_bit_offset(t, member); 4473 bitfield_size = __btf_member_bitfield_size(t, member); 4474 bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset); 4475 bits8_offset = BITS_PER_BYTE_MASKED(member_offset); 4476 if (bitfield_size) { 4477 safe_data = btf_show_start_type(show, member_type, 4478 member->type, 4479 data + bytes_offset); 4480 if (safe_data) 4481 btf_bitfield_show(safe_data, 4482 bits8_offset, 4483 bitfield_size, show); 4484 btf_show_end_type(show); 4485 } else { 4486 ops = btf_type_ops(member_type); 4487 ops->show(btf, member_type, member->type, 4488 data + bytes_offset, bits8_offset, show); 4489 } 4490 4491 btf_show_end_member(show); 4492 } 4493 4494 btf_show_end_struct_type(show); 4495 } 4496 4497 static void btf_struct_show(const struct btf *btf, const struct btf_type *t, 4498 u32 type_id, void *data, u8 bits_offset, 4499 struct btf_show *show) 4500 { 4501 const struct btf_member *m = show->state.member; 4502 4503 /* 4504 * First check if any members would be shown (are non-zero). 4505 * See comments above "struct btf_show" definition for more 4506 * details on how this works at a high-level. 4507 */ 4508 if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) { 4509 if (!show->state.depth_check) { 4510 show->state.depth_check = show->state.depth + 1; 4511 show->state.depth_to_show = 0; 4512 } 4513 __btf_struct_show(btf, t, type_id, data, bits_offset, show); 4514 /* Restore saved member data here */ 4515 show->state.member = m; 4516 if (show->state.depth_check != show->state.depth + 1) 4517 return; 4518 show->state.depth_check = 0; 4519 4520 if (show->state.depth_to_show <= show->state.depth) 4521 return; 4522 /* 4523 * Reaching here indicates we have recursed and found 4524 * non-zero child values. 4525 */ 4526 } 4527 4528 __btf_struct_show(btf, t, type_id, data, bits_offset, show); 4529 } 4530 4531 static const struct btf_kind_operations struct_ops = { 4532 .check_meta = btf_struct_check_meta, 4533 .resolve = btf_struct_resolve, 4534 .check_member = btf_struct_check_member, 4535 .check_kflag_member = btf_generic_check_kflag_member, 4536 .log_details = btf_struct_log, 4537 .show = btf_struct_show, 4538 }; 4539 4540 static int btf_enum_check_member(struct btf_verifier_env *env, 4541 const struct btf_type *struct_type, 4542 const struct btf_member *member, 4543 const struct btf_type *member_type) 4544 { 4545 u32 struct_bits_off = member->offset; 4546 u32 struct_size, bytes_offset; 4547 4548 if (BITS_PER_BYTE_MASKED(struct_bits_off)) { 4549 btf_verifier_log_member(env, struct_type, member, 4550 "Member is not byte aligned"); 4551 return -EINVAL; 4552 } 4553 4554 struct_size = struct_type->size; 4555 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); 4556 if (struct_size - bytes_offset < member_type->size) { 4557 btf_verifier_log_member(env, struct_type, member, 4558 "Member exceeds struct_size"); 4559 return -EINVAL; 4560 } 4561 4562 return 0; 4563 } 4564 4565 static int btf_enum_check_kflag_member(struct btf_verifier_env *env, 4566 const struct btf_type *struct_type, 4567 const struct btf_member *member, 4568 const struct btf_type *member_type) 4569 { 4570 u32 struct_bits_off, nr_bits, bytes_end, struct_size; 4571 u32 int_bitsize = sizeof(int) * BITS_PER_BYTE; 4572 4573 struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset); 4574 nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset); 4575 if (!nr_bits) { 4576 if (BITS_PER_BYTE_MASKED(struct_bits_off)) { 4577 btf_verifier_log_member(env, struct_type, member, 4578 "Member is not byte aligned"); 4579 return -EINVAL; 4580 } 4581 4582 nr_bits = int_bitsize; 4583 } else if (nr_bits > int_bitsize) { 4584 btf_verifier_log_member(env, struct_type, member, 4585 "Invalid member bitfield_size"); 4586 return -EINVAL; 4587 } 4588 4589 struct_size = struct_type->size; 4590 bytes_end = BITS_ROUNDUP_BYTES(struct_bits_off + nr_bits); 4591 if (struct_size < bytes_end) { 4592 btf_verifier_log_member(env, struct_type, member, 4593 "Member exceeds struct_size"); 4594 return -EINVAL; 4595 } 4596 4597 return 0; 4598 } 4599 4600 static s32 btf_enum_check_meta(struct btf_verifier_env *env, 4601 const struct btf_type *t, 4602 u32 meta_left) 4603 { 4604 const struct btf_enum *enums = btf_type_enum(t); 4605 struct btf *btf = env->btf; 4606 const char *fmt_str; 4607 u32 i, nr_enums; 4608 u32 meta_needed; 4609 4610 nr_enums = btf_type_vlen(t); 4611 meta_needed = nr_enums * sizeof(*enums); 4612 4613 if (meta_left < meta_needed) { 4614 btf_verifier_log_basic(env, t, 4615 "meta_left:%u meta_needed:%u", 4616 meta_left, meta_needed); 4617 return -EINVAL; 4618 } 4619 4620 if (t->size > 8 || !is_power_of_2(t->size)) { 4621 btf_verifier_log_type(env, t, "Unexpected size"); 4622 return -EINVAL; 4623 } 4624 4625 /* enum type either no name or a valid one */ 4626 if (t->name_off && 4627 !btf_name_valid_identifier(env->btf, t->name_off)) { 4628 btf_verifier_log_type(env, t, "Invalid name"); 4629 return -EINVAL; 4630 } 4631 4632 btf_verifier_log_type(env, t, NULL); 4633 4634 for (i = 0; i < nr_enums; i++) { 4635 if (!btf_name_offset_valid(btf, enums[i].name_off)) { 4636 btf_verifier_log(env, "\tInvalid name_offset:%u", 4637 enums[i].name_off); 4638 return -EINVAL; 4639 } 4640 4641 /* enum member must have a valid name */ 4642 if (!enums[i].name_off || 4643 !btf_name_valid_identifier(btf, enums[i].name_off)) { 4644 btf_verifier_log_type(env, t, "Invalid name"); 4645 return -EINVAL; 4646 } 4647 4648 if (env->log.level == BPF_LOG_KERNEL) 4649 continue; 4650 fmt_str = btf_type_kflag(t) ? "\t%s val=%d\n" : "\t%s val=%u\n"; 4651 btf_verifier_log(env, fmt_str, 4652 __btf_name_by_offset(btf, enums[i].name_off), 4653 enums[i].val); 4654 } 4655 4656 return meta_needed; 4657 } 4658 4659 static void btf_enum_log(struct btf_verifier_env *env, 4660 const struct btf_type *t) 4661 { 4662 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); 4663 } 4664 4665 static void btf_enum_show(const struct btf *btf, const struct btf_type *t, 4666 u32 type_id, void *data, u8 bits_offset, 4667 struct btf_show *show) 4668 { 4669 const struct btf_enum *enums = btf_type_enum(t); 4670 u32 i, nr_enums = btf_type_vlen(t); 4671 void *safe_data; 4672 int v; 4673 4674 safe_data = btf_show_start_type(show, t, type_id, data); 4675 if (!safe_data) 4676 return; 4677 4678 v = *(int *)safe_data; 4679 4680 for (i = 0; i < nr_enums; i++) { 4681 if (v != enums[i].val) 4682 continue; 4683 4684 btf_show_type_value(show, "%s", 4685 __btf_name_by_offset(btf, 4686 enums[i].name_off)); 4687 4688 btf_show_end_type(show); 4689 return; 4690 } 4691 4692 if (btf_type_kflag(t)) 4693 btf_show_type_value(show, "%d", v); 4694 else 4695 btf_show_type_value(show, "%u", v); 4696 btf_show_end_type(show); 4697 } 4698 4699 static const struct btf_kind_operations enum_ops = { 4700 .check_meta = btf_enum_check_meta, 4701 .resolve = btf_df_resolve, 4702 .check_member = btf_enum_check_member, 4703 .check_kflag_member = btf_enum_check_kflag_member, 4704 .log_details = btf_enum_log, 4705 .show = btf_enum_show, 4706 }; 4707 4708 static s32 btf_enum64_check_meta(struct btf_verifier_env *env, 4709 const struct btf_type *t, 4710 u32 meta_left) 4711 { 4712 const struct btf_enum64 *enums = btf_type_enum64(t); 4713 struct btf *btf = env->btf; 4714 const char *fmt_str; 4715 u32 i, nr_enums; 4716 u32 meta_needed; 4717 4718 nr_enums = btf_type_vlen(t); 4719 meta_needed = nr_enums * sizeof(*enums); 4720 4721 if (meta_left < meta_needed) { 4722 btf_verifier_log_basic(env, t, 4723 "meta_left:%u meta_needed:%u", 4724 meta_left, meta_needed); 4725 return -EINVAL; 4726 } 4727 4728 if (t->size > 8 || !is_power_of_2(t->size)) { 4729 btf_verifier_log_type(env, t, "Unexpected size"); 4730 return -EINVAL; 4731 } 4732 4733 /* enum type either no name or a valid one */ 4734 if (t->name_off && 4735 !btf_name_valid_identifier(env->btf, t->name_off)) { 4736 btf_verifier_log_type(env, t, "Invalid name"); 4737 return -EINVAL; 4738 } 4739 4740 btf_verifier_log_type(env, t, NULL); 4741 4742 for (i = 0; i < nr_enums; i++) { 4743 if (!btf_name_offset_valid(btf, enums[i].name_off)) { 4744 btf_verifier_log(env, "\tInvalid name_offset:%u", 4745 enums[i].name_off); 4746 return -EINVAL; 4747 } 4748 4749 /* enum member must have a valid name */ 4750 if (!enums[i].name_off || 4751 !btf_name_valid_identifier(btf, enums[i].name_off)) { 4752 btf_verifier_log_type(env, t, "Invalid name"); 4753 return -EINVAL; 4754 } 4755 4756 if (env->log.level == BPF_LOG_KERNEL) 4757 continue; 4758 4759 fmt_str = btf_type_kflag(t) ? "\t%s val=%lld\n" : "\t%s val=%llu\n"; 4760 btf_verifier_log(env, fmt_str, 4761 __btf_name_by_offset(btf, enums[i].name_off), 4762 btf_enum64_value(enums + i)); 4763 } 4764 4765 return meta_needed; 4766 } 4767 4768 static void btf_enum64_show(const struct btf *btf, const struct btf_type *t, 4769 u32 type_id, void *data, u8 bits_offset, 4770 struct btf_show *show) 4771 { 4772 const struct btf_enum64 *enums = btf_type_enum64(t); 4773 u32 i, nr_enums = btf_type_vlen(t); 4774 void *safe_data; 4775 s64 v; 4776 4777 safe_data = btf_show_start_type(show, t, type_id, data); 4778 if (!safe_data) 4779 return; 4780 4781 v = *(u64 *)safe_data; 4782 4783 for (i = 0; i < nr_enums; i++) { 4784 if (v != btf_enum64_value(enums + i)) 4785 continue; 4786 4787 btf_show_type_value(show, "%s", 4788 __btf_name_by_offset(btf, 4789 enums[i].name_off)); 4790 4791 btf_show_end_type(show); 4792 return; 4793 } 4794 4795 if (btf_type_kflag(t)) 4796 btf_show_type_value(show, "%lld", v); 4797 else 4798 btf_show_type_value(show, "%llu", v); 4799 btf_show_end_type(show); 4800 } 4801 4802 static const struct btf_kind_operations enum64_ops = { 4803 .check_meta = btf_enum64_check_meta, 4804 .resolve = btf_df_resolve, 4805 .check_member = btf_enum_check_member, 4806 .check_kflag_member = btf_enum_check_kflag_member, 4807 .log_details = btf_enum_log, 4808 .show = btf_enum64_show, 4809 }; 4810 4811 static s32 btf_loc_param_check_meta(struct btf_verifier_env *env, 4812 const struct btf_type *t, 4813 u32 meta_left) 4814 { 4815 const struct btf_loc_param *p = btf_loc_param(t); 4816 u32 size, meta_needed, vlen = btf_vlen(t); 4817 4818 meta_needed = sizeof(*p) + sizeof(__u32) * vlen; 4819 if (meta_left < meta_needed) { 4820 btf_verifier_log_basic(env, t, 4821 "meta_left:%u meta_needed:%u", 4822 meta_left, meta_needed); 4823 return -EINVAL; 4824 } 4825 4826 if (t->name_off) { 4827 btf_verifier_log_type(env, t, "Invalid name"); 4828 return -EINVAL; 4829 } 4830 size = t->size; 4831 if (!size || size > 16) { 4832 btf_verifier_log_type(env, t, "Unexpected size"); 4833 return -EINVAL; 4834 } 4835 4836 if (btf_type_kflag(t)) { 4837 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 4838 return -EINVAL; 4839 } 4840 /* All LOC_PARAMs have vlen of at least 1 */ 4841 if (vlen < 1) 4842 goto invalid_vlen; 4843 4844 switch (p->flags) { 4845 case BTF_LOC_PARAM_CONST: 4846 case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_SIGNED: 4847 case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_ADDR: 4848 case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_ADDR | BTF_LOC_PARAM_SIGNED: 4849 if (vlen < 1 || vlen > 2) 4850 goto invalid_vlen; 4851 if (!size || size > sizeof(__u64)) 4852 goto invalid_vlen; 4853 /* Ensure size/vlen are consistent */ 4854 if (vlen != (size > sizeof(__u32) ? 2 : 1)) 4855 goto invalid_vlen; 4856 break; 4857 case BTF_LOC_PARAM_REG: 4858 /* Reg or reg/reg pair */ 4859 if (vlen > 2) 4860 goto invalid_vlen; 4861 break; 4862 case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF: 4863 /* A reg deref without offset can only have vlen 1 */ 4864 if (vlen != 1) 4865 goto invalid_vlen; 4866 break; 4867 case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_OFFSET: 4868 case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_OFFSET | BTF_LOC_PARAM_SIGNED: 4869 case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET: 4870 case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET | 4871 BTF_LOC_PARAM_SIGNED: 4872 if (vlen < 2 || vlen > 3) 4873 goto invalid_vlen; 4874 break; 4875 default: 4876 btf_verifier_log_type(env, t, "Invalid flags"); 4877 return -EINVAL; 4878 } 4879 4880 btf_verifier_log_type(env, t, NULL); 4881 4882 return meta_needed; 4883 4884 invalid_vlen: 4885 btf_verifier_log_type(env, t, "Invalid vlen"); 4886 return -EINVAL; 4887 } 4888 4889 static void btf_loc_param_log(struct btf_verifier_env *env, 4890 const struct btf_type *t) 4891 { 4892 const struct btf_loc_param *p = btf_loc_param(t); 4893 u32 i, vlen = btf_vlen(t); 4894 4895 btf_verifier_log(env, "size=%u vlen=%u flags=0x%x", t->size, vlen, p->flags); 4896 for (i = 0; i < vlen; i++) 4897 btf_verifier_log(env, ", %u", p->values[i]); 4898 } 4899 4900 static const struct btf_kind_operations loc_param_ops = { 4901 .check_meta = btf_loc_param_check_meta, 4902 .resolve = btf_df_resolve, 4903 .check_member = btf_df_check_member, 4904 .check_kflag_member = btf_df_check_kflag_member, 4905 .log_details = btf_loc_param_log, 4906 .show = btf_df_show, 4907 }; 4908 4909 static s32 btf_loc_proto_check_meta(struct btf_verifier_env *env, 4910 const struct btf_type *t, 4911 u32 meta_left) 4912 { 4913 u32 meta_needed; 4914 4915 meta_needed = sizeof(__u32) * btf_type_vlen(t); 4916 4917 if (meta_left < meta_needed) { 4918 btf_verifier_log_basic(env, t, 4919 "meta_left:%u meta_needed:%u", 4920 meta_left, meta_needed); 4921 return -EINVAL; 4922 } 4923 4924 if (t->name_off) { 4925 btf_verifier_log_type(env, t, "Invalid name"); 4926 return -EINVAL; 4927 } 4928 4929 if (btf_type_kflag(t)) { 4930 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 4931 return -EINVAL; 4932 } 4933 4934 btf_verifier_log_type(env, t, NULL); 4935 4936 return meta_needed; 4937 } 4938 4939 static void btf_loc_proto_log(struct btf_verifier_env *env, 4940 const struct btf_type *t) 4941 { 4942 const __u32 *params = btf_loc_proto_params(t); 4943 u32 i, nr_params = btf_type_vlen(t); 4944 4945 btf_verifier_log(env, "vlen=%u", nr_params); 4946 for (i = 0; i < nr_params; i++, params++) 4947 btf_verifier_log(env, ", %u", *params); 4948 } 4949 4950 static int btf_loc_proto_resolve(struct btf_verifier_env *env, 4951 const struct resolve_vertex *v) 4952 { 4953 const struct btf_type *t = v->t; 4954 const __u32 *params = btf_loc_proto_params(t); 4955 u32 i, nr_params = btf_type_vlen(t); 4956 struct btf *btf = env->btf; 4957 4958 if (t->type) { 4959 btf_verifier_log_type(env, t, "Invalid loc_proto type"); 4960 return -EINVAL; 4961 } 4962 4963 for (i = 0; i < nr_params; i++) { 4964 const struct btf_type *param_type; 4965 u32 param_type_id = params[i]; 4966 4967 if (!param_type_id) 4968 continue; 4969 4970 param_type = btf_type_by_id(btf, param_type_id); 4971 if (!param_type || !btf_type_is_loc_param(param_type)) { 4972 btf_verifier_log_type(env, t, 4973 "Invalid loc_param#%u", i + 1); 4974 return -EINVAL; 4975 } 4976 } 4977 4978 env_stack_pop_resolved(env, 0, 0); 4979 return 0; 4980 } 4981 4982 static const struct btf_kind_operations loc_proto_ops = { 4983 .check_meta = btf_loc_proto_check_meta, 4984 .resolve = btf_loc_proto_resolve, 4985 .check_member = btf_df_check_member, 4986 .check_kflag_member = btf_df_check_kflag_member, 4987 .log_details = btf_loc_proto_log, 4988 .show = btf_df_show, 4989 }; 4990 4991 __printf(4, 5) 4992 static void btf_verifier_log_loc(struct btf_verifier_env *env, 4993 const struct btf_type *locsec_type, 4994 const struct btf_loc *loc, 4995 const char *fmt, ...) 4996 { 4997 struct bpf_verifier_log *log = &env->log; 4998 va_list args; 4999 5000 if (!bpf_verifier_log_needed(log)) 5001 return; 5002 if (log->level == BPF_LOG_KERNEL && !fmt) 5003 return; 5004 if (env->phase != CHECK_META) 5005 btf_verifier_log_type(env, locsec_type, NULL); 5006 5007 __btf_verifier_log(log, "\t func=%u loc_proto=%u offset=%u", 5008 loc->func, loc->loc_proto, loc->offset); 5009 if (fmt && *fmt) { 5010 __btf_verifier_log(log, " "); 5011 va_start(args, fmt); 5012 bpf_verifier_vlog(log, fmt, args); 5013 va_end(args); 5014 } 5015 5016 __btf_verifier_log(log, "\n"); 5017 } 5018 5019 static s32 btf_locsec_check_meta(struct btf_verifier_env *env, 5020 const struct btf_type *t, 5021 u32 meta_left) 5022 { 5023 u32 i, meta_needed, vlen = btf_type_vlen(t); 5024 const struct btf_loc *loc; 5025 5026 meta_needed = sizeof(struct btf_loc) * vlen; 5027 5028 if (meta_left < meta_needed) { 5029 btf_verifier_log_basic(env, t, 5030 "meta_left:%u meta_needed:%u", 5031 meta_left, meta_needed); 5032 return -EINVAL; 5033 } 5034 5035 if (btf_type_kflag(t)) { 5036 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 5037 return -EINVAL; 5038 } 5039 5040 if (!t->name_off || 5041 !btf_name_valid_section(env->btf, t->name_off)) { 5042 btf_verifier_log_type(env, t, "Invalid name"); 5043 return -EINVAL; 5044 } 5045 5046 for_each_loc(i, t, loc) { 5047 /* A loc func, loc proto cannot be in type void */ 5048 if (!loc->func || !BTF_TYPE_ID_VALID(loc->func)) { 5049 btf_verifier_log_loc(env, t, loc, "Invalid func"); 5050 return -EINVAL; 5051 } 5052 if (!loc->loc_proto || !BTF_TYPE_ID_VALID(loc->loc_proto)) { 5053 btf_verifier_log_loc(env, t, loc, "Invalid loc_proto"); 5054 return -EINVAL; 5055 } 5056 btf_verifier_log_loc(env, t, loc, NULL); 5057 } 5058 5059 return meta_needed; 5060 } 5061 5062 static void btf_locsec_log(struct btf_verifier_env *env, 5063 const struct btf_type *t) 5064 { 5065 btf_verifier_log(env, "vlen=%u", btf_type_vlen(t)); 5066 } 5067 5068 static int btf_locsec_resolve(struct btf_verifier_env *env, 5069 const struct resolve_vertex *v) 5070 { 5071 const struct btf_type *t = v->t; 5072 const struct btf_loc *loc; 5073 struct btf *btf = env->btf; 5074 u32 i; 5075 5076 if (t->type) { 5077 btf_verifier_log_type(env, t, "Invalid locsec type"); 5078 return -EINVAL; 5079 } 5080 5081 env->resolve_mode = RESOLVE_TBD; 5082 for (i = v->next_member, loc = btf_type_loc_secinfo(t) + i; 5083 i < btf_type_vlen(t); i++, loc++) { 5084 const struct btf_type *func_type, *func_proto_type; 5085 const struct btf_type *loc_proto_type; 5086 u32 func_type_id = loc->func; 5087 u32 loc_proto_type_id = loc->loc_proto; 5088 u32 proto_vlen; 5089 5090 func_type = btf_type_by_id(btf, func_type_id); 5091 if (!func_type || !btf_type_is_func(func_type)) { 5092 btf_verifier_log_type(env, t, 5093 "Invalid func#%u", i + 1); 5094 return -EINVAL; 5095 } 5096 func_proto_type = btf_type_by_id(btf, func_type->type); 5097 if (!func_proto_type || !btf_type_is_func_proto(func_proto_type)) { 5098 btf_verifier_log_type(env, t, 5099 "Invalid func#%u", i + 1); 5100 return -EINVAL; 5101 } 5102 proto_vlen = btf_vlen(func_proto_type); 5103 5104 if (!env_type_is_resolved(env, func_type_id)) { 5105 env_stack_set_next_member(env, i); 5106 return env_stack_push(env, func_type, func_type_id); 5107 } 5108 5109 loc_proto_type = btf_type_by_id(btf, loc_proto_type_id); 5110 if (!loc_proto_type || !btf_type_is_loc_proto(loc_proto_type)) { 5111 btf_verifier_log_type(env, t, 5112 "Invalid loc_proto#%u", i + 1); 5113 return -EINVAL; 5114 } 5115 if (proto_vlen != btf_vlen(loc_proto_type)) { 5116 btf_verifier_log_type(env, t, 5117 "Mismatched vlen for loc_proto#%u", i + 1); 5118 return -EINVAL; 5119 } 5120 5121 if (!env_type_is_resolved(env, loc_proto_type_id)) { 5122 env_stack_set_next_member(env, i + 1); 5123 return env_stack_push(env, loc_proto_type, 5124 loc_proto_type_id); 5125 } 5126 } 5127 5128 env_stack_pop_resolved(env, 0, 0); 5129 return 0; 5130 } 5131 5132 static const struct btf_kind_operations locsec_ops = { 5133 .check_meta = btf_locsec_check_meta, 5134 .resolve = btf_locsec_resolve, 5135 .check_member = btf_df_check_member, 5136 .check_kflag_member = btf_df_check_kflag_member, 5137 .log_details = btf_locsec_log, 5138 .show = btf_df_show, 5139 }; 5140 5141 static s32 btf_func_proto_check_meta(struct btf_verifier_env *env, 5142 const struct btf_type *t, 5143 u32 meta_left) 5144 { 5145 u32 meta_needed = btf_type_vlen(t) * sizeof(struct btf_param); 5146 5147 if (meta_left < meta_needed) { 5148 btf_verifier_log_basic(env, t, 5149 "meta_left:%u meta_needed:%u", 5150 meta_left, meta_needed); 5151 return -EINVAL; 5152 } 5153 5154 if (t->name_off) { 5155 btf_verifier_log_type(env, t, "Invalid name"); 5156 return -EINVAL; 5157 } 5158 5159 if (btf_type_kflag(t)) { 5160 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 5161 return -EINVAL; 5162 } 5163 5164 btf_verifier_log_type(env, t, NULL); 5165 5166 return meta_needed; 5167 } 5168 5169 static void btf_func_proto_log(struct btf_verifier_env *env, 5170 const struct btf_type *t) 5171 { 5172 const struct btf_param *args = (const struct btf_param *)(t + 1); 5173 u32 nr_args = btf_type_vlen(t), i; 5174 5175 btf_verifier_log(env, "return=%u args=(", t->type); 5176 if (!nr_args) { 5177 btf_verifier_log(env, "void"); 5178 goto done; 5179 } 5180 5181 if (nr_args == 1 && !args[0].type) { 5182 /* Only one vararg */ 5183 btf_verifier_log(env, "vararg"); 5184 goto done; 5185 } 5186 5187 btf_verifier_log(env, "%u %s", args[0].type, 5188 __btf_name_by_offset(env->btf, 5189 args[0].name_off)); 5190 for (i = 1; i < nr_args - 1; i++) 5191 btf_verifier_log(env, ", %u %s", args[i].type, 5192 __btf_name_by_offset(env->btf, 5193 args[i].name_off)); 5194 5195 if (nr_args > 1) { 5196 const struct btf_param *last_arg = &args[nr_args - 1]; 5197 5198 if (last_arg->type) 5199 btf_verifier_log(env, ", %u %s", last_arg->type, 5200 __btf_name_by_offset(env->btf, 5201 last_arg->name_off)); 5202 else 5203 btf_verifier_log(env, ", vararg"); 5204 } 5205 5206 done: 5207 btf_verifier_log(env, ")"); 5208 } 5209 5210 static const struct btf_kind_operations func_proto_ops = { 5211 .check_meta = btf_func_proto_check_meta, 5212 .resolve = btf_df_resolve, 5213 /* 5214 * BTF_KIND_FUNC_PROTO cannot be directly referred by 5215 * a struct's member. 5216 * 5217 * It should be a function pointer instead. 5218 * (i.e. struct's member -> BTF_KIND_PTR -> BTF_KIND_FUNC_PROTO) 5219 * 5220 * Hence, there is no btf_func_check_member(). 5221 */ 5222 .check_member = btf_df_check_member, 5223 .check_kflag_member = btf_df_check_kflag_member, 5224 .log_details = btf_func_proto_log, 5225 .show = btf_df_show, 5226 }; 5227 5228 static s32 btf_func_check_meta(struct btf_verifier_env *env, 5229 const struct btf_type *t, 5230 u32 meta_left) 5231 { 5232 if (!t->name_off || 5233 !btf_name_valid_identifier(env->btf, t->name_off)) { 5234 btf_verifier_log_type(env, t, "Invalid name"); 5235 return -EINVAL; 5236 } 5237 5238 if (btf_type_vlen(t) > BTF_FUNC_GLOBAL) { 5239 btf_verifier_log_type(env, t, "Invalid func linkage"); 5240 return -EINVAL; 5241 } 5242 5243 if (btf_type_kflag(t)) { 5244 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 5245 return -EINVAL; 5246 } 5247 5248 btf_verifier_log_type(env, t, NULL); 5249 5250 return 0; 5251 } 5252 5253 static int btf_func_resolve(struct btf_verifier_env *env, 5254 const struct resolve_vertex *v) 5255 { 5256 const struct btf_type *t = v->t; 5257 u32 next_type_id = t->type; 5258 int err; 5259 5260 err = btf_func_check(env, t); 5261 if (err) 5262 return err; 5263 5264 env_stack_pop_resolved(env, next_type_id, 0); 5265 return 0; 5266 } 5267 5268 static const struct btf_kind_operations func_ops = { 5269 .check_meta = btf_func_check_meta, 5270 .resolve = btf_func_resolve, 5271 .check_member = btf_df_check_member, 5272 .check_kflag_member = btf_df_check_kflag_member, 5273 .log_details = btf_ref_type_log, 5274 .show = btf_df_show, 5275 }; 5276 5277 static s32 btf_var_check_meta(struct btf_verifier_env *env, 5278 const struct btf_type *t, 5279 u32 meta_left) 5280 { 5281 const struct btf_var *var; 5282 u32 meta_needed = sizeof(*var); 5283 5284 if (meta_left < meta_needed) { 5285 btf_verifier_log_basic(env, t, 5286 "meta_left:%u meta_needed:%u", 5287 meta_left, meta_needed); 5288 return -EINVAL; 5289 } 5290 5291 if (btf_type_vlen(t)) { 5292 btf_verifier_log_type(env, t, "vlen != 0"); 5293 return -EINVAL; 5294 } 5295 5296 if (btf_type_kflag(t)) { 5297 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 5298 return -EINVAL; 5299 } 5300 5301 if (!t->name_off || 5302 !btf_name_valid_identifier(env->btf, t->name_off)) { 5303 btf_verifier_log_type(env, t, "Invalid name"); 5304 return -EINVAL; 5305 } 5306 5307 /* A var cannot be in type void */ 5308 if (!t->type || !BTF_TYPE_ID_VALID(t->type)) { 5309 btf_verifier_log_type(env, t, "Invalid type_id"); 5310 return -EINVAL; 5311 } 5312 5313 var = btf_type_var(t); 5314 if (var->linkage != BTF_VAR_STATIC && 5315 var->linkage != BTF_VAR_GLOBAL_ALLOCATED) { 5316 btf_verifier_log_type(env, t, "Linkage not supported"); 5317 return -EINVAL; 5318 } 5319 5320 btf_verifier_log_type(env, t, NULL); 5321 5322 return meta_needed; 5323 } 5324 5325 static void btf_var_log(struct btf_verifier_env *env, const struct btf_type *t) 5326 { 5327 const struct btf_var *var = btf_type_var(t); 5328 5329 btf_verifier_log(env, "type_id=%u linkage=%u", t->type, var->linkage); 5330 } 5331 5332 static const struct btf_kind_operations var_ops = { 5333 .check_meta = btf_var_check_meta, 5334 .resolve = btf_var_resolve, 5335 .check_member = btf_df_check_member, 5336 .check_kflag_member = btf_df_check_kflag_member, 5337 .log_details = btf_var_log, 5338 .show = btf_var_show, 5339 }; 5340 5341 static s32 btf_datasec_check_meta(struct btf_verifier_env *env, 5342 const struct btf_type *t, 5343 u32 meta_left) 5344 { 5345 const struct btf_var_secinfo *vsi; 5346 u64 last_vsi_end_off = 0, sum = 0; 5347 u32 i, meta_needed; 5348 5349 meta_needed = btf_type_vlen(t) * sizeof(*vsi); 5350 if (meta_left < meta_needed) { 5351 btf_verifier_log_basic(env, t, 5352 "meta_left:%u meta_needed:%u", 5353 meta_left, meta_needed); 5354 return -EINVAL; 5355 } 5356 5357 if (!t->size) { 5358 btf_verifier_log_type(env, t, "size == 0"); 5359 return -EINVAL; 5360 } 5361 5362 if (btf_type_kflag(t)) { 5363 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 5364 return -EINVAL; 5365 } 5366 5367 if (!t->name_off || 5368 !btf_name_valid_section(env->btf, t->name_off)) { 5369 btf_verifier_log_type(env, t, "Invalid name"); 5370 return -EINVAL; 5371 } 5372 5373 btf_verifier_log_type(env, t, NULL); 5374 5375 for_each_vsi(i, t, vsi) { 5376 /* A var cannot be in type void */ 5377 if (!vsi->type || !BTF_TYPE_ID_VALID(vsi->type)) { 5378 btf_verifier_log_vsi(env, t, vsi, 5379 "Invalid type_id"); 5380 return -EINVAL; 5381 } 5382 5383 if (vsi->offset < last_vsi_end_off || vsi->offset >= t->size) { 5384 btf_verifier_log_vsi(env, t, vsi, 5385 "Invalid offset"); 5386 return -EINVAL; 5387 } 5388 5389 if (!vsi->size || vsi->size > t->size) { 5390 btf_verifier_log_vsi(env, t, vsi, 5391 "Invalid size"); 5392 return -EINVAL; 5393 } 5394 5395 last_vsi_end_off = vsi->offset + vsi->size; 5396 if (last_vsi_end_off > t->size) { 5397 btf_verifier_log_vsi(env, t, vsi, 5398 "Invalid offset+size"); 5399 return -EINVAL; 5400 } 5401 5402 btf_verifier_log_vsi(env, t, vsi, NULL); 5403 sum += vsi->size; 5404 } 5405 5406 if (t->size < sum) { 5407 btf_verifier_log_type(env, t, "Invalid btf_info size"); 5408 return -EINVAL; 5409 } 5410 5411 return meta_needed; 5412 } 5413 5414 static int btf_datasec_resolve(struct btf_verifier_env *env, 5415 const struct resolve_vertex *v) 5416 { 5417 const struct btf_var_secinfo *vsi; 5418 struct btf *btf = env->btf; 5419 u32 i; 5420 5421 env->resolve_mode = RESOLVE_TBD; 5422 for_each_vsi_from(i, v->next_member, v->t, vsi) { 5423 u32 var_type_id = vsi->type, type_id; 5424 const struct btf_type *var_type = btf_type_by_id(env->btf, 5425 var_type_id); 5426 if (!var_type || !btf_type_is_var(var_type)) { 5427 btf_verifier_log_vsi(env, v->t, vsi, 5428 "Not a VAR kind member"); 5429 return -EINVAL; 5430 } 5431 5432 if (!env_type_is_resolve_sink(env, var_type) && 5433 !env_type_is_resolved(env, var_type_id)) { 5434 env_stack_set_next_member(env, i + 1); 5435 return env_stack_push(env, var_type, var_type_id); 5436 } 5437 5438 /* 5439 * The variable can be smaller than its type. It's a piece of 5440 * a variable that the compiler split then, with the type of 5441 * the whole variable. 5442 */ 5443 type_id = var_type->type; 5444 if (!btf_type_id_size(btf, &type_id, NULL)) { 5445 btf_verifier_log_vsi(env, v->t, vsi, "Invalid type"); 5446 return -EINVAL; 5447 } 5448 } 5449 5450 env_stack_pop_resolved(env, 0, 0); 5451 return 0; 5452 } 5453 5454 static void btf_datasec_log(struct btf_verifier_env *env, 5455 const struct btf_type *t) 5456 { 5457 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); 5458 } 5459 5460 /* A piece of a variable is smaller than its type, which is the one of the whole variable */ 5461 static bool btf_var_is_piece(const struct btf *btf, const struct btf_type *var, 5462 const struct btf_var_secinfo *vsi) 5463 { 5464 u32 size; 5465 5466 return IS_ERR(btf_resolve_size(btf, btf_type_by_id(btf, var->type), &size)) || 5467 vsi->size < size; 5468 } 5469 5470 static void btf_datasec_show(const struct btf *btf, 5471 const struct btf_type *t, u32 type_id, 5472 void *data, u8 bits_offset, 5473 struct btf_show *show) 5474 { 5475 const struct btf_var_secinfo *vsi; 5476 const struct btf_type *var; 5477 bool comma = false; 5478 u32 i; 5479 5480 if (!btf_show_start_type(show, t, type_id, data)) 5481 return; 5482 5483 btf_show_type_value(show, "section (\"%s\") = {", 5484 __btf_name_by_offset(btf, t->name_off)); 5485 for_each_vsi(i, t, vsi) { 5486 var = btf_type_by_id(btf, vsi->type); 5487 /* there is less data than the type takes */ 5488 if (btf_var_is_piece(btf, var, vsi)) 5489 continue; 5490 if (comma) 5491 btf_show(show, ","); 5492 comma = true; 5493 btf_type_ops(var)->show(btf, var, vsi->type, 5494 data + vsi->offset, bits_offset, show); 5495 } 5496 btf_show_end_type(show); 5497 } 5498 5499 static const struct btf_kind_operations datasec_ops = { 5500 .check_meta = btf_datasec_check_meta, 5501 .resolve = btf_datasec_resolve, 5502 .check_member = btf_df_check_member, 5503 .check_kflag_member = btf_df_check_kflag_member, 5504 .log_details = btf_datasec_log, 5505 .show = btf_datasec_show, 5506 }; 5507 5508 static s32 btf_float_check_meta(struct btf_verifier_env *env, 5509 const struct btf_type *t, 5510 u32 meta_left) 5511 { 5512 if (btf_type_vlen(t)) { 5513 btf_verifier_log_type(env, t, "vlen != 0"); 5514 return -EINVAL; 5515 } 5516 5517 if (btf_type_kflag(t)) { 5518 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 5519 return -EINVAL; 5520 } 5521 5522 if (t->size != 2 && t->size != 4 && t->size != 8 && t->size != 12 && 5523 t->size != 16) { 5524 btf_verifier_log_type(env, t, "Invalid type_size"); 5525 return -EINVAL; 5526 } 5527 5528 btf_verifier_log_type(env, t, NULL); 5529 5530 return 0; 5531 } 5532 5533 static int btf_float_check_member(struct btf_verifier_env *env, 5534 const struct btf_type *struct_type, 5535 const struct btf_member *member, 5536 const struct btf_type *member_type) 5537 { 5538 u64 start_offset_bytes; 5539 u64 end_offset_bytes; 5540 u64 misalign_bits; 5541 u64 align_bytes; 5542 u64 align_bits; 5543 5544 /* Different architectures have different alignment requirements, so 5545 * here we check only for the reasonable minimum. This way we ensure 5546 * that types after CO-RE can pass the kernel BTF verifier. 5547 */ 5548 align_bytes = min_t(u64, sizeof(void *), member_type->size); 5549 align_bits = align_bytes * BITS_PER_BYTE; 5550 div64_u64_rem(member->offset, align_bits, &misalign_bits); 5551 if (misalign_bits) { 5552 btf_verifier_log_member(env, struct_type, member, 5553 "Member is not properly aligned"); 5554 return -EINVAL; 5555 } 5556 5557 start_offset_bytes = member->offset / BITS_PER_BYTE; 5558 end_offset_bytes = start_offset_bytes + member_type->size; 5559 if (end_offset_bytes > struct_type->size) { 5560 btf_verifier_log_member(env, struct_type, member, 5561 "Member exceeds struct_size"); 5562 return -EINVAL; 5563 } 5564 5565 return 0; 5566 } 5567 5568 static void btf_float_log(struct btf_verifier_env *env, 5569 const struct btf_type *t) 5570 { 5571 btf_verifier_log(env, "size=%u", t->size); 5572 } 5573 5574 static const struct btf_kind_operations float_ops = { 5575 .check_meta = btf_float_check_meta, 5576 .resolve = btf_df_resolve, 5577 .check_member = btf_float_check_member, 5578 .check_kflag_member = btf_generic_check_kflag_member, 5579 .log_details = btf_float_log, 5580 .show = btf_df_show, 5581 }; 5582 5583 static s32 btf_decl_tag_check_meta(struct btf_verifier_env *env, 5584 const struct btf_type *t, 5585 u32 meta_left) 5586 { 5587 const struct btf_decl_tag *tag; 5588 u32 meta_needed = sizeof(*tag); 5589 s32 component_idx; 5590 const char *value; 5591 5592 if (meta_left < meta_needed) { 5593 btf_verifier_log_basic(env, t, 5594 "meta_left:%u meta_needed:%u", 5595 meta_left, meta_needed); 5596 return -EINVAL; 5597 } 5598 5599 value = btf_name_by_offset(env->btf, t->name_off); 5600 if (!value || !value[0]) { 5601 btf_verifier_log_type(env, t, "Invalid value"); 5602 return -EINVAL; 5603 } 5604 5605 if (btf_type_vlen(t)) { 5606 btf_verifier_log_type(env, t, "vlen != 0"); 5607 return -EINVAL; 5608 } 5609 5610 component_idx = btf_type_decl_tag(t)->component_idx; 5611 if (component_idx < -1) { 5612 btf_verifier_log_type(env, t, "Invalid component_idx"); 5613 return -EINVAL; 5614 } 5615 5616 btf_verifier_log_type(env, t, NULL); 5617 5618 return meta_needed; 5619 } 5620 5621 static int btf_decl_tag_resolve(struct btf_verifier_env *env, 5622 const struct resolve_vertex *v) 5623 { 5624 const struct btf_type *next_type; 5625 const struct btf_type *t = v->t; 5626 u32 next_type_id = t->type; 5627 struct btf *btf = env->btf; 5628 s32 component_idx; 5629 u32 vlen; 5630 5631 next_type = btf_type_by_id(btf, next_type_id); 5632 if (!next_type || !btf_type_is_decl_tag_target(next_type)) { 5633 btf_verifier_log_type(env, v->t, "Invalid type_id"); 5634 return -EINVAL; 5635 } 5636 5637 if (!env_type_is_resolve_sink(env, next_type) && 5638 !env_type_is_resolved(env, next_type_id)) 5639 return env_stack_push(env, next_type, next_type_id); 5640 5641 component_idx = btf_type_decl_tag(t)->component_idx; 5642 if (component_idx != -1) { 5643 if (btf_type_is_var(next_type) || btf_type_is_typedef(next_type)) { 5644 btf_verifier_log_type(env, v->t, "Invalid component_idx"); 5645 return -EINVAL; 5646 } 5647 5648 if (btf_type_is_struct(next_type)) { 5649 vlen = btf_type_vlen(next_type); 5650 } else { 5651 /* next_type should be a function */ 5652 next_type = btf_type_by_id(btf, next_type->type); 5653 vlen = btf_type_vlen(next_type); 5654 } 5655 5656 if ((u32)component_idx >= vlen) { 5657 btf_verifier_log_type(env, v->t, "Invalid component_idx"); 5658 return -EINVAL; 5659 } 5660 } 5661 5662 env_stack_pop_resolved(env, next_type_id, 0); 5663 5664 return 0; 5665 } 5666 5667 static void btf_decl_tag_log(struct btf_verifier_env *env, const struct btf_type *t) 5668 { 5669 btf_verifier_log(env, "type=%u component_idx=%d", t->type, 5670 btf_type_decl_tag(t)->component_idx); 5671 } 5672 5673 static const struct btf_kind_operations decl_tag_ops = { 5674 .check_meta = btf_decl_tag_check_meta, 5675 .resolve = btf_decl_tag_resolve, 5676 .check_member = btf_df_check_member, 5677 .check_kflag_member = btf_df_check_kflag_member, 5678 .log_details = btf_decl_tag_log, 5679 .show = btf_df_show, 5680 }; 5681 5682 static int btf_func_proto_check(struct btf_verifier_env *env, 5683 const struct btf_type *t) 5684 { 5685 const struct btf_type *ret_type; 5686 const struct btf_param *args; 5687 const struct btf *btf; 5688 u32 nr_args, i; 5689 int err; 5690 5691 btf = env->btf; 5692 args = (const struct btf_param *)(t + 1); 5693 nr_args = btf_type_vlen(t); 5694 5695 /* Check func return type which could be "void" (t->type == 0) */ 5696 if (t->type) { 5697 u32 ret_type_id = t->type; 5698 5699 ret_type = btf_type_by_id(btf, ret_type_id); 5700 if (!ret_type) { 5701 btf_verifier_log_type(env, t, "Invalid return type"); 5702 return -EINVAL; 5703 } 5704 5705 if (btf_type_is_resolve_source_only(ret_type)) { 5706 btf_verifier_log_type(env, t, "Invalid return type"); 5707 return -EINVAL; 5708 } 5709 5710 if (btf_type_needs_resolve(ret_type) && 5711 !env_type_is_resolved(env, ret_type_id)) { 5712 err = btf_resolve(env, ret_type, ret_type_id); 5713 if (err) 5714 return err; 5715 } 5716 5717 /* Ensure the return type is a type that has a size */ 5718 if (!btf_type_id_size(btf, &ret_type_id, NULL)) { 5719 btf_verifier_log_type(env, t, "Invalid return type"); 5720 return -EINVAL; 5721 } 5722 } 5723 5724 if (!nr_args) 5725 return 0; 5726 5727 /* Last func arg type_id could be 0 if it is a vararg */ 5728 if (!args[nr_args - 1].type) { 5729 if (args[nr_args - 1].name_off) { 5730 btf_verifier_log_type(env, t, "Invalid arg#%u", 5731 nr_args); 5732 return -EINVAL; 5733 } 5734 nr_args--; 5735 } 5736 5737 for (i = 0; i < nr_args; i++) { 5738 const struct btf_type *arg_type; 5739 u32 arg_type_id; 5740 5741 arg_type_id = args[i].type; 5742 arg_type = btf_type_by_id(btf, arg_type_id); 5743 if (!arg_type) { 5744 btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1); 5745 return -EINVAL; 5746 } 5747 5748 if (btf_type_is_resolve_source_only(arg_type)) { 5749 btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1); 5750 return -EINVAL; 5751 } 5752 5753 if (args[i].name_off && 5754 (!btf_name_offset_valid(btf, args[i].name_off) || 5755 !btf_name_valid_identifier(btf, args[i].name_off))) { 5756 btf_verifier_log_type(env, t, 5757 "Invalid arg#%u", i + 1); 5758 return -EINVAL; 5759 } 5760 5761 if (btf_type_needs_resolve(arg_type) && 5762 !env_type_is_resolved(env, arg_type_id)) { 5763 err = btf_resolve(env, arg_type, arg_type_id); 5764 if (err) 5765 return err; 5766 } 5767 5768 if (!btf_type_id_size(btf, &arg_type_id, NULL)) { 5769 btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1); 5770 return -EINVAL; 5771 } 5772 } 5773 5774 return 0; 5775 } 5776 5777 static int btf_func_check(struct btf_verifier_env *env, 5778 const struct btf_type *t) 5779 { 5780 const struct btf_type *proto_type; 5781 const struct btf_param *args; 5782 const struct btf *btf; 5783 u32 nr_args, i; 5784 5785 btf = env->btf; 5786 proto_type = btf_type_by_id(btf, t->type); 5787 5788 if (!proto_type || !btf_type_is_func_proto(proto_type)) { 5789 btf_verifier_log_type(env, t, "Invalid type_id"); 5790 return -EINVAL; 5791 } 5792 5793 /* Rust leaves out names of some arguments of static functions */ 5794 if (btf_func_linkage(t) == BTF_FUNC_STATIC) 5795 return 0; 5796 5797 args = (const struct btf_param *)(proto_type + 1); 5798 nr_args = btf_type_vlen(proto_type); 5799 for (i = 0; i < nr_args; i++) { 5800 if (!args[i].name_off && args[i].type) { 5801 btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1); 5802 return -EINVAL; 5803 } 5804 } 5805 5806 return 0; 5807 } 5808 5809 static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = { 5810 [BTF_KIND_INT] = &int_ops, 5811 [BTF_KIND_PTR] = &ptr_ops, 5812 [BTF_KIND_ARRAY] = &array_ops, 5813 [BTF_KIND_STRUCT] = &struct_ops, 5814 [BTF_KIND_UNION] = &struct_ops, 5815 [BTF_KIND_ENUM] = &enum_ops, 5816 [BTF_KIND_FWD] = &fwd_ops, 5817 [BTF_KIND_TYPEDEF] = &modifier_ops, 5818 [BTF_KIND_VOLATILE] = &modifier_ops, 5819 [BTF_KIND_CONST] = &modifier_ops, 5820 [BTF_KIND_RESTRICT] = &modifier_ops, 5821 [BTF_KIND_FUNC] = &func_ops, 5822 [BTF_KIND_FUNC_PROTO] = &func_proto_ops, 5823 [BTF_KIND_VAR] = &var_ops, 5824 [BTF_KIND_DATASEC] = &datasec_ops, 5825 [BTF_KIND_FLOAT] = &float_ops, 5826 [BTF_KIND_DECL_TAG] = &decl_tag_ops, 5827 [BTF_KIND_TYPE_TAG] = &modifier_ops, 5828 [BTF_KIND_ENUM64] = &enum64_ops, 5829 [BTF_KIND_LOC_PARAM] = &loc_param_ops, 5830 [BTF_KIND_LOC_PROTO] = &loc_proto_ops, 5831 [BTF_KIND_LOCSEC] = &locsec_ops, 5832 }; 5833 5834 static s32 btf_check_meta(struct btf_verifier_env *env, 5835 const struct btf_type *t, 5836 u32 meta_left) 5837 { 5838 u32 saved_meta_left = meta_left; 5839 s32 var_meta_size; 5840 5841 if (meta_left < sizeof(*t)) { 5842 btf_verifier_log(env, "[%u] meta_left:%u meta_needed:%zu", 5843 env->log_type_id, meta_left, sizeof(*t)); 5844 return -EINVAL; 5845 } 5846 meta_left -= sizeof(*t); 5847 5848 if (BTF_INFO_KIND(t->info) > BTF_KIND_MAX || 5849 BTF_INFO_KIND(t->info) == BTF_KIND_UNKN) { 5850 btf_verifier_log(env, "[%u] Invalid kind:%u", 5851 env->log_type_id, BTF_INFO_KIND(t->info)); 5852 return -EINVAL; 5853 } 5854 5855 if (!btf_name_offset_valid(env->btf, t->name_off)) { 5856 btf_verifier_log(env, "[%u] Invalid name_offset:%u", 5857 env->log_type_id, t->name_off); 5858 return -EINVAL; 5859 } 5860 5861 var_meta_size = btf_type_ops(t)->check_meta(env, t, meta_left); 5862 if (var_meta_size < 0) 5863 return var_meta_size; 5864 5865 meta_left -= var_meta_size; 5866 5867 return saved_meta_left - meta_left; 5868 } 5869 5870 static int btf_check_all_metas(struct btf_verifier_env *env) 5871 { 5872 struct btf *btf = env->btf; 5873 struct btf_header *hdr; 5874 void *cur, *end; 5875 int err; 5876 5877 hdr = &btf->hdr; 5878 cur = btf->nohdr_data + hdr->type_off; 5879 end = cur + hdr->type_len; 5880 5881 env->log_type_id = btf->base_btf ? btf->start_id : 1; 5882 while (cur < end) { 5883 struct btf_type *t = cur; 5884 s32 meta_size; 5885 5886 meta_size = btf_check_meta(env, t, end - cur); 5887 if (meta_size < 0) 5888 return meta_size; 5889 5890 err = btf_add_type(env, t); 5891 if (err) 5892 return err; 5893 5894 cur += meta_size; 5895 env->log_type_id++; 5896 } 5897 5898 return 0; 5899 } 5900 5901 static bool btf_resolve_valid(struct btf_verifier_env *env, 5902 const struct btf_type *t, 5903 u32 type_id) 5904 { 5905 struct btf *btf = env->btf; 5906 5907 if (!env_type_is_resolved(env, type_id)) 5908 return false; 5909 5910 if (btf_type_is_struct(t) || btf_type_is_datasec(t) || 5911 btf_type_is_loc_proto(t) || btf_type_is_locsec(t)) 5912 return !btf_resolved_type_id(btf, type_id) && 5913 !btf_resolved_type_size(btf, type_id); 5914 5915 if (btf_type_is_decl_tag(t) || btf_type_is_func(t)) 5916 return btf_resolved_type_id(btf, type_id) && 5917 !btf_resolved_type_size(btf, type_id); 5918 5919 if (btf_type_is_modifier(t) || btf_type_is_ptr(t) || 5920 btf_type_is_var(t)) { 5921 t = btf_type_id_resolve(btf, &type_id); 5922 return t && 5923 !btf_type_is_modifier(t) && 5924 !btf_type_is_var(t) && 5925 !btf_type_is_datasec(t); 5926 } 5927 5928 if (btf_type_is_array(t)) { 5929 const struct btf_array *array = btf_type_array(t); 5930 const struct btf_type *elem_type; 5931 u32 elem_type_id = array->type; 5932 u32 elem_size; 5933 5934 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size); 5935 return elem_type && !btf_type_is_modifier(elem_type) && 5936 (array->nelems * elem_size == 5937 btf_resolved_type_size(btf, type_id)); 5938 } 5939 5940 return false; 5941 } 5942 5943 static int btf_resolve(struct btf_verifier_env *env, 5944 const struct btf_type *t, u32 type_id) 5945 { 5946 u32 save_log_type_id = env->log_type_id; 5947 const struct resolve_vertex *v; 5948 int err = 0; 5949 5950 env->resolve_mode = RESOLVE_TBD; 5951 env_stack_push(env, t, type_id); 5952 while (!err && (v = env_stack_peak(env))) { 5953 env->log_type_id = v->type_id; 5954 err = btf_type_ops(v->t)->resolve(env, v); 5955 } 5956 5957 env->log_type_id = type_id; 5958 if (err == -E2BIG) { 5959 btf_verifier_log_type(env, t, 5960 "Exceeded max resolving depth:%u", 5961 MAX_RESOLVE_DEPTH); 5962 } else if (err == -EEXIST) { 5963 btf_verifier_log_type(env, t, "Loop detected"); 5964 } 5965 5966 /* Final sanity check */ 5967 if (!err && !btf_resolve_valid(env, t, type_id)) { 5968 btf_verifier_log_type(env, t, "Invalid resolve state"); 5969 err = -EINVAL; 5970 } 5971 5972 env->log_type_id = save_log_type_id; 5973 return err; 5974 } 5975 5976 static int btf_check_all_types(struct btf_verifier_env *env) 5977 { 5978 struct btf *btf = env->btf; 5979 const struct btf_type *t; 5980 u32 type_id, i; 5981 int err; 5982 5983 err = env_resolve_init(env); 5984 if (err) 5985 return err; 5986 5987 env->phase++; 5988 for (i = btf->base_btf ? 0 : 1; i < btf->nr_types; i++) { 5989 type_id = btf->start_id + i; 5990 t = btf_type_by_id(btf, type_id); 5991 5992 env->log_type_id = type_id; 5993 if (btf_type_needs_resolve(t) && 5994 !env_type_is_resolved(env, type_id)) { 5995 err = btf_resolve(env, t, type_id); 5996 if (err) 5997 return err; 5998 } 5999 6000 if (btf_type_is_func_proto(t)) { 6001 err = btf_func_proto_check(env, t); 6002 if (err) 6003 return err; 6004 } 6005 } 6006 6007 return 0; 6008 } 6009 6010 static int btf_parse_type_sec(struct btf_verifier_env *env) 6011 { 6012 const struct btf_header *hdr = &env->btf->hdr; 6013 int err; 6014 6015 /* Type section must align to 4 bytes */ 6016 if (hdr->type_off & (sizeof(u32) - 1)) { 6017 btf_verifier_log(env, "Unaligned type_off"); 6018 return -EINVAL; 6019 } 6020 6021 if (!env->btf->base_btf && !hdr->type_len) { 6022 btf_verifier_log(env, "No type found"); 6023 return -EINVAL; 6024 } 6025 6026 err = btf_check_all_metas(env); 6027 if (err) 6028 return err; 6029 6030 return btf_check_all_types(env); 6031 } 6032 6033 static int btf_parse_str_sec(struct btf_verifier_env *env) 6034 { 6035 const struct btf_header *hdr; 6036 struct btf *btf = env->btf; 6037 const char *start, *end; 6038 6039 hdr = &btf->hdr; 6040 start = btf->nohdr_data + hdr->str_off; 6041 end = start + hdr->str_len; 6042 6043 if (hdr->hdr_len < sizeof(struct btf_header) && 6044 end != btf->data + btf->data_size) { 6045 btf_verifier_log(env, "String section is not at the end"); 6046 return -EINVAL; 6047 } 6048 6049 btf->strings = start; 6050 6051 if (btf->base_btf && !hdr->str_len) 6052 return 0; 6053 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET || end[-1]) { 6054 btf_verifier_log(env, "Invalid string section"); 6055 return -EINVAL; 6056 } 6057 if (!btf->base_btf && start[0]) { 6058 btf_verifier_log(env, "Invalid string section"); 6059 return -EINVAL; 6060 } 6061 6062 return 0; 6063 } 6064 6065 static int btf_parse_layout_sec(struct btf_verifier_env *env) 6066 { 6067 const struct btf_header *hdr = &env->btf->hdr; 6068 struct btf *btf = env->btf; 6069 void *start, *end; 6070 6071 if (hdr->hdr_len < sizeof(struct btf_header) || 6072 hdr->layout_len == 0) 6073 return 0; 6074 6075 /* Layout section must align to 4 bytes */ 6076 if (hdr->layout_off & (sizeof(u32) - 1)) { 6077 btf_verifier_log(env, "Unaligned layout_off"); 6078 return -EINVAL; 6079 } 6080 start = btf->nohdr_data + hdr->layout_off; 6081 end = start + hdr->layout_len; 6082 6083 if (hdr->layout_len < sizeof(struct btf_layout)) { 6084 btf_verifier_log(env, "Layout section is too small"); 6085 return -EINVAL; 6086 } 6087 if (hdr->layout_len % sizeof(struct btf_layout) != 0) { 6088 btf_verifier_log(env, "layout_len is not multiple of %zu", 6089 sizeof(struct btf_layout)); 6090 return -EINVAL; 6091 } 6092 if (end > btf->data + btf->data_size) { 6093 btf_verifier_log(env, "Layout section is too big"); 6094 return -EINVAL; 6095 } 6096 btf->layout = start; 6097 6098 return 0; 6099 } 6100 6101 static const size_t btf_sec_info_offset[] = { 6102 offsetof(struct btf_header, type_off), 6103 offsetof(struct btf_header, str_off), 6104 offsetof(struct btf_header, layout_off) 6105 }; 6106 6107 static int btf_sec_info_cmp(const void *a, const void *b) 6108 { 6109 const struct btf_sec_info *x = a; 6110 const struct btf_sec_info *y = b; 6111 6112 return (int)(x->off - y->off) ? : (int)(x->len - y->len); 6113 } 6114 6115 static int btf_check_sec_info(struct btf_verifier_env *env, 6116 u32 btf_data_size) 6117 { 6118 struct btf_sec_info secs[ARRAY_SIZE(btf_sec_info_offset)]; 6119 u32 total, expected_total, i; 6120 u32 nr_secs = ARRAY_SIZE(btf_sec_info_offset); 6121 const struct btf_header *hdr; 6122 const struct btf *btf; 6123 6124 btf = env->btf; 6125 hdr = &btf->hdr; 6126 6127 if (hdr->hdr_len < sizeof(struct btf_header) || hdr->layout_len == 0) 6128 nr_secs--; 6129 6130 /* Populate the secs from hdr */ 6131 for (i = 0; i < nr_secs; i++) 6132 secs[i] = *(struct btf_sec_info *)((void *)hdr + 6133 btf_sec_info_offset[i]); 6134 6135 sort(secs, nr_secs, 6136 sizeof(struct btf_sec_info), btf_sec_info_cmp, NULL); 6137 6138 /* Check for gaps and overlap among sections */ 6139 total = 0; 6140 expected_total = btf_data_size - hdr->hdr_len; 6141 for (i = 0; i < nr_secs; i++) { 6142 if (expected_total < secs[i].off) { 6143 btf_verifier_log(env, "Invalid section offset"); 6144 return -EINVAL; 6145 } 6146 if (total < secs[i].off) { 6147 /* gap */ 6148 btf_verifier_log(env, "Unsupported section found"); 6149 return -EINVAL; 6150 } 6151 if (total > secs[i].off) { 6152 btf_verifier_log(env, "Section overlap found"); 6153 return -EINVAL; 6154 } 6155 if (expected_total - total < secs[i].len) { 6156 btf_verifier_log(env, 6157 "Total section length too long"); 6158 return -EINVAL; 6159 } 6160 total += secs[i].len; 6161 } 6162 6163 /* There is data other than hdr and known sections */ 6164 if (expected_total != total) { 6165 btf_verifier_log(env, "Unsupported section found"); 6166 return -EINVAL; 6167 } 6168 6169 return 0; 6170 } 6171 6172 static int btf_parse_hdr(struct btf_verifier_env *env) 6173 { 6174 u32 hdr_len, hdr_copy, btf_data_size; 6175 const struct btf_header *hdr; 6176 struct btf *btf; 6177 6178 btf = env->btf; 6179 btf_data_size = btf->data_size; 6180 6181 if (btf_data_size < offsetofend(struct btf_header, hdr_len)) { 6182 btf_verifier_log(env, "hdr_len not found"); 6183 return -EINVAL; 6184 } 6185 6186 hdr = btf->data; 6187 hdr_len = hdr->hdr_len; 6188 if (btf_data_size < hdr_len) { 6189 btf_verifier_log(env, "btf_header not found"); 6190 return -EINVAL; 6191 } 6192 6193 /* Ensure the unsupported header fields are zero */ 6194 if (hdr_len > sizeof(btf->hdr)) { 6195 u8 *expected_zero = btf->data + sizeof(btf->hdr); 6196 u8 *end = btf->data + hdr_len; 6197 6198 for (; expected_zero < end; expected_zero++) { 6199 if (*expected_zero) { 6200 btf_verifier_log(env, "Unsupported btf_header"); 6201 return -E2BIG; 6202 } 6203 } 6204 } 6205 6206 hdr_copy = min_t(u32, hdr_len, sizeof(btf->hdr)); 6207 memcpy(&btf->hdr, btf->data, hdr_copy); 6208 6209 hdr = &btf->hdr; 6210 6211 btf_verifier_log_hdr(env, btf_data_size); 6212 6213 if (hdr->magic != BTF_MAGIC) { 6214 btf_verifier_log(env, "Invalid magic"); 6215 return -EINVAL; 6216 } 6217 6218 if (hdr->version != BTF_VERSION) { 6219 btf_verifier_log(env, "Unsupported version"); 6220 return -ENOTSUPP; 6221 } 6222 6223 if (hdr->flags) { 6224 btf_verifier_log(env, "Unsupported flags"); 6225 return -ENOTSUPP; 6226 } 6227 6228 if (!btf->base_btf && btf_data_size == hdr->hdr_len) { 6229 btf_verifier_log(env, "No data"); 6230 return -EINVAL; 6231 } 6232 6233 return btf_check_sec_info(env, btf_data_size); 6234 } 6235 6236 static const char * const alloc_obj_fields[] = { 6237 "bpf_spin_lock", 6238 "bpf_list_head", 6239 "bpf_list_node", 6240 "bpf_rb_root", 6241 "bpf_rb_node", 6242 "bpf_refcount", 6243 }; 6244 6245 static struct btf_struct_metas * 6246 btf_parse_struct_metas(struct bpf_verifier_log *log, struct btf *btf) 6247 { 6248 struct btf_struct_metas *tab = NULL; 6249 struct btf_id_set *aof; 6250 int i, n, id, ret; 6251 6252 BUILD_BUG_ON(offsetof(struct btf_id_set, cnt) != 0); 6253 BUILD_BUG_ON(sizeof(struct btf_id_set) != sizeof(u32)); 6254 6255 aof = kmalloc_obj(*aof, GFP_KERNEL | __GFP_NOWARN); 6256 if (!aof) 6257 return ERR_PTR(-ENOMEM); 6258 aof->cnt = 0; 6259 6260 for (i = 0; i < ARRAY_SIZE(alloc_obj_fields); i++) { 6261 /* Try to find whether this special type exists in user BTF, and 6262 * if so remember its ID so we can easily find it among members 6263 * of structs that we iterate in the next loop. 6264 */ 6265 struct btf_id_set *new_aof; 6266 6267 id = btf_find_by_name_kind(btf, alloc_obj_fields[i], BTF_KIND_STRUCT); 6268 if (id < 0) 6269 continue; 6270 6271 new_aof = krealloc(aof, struct_size(new_aof, ids, aof->cnt + 1), 6272 GFP_KERNEL | __GFP_NOWARN); 6273 if (!new_aof) { 6274 ret = -ENOMEM; 6275 goto free_aof; 6276 } 6277 aof = new_aof; 6278 aof->ids[aof->cnt++] = id; 6279 } 6280 6281 n = btf_nr_types(btf); 6282 for (i = 1; i < n; i++) { 6283 /* Try to find if there are kptrs in user BTF and remember their ID */ 6284 struct btf_id_set *new_aof; 6285 struct btf_field_info tmp; 6286 const struct btf_type *t; 6287 6288 t = btf_type_by_id(btf, i); 6289 if (!t) { 6290 ret = -EINVAL; 6291 goto free_aof; 6292 } 6293 6294 ret = btf_find_kptr(btf, t, 0, 0, &tmp, BPF_KPTR); 6295 if (ret != BTF_FIELD_FOUND) 6296 continue; 6297 6298 new_aof = krealloc(aof, struct_size(new_aof, ids, aof->cnt + 1), 6299 GFP_KERNEL | __GFP_NOWARN); 6300 if (!new_aof) { 6301 ret = -ENOMEM; 6302 goto free_aof; 6303 } 6304 aof = new_aof; 6305 aof->ids[aof->cnt++] = i; 6306 } 6307 6308 if (!aof->cnt) { 6309 kfree(aof); 6310 return NULL; 6311 } 6312 sort(&aof->ids, aof->cnt, sizeof(aof->ids[0]), btf_id_cmp_func, NULL); 6313 6314 for (i = 1; i < n; i++) { 6315 struct btf_struct_metas *new_tab; 6316 const struct btf_member *member; 6317 struct btf_struct_meta *type; 6318 struct btf_record *record; 6319 const struct btf_type *t; 6320 int j, tab_cnt; 6321 6322 t = btf_type_by_id(btf, i); 6323 if (!__btf_type_is_struct(t)) 6324 continue; 6325 6326 cond_resched(); 6327 6328 for_each_member(j, t, member) { 6329 if (btf_id_set_contains(aof, member->type)) 6330 goto parse; 6331 } 6332 continue; 6333 parse: 6334 tab_cnt = tab ? tab->cnt : 0; 6335 new_tab = krealloc(tab, struct_size(new_tab, types, tab_cnt + 1), 6336 GFP_KERNEL | __GFP_NOWARN); 6337 if (!new_tab) { 6338 ret = -ENOMEM; 6339 goto free; 6340 } 6341 if (!tab) 6342 new_tab->cnt = 0; 6343 tab = new_tab; 6344 6345 type = &tab->types[tab->cnt]; 6346 type->btf_id = i; 6347 record = btf_parse_fields(btf, t, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK | BPF_LIST_HEAD | BPF_LIST_NODE | 6348 BPF_RB_ROOT | BPF_RB_NODE | BPF_REFCOUNT | 6349 BPF_KPTR, t->size); 6350 /* The record cannot be unset, treat it as an error if so */ 6351 if (IS_ERR_OR_NULL(record)) { 6352 ret = PTR_ERR_OR_ZERO(record) ?: -EFAULT; 6353 goto free; 6354 } 6355 type->record = record; 6356 tab->cnt++; 6357 } 6358 kfree(aof); 6359 return tab; 6360 free: 6361 btf_struct_metas_free(tab); 6362 free_aof: 6363 kfree(aof); 6364 return ERR_PTR(ret); 6365 } 6366 6367 struct btf_struct_meta *btf_find_struct_meta(const struct btf *btf, u32 btf_id) 6368 { 6369 struct btf_struct_metas *tab; 6370 6371 BUILD_BUG_ON(offsetof(struct btf_struct_meta, btf_id) != 0); 6372 tab = btf->struct_meta_tab; 6373 if (!tab) 6374 return NULL; 6375 return bsearch(&btf_id, tab->types, tab->cnt, sizeof(tab->types[0]), btf_id_cmp_func); 6376 } 6377 6378 static int btf_check_modifier_chain_length(struct btf_verifier_env *env, 6379 struct btf *btf, int start_id) 6380 { 6381 int i, n, good_id = start_id - 1; 6382 6383 n = btf_nr_types(btf); 6384 for (i = start_id; i < n; i++) { 6385 const struct btf_type *t; 6386 int chain_limit = 32; 6387 u32 cur_id = i; 6388 6389 t = btf_type_by_id(btf, i); 6390 if (!t) 6391 return -EINVAL; 6392 if (!btf_type_is_modifier(t)) 6393 continue; 6394 6395 cond_resched(); 6396 6397 while (btf_type_is_modifier(t)) { 6398 if (!chain_limit--) { 6399 btf_verifier_log(env, "Max chain length or cycle detected"); 6400 return -ELOOP; 6401 } 6402 6403 if (cur_id <= good_id) 6404 break; 6405 /* Move to next type */ 6406 cur_id = t->type; 6407 t = btf_type_by_id(btf, cur_id); 6408 if (!t) 6409 return -EINVAL; 6410 } 6411 good_id = i; 6412 } 6413 return 0; 6414 } 6415 6416 static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, 6417 struct bpf_log_attr *attr_log) 6418 { 6419 bpfptr_t btf_data = make_bpfptr(attr->btf, uattr.is_kernel); 6420 struct btf_struct_metas *struct_meta_tab; 6421 struct btf_verifier_env *env = NULL; 6422 struct btf *btf = NULL; 6423 u8 *data; 6424 int err, ret; 6425 6426 if (attr->btf_size > BTF_MAX_SIZE) 6427 return ERR_PTR(-E2BIG); 6428 6429 env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN); 6430 if (!env) 6431 return ERR_PTR(-ENOMEM); 6432 6433 /* user could have requested verbose verifier output 6434 * and supplied buffer to store the verification trace 6435 */ 6436 err = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size); 6437 if (err) 6438 goto errout_free; 6439 6440 btf = kzalloc_obj(*btf, GFP_KERNEL | __GFP_NOWARN); 6441 if (!btf) { 6442 err = -ENOMEM; 6443 goto errout; 6444 } 6445 env->btf = btf; 6446 btf->named_start_id = 0; 6447 6448 data = kvmalloc(attr->btf_size, GFP_KERNEL | __GFP_NOWARN); 6449 if (!data) { 6450 err = -ENOMEM; 6451 goto errout; 6452 } 6453 6454 btf->data = data; 6455 btf->data_size = attr->btf_size; 6456 6457 if (copy_from_bpfptr(data, btf_data, attr->btf_size)) { 6458 err = -EFAULT; 6459 goto errout; 6460 } 6461 6462 err = btf_parse_hdr(env); 6463 if (err) 6464 goto errout; 6465 6466 btf->nohdr_data = btf->data + btf->hdr.hdr_len; 6467 6468 err = btf_parse_str_sec(env); 6469 if (err) 6470 goto errout; 6471 6472 err = btf_parse_layout_sec(env); 6473 if (err) 6474 goto errout; 6475 6476 err = btf_parse_type_sec(env); 6477 if (err) 6478 goto errout; 6479 6480 err = btf_check_modifier_chain_length(env, btf, 1); 6481 if (err) 6482 goto errout; 6483 6484 struct_meta_tab = btf_parse_struct_metas(&env->log, btf); 6485 if (IS_ERR(struct_meta_tab)) { 6486 err = PTR_ERR(struct_meta_tab); 6487 goto errout; 6488 } 6489 btf->struct_meta_tab = struct_meta_tab; 6490 6491 if (struct_meta_tab) { 6492 int i; 6493 6494 for (i = 0; i < struct_meta_tab->cnt; i++) { 6495 err = btf_check_and_fixup_fields(btf, struct_meta_tab->types[i].record); 6496 if (err < 0) 6497 goto errout_meta; 6498 } 6499 6500 err = btf_check_ownership_depth(btf, struct_meta_tab); 6501 if (err < 0) 6502 goto errout_meta; 6503 } 6504 6505 err = bpf_log_attr_finalize(attr_log, &env->log); 6506 if (err) 6507 goto errout_free; 6508 6509 btf_verifier_env_free(env); 6510 refcount_set(&btf->refcnt, 1); 6511 return btf; 6512 6513 errout_meta: 6514 btf_free_struct_meta_tab(btf); 6515 errout: 6516 /* overwrite err with -ENOSPC or -EFAULT */ 6517 ret = bpf_log_attr_finalize(attr_log, &env->log); 6518 if (ret) 6519 err = ret; 6520 errout_free: 6521 btf_verifier_env_free(env); 6522 if (btf) 6523 btf_free(btf); 6524 return ERR_PTR(err); 6525 } 6526 6527 extern char __start_BTF[]; 6528 extern char __stop_BTF[]; 6529 extern struct btf *btf_vmlinux; 6530 6531 #define BPF_MAP_TYPE(_id, _ops) 6532 #define BPF_LINK_TYPE(_id, _name) 6533 static union { 6534 struct bpf_ctx_convert { 6535 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 6536 prog_ctx_type _id##_prog; \ 6537 kern_ctx_type _id##_kern; 6538 #include <linux/bpf_types.h> 6539 #undef BPF_PROG_TYPE 6540 } *__t; 6541 /* 't' is written once under lock. Read many times. */ 6542 const struct btf_type *t; 6543 } bpf_ctx_convert; 6544 enum { 6545 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 6546 __ctx_convert##_id, 6547 #include <linux/bpf_types.h> 6548 #undef BPF_PROG_TYPE 6549 __ctx_convert_unused, /* to avoid empty enum in extreme .config */ 6550 }; 6551 static u8 bpf_ctx_convert_map[] = { 6552 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 6553 [_id] = __ctx_convert##_id, 6554 #include <linux/bpf_types.h> 6555 #undef BPF_PROG_TYPE 6556 0, /* avoid empty array */ 6557 }; 6558 #undef BPF_MAP_TYPE 6559 #undef BPF_LINK_TYPE 6560 6561 static const struct btf_type *find_canonical_prog_ctx_type(enum bpf_prog_type prog_type) 6562 { 6563 const struct btf_type *conv_struct; 6564 const struct btf_member *ctx_type; 6565 6566 conv_struct = bpf_ctx_convert.t; 6567 if (!conv_struct) 6568 return NULL; 6569 /* prog_type is valid bpf program type. No need for bounds check. */ 6570 ctx_type = btf_type_member(conv_struct) + bpf_ctx_convert_map[prog_type] * 2; 6571 /* ctx_type is a pointer to prog_ctx_type in vmlinux. 6572 * Like 'struct __sk_buff' 6573 */ 6574 return btf_type_by_id(btf_vmlinux, ctx_type->type); 6575 } 6576 6577 static int find_kern_ctx_type_id(enum bpf_prog_type prog_type) 6578 { 6579 const struct btf_type *conv_struct; 6580 const struct btf_member *ctx_type; 6581 6582 conv_struct = bpf_ctx_convert.t; 6583 if (!conv_struct) 6584 return -EFAULT; 6585 /* prog_type is valid bpf program type. No need for bounds check. */ 6586 ctx_type = btf_type_member(conv_struct) + bpf_ctx_convert_map[prog_type] * 2 + 1; 6587 /* ctx_type is a pointer to prog_ctx_type in vmlinux. 6588 * Like 'struct sk_buff' 6589 */ 6590 return ctx_type->type; 6591 } 6592 6593 /* 6594 * A projection type is the view that programs have of a kernel object with 6595 * another layout: see btf_is_projection_of() for the two types behind them. 6596 */ 6597 bool btf_is_projection_type(const char *pname) 6598 { 6599 return strcmp(pname, "__sk_buff") == 0 || strcmp(pname, "xdp_md") == 0; 6600 } 6601 6602 bool btf_is_projection_of(const char *pname, const char *tname) 6603 { 6604 if (strcmp(pname, "__sk_buff") == 0 && strcmp(tname, "sk_buff") == 0) 6605 return true; 6606 if (strcmp(pname, "xdp_md") == 0 && strcmp(tname, "xdp_buff") == 0) 6607 return true; 6608 return false; 6609 } 6610 6611 bool btf_is_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf, 6612 const struct btf_type *t, enum bpf_prog_type prog_type, 6613 int arg) 6614 { 6615 const struct btf_type *ctx_type; 6616 const char *tname, *ctx_tname; 6617 6618 t = btf_type_by_id(btf, t->type); 6619 6620 /* KPROBE programs allow bpf_user_pt_regs_t typedef, which we need to 6621 * check before we skip all the typedef below. 6622 */ 6623 if (prog_type == BPF_PROG_TYPE_KPROBE) { 6624 while (btf_type_is_modifier(t) && !btf_type_is_typedef(t)) 6625 t = btf_type_by_id(btf, t->type); 6626 6627 if (btf_type_is_typedef(t)) { 6628 tname = btf_name_by_offset(btf, t->name_off); 6629 if (tname && strcmp(tname, "bpf_user_pt_regs_t") == 0) 6630 return true; 6631 } 6632 } 6633 6634 while (btf_type_is_modifier(t)) 6635 t = btf_type_by_id(btf, t->type); 6636 if (!btf_type_is_struct(t)) { 6637 /* Only pointer to struct is supported for now. 6638 * That means that BPF_PROG_TYPE_TRACEPOINT with BTF 6639 * is not supported yet. 6640 * BPF_PROG_TYPE_RAW_TRACEPOINT is fine. 6641 */ 6642 return false; 6643 } 6644 tname = btf_name_by_offset(btf, t->name_off); 6645 if (!tname) { 6646 bpf_log(log, "arg#%d struct doesn't have a name\n", arg); 6647 return false; 6648 } 6649 6650 ctx_type = find_canonical_prog_ctx_type(prog_type); 6651 if (!ctx_type) { 6652 bpf_log(log, "btf_vmlinux is malformed\n"); 6653 /* should not happen */ 6654 return false; 6655 } 6656 again: 6657 ctx_tname = btf_name_by_offset(btf_vmlinux, ctx_type->name_off); 6658 if (!ctx_tname) { 6659 /* should not happen */ 6660 bpf_log(log, "Please fix kernel include/linux/bpf_types.h\n"); 6661 return false; 6662 } 6663 /* program types without named context types work only with arg:ctx tag */ 6664 if (ctx_tname[0] == '\0') 6665 return false; 6666 /* only compare that prog's ctx type name is the same as 6667 * kernel expects. No need to compare field by field. 6668 * It's ok for bpf prog to do: 6669 * struct __sk_buff {}; 6670 * int socket_filter_bpf_prog(struct __sk_buff *skb) 6671 * { // no fields of skb are ever used } 6672 */ 6673 if (btf_is_projection_of(ctx_tname, tname)) 6674 return true; 6675 if (strcmp(ctx_tname, tname)) { 6676 /* bpf_user_pt_regs_t is a typedef, so resolve it to 6677 * underlying struct and check name again 6678 */ 6679 if (!btf_type_is_modifier(ctx_type)) 6680 return false; 6681 while (btf_type_is_modifier(ctx_type)) 6682 ctx_type = btf_type_by_id(btf_vmlinux, ctx_type->type); 6683 goto again; 6684 } 6685 return true; 6686 } 6687 6688 /* forward declarations for arch-specific underlying types of 6689 * bpf_user_pt_regs_t; this avoids the need for arch-specific #ifdef 6690 * compilation guards below for BPF_PROG_TYPE_PERF_EVENT checks, but still 6691 * works correctly with __builtin_types_compatible_p() on respective 6692 * architectures 6693 */ 6694 struct user_regs_struct; 6695 struct user_pt_regs; 6696 6697 static int btf_validate_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf, 6698 const struct btf_type *t, int arg, 6699 enum bpf_prog_type prog_type, 6700 enum bpf_attach_type attach_type) 6701 { 6702 const struct btf_type *ctx_type; 6703 const char *tname, *ctx_tname; 6704 6705 if (!btf_is_ptr(t)) { 6706 bpf_log(log, "arg#%d type isn't a pointer\n", arg); 6707 return -EINVAL; 6708 } 6709 t = btf_type_by_id(btf, t->type); 6710 6711 /* KPROBE and PERF_EVENT programs allow bpf_user_pt_regs_t typedef */ 6712 if (prog_type == BPF_PROG_TYPE_KPROBE || prog_type == BPF_PROG_TYPE_PERF_EVENT) { 6713 while (btf_type_is_modifier(t) && !btf_type_is_typedef(t)) 6714 t = btf_type_by_id(btf, t->type); 6715 6716 if (btf_type_is_typedef(t)) { 6717 tname = btf_name_by_offset(btf, t->name_off); 6718 if (tname && strcmp(tname, "bpf_user_pt_regs_t") == 0) 6719 return 0; 6720 } 6721 } 6722 6723 /* all other program types don't use typedefs for context type */ 6724 while (btf_type_is_modifier(t)) 6725 t = btf_type_by_id(btf, t->type); 6726 6727 /* `void *ctx __arg_ctx` is always valid */ 6728 if (btf_type_is_void(t)) 6729 return 0; 6730 6731 tname = btf_name_by_offset(btf, t->name_off); 6732 if (str_is_empty(tname)) { 6733 bpf_log(log, "arg#%d type doesn't have a name\n", arg); 6734 return -EINVAL; 6735 } 6736 6737 /* special cases */ 6738 switch (prog_type) { 6739 case BPF_PROG_TYPE_KPROBE: 6740 if (__btf_type_is_struct(t) && strcmp(tname, "pt_regs") == 0) 6741 return 0; 6742 break; 6743 case BPF_PROG_TYPE_PERF_EVENT: 6744 if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct pt_regs) && 6745 __btf_type_is_struct(t) && strcmp(tname, "pt_regs") == 0) 6746 return 0; 6747 if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct user_pt_regs) && 6748 __btf_type_is_struct(t) && strcmp(tname, "user_pt_regs") == 0) 6749 return 0; 6750 if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct user_regs_struct) && 6751 __btf_type_is_struct(t) && strcmp(tname, "user_regs_struct") == 0) 6752 return 0; 6753 break; 6754 case BPF_PROG_TYPE_RAW_TRACEPOINT: 6755 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 6756 /* allow u64* as ctx */ 6757 if (btf_is_int(t) && t->size == 8) 6758 return 0; 6759 break; 6760 case BPF_PROG_TYPE_TRACING: 6761 switch (attach_type) { 6762 case BPF_TRACE_RAW_TP: 6763 /* tp_btf program is TRACING, so need special case here */ 6764 if (__btf_type_is_struct(t) && 6765 strcmp(tname, "bpf_raw_tracepoint_args") == 0) 6766 return 0; 6767 /* allow u64* as ctx */ 6768 if (btf_is_int(t) && t->size == 8) 6769 return 0; 6770 break; 6771 case BPF_TRACE_ITER: 6772 /* allow struct bpf_iter__xxx types only */ 6773 if (__btf_type_is_struct(t) && 6774 strncmp(tname, "bpf_iter__", sizeof("bpf_iter__") - 1) == 0) 6775 return 0; 6776 break; 6777 case BPF_TRACE_FENTRY: 6778 case BPF_TRACE_FEXIT: 6779 case BPF_MODIFY_RETURN: 6780 case BPF_TRACE_FSESSION: 6781 /* allow u64* as ctx */ 6782 if (btf_is_int(t) && t->size == 8) 6783 return 0; 6784 break; 6785 default: 6786 break; 6787 } 6788 break; 6789 case BPF_PROG_TYPE_LSM: 6790 case BPF_PROG_TYPE_STRUCT_OPS: 6791 /* allow u64* as ctx */ 6792 if (btf_is_int(t) && t->size == 8) 6793 return 0; 6794 break; 6795 case BPF_PROG_TYPE_TRACEPOINT: 6796 case BPF_PROG_TYPE_SYSCALL: 6797 case BPF_PROG_TYPE_EXT: 6798 return 0; /* anything goes */ 6799 default: 6800 break; 6801 } 6802 6803 ctx_type = find_canonical_prog_ctx_type(prog_type); 6804 if (!ctx_type) { 6805 /* should not happen */ 6806 bpf_log(log, "btf_vmlinux is malformed\n"); 6807 return -EINVAL; 6808 } 6809 6810 /* resolve typedefs and check that underlying structs are matching as well */ 6811 while (btf_type_is_modifier(ctx_type)) 6812 ctx_type = btf_type_by_id(btf_vmlinux, ctx_type->type); 6813 6814 /* if program type doesn't have distinctly named struct type for 6815 * context, then __arg_ctx argument can only be `void *`, which we 6816 * already checked above 6817 */ 6818 if (!__btf_type_is_struct(ctx_type)) { 6819 bpf_log(log, "arg#%d should be void pointer\n", arg); 6820 return -EINVAL; 6821 } 6822 6823 ctx_tname = btf_name_by_offset(btf_vmlinux, ctx_type->name_off); 6824 if (!__btf_type_is_struct(t) || strcmp(ctx_tname, tname) != 0) { 6825 bpf_log(log, "arg#%d should be `struct %s *`\n", arg, ctx_tname); 6826 return -EINVAL; 6827 } 6828 6829 return 0; 6830 } 6831 6832 static int btf_translate_to_vmlinux(struct bpf_verifier_log *log, 6833 struct btf *btf, 6834 const struct btf_type *t, 6835 enum bpf_prog_type prog_type, 6836 int arg) 6837 { 6838 if (!btf_is_prog_ctx_type(log, btf, t, prog_type, arg)) 6839 return -ENOENT; 6840 return find_kern_ctx_type_id(prog_type); 6841 } 6842 6843 int get_kern_ctx_btf_id(struct bpf_verifier_log *log, enum bpf_prog_type prog_type) 6844 { 6845 const struct btf_member *kctx_member; 6846 const struct btf_type *conv_struct; 6847 const struct btf_type *kctx_type; 6848 u32 kctx_type_id; 6849 6850 conv_struct = bpf_ctx_convert.t; 6851 /* get member for kernel ctx type */ 6852 kctx_member = btf_type_member(conv_struct) + bpf_ctx_convert_map[prog_type] * 2 + 1; 6853 kctx_type_id = kctx_member->type; 6854 kctx_type = btf_type_by_id(btf_vmlinux, kctx_type_id); 6855 if (!btf_type_is_struct(kctx_type)) { 6856 bpf_log(log, "kern ctx type id %u is not a struct\n", kctx_type_id); 6857 return -EINVAL; 6858 } 6859 6860 return kctx_type_id; 6861 } 6862 6863 BTF_ID_LIST_SINGLE(bpf_ctx_convert_btf_id, struct, bpf_ctx_convert) 6864 6865 static struct btf *btf_parse_base(struct btf_verifier_env *env, const char *name, 6866 void *data, unsigned int data_size) 6867 { 6868 struct btf *btf = NULL; 6869 int err; 6870 6871 if (!IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) 6872 return ERR_PTR(-ENOENT); 6873 6874 btf = kzalloc_obj(*btf, GFP_KERNEL | __GFP_NOWARN); 6875 if (!btf) { 6876 err = -ENOMEM; 6877 goto errout; 6878 } 6879 env->btf = btf; 6880 6881 btf->data = data; 6882 btf->data_size = data_size; 6883 btf->kernel_btf = true; 6884 btf->named_start_id = 0; 6885 strscpy(btf->name, name); 6886 6887 err = btf_parse_hdr(env); 6888 if (err) 6889 goto errout; 6890 6891 btf->nohdr_data = btf->data + btf->hdr.hdr_len; 6892 6893 err = btf_parse_str_sec(env); 6894 if (err) 6895 goto errout; 6896 6897 err = btf_check_all_metas(env); 6898 if (err) 6899 goto errout; 6900 6901 err = btf_check_modifier_chain_length(env, btf, 1); 6902 if (err) 6903 goto errout; 6904 6905 btf_check_sorted(btf); 6906 refcount_set(&btf->refcnt, 1); 6907 6908 return btf; 6909 6910 errout: 6911 if (btf) { 6912 kvfree(btf->types); 6913 kfree(btf); 6914 } 6915 return ERR_PTR(err); 6916 } 6917 6918 struct btf *btf_parse_vmlinux(void) 6919 { 6920 struct btf_verifier_env *env = NULL; 6921 struct bpf_verifier_log *log; 6922 struct btf *btf; 6923 int err; 6924 6925 env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN); 6926 if (!env) 6927 return ERR_PTR(-ENOMEM); 6928 6929 log = &env->log; 6930 log->level = BPF_LOG_KERNEL; 6931 btf = btf_parse_base(env, "vmlinux", __start_BTF, __stop_BTF - __start_BTF); 6932 if (IS_ERR(btf)) 6933 goto err_out; 6934 6935 /* btf_parse_vmlinux() runs under btf_vmlinux_lock */ 6936 bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]); 6937 err = btf_alloc_id(btf); 6938 if (err) { 6939 btf_free(btf); 6940 btf = ERR_PTR(err); 6941 } 6942 err_out: 6943 btf_verifier_env_free(env); 6944 return btf; 6945 } 6946 6947 /* If .BTF_ids section was created with distilled base BTF, both base and 6948 * split BTF ids will need to be mapped to actual base/split ids for 6949 * BTF now that it has been relocated. 6950 */ 6951 __u32 btf_relocate_id(const struct btf *btf, __u32 id) 6952 { 6953 if (!btf->base_btf || !btf->base_id_map) 6954 return id; 6955 return btf->base_id_map[id]; 6956 } 6957 6958 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES 6959 6960 static struct btf *btf_parse_split(struct btf_verifier_env *env, 6961 const char *name, const void *data, 6962 unsigned int data_size, struct btf *base_btf) 6963 { 6964 struct btf *btf; 6965 int err; 6966 6967 btf = kzalloc_obj(*btf, GFP_KERNEL | __GFP_NOWARN); 6968 if (!btf) 6969 return ERR_PTR(-ENOMEM); 6970 env->btf = btf; 6971 6972 btf_set_base_btf(btf, base_btf); 6973 btf->kernel_btf = true; 6974 btf->named_start_id = 0; 6975 strscpy(btf->name, name); 6976 6977 btf->data = kvmemdup(data, data_size, GFP_KERNEL | __GFP_NOWARN); 6978 if (!btf->data) { 6979 err = -ENOMEM; 6980 goto errout; 6981 } 6982 btf->data_size = data_size; 6983 6984 err = btf_parse_hdr(env); 6985 if (err) 6986 goto errout; 6987 6988 btf->nohdr_data = btf->data + btf->hdr.hdr_len; 6989 6990 err = btf_parse_str_sec(env); 6991 if (err) 6992 goto errout; 6993 6994 err = btf_check_all_metas(env); 6995 if (err) 6996 goto errout; 6997 6998 err = btf_check_modifier_chain_length(env, btf, btf_nr_types(base_btf)); 6999 if (err) 7000 goto errout; 7001 7002 return btf; 7003 7004 errout: 7005 btf_free(btf); 7006 return ERR_PTR(err); 7007 } 7008 7009 static int btf_rebase_inline(struct btf *inline_btf, 7010 const struct btf *module_btf, 7011 const u32 *module_id_map, 7012 const u32 *module_str_map, 7013 u32 old_module_type_cnt) 7014 { 7015 u32 old_start_id = inline_btf->start_id; 7016 u32 old_start_str_off = inline_btf->start_str_off; 7017 u32 old_module_start_str_off = old_start_str_off - module_btf->hdr.str_len; 7018 u32 new_start_id = btf_nr_types(module_btf); 7019 u32 new_start_str_off = module_btf->start_str_off + module_btf->hdr.str_len; 7020 s64 id_delta = (s64)new_start_id - old_start_id; 7021 s64 str_delta = (s64)new_start_str_off - old_start_str_off; 7022 u32 i; 7023 7024 /* 7025 * The inline BTF was parsed relative to the original module BTF. Its 7026 * base IDs must therefore use the map generated when that BTF was 7027 * relocated, while IDs for inline-local types only move by the change 7028 * in the module BTF's starting ID. 7029 */ 7030 for (i = 0; i < inline_btf->nr_types; i++) { 7031 struct btf_field_iter it; 7032 struct btf_type *t = inline_btf->types[i]; 7033 u32 *id, *str_off; 7034 int err; 7035 7036 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS); 7037 if (err) 7038 return err; 7039 while ((id = btf_field_iter_next(&it))) { 7040 if (!*id) 7041 continue; 7042 if (*id < old_module_type_cnt) { 7043 if (module_id_map) 7044 *id = module_id_map[*id]; 7045 } else if (*id >= old_start_id) { 7046 *id += id_delta; 7047 } else { 7048 return -EINVAL; 7049 } 7050 } 7051 7052 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS); 7053 if (err) 7054 return err; 7055 while ((str_off = btf_field_iter_next(&it))) { 7056 if (!*str_off) 7057 continue; 7058 if (*str_off < old_module_start_str_off) { 7059 /* Vmlinux strings retain their offsets for in-tree modules. */ 7060 if (!module_id_map) 7061 continue; 7062 if (!module_str_map || !module_str_map[*str_off]) 7063 return -EINVAL; 7064 *str_off = module_str_map[*str_off]; 7065 continue; 7066 } 7067 *str_off += str_delta; 7068 } 7069 } 7070 7071 btf_set_base_btf(inline_btf, module_btf); 7072 btf_check_sorted(inline_btf); 7073 return 0; 7074 } 7075 7076 static struct btf *btf_parse_module(const char *module_name, const void *data, 7077 unsigned int data_size, void *base_data, 7078 unsigned int base_data_size, 7079 const void *inline_data, 7080 unsigned int inline_data_size, 7081 void **relocated_inline_data) 7082 { 7083 struct btf *btf = NULL, *inline_btf = NULL, *vmlinux_btf, *base_btf = NULL; 7084 struct btf_verifier_env *env = NULL; 7085 struct bpf_verifier_log *log; 7086 u32 *module_str_map = NULL; 7087 u32 old_module_type_cnt; 7088 int err = 0; 7089 7090 *relocated_inline_data = NULL; 7091 vmlinux_btf = bpf_get_btf_vmlinux(); 7092 if (IS_ERR(vmlinux_btf)) 7093 return vmlinux_btf; 7094 if (!vmlinux_btf) 7095 return ERR_PTR(-EINVAL); 7096 7097 env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN); 7098 if (!env) 7099 return ERR_PTR(-ENOMEM); 7100 7101 log = &env->log; 7102 log->level = BPF_LOG_KERNEL; 7103 7104 if (base_data) { 7105 base_btf = btf_parse_base(env, ".BTF.base", base_data, base_data_size); 7106 if (IS_ERR(base_btf)) { 7107 err = PTR_ERR(base_btf); 7108 goto errout; 7109 } 7110 } else { 7111 base_btf = vmlinux_btf; 7112 } 7113 7114 btf = btf_parse_split(env, module_name, data, data_size, base_btf); 7115 if (IS_ERR(btf)) { 7116 err = PTR_ERR(btf); 7117 btf = NULL; 7118 goto errout; 7119 } 7120 old_module_type_cnt = btf_nr_types(btf); 7121 7122 if (inline_data_size) { 7123 inline_btf = btf_parse_split(env, module_name, inline_data, 7124 inline_data_size, btf); 7125 if (IS_ERR(inline_btf)) { 7126 pr_warn("failed to validate module [%s] inline BTF: %ld\n", 7127 module_name, PTR_ERR(inline_btf)); 7128 inline_btf = NULL; 7129 } 7130 } 7131 7132 if (base_btf != vmlinux_btf) { 7133 err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map, 7134 &module_str_map); 7135 if (err) 7136 goto errout; 7137 btf_free(base_btf); 7138 base_btf = vmlinux_btf; 7139 } 7140 7141 if (inline_btf) { 7142 err = btf_rebase_inline(inline_btf, btf, btf->base_id_map, 7143 module_str_map, old_module_type_cnt); 7144 if (err) { 7145 pr_warn("failed to relocate module [%s] inline BTF: %d\n", 7146 module_name, err); 7147 btf_free(inline_btf); 7148 } else { 7149 *relocated_inline_data = inline_btf->data; 7150 inline_btf->data = NULL; 7151 btf_free(inline_btf); 7152 } 7153 } 7154 7155 /* 7156 * With a distilled base, btf_relocate() replaces the base BTF and 7157 * rewrites string offsets. Check ordering only after that final BTF 7158 * view has been established, so named_start_id describes the BTF used 7159 * by name lookups. 7160 */ 7161 btf_check_sorted(btf); 7162 btf_verifier_env_free(env); 7163 kvfree(module_str_map); 7164 refcount_set(&btf->refcnt, 1); 7165 return btf; 7166 7167 errout: 7168 kvfree(module_str_map); 7169 btf_verifier_env_free(env); 7170 if (inline_btf) 7171 btf_free(inline_btf); 7172 if (!IS_ERR(base_btf) && base_btf != vmlinux_btf) 7173 btf_free(base_btf); 7174 if (btf) 7175 btf_free(btf); 7176 return ERR_PTR(err); 7177 } 7178 7179 #endif /* CONFIG_DEBUG_INFO_BTF_MODULES */ 7180 7181 struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog) 7182 { 7183 struct bpf_prog *tgt_prog = prog->aux->dst_prog; 7184 7185 if (tgt_prog) 7186 return tgt_prog->aux->btf; 7187 else 7188 return prog->aux->attach_btf; 7189 } 7190 7191 u32 btf_ctx_arg_idx(struct btf *btf, const struct btf_type *func_proto, 7192 int off) 7193 { 7194 const struct btf_param *args; 7195 const struct btf_type *t; 7196 u32 offset = 0, nr_args; 7197 int i; 7198 7199 if (!func_proto) 7200 return off / 8; 7201 7202 nr_args = btf_type_vlen(func_proto); 7203 args = (const struct btf_param *)(func_proto + 1); 7204 for (i = 0; i < nr_args; i++) { 7205 t = btf_type_skip_modifiers(btf, args[i].type, NULL); 7206 offset += btf_type_is_ptr(t) ? 8 : roundup(t->size, 8); 7207 if (off < offset) 7208 return i; 7209 } 7210 7211 t = btf_type_skip_modifiers(btf, func_proto->type, NULL); 7212 offset += btf_type_is_ptr(t) ? 8 : roundup(t->size, 8); 7213 if (off < offset) 7214 return nr_args; 7215 7216 return nr_args + 1; 7217 } 7218 7219 static bool prog_args_trusted(const struct bpf_prog *prog) 7220 { 7221 enum bpf_attach_type atype = prog->expected_attach_type; 7222 7223 switch (prog->type) { 7224 case BPF_PROG_TYPE_TRACING: 7225 return atype == BPF_TRACE_RAW_TP || atype == BPF_TRACE_ITER; 7226 case BPF_PROG_TYPE_LSM: 7227 return bpf_lsm_is_trusted(prog); 7228 case BPF_PROG_TYPE_STRUCT_OPS: 7229 return true; 7230 default: 7231 return false; 7232 } 7233 } 7234 7235 int btf_ctx_arg_offset(const struct btf *btf, const struct btf_type *func_proto, 7236 u32 arg_no) 7237 { 7238 const struct btf_param *args; 7239 const struct btf_type *t; 7240 int off = 0, i; 7241 u32 sz; 7242 7243 args = btf_params(func_proto); 7244 for (i = 0; i < arg_no; i++) { 7245 t = btf_type_by_id(btf, args[i].type); 7246 t = btf_resolve_size(btf, t, &sz); 7247 if (IS_ERR(t)) 7248 return PTR_ERR(t); 7249 off += roundup(sz, 8); 7250 } 7251 7252 return off; 7253 } 7254 7255 struct bpf_raw_tp_null_args { 7256 const char *func; 7257 u64 mask; 7258 }; 7259 7260 static const struct bpf_raw_tp_null_args raw_tp_null_args[] = { 7261 /* sched */ 7262 { "sched_pi_setprio", 0x10 }, 7263 /* 7264 * do_wait() passes NULL for wait4(-1) and waitid(P_ALL). 7265 */ 7266 { "sched_process_wait", 0x1 }, 7267 /* ... from sched_numa_pair_template event class */ 7268 { "sched_stick_numa", 0x100 }, 7269 { "sched_swap_numa", 0x100 }, 7270 /* afs */ 7271 { "afs_make_fs_call", 0x10 }, 7272 { "afs_make_fs_calli", 0x10 }, 7273 { "afs_make_fs_call1", 0x10 }, 7274 { "afs_make_fs_call2", 0x10 }, 7275 { "afs_protocol_error", 0x1 }, 7276 { "afs_flock_ev", 0x10 }, 7277 /* cachefiles */ 7278 { "cachefiles_lookup", 0x1 | 0x200 }, 7279 { "cachefiles_unlink", 0x1 }, 7280 { "cachefiles_rename", 0x1 }, 7281 { "cachefiles_prep_read", 0x1 }, 7282 { "cachefiles_mark_active", 0x1 }, 7283 { "cachefiles_mark_failed", 0x1 }, 7284 { "cachefiles_mark_inactive", 0x1 }, 7285 { "cachefiles_vfs_error", 0x1 }, 7286 { "cachefiles_io_error", 0x1 }, 7287 /* ext4, from ext4__mballoc event class */ 7288 { "ext4_mballoc_discard", 0x10 }, 7289 { "ext4_mballoc_free", 0x10 }, 7290 /* fib */ 7291 { "fib_table_lookup", 0x100 }, 7292 /* filelock */ 7293 /* ... from filelock_lock event class */ 7294 { "posix_lock_inode", 0x10 }, 7295 { "fcntl_setlk", 0x10 }, 7296 { "locks_remove_posix", 0x10 }, 7297 { "flock_lock_inode", 0x10 }, 7298 /* ... from filelock_lease event class */ 7299 { "break_lease_noblock", 0x10 }, 7300 { "break_lease_block", 0x10 }, 7301 { "break_lease_unblock", 0x10 }, 7302 { "generic_delete_lease", 0x10 }, 7303 { "time_out_leases", 0x10 }, 7304 /* host1x */ 7305 { "host1x_cdma_push_gather", 0x10000 }, 7306 /* huge_memory */ 7307 { "mm_khugepaged_scan_pmd", 0x10 }, 7308 { "mm_collapse_huge_page_isolate", 0x1 }, 7309 { "mm_khugepaged_scan_file", 0x10 }, 7310 { "mm_khugepaged_collapse_file", 0x10 }, 7311 /* kmem */ 7312 { "mm_page_alloc", 0x1 }, 7313 { "mm_page_pcpu_drain", 0x1 }, 7314 /* .. from mm_page event class */ 7315 { "mm_page_alloc_zone_locked", 0x1 }, 7316 /* netfs */ 7317 { "netfs_failure", 0x10 }, 7318 /* power */ 7319 { "device_pm_callback_start", 0x10 }, 7320 /* qdisc */ 7321 { "qdisc_dequeue", 0x1000 }, 7322 /* rxrpc */ 7323 { "rxrpc_recvdata", 0x1 }, 7324 { "rxrpc_resend", 0x10 }, 7325 { "rxrpc_tq", 0x10 }, 7326 { "rxrpc_client", 0x1 }, 7327 /* signal */ 7328 { "signal_generate", 0x20 }, 7329 { "signal_deliver", 0x20 }, 7330 /* skb */ 7331 {"kfree_skb", 0x1000}, 7332 /* sunrpc */ 7333 { "xs_stream_read_data", 0x1 }, 7334 /* ... from xprt_cong_event event class */ 7335 { "xprt_reserve_cong", 0x10 }, 7336 { "xprt_release_cong", 0x10 }, 7337 { "xprt_get_cong", 0x10 }, 7338 { "xprt_put_cong", 0x10 }, 7339 /* tcp */ 7340 { "tcp_send_reset", 0x11 }, 7341 { "tcp_sendmsg_locked", 0x100 }, 7342 /* tegra_apb_dma */ 7343 { "tegra_dma_tx_status", 0x100 }, 7344 /* timer_migration */ 7345 { "tmigr_update_events", 0x1 }, 7346 /* writeback, from writeback_folio_template event class */ 7347 { "writeback_dirty_folio", 0x10 }, 7348 { "folio_wait_writeback", 0x10 }, 7349 /* rdma */ 7350 { "mr_integ_alloc", 0x2000 }, 7351 /* bpf_testmod */ 7352 { "bpf_testmod_test_read", 0x0 }, 7353 /* amdgpu */ 7354 { "amdgpu_vm_bo_map", 0x1 }, 7355 { "amdgpu_vm_bo_unmap", 0x1 }, 7356 /* netfs */ 7357 { "netfs_folioq", 0x1 }, 7358 /* xfs from xfs_defer_pending_class */ 7359 { "xfs_defer_create_intent", 0x1 }, 7360 { "xfs_defer_cancel_list", 0x1 }, 7361 { "xfs_defer_pending_finish", 0x1 }, 7362 { "xfs_defer_pending_abort", 0x1 }, 7363 { "xfs_defer_relog_intent", 0x1 }, 7364 { "xfs_defer_isolate_paused", 0x1 }, 7365 { "xfs_defer_item_pause", 0x1 }, 7366 { "xfs_defer_item_unpause", 0x1 }, 7367 /* xfs from xfs_defer_pending_item_class */ 7368 { "xfs_defer_add_item", 0x1 }, 7369 { "xfs_defer_cancel_item", 0x1 }, 7370 { "xfs_defer_finish_item", 0x1 }, 7371 /* xfs from xfs_icwalk_class */ 7372 { "xfs_ioc_free_eofblocks", 0x10 }, 7373 { "xfs_blockgc_free_space", 0x10 }, 7374 /* xfs from xfs_btree_cur_class */ 7375 { "xfs_btree_updkeys", 0x100 }, 7376 { "xfs_btree_overlapped_query_range", 0x100 }, 7377 /* xfs from xfs_imap_class*/ 7378 { "xfs_map_blocks_found", 0x10000 }, 7379 { "xfs_map_blocks_alloc", 0x10000 }, 7380 { "xfs_iomap_alloc", 0x1000 }, 7381 { "xfs_iomap_found", 0x1000 }, 7382 /* xfs from xfs_fs_class */ 7383 { "xfs_inodegc_flush", 0x1 }, 7384 { "xfs_inodegc_push", 0x1 }, 7385 { "xfs_inodegc_start", 0x1 }, 7386 { "xfs_inodegc_stop", 0x1 }, 7387 { "xfs_inodegc_queue", 0x1 }, 7388 { "xfs_inodegc_throttle", 0x1 }, 7389 { "xfs_fs_sync_fs", 0x1 }, 7390 { "xfs_blockgc_start", 0x1 }, 7391 { "xfs_blockgc_stop", 0x1 }, 7392 { "xfs_blockgc_worker", 0x1 }, 7393 { "xfs_blockgc_flush_all", 0x1 }, 7394 /* xfs_scrub */ 7395 { "xchk_nlinks_live_update", 0x10 }, 7396 /* xfs_scrub from xchk_metapath_class */ 7397 { "xchk_metapath_lookup", 0x100 }, 7398 /* nfsd */ 7399 { "nfsd_dirent", 0x1 }, 7400 { "nfsd_file_acquire", 0x1001 }, 7401 { "nfsd_file_insert_err", 0x1 }, 7402 { "nfsd_file_cons_err", 0x1 }, 7403 /* nfs4 */ 7404 { "nfs4_setup_sequence", 0x1 }, 7405 { "pnfs_update_layout", 0x10000 }, 7406 { "nfs4_inode_callback_event", 0x200 }, 7407 { "nfs4_inode_stateid_callback_event", 0x200 }, 7408 /* nfs from pnfs_layout_event */ 7409 { "pnfs_mds_fallback_pg_init_read", 0x10000 }, 7410 { "pnfs_mds_fallback_pg_init_write", 0x10000 }, 7411 { "pnfs_mds_fallback_pg_get_mirror_count", 0x10000 }, 7412 { "pnfs_mds_fallback_read_done", 0x10000 }, 7413 { "pnfs_mds_fallback_write_done", 0x10000 }, 7414 { "pnfs_mds_fallback_read_pagelist", 0x10000 }, 7415 { "pnfs_mds_fallback_write_pagelist", 0x10000 }, 7416 /* coda */ 7417 { "coda_dec_pic_run", 0x10 }, 7418 { "coda_dec_pic_done", 0x10 }, 7419 /* cfg80211 */ 7420 { "cfg80211_scan_done", 0x11 }, 7421 { "rdev_set_coalesce", 0x10 }, 7422 { "cfg80211_report_wowlan_wakeup", 0x100 }, 7423 { "cfg80211_inform_bss_frame", 0x100 }, 7424 { "cfg80211_michael_mic_failure", 0x10000 }, 7425 /* cfg80211 from wiphy_work_event */ 7426 { "wiphy_work_queue", 0x10 }, 7427 { "wiphy_work_run", 0x10 }, 7428 { "wiphy_work_cancel", 0x10 }, 7429 { "wiphy_work_flush", 0x10 }, 7430 /* hugetlbfs */ 7431 { "hugetlbfs_alloc_inode", 0x10 }, 7432 /* spufs */ 7433 { "spufs_context", 0x10 }, 7434 /* kvm_hv */ 7435 { "kvm_page_fault_enter", 0x100 }, 7436 /* dpu */ 7437 { "dpu_crtc_setup_mixer", 0x100 }, 7438 /* binder */ 7439 { "binder_transaction", 0x100 }, 7440 /* bcachefs */ 7441 { "btree_path_free", 0x100 }, 7442 /* hfi1_tx */ 7443 { "hfi1_sdma_progress", 0x1000 }, 7444 /* iptfs */ 7445 { "iptfs_ingress_postq_event", 0x1000 }, 7446 /* neigh */ 7447 { "neigh_update", 0x10 }, 7448 /* snd_firewire_lib */ 7449 { "amdtp_packet", 0x100 }, 7450 }; 7451 7452 bool btf_ctx_access(int off, int size, enum bpf_access_type type, 7453 const struct bpf_prog *prog, 7454 struct bpf_insn_access_aux *info) 7455 { 7456 static const struct btf_type_tag_match ctx_type_tags[] = { 7457 { "user", MEM_USER }, 7458 { "percpu", MEM_PERCPU }, 7459 }; 7460 const struct btf_type *t = prog->aux->attach_func_proto; 7461 struct bpf_prog *tgt_prog = prog->aux->dst_prog; 7462 struct btf *btf = bpf_prog_get_target_btf(prog); 7463 const char *tname = prog->aux->attach_func_name; 7464 struct bpf_verifier_log *log = info->log; 7465 struct btf_type_tag_walk_ctx ctx; 7466 const struct btf_param *args; 7467 bool ptr_err_raw_tp = false; 7468 u32 nr_args, arg; 7469 int i, ret; 7470 7471 if (off % 8) { 7472 bpf_log(log, "func '%s' offset %d is not multiple of 8\n", 7473 tname, off); 7474 return false; 7475 } 7476 arg = btf_ctx_arg_idx(btf, t, off); 7477 args = (const struct btf_param *)(t + 1); 7478 /* if (t == NULL) Fall back to default BPF prog with 7479 * MAX_BPF_FUNC_REG_ARGS u64 arguments. 7480 */ 7481 nr_args = t ? btf_type_vlen(t) : MAX_BPF_FUNC_REG_ARGS; 7482 if (prog->aux->attach_btf_trace) { 7483 /* skip first 'void *__data' argument in btf_trace_##name typedef */ 7484 args++; 7485 nr_args--; 7486 } 7487 7488 if (arg > nr_args) { 7489 bpf_log(log, "func '%s' doesn't have %d-th argument\n", 7490 tname, arg + 1); 7491 return false; 7492 } 7493 7494 if (arg == nr_args) { 7495 switch (prog->expected_attach_type) { 7496 case BPF_LSM_MAC: 7497 /* mark we are accessing the return value */ 7498 info->is_retval = true; 7499 fallthrough; 7500 case BPF_LSM_CGROUP: 7501 case BPF_TRACE_FEXIT: 7502 case BPF_TRACE_FSESSION: 7503 /* When LSM programs are attached to void LSM hooks 7504 * they use FEXIT trampolines and when attached to 7505 * int LSM hooks, they use MODIFY_RETURN trampolines. 7506 * 7507 * While the LSM programs are BPF_MODIFY_RETURN-like 7508 * the check: 7509 * 7510 * if (ret_type != 'int') 7511 * return -EINVAL; 7512 * 7513 * is _not_ done here. This is still safe as LSM hooks 7514 * have only void and int return types. 7515 */ 7516 if (!t) 7517 return true; 7518 t = btf_type_by_id(btf, t->type); 7519 break; 7520 case BPF_MODIFY_RETURN: 7521 /* For now the BPF_MODIFY_RETURN can only be attached to 7522 * functions that return an int. 7523 */ 7524 if (!t) 7525 return false; 7526 7527 t = btf_type_skip_modifiers(btf, t->type, NULL); 7528 if (!btf_type_is_small_int(t)) { 7529 bpf_log(log, 7530 "ret type %s not allowed for fmod_ret\n", 7531 btf_type_str(t)); 7532 return false; 7533 } 7534 break; 7535 default: 7536 bpf_log(log, "func '%s' doesn't have %d-th argument\n", 7537 tname, arg + 1); 7538 return false; 7539 } 7540 } else { 7541 if (!t) 7542 /* Default prog with MAX_BPF_FUNC_REG_ARGS args */ 7543 return true; 7544 t = btf_type_by_id(btf, args[arg].type); 7545 } 7546 7547 /* skip modifiers */ 7548 while (btf_type_is_modifier(t)) 7549 t = btf_type_by_id(btf, t->type); 7550 if (btf_type_is_small_int(t) || btf_is_any_enum(t) || btf_type_is_struct(t)) 7551 /* accessing a scalar */ 7552 return true; 7553 if (!btf_type_is_ptr(t)) { 7554 bpf_log(log, 7555 "func '%s' arg%d '%s' has type %s. Only pointer access is allowed\n", 7556 tname, arg, 7557 __btf_name_by_offset(btf, t->name_off), 7558 btf_type_str(t)); 7559 return false; 7560 } 7561 7562 if (size != sizeof(u64)) { 7563 bpf_log(log, "func '%s' size %d must be 8\n", 7564 tname, size); 7565 return false; 7566 } 7567 7568 /* 7569 * Check for PTR_TO_RDONLY_BUF_OR_NULL, PTR_TO_RDWR_BUF_OR_NULL, 7570 * PTR_TO_ARENA (both nullable and non-nullable cases) or fixed-size 7571 * PTR_TO_MEM. 7572 */ 7573 for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { 7574 const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; 7575 u32 type, flag; 7576 7577 type = base_type(ctx_arg_info->reg_type); 7578 flag = type_flag(ctx_arg_info->reg_type); 7579 if (ctx_arg_info->offset == off && 7580 (type == PTR_TO_ARENA || 7581 type == PTR_TO_MEM || 7582 (type == PTR_TO_BUF && (flag & PTR_MAYBE_NULL)))) { 7583 info->reg_type = ctx_arg_info->reg_type; 7584 info->mem_size = ctx_arg_info->mem_size; 7585 return true; 7586 } 7587 } 7588 7589 /* 7590 * If it's a single or multilevel pointer, except a pointer 7591 * to a structure, it's the same as scalar from the verifier 7592 * safety POV. Multilevel pointers to structures are treated as 7593 * scalars. The verifier lacks the context to infer the size of 7594 * their target memory regions. Either way, no further pointer 7595 * walking is allowed. 7596 */ 7597 if (!btf_type_is_struct_ptr(btf, t)) 7598 return true; 7599 7600 /* this is a pointer to another type */ 7601 for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { 7602 const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; 7603 7604 if (ctx_arg_info->offset == off) { 7605 if (!ctx_arg_info->btf_id) { 7606 bpf_log(log,"invalid btf_id for context argument offset %u\n", off); 7607 return false; 7608 } 7609 7610 info->reg_type = ctx_arg_info->reg_type; 7611 info->btf = ctx_arg_info->btf ? : btf_vmlinux; 7612 info->btf_id = ctx_arg_info->btf_id; 7613 info->ref_id = ctx_arg_info->ref_id; 7614 return true; 7615 } 7616 } 7617 7618 info->reg_type = PTR_TO_BTF_ID; 7619 if (prog_args_trusted(prog)) 7620 info->reg_type |= PTR_TRUSTED; 7621 7622 if (btf_param_match_suffix(btf, &args[arg], "__nullable")) 7623 info->reg_type |= PTR_MAYBE_NULL; 7624 7625 if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { 7626 struct btf *btf = prog->aux->attach_btf; 7627 const struct btf_type *t; 7628 const char *tname; 7629 7630 /* BTF lookups cannot fail, return false on error */ 7631 t = btf_type_by_id(btf, prog->aux->attach_btf_id); 7632 if (!t) 7633 return false; 7634 tname = btf_name_by_offset(btf, t->name_off); 7635 if (!tname) 7636 return false; 7637 /* Checked by bpf_check_attach_target */ 7638 tname += sizeof("btf_trace_") - 1; 7639 for (i = 0; i < ARRAY_SIZE(raw_tp_null_args); i++) { 7640 /* Is this a func with potential NULL args? */ 7641 if (strcmp(tname, raw_tp_null_args[i].func)) 7642 continue; 7643 if (raw_tp_null_args[i].mask & (0x1ULL << (arg * 4))) 7644 info->reg_type |= PTR_MAYBE_NULL; 7645 /* Is the current arg IS_ERR? */ 7646 if (raw_tp_null_args[i].mask & (0x2ULL << (arg * 4))) 7647 ptr_err_raw_tp = true; 7648 break; 7649 } 7650 /* If we don't know NULL-ness specification and the tracepoint 7651 * is coming from a loadable module, be conservative and mark 7652 * argument as PTR_MAYBE_NULL. 7653 */ 7654 if (i == ARRAY_SIZE(raw_tp_null_args) && btf_is_module(btf)) 7655 info->reg_type |= PTR_MAYBE_NULL; 7656 } 7657 7658 if (tgt_prog) { 7659 enum bpf_prog_type tgt_type; 7660 7661 if (tgt_prog->type == BPF_PROG_TYPE_EXT) 7662 tgt_type = tgt_prog->aux->saved_dst_prog_type; 7663 else 7664 tgt_type = tgt_prog->type; 7665 7666 ret = btf_translate_to_vmlinux(log, btf, t, tgt_type, arg); 7667 if (ret > 0) { 7668 info->btf = btf_vmlinux; 7669 info->btf_id = ret; 7670 return true; 7671 } else { 7672 return false; 7673 } 7674 } 7675 7676 info->btf = btf; 7677 ctx.t = t; 7678 ret = btf_type_tag_walk(btf, &ctx, ctx_type_tags, 7679 ARRAY_SIZE(ctx_type_tags)); 7680 if (ret) { 7681 bpf_log(log, "func '%s' arg%d type %s has multiple type tags\n", 7682 tname, arg, btf_type_str(t)); 7683 return false; 7684 } 7685 info->reg_type |= ctx.res; 7686 info->btf_id = ctx.id; 7687 t = ctx.t; 7688 7689 if (!btf_type_is_struct(t)) { 7690 bpf_log(log, 7691 "func '%s' arg%d type %s is not a struct\n", 7692 tname, arg, btf_type_str(t)); 7693 return false; 7694 } 7695 bpf_log(log, "func '%s' arg%d has btf_id %d type %s '%s'\n", 7696 tname, arg, info->btf_id, btf_type_str(t), 7697 __btf_name_by_offset(btf, t->name_off)); 7698 7699 /* Perform all checks on the validity of type for this argument, but if 7700 * we know it can be IS_ERR at runtime, scrub pointer type and mark as 7701 * scalar. 7702 */ 7703 if (ptr_err_raw_tp) { 7704 bpf_log(log, "marking pointer arg%d as scalar as it may encode error", arg); 7705 info->reg_type = SCALAR_VALUE; 7706 } 7707 return true; 7708 } 7709 EXPORT_SYMBOL_GPL(btf_ctx_access); 7710 7711 #define MAX_ARRAYS_WALK 16 7712 7713 /* Description of an array crossed while walking a BTF access chain */ 7714 struct array_access { 7715 u32 id; /* BTF_KIND_ARRAY type id */ 7716 u32 off; /* offset of the access within the array */ 7717 }; 7718 7719 enum bpf_struct_walk_result { 7720 /* < 0 error */ 7721 WALK_SCALAR = 0, 7722 WALK_PTR, 7723 WALK_PTR_UNTRUSTED, 7724 WALK_PTR_ARENA, 7725 WALK_STRUCT, 7726 }; 7727 7728 static const struct btf_member *find_flex_member(const struct btf *btf, const struct btf_type *t) 7729 { 7730 const struct btf_array *array; 7731 const struct btf_type *mtype; 7732 const struct btf_member *member; 7733 u32 vlen = btf_type_vlen(t); 7734 7735 if (vlen == 0) 7736 return NULL; 7737 7738 member = btf_type_member(t) + vlen - 1; 7739 mtype = btf_type_skip_modifiers(btf, member->type, NULL); 7740 if (!btf_type_is_array(mtype)) 7741 return NULL; 7742 7743 array = (const struct btf_array *)(mtype + 1); 7744 if (array->nelems != 0) 7745 return NULL; 7746 7747 return member; 7748 } 7749 7750 static int push_array(struct array_access *arrays, u32 *arrays_cnt, struct array_access item) 7751 { 7752 if (arrays == NULL) 7753 return 0; 7754 if (*arrays_cnt == MAX_ARRAYS_WALK) 7755 return -1; 7756 arrays[(*arrays_cnt)++] = item; 7757 return 0; 7758 } 7759 7760 static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, 7761 const struct btf_type *t, int off, int size, 7762 u32 *next_btf_id, enum bpf_type_flag *flag, 7763 const char **field_name, bool walk_flex_arrays, 7764 struct array_access *arrays, u32 *arrays_cnt) 7765 { 7766 u32 i, moff, mtrue_end, msize = 0, total_nelems = 0; 7767 const struct btf_type *mtype, *elem_type = NULL; 7768 const struct btf_member *member; 7769 const char *tname, *mname; 7770 u32 vlen, elem_id, mid; 7771 7772 again: 7773 if (btf_type_is_modifier(t)) 7774 t = btf_type_skip_modifiers(btf, t->type, NULL); 7775 tname = __btf_name_by_offset(btf, t->name_off); 7776 if (!btf_type_is_struct(t)) { 7777 bpf_log(log, "Type '%s' is not a struct\n", tname); 7778 return -EINVAL; 7779 } 7780 7781 vlen = btf_type_vlen(t); 7782 if (BTF_INFO_KIND(t->info) == BTF_KIND_UNION && vlen != 1 && !(*flag & PTR_UNTRUSTED)) 7783 /* 7784 * walking unions yields untrusted pointers 7785 * with exception of __bpf_md_ptr and other 7786 * unions with a single member 7787 */ 7788 *flag |= PTR_UNTRUSTED; 7789 7790 if (off + size > t->size) { 7791 struct btf_array *array_elem; 7792 7793 if (!walk_flex_arrays) 7794 goto error; 7795 7796 /* If the last element is a variable size array, we may 7797 * need to relax the rule. 7798 */ 7799 member = find_flex_member(btf, t); 7800 if (!member) 7801 goto error; 7802 7803 moff = __btf_member_bit_offset(t, member) / 8; 7804 if (off < moff) 7805 goto error; 7806 7807 mtype = btf_type_skip_modifiers(btf, member->type, &mid); 7808 array_elem = (struct btf_array *)(mtype + 1); 7809 7810 if (push_array(arrays, arrays_cnt, (struct array_access){ mid, off - moff })) 7811 return -E2BIG; 7812 7813 /* allow structure and integer */ 7814 t = btf_type_skip_modifiers(btf, array_elem->type, 7815 NULL); 7816 7817 if (btf_type_is_int(t)) 7818 return WALK_SCALAR; 7819 7820 if (!btf_type_is_struct(t) || !t->size) 7821 goto error; 7822 7823 off = (off - moff) % t->size; 7824 goto again; 7825 7826 error: 7827 bpf_log(log, "access beyond struct %s at off %u size %u\n", 7828 tname, off, size); 7829 return -EACCES; 7830 } 7831 7832 for_each_member(i, t, member) { 7833 /* offset of the field in bytes */ 7834 moff = __btf_member_bit_offset(t, member) / 8; 7835 if (off + size <= moff) 7836 /* won't find anything, field is already too far */ 7837 break; 7838 7839 if (__btf_member_bitfield_size(t, member)) { 7840 u32 end_bit = __btf_member_bit_offset(t, member) + 7841 __btf_member_bitfield_size(t, member); 7842 7843 /* off <= moff instead of off == moff because clang 7844 * does not generate a BTF member for anonymous 7845 * bitfield like the ":16" here: 7846 * struct { 7847 * int :16; 7848 * int x:8; 7849 * }; 7850 */ 7851 if (off <= moff && 7852 BITS_ROUNDUP_BYTES(end_bit) <= off + size) 7853 return WALK_SCALAR; 7854 7855 /* off may be accessing a following member 7856 * 7857 * or 7858 * 7859 * Doing partial access at either end of this 7860 * bitfield. Continue on this case also to 7861 * treat it as not accessing this bitfield 7862 * and eventually error out as field not 7863 * found to keep it simple. 7864 * It could be relaxed if there was a legit 7865 * partial access case later. 7866 */ 7867 continue; 7868 } 7869 7870 /* In case of "off" is pointing to holes of a struct */ 7871 if (off < moff) 7872 break; 7873 7874 /* type of the field */ 7875 mid = member->type; 7876 mtype = btf_type_by_id(btf, member->type); 7877 mname = __btf_name_by_offset(btf, member->name_off); 7878 7879 mtype = __btf_resolve_size(btf, mtype, &msize, 7880 &elem_type, &elem_id, &total_nelems, 7881 &mid); 7882 if (IS_ERR(mtype)) { 7883 bpf_log(log, "field %s doesn't have size\n", mname); 7884 return -EFAULT; 7885 } 7886 7887 mtrue_end = moff + msize; 7888 if (off >= mtrue_end) 7889 /* no overlap with member, keep iterating */ 7890 continue; 7891 7892 if (btf_type_is_array(mtype)) { 7893 u32 elem_idx; 7894 7895 /* __btf_resolve_size() above helps to 7896 * linearize a multi-dimensional array. 7897 * 7898 * The logic here is treating an array 7899 * in a struct as the following way: 7900 * 7901 * struct outer { 7902 * struct inner array[2][2]; 7903 * }; 7904 * 7905 * looks like: 7906 * 7907 * struct outer { 7908 * struct inner array_elem0; 7909 * struct inner array_elem1; 7910 * struct inner array_elem2; 7911 * struct inner array_elem3; 7912 * }; 7913 * 7914 * When accessing outer->array[1][0], it moves 7915 * moff to "array_elem2", set mtype to 7916 * "struct inner", and msize also becomes 7917 * sizeof(struct inner). Then most of the 7918 * remaining logic will fall through without 7919 * caring the current member is an array or 7920 * not. 7921 * 7922 * Unlike mtype/msize/moff, mtrue_end does not 7923 * change. The naming difference ("_true") tells 7924 * that it is not always corresponding to 7925 * the current mtype/msize/moff. 7926 * It is the true end of the current 7927 * member (i.e. array in this case). That 7928 * will allow an int array to be accessed like 7929 * a scratch space, 7930 * i.e. allow access beyond the size of 7931 * the array's element as long as it is 7932 * within the mtrue_end boundary. 7933 */ 7934 if (push_array(arrays, arrays_cnt, 7935 (struct array_access){ mid, off - moff })) 7936 return -E2BIG; 7937 7938 /* skip empty array */ 7939 if (moff == mtrue_end) 7940 continue; 7941 7942 msize /= total_nelems; 7943 elem_idx = (off - moff) / msize; 7944 moff += elem_idx * msize; 7945 mtype = elem_type; 7946 mid = elem_id; 7947 } 7948 7949 /* the 'off' we're looking for is either equal to start 7950 * of this field or inside of this struct 7951 */ 7952 if (btf_type_is_struct(mtype)) { 7953 /* our field must be inside that union or struct */ 7954 t = mtype; 7955 7956 /* return if the offset matches the member offset */ 7957 if (off == moff) { 7958 *next_btf_id = mid; 7959 return WALK_STRUCT; 7960 } 7961 7962 /* adjust offset we're looking for */ 7963 off -= moff; 7964 goto again; 7965 } 7966 7967 if (btf_type_is_ptr(mtype)) { 7968 static const struct btf_type_tag_match walk_type_tags[] = { 7969 { "user", MEM_USER }, 7970 { "percpu", MEM_PERCPU }, 7971 { "rcu", MEM_RCU }, 7972 }; 7973 enum bpf_type_flag tmp_flag = 0; 7974 struct btf_type_tag_walk_ctx ctx = { .t = mtype }; 7975 const struct btf_type *stype; 7976 int err; 7977 u32 id; 7978 7979 if (msize != size || off != moff) { 7980 bpf_log(log, 7981 "cannot access ptr member %s with moff %u in struct %s with off %u size %u\n", 7982 mname, moff, tname, off, size); 7983 return -EACCES; 7984 } 7985 7986 err = btf_type_tag_walk(btf, &ctx, walk_type_tags, 7987 ARRAY_SIZE(walk_type_tags)); 7988 if (err) { 7989 bpf_log(log, "type '%s' has multiple type tags\n", 7990 btf_type_str(mtype)); 7991 return err; 7992 } 7993 tmp_flag = ctx.res; 7994 id = ctx.id; 7995 stype = ctx.t; 7996 7997 /* 7998 * __nullable combines with any of the above: it only says 7999 * that the pointer may be NULL. The caller decides whether 8000 * that matters for the register type it derives. 8001 */ 8002 if (btf_ptr_has_type_tag(btf, mtype, "nullable")) 8003 tmp_flag |= PTR_MAYBE_NULL; 8004 8005 /* 8006 * An __arena member of a kernel struct holds the kernel 8007 * address of arena memory, which the verifier turns into 8008 * an arena pointer on load. Program BTF is not affected: 8009 * there the tag is what the compiler emits for the arena 8010 * address space. An offset into a union resolves to its 8011 * first member, so a program reading another member there 8012 * would get an arena pointer it did not ask for; the walk 8013 * marked such a union untrusted. 8014 */ 8015 if (btf_is_kernel(btf) && btf_type_is_arena_ptr(btf, mtype)) { 8016 if (*flag & PTR_UNTRUSTED) { 8017 bpf_log(log, 8018 "arena pointer member %s of %s is inside a union\n", 8019 mname, tname); 8020 return -EACCES; 8021 } 8022 *next_btf_id = id; 8023 *flag |= tmp_flag & PTR_MAYBE_NULL; 8024 if (field_name) 8025 *field_name = mname; 8026 return WALK_PTR_ARENA; 8027 } 8028 8029 if (btf_type_is_struct(stype)) { 8030 *next_btf_id = id; 8031 *flag |= tmp_flag; 8032 if (field_name) 8033 *field_name = mname; 8034 return WALK_PTR; 8035 } 8036 8037 return WALK_PTR_UNTRUSTED; 8038 } 8039 8040 /* Allow more flexible access within an int as long as 8041 * it is within mtrue_end. 8042 * Since mtrue_end could be the end of an array, 8043 * that also allows using an array of int as a scratch 8044 * space. e.g. skb->cb[]. 8045 */ 8046 if (off + size > mtrue_end && !(*flag & PTR_UNTRUSTED)) { 8047 bpf_log(log, 8048 "access beyond the end of member %s (mend:%u) in struct %s with off %u size %u\n", 8049 mname, mtrue_end, tname, off, size); 8050 return -EACCES; 8051 } 8052 8053 return WALK_SCALAR; 8054 } 8055 bpf_log(log, "struct %s doesn't have field at offset %d\n", tname, off); 8056 return -EINVAL; 8057 } 8058 8059 int btf_struct_access(struct bpf_verifier_log *log, 8060 const struct bpf_reg_state *reg, 8061 int _off, int size, enum bpf_access_type atype __maybe_unused, 8062 u32 *next_btf_id, enum bpf_type_flag *flag, 8063 const char **field_name) 8064 { 8065 const struct btf *btf = reg->btf; 8066 enum bpf_type_flag tmp_flag = 0; 8067 struct array_access arrays[MAX_ARRAYS_WALK]; 8068 const struct btf_member *member; 8069 const struct btf_type *t; 8070 u32 i, arrays_cnt = 0; 8071 u32 id = reg->btf_id; 8072 s64 min_off, max_off, flex_off; 8073 int err, off, ret; 8074 8075 if (check_add_overflow(reg_smin(reg), (s64)_off, &min_off) || 8076 check_add_overflow(reg_smax(reg), (s64)_off, &max_off) || 8077 min_off != (int)min_off) { 8078 bpf_log(log, 8079 "offset computation overflows: register offset range is [%lld, %lld], instruction offset is %d\n", 8080 reg_smin(reg), reg_smax(reg), _off); 8081 return -EINVAL; 8082 } 8083 off = min_off; 8084 8085 while (type_is_alloc(reg->type)) { 8086 struct btf_struct_meta *meta; 8087 struct btf_record *rec; 8088 int i; 8089 8090 meta = btf_find_struct_meta(btf, id); 8091 if (!meta) 8092 break; 8093 rec = meta->record; 8094 for (i = 0; i < rec->cnt; i++) { 8095 struct btf_field *field = &rec->fields[i]; 8096 u32 offset = field->offset; 8097 if (off < offset + field->size && offset < off + size) { 8098 bpf_log(log, 8099 "direct access to %s is disallowed\n", 8100 btf_field_type_name(field->type)); 8101 return -EACCES; 8102 } 8103 } 8104 break; 8105 } 8106 8107 t = btf_type_by_id(btf, id); 8108 do { 8109 err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, 8110 field_name, !type_is_alloc(reg->type), arrays, &arrays_cnt); 8111 switch (err) { 8112 case WALK_PTR: 8113 /* For local types, the destination register cannot 8114 * become a pointer again. 8115 */ 8116 if (type_is_alloc(reg->type)) { 8117 ret = SCALAR_VALUE; 8118 goto check_variable_offset; 8119 } 8120 /* If we found the pointer or scalar on t+off, 8121 * we're done. 8122 */ 8123 *next_btf_id = id; 8124 *flag = tmp_flag; 8125 ret = PTR_TO_BTF_ID; 8126 goto check_variable_offset; 8127 case WALK_PTR_UNTRUSTED: 8128 *flag = MEM_RDONLY | PTR_UNTRUSTED; 8129 ret = PTR_TO_MEM; 8130 goto check_variable_offset; 8131 case WALK_PTR_ARENA: 8132 *next_btf_id = id; 8133 *flag = tmp_flag; 8134 ret = PTR_TO_ARENA; 8135 goto check_variable_offset; 8136 case WALK_SCALAR: 8137 ret = SCALAR_VALUE; 8138 goto check_variable_offset; 8139 case WALK_STRUCT: 8140 /* We found nested struct, so continue the search 8141 * by diving in it. At this point the offset is 8142 * aligned with the new type, so set it to 0. 8143 */ 8144 t = btf_type_by_id(btf, id); 8145 off = 0; 8146 break; 8147 default: 8148 /* It's either error or unknown return value.. 8149 * scream and leave. 8150 */ 8151 if (WARN_ONCE(err > 0, "unknown btf_struct_walk return value")) 8152 return -EINVAL; 8153 return err; 8154 } 8155 } while (t); 8156 8157 return -EINVAL; 8158 8159 check_variable_offset: 8160 if (min_off == max_off) 8161 return ret; 8162 8163 /* Find an offset at which access would go to a flexible array tail (if any). */ 8164 t = btf_type_skip_modifiers(btf, reg->btf_id, NULL); 8165 member = find_flex_member(btf, t); 8166 flex_off = member ? __btf_member_bit_offset(t, member) / 8 : S64_MAX; 8167 8168 /* 8169 * If this is a varying offset access, the step recorded within a register 8170 * should correspond to one of the arrays visited while walking. 8171 */ 8172 for (i = 0; i < arrays_cnt; i++) { 8173 s64 array_start, array_end, access_end; 8174 u32 asize, esize, elem_id; 8175 8176 /* 8177 * 'arrays' records top-level arrays only, use __btf_resolve_size() 8178 * to get flattened representation. 8179 */ 8180 t = btf_type_by_id(btf, arrays[i].id); 8181 if (IS_ERR(__btf_resolve_size(btf, t, &asize, NULL, &elem_id, NULL, NULL)) || 8182 IS_ERR(btf_resolve_size(btf, btf_type_by_id(btf, elem_id), &esize))) 8183 continue; 8184 /* Make sure every accessed offset lands on the same field of some element. */ 8185 if (esize == 0 || reg->step % esize != 0) 8186 continue; 8187 8188 array_start = min_off - arrays[i].off; 8189 /* An array within the flexible tail has no upper bound to exceed */ 8190 if (array_start >= flex_off && btf_type_is_array(t) && 8191 btf_type_array(t)->nelems == 0) 8192 return ret; 8193 8194 /* the 'size' bytes read at max_off must stay within the array */ 8195 if (check_add_overflow(array_start, (s64)asize, &array_end) || 8196 check_add_overflow(max_off, (s64)size, &access_end) || 8197 access_end > array_end) 8198 continue; 8199 return ret; 8200 } 8201 8202 bpf_log(log, 8203 "invalid variable offset access into struct %s: offsets [%lld, %lld] size %d step %d do not match any array\n", 8204 btf_name_by_offset(btf, btf_type_by_id(btf, reg->btf_id)->name_off), 8205 min_off, max_off, size, reg->step); 8206 return -EINVAL; 8207 } 8208 8209 /* Check that two BTF types, each specified as an BTF object + id, are exactly 8210 * the same. Trivial ID check is not enough due to module BTFs, because we can 8211 * end up with two different module BTFs, but IDs point to the common type in 8212 * vmlinux BTF. 8213 */ 8214 bool btf_types_are_same(const struct btf *btf1, u32 id1, 8215 const struct btf *btf2, u32 id2) 8216 { 8217 if (id1 != id2) 8218 return false; 8219 if (btf1 == btf2) 8220 return true; 8221 return btf_type_by_id(btf1, id1) == btf_type_by_id(btf2, id2); 8222 } 8223 8224 bool btf_struct_ids_match(struct bpf_verifier_log *log, 8225 const struct btf *btf, u32 id, int off, 8226 const struct btf *need_btf, u32 need_type_id, 8227 bool strict, bool walk_flex_arrays) 8228 { 8229 const struct btf_type *type; 8230 enum bpf_type_flag flag = 0; 8231 int err; 8232 8233 /* Are we already done? */ 8234 if (off == 0 && btf_types_are_same(btf, id, need_btf, need_type_id)) 8235 return true; 8236 /* In case of strict type match, we do not walk struct, the top level 8237 * type match must succeed. When strict is true, off should have already 8238 * been 0. 8239 */ 8240 if (strict) 8241 return false; 8242 again: 8243 type = btf_type_by_id(btf, id); 8244 if (!type) 8245 return false; 8246 err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL, 8247 walk_flex_arrays, NULL, NULL); 8248 if (err != WALK_STRUCT) 8249 return false; 8250 8251 /* We found nested struct object. If it matches 8252 * the requested ID, we're done. Otherwise let's 8253 * continue the search with offset 0 in the new 8254 * type. 8255 */ 8256 if (!btf_types_are_same(btf, id, need_btf, need_type_id)) { 8257 off = 0; 8258 goto again; 8259 } 8260 8261 return true; 8262 } 8263 8264 static int __get_type_size(struct btf *btf, u32 btf_id, 8265 const struct btf_type **ret_type) 8266 { 8267 const struct btf_type *t; 8268 8269 *ret_type = btf_type_by_id(btf, 0); 8270 if (!btf_id) 8271 /* void */ 8272 return 0; 8273 t = btf_type_by_id(btf, btf_id); 8274 while (t && btf_type_is_modifier(t)) 8275 t = btf_type_by_id(btf, t->type); 8276 if (!t) 8277 return -EINVAL; 8278 *ret_type = t; 8279 if (btf_type_is_ptr(t)) 8280 /* kernel size of pointer. Not BPF's size of pointer*/ 8281 return sizeof(void *); 8282 if (btf_type_is_int(t) || btf_is_any_enum(t) || btf_type_is_struct(t)) 8283 return t->size; 8284 return -EINVAL; 8285 } 8286 8287 static u8 __get_type_fmodel_flags(const struct btf_type *t) 8288 { 8289 u8 flags = 0; 8290 8291 if (btf_type_is_signed_int(t)) 8292 flags |= BTF_FMODEL_SIGNED_ARG; 8293 8294 return flags; 8295 } 8296 8297 static u8 __get_arg_fmodel_flags(const struct btf *btf, 8298 const struct btf_param *arg, 8299 const struct btf_type *t) 8300 { 8301 u8 flags = __get_type_fmodel_flags(t); 8302 8303 if (btf_func_arg_align(btf, t) > sizeof(u64)) 8304 flags |= BTF_FMODEL_ALIGN16_ARG; 8305 8306 if (btf_param_match_suffix(btf, arg, "__arena__nullable")) 8307 flags |= BTF_FMODEL_ARENA_ARG | BTF_FMODEL_NULLABLE_ARG; 8308 else if (btf_param_match_suffix(btf, arg, "__arena")) 8309 flags |= BTF_FMODEL_ARENA_ARG; 8310 else if (btf_param_match_suffix(btf, arg, "__nullable")) 8311 flags |= BTF_FMODEL_NULLABLE_ARG; 8312 8313 return flags; 8314 } 8315 8316 int btf_distill_func_proto(struct bpf_verifier_log *log, 8317 struct btf *btf, 8318 const struct btf_type *func, 8319 const char *tname, 8320 struct btf_func_model *m) 8321 { 8322 const struct btf_param *args; 8323 const struct btf_type *t; 8324 u32 i, nargs; 8325 int ret; 8326 8327 if (!func) { 8328 /* BTF function prototype doesn't match the verifier types. 8329 * Fall back to MAX_BPF_FUNC_REG_ARGS u64 args. 8330 */ 8331 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { 8332 m->arg_size[i] = 8; 8333 m->arg_flags[i] = 0; 8334 } 8335 m->ret_size = 8; 8336 m->ret_flags = 0; 8337 m->nr_args = MAX_BPF_FUNC_REG_ARGS; 8338 return 0; 8339 } 8340 args = (const struct btf_param *)(func + 1); 8341 nargs = btf_type_vlen(func); 8342 if (nargs > MAX_BPF_FUNC_ARGS) { 8343 bpf_log(log, 8344 "The function %s has %d arguments. Too many.\n", 8345 tname, nargs); 8346 return -EINVAL; 8347 } 8348 ret = __get_type_size(btf, func->type, &t); 8349 if (ret < 0 || ret > 16) { 8350 bpf_log(log, 8351 "The function %s return type %s is unsupported.\n", 8352 tname, btf_type_str(t)); 8353 return -EINVAL; 8354 } 8355 m->ret_size = ret; 8356 m->ret_flags = __get_type_fmodel_flags(t); 8357 8358 for (i = 0; i < nargs; i++) { 8359 if (i == nargs - 1 && args[i].type == 0) { 8360 bpf_log(log, 8361 "The function %s with variable args is unsupported.\n", 8362 tname); 8363 return -EINVAL; 8364 } 8365 ret = __get_type_size(btf, args[i].type, &t); 8366 8367 /* No support of struct argument size greater than 16 bytes */ 8368 if (ret < 0 || ret > 16) { 8369 bpf_log(log, 8370 "The function %s arg%d type %s is unsupported.\n", 8371 tname, i, btf_type_str(t)); 8372 return -EINVAL; 8373 } 8374 if (ret == 0) { 8375 bpf_log(log, 8376 "The function %s has malformed void argument.\n", 8377 tname); 8378 return -EINVAL; 8379 } 8380 m->arg_size[i] = ret; 8381 m->arg_flags[i] = __get_arg_fmodel_flags(btf, &args[i], t); 8382 } 8383 m->nr_args = nargs; 8384 return 0; 8385 } 8386 8387 /* Compare BTFs of two functions assuming only scalars and pointers to context. 8388 * t1 points to BTF_KIND_FUNC in btf1 8389 * t2 points to BTF_KIND_FUNC in btf2 8390 * Returns: 8391 * EINVAL - function prototype mismatch 8392 * EFAULT - verifier bug 8393 * 0 - 99% match. The last 1% is validated by the verifier. 8394 */ 8395 static int btf_check_func_type_match(struct bpf_verifier_log *log, 8396 struct btf *btf1, const struct btf_type *t1, 8397 struct btf *btf2, const struct btf_type *t2) 8398 { 8399 const struct btf_param *args1, *args2; 8400 const char *fn1, *fn2, *s1, *s2; 8401 u32 nargs1, nargs2, i; 8402 8403 fn1 = btf_name_by_offset(btf1, t1->name_off); 8404 fn2 = btf_name_by_offset(btf2, t2->name_off); 8405 8406 if (btf_func_linkage(t1) != BTF_FUNC_GLOBAL) { 8407 bpf_log(log, "%s() is not a global function\n", fn1); 8408 return -EINVAL; 8409 } 8410 if (btf_func_linkage(t2) != BTF_FUNC_GLOBAL) { 8411 bpf_log(log, "%s() is not a global function\n", fn2); 8412 return -EINVAL; 8413 } 8414 8415 t1 = btf_type_by_id(btf1, t1->type); 8416 if (!t1 || !btf_type_is_func_proto(t1)) 8417 return -EFAULT; 8418 t2 = btf_type_by_id(btf2, t2->type); 8419 if (!t2 || !btf_type_is_func_proto(t2)) 8420 return -EFAULT; 8421 8422 args1 = (const struct btf_param *)(t1 + 1); 8423 nargs1 = btf_type_vlen(t1); 8424 args2 = (const struct btf_param *)(t2 + 1); 8425 nargs2 = btf_type_vlen(t2); 8426 8427 if (nargs1 != nargs2) { 8428 bpf_log(log, "%s() has %d args while %s() has %d args\n", 8429 fn1, nargs1, fn2, nargs2); 8430 return -EINVAL; 8431 } 8432 8433 t1 = btf_type_skip_modifiers(btf1, t1->type, NULL); 8434 t2 = btf_type_skip_modifiers(btf2, t2->type, NULL); 8435 if (t1->info != t2->info) { 8436 bpf_log(log, 8437 "Return type %s of %s() doesn't match type %s of %s()\n", 8438 btf_type_str(t1), fn1, 8439 btf_type_str(t2), fn2); 8440 return -EINVAL; 8441 } 8442 if (btf_type_has_size(t1) && (t1->size > 8 || t2->size > 8)) { 8443 bpf_log(log, 8444 "Return type of %s() has size %u and of %s() size %u, and a size above 8 bytes cannot be replaced\n", 8445 fn1, t1->size, fn2, t2->size); 8446 return -EINVAL; 8447 } 8448 8449 for (i = 0; i < nargs1; i++) { 8450 t1 = btf_type_skip_modifiers(btf1, args1[i].type, NULL); 8451 t2 = btf_type_skip_modifiers(btf2, args2[i].type, NULL); 8452 8453 if (t1->info != t2->info) { 8454 bpf_log(log, "arg%d in %s() is %s while %s() has %s\n", 8455 i, fn1, btf_type_str(t1), 8456 fn2, btf_type_str(t2)); 8457 return -EINVAL; 8458 } 8459 if (btf_type_has_size(t1) && t1->size != t2->size) { 8460 bpf_log(log, 8461 "arg%d in %s() has size %d while %s() has %d\n", 8462 i, fn1, t1->size, 8463 fn2, t2->size); 8464 return -EINVAL; 8465 } 8466 8467 /* global functions are validated with scalars and pointers 8468 * to context only. And only global functions can be replaced. 8469 * Hence type check only those types. 8470 */ 8471 if (btf_type_is_int(t1) || btf_is_any_enum(t1)) 8472 continue; 8473 if (!btf_type_is_ptr(t1)) { 8474 bpf_log(log, 8475 "arg%d in %s() has unrecognized type\n", 8476 i, fn1); 8477 return -EINVAL; 8478 } 8479 t1 = btf_type_skip_modifiers(btf1, t1->type, NULL); 8480 t2 = btf_type_skip_modifiers(btf2, t2->type, NULL); 8481 if (!btf_type_is_struct(t1)) { 8482 bpf_log(log, 8483 "arg%d in %s() is not a pointer to context\n", 8484 i, fn1); 8485 return -EINVAL; 8486 } 8487 if (!btf_type_is_struct(t2)) { 8488 bpf_log(log, 8489 "arg%d in %s() is not a pointer to context\n", 8490 i, fn2); 8491 return -EINVAL; 8492 } 8493 /* This is an optional check to make program writing easier. 8494 * Compare names of structs and report an error to the user. 8495 * check_ext_args() compares what btf_prepare_func_args() 8496 * derives for the two arguments. 8497 */ 8498 s1 = btf_name_by_offset(btf1, t1->name_off); 8499 s2 = btf_name_by_offset(btf2, t2->name_off); 8500 if (strcmp(s1, s2)) { 8501 bpf_log(log, 8502 "arg%d %s(struct %s *) doesn't match %s(struct %s *)\n", 8503 i, fn1, s1, fn2, s2); 8504 return -EINVAL; 8505 } 8506 } 8507 return 0; 8508 } 8509 8510 /* Compare BTFs of given program with BTF of target program */ 8511 int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, 8512 struct btf *btf2, const struct btf_type *t2) 8513 { 8514 struct btf *btf1 = prog->aux->btf; 8515 const struct btf_type *t1; 8516 u32 btf_id = 0; 8517 8518 if (!prog->aux->func_info) { 8519 bpf_log(log, "Program extension requires BTF\n"); 8520 return -EINVAL; 8521 } 8522 8523 btf_id = prog->aux->func_info[0].type_id; 8524 if (!btf_id) 8525 return -EFAULT; 8526 8527 t1 = btf_type_by_id(btf1, btf_id); 8528 if (!t1 || !btf_type_is_func(t1)) 8529 return -EFAULT; 8530 8531 return btf_check_func_type_match(log, btf1, t1, btf2, t2); 8532 } 8533 8534 static bool btf_is_dynptr_ptr(const struct btf *btf, const struct btf_type *t) 8535 { 8536 const char *name; 8537 8538 t = btf_type_by_id(btf, t->type); /* skip PTR */ 8539 8540 while (btf_type_is_modifier(t)) 8541 t = btf_type_by_id(btf, t->type); 8542 8543 /* allow either struct or struct forward declaration */ 8544 if (btf_type_is_struct(t) || 8545 (btf_type_is_fwd(t) && btf_type_kflag(t) == 0)) { 8546 name = btf_str_by_offset(btf, t->name_off); 8547 return name && strcmp(name, "bpf_dynptr") == 0; 8548 } 8549 8550 return false; 8551 } 8552 8553 struct bpf_cand_cache { 8554 const char *name; 8555 u32 name_len; 8556 u16 kind; 8557 u16 cnt; 8558 struct { 8559 const struct btf *btf; 8560 u32 id; 8561 } cands[]; 8562 }; 8563 8564 static DEFINE_MUTEX(cand_cache_mutex); 8565 8566 static struct bpf_cand_cache * 8567 bpf_core_find_cands(struct bpf_core_ctx *ctx, u32 local_type_id); 8568 8569 static int btf_get_ptr_to_btf_id(struct bpf_verifier_log *log, int arg_idx, 8570 const struct btf *btf, const struct btf_type *t) 8571 { 8572 struct bpf_cand_cache *cc; 8573 struct bpf_core_ctx ctx = { 8574 .btf = btf, 8575 .log = log, 8576 }; 8577 u32 kern_type_id, type_id; 8578 int err = 0; 8579 8580 /* skip PTR and modifiers */ 8581 type_id = t->type; 8582 t = btf_type_by_id(btf, t->type); 8583 while (btf_type_is_modifier(t)) { 8584 type_id = t->type; 8585 t = btf_type_by_id(btf, t->type); 8586 } 8587 8588 mutex_lock(&cand_cache_mutex); 8589 cc = bpf_core_find_cands(&ctx, type_id); 8590 if (IS_ERR(cc)) { 8591 err = PTR_ERR(cc); 8592 bpf_log(log, "arg#%d reference type('%s %s') candidate matching error: %d\n", 8593 arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off), 8594 err); 8595 goto cand_cache_unlock; 8596 } 8597 if (cc->cnt != 1) { 8598 bpf_log(log, "arg#%d reference type('%s %s') %s\n", 8599 arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off), 8600 cc->cnt == 0 ? "has no matches" : "is ambiguous"); 8601 err = cc->cnt == 0 ? -ENOENT : -ESRCH; 8602 goto cand_cache_unlock; 8603 } 8604 if (btf_is_module(cc->cands[0].btf)) { 8605 bpf_log(log, "arg#%d reference type('%s %s') points to kernel module type (unsupported)\n", 8606 arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off)); 8607 err = -EOPNOTSUPP; 8608 goto cand_cache_unlock; 8609 } 8610 kern_type_id = cc->cands[0].id; 8611 8612 cand_cache_unlock: 8613 mutex_unlock(&cand_cache_mutex); 8614 if (err) 8615 return err; 8616 8617 return kern_type_id; 8618 } 8619 8620 enum btf_arg_tag { 8621 ARG_TAG_CTX = BIT_ULL(0), 8622 ARG_TAG_NONNULL = BIT_ULL(1), 8623 ARG_TAG_TRUSTED = BIT_ULL(2), 8624 ARG_TAG_UNTRUSTED = BIT_ULL(3), 8625 ARG_TAG_NULLABLE = BIT_ULL(4), 8626 ARG_TAG_ARENA = BIT_ULL(5), 8627 }; 8628 8629 static int btf_scan_decl_tags(struct bpf_verifier_env *env, 8630 const struct btf *btf, 8631 const struct btf_type *fn_t, 8632 u32 arg_idx, bool is_global, u32 *tags) 8633 { 8634 int id = btf_named_start_id(btf, false) - 1; 8635 const char tag_key[] = "arg:"; 8636 static const struct { 8637 const char *tag_value; 8638 enum btf_arg_tag arg_tag; 8639 } tag_values[] = { 8640 { "ctx", ARG_TAG_CTX }, 8641 { "trusted", ARG_TAG_TRUSTED }, 8642 { "untrusted", ARG_TAG_UNTRUSTED }, 8643 { "nonnull", ARG_TAG_NONNULL }, 8644 { "nullable", ARG_TAG_NULLABLE }, 8645 { "arena", ARG_TAG_ARENA }, 8646 }; 8647 8648 /* 8649 * The 'arg:<tag>' decl_tag takes precedence over the derivation 8650 * of the register type from the BTF type itself. 8651 */ 8652 while ((id = btf_find_next_decl_tag(btf, fn_t, arg_idx, tag_key, id)) > 0) { 8653 const struct btf_type *tag_t; 8654 const char *tag; 8655 int i; 8656 bool found; 8657 8658 /* disallow arg tags in static subprogs */ 8659 if (!is_global) { 8660 bpf_log(&env->log, 8661 "arg#%d type tag is not supported in static functions\n", 8662 arg_idx); 8663 return -EOPNOTSUPP; 8664 } 8665 8666 tag_t = btf_type_by_id(btf, id); 8667 tag = __btf_name_by_offset(btf, tag_t->name_off) + (sizeof(tag_key) - 1); 8668 8669 found = false; 8670 for (i = 0; i < ARRAY_SIZE(tag_values); ++i) { 8671 if (!strcmp(tag, tag_values[i].tag_value)) { 8672 *tags |= tag_values[i].arg_tag; 8673 found = true; 8674 break; 8675 } 8676 } 8677 8678 if (!found) { 8679 bpf_log(&env->log, "arg#%d has unsupported set of tags\n", arg_idx); 8680 return -EOPNOTSUPP; 8681 } 8682 } 8683 if (id != -ENOENT) { 8684 bpf_log(&env->log, "arg#%d type tag fetching failure: %d\n", arg_idx, id); 8685 return id; 8686 } 8687 8688 return 0; 8689 } 8690 8691 static void btf_scan_type_tags(const struct btf *btf, u32 type_id, u32 *tags) 8692 { 8693 /* Find the first pointer type in the chain. */ 8694 const struct btf_type *t = btf_type_skip_modifiers(btf, type_id, NULL); 8695 8696 if (btf_type_is_arena_ptr(btf, t)) 8697 *tags |= ARG_TAG_ARENA; 8698 } 8699 8700 /* Check whether the type is a valid return type. */ 8701 static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf, 8702 const struct btf_type *t, int subprog, bool is_global) 8703 { 8704 t = btf_type_skip_modifiers(btf, t->type, NULL); 8705 8706 /* 8707 * We allow all subprogs except for the main one to return any kind of arena pointer. 8708 * General arena variables are not allowed, since it makes no sense to return by value 8709 * a variable that's on the heap in the first place. 8710 */ 8711 if (subprog && btf_type_is_arena_ptr(btf, t)) 8712 return 0; 8713 8714 /* We always accept void or scalars. */ 8715 if (btf_type_is_void(t) || btf_type_is_int(t) || btf_is_any_enum(t)) 8716 return 0; 8717 8718 if (btf_type_is_struct(t) && t->size <= 16) { 8719 /* 8720 * A global function's caller models the return as an opaque 8721 * scalar pair, so a pointer member would be laundered into a 8722 * scalar and escape provenance and reference tracking. Only 8723 * scalars and arena pointers are allowed: an arena pointer has 8724 * no provenance to lose, since a program may already derive one 8725 * from any scalar with addr_space_cast(), which confines the 8726 * result to the arena. The main program is the exception: it 8727 * returns to the kernel, which has no arena to cast the address 8728 * back into. A local function is verified inline, so its caller 8729 * receives the real register state and any member is fine. 8730 */ 8731 bool local_func = subprog && !is_global; 8732 u32 member_kinds = BTF_MEMBER_SCALAR; 8733 8734 if (subprog) 8735 member_kinds |= BTF_MEMBER_ARENA_PTR; 8736 8737 if (local_func || btf_struct_is_composed_of(env, btf, t, member_kinds)) 8738 return 0; 8739 } 8740 8741 return -EOPNOTSUPP; 8742 } 8743 8744 static int btf_check_arg_slots(struct bpf_verifier_log *log, const char *tname, 8745 bool is_global, u32 slot_cnt, 8746 struct bpf_subprog_info *sub) 8747 { 8748 if (slot_cnt <= MAX_BPF_FUNC_REG_ARGS) 8749 return 0; 8750 8751 if (is_global) { 8752 bpf_log(log, 8753 "global function %s() needs %d > %d argument slots, " 8754 "stack args not supported\n", 8755 tname, slot_cnt, MAX_BPF_FUNC_REG_ARGS); 8756 return -EINVAL; 8757 } 8758 if (!bpf_jit_supports_stack_args()) { 8759 bpf_log(log, "JIT does not support function %s() with %d argument slots\n", 8760 tname, slot_cnt); 8761 return -EFAULT; 8762 } 8763 sub->stack_arg_cnt = slot_cnt - MAX_BPF_FUNC_REG_ARGS; 8764 return 0; 8765 } 8766 8767 /* Process BTF of a function to produce high-level expectation of function 8768 * arguments (like ARG_PTR_TO_CTX, or ARG_PTR_TO_MEM, etc). This information 8769 * is cached in subprog info for reuse. 8770 * Returns: 8771 * EFAULT - there is a verifier bug. Abort verification. 8772 * EINVAL - cannot convert BTF. 8773 * 0 - Successfully processed BTF and constructed argument expectations. 8774 */ 8775 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) 8776 { 8777 bool is_global = subprog_aux(env, subprog)->linkage == BTF_FUNC_GLOBAL; 8778 struct bpf_subprog_info *sub = subprog_info(env, subprog); 8779 struct bpf_verifier_log *log = &env->log; 8780 struct bpf_prog *prog = env->prog; 8781 enum bpf_prog_type prog_type = prog->type; 8782 struct btf *btf = prog->aux->btf; 8783 const struct btf_param *args; 8784 const struct btf_type *t, *ref_t, *fn_t; 8785 int err; 8786 u32 i, slots_used, nargs, btf_id; 8787 const char *tname; 8788 8789 if (sub->args_cached) 8790 return 0; 8791 8792 if (!prog->aux->func_info) { 8793 verifier_bug(env, "func_info undefined"); 8794 return -EFAULT; 8795 } 8796 8797 btf_id = prog->aux->func_info[subprog].type_id; 8798 if (!btf_id) { 8799 if (!is_global) /* not fatal for static funcs */ 8800 return -EINVAL; 8801 bpf_log(log, "Global functions need valid BTF\n"); 8802 return -EFAULT; 8803 } 8804 8805 fn_t = btf_type_by_id(btf, btf_id); 8806 if (!fn_t || !btf_type_is_func(fn_t)) { 8807 /* These checks were already done by the verifier while loading 8808 * struct bpf_func_info 8809 */ 8810 bpf_log(log, "BTF of func#%d doesn't point to KIND_FUNC\n", 8811 subprog); 8812 return -EFAULT; 8813 } 8814 tname = btf_name_by_offset(btf, fn_t->name_off); 8815 8816 if (prog->aux->func_info_aux[subprog].unreliable) { 8817 verifier_bug(env, "unreliable BTF for function %s()", tname); 8818 return -EFAULT; 8819 } 8820 if (prog_type == BPF_PROG_TYPE_EXT) 8821 prog_type = prog->aux->dst_prog->type; 8822 8823 t = btf_type_by_id(btf, fn_t->type); 8824 if (!t || !btf_type_is_func_proto(t)) { 8825 bpf_log(log, "Invalid type of function %s()\n", tname); 8826 return -EFAULT; 8827 } 8828 args = (const struct btf_param *)(t + 1); 8829 nargs = btf_type_vlen(t); 8830 sub->arg_slot_cnt = nargs; 8831 if (nargs > MAX_BPF_FUNC_ARGS) { 8832 bpf_log(log, "kernel supports at most %d parameters, function %s has %d\n", 8833 MAX_BPF_FUNC_ARGS, tname, nargs); 8834 return -EFAULT; 8835 } 8836 err = btf_check_arg_slots(log, tname, is_global, nargs, sub); 8837 if (err) 8838 return err; 8839 8840 err = btf_validate_return_type(env, btf, t, subprog, is_global); 8841 if (err) { 8842 if (is_global) { 8843 /* Only a subprogram may return arena pointers. */ 8844 const char *supported = subprog ? 8845 "void, scalar, arena pointer, or a struct/union of " 8846 "scalars and arena pointers" : 8847 "void, scalar, or a scalar-only struct/union"; 8848 8849 bpf_log(log, 8850 "Global function %s() has unsupported return type. " 8851 "Only %s up to 16 bytes is supported.\n", 8852 tname, supported); 8853 } 8854 return err; 8855 } 8856 8857 /* Convert BTF function arguments into verifier types. 8858 * Only PTR_TO_CTX and SCALAR are supported atm. 8859 */ 8860 for (i = 0, slots_used = 0; i < nargs; i++) { 8861 u32 tags = 0; 8862 8863 if (slots_used >= MAX_BPF_FUNC_ARGS) 8864 goto too_many_slots; 8865 8866 err = btf_scan_decl_tags(env, btf, fn_t, i, is_global, &tags); 8867 if (err) 8868 return err; 8869 8870 btf_scan_type_tags(btf, args[i].type, &tags); 8871 8872 t = btf_type_by_id(btf, args[i].type); 8873 while (btf_type_is_modifier(t)) 8874 t = btf_type_by_id(btf, t->type); 8875 if (!btf_type_is_ptr(t)) 8876 goto skip_pointer; 8877 8878 if ((tags & ARG_TAG_CTX) || btf_is_prog_ctx_type(log, btf, t, prog_type, i)) { 8879 if (tags & ~ARG_TAG_CTX) { 8880 bpf_log(log, "arg#%d has invalid combination of tags\n", i); 8881 return -EINVAL; 8882 } 8883 if ((tags & ARG_TAG_CTX) && 8884 btf_validate_prog_ctx_type(log, btf, t, i, prog_type, 8885 prog->expected_attach_type)) 8886 return -EINVAL; 8887 sub->args[slots_used++].arg_type = ARG_PTR_TO_CTX; 8888 continue; 8889 } 8890 if (btf_is_dynptr_ptr(btf, t)) { 8891 if (tags) { 8892 bpf_log(log, "arg#%d has invalid combination of tags\n", i); 8893 return -EINVAL; 8894 } 8895 sub->args[slots_used++].arg_type = ARG_PTR_TO_DYNPTR; 8896 continue; 8897 } 8898 if (tags & ARG_TAG_TRUSTED) { 8899 int kern_type_id; 8900 8901 if (tags & ARG_TAG_NONNULL) { 8902 bpf_log(log, "arg#%d has invalid combination of tags\n", i); 8903 return -EINVAL; 8904 } 8905 8906 kern_type_id = btf_get_ptr_to_btf_id(log, i, btf, t); 8907 if (kern_type_id < 0) 8908 return kern_type_id; 8909 8910 sub->args[slots_used].arg_type = ARG_PTR_TO_BTF_ID | PTR_TRUSTED; 8911 if (tags & ARG_TAG_NULLABLE) 8912 sub->args[slots_used].arg_type |= PTR_MAYBE_NULL; 8913 sub->args[slots_used].btf_id = kern_type_id; 8914 slots_used++; 8915 continue; 8916 } 8917 if (tags & ARG_TAG_UNTRUSTED) { 8918 struct btf *vmlinux_btf; 8919 int kern_type_id; 8920 8921 if (tags & ~ARG_TAG_UNTRUSTED) { 8922 bpf_log(log, "arg#%d untrusted cannot be combined with any other tags\n", i); 8923 return -EINVAL; 8924 } 8925 8926 ref_t = btf_type_skip_modifiers(btf, t->type, NULL); 8927 if (btf_type_is_void(ref_t) || btf_type_is_primitive(ref_t)) { 8928 sub->args[slots_used].arg_type = ARG_PTR_TO_MEM | MEM_RDONLY | 8929 PTR_UNTRUSTED; 8930 sub->args[slots_used].mem_size = 0; 8931 slots_used++; 8932 continue; 8933 } 8934 8935 kern_type_id = btf_get_ptr_to_btf_id(log, i, btf, t); 8936 if (kern_type_id < 0) 8937 return kern_type_id; 8938 8939 vmlinux_btf = bpf_get_btf_vmlinux(); 8940 ref_t = btf_type_by_id(vmlinux_btf, kern_type_id); 8941 if (!btf_type_is_struct(ref_t)) { 8942 tname = __btf_name_by_offset(vmlinux_btf, t->name_off); 8943 bpf_log(log, "arg#%d has type %s '%s', but only struct or primitive types are allowed\n", 8944 i, btf_type_str(ref_t), tname); 8945 return -EINVAL; 8946 } 8947 sub->args[slots_used].arg_type = ARG_PTR_TO_BTF_ID | PTR_UNTRUSTED; 8948 sub->args[slots_used].btf_id = kern_type_id; 8949 slots_used++; 8950 continue; 8951 } 8952 if (tags & ARG_TAG_ARENA) { 8953 if (tags & ~ARG_TAG_ARENA) { 8954 bpf_log(log, "arg#%d arena cannot be combined with any other tags\n", i); 8955 return -EINVAL; 8956 } 8957 sub->args[slots_used++].arg_type = ARG_PTR_TO_ARENA; 8958 continue; 8959 } 8960 if (is_global) { /* generic user data pointer */ 8961 u32 mem_size; 8962 8963 if (tags & ARG_TAG_NULLABLE) { 8964 bpf_log(log, "arg#%d has invalid combination of tags\n", i); 8965 return -EINVAL; 8966 } 8967 8968 t = btf_type_skip_modifiers(btf, t->type, NULL); 8969 ref_t = btf_resolve_size(btf, t, &mem_size); 8970 if (IS_ERR(ref_t)) { 8971 bpf_log(log, "arg#%d reference type('%s %s') size cannot be determined: %ld\n", 8972 i, btf_type_str(t), btf_name_by_offset(btf, t->name_off), 8973 PTR_ERR(ref_t)); 8974 return -EINVAL; 8975 } 8976 8977 sub->args[slots_used].arg_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL; 8978 if (tags & ARG_TAG_NONNULL) 8979 sub->args[slots_used].arg_type &= ~PTR_MAYBE_NULL; 8980 sub->args[slots_used].mem_size = mem_size; 8981 slots_used++; 8982 continue; 8983 } 8984 8985 skip_pointer: 8986 if (tags) { 8987 bpf_log(log, "arg#%d has pointer tag, but is not a pointer type\n", i); 8988 return -EINVAL; 8989 } 8990 if (btf_type_is_int(t) || btf_is_any_enum(t) || btf_type_is_struct(t)) { 8991 u32 nslots; 8992 8993 if (!t->size || t->size > 2 * BPF_REG_SIZE) { 8994 if (!is_global) 8995 return -EINVAL; 8996 bpf_log(log, 8997 "Arg#%d type %s in %s() has size %u, only 1 to %d bytes " 8998 "can be passed by value\n", 8999 i, btf_type_str(t), tname, t->size, 2 * BPF_REG_SIZE); 9000 return -EINVAL; 9001 } 9002 if (btf_type_is_struct(t) && 9003 !btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) { 9004 if (!is_global) 9005 return -EINVAL; 9006 bpf_log(log, "Arg#%d type %s in %s() is not composed of scalars\n", 9007 i, btf_type_str(t), tname); 9008 return -EINVAL; 9009 } 9010 9011 nslots = (t->size + BPF_REG_SIZE - 1) / BPF_REG_SIZE; 9012 if (slots_used + nslots > MAX_BPF_FUNC_ARGS) 9013 goto too_many_slots; 9014 while (nslots--) 9015 sub->args[slots_used++].arg_type = ARG_SCALAR; 9016 continue; 9017 } 9018 if (!is_global) 9019 return -EINVAL; 9020 bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n", 9021 i, btf_type_str(t), tname); 9022 return -EINVAL; 9023 } 9024 9025 err = btf_check_arg_slots(log, tname, is_global, slots_used, sub); 9026 if (err) 9027 return err; 9028 sub->arg_slot_cnt = slots_used; 9029 9030 /* check_ext_args() needs them after env is freed */ 9031 if (is_global) 9032 memcpy(subprog_aux(env, subprog)->args, sub->args, 9033 sizeof(subprog_aux(env, subprog)->args)); 9034 9035 sub->args_cached = true; 9036 9037 return 0; 9038 9039 too_many_slots: 9040 if (!is_global) 9041 return -EINVAL; 9042 bpf_log(log, "Arguments of %s() need more than %d argument slots\n", 9043 tname, MAX_BPF_FUNC_ARGS); 9044 return -EINVAL; 9045 } 9046 9047 static void btf_type_show(const struct btf *btf, u32 type_id, void *obj, 9048 struct btf_show *show) 9049 { 9050 const struct btf_type *t = btf_type_by_id(btf, type_id); 9051 9052 show->btf = btf; 9053 memset(&show->state, 0, sizeof(show->state)); 9054 memset(&show->obj, 0, sizeof(show->obj)); 9055 9056 btf_type_ops(t)->show(btf, t, type_id, obj, 0, show); 9057 } 9058 9059 __printf(2, 0) static void btf_seq_show(struct btf_show *show, const char *fmt, 9060 va_list args) 9061 { 9062 seq_vprintf((struct seq_file *)show->target, fmt, args); 9063 } 9064 9065 int btf_type_seq_show_flags(const struct btf *btf, u32 type_id, 9066 void *obj, struct seq_file *m, u64 flags) 9067 { 9068 struct btf_show sseq; 9069 9070 sseq.target = m; 9071 sseq.showfn = btf_seq_show; 9072 sseq.flags = flags; 9073 9074 btf_type_show(btf, type_id, obj, &sseq); 9075 9076 return sseq.state.status; 9077 } 9078 9079 void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj, 9080 struct seq_file *m) 9081 { 9082 (void) btf_type_seq_show_flags(btf, type_id, obj, m, 9083 BTF_SHOW_NONAME | BTF_SHOW_COMPACT | 9084 BTF_SHOW_ZERO | BTF_SHOW_UNSAFE); 9085 } 9086 9087 struct btf_show_snprintf { 9088 struct btf_show show; 9089 int len_left; /* space left in string */ 9090 int len; /* length we would have written */ 9091 }; 9092 9093 __printf(2, 0) static void btf_snprintf_show(struct btf_show *show, const char *fmt, 9094 va_list args) 9095 { 9096 struct btf_show_snprintf *ssnprintf = (struct btf_show_snprintf *)show; 9097 int len; 9098 9099 len = vsnprintf(show->target, ssnprintf->len_left, fmt, args); 9100 9101 if (len < 0) { 9102 ssnprintf->len_left = 0; 9103 ssnprintf->len = len; 9104 } else if (len >= ssnprintf->len_left) { 9105 /* no space, drive on to get length we would have written */ 9106 ssnprintf->len_left = 0; 9107 ssnprintf->len += len; 9108 } else { 9109 ssnprintf->len_left -= len; 9110 ssnprintf->len += len; 9111 show->target += len; 9112 } 9113 } 9114 9115 int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, 9116 char *buf, int len, u64 flags) 9117 { 9118 struct btf_show_snprintf ssnprintf; 9119 9120 ssnprintf.show.target = buf; 9121 ssnprintf.show.flags = flags; 9122 ssnprintf.show.showfn = btf_snprintf_show; 9123 ssnprintf.len_left = len; 9124 ssnprintf.len = 0; 9125 9126 btf_type_show(btf, type_id, obj, (struct btf_show *)&ssnprintf); 9127 9128 /* If we encountered an error, return it. */ 9129 if (ssnprintf.show.state.status) 9130 return ssnprintf.show.state.status; 9131 9132 /* Otherwise return length we would have written */ 9133 return ssnprintf.len; 9134 } 9135 9136 int btf_type_name_to_buf(const struct btf *btf, u32 type_id, char *buf, int len) 9137 { 9138 struct btf_show show = { 9139 .btf = btf, 9140 .state.type_id = type_id, 9141 }; 9142 9143 return snprintf(buf, len, "%s", btf_show_name(&show)); 9144 } 9145 9146 #ifdef CONFIG_PROC_FS 9147 static void bpf_btf_show_fdinfo(struct seq_file *m, struct file *filp) 9148 { 9149 const struct btf *btf = filp->private_data; 9150 9151 seq_printf(m, "btf_id:\t%u\n", READ_ONCE(btf->id)); 9152 } 9153 #endif 9154 9155 static int btf_release(struct inode *inode, struct file *filp) 9156 { 9157 btf_put(filp->private_data); 9158 return 0; 9159 } 9160 9161 const struct file_operations btf_fops = { 9162 #ifdef CONFIG_PROC_FS 9163 .show_fdinfo = bpf_btf_show_fdinfo, 9164 #endif 9165 .release = btf_release, 9166 }; 9167 9168 static int __btf_new_fd(struct btf *btf) 9169 { 9170 return anon_inode_getfd("btf", &btf_fops, btf, O_RDONLY | O_CLOEXEC); 9171 } 9172 9173 int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log) 9174 { 9175 struct btf *btf; 9176 int ret; 9177 9178 btf = btf_parse(attr, uattr, attr_log); 9179 if (IS_ERR(btf)) 9180 return PTR_ERR(btf); 9181 9182 ret = btf_alloc_id(btf); 9183 if (ret) { 9184 btf_free(btf); 9185 return ret; 9186 } 9187 9188 /* 9189 * The BTF ID is published to the userspace. 9190 * All BTF free must go through call_rcu() from 9191 * now on (i.e. free by calling btf_put()). 9192 */ 9193 9194 ret = __btf_new_fd(btf); 9195 if (ret < 0) 9196 btf_put(btf); 9197 9198 return ret; 9199 } 9200 9201 struct btf *btf_get_by_fd(int fd) 9202 { 9203 struct btf *btf; 9204 CLASS(fd, f)(fd); 9205 9206 btf = __btf_get_by_fd(f); 9207 if (!IS_ERR(btf)) 9208 refcount_inc(&btf->refcnt); 9209 9210 return btf; 9211 } 9212 9213 int btf_get_info_by_fd(const struct btf *btf, 9214 const union bpf_attr *attr, 9215 union bpf_attr __user *uattr) 9216 { 9217 struct bpf_btf_info __user *uinfo; 9218 struct bpf_btf_info info; 9219 u32 info_copy, btf_copy; 9220 void __user *ubtf; 9221 char __user *uname; 9222 u32 uinfo_len, uname_len, name_len; 9223 int ret = 0; 9224 9225 uinfo = u64_to_user_ptr(attr->info.info); 9226 uinfo_len = attr->info.info_len; 9227 9228 info_copy = min_t(u32, uinfo_len, sizeof(info)); 9229 memset(&info, 0, sizeof(info)); 9230 if (copy_from_user(&info, uinfo, info_copy)) 9231 return -EFAULT; 9232 9233 info.id = READ_ONCE(btf->id); 9234 ubtf = u64_to_user_ptr(info.btf); 9235 btf_copy = min_t(u32, btf->data_size, info.btf_size); 9236 if (copy_to_user(ubtf, btf->data, btf_copy)) 9237 return -EFAULT; 9238 info.btf_size = btf->data_size; 9239 9240 info.kernel_btf = btf->kernel_btf; 9241 9242 uname = u64_to_user_ptr(info.name); 9243 uname_len = info.name_len; 9244 if (!uname ^ !uname_len) 9245 return -EINVAL; 9246 9247 name_len = strlen(btf->name); 9248 info.name_len = name_len; 9249 9250 if (uname) { 9251 if (uname_len >= name_len + 1) { 9252 if (copy_to_user(uname, btf->name, name_len + 1)) 9253 return -EFAULT; 9254 } else { 9255 char zero = '\0'; 9256 9257 if (copy_to_user(uname, btf->name, uname_len - 1)) 9258 return -EFAULT; 9259 if (put_user(zero, uname + uname_len - 1)) 9260 return -EFAULT; 9261 /* let user-space know about too short buffer */ 9262 ret = -ENOSPC; 9263 } 9264 } 9265 9266 if (copy_to_user(uinfo, &info, info_copy) || 9267 put_user(info_copy, &uattr->info.info_len)) 9268 return -EFAULT; 9269 9270 return ret; 9271 } 9272 9273 int btf_get_fd_by_id(u32 id) 9274 { 9275 struct btf *btf; 9276 int fd; 9277 9278 rcu_read_lock(); 9279 btf = idr_find(&btf_idr, id); 9280 if (!btf || !refcount_inc_not_zero(&btf->refcnt)) 9281 btf = ERR_PTR(-ENOENT); 9282 rcu_read_unlock(); 9283 9284 if (IS_ERR(btf)) 9285 return PTR_ERR(btf); 9286 9287 fd = __btf_new_fd(btf); 9288 if (fd < 0) 9289 btf_put(btf); 9290 9291 return fd; 9292 } 9293 9294 u32 btf_obj_id(const struct btf *btf) 9295 { 9296 return READ_ONCE(btf->id); 9297 } 9298 9299 bool btf_is_kernel(const struct btf *btf) 9300 { 9301 return btf->kernel_btf; 9302 } 9303 9304 bool btf_is_module(const struct btf *btf) 9305 { 9306 return btf->kernel_btf && strcmp(btf->name, "vmlinux") != 0; 9307 } 9308 9309 enum { 9310 BTF_MODULE_F_LIVE = (1 << 0), 9311 }; 9312 9313 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES 9314 struct btf_module { 9315 struct list_head list; 9316 struct module *module; 9317 struct btf *btf; 9318 struct bin_attribute *sysfs_attr; 9319 struct bin_attribute *sysfs_inline_attr; 9320 void *btf_inline_data; 9321 int flags; 9322 }; 9323 9324 static LIST_HEAD(btf_modules); 9325 static DEFINE_MUTEX(btf_module_mutex); 9326 9327 static void purge_cand_cache(struct btf *btf); 9328 9329 #if IS_MODULE(CONFIG_DEBUG_INFO_BTF_INLINE) 9330 extern char __start_BTF_inline_link[]; 9331 extern char __stop_BTF_inline_link[]; 9332 9333 static struct bin_attribute *vmlinux_inline_attr; 9334 static const struct btf_link *vmlinux_inline_link; 9335 9336 static const struct btf_link *btf_parse_link(void *start, unsigned int size) 9337 { 9338 const struct btf_link *link = start; 9339 unsigned int module_name_len; 9340 9341 if (!link || size != sizeof(*link) || link->btf_size == 0) 9342 return NULL; 9343 9344 module_name_len = strnlen(link->module_name, sizeof(link->module_name)); 9345 if (!module_name_len || module_name_len == sizeof(link->module_name) || 9346 memchr_inv(link->module_name + module_name_len + 1, 0, 9347 sizeof(link->module_name) - module_name_len - 1)) 9348 return NULL; 9349 return link; 9350 } 9351 9352 static bool btf_verify_link(const struct btf_link *link, const struct module *mod, 9353 const void *data, size_t data_size) 9354 { 9355 u8 actual_digest[BTF_LINK_SHA256_LEN]; 9356 9357 if (!link || data_size != link->btf_size) { 9358 pr_warn(" BTF link from module [%s] missing or has unexpected size\n", mod->name); 9359 return false; 9360 } 9361 sha256(data, data_size, actual_digest); 9362 if (memcmp(actual_digest, link->sha256, sizeof(actual_digest))) { 9363 pr_warn("BTF link from module [%s] has unexpected digest\n", mod->name); 9364 return false; 9365 } 9366 9367 return true; 9368 } 9369 9370 static void btf_vmlinux_inline_lazy_init(void) 9371 { 9372 vmlinux_inline_link = btf_parse_link(__start_BTF_inline_link, 9373 __stop_BTF_inline_link - 9374 __start_BTF_inline_link); 9375 if (!vmlinux_inline_link) { 9376 pr_warn("invalid vmlinux inline BTF link\n"); 9377 return; 9378 } 9379 vmlinux_inline_attr = sysfs_btf_add("vmlinux.inline", NULL, 9380 vmlinux_inline_link->btf_size, true, 9381 vmlinux_inline_link->module_name); 9382 if (IS_ERR(vmlinux_inline_attr)) { 9383 pr_warn("failed to register vmlinux inline BTF in sysfs: %ld\n", 9384 PTR_ERR(vmlinux_inline_attr)); 9385 vmlinux_inline_attr = NULL; 9386 } 9387 } 9388 9389 static bool btf_module_is_vmlinux_inline(const struct module *mod) 9390 { 9391 return vmlinux_inline_link && !strcmp(mod->name, vmlinux_inline_link->module_name); 9392 } 9393 9394 static int btf_module_handle_vmlinux_inline(const struct module *mod) 9395 { 9396 void *data; 9397 9398 if (!mod->btf_inline_data || 9399 mod->btf_inline_data_size != vmlinux_inline_link->btf_size) 9400 return -EINVAL; 9401 9402 data = vmalloc_user(mod->btf_inline_data_size); 9403 if (!data) 9404 return -ENOMEM; 9405 memcpy(data, mod->btf_inline_data, mod->btf_inline_data_size); 9406 9407 if (!btf_verify_link(vmlinux_inline_link, mod, data, mod->btf_inline_data_size) || 9408 !sysfs_btf_update(vmlinux_inline_attr, data)) { 9409 kvfree(data); 9410 return -EINVAL; 9411 } 9412 return 0; 9413 } 9414 9415 #else 9416 9417 static bool btf_module_is_vmlinux_inline(const struct module *mod) 9418 { 9419 return false; 9420 } 9421 9422 static int btf_module_handle_vmlinux_inline(const struct module *mod) 9423 { 9424 return 0; 9425 } 9426 9427 static void btf_vmlinux_inline_lazy_init(void) 9428 { 9429 } 9430 9431 #endif /* IS_MODULE(CONFIG_DEBUG_INFO_BTF_INLINE) */ 9432 9433 static int btf_module_notify(struct notifier_block *nb, unsigned long op, 9434 void *module) 9435 { 9436 struct btf_module *btf_mod, *tmp; 9437 struct module *mod = module; 9438 struct bin_attribute *attr; 9439 struct btf *btf; 9440 void *inline_data = NULL, *relocated_inline_data = NULL; 9441 unsigned int inline_data_size = 0; 9442 int err = 0; 9443 9444 if (op != MODULE_STATE_COMING && op != MODULE_STATE_LIVE && 9445 op != MODULE_STATE_GOING) 9446 goto out; 9447 9448 switch (op) { 9449 case MODULE_STATE_COMING: 9450 if (btf_module_is_vmlinux_inline(mod)) { 9451 err = btf_module_handle_vmlinux_inline(mod); 9452 break; 9453 } 9454 9455 #if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) 9456 if (mod->btf_inline_data_size) { 9457 inline_data = mod->btf_inline_data; 9458 inline_data_size = mod->btf_inline_data_size; 9459 } 9460 #endif 9461 if (!mod->btf_data_size) 9462 break; 9463 btf_mod = kzalloc_obj(*btf_mod); 9464 if (!btf_mod) { 9465 err = -ENOMEM; 9466 goto out; 9467 } 9468 btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size, 9469 mod->btf_base_data, mod->btf_base_data_size, 9470 inline_data, inline_data_size, 9471 &relocated_inline_data); 9472 if (IS_ERR(btf)) { 9473 kfree(btf_mod); 9474 if (!IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) { 9475 pr_warn("failed to validate module [%s] BTF: %ld\n", 9476 mod->name, PTR_ERR(btf)); 9477 err = PTR_ERR(btf); 9478 } else { 9479 pr_warn_once("Kernel module BTF mismatch detected, BTF debug info may be unavailable for some modules\n"); 9480 } 9481 goto out; 9482 } 9483 err = btf_alloc_id(btf); 9484 if (err) { 9485 btf_free(btf); 9486 kvfree(relocated_inline_data); 9487 kfree(btf_mod); 9488 goto out; 9489 } 9490 9491 purge_cand_cache(NULL); 9492 mutex_lock(&btf_module_mutex); 9493 btf_mod->module = module; 9494 btf_mod->btf = btf; 9495 btf_mod->btf_inline_data = relocated_inline_data; 9496 list_add(&btf_mod->list, &btf_modules); 9497 mutex_unlock(&btf_module_mutex); 9498 9499 attr = sysfs_btf_add(btf->name, btf->data, btf->data_size, false, NULL); 9500 if (IS_ERR(attr)) { 9501 err = 0; 9502 goto out; 9503 } 9504 btf_mod->sysfs_attr = attr; 9505 9506 if (relocated_inline_data) { 9507 char name[MODULE_NAME_LEN + sizeof(".inline")]; 9508 9509 snprintf(name, sizeof(name), "%s.inline", mod->name); 9510 attr = sysfs_btf_add(name, relocated_inline_data, 9511 inline_data_size, false, NULL); 9512 if (IS_ERR(attr)) { 9513 kvfree(relocated_inline_data); 9514 btf_mod->btf_inline_data = NULL; 9515 err = 0; 9516 break; 9517 } 9518 btf_mod->sysfs_inline_attr = attr; 9519 } 9520 break; 9521 case MODULE_STATE_LIVE: 9522 if (!mod->btf_data_size) 9523 break; 9524 mutex_lock(&btf_module_mutex); 9525 list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { 9526 if (btf_mod->module != module) 9527 continue; 9528 9529 btf_mod->flags |= BTF_MODULE_F_LIVE; 9530 break; 9531 } 9532 mutex_unlock(&btf_module_mutex); 9533 break; 9534 case MODULE_STATE_GOING: 9535 if (!mod->btf_data_size) 9536 break; 9537 mutex_lock(&btf_module_mutex); 9538 list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { 9539 if (btf_mod->module != module) 9540 continue; 9541 9542 /* 9543 * For modules, we do the freeing of BTF IDR as soon as 9544 * module goes away to disable BTF discovery, since the 9545 * btf_try_get_module() on such BTFs will fail. This may 9546 * be called again on btf_put(), but it's ok to do so. 9547 */ 9548 btf_free_id(btf_mod->btf); 9549 list_del(&btf_mod->list); 9550 if (btf_mod->sysfs_attr) 9551 sysfs_btf_remove(btf_mod->sysfs_attr); 9552 if (btf_mod->sysfs_inline_attr) 9553 sysfs_btf_remove(btf_mod->sysfs_inline_attr); 9554 kvfree(btf_mod->btf_inline_data); 9555 purge_cand_cache(btf_mod->btf); 9556 btf_put(btf_mod->btf); 9557 kfree(btf_mod); 9558 break; 9559 } 9560 mutex_unlock(&btf_module_mutex); 9561 break; 9562 } 9563 out: 9564 return notifier_from_errno(err); 9565 } 9566 9567 static struct notifier_block btf_module_nb = { 9568 .notifier_call = btf_module_notify, 9569 }; 9570 9571 static int __init btf_module_init(void) 9572 { 9573 btf_vmlinux_inline_lazy_init(); 9574 register_module_notifier(&btf_module_nb); 9575 return 0; 9576 } 9577 9578 fs_initcall(btf_module_init); 9579 #endif /* CONFIG_DEBUG_INFO_BTF_MODULES */ 9580 9581 struct module *btf_try_get_module(const struct btf *btf) 9582 { 9583 struct module *res = NULL; 9584 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES 9585 struct btf_module *btf_mod, *tmp; 9586 9587 mutex_lock(&btf_module_mutex); 9588 list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { 9589 if (btf_mod->btf != btf) 9590 continue; 9591 9592 /* We must only consider module whose __init routine has 9593 * finished, hence we must check for BTF_MODULE_F_LIVE flag, 9594 * which is set from the notifier callback for 9595 * MODULE_STATE_LIVE. 9596 */ 9597 if ((btf_mod->flags & BTF_MODULE_F_LIVE) && try_module_get(btf_mod->module)) 9598 res = btf_mod->module; 9599 9600 break; 9601 } 9602 mutex_unlock(&btf_module_mutex); 9603 #endif 9604 9605 return res; 9606 } 9607 9608 /* Returns struct btf corresponding to the struct module. 9609 * This function can return NULL or ERR_PTR. 9610 */ 9611 struct btf *btf_get_module_btf(const struct module *module) 9612 { 9613 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES 9614 struct btf_module *btf_mod, *tmp; 9615 #endif 9616 struct btf *btf = NULL; 9617 9618 if (!module) { 9619 btf = bpf_get_btf_vmlinux(); 9620 if (!IS_ERR_OR_NULL(btf)) 9621 btf_get(btf); 9622 return btf; 9623 } 9624 9625 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES 9626 mutex_lock(&btf_module_mutex); 9627 list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { 9628 if (btf_mod->module != module) 9629 continue; 9630 9631 btf_get(btf_mod->btf); 9632 btf = btf_mod->btf; 9633 break; 9634 } 9635 mutex_unlock(&btf_module_mutex); 9636 #endif 9637 9638 return btf; 9639 } 9640 9641 static int check_btf_kconfigs(const struct module *module, const char *feature) 9642 { 9643 if (!module && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { 9644 pr_err("missing vmlinux BTF, cannot register %s\n", feature); 9645 return -ENOENT; 9646 } 9647 if (module && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES)) 9648 pr_warn("missing module BTF, cannot register %s\n", feature); 9649 return 0; 9650 } 9651 9652 BPF_CALL_4(bpf_btf_find_by_name_kind, char *, name, int, name_sz, u32, kind, int, flags) 9653 { 9654 struct btf *btf = NULL; 9655 int btf_obj_fd = 0; 9656 long ret; 9657 9658 if (flags) 9659 return -EINVAL; 9660 9661 if (name_sz <= 1 || name[name_sz - 1]) 9662 return -EINVAL; 9663 9664 ret = bpf_find_btf_id(name, kind, &btf); 9665 if (ret > 0 && btf_is_module(btf)) { 9666 btf_obj_fd = __btf_new_fd(btf); 9667 if (btf_obj_fd < 0) { 9668 btf_put(btf); 9669 return btf_obj_fd; 9670 } 9671 return ret | (((u64)btf_obj_fd) << 32); 9672 } 9673 if (ret > 0) 9674 btf_put(btf); 9675 return ret; 9676 } 9677 9678 const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { 9679 .func = bpf_btf_find_by_name_kind, 9680 .gpl_only = false, 9681 .might_sleep = true, 9682 .ret_type = RET_INTEGER, 9683 .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, 9684 .arg2_type = ARG_MEM_SIZE, 9685 .arg3_type = ARG_ANYTHING, 9686 .arg4_type = ARG_ANYTHING, 9687 }; 9688 9689 BTF_ID_LIST_GLOBAL(btf_tracing_ids, MAX_BTF_TRACING_TYPE) 9690 #define BTF_TRACING_TYPE(name, type) BTF_ID(struct, type) 9691 BTF_TRACING_TYPE_xxx 9692 #undef BTF_TRACING_TYPE 9693 9694 /* Validate well-formedness of iter argument type. 9695 * On success, return positive BTF ID of iter state's STRUCT type. 9696 * On error, negative error is returned. 9697 */ 9698 int btf_check_iter_arg(struct btf *btf, const struct btf_type *func, int arg_idx) 9699 { 9700 const struct btf_param *arg; 9701 const struct btf_type *t; 9702 const char *name; 9703 int btf_id; 9704 9705 if (btf_type_vlen(func) <= arg_idx) 9706 return -EINVAL; 9707 9708 arg = &btf_params(func)[arg_idx]; 9709 t = btf_type_skip_modifiers(btf, arg->type, NULL); 9710 if (!t || !btf_type_is_ptr(t)) 9711 return -EINVAL; 9712 t = btf_type_skip_modifiers(btf, t->type, &btf_id); 9713 if (!t || !__btf_type_is_struct(t)) 9714 return -EINVAL; 9715 9716 name = btf_name_by_offset(btf, t->name_off); 9717 if (!name || strncmp(name, ITER_PREFIX, sizeof(ITER_PREFIX) - 1)) 9718 return -EINVAL; 9719 9720 return btf_id; 9721 } 9722 9723 static int btf_check_iter_kfuncs(struct btf *btf, const char *func_name, 9724 const struct btf_type *func, u32 func_flags) 9725 { 9726 u32 flags = func_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY); 9727 const char *sfx, *iter_name; 9728 const struct btf_type *t; 9729 char exp_name[128]; 9730 u32 nr_args; 9731 int btf_id; 9732 9733 /* exactly one of KF_ITER_{NEW,NEXT,DESTROY} can be set */ 9734 if (!flags || (flags & (flags - 1))) 9735 return -EINVAL; 9736 9737 if ((func_flags & KF_ITER_RET_INDEPENDENT) && 9738 (!(flags & KF_ITER_NEXT) || (func_flags & KF_ACQUIRE))) 9739 return -EINVAL; 9740 9741 /* any BPF iter kfunc should have `struct bpf_iter_<type> *` first arg */ 9742 nr_args = btf_type_vlen(func); 9743 if (nr_args < 1) 9744 return -EINVAL; 9745 9746 btf_id = btf_check_iter_arg(btf, func, 0); 9747 if (btf_id < 0) 9748 return btf_id; 9749 9750 /* sizeof(struct bpf_iter_<type>) should be a multiple of 8 to 9751 * fit nicely in stack slots 9752 */ 9753 t = btf_type_by_id(btf, btf_id); 9754 if (t->size == 0 || (t->size % 8)) 9755 return -EINVAL; 9756 9757 /* validate bpf_iter_<type>_{new,next,destroy}(struct bpf_iter_<type> *) 9758 * naming pattern 9759 */ 9760 iter_name = btf_name_by_offset(btf, t->name_off) + sizeof(ITER_PREFIX) - 1; 9761 if (flags & KF_ITER_NEW) 9762 sfx = "new"; 9763 else if (flags & KF_ITER_NEXT) 9764 sfx = "next"; 9765 else /* (flags & KF_ITER_DESTROY) */ 9766 sfx = "destroy"; 9767 9768 snprintf(exp_name, sizeof(exp_name), "bpf_iter_%s_%s", iter_name, sfx); 9769 if (strcmp(func_name, exp_name)) 9770 return -EINVAL; 9771 9772 /* only iter constructor should have extra arguments */ 9773 if (!(flags & KF_ITER_NEW) && nr_args != 1) 9774 return -EINVAL; 9775 9776 if (flags & KF_ITER_NEXT) { 9777 /* bpf_iter_<type>_next() should return pointer */ 9778 t = btf_type_skip_modifiers(btf, func->type, NULL); 9779 if (!t || !btf_type_is_ptr(t)) 9780 return -EINVAL; 9781 /* The opt-out applies only to non-acquired struct results. */ 9782 if (func_flags & KF_ITER_RET_INDEPENDENT) { 9783 t = btf_type_skip_modifiers(btf, t->type, NULL); 9784 if (!t || !__btf_type_is_struct(t)) 9785 return -EINVAL; 9786 } 9787 } 9788 9789 if (flags & KF_ITER_DESTROY) { 9790 /* bpf_iter_<type>_destroy() should return void */ 9791 t = btf_type_by_id(btf, func->type); 9792 if (!t || !btf_type_is_void(t)) 9793 return -EINVAL; 9794 } 9795 9796 return 0; 9797 } 9798 9799 static int btf_check_kfunc_name(struct btf *btf, const char *func_name, u32 kind) 9800 { 9801 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES 9802 struct btf_module *btf_mod, *tmp; 9803 #endif 9804 s32 id; 9805 9806 if (!btf_is_module(btf)) 9807 return 0; 9808 9809 id = btf_find_by_name_kind(bpf_get_btf_vmlinux(), func_name, kind); 9810 if (id >= 0) { 9811 pr_err("kfunc %s (id: %d) is already present in vmlinux.\n", 9812 func_name, id); 9813 return -EINVAL; 9814 } 9815 9816 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES 9817 guard(mutex)(&btf_module_mutex); 9818 list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { 9819 if (btf_mod->btf == btf) 9820 continue; 9821 id = btf_find_by_name_kind(btf_mod->btf, func_name, kind); 9822 if (id >= 0) { 9823 pr_err("kfunc %s (id: %d) is already present in module %s.\n", 9824 func_name, id, btf_mod->module->name); 9825 return -EINVAL; 9826 } 9827 } 9828 #endif 9829 return 0; 9830 } 9831 9832 static int btf_check_kfunc_protos(struct btf *btf, u32 func_id, u32 func_flags) 9833 { 9834 const struct btf_type *func; 9835 const char *func_name; 9836 int err; 9837 9838 /* any kfunc should be FUNC -> FUNC_PROTO */ 9839 func = btf_type_by_id(btf, func_id); 9840 if (!func || !btf_type_is_func(func)) 9841 return -EINVAL; 9842 9843 /* sanity check kfunc name */ 9844 func_name = btf_name_by_offset(btf, func->name_off); 9845 if (!func_name || !func_name[0] || 9846 btf_check_kfunc_name(btf, func_name, BTF_INFO_KIND(func->info))) 9847 return -EINVAL; 9848 9849 func = btf_type_by_id(btf, func->type); 9850 if (!func || !btf_type_is_func_proto(func)) 9851 return -EINVAL; 9852 9853 if (func_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY | 9854 KF_ITER_RET_INDEPENDENT)) { 9855 err = btf_check_iter_kfuncs(btf, func_name, func, func_flags); 9856 if (err) 9857 return err; 9858 } 9859 9860 return 0; 9861 } 9862 9863 /* Kernel Function (kfunc) BTF ID set registration API */ 9864 9865 static int btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, 9866 const struct btf_kfunc_id_set *kset) 9867 { 9868 struct btf_kfunc_hook_filter *hook_filter; 9869 struct btf_id_set8 *add_set = kset->set; 9870 bool vmlinux_set = !btf_is_module(btf); 9871 bool add_filter = !!kset->filter; 9872 struct btf_kfunc_set_tab *tab; 9873 struct btf_id_set8 *set; 9874 u32 set_cnt, i; 9875 int ret; 9876 9877 if (hook >= BTF_KFUNC_HOOK_MAX) { 9878 ret = -EINVAL; 9879 goto end; 9880 } 9881 9882 if (!add_set->cnt) 9883 return 0; 9884 9885 tab = btf->kfunc_set_tab; 9886 9887 if (tab && add_filter) { 9888 u32 i; 9889 9890 hook_filter = &tab->hook_filters[hook]; 9891 for (i = 0; i < hook_filter->nr_filters; i++) { 9892 if (hook_filter->filters[i] == kset->filter) { 9893 add_filter = false; 9894 break; 9895 } 9896 } 9897 9898 if (add_filter && hook_filter->nr_filters == BTF_KFUNC_FILTER_MAX_CNT) { 9899 ret = -E2BIG; 9900 goto end; 9901 } 9902 } 9903 9904 if (!tab) { 9905 tab = kzalloc_obj(*tab, GFP_KERNEL | __GFP_NOWARN); 9906 if (!tab) 9907 return -ENOMEM; 9908 btf->kfunc_set_tab = tab; 9909 } 9910 9911 set = tab->sets[hook]; 9912 /* Warn when register_btf_kfunc_id_set is called twice for the same hook 9913 * for module sets. 9914 */ 9915 if (WARN_ON_ONCE(set && !vmlinux_set)) { 9916 ret = -EINVAL; 9917 goto end; 9918 } 9919 9920 /* In case of vmlinux sets, there may be more than one set being 9921 * registered per hook. To create a unified set, we allocate a new set 9922 * and concatenate all individual sets being registered. While each set 9923 * is individually sorted, they may become unsorted when concatenated, 9924 * hence re-sorting the final set again is required to make binary 9925 * searching the set using btf_id_set8_contains function work. 9926 * 9927 * For module sets, we need to allocate as we may need to relocate 9928 * BTF ids. 9929 */ 9930 set_cnt = set ? set->cnt : 0; 9931 9932 if (set_cnt > U32_MAX - add_set->cnt) { 9933 ret = -EOVERFLOW; 9934 goto end; 9935 } 9936 9937 if (set_cnt + add_set->cnt > BTF_KFUNC_SET_MAX_CNT) { 9938 ret = -E2BIG; 9939 goto end; 9940 } 9941 9942 /* Grow set */ 9943 set = krealloc(tab->sets[hook], 9944 struct_size(set, pairs, set_cnt + add_set->cnt), 9945 GFP_KERNEL | __GFP_NOWARN); 9946 if (!set) { 9947 ret = -ENOMEM; 9948 goto end; 9949 } 9950 9951 /* For newly allocated set, initialize set->cnt to 0 */ 9952 if (!tab->sets[hook]) 9953 set->cnt = 0; 9954 tab->sets[hook] = set; 9955 9956 /* Concatenate the two sets */ 9957 memcpy(set->pairs + set->cnt, add_set->pairs, add_set->cnt * sizeof(set->pairs[0])); 9958 /* Now that the set is copied, update with relocated BTF ids */ 9959 for (i = set->cnt; i < set->cnt + add_set->cnt; i++) 9960 set->pairs[i].id = btf_relocate_id(btf, set->pairs[i].id); 9961 9962 set->cnt += add_set->cnt; 9963 9964 sort(set->pairs, set->cnt, sizeof(set->pairs[0]), btf_id_cmp_func, NULL); 9965 9966 if (add_filter) { 9967 hook_filter = &tab->hook_filters[hook]; 9968 hook_filter->filters[hook_filter->nr_filters++] = kset->filter; 9969 } 9970 return 0; 9971 end: 9972 btf_free_kfunc_set_tab(btf); 9973 return ret; 9974 } 9975 9976 static u32 *btf_kfunc_id_set_contains(const struct btf *btf, 9977 enum btf_kfunc_hook hook, 9978 u32 kfunc_btf_id) 9979 { 9980 struct btf_id_set8 *set; 9981 u32 *id; 9982 9983 if (hook >= BTF_KFUNC_HOOK_MAX) 9984 return NULL; 9985 if (!btf->kfunc_set_tab) 9986 return NULL; 9987 set = btf->kfunc_set_tab->sets[hook]; 9988 if (!set) 9989 return NULL; 9990 id = btf_id_set8_contains(set, kfunc_btf_id); 9991 if (!id) 9992 return NULL; 9993 /* The flags for BTF ID are located next to it */ 9994 return id + 1; 9995 } 9996 9997 static bool __btf_kfunc_is_allowed(const struct btf *btf, 9998 enum btf_kfunc_hook hook, 9999 u32 kfunc_btf_id, 10000 const struct bpf_prog *prog) 10001 { 10002 struct btf_kfunc_hook_filter *hook_filter; 10003 int i; 10004 10005 if (hook >= BTF_KFUNC_HOOK_MAX) 10006 return false; 10007 if (!btf->kfunc_set_tab) 10008 return false; 10009 10010 hook_filter = &btf->kfunc_set_tab->hook_filters[hook]; 10011 for (i = 0; i < hook_filter->nr_filters; i++) { 10012 if (hook_filter->filters[i](prog, kfunc_btf_id)) 10013 return false; 10014 } 10015 10016 return true; 10017 } 10018 10019 static int bpf_prog_type_to_kfunc_hook(enum bpf_prog_type prog_type) 10020 { 10021 switch (prog_type) { 10022 case BPF_PROG_TYPE_UNSPEC: 10023 return BTF_KFUNC_HOOK_COMMON; 10024 case BPF_PROG_TYPE_XDP: 10025 return BTF_KFUNC_HOOK_XDP; 10026 case BPF_PROG_TYPE_SCHED_CLS: 10027 return BTF_KFUNC_HOOK_TC; 10028 case BPF_PROG_TYPE_STRUCT_OPS: 10029 return BTF_KFUNC_HOOK_STRUCT_OPS; 10030 case BPF_PROG_TYPE_TRACING: 10031 case BPF_PROG_TYPE_TRACEPOINT: 10032 case BPF_PROG_TYPE_RAW_TRACEPOINT: 10033 case BPF_PROG_TYPE_PERF_EVENT: 10034 case BPF_PROG_TYPE_LSM: 10035 return BTF_KFUNC_HOOK_TRACING; 10036 case BPF_PROG_TYPE_SYSCALL: 10037 return BTF_KFUNC_HOOK_SYSCALL; 10038 case BPF_PROG_TYPE_CGROUP_SKB: 10039 case BPF_PROG_TYPE_CGROUP_SOCK: 10040 case BPF_PROG_TYPE_CGROUP_DEVICE: 10041 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 10042 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 10043 case BPF_PROG_TYPE_CGROUP_SYSCTL: 10044 case BPF_PROG_TYPE_SOCK_OPS: 10045 return BTF_KFUNC_HOOK_CGROUP; 10046 case BPF_PROG_TYPE_SCHED_ACT: 10047 return BTF_KFUNC_HOOK_SCHED_ACT; 10048 case BPF_PROG_TYPE_SK_SKB: 10049 return BTF_KFUNC_HOOK_SK_SKB; 10050 case BPF_PROG_TYPE_SOCKET_FILTER: 10051 return BTF_KFUNC_HOOK_SOCKET_FILTER; 10052 case BPF_PROG_TYPE_LWT_OUT: 10053 case BPF_PROG_TYPE_LWT_IN: 10054 case BPF_PROG_TYPE_LWT_XMIT: 10055 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 10056 return BTF_KFUNC_HOOK_LWT; 10057 case BPF_PROG_TYPE_NETFILTER: 10058 return BTF_KFUNC_HOOK_NETFILTER; 10059 case BPF_PROG_TYPE_KPROBE: 10060 return BTF_KFUNC_HOOK_KPROBE; 10061 default: 10062 return BTF_KFUNC_HOOK_MAX; 10063 } 10064 } 10065 10066 bool btf_kfunc_is_allowed(const struct btf *btf, 10067 u32 kfunc_btf_id, 10068 const struct bpf_prog *prog) 10069 { 10070 enum bpf_prog_type prog_type = resolve_prog_type(prog); 10071 enum btf_kfunc_hook hook; 10072 u32 *kfunc_flags; 10073 10074 kfunc_flags = btf_kfunc_id_set_contains(btf, BTF_KFUNC_HOOK_COMMON, kfunc_btf_id); 10075 if (kfunc_flags && __btf_kfunc_is_allowed(btf, BTF_KFUNC_HOOK_COMMON, kfunc_btf_id, prog)) 10076 return true; 10077 10078 hook = bpf_prog_type_to_kfunc_hook(prog_type); 10079 kfunc_flags = btf_kfunc_id_set_contains(btf, hook, kfunc_btf_id); 10080 if (kfunc_flags && __btf_kfunc_is_allowed(btf, hook, kfunc_btf_id, prog)) 10081 return true; 10082 10083 return false; 10084 } 10085 10086 /* Caution: 10087 * Reference to the module (obtained using btf_try_get_module) corresponding to 10088 * the struct btf *MUST* be held when calling this function from verifier 10089 * context. This is usually true as we stash references in prog's kfunc_btf_tab; 10090 * keeping the reference for the duration of the call provides the necessary 10091 * protection for looking up a well-formed btf->kfunc_set_tab. 10092 */ 10093 u32 *btf_kfunc_flags(const struct btf *btf, u32 kfunc_btf_id, const struct bpf_prog *prog) 10094 { 10095 enum bpf_prog_type prog_type = resolve_prog_type(prog); 10096 enum btf_kfunc_hook hook; 10097 u32 *kfunc_flags; 10098 10099 kfunc_flags = btf_kfunc_id_set_contains(btf, BTF_KFUNC_HOOK_COMMON, kfunc_btf_id); 10100 if (kfunc_flags) 10101 return kfunc_flags; 10102 10103 hook = bpf_prog_type_to_kfunc_hook(prog_type); 10104 return btf_kfunc_id_set_contains(btf, hook, kfunc_btf_id); 10105 } 10106 10107 /* 10108 * Check a single KF_* @flag on a kfunc across all of its hook sets. 10109 * Returns: 10110 * * 1 if @flag is set 10111 * * 0 if @flag is not set 10112 * * -EINVAL if @flag is set inconsistently across the sets 10113 * * -ENOENT if kfunc_btf_id is not a registered kfunc 10114 */ 10115 int btf_kfunc_check_flag(const struct btf *btf, u32 kfunc_btf_id, u32 flag) 10116 { 10117 enum btf_kfunc_hook hook; 10118 int res = -ENOENT; 10119 bool is_set; 10120 u32 *flags; 10121 10122 for (hook = 0; hook < BTF_KFUNC_HOOK_MAX; hook++) { 10123 flags = btf_kfunc_id_set_contains(btf, hook, kfunc_btf_id); 10124 if (!flags) 10125 continue; 10126 is_set = *flags & flag; 10127 if (res < 0) 10128 res = is_set; 10129 else if (res != is_set) 10130 return -EINVAL; 10131 } 10132 10133 return res; 10134 } 10135 10136 u32 *btf_kfunc_is_modify_return(const struct btf *btf, u32 kfunc_btf_id, 10137 const struct bpf_prog *prog) 10138 { 10139 if (!__btf_kfunc_is_allowed(btf, BTF_KFUNC_HOOK_FMODRET, kfunc_btf_id, prog)) 10140 return NULL; 10141 10142 return btf_kfunc_id_set_contains(btf, BTF_KFUNC_HOOK_FMODRET, kfunc_btf_id); 10143 } 10144 10145 static int __register_btf_kfunc_id_set(enum btf_kfunc_hook hook, 10146 const struct btf_kfunc_id_set *kset) 10147 { 10148 struct btf *btf; 10149 int ret, i; 10150 10151 btf = btf_get_module_btf(kset->owner); 10152 if (!btf) 10153 return check_btf_kconfigs(kset->owner, "kfunc"); 10154 if (IS_ERR(btf)) 10155 return PTR_ERR(btf); 10156 10157 for (i = 0; i < kset->set->cnt; i++) { 10158 ret = btf_check_kfunc_protos(btf, btf_relocate_id(btf, kset->set->pairs[i].id), 10159 kset->set->pairs[i].flags); 10160 if (ret) 10161 goto err_out; 10162 } 10163 10164 ret = btf_populate_kfunc_set(btf, hook, kset); 10165 10166 err_out: 10167 btf_put(btf); 10168 return ret; 10169 } 10170 10171 /* This function must be invoked only from initcalls/module init functions */ 10172 int register_btf_kfunc_id_set(enum bpf_prog_type prog_type, 10173 const struct btf_kfunc_id_set *kset) 10174 { 10175 enum btf_kfunc_hook hook; 10176 10177 /* All kfuncs need to be tagged as such in BTF. 10178 * WARN() for initcall registrations that do not check errors. 10179 */ 10180 if (!(kset->set->flags & BTF_SET8_KFUNCS)) { 10181 WARN_ON(!kset->owner); 10182 return -EINVAL; 10183 } 10184 10185 hook = bpf_prog_type_to_kfunc_hook(prog_type); 10186 return __register_btf_kfunc_id_set(hook, kset); 10187 } 10188 EXPORT_SYMBOL_GPL(register_btf_kfunc_id_set); 10189 10190 /* This function must be invoked only from initcalls/module init functions */ 10191 int register_btf_fmodret_id_set(const struct btf_kfunc_id_set *kset) 10192 { 10193 return __register_btf_kfunc_id_set(BTF_KFUNC_HOOK_FMODRET, kset); 10194 } 10195 EXPORT_SYMBOL_GPL(register_btf_fmodret_id_set); 10196 10197 s32 btf_find_dtor_kfunc(struct btf *btf, u32 btf_id) 10198 { 10199 struct btf_id_dtor_kfunc_tab *tab = btf->dtor_kfunc_tab; 10200 struct btf_id_dtor_kfunc *dtor; 10201 10202 if (!tab) 10203 return -ENOENT; 10204 /* Even though the size of tab->dtors[0] is > sizeof(u32), we only need 10205 * to compare the first u32 with btf_id, so we can reuse btf_id_cmp_func. 10206 */ 10207 BUILD_BUG_ON(offsetof(struct btf_id_dtor_kfunc, btf_id) != 0); 10208 dtor = bsearch(&btf_id, tab->dtors, tab->cnt, sizeof(tab->dtors[0]), btf_id_cmp_func); 10209 if (!dtor) 10210 return -ENOENT; 10211 return dtor->kfunc_btf_id; 10212 } 10213 10214 static int btf_check_dtor_kfuncs(struct btf *btf, const struct btf_id_dtor_kfunc *dtors, u32 cnt) 10215 { 10216 const struct btf_type *dtor_func, *dtor_func_proto, *t; 10217 const struct btf_param *args; 10218 s32 dtor_btf_id; 10219 u32 nr_args, i; 10220 10221 for (i = 0; i < cnt; i++) { 10222 dtor_btf_id = btf_relocate_id(btf, dtors[i].kfunc_btf_id); 10223 10224 dtor_func = btf_type_by_id(btf, dtor_btf_id); 10225 if (!dtor_func || !btf_type_is_func(dtor_func)) 10226 return -EINVAL; 10227 10228 dtor_func_proto = btf_type_by_id(btf, dtor_func->type); 10229 if (!dtor_func_proto || !btf_type_is_func_proto(dtor_func_proto)) 10230 return -EINVAL; 10231 10232 /* Make sure the prototype of the destructor kfunc is 'void func(type *)' */ 10233 t = btf_type_by_id(btf, dtor_func_proto->type); 10234 if (!t || !btf_type_is_void(t)) 10235 return -EINVAL; 10236 10237 nr_args = btf_type_vlen(dtor_func_proto); 10238 if (nr_args != 1) 10239 return -EINVAL; 10240 args = btf_params(dtor_func_proto); 10241 t = btf_type_by_id(btf, args[0].type); 10242 /* Allow any pointer type, as width on targets Linux supports 10243 * will be same for all pointer types (i.e. sizeof(void *)) 10244 */ 10245 if (!t || !btf_type_is_ptr(t)) 10246 return -EINVAL; 10247 10248 if (IS_ENABLED(CONFIG_CFI)) { 10249 /* Ensure the destructor kfunc type matches btf_dtor_kfunc_t */ 10250 t = btf_type_by_id(btf, t->type); 10251 if (!btf_type_is_void(t)) 10252 return -EINVAL; 10253 } 10254 } 10255 return 0; 10256 } 10257 10258 /* This function must be invoked only from initcalls/module init functions */ 10259 int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dtors, u32 add_cnt, 10260 struct module *owner) 10261 { 10262 struct btf_id_dtor_kfunc_tab *tab; 10263 struct btf *btf; 10264 u32 tab_cnt, i; 10265 int ret; 10266 10267 btf = btf_get_module_btf(owner); 10268 if (!btf) 10269 return check_btf_kconfigs(owner, "dtor kfuncs"); 10270 if (IS_ERR(btf)) 10271 return PTR_ERR(btf); 10272 10273 if (add_cnt >= BTF_DTOR_KFUNC_MAX_CNT) { 10274 pr_err("cannot register more than %d kfunc destructors\n", BTF_DTOR_KFUNC_MAX_CNT); 10275 ret = -E2BIG; 10276 goto end; 10277 } 10278 10279 /* Ensure that the prototype of dtor kfuncs being registered is sane */ 10280 ret = btf_check_dtor_kfuncs(btf, dtors, add_cnt); 10281 if (ret < 0) 10282 goto end; 10283 10284 tab = btf->dtor_kfunc_tab; 10285 /* Only one call allowed for modules */ 10286 if (WARN_ON_ONCE(tab && btf_is_module(btf))) { 10287 ret = -EINVAL; 10288 goto end; 10289 } 10290 10291 tab_cnt = tab ? tab->cnt : 0; 10292 if (tab_cnt > U32_MAX - add_cnt) { 10293 ret = -EOVERFLOW; 10294 goto end; 10295 } 10296 if (tab_cnt + add_cnt >= BTF_DTOR_KFUNC_MAX_CNT) { 10297 pr_err("cannot register more than %d kfunc destructors\n", BTF_DTOR_KFUNC_MAX_CNT); 10298 ret = -E2BIG; 10299 goto end; 10300 } 10301 10302 tab = krealloc(btf->dtor_kfunc_tab, 10303 struct_size(tab, dtors, tab_cnt + add_cnt), 10304 GFP_KERNEL | __GFP_NOWARN); 10305 if (!tab) { 10306 ret = -ENOMEM; 10307 goto end; 10308 } 10309 10310 if (!btf->dtor_kfunc_tab) 10311 tab->cnt = 0; 10312 btf->dtor_kfunc_tab = tab; 10313 10314 memcpy(tab->dtors + tab->cnt, dtors, add_cnt * sizeof(tab->dtors[0])); 10315 10316 /* remap BTF ids based on BTF relocation (if any) */ 10317 for (i = tab_cnt; i < tab_cnt + add_cnt; i++) { 10318 tab->dtors[i].btf_id = btf_relocate_id(btf, tab->dtors[i].btf_id); 10319 tab->dtors[i].kfunc_btf_id = btf_relocate_id(btf, tab->dtors[i].kfunc_btf_id); 10320 } 10321 10322 tab->cnt += add_cnt; 10323 10324 sort(tab->dtors, tab->cnt, sizeof(tab->dtors[0]), btf_id_cmp_func, NULL); 10325 10326 end: 10327 if (ret) 10328 btf_free_dtor_kfunc_tab(btf); 10329 btf_put(btf); 10330 return ret; 10331 } 10332 EXPORT_SYMBOL_GPL(register_btf_id_dtor_kfuncs); 10333 10334 #define MAX_TYPES_ARE_COMPAT_DEPTH 2 10335 10336 /* Check local and target types for compatibility. This check is used for 10337 * type-based CO-RE relocations and follow slightly different rules than 10338 * field-based relocations. This function assumes that root types were already 10339 * checked for name match. Beyond that initial root-level name check, names 10340 * are completely ignored. Compatibility rules are as follows: 10341 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs/ENUM64s are considered compatible, but 10342 * kind should match for local and target types (i.e., STRUCT is not 10343 * compatible with UNION); 10344 * - for ENUMs/ENUM64s, the size is ignored; 10345 * - for INT, size and signedness are ignored; 10346 * - for ARRAY, dimensionality is ignored, element types are checked for 10347 * compatibility recursively; 10348 * - CONST/VOLATILE/RESTRICT modifiers are ignored; 10349 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible; 10350 * - FUNC_PROTOs are compatible if they have compatible signature: same 10351 * number of input args and compatible return and argument types. 10352 * These rules are not set in stone and probably will be adjusted as we get 10353 * more experience with using BPF CO-RE relocations. 10354 */ 10355 int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, 10356 const struct btf *targ_btf, __u32 targ_id) 10357 { 10358 return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 10359 MAX_TYPES_ARE_COMPAT_DEPTH); 10360 } 10361 10362 #define MAX_TYPES_MATCH_DEPTH 2 10363 10364 int bpf_core_types_match(const struct btf *local_btf, u32 local_id, 10365 const struct btf *targ_btf, u32 targ_id) 10366 { 10367 return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 10368 MAX_TYPES_MATCH_DEPTH); 10369 } 10370 10371 static bool bpf_core_is_flavor_sep(const char *s) 10372 { 10373 /* check X___Y name pattern, where X and Y are not underscores */ 10374 return s[0] != '_' && /* X */ 10375 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ 10376 s[4] != '_'; /* Y */ 10377 } 10378 10379 size_t bpf_core_essential_name_len(const char *name) 10380 { 10381 size_t n = strlen(name); 10382 int i; 10383 10384 for (i = n - 5; i >= 0; i--) { 10385 if (bpf_core_is_flavor_sep(name + i)) 10386 return i + 1; 10387 } 10388 return n; 10389 } 10390 10391 static void bpf_free_cands(struct bpf_cand_cache *cands) 10392 { 10393 if (!cands->cnt) 10394 /* empty candidate array was allocated on stack */ 10395 return; 10396 kfree(cands); 10397 } 10398 10399 static void bpf_free_cands_from_cache(struct bpf_cand_cache *cands) 10400 { 10401 kfree(cands->name); 10402 kfree(cands); 10403 } 10404 10405 #define VMLINUX_CAND_CACHE_SIZE 31 10406 static struct bpf_cand_cache *vmlinux_cand_cache[VMLINUX_CAND_CACHE_SIZE]; 10407 10408 #define MODULE_CAND_CACHE_SIZE 31 10409 static struct bpf_cand_cache *module_cand_cache[MODULE_CAND_CACHE_SIZE]; 10410 10411 static void __print_cand_cache(struct bpf_verifier_log *log, 10412 struct bpf_cand_cache **cache, 10413 int cache_size) 10414 { 10415 struct bpf_cand_cache *cc; 10416 int i, j; 10417 10418 for (i = 0; i < cache_size; i++) { 10419 cc = cache[i]; 10420 if (!cc) 10421 continue; 10422 bpf_log(log, "[%d]%s(", i, cc->name); 10423 for (j = 0; j < cc->cnt; j++) { 10424 bpf_log(log, "%d", cc->cands[j].id); 10425 if (j < cc->cnt - 1) 10426 bpf_log(log, " "); 10427 } 10428 bpf_log(log, "), "); 10429 } 10430 } 10431 10432 static void print_cand_cache(struct bpf_verifier_log *log) 10433 { 10434 mutex_lock(&cand_cache_mutex); 10435 bpf_log(log, "vmlinux_cand_cache:"); 10436 __print_cand_cache(log, vmlinux_cand_cache, VMLINUX_CAND_CACHE_SIZE); 10437 bpf_log(log, "\nmodule_cand_cache:"); 10438 __print_cand_cache(log, module_cand_cache, MODULE_CAND_CACHE_SIZE); 10439 bpf_log(log, "\n"); 10440 mutex_unlock(&cand_cache_mutex); 10441 } 10442 10443 static u32 hash_cands(struct bpf_cand_cache *cands) 10444 { 10445 return jhash(cands->name, cands->name_len, 0); 10446 } 10447 10448 static struct bpf_cand_cache *check_cand_cache(struct bpf_cand_cache *cands, 10449 struct bpf_cand_cache **cache, 10450 int cache_size) 10451 { 10452 struct bpf_cand_cache *cc = cache[hash_cands(cands) % cache_size]; 10453 10454 if (cc && cc->name_len == cands->name_len && 10455 !strncmp(cc->name, cands->name, cands->name_len)) 10456 return cc; 10457 return NULL; 10458 } 10459 10460 static size_t sizeof_cands(int cnt) 10461 { 10462 return offsetof(struct bpf_cand_cache, cands[cnt]); 10463 } 10464 10465 static struct bpf_cand_cache *populate_cand_cache(struct bpf_cand_cache *cands, 10466 struct bpf_cand_cache **cache, 10467 int cache_size) 10468 { 10469 struct bpf_cand_cache **cc = &cache[hash_cands(cands) % cache_size], *new_cands; 10470 10471 if (*cc) { 10472 bpf_free_cands_from_cache(*cc); 10473 *cc = NULL; 10474 } 10475 new_cands = kmemdup(cands, sizeof_cands(cands->cnt), GFP_KERNEL_ACCOUNT); 10476 if (!new_cands) { 10477 bpf_free_cands(cands); 10478 return ERR_PTR(-ENOMEM); 10479 } 10480 /* strdup the name, since it will stay in cache. 10481 * the cands->name points to strings in prog's BTF and the prog can be unloaded. 10482 */ 10483 new_cands->name = kmemdup_nul(cands->name, cands->name_len, GFP_KERNEL_ACCOUNT); 10484 bpf_free_cands(cands); 10485 if (!new_cands->name) { 10486 kfree(new_cands); 10487 return ERR_PTR(-ENOMEM); 10488 } 10489 *cc = new_cands; 10490 return new_cands; 10491 } 10492 10493 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES 10494 static void __purge_cand_cache(struct btf *btf, struct bpf_cand_cache **cache, 10495 int cache_size) 10496 { 10497 struct bpf_cand_cache *cc; 10498 int i, j; 10499 10500 for (i = 0; i < cache_size; i++) { 10501 cc = cache[i]; 10502 if (!cc) 10503 continue; 10504 if (!btf) { 10505 /* when new module is loaded purge all of module_cand_cache, 10506 * since new module might have candidates with the name 10507 * that matches cached cands. 10508 */ 10509 bpf_free_cands_from_cache(cc); 10510 cache[i] = NULL; 10511 continue; 10512 } 10513 /* when module is unloaded purge cache entries 10514 * that match module's btf 10515 */ 10516 for (j = 0; j < cc->cnt; j++) 10517 if (cc->cands[j].btf == btf) { 10518 bpf_free_cands_from_cache(cc); 10519 cache[i] = NULL; 10520 break; 10521 } 10522 } 10523 10524 } 10525 10526 static void purge_cand_cache(struct btf *btf) 10527 { 10528 mutex_lock(&cand_cache_mutex); 10529 __purge_cand_cache(btf, module_cand_cache, MODULE_CAND_CACHE_SIZE); 10530 mutex_unlock(&cand_cache_mutex); 10531 } 10532 #endif 10533 10534 static struct bpf_cand_cache * 10535 bpf_core_add_cands(struct bpf_cand_cache *cands, const struct btf *targ_btf, 10536 int targ_start_id) 10537 { 10538 struct bpf_cand_cache *new_cands; 10539 const struct btf_type *t; 10540 const char *targ_name; 10541 size_t targ_essent_len; 10542 int n, i; 10543 10544 n = btf_nr_types(targ_btf); 10545 for (i = targ_start_id; i < n; i++) { 10546 t = btf_type_by_id(targ_btf, i); 10547 if (btf_kind(t) != cands->kind) 10548 continue; 10549 10550 targ_name = btf_name_by_offset(targ_btf, t->name_off); 10551 if (!targ_name) 10552 continue; 10553 10554 /* the resched point is before strncmp to make sure that search 10555 * for non-existing name will have a chance to schedule(). 10556 */ 10557 cond_resched(); 10558 10559 if (strncmp(cands->name, targ_name, cands->name_len) != 0) 10560 continue; 10561 10562 targ_essent_len = bpf_core_essential_name_len(targ_name); 10563 if (targ_essent_len != cands->name_len) 10564 continue; 10565 10566 /* most of the time there is only one candidate for a given kind+name pair */ 10567 new_cands = kmalloc(sizeof_cands(cands->cnt + 1), GFP_KERNEL_ACCOUNT); 10568 if (!new_cands) { 10569 bpf_free_cands(cands); 10570 return ERR_PTR(-ENOMEM); 10571 } 10572 10573 memcpy(new_cands, cands, sizeof_cands(cands->cnt)); 10574 bpf_free_cands(cands); 10575 cands = new_cands; 10576 cands->cands[cands->cnt].btf = targ_btf; 10577 cands->cands[cands->cnt].id = i; 10578 cands->cnt++; 10579 } 10580 return cands; 10581 } 10582 10583 static struct bpf_cand_cache * 10584 bpf_core_find_cands(struct bpf_core_ctx *ctx, u32 local_type_id) 10585 { 10586 struct bpf_cand_cache *cands, *cc, local_cand = {}; 10587 const struct btf *local_btf = ctx->btf; 10588 const struct btf_type *local_type; 10589 const struct btf *main_btf; 10590 size_t local_essent_len; 10591 struct btf *mod_btf; 10592 const char *name; 10593 int id; 10594 10595 main_btf = bpf_get_btf_vmlinux(); 10596 if (IS_ERR(main_btf)) 10597 return ERR_CAST(main_btf); 10598 if (!main_btf) 10599 return ERR_PTR(-EINVAL); 10600 10601 local_type = btf_type_by_id(local_btf, local_type_id); 10602 if (!local_type) 10603 return ERR_PTR(-EINVAL); 10604 10605 name = btf_name_by_offset(local_btf, local_type->name_off); 10606 if (str_is_empty(name)) 10607 return ERR_PTR(-EINVAL); 10608 local_essent_len = bpf_core_essential_name_len(name); 10609 10610 cands = &local_cand; 10611 cands->name = name; 10612 cands->kind = btf_kind(local_type); 10613 cands->name_len = local_essent_len; 10614 10615 cc = check_cand_cache(cands, vmlinux_cand_cache, VMLINUX_CAND_CACHE_SIZE); 10616 /* cands is a pointer to stack here */ 10617 if (cc) { 10618 if (cc->cnt) 10619 return cc; 10620 goto check_modules; 10621 } 10622 10623 /* Attempt to find target candidates in vmlinux BTF first */ 10624 cands = bpf_core_add_cands(cands, main_btf, btf_named_start_id(main_btf, true)); 10625 if (IS_ERR(cands)) 10626 return ERR_CAST(cands); 10627 10628 /* cands is a pointer to kmalloced memory here if cands->cnt > 0 */ 10629 10630 /* populate cache even when cands->cnt == 0 */ 10631 cc = populate_cand_cache(cands, vmlinux_cand_cache, VMLINUX_CAND_CACHE_SIZE); 10632 if (IS_ERR(cc)) 10633 return ERR_CAST(cc); 10634 10635 /* if vmlinux BTF has any candidate, don't go for module BTFs */ 10636 if (cc->cnt) 10637 return cc; 10638 10639 check_modules: 10640 /* cands is a pointer to stack here and cands->cnt == 0 */ 10641 cc = check_cand_cache(cands, module_cand_cache, MODULE_CAND_CACHE_SIZE); 10642 if (cc) 10643 /* if cache has it return it even if cc->cnt == 0 */ 10644 return cc; 10645 10646 /* If candidate is not found in vmlinux's BTF then search in module's BTFs */ 10647 spin_lock_bh(&btf_idr_lock); 10648 idr_for_each_entry(&btf_idr, mod_btf, id) { 10649 if (!btf_is_module(mod_btf)) 10650 continue; 10651 /* linear search could be slow hence unlock/lock 10652 * the IDR to avoiding holding it for too long 10653 */ 10654 btf_get(mod_btf); 10655 spin_unlock_bh(&btf_idr_lock); 10656 cands = bpf_core_add_cands(cands, mod_btf, btf_named_start_id(mod_btf, true)); 10657 btf_put(mod_btf); 10658 if (IS_ERR(cands)) 10659 return ERR_CAST(cands); 10660 spin_lock_bh(&btf_idr_lock); 10661 } 10662 spin_unlock_bh(&btf_idr_lock); 10663 /* cands is a pointer to kmalloced memory here if cands->cnt > 0 10664 * or pointer to stack if cands->cnd == 0. 10665 * Copy it into the cache even when cands->cnt == 0 and 10666 * return the result. 10667 */ 10668 return populate_cand_cache(cands, module_cand_cache, MODULE_CAND_CACHE_SIZE); 10669 } 10670 10671 int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo, 10672 int relo_idx, void *insn) 10673 { 10674 bool need_cands = relo->kind != BPF_CORE_TYPE_ID_LOCAL; 10675 struct bpf_core_cand_list cands = {}; 10676 struct bpf_core_relo_res targ_res; 10677 struct bpf_core_spec *specs; 10678 const struct btf_type *type; 10679 int err; 10680 10681 /* ~4k of temp memory necessary to convert LLVM spec like "0:1:0:5" 10682 * into arrays of btf_ids of struct fields and array indices. 10683 */ 10684 specs = kzalloc_objs(*specs, 3, GFP_KERNEL_ACCOUNT); 10685 if (!specs) 10686 return -ENOMEM; 10687 10688 type = btf_type_by_id(ctx->btf, relo->type_id); 10689 if (!type) { 10690 bpf_log(ctx->log, "relo #%u: bad type id %u\n", 10691 relo_idx, relo->type_id); 10692 kfree(specs); 10693 return -EINVAL; 10694 } 10695 10696 if (need_cands) { 10697 struct bpf_cand_cache *cc; 10698 int i; 10699 10700 mutex_lock(&cand_cache_mutex); 10701 cc = bpf_core_find_cands(ctx, relo->type_id); 10702 if (IS_ERR(cc)) { 10703 bpf_log(ctx->log, "target candidate search failed for %d\n", 10704 relo->type_id); 10705 err = PTR_ERR(cc); 10706 goto out; 10707 } 10708 if (cc->cnt) { 10709 cands.cands = kzalloc_objs(*cands.cands, cc->cnt, 10710 GFP_KERNEL_ACCOUNT); 10711 if (!cands.cands) { 10712 err = -ENOMEM; 10713 goto out; 10714 } 10715 } 10716 for (i = 0; i < cc->cnt; i++) { 10717 bpf_log(ctx->log, 10718 "CO-RE relocating %s %s: found target candidate [%d]\n", 10719 btf_kind_str[cc->kind], cc->name, cc->cands[i].id); 10720 cands.cands[i].btf = cc->cands[i].btf; 10721 cands.cands[i].id = cc->cands[i].id; 10722 } 10723 cands.len = cc->cnt; 10724 /* cand_cache_mutex needs to span the cache lookup and 10725 * copy of btf pointer into bpf_core_cand_list, 10726 * since module can be unloaded while bpf_core_calc_relo_insn 10727 * is working with module's btf. 10728 */ 10729 } 10730 10731 err = bpf_core_calc_relo_insn((void *)ctx->log, relo, relo_idx, ctx->btf, &cands, specs, 10732 &targ_res); 10733 if (err) 10734 goto out; 10735 10736 err = bpf_core_patch_insn((void *)ctx->log, insn, relo->insn_off / 8, relo, relo_idx, 10737 &targ_res); 10738 10739 out: 10740 kfree(specs); 10741 if (need_cands) { 10742 kfree(cands.cands); 10743 mutex_unlock(&cand_cache_mutex); 10744 if (ctx->log->level & BPF_LOG_LEVEL2) 10745 print_cand_cache(ctx->log); 10746 } 10747 return err; 10748 } 10749 10750 bool btf_nested_type_is_trusted(struct bpf_verifier_log *log, 10751 const struct bpf_reg_state *reg, 10752 const char *field_name, u32 btf_id, const char *suffix) 10753 { 10754 struct btf *btf = reg->btf; 10755 const struct btf_type *walk_type, *safe_type; 10756 const char *tname; 10757 char safe_tname[64]; 10758 long ret, safe_id; 10759 const struct btf_member *member; 10760 u32 i; 10761 10762 walk_type = btf_type_by_id(btf, reg->btf_id); 10763 if (!walk_type) 10764 return false; 10765 10766 tname = btf_name_by_offset(btf, walk_type->name_off); 10767 10768 ret = snprintf(safe_tname, sizeof(safe_tname), "%s%s", tname, suffix); 10769 if (ret >= sizeof(safe_tname)) 10770 return false; 10771 10772 safe_id = btf_find_by_name_kind(btf, safe_tname, BTF_INFO_KIND(walk_type->info)); 10773 if (safe_id < 0) 10774 return false; 10775 10776 safe_type = btf_type_by_id(btf, safe_id); 10777 if (!safe_type) 10778 return false; 10779 10780 for_each_member(i, safe_type, member) { 10781 const char *m_name = __btf_name_by_offset(btf, member->name_off); 10782 const struct btf_type *mtype = btf_type_by_id(btf, member->type); 10783 u32 id; 10784 10785 if (!btf_type_is_ptr(mtype)) 10786 continue; 10787 10788 btf_type_skip_modifiers(btf, mtype->type, &id); 10789 /* If we match on both type and name, the field is considered trusted. */ 10790 if (btf_id == id && !strcmp(field_name, m_name)) 10791 return true; 10792 } 10793 10794 return false; 10795 } 10796 10797 bool btf_type_ids_nocast_alias(struct bpf_verifier_log *log, 10798 const struct btf *reg_btf, u32 reg_id, 10799 const struct btf *arg_btf, u32 arg_id) 10800 { 10801 const char *reg_name, *arg_name, *search_needle; 10802 const struct btf_type *reg_type, *arg_type; 10803 int reg_len, arg_len, cmp_len; 10804 size_t pattern_len = sizeof(NOCAST_ALIAS_SUFFIX) - sizeof(char); 10805 10806 reg_type = btf_type_by_id(reg_btf, reg_id); 10807 if (!reg_type) 10808 return false; 10809 10810 arg_type = btf_type_by_id(arg_btf, arg_id); 10811 if (!arg_type) 10812 return false; 10813 10814 reg_name = btf_name_by_offset(reg_btf, reg_type->name_off); 10815 arg_name = btf_name_by_offset(arg_btf, arg_type->name_off); 10816 10817 reg_len = strlen(reg_name); 10818 arg_len = strlen(arg_name); 10819 10820 /* Exactly one of the two type names may be suffixed with ___init, so 10821 * if the strings are the same size, they can't possibly be no-cast 10822 * aliases of one another. If you have two of the same type names, e.g. 10823 * they're both nf_conn___init, it would be improper to return true 10824 * because they are _not_ no-cast aliases, they are the same type. 10825 */ 10826 if (reg_len == arg_len) 10827 return false; 10828 10829 /* Either of the two names must be the other name, suffixed with ___init. */ 10830 if ((reg_len != arg_len + pattern_len) && 10831 (arg_len != reg_len + pattern_len)) 10832 return false; 10833 10834 if (reg_len < arg_len) { 10835 search_needle = strstr(arg_name, NOCAST_ALIAS_SUFFIX); 10836 cmp_len = reg_len; 10837 } else { 10838 search_needle = strstr(reg_name, NOCAST_ALIAS_SUFFIX); 10839 cmp_len = arg_len; 10840 } 10841 10842 if (!search_needle) 10843 return false; 10844 10845 /* ___init suffix must come at the end of the name */ 10846 if (*(search_needle + pattern_len) != '\0') 10847 return false; 10848 10849 return !strncmp(reg_name, arg_name, cmp_len); 10850 } 10851 10852 #ifdef CONFIG_BPF_JIT 10853 static int 10854 btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, 10855 struct bpf_verifier_log *log) 10856 { 10857 struct btf_struct_ops_tab *tab, *new_tab; 10858 int cgroup_atype; 10859 int i, err; 10860 10861 tab = btf->struct_ops_tab; 10862 if (!tab) { 10863 tab = kzalloc_flex(*tab, ops, 4); 10864 if (!tab) 10865 return -ENOMEM; 10866 tab->capacity = 4; 10867 btf->struct_ops_tab = tab; 10868 } 10869 10870 cgroup_atype = st_ops->cgroup_atype; 10871 for (i = 0; i < tab->cnt; i++) 10872 if (tab->ops[i].st_ops == st_ops || 10873 (cgroup_atype && cgroup_atype == tab->ops[i].st_ops->cgroup_atype)) 10874 return -EEXIST; 10875 10876 if (tab->cnt == tab->capacity) { 10877 new_tab = krealloc(tab, 10878 struct_size(tab, ops, tab->capacity * 2), 10879 GFP_KERNEL); 10880 if (!new_tab) 10881 return -ENOMEM; 10882 tab = new_tab; 10883 tab->capacity *= 2; 10884 btf->struct_ops_tab = tab; 10885 } 10886 10887 tab->ops[btf->struct_ops_tab->cnt].st_ops = st_ops; 10888 10889 err = bpf_struct_ops_desc_init(&tab->ops[btf->struct_ops_tab->cnt], btf, log); 10890 if (err) 10891 return err; 10892 10893 if (cgroup_atype) { 10894 /* 10895 * Cgroup struct_ops callers hold the RCU read lock around the 10896 * entire trampoline call, including its trailing instructions. 10897 * A regular RCU grace period therefore protects both the kdata 10898 * and the trampoline image, so a tasks RCU grace period is not 10899 * needed. 10900 */ 10901 if (!cgroup_bpf_is_struct_ops_atype(cgroup_atype) || 10902 st_ops->reg || st_ops->unreg || st_ops->free_after_tasks_rcu_gp) { 10903 bpf_struct_ops_desc_release(&tab->ops[btf->struct_ops_tab->cnt]); 10904 return -EINVAL; 10905 } 10906 10907 /* 10908 * There is no need to unregister from the cgroup when btf_free() 10909 * runs. No struct_ops map or cgroup link can be created once its 10910 * BTF is gone. 10911 */ 10912 cgroup_bpf_struct_ops_register(cgroup_atype, 10913 tab->ops[btf->struct_ops_tab->cnt].type_id, 10914 st_ops->cfi_stubs, 10915 st_ops->free_after_mult_rcu_gp); 10916 } 10917 10918 btf->struct_ops_tab->cnt++; 10919 10920 return 0; 10921 } 10922 10923 const struct bpf_struct_ops_desc * 10924 bpf_struct_ops_find_value(struct btf *btf, u32 value_id) 10925 { 10926 const struct bpf_struct_ops_desc *st_ops_list; 10927 unsigned int i; 10928 u32 cnt; 10929 10930 if (!value_id) 10931 return NULL; 10932 if (!btf->struct_ops_tab) 10933 return NULL; 10934 10935 cnt = btf->struct_ops_tab->cnt; 10936 st_ops_list = btf->struct_ops_tab->ops; 10937 for (i = 0; i < cnt; i++) { 10938 if (st_ops_list[i].value_id == value_id) 10939 return &st_ops_list[i]; 10940 } 10941 10942 return NULL; 10943 } 10944 10945 const struct bpf_struct_ops_desc * 10946 bpf_struct_ops_find(struct btf *btf, u32 type_id) 10947 { 10948 const struct bpf_struct_ops_desc *st_ops_list; 10949 unsigned int i; 10950 u32 cnt; 10951 10952 if (!type_id) 10953 return NULL; 10954 if (!btf->struct_ops_tab) 10955 return NULL; 10956 10957 cnt = btf->struct_ops_tab->cnt; 10958 st_ops_list = btf->struct_ops_tab->ops; 10959 for (i = 0; i < cnt; i++) { 10960 if (st_ops_list[i].type_id == type_id) 10961 return &st_ops_list[i]; 10962 } 10963 10964 return NULL; 10965 } 10966 10967 int __register_bpf_struct_ops(struct bpf_struct_ops *st_ops) 10968 { 10969 struct bpf_verifier_log *log; 10970 struct btf *btf; 10971 int err = 0; 10972 10973 btf = btf_get_module_btf(st_ops->owner); 10974 if (!btf) 10975 return check_btf_kconfigs(st_ops->owner, "struct_ops"); 10976 if (IS_ERR(btf)) 10977 return PTR_ERR(btf); 10978 10979 log = kzalloc_obj(*log, GFP_KERNEL | __GFP_NOWARN); 10980 if (!log) { 10981 err = -ENOMEM; 10982 goto errout; 10983 } 10984 10985 log->level = BPF_LOG_KERNEL; 10986 10987 err = btf_add_struct_ops(btf, st_ops, log); 10988 10989 errout: 10990 kfree(log); 10991 btf_put(btf); 10992 10993 return err; 10994 } 10995 EXPORT_SYMBOL_GPL(__register_bpf_struct_ops); 10996 #endif 10997 10998 bool btf_param_match_suffix(const struct btf *btf, 10999 const struct btf_param *arg, 11000 const char *suffix) 11001 { 11002 int suffix_len = strlen(suffix), len; 11003 const char *param_name; 11004 11005 /* In the future, this can be ported to use BTF tagging */ 11006 param_name = btf_name_by_offset(btf, arg->name_off); 11007 if (str_is_empty(param_name)) 11008 return false; 11009 len = strlen(param_name); 11010 if (len <= suffix_len) 11011 return false; 11012 param_name += len - suffix_len; 11013 return !strncmp(param_name, suffix, suffix_len); 11014 } 11015