Lines Matching +full:multi +full:- +full:ported
1 // SPDX-License-Identifier: GPL-2.0
74 * NOTE that we cannot assume any reference-order.
80 * object describing "void *". This type-reference is done
87 * - Each line started with "[?]" is a btf_type object
88 * - [?] is the type_id of the btf_type object.
89 * - CONST/PTR is the BTF_KIND_XXX
90 * - "(anon)" is the name of the type. It just
92 * - type_id=XXX is the 'u32 type' in btf_type
116 * an array: "btf->types".
126 * check this type-reference in the first pass.
137 * 1) does exist in the BTF (i.e. in btf->types[])
170 * BTF_KIND_CONST -> BTF_KIND_PTR -> BTF_KIND_CONST -> BTF_KIND_PTR +
172 * +-----------------------------------------+
177 #define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1)
277 __u32 *base_id_map; /* map from distilled base BTF -> vmlinux BTF ids */
348 return btf_kind_str[BTF_INFO_KIND(t->info)]; in btf_type_str()
356 * 128-bit int); if we are at the end of our safe buffer and have
377 * One challenge with showing nested data is we want to skip 0-valued
382 * pass is signalled by show->state.depth_check being set, and if we
383 * encounter a non-zero value we set show->state.depth_to_show to
401 * as we traverse the object's data. skbuff-like semantics are
404 * - obj.head points to the start of the toplevel object for display
405 * - obj.size is the size of the toplevel object
406 * - obj.data points to the current point in the original data at
429 int status; /* non-zero for error */
478 * type through t->type AND its size cannot in btf_type_is_modifier()
479 * be determined without following the t->type. in btf_type_is_modifier()
484 switch (BTF_INFO_KIND(t->info)) { in btf_type_is_modifier()
503 return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC; in btf_type_is_datasec()
508 return BTF_INFO_KIND(t->info) == BTF_KIND_DECL_TAG; in btf_type_is_decl_tag()
531 return btf->kernel_btf && !btf->base_btf; in btf_is_vmlinux()
539 total += btf->nr_types; in btf_nr_types()
540 btf = btf->base_btf; in btf_nr_types()
555 if (BTF_INFO_KIND(t->info) != kind) in btf_find_by_name_kind()
558 tname = btf_name_by_offset(btf, t->name_off); in btf_find_by_name_kind()
563 return -ENOENT; in btf_find_by_name_kind()
576 return -EINVAL; in bpf_find_btf_id()
616 id = t->type; in btf_type_skip_modifiers()
617 t = btf_type_by_id(btf, t->type); in btf_type_skip_modifiers()
635 return btf_type_skip_modifiers(btf, t->type, res_id); in btf_type_resolve_ptr()
665 * another type (through member->type).
671 * btf_type_is_array() because its element (array->type)
674 * member-type repeated by array->nelems of times.
688 /* t->size can be used */
691 switch (BTF_INFO_KIND(t->info)) { in btf_type_has_size()
751 return kind_ops[BTF_INFO_KIND(t->info)]; in btf_type_ops()
759 while (offset < btf->start_str_off) in btf_name_offset_valid()
760 btf = btf->base_btf; in btf_name_offset_valid()
762 offset -= btf->start_str_off; in btf_name_offset_valid()
763 return offset < btf->hdr.str_len; in btf_name_offset_valid()
778 while (offset < btf->start_str_off) in btf_str_by_offset()
779 btf = btf->base_btf; in btf_str_by_offset()
781 offset -= btf->start_str_off; in btf_str_by_offset()
782 if (offset < btf->hdr.str_len) in btf_str_by_offset()
783 return &btf->strings[offset]; in btf_str_by_offset()
838 return name ?: "(invalid-name-offset)"; in __btf_name_by_offset()
848 while (type_id < btf->start_id) in btf_type_by_id()
849 btf = btf->base_btf; in btf_type_by_id()
851 type_id -= btf->start_id; in btf_type_by_id()
852 if (type_id >= btf->nr_types) in btf_type_by_id()
854 return btf->types[type_id]; in btf_type_by_id()
893 id = m->type; in btf_member_is_reg_int()
901 u32 bitfield_size = BTF_MEMBER_BITFIELD_SIZE(m->offset); in btf_member_is_reg_int()
902 u32 bit_offset = BTF_MEMBER_BIT_OFFSET(m->offset); in btf_member_is_reg_int()
913 BITS_PER_BYTE_MASKED(m->offset) || in btf_member_is_reg_int()
914 BITS_ROUNDUP_BYTES(m->offset) != expected_offset || in btf_member_is_reg_int()
929 BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF) { in btf_type_skip_qualifiers()
930 t = btf_type_by_id(btf, t->type); in btf_type_skip_qualifiers()
941 * Populate show->state.name with type name information.
955 const struct btf_member *m = show->state.member; in btf_show_name()
958 u32 id = show->state.type_id; in btf_show_name()
964 show->state.name[0] = '\0'; in btf_show_name()
971 if (show->state.array_member) in btf_show_name()
976 member = btf_name_by_offset(show->btf, m->name_off); in btf_show_name()
978 id = m->type; in btf_show_name()
986 * in our show->state points at the resolved type of the typedef. in btf_show_name()
988 t = btf_type_by_id(show->btf, id); in btf_show_name()
1016 switch (BTF_INFO_KIND(t->info)) { in btf_show_name()
1019 name = btf_name_by_offset(show->btf, in btf_show_name()
1020 t->name_off); in btf_show_name()
1022 id = t->type; in btf_show_name()
1031 array_suffix -= 2; in btf_show_name()
1032 id = array->type; in btf_show_name()
1037 ptr_suffix -= 1; in btf_show_name()
1038 id = t->type; in btf_show_name()
1046 t = btf_type_skip_qualifiers(show->btf, id); in btf_show_name()
1053 name = btf_name_by_offset(show->btf, t->name_off); in btf_show_name()
1055 switch (BTF_INFO_KIND(t->info)) { in btf_show_name()
1058 prefix = BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT ? in btf_show_name()
1083 if (show->flags & BTF_SHOW_NONAME) in btf_show_name()
1084 snprintf(show->state.name, sizeof(show->state.name), "%s", in btf_show_name()
1087 snprintf(show->state.name, sizeof(show->state.name), in btf_show_name()
1102 return show->state.name; in btf_show_name()
1110 if ((indent - show->state.depth) >= indents) in __btf_show_indent()
1111 return indent - show->state.depth; in __btf_show_indent()
1117 return show->flags & BTF_SHOW_COMPACT ? "" : __btf_show_indent(show); in btf_show_indent()
1122 return show->flags & BTF_SHOW_COMPACT ? "" : "\n"; in btf_show_newline()
1127 if (show->state.depth == 0) in btf_show_delim()
1130 if ((show->flags & BTF_SHOW_COMPACT) && show->state.type && in btf_show_delim()
1131 BTF_INFO_KIND(show->state.type->info) == BTF_KIND_UNION) in btf_show_delim()
1141 if (!show->state.depth_check) { in btf_show()
1143 show->showfn(show, fmt, args); in btf_show()
1156 (show->flags & BTF_SHOW_ZERO) || \
1157 show->state.depth == 0) { \
1163 if (show->state.depth > show->state.depth_to_show) \
1164 show->state.depth_to_show = show->state.depth; \
1174 if (show->state.depth > show->state.depth_to_show) \
1175 show->state.depth_to_show = show->state.depth; \
1181 return show->obj.head + show->obj.size - data; in btf_show_obj_size_left()
1187 return data >= show->obj.data && in btf_show_obj_is_safe()
1188 (data + size) < (show->obj.data + BTF_SHOW_OBJ_SAFE_SIZE); in btf_show_obj_is_safe()
1200 return show->obj.safe + (data - show->obj.data); in __btf_show_obj_safe()
1205 * Return a safe-to-access version of data pointed to by @data.
1209 * If BTF_SHOW_UNSAFE is specified, just return data as-is; no
1228 * We use stack data as opposed to per-CPU buffers because the
1240 if (show->flags & BTF_SHOW_UNSAFE) in btf_show_obj_safe()
1243 rt = btf_resolve_size(show->btf, t, &size); in btf_show_obj_safe()
1245 show->state.status = PTR_ERR(rt); in btf_show_obj_safe()
1254 if (show->state.depth == 0) { in btf_show_obj_safe()
1255 show->obj.size = size; in btf_show_obj_safe()
1256 show->obj.head = data; in btf_show_obj_safe()
1275 * - the current type size is within the safe buffer; or in btf_show_obj_safe()
1276 * - at least BTF_SHOW_OBJ_BASE_TYPE_SIZE bytes are left in in btf_show_obj_safe()
1293 show->state.status = copy_from_kernel_nofault(show->obj.safe, in btf_show_obj_safe()
1295 if (!show->state.status) { in btf_show_obj_safe()
1296 show->obj.data = data; in btf_show_obj_safe()
1297 safe = show->obj.safe; in btf_show_obj_safe()
1312 show->state.type = t; in btf_show_start_type()
1313 show->state.type_id = type_id; in btf_show_start_type()
1314 show->state.name[0] = '\0'; in btf_show_start_type()
1321 show->state.type = NULL; in btf_show_end_type()
1322 show->state.type_id = 0; in btf_show_end_type()
1323 show->state.name[0] = '\0'; in btf_show_end_type()
1338 show->state.depth++; in btf_show_start_aggr_type()
1345 show->state.depth--; in btf_show_end_aggr_type()
1354 show->state.member = m; in btf_show_start_member()
1359 show->state.array_member = 1; in btf_show_start_array_member()
1365 show->state.member = NULL; in btf_show_end_member()
1370 show->state.array_member = 0; in btf_show_end_array_member()
1380 show->state.array_encoding = array_encoding; in btf_show_start_array_type()
1381 show->state.array_terminated = 0; in btf_show_start_array_type()
1387 show->state.array_encoding = 0; in btf_show_end_array_type()
1388 show->state.array_terminated = 0; in btf_show_end_array_type()
1418 struct bpf_verifier_log *log = &env->log; in btf_verifier_log()
1434 struct bpf_verifier_log *log = &env->log; in __btf_verifier_log_type()
1435 struct btf *btf = env->btf; in __btf_verifier_log_type()
1441 if (log->level == BPF_LOG_KERNEL) { in __btf_verifier_log_type()
1444 * Skip those prints for in-kernel BTF verification. in __btf_verifier_log_type()
1450 if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) in __btf_verifier_log_type()
1455 env->log_type_id, in __btf_verifier_log_type()
1457 __btf_name_by_offset(btf, t->name_off), in __btf_verifier_log_type()
1461 btf_type_ops(t)->log_details(env, t); in __btf_verifier_log_type()
1484 struct bpf_verifier_log *log = &env->log; in btf_verifier_log_member()
1485 struct btf *btf = env->btf; in btf_verifier_log_member()
1491 if (log->level == BPF_LOG_KERNEL) { in btf_verifier_log_member()
1496 if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) in btf_verifier_log_member()
1506 if (env->phase != CHECK_META) in btf_verifier_log_member()
1512 __btf_name_by_offset(btf, member->name_off), in btf_verifier_log_member()
1513 member->type, in btf_verifier_log_member()
1514 BTF_MEMBER_BITFIELD_SIZE(member->offset), in btf_verifier_log_member()
1515 BTF_MEMBER_BIT_OFFSET(member->offset)); in btf_verifier_log_member()
1518 __btf_name_by_offset(btf, member->name_off), in btf_verifier_log_member()
1519 member->type, member->offset); in btf_verifier_log_member()
1537 struct bpf_verifier_log *log = &env->log; in btf_verifier_log_vsi()
1542 if (log->level == BPF_LOG_KERNEL && !fmt) in btf_verifier_log_vsi()
1544 if (env->phase != CHECK_META) in btf_verifier_log_vsi()
1548 vsi->type, vsi->offset, vsi->size); in btf_verifier_log_vsi()
1562 struct bpf_verifier_log *log = &env->log; in btf_verifier_log_hdr()
1563 const struct btf *btf = env->btf; in btf_verifier_log_hdr()
1569 if (log->level == BPF_LOG_KERNEL) in btf_verifier_log_hdr()
1571 hdr = &btf->hdr; in btf_verifier_log_hdr()
1572 __btf_verifier_log(log, "magic: 0x%x\n", hdr->magic); in btf_verifier_log_hdr()
1573 __btf_verifier_log(log, "version: %u\n", hdr->version); in btf_verifier_log_hdr()
1574 __btf_verifier_log(log, "flags: 0x%x\n", hdr->flags); in btf_verifier_log_hdr()
1575 __btf_verifier_log(log, "hdr_len: %u\n", hdr->hdr_len); in btf_verifier_log_hdr()
1576 __btf_verifier_log(log, "type_off: %u\n", hdr->type_off); in btf_verifier_log_hdr()
1577 __btf_verifier_log(log, "type_len: %u\n", hdr->type_len); in btf_verifier_log_hdr()
1578 __btf_verifier_log(log, "str_off: %u\n", hdr->str_off); in btf_verifier_log_hdr()
1579 __btf_verifier_log(log, "str_len: %u\n", hdr->str_len); in btf_verifier_log_hdr()
1585 struct btf *btf = env->btf; in btf_add_type()
1587 if (btf->types_size == btf->nr_types) { in btf_add_type()
1593 if (btf->start_id + btf->types_size == BTF_MAX_TYPE) { in btf_add_type()
1595 return -E2BIG; in btf_add_type()
1598 expand_by = max_t(u32, btf->types_size >> 2, 16); in btf_add_type()
1600 btf->types_size + expand_by); in btf_add_type()
1605 return -ENOMEM; in btf_add_type()
1607 if (btf->nr_types == 0) { in btf_add_type()
1608 if (!btf->base_btf) { in btf_add_type()
1611 btf->nr_types++; in btf_add_type()
1614 memcpy(new_types, btf->types, in btf_add_type()
1615 sizeof(*btf->types) * btf->nr_types); in btf_add_type()
1618 kvfree(btf->types); in btf_add_type()
1619 btf->types = new_types; in btf_add_type()
1620 btf->types_size = new_size; in btf_add_type()
1623 btf->types[btf->nr_types++] = t; in btf_add_type()
1636 btf->id = id; in btf_alloc_id()
1641 return -ENOSPC; in btf_alloc_id()
1651 * In map-in-map, calling map_delete_elem() on outer in btf_free_id()
1660 idr_remove(&btf_idr, btf->id); in btf_free_id()
1666 struct btf_kfunc_set_tab *tab = btf->kfunc_set_tab; in btf_free_kfunc_set_tab()
1671 for (hook = 0; hook < ARRAY_SIZE(tab->sets); hook++) in btf_free_kfunc_set_tab()
1672 kfree(tab->sets[hook]); in btf_free_kfunc_set_tab()
1674 btf->kfunc_set_tab = NULL; in btf_free_kfunc_set_tab()
1679 struct btf_id_dtor_kfunc_tab *tab = btf->dtor_kfunc_tab; in btf_free_dtor_kfunc_tab()
1684 btf->dtor_kfunc_tab = NULL; in btf_free_dtor_kfunc_tab()
1693 for (i = 0; i < tab->cnt; i++) in btf_struct_metas_free()
1694 btf_record_free(tab->types[i].record); in btf_struct_metas_free()
1700 struct btf_struct_metas *tab = btf->struct_meta_tab; in btf_free_struct_meta_tab()
1703 btf->struct_meta_tab = NULL; in btf_free_struct_meta_tab()
1708 struct btf_struct_ops_tab *tab = btf->struct_ops_tab; in btf_free_struct_ops_tab()
1714 for (i = 0; i < tab->cnt; i++) in btf_free_struct_ops_tab()
1715 bpf_struct_ops_desc_release(&tab->ops[i]); in btf_free_struct_ops_tab()
1718 btf->struct_ops_tab = NULL; in btf_free_struct_ops_tab()
1727 kvfree(btf->types); in btf_free()
1728 kvfree(btf->resolved_sizes); in btf_free()
1729 kvfree(btf->resolved_ids); in btf_free()
1730 /* vmlinux does not allocate btf->data, it simply points it at in btf_free()
1734 kvfree(btf->data); in btf_free()
1735 kvfree(btf->base_id_map); in btf_free()
1748 return btf->name; in btf_get_name()
1753 refcount_inc(&btf->refcnt); in btf_get()
1758 if (btf && refcount_dec_and_test(&btf->refcnt)) { in btf_put()
1760 call_rcu(&btf->rcu, btf_free_rcu); in btf_put()
1766 return btf->base_btf; in btf_base_btf()
1771 return &btf->hdr; in btf_header()
1776 btf->base_btf = (struct btf *)base_btf; in btf_set_base_btf()
1777 btf->start_id = btf_nr_types(base_btf); in btf_set_base_btf()
1778 btf->start_str_off = base_btf->hdr.str_len; in btf_set_base_btf()
1783 struct btf *btf = env->btf; in env_resolve_init()
1784 u32 nr_types = btf->nr_types; in env_resolve_init()
1804 btf->resolved_sizes = resolved_sizes; in env_resolve_init()
1805 btf->resolved_ids = resolved_ids; in env_resolve_init()
1806 env->visit_states = visit_states; in env_resolve_init()
1814 return -ENOMEM; in env_resolve_init()
1819 kvfree(env->visit_states); in btf_verifier_env_free()
1826 switch (env->resolve_mode) { in env_type_is_resolve_sink()
1852 if (type_id < env->btf->start_id) in env_type_is_resolved()
1855 return env->visit_states[type_id - env->btf->start_id] == RESOLVED; in env_type_is_resolved()
1861 const struct btf *btf = env->btf; in env_stack_push()
1864 if (env->top_stack == MAX_RESOLVE_DEPTH) in env_stack_push()
1865 return -E2BIG; in env_stack_push()
1867 if (type_id < btf->start_id in env_stack_push()
1868 || env->visit_states[type_id - btf->start_id] != NOT_VISITED) in env_stack_push()
1869 return -EEXIST; in env_stack_push()
1871 env->visit_states[type_id - btf->start_id] = VISITED; in env_stack_push()
1873 v = &env->stack[env->top_stack++]; in env_stack_push()
1874 v->t = t; in env_stack_push()
1875 v->type_id = type_id; in env_stack_push()
1876 v->next_member = 0; in env_stack_push()
1878 if (env->resolve_mode == RESOLVE_TBD) { in env_stack_push()
1880 env->resolve_mode = RESOLVE_PTR; in env_stack_push()
1882 env->resolve_mode = RESOLVE_STRUCT_OR_ARRAY; in env_stack_push()
1891 env->stack[env->top_stack - 1].next_member = next_member; in env_stack_set_next_member()
1898 u32 type_id = env->stack[--(env->top_stack)].type_id; in env_stack_pop_resolved()
1899 struct btf *btf = env->btf; in env_stack_pop_resolved()
1901 type_id -= btf->start_id; /* adjust to local type id */ in env_stack_pop_resolved()
1902 btf->resolved_sizes[type_id] = resolved_size; in env_stack_pop_resolved()
1903 btf->resolved_ids[type_id] = resolved_type_id; in env_stack_pop_resolved()
1904 env->visit_states[type_id] = RESOLVED; in env_stack_pop_resolved()
1909 return env->top_stack ? &env->stack[env->top_stack - 1] : NULL; in env_stack_peak()
1912 /* Resolve the size of a passed-in "type"
1942 switch (BTF_INFO_KIND(type->info)) { in __btf_resolve_size()
1943 /* type->size can be used */ in __btf_resolve_size()
1950 size = type->size; in __btf_resolve_size()
1963 id = type->type; in __btf_resolve_size()
1964 type = btf_type_by_id(btf, type->type); in __btf_resolve_size()
1971 if (nelems && array->nelems > U32_MAX / nelems) in __btf_resolve_size()
1972 return ERR_PTR(-EINVAL); in __btf_resolve_size()
1973 nelems *= array->nelems; in __btf_resolve_size()
1974 type = btf_type_by_id(btf, array->type); in __btf_resolve_size()
1979 return ERR_PTR(-EINVAL); in __btf_resolve_size()
1983 return ERR_PTR(-EINVAL); in __btf_resolve_size()
1987 return ERR_PTR(-EINVAL); in __btf_resolve_size()
1995 *elem_id = array ? array->type : 0; in __btf_resolve_size()
2011 while (type_id < btf->start_id) in btf_resolved_type_id()
2012 btf = btf->base_btf; in btf_resolved_type_id()
2014 return btf->resolved_ids[type_id - btf->start_id]; in btf_resolved_type_id()
2027 while (type_id < btf->start_id) in btf_resolved_type_size()
2028 btf = btf->base_btf; in btf_resolved_type_size()
2030 return btf->resolved_sizes[type_id - btf->start_id]; in btf_resolved_type_size()
2045 size = size_type->size; in btf_type_id_size()
2060 size = size_type->size; in btf_type_id_size()
2083 return -EINVAL; in btf_df_check_member()
2093 return -EINVAL; in btf_df_check_kflag_member()
2104 if (BTF_MEMBER_BITFIELD_SIZE(member->offset)) { in btf_generic_check_kflag_member()
2107 return -EINVAL; in btf_generic_check_kflag_member()
2110 /* bitfield size is 0, so member->offset represents bit offset only. in btf_generic_check_kflag_member()
2113 return btf_type_ops(member_type)->check_member(env, struct_type, in btf_generic_check_kflag_member()
2121 btf_verifier_log_basic(env, v->t, "Unsupported resolve"); in btf_df_resolve()
2122 return -EINVAL; in btf_df_resolve()
2129 btf_show(show, "<unsupported kind:%u>", BTF_INFO_KIND(t->info)); in btf_df_show()
2138 u32 struct_bits_off = member->offset; in btf_int_check_member()
2139 u32 struct_size = struct_type->size; in btf_int_check_member()
2143 if (U32_MAX - struct_bits_off < BTF_INT_OFFSET(int_data)) { in btf_int_check_member()
2146 return -EINVAL; in btf_int_check_member()
2157 return -EINVAL; in btf_int_check_member()
2161 struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) { in btf_int_check_member()
2164 return -EINVAL; in btf_int_check_member()
2177 u32 struct_size = struct_type->size; in btf_int_check_kflag_member()
2184 return -EINVAL; in btf_int_check_kflag_member()
2188 nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset); in btf_int_check_kflag_member()
2189 struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset); in btf_int_check_kflag_member()
2198 return -EINVAL; in btf_int_check_kflag_member()
2205 return -EINVAL; in btf_int_check_kflag_member()
2213 return -EINVAL; in btf_int_check_kflag_member()
2217 struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) { in btf_int_check_kflag_member()
2220 return -EINVAL; in btf_int_check_kflag_member()
2237 return -EINVAL; in btf_int_check_meta()
2242 return -EINVAL; in btf_int_check_meta()
2247 return -EINVAL; in btf_int_check_meta()
2254 return -EINVAL; in btf_int_check_meta()
2262 return -EINVAL; in btf_int_check_meta()
2265 if (BITS_ROUNDUP_BYTES(nr_bits) > t->size) { in btf_int_check_meta()
2267 return -EINVAL; in btf_int_check_meta()
2282 return -ENOTSUPP; in btf_int_check_meta()
2297 t->size, BTF_INT_OFFSET(int_data), in btf_int_log()
2340 /* shake out un-needed bits by shift/or operations */ in btf_int128_shift()
2342 upper_num = lower_num << (left_shift_bits - 64); in btf_int128_shift()
2346 (lower_num >> (64 - left_shift_bits)); in btf_int128_shift()
2351 lower_num = upper_num >> (right_shift_bits - 64); in btf_int128_shift()
2355 (upper_num << (64 - right_shift_bits)); in btf_int128_shift()
2384 left_shift_bits = BITS_PER_U128 - nr_copy_bits; in btf_bitfield_show()
2386 right_shift_bits = BITS_PER_U128 - nr_bits; in btf_bitfield_show()
2455 if (show->state.array_encoding == BTF_INT_CHAR) { in btf_int_show()
2457 if (show->state.array_terminated) in btf_int_show()
2460 show->state.array_terminated = 1; in btf_int_show()
2497 u32 resolved_type_id = member->type; in btf_modifier_check_member()
2499 struct btf *btf = env->btf; in btf_modifier_check_member()
2505 return -EINVAL; in btf_modifier_check_member()
2511 return btf_type_ops(resolved_type)->check_member(env, struct_type, in btf_modifier_check_member()
2522 u32 resolved_type_id = member->type; in btf_modifier_check_kflag_member()
2524 struct btf *btf = env->btf; in btf_modifier_check_kflag_member()
2530 return -EINVAL; in btf_modifier_check_kflag_member()
2536 return btf_type_ops(resolved_type)->check_kflag_member(env, struct_type, in btf_modifier_check_kflag_member()
2548 struct_size = struct_type->size; in btf_ptr_check_member()
2549 struct_bits_off = member->offset; in btf_ptr_check_member()
2555 return -EINVAL; in btf_ptr_check_member()
2558 if (struct_size - bytes_offset < sizeof(void *)) { in btf_ptr_check_member()
2561 return -EINVAL; in btf_ptr_check_member()
2575 return -EINVAL; in btf_ref_type_check_meta()
2580 return -EINVAL; in btf_ref_type_check_meta()
2583 if (!BTF_TYPE_ID_VALID(t->type)) { in btf_ref_type_check_meta()
2585 return -EINVAL; in btf_ref_type_check_meta()
2591 if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF) { in btf_ref_type_check_meta()
2592 if (!t->name_off || in btf_ref_type_check_meta()
2593 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_ref_type_check_meta()
2595 return -EINVAL; in btf_ref_type_check_meta()
2597 } else if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPE_TAG) { in btf_ref_type_check_meta()
2598 value = btf_name_by_offset(env->btf, t->name_off); in btf_ref_type_check_meta()
2601 return -EINVAL; in btf_ref_type_check_meta()
2604 if (t->name_off) { in btf_ref_type_check_meta()
2606 return -EINVAL; in btf_ref_type_check_meta()
2618 const struct btf_type *t = v->t; in btf_modifier_resolve()
2620 u32 next_type_id = t->type; in btf_modifier_resolve()
2621 struct btf *btf = env->btf; in btf_modifier_resolve()
2625 btf_verifier_log_type(env, v->t, "Invalid type_id"); in btf_modifier_resolve()
2626 return -EINVAL; in btf_modifier_resolve()
2636 * save us a few type-following when we use it later (e.g. in in btf_modifier_resolve()
2647 btf_verifier_log_type(env, v->t, "Invalid type_id"); in btf_modifier_resolve()
2648 return -EINVAL; in btf_modifier_resolve()
2661 const struct btf_type *t = v->t; in btf_var_resolve()
2662 u32 next_type_id = t->type; in btf_var_resolve()
2663 struct btf *btf = env->btf; in btf_var_resolve()
2667 btf_verifier_log_type(env, v->t, "Invalid type_id"); in btf_var_resolve()
2668 return -EINVAL; in btf_var_resolve()
2694 btf_verifier_log_type(env, v->t, "Invalid type_id"); in btf_var_resolve()
2695 return -EINVAL; in btf_var_resolve()
2707 const struct btf_type *t = v->t; in btf_ptr_resolve()
2708 u32 next_type_id = t->type; in btf_ptr_resolve()
2709 struct btf *btf = env->btf; in btf_ptr_resolve()
2713 btf_verifier_log_type(env, v->t, "Invalid type_id"); in btf_ptr_resolve()
2714 return -EINVAL; in btf_ptr_resolve()
2723 * to a ptr (last-resolved-ptr). in btf_ptr_resolve()
2725 * We now need to continue from the last-resolved-ptr to in btf_ptr_resolve()
2726 * ensure the last-resolved-ptr will not referring back to in btf_ptr_resolve()
2750 btf_verifier_log_type(env, v->t, "Invalid type_id"); in btf_ptr_resolve()
2751 return -EINVAL; in btf_ptr_resolve()
2765 if (btf->resolved_ids) in btf_modifier_show()
2770 btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); in btf_modifier_show()
2779 btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); in btf_var_show()
2793 if (show->flags & BTF_SHOW_PTR_RAW) in btf_ptr_show()
2803 btf_verifier_log(env, "type_id=%u", t->type); in btf_ref_type_log()
2830 return -EINVAL; in btf_fwd_check_meta()
2833 if (t->type) { in btf_fwd_check_meta()
2835 return -EINVAL; in btf_fwd_check_meta()
2839 if (!t->name_off || in btf_fwd_check_meta()
2840 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_fwd_check_meta()
2842 return -EINVAL; in btf_fwd_check_meta()
2870 u32 struct_bits_off = member->offset; in btf_array_check_member()
2873 struct btf *btf = env->btf; in btf_array_check_member()
2878 return -EINVAL; in btf_array_check_member()
2881 array_type_id = member->type; in btf_array_check_member()
2883 struct_size = struct_type->size; in btf_array_check_member()
2885 if (struct_size - bytes_offset < array_size) { in btf_array_check_member()
2888 return -EINVAL; in btf_array_check_member()
2905 return -EINVAL; in btf_array_check_meta()
2909 if (t->name_off) { in btf_array_check_meta()
2911 return -EINVAL; in btf_array_check_meta()
2916 return -EINVAL; in btf_array_check_meta()
2921 return -EINVAL; in btf_array_check_meta()
2924 if (t->size) { in btf_array_check_meta()
2926 return -EINVAL; in btf_array_check_meta()
2930 * so !array->type and !array->index_type are not allowed. in btf_array_check_meta()
2932 if (!array->type || !BTF_TYPE_ID_VALID(array->type)) { in btf_array_check_meta()
2934 return -EINVAL; in btf_array_check_meta()
2937 if (!array->index_type || !BTF_TYPE_ID_VALID(array->index_type)) { in btf_array_check_meta()
2939 return -EINVAL; in btf_array_check_meta()
2950 const struct btf_array *array = btf_type_array(v->t); in btf_array_resolve()
2953 struct btf *btf = env->btf; in btf_array_resolve()
2956 /* Check array->index_type */ in btf_array_resolve()
2957 index_type_id = array->index_type; in btf_array_resolve()
2961 btf_verifier_log_type(env, v->t, "Invalid index"); in btf_array_resolve()
2962 return -EINVAL; in btf_array_resolve()
2972 btf_verifier_log_type(env, v->t, "Invalid index"); in btf_array_resolve()
2973 return -EINVAL; in btf_array_resolve()
2976 /* Check array->type */ in btf_array_resolve()
2977 elem_type_id = array->type; in btf_array_resolve()
2981 btf_verifier_log_type(env, v->t, in btf_array_resolve()
2983 return -EINVAL; in btf_array_resolve()
2992 btf_verifier_log_type(env, v->t, "Invalid elem"); in btf_array_resolve()
2993 return -EINVAL; in btf_array_resolve()
2997 btf_verifier_log_type(env, v->t, "Invalid array of int"); in btf_array_resolve()
2998 return -EINVAL; in btf_array_resolve()
3001 if (array->nelems && elem_size > U32_MAX / array->nelems) { in btf_array_resolve()
3002 btf_verifier_log_type(env, v->t, in btf_array_resolve()
3004 return -EINVAL; in btf_array_resolve()
3007 env_stack_pop_resolved(env, elem_type_id, elem_size * array->nelems); in btf_array_resolve()
3018 array->type, array->index_type, array->nelems); in btf_array_log()
3031 elem_type_id = array->type; in __btf_array_show()
3034 elem_size = elem_type->size; in __btf_array_show()
3057 for (i = 0; i < array->nelems; i++) { in __btf_array_show()
3061 elem_ops->show(btf, elem_type, elem_type_id, data, in __btf_array_show()
3067 if (show->state.array_terminated) in __btf_array_show()
3078 const struct btf_member *m = show->state.member; in btf_array_show()
3081 * First check if any members would be shown (are non-zero). in btf_array_show()
3083 * details on how this works at a high-level. in btf_array_show()
3085 if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) { in btf_array_show()
3086 if (!show->state.depth_check) { in btf_array_show()
3087 show->state.depth_check = show->state.depth + 1; in btf_array_show()
3088 show->state.depth_to_show = 0; in btf_array_show()
3091 show->state.member = m; in btf_array_show()
3093 if (show->state.depth_check != show->state.depth + 1) in btf_array_show()
3095 show->state.depth_check = 0; in btf_array_show()
3097 if (show->state.depth_to_show <= show->state.depth) in btf_array_show()
3101 * non-zero array member(s). in btf_array_show()
3121 u32 struct_bits_off = member->offset; in btf_struct_check_member()
3127 return -EINVAL; in btf_struct_check_member()
3130 struct_size = struct_type->size; in btf_struct_check_member()
3132 if (struct_size - bytes_offset < member_type->size) { in btf_struct_check_member()
3135 return -EINVAL; in btf_struct_check_member()
3145 bool is_union = BTF_INFO_KIND(t->info) == BTF_KIND_UNION; in btf_struct_check_meta()
3148 struct btf *btf = env->btf; in btf_struct_check_meta()
3149 u32 struct_size = t->size; in btf_struct_check_meta()
3158 return -EINVAL; in btf_struct_check_meta()
3162 if (t->name_off && in btf_struct_check_meta()
3163 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_struct_check_meta()
3165 return -EINVAL; in btf_struct_check_meta()
3172 if (!btf_name_offset_valid(btf, member->name_off)) { in btf_struct_check_meta()
3175 member->name_off); in btf_struct_check_meta()
3176 return -EINVAL; in btf_struct_check_meta()
3180 if (member->name_off && in btf_struct_check_meta()
3181 !btf_name_valid_identifier(btf, member->name_off)) { in btf_struct_check_meta()
3183 return -EINVAL; in btf_struct_check_meta()
3186 if (!member->type || !BTF_TYPE_ID_VALID(member->type)) { in btf_struct_check_meta()
3189 return -EINVAL; in btf_struct_check_meta()
3196 return -EINVAL; in btf_struct_check_meta()
3206 return -EINVAL; in btf_struct_check_meta()
3212 return -EINVAL; in btf_struct_check_meta()
3233 if (v->next_member) { in btf_struct_resolve()
3238 last_member = btf_type_member(v->t) + v->next_member - 1; in btf_struct_resolve()
3239 last_member_type_id = last_member->type; in btf_struct_resolve()
3242 return -EINVAL; in btf_struct_resolve()
3244 last_member_type = btf_type_by_id(env->btf, in btf_struct_resolve()
3246 if (btf_type_kflag(v->t)) in btf_struct_resolve()
3247 err = btf_type_ops(last_member_type)->check_kflag_member(env, v->t, in btf_struct_resolve()
3251 err = btf_type_ops(last_member_type)->check_member(env, v->t, in btf_struct_resolve()
3258 for_each_member_from(i, v->next_member, v->t, member) { in btf_struct_resolve()
3259 u32 member_type_id = member->type; in btf_struct_resolve()
3260 const struct btf_type *member_type = btf_type_by_id(env->btf, in btf_struct_resolve()
3265 btf_verifier_log_member(env, v->t, member, in btf_struct_resolve()
3267 return -EINVAL; in btf_struct_resolve()
3276 if (btf_type_kflag(v->t)) in btf_struct_resolve()
3277 err = btf_type_ops(member_type)->check_kflag_member(env, v->t, in btf_struct_resolve()
3281 err = btf_type_ops(member_type)->check_member(env, v->t, in btf_struct_resolve()
3296 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); in btf_struct_log()
3324 if (t->size != sz) in btf_find_struct()
3326 info->type = field_type; in btf_find_struct()
3327 info->off = off; in btf_find_struct()
3339 t = btf_type_by_id(btf, t->type); in btf_find_kptr()
3343 t = btf_type_by_id(btf, t->type); in btf_find_kptr()
3348 if (btf_type_is_type_tag(btf_type_by_id(btf, t->type))) in btf_find_kptr()
3349 return -EINVAL; in btf_find_kptr()
3350 if (!strcmp("kptr_untrusted", __btf_name_by_offset(btf, t->name_off))) in btf_find_kptr()
3352 else if (!strcmp("kptr", __btf_name_by_offset(btf, t->name_off))) in btf_find_kptr()
3354 else if (!strcmp("percpu_kptr", __btf_name_by_offset(btf, t->name_off))) in btf_find_kptr()
3356 else if (!strcmp("uptr", __btf_name_by_offset(btf, t->name_off))) in btf_find_kptr()
3359 return -EINVAL; in btf_find_kptr()
3365 t = btf_type_skip_modifiers(btf, t->type, &res_id); in btf_find_kptr()
3368 return -EINVAL; in btf_find_kptr()
3370 info->type = type; in btf_find_kptr()
3371 info->off = off; in btf_find_kptr()
3372 info->kptr.type_id = res_id; in btf_find_kptr()
3387 if (pt != btf_type_by_id(btf, t->type)) in btf_find_next_decl_tag()
3389 if (btf_type_decl_tag(t)->component_idx != comp_idx) in btf_find_next_decl_tag()
3391 if (strncmp(__btf_name_by_offset(btf, t->name_off), tag_key, len)) in btf_find_next_decl_tag()
3395 return -ENOENT; in btf_find_next_decl_tag()
3411 value = __btf_name_by_offset(btf, t->name_off) + len; in btf_find_decl_tag_value()
3416 return ERR_PTR(-EEXIST); in btf_find_decl_tag_value()
3433 if (t->size != sz) in btf_find_graph_root()
3437 return -EINVAL; in btf_find_graph_root()
3440 return -EINVAL; in btf_find_graph_root()
3441 value_type = kstrndup(value_type, node_field_name - value_type, GFP_KERNEL | __GFP_NOWARN); in btf_find_graph_root()
3443 return -ENOMEM; in btf_find_graph_root()
3450 return -EINVAL; in btf_find_graph_root()
3451 info->type = head_type; in btf_find_graph_root()
3452 info->off = off; in btf_find_graph_root()
3453 info->graph_root.value_btf_id = id; in btf_find_graph_root()
3454 info->graph_root.node_name = node_field_name; in btf_find_graph_root()
3469 const char *name = __btf_name_by_offset(btf, var_type->name_off); in btf_get_field_type()
3474 return -E2BIG; in btf_get_field_type()
3483 return -E2BIG; in btf_get_field_type()
3492 return -E2BIG; in btf_get_field_type()
3543 return -EINVAL; in btf_repeat_fields()
3551 return -E2BIG; in btf_repeat_fields()
3583 return -E2BIG; in btf_find_nested_struct()
3597 err = btf_repeat_fields(info, info_cnt, ret, nelems - 1, t->size); in btf_find_nested_struct()
3626 nelems *= array->nelems; in btf_find_field_one()
3627 var_type = btf_type_by_id(btf, array->type); in btf_find_field_one()
3630 return -E2BIG; in btf_find_field_one()
3638 sz = var_type->size; in btf_find_field_one()
3687 return -EFAULT; in btf_find_field_one()
3693 return -E2BIG; in btf_find_field_one()
3695 ret = btf_repeat_fields(info, info_cnt, 1, nelems - 1, sz); in btf_find_field_one()
3713 member->type); in btf_find_struct_field()
3718 return -EINVAL; in btf_find_struct_field()
3724 &info[idx], info_cnt - idx, level); in btf_find_struct_field()
3741 const struct btf_type *var = btf_type_by_id(btf, vsi->type); in btf_find_datasec_var()
3742 const struct btf_type *var_type = btf_type_by_id(btf, var->type); in btf_find_datasec_var()
3744 off = vsi->offset; in btf_find_datasec_var()
3745 ret = btf_find_field_one(btf, var, var_type, -1, off, vsi->size, in btf_find_datasec_var()
3747 &info[idx], info_cnt - idx, in btf_find_datasec_var()
3764 return -EINVAL; in btf_find_field()
3783 t = btf_type_by_id(btf, info->kptr.type_id); in btf_parse_kptr()
3784 id = bpf_find_btf_id(__btf_name_by_offset(btf, t->name_off), BTF_INFO_KIND(t->info), in btf_parse_kptr()
3786 if (id == -ENOENT) { in btf_parse_kptr()
3793 field->kptr.dtor = NULL; in btf_parse_kptr()
3794 id = info->kptr.type_id; in btf_parse_kptr()
3804 if (info->type == BPF_KPTR_REF) { in btf_parse_kptr()
3822 ret = -ENOENT; in btf_parse_kptr()
3829 ret = -ENXIO; in btf_parse_kptr()
3837 dtor_func_name = __btf_name_by_offset(kptr_btf, dtor_func->name_off); in btf_parse_kptr()
3840 ret = -EINVAL; in btf_parse_kptr()
3843 field->kptr.dtor = (void *)addr; in btf_parse_kptr()
3847 field->kptr.btf_id = id; in btf_parse_kptr()
3848 field->kptr.btf = kptr_btf; in btf_parse_kptr()
3849 field->kptr.module = mod; in btf_parse_kptr()
3869 t = btf_type_by_id(btf, info->graph_root.value_btf_id); in btf_parse_graph_root()
3875 if (strcmp(info->graph_root.node_name, in btf_parse_graph_root()
3876 __btf_name_by_offset(btf, member->name_off))) in btf_parse_graph_root()
3880 return -EINVAL; in btf_parse_graph_root()
3881 n = btf_type_by_id(btf, member->type); in btf_parse_graph_root()
3883 return -EINVAL; in btf_parse_graph_root()
3884 if (strcmp(node_type_name, __btf_name_by_offset(btf, n->name_off))) in btf_parse_graph_root()
3885 return -EINVAL; in btf_parse_graph_root()
3888 return -EINVAL; in btf_parse_graph_root()
3891 return -EINVAL; in btf_parse_graph_root()
3893 field->graph_root.btf = (struct btf *)btf; in btf_parse_graph_root()
3894 field->graph_root.value_btf_id = info->graph_root.value_btf_id; in btf_parse_graph_root()
3895 field->graph_root.node_offset = offset; in btf_parse_graph_root()
3898 return -ENOENT; in btf_parse_graph_root()
3921 if (a->offset < b->offset) in btf_field_cmp()
3922 return -1; in btf_field_cmp()
3923 else if (a->offset > b->offset) in btf_field_cmp()
3948 return ERR_PTR(-ENOMEM); in btf_parse_fields()
3950 rec->spin_lock_off = -EINVAL; in btf_parse_fields()
3951 rec->timer_off = -EINVAL; in btf_parse_fields()
3952 rec->wq_off = -EINVAL; in btf_parse_fields()
3953 rec->refcount_off = -EINVAL; in btf_parse_fields()
3958 ret = -EFAULT; in btf_parse_fields()
3962 ret = -EEXIST; in btf_parse_fields()
3967 rec->field_mask |= info_arr[i].type; in btf_parse_fields()
3968 rec->fields[i].offset = info_arr[i].off; in btf_parse_fields()
3969 rec->fields[i].type = info_arr[i].type; in btf_parse_fields()
3970 rec->fields[i].size = field_type_size; in btf_parse_fields()
3974 WARN_ON_ONCE(rec->spin_lock_off >= 0); in btf_parse_fields()
3976 rec->spin_lock_off = rec->fields[i].offset; in btf_parse_fields()
3979 WARN_ON_ONCE(rec->timer_off >= 0); in btf_parse_fields()
3981 rec->timer_off = rec->fields[i].offset; in btf_parse_fields()
3984 WARN_ON_ONCE(rec->wq_off >= 0); in btf_parse_fields()
3986 rec->wq_off = rec->fields[i].offset; in btf_parse_fields()
3989 WARN_ON_ONCE(rec->refcount_off >= 0); in btf_parse_fields()
3991 rec->refcount_off = rec->fields[i].offset; in btf_parse_fields()
3997 ret = btf_parse_kptr(btf, &rec->fields[i], &info_arr[i]); in btf_parse_fields()
4002 ret = btf_parse_list_head(btf, &rec->fields[i], &info_arr[i]); in btf_parse_fields()
4007 ret = btf_parse_rb_root(btf, &rec->fields[i], &info_arr[i]); in btf_parse_fields()
4015 ret = -EFAULT; in btf_parse_fields()
4018 rec->cnt++; in btf_parse_fields()
4023 btf_record_has_field(rec, BPF_RB_ROOT)) && rec->spin_lock_off < 0) { in btf_parse_fields()
4024 ret = -EINVAL; in btf_parse_fields()
4028 if (rec->refcount_off < 0 && in btf_parse_fields()
4031 ret = -EINVAL; in btf_parse_fields()
4035 sort_r(rec->fields, rec->cnt, sizeof(struct btf_field), btf_field_cmp, in btf_parse_fields()
4056 if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & (BPF_GRAPH_ROOT | BPF_UPTR))) in btf_check_and_fixup_fields()
4058 for (i = 0; i < rec->cnt; i++) { in btf_check_and_fixup_fields()
4063 if (rec->fields[i].type == BPF_UPTR) { in btf_check_and_fixup_fields()
4067 if (btf_is_kernel(rec->fields[i].kptr.btf)) in btf_check_and_fixup_fields()
4068 return -EINVAL; in btf_check_and_fixup_fields()
4069 t = btf_type_by_id(rec->fields[i].kptr.btf, in btf_check_and_fixup_fields()
4070 rec->fields[i].kptr.btf_id); in btf_check_and_fixup_fields()
4071 if (!t->size) in btf_check_and_fixup_fields()
4072 return -EINVAL; in btf_check_and_fixup_fields()
4073 if (t->size > PAGE_SIZE) in btf_check_and_fixup_fields()
4074 return -E2BIG; in btf_check_and_fixup_fields()
4078 if (!(rec->fields[i].type & BPF_GRAPH_ROOT)) in btf_check_and_fixup_fields()
4080 btf_id = rec->fields[i].graph_root.value_btf_id; in btf_check_and_fixup_fields()
4083 return -EFAULT; in btf_check_and_fixup_fields()
4084 rec->fields[i].graph_root.value_rec = meta->record; in btf_check_and_fixup_fields()
4090 if (!(rec->field_mask & BPF_GRAPH_NODE)) in btf_check_and_fixup_fields()
4099 * - A type can only be owned by another type in user BTF if it in btf_check_and_fixup_fields()
4101 * - A type can only _own_ another type in user BTF if it has a in btf_check_and_fixup_fields()
4111 * A -> B -> C in btf_check_and_fixup_fields()
4113 * - A is an root, e.g. has bpf_rb_root. in btf_check_and_fixup_fields()
4114 * - B is both a root and node, e.g. has bpf_rb_node and in btf_check_and_fixup_fields()
4116 * - C is only an root, e.g. has bpf_list_node in btf_check_and_fixup_fields()
4121 * A -> B in btf_check_and_fixup_fields()
4123 * - A is both an root and node. in btf_check_and_fixup_fields()
4124 * - B is only an node. in btf_check_and_fixup_fields()
4126 if (meta->record->field_mask & BPF_GRAPH_ROOT) in btf_check_and_fixup_fields()
4127 return -ELOOP; in btf_check_and_fixup_fields()
4146 member->type); in __btf_struct_show()
4160 member->type, in __btf_struct_show()
4169 ops->show(btf, member_type, member->type, in __btf_struct_show()
4183 const struct btf_member *m = show->state.member; in btf_struct_show()
4186 * First check if any members would be shown (are non-zero). in btf_struct_show()
4188 * details on how this works at a high-level. in btf_struct_show()
4190 if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) { in btf_struct_show()
4191 if (!show->state.depth_check) { in btf_struct_show()
4192 show->state.depth_check = show->state.depth + 1; in btf_struct_show()
4193 show->state.depth_to_show = 0; in btf_struct_show()
4197 show->state.member = m; in btf_struct_show()
4198 if (show->state.depth_check != show->state.depth + 1) in btf_struct_show()
4200 show->state.depth_check = 0; in btf_struct_show()
4202 if (show->state.depth_to_show <= show->state.depth) in btf_struct_show()
4206 * non-zero child values. in btf_struct_show()
4227 u32 struct_bits_off = member->offset; in btf_enum_check_member()
4233 return -EINVAL; in btf_enum_check_member()
4236 struct_size = struct_type->size; in btf_enum_check_member()
4238 if (struct_size - bytes_offset < member_type->size) { in btf_enum_check_member()
4241 return -EINVAL; in btf_enum_check_member()
4255 struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset); in btf_enum_check_kflag_member()
4256 nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset); in btf_enum_check_kflag_member()
4261 return -EINVAL; in btf_enum_check_kflag_member()
4268 return -EINVAL; in btf_enum_check_kflag_member()
4271 struct_size = struct_type->size; in btf_enum_check_kflag_member()
4276 return -EINVAL; in btf_enum_check_kflag_member()
4287 struct btf *btf = env->btf; in btf_enum_check_meta()
4299 return -EINVAL; in btf_enum_check_meta()
4302 if (t->size > 8 || !is_power_of_2(t->size)) { in btf_enum_check_meta()
4304 return -EINVAL; in btf_enum_check_meta()
4308 if (t->name_off && in btf_enum_check_meta()
4309 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_enum_check_meta()
4311 return -EINVAL; in btf_enum_check_meta()
4320 return -EINVAL; in btf_enum_check_meta()
4327 return -EINVAL; in btf_enum_check_meta()
4330 if (env->log.level == BPF_LOG_KERNEL) in btf_enum_check_meta()
4344 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); in btf_enum_log()
4395 struct btf *btf = env->btf; in btf_enum64_check_meta()
4407 return -EINVAL; in btf_enum64_check_meta()
4410 if (t->size > 8 || !is_power_of_2(t->size)) { in btf_enum64_check_meta()
4412 return -EINVAL; in btf_enum64_check_meta()
4416 if (t->name_off && in btf_enum64_check_meta()
4417 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_enum64_check_meta()
4419 return -EINVAL; in btf_enum64_check_meta()
4428 return -EINVAL; in btf_enum64_check_meta()
4435 return -EINVAL; in btf_enum64_check_meta()
4438 if (env->log.level == BPF_LOG_KERNEL) in btf_enum64_check_meta()
4503 return -EINVAL; in btf_func_proto_check_meta()
4506 if (t->name_off) { in btf_func_proto_check_meta()
4508 return -EINVAL; in btf_func_proto_check_meta()
4513 return -EINVAL; in btf_func_proto_check_meta()
4527 btf_verifier_log(env, "return=%u args=(", t->type); in btf_func_proto_log()
4540 __btf_name_by_offset(env->btf, in btf_func_proto_log()
4542 for (i = 1; i < nr_args - 1; i++) in btf_func_proto_log()
4544 __btf_name_by_offset(env->btf, in btf_func_proto_log()
4548 const struct btf_param *last_arg = &args[nr_args - 1]; in btf_func_proto_log()
4550 if (last_arg->type) in btf_func_proto_log()
4551 btf_verifier_log(env, ", %u %s", last_arg->type, in btf_func_proto_log()
4552 __btf_name_by_offset(env->btf, in btf_func_proto_log()
4553 last_arg->name_off)); in btf_func_proto_log()
4570 * (i.e. struct's member -> BTF_KIND_PTR -> BTF_KIND_FUNC_PROTO)
4584 if (!t->name_off || in btf_func_check_meta()
4585 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_func_check_meta()
4587 return -EINVAL; in btf_func_check_meta()
4592 return -EINVAL; in btf_func_check_meta()
4597 return -EINVAL; in btf_func_check_meta()
4608 const struct btf_type *t = v->t; in btf_func_resolve()
4609 u32 next_type_id = t->type; in btf_func_resolve()
4640 return -EINVAL; in btf_var_check_meta()
4645 return -EINVAL; in btf_var_check_meta()
4650 return -EINVAL; in btf_var_check_meta()
4653 if (!t->name_off || in btf_var_check_meta()
4654 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_var_check_meta()
4656 return -EINVAL; in btf_var_check_meta()
4660 if (!t->type || !BTF_TYPE_ID_VALID(t->type)) { in btf_var_check_meta()
4662 return -EINVAL; in btf_var_check_meta()
4666 if (var->linkage != BTF_VAR_STATIC && in btf_var_check_meta()
4667 var->linkage != BTF_VAR_GLOBAL_ALLOCATED) { in btf_var_check_meta()
4669 return -EINVAL; in btf_var_check_meta()
4681 btf_verifier_log(env, "type_id=%u linkage=%u", t->type, var->linkage); in btf_var_log()
4706 return -EINVAL; in btf_datasec_check_meta()
4709 if (!t->size) { in btf_datasec_check_meta()
4711 return -EINVAL; in btf_datasec_check_meta()
4716 return -EINVAL; in btf_datasec_check_meta()
4719 if (!t->name_off || in btf_datasec_check_meta()
4720 !btf_name_valid_section(env->btf, t->name_off)) { in btf_datasec_check_meta()
4722 return -EINVAL; in btf_datasec_check_meta()
4729 if (!vsi->type || !BTF_TYPE_ID_VALID(vsi->type)) { in btf_datasec_check_meta()
4732 return -EINVAL; in btf_datasec_check_meta()
4735 if (vsi->offset < last_vsi_end_off || vsi->offset >= t->size) { in btf_datasec_check_meta()
4738 return -EINVAL; in btf_datasec_check_meta()
4741 if (!vsi->size || vsi->size > t->size) { in btf_datasec_check_meta()
4744 return -EINVAL; in btf_datasec_check_meta()
4747 last_vsi_end_off = vsi->offset + vsi->size; in btf_datasec_check_meta()
4748 if (last_vsi_end_off > t->size) { in btf_datasec_check_meta()
4751 return -EINVAL; in btf_datasec_check_meta()
4755 sum += vsi->size; in btf_datasec_check_meta()
4758 if (t->size < sum) { in btf_datasec_check_meta()
4760 return -EINVAL; in btf_datasec_check_meta()
4770 struct btf *btf = env->btf; in btf_datasec_resolve()
4773 env->resolve_mode = RESOLVE_TBD; in btf_datasec_resolve()
4774 for_each_vsi_from(i, v->next_member, v->t, vsi) { in btf_datasec_resolve()
4775 u32 var_type_id = vsi->type, type_id, type_size = 0; in btf_datasec_resolve()
4776 const struct btf_type *var_type = btf_type_by_id(env->btf, in btf_datasec_resolve()
4779 btf_verifier_log_vsi(env, v->t, vsi, in btf_datasec_resolve()
4781 return -EINVAL; in btf_datasec_resolve()
4790 type_id = var_type->type; in btf_datasec_resolve()
4792 btf_verifier_log_vsi(env, v->t, vsi, "Invalid type"); in btf_datasec_resolve()
4793 return -EINVAL; in btf_datasec_resolve()
4796 if (vsi->size < type_size) { in btf_datasec_resolve()
4797 btf_verifier_log_vsi(env, v->t, vsi, "Invalid size"); in btf_datasec_resolve()
4798 return -EINVAL; in btf_datasec_resolve()
4809 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); in btf_datasec_log()
4825 __btf_name_by_offset(btf, t->name_off)); in btf_datasec_show()
4827 var = btf_type_by_id(btf, vsi->type); in btf_datasec_show()
4830 btf_type_ops(var)->show(btf, var, vsi->type, in btf_datasec_show()
4831 data + vsi->offset, bits_offset, show); in btf_datasec_show()
4851 return -EINVAL; in btf_float_check_meta()
4856 return -EINVAL; in btf_float_check_meta()
4859 if (t->size != 2 && t->size != 4 && t->size != 8 && t->size != 12 && in btf_float_check_meta()
4860 t->size != 16) { in btf_float_check_meta()
4862 return -EINVAL; in btf_float_check_meta()
4883 * that types after CO-RE can pass the kernel BTF verifier. in btf_float_check_member()
4885 align_bytes = min_t(u64, sizeof(void *), member_type->size); in btf_float_check_member()
4887 div64_u64_rem(member->offset, align_bits, &misalign_bits); in btf_float_check_member()
4891 return -EINVAL; in btf_float_check_member()
4894 start_offset_bytes = member->offset / BITS_PER_BYTE; in btf_float_check_member()
4895 end_offset_bytes = start_offset_bytes + member_type->size; in btf_float_check_member()
4896 if (end_offset_bytes > struct_type->size) { in btf_float_check_member()
4899 return -EINVAL; in btf_float_check_member()
4908 btf_verifier_log(env, "size=%u", t->size); in btf_float_log()
4933 return -EINVAL; in btf_decl_tag_check_meta()
4936 value = btf_name_by_offset(env->btf, t->name_off); in btf_decl_tag_check_meta()
4939 return -EINVAL; in btf_decl_tag_check_meta()
4944 return -EINVAL; in btf_decl_tag_check_meta()
4949 return -EINVAL; in btf_decl_tag_check_meta()
4952 component_idx = btf_type_decl_tag(t)->component_idx; in btf_decl_tag_check_meta()
4953 if (component_idx < -1) { in btf_decl_tag_check_meta()
4955 return -EINVAL; in btf_decl_tag_check_meta()
4967 const struct btf_type *t = v->t; in btf_decl_tag_resolve()
4968 u32 next_type_id = t->type; in btf_decl_tag_resolve()
4969 struct btf *btf = env->btf; in btf_decl_tag_resolve()
4975 btf_verifier_log_type(env, v->t, "Invalid type_id"); in btf_decl_tag_resolve()
4976 return -EINVAL; in btf_decl_tag_resolve()
4983 component_idx = btf_type_decl_tag(t)->component_idx; in btf_decl_tag_resolve()
4984 if (component_idx != -1) { in btf_decl_tag_resolve()
4986 btf_verifier_log_type(env, v->t, "Invalid component_idx"); in btf_decl_tag_resolve()
4987 return -EINVAL; in btf_decl_tag_resolve()
4994 next_type = btf_type_by_id(btf, next_type->type); in btf_decl_tag_resolve()
4999 btf_verifier_log_type(env, v->t, "Invalid component_idx"); in btf_decl_tag_resolve()
5000 return -EINVAL; in btf_decl_tag_resolve()
5011 btf_verifier_log(env, "type=%u component_idx=%d", t->type, in btf_decl_tag_log()
5012 btf_type_decl_tag(t)->component_idx); in btf_decl_tag_log()
5033 btf = env->btf; in btf_func_proto_check()
5037 /* Check func return type which could be "void" (t->type == 0) */ in btf_func_proto_check()
5038 if (t->type) { in btf_func_proto_check()
5039 u32 ret_type_id = t->type; in btf_func_proto_check()
5044 return -EINVAL; in btf_func_proto_check()
5049 return -EINVAL; in btf_func_proto_check()
5062 return -EINVAL; in btf_func_proto_check()
5070 if (!args[nr_args - 1].type) { in btf_func_proto_check()
5071 if (args[nr_args - 1].name_off) { in btf_func_proto_check()
5074 return -EINVAL; in btf_func_proto_check()
5076 nr_args--; in btf_func_proto_check()
5087 return -EINVAL; in btf_func_proto_check()
5092 return -EINVAL; in btf_func_proto_check()
5100 return -EINVAL; in btf_func_proto_check()
5112 return -EINVAL; in btf_func_proto_check()
5127 btf = env->btf; in btf_func_check()
5128 proto_type = btf_type_by_id(btf, t->type); in btf_func_check()
5132 return -EINVAL; in btf_func_check()
5140 return -EINVAL; in btf_func_check()
5178 env->log_type_id, meta_left, sizeof(*t)); in btf_check_meta()
5179 return -EINVAL; in btf_check_meta()
5181 meta_left -= sizeof(*t); in btf_check_meta()
5183 if (t->info & ~BTF_INFO_MASK) { in btf_check_meta()
5185 env->log_type_id, t->info); in btf_check_meta()
5186 return -EINVAL; in btf_check_meta()
5189 if (BTF_INFO_KIND(t->info) > BTF_KIND_MAX || in btf_check_meta()
5190 BTF_INFO_KIND(t->info) == BTF_KIND_UNKN) { in btf_check_meta()
5192 env->log_type_id, BTF_INFO_KIND(t->info)); in btf_check_meta()
5193 return -EINVAL; in btf_check_meta()
5196 if (!btf_name_offset_valid(env->btf, t->name_off)) { in btf_check_meta()
5198 env->log_type_id, t->name_off); in btf_check_meta()
5199 return -EINVAL; in btf_check_meta()
5202 var_meta_size = btf_type_ops(t)->check_meta(env, t, meta_left); in btf_check_meta()
5206 meta_left -= var_meta_size; in btf_check_meta()
5208 return saved_meta_left - meta_left; in btf_check_meta()
5213 struct btf *btf = env->btf; in btf_check_all_metas()
5217 hdr = &btf->hdr; in btf_check_all_metas()
5218 cur = btf->nohdr_data + hdr->type_off; in btf_check_all_metas()
5219 end = cur + hdr->type_len; in btf_check_all_metas()
5221 env->log_type_id = btf->base_btf ? btf->start_id : 1; in btf_check_all_metas()
5226 meta_size = btf_check_meta(env, t, end - cur); in btf_check_all_metas()
5232 env->log_type_id++; in btf_check_all_metas()
5242 struct btf *btf = env->btf; in btf_resolve_valid()
5267 u32 elem_type_id = array->type; in btf_resolve_valid()
5272 (array->nelems * elem_size == in btf_resolve_valid()
5282 u32 save_log_type_id = env->log_type_id; in btf_resolve()
5286 env->resolve_mode = RESOLVE_TBD; in btf_resolve()
5289 env->log_type_id = v->type_id; in btf_resolve()
5290 err = btf_type_ops(v->t)->resolve(env, v); in btf_resolve()
5293 env->log_type_id = type_id; in btf_resolve()
5294 if (err == -E2BIG) { in btf_resolve()
5298 } else if (err == -EEXIST) { in btf_resolve()
5305 err = -EINVAL; in btf_resolve()
5308 env->log_type_id = save_log_type_id; in btf_resolve()
5314 struct btf *btf = env->btf; in btf_check_all_types()
5323 env->phase++; in btf_check_all_types()
5324 for (i = btf->base_btf ? 0 : 1; i < btf->nr_types; i++) { in btf_check_all_types()
5325 type_id = btf->start_id + i; in btf_check_all_types()
5328 env->log_type_id = type_id; in btf_check_all_types()
5348 const struct btf_header *hdr = &env->btf->hdr; in btf_parse_type_sec()
5352 if (hdr->type_off & (sizeof(u32) - 1)) { in btf_parse_type_sec()
5354 return -EINVAL; in btf_parse_type_sec()
5357 if (!env->btf->base_btf && !hdr->type_len) { in btf_parse_type_sec()
5359 return -EINVAL; in btf_parse_type_sec()
5372 struct btf *btf = env->btf; in btf_parse_str_sec()
5375 hdr = &btf->hdr; in btf_parse_str_sec()
5376 start = btf->nohdr_data + hdr->str_off; in btf_parse_str_sec()
5377 end = start + hdr->str_len; in btf_parse_str_sec()
5379 if (end != btf->data + btf->data_size) { in btf_parse_str_sec()
5381 return -EINVAL; in btf_parse_str_sec()
5384 btf->strings = start; in btf_parse_str_sec()
5386 if (btf->base_btf && !hdr->str_len) in btf_parse_str_sec()
5388 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET || end[-1]) { in btf_parse_str_sec()
5390 return -EINVAL; in btf_parse_str_sec()
5392 if (!btf->base_btf && start[0]) { in btf_parse_str_sec()
5394 return -EINVAL; in btf_parse_str_sec()
5410 return (int)(x->off - y->off) ? : (int)(x->len - y->len); in btf_sec_info_cmp()
5421 btf = env->btf; in btf_check_sec_info()
5422 hdr = &btf->hdr; in btf_check_sec_info()
5434 expected_total = btf_data_size - hdr->hdr_len; in btf_check_sec_info()
5438 return -EINVAL; in btf_check_sec_info()
5443 return -EINVAL; in btf_check_sec_info()
5447 return -EINVAL; in btf_check_sec_info()
5449 if (expected_total - total < secs[i].len) { in btf_check_sec_info()
5452 return -EINVAL; in btf_check_sec_info()
5460 return -EINVAL; in btf_check_sec_info()
5472 btf = env->btf; in btf_parse_hdr()
5473 btf_data_size = btf->data_size; in btf_parse_hdr()
5477 return -EINVAL; in btf_parse_hdr()
5480 hdr = btf->data; in btf_parse_hdr()
5481 hdr_len = hdr->hdr_len; in btf_parse_hdr()
5484 return -EINVAL; in btf_parse_hdr()
5488 if (hdr_len > sizeof(btf->hdr)) { in btf_parse_hdr()
5489 u8 *expected_zero = btf->data + sizeof(btf->hdr); in btf_parse_hdr()
5490 u8 *end = btf->data + hdr_len; in btf_parse_hdr()
5495 return -E2BIG; in btf_parse_hdr()
5500 hdr_copy = min_t(u32, hdr_len, sizeof(btf->hdr)); in btf_parse_hdr()
5501 memcpy(&btf->hdr, btf->data, hdr_copy); in btf_parse_hdr()
5503 hdr = &btf->hdr; in btf_parse_hdr()
5507 if (hdr->magic != BTF_MAGIC) { in btf_parse_hdr()
5509 return -EINVAL; in btf_parse_hdr()
5512 if (hdr->version != BTF_VERSION) { in btf_parse_hdr()
5514 return -ENOTSUPP; in btf_parse_hdr()
5517 if (hdr->flags) { in btf_parse_hdr()
5519 return -ENOTSUPP; in btf_parse_hdr()
5522 if (!btf->base_btf && btf_data_size == hdr->hdr_len) { in btf_parse_hdr()
5524 return -EINVAL; in btf_parse_hdr()
5551 return ERR_PTR(-ENOMEM); in btf_parse_struct_metas()
5552 aof->cnt = 0; in btf_parse_struct_metas()
5565 new_aof = krealloc(aof, offsetof(struct btf_id_set, ids[aof->cnt + 1]), in btf_parse_struct_metas()
5568 ret = -ENOMEM; in btf_parse_struct_metas()
5572 aof->ids[aof->cnt++] = id; in btf_parse_struct_metas()
5584 ret = -EINVAL; in btf_parse_struct_metas()
5592 new_aof = krealloc(aof, offsetof(struct btf_id_set, ids[aof->cnt + 1]), in btf_parse_struct_metas()
5595 ret = -ENOMEM; in btf_parse_struct_metas()
5599 aof->ids[aof->cnt++] = i; in btf_parse_struct_metas()
5602 if (!aof->cnt) { in btf_parse_struct_metas()
5606 sort(&aof->ids, aof->cnt, sizeof(aof->ids[0]), btf_id_cmp_func, NULL); in btf_parse_struct_metas()
5623 if (btf_id_set_contains(aof, member->type)) in btf_parse_struct_metas()
5628 tab_cnt = tab ? tab->cnt : 0; in btf_parse_struct_metas()
5632 ret = -ENOMEM; in btf_parse_struct_metas()
5636 new_tab->cnt = 0; in btf_parse_struct_metas()
5639 type = &tab->types[tab->cnt]; in btf_parse_struct_metas()
5640 type->btf_id = i; in btf_parse_struct_metas()
5643 BPF_KPTR, t->size); in btf_parse_struct_metas()
5646 ret = PTR_ERR_OR_ZERO(record) ?: -EFAULT; in btf_parse_struct_metas()
5649 type->record = record; in btf_parse_struct_metas()
5650 tab->cnt++; in btf_parse_struct_metas()
5666 tab = btf->struct_meta_tab; in btf_find_struct_meta()
5669 return bsearch(&btf_id, tab->types, tab->cnt, sizeof(tab->types[0]), btf_id_cmp_func); in btf_find_struct_meta()
5675 int i, n, good_id = start_id - 1; in btf_check_type_tags()
5686 return -EINVAL; in btf_check_type_tags()
5694 if (!chain_limit--) { in btf_check_type_tags()
5696 return -ELOOP; in btf_check_type_tags()
5701 return -EINVAL; in btf_check_type_tags()
5709 cur_id = t->type; in btf_check_type_tags()
5712 return -EINVAL; in btf_check_type_tags()
5729 err = -EFAULT; in finalize_log()
5736 bpfptr_t btf_data = make_bpfptr(attr->btf, uattr.is_kernel); in btf_parse()
5737 char __user *log_ubuf = u64_to_user_ptr(attr->btf_log_buf); in btf_parse()
5744 if (attr->btf_size > BTF_MAX_SIZE) in btf_parse()
5745 return ERR_PTR(-E2BIG); in btf_parse()
5749 return ERR_PTR(-ENOMEM); in btf_parse()
5754 err = bpf_vlog_init(&env->log, attr->btf_log_level, in btf_parse()
5755 log_ubuf, attr->btf_log_size); in btf_parse()
5761 err = -ENOMEM; in btf_parse()
5764 env->btf = btf; in btf_parse()
5766 data = kvmalloc(attr->btf_size, GFP_KERNEL | __GFP_NOWARN); in btf_parse()
5768 err = -ENOMEM; in btf_parse()
5772 btf->data = data; in btf_parse()
5773 btf->data_size = attr->btf_size; in btf_parse()
5775 if (copy_from_bpfptr(data, btf_data, attr->btf_size)) { in btf_parse()
5776 err = -EFAULT; in btf_parse()
5784 btf->nohdr_data = btf->data + btf->hdr.hdr_len; in btf_parse()
5798 struct_meta_tab = btf_parse_struct_metas(&env->log, btf); in btf_parse()
5803 btf->struct_meta_tab = struct_meta_tab; in btf_parse()
5808 for (i = 0; i < struct_meta_tab->cnt; i++) { in btf_parse()
5809 err = btf_check_and_fixup_fields(btf, struct_meta_tab->types[i].record); in btf_parse()
5815 err = finalize_log(&env->log, uattr, uattr_size); in btf_parse()
5820 refcount_set(&btf->refcnt, 1); in btf_parse()
5826 /* overwrite err with -ENOSPC or -EFAULT */ in btf_parse()
5827 ret = finalize_log(&env->log, uattr, uattr_size); in btf_parse()
5884 return btf_type_by_id(btf_vmlinux, ctx_type->type); in find_canonical_prog_ctx_type()
5894 return -EFAULT; in find_kern_ctx_type_id()
5900 return ctx_type->type; in find_kern_ctx_type_id()
5919 t = btf_type_by_id(btf, t->type); in btf_is_prog_ctx_type()
5926 t = btf_type_by_id(btf, t->type); in btf_is_prog_ctx_type()
5929 tname = btf_name_by_offset(btf, t->name_off); in btf_is_prog_ctx_type()
5936 t = btf_type_by_id(btf, t->type); in btf_is_prog_ctx_type()
5945 tname = btf_name_by_offset(btf, t->name_off); in btf_is_prog_ctx_type()
5958 ctx_tname = btf_name_by_offset(btf_vmlinux, ctx_type->name_off); in btf_is_prog_ctx_type()
5983 ctx_type = btf_type_by_id(btf_vmlinux, ctx_type->type); in btf_is_prog_ctx_type()
5989 /* forward declarations for arch-specific underlying types of
5990 * bpf_user_pt_regs_t; this avoids the need for arch-specific #ifdef
6008 return -EINVAL; in btf_validate_prog_ctx_type()
6010 t = btf_type_by_id(btf, t->type); in btf_validate_prog_ctx_type()
6015 t = btf_type_by_id(btf, t->type); in btf_validate_prog_ctx_type()
6018 tname = btf_name_by_offset(btf, t->name_off); in btf_validate_prog_ctx_type()
6026 t = btf_type_by_id(btf, t->type); in btf_validate_prog_ctx_type()
6032 tname = btf_name_by_offset(btf, t->name_off); in btf_validate_prog_ctx_type()
6035 return -EINVAL; in btf_validate_prog_ctx_type()
6058 if (btf_is_int(t) && t->size == 8) in btf_validate_prog_ctx_type()
6069 if (btf_is_int(t) && t->size == 8) in btf_validate_prog_ctx_type()
6075 strncmp(tname, "bpf_iter__", sizeof("bpf_iter__") - 1) == 0) in btf_validate_prog_ctx_type()
6082 if (btf_is_int(t) && t->size == 8) in btf_validate_prog_ctx_type()
6092 if (btf_is_int(t) && t->size == 8) in btf_validate_prog_ctx_type()
6107 return -EINVAL; in btf_validate_prog_ctx_type()
6112 ctx_type = btf_type_by_id(btf_vmlinux, ctx_type->type); in btf_validate_prog_ctx_type()
6120 return -EINVAL; in btf_validate_prog_ctx_type()
6123 ctx_tname = btf_name_by_offset(btf_vmlinux, ctx_type->name_off); in btf_validate_prog_ctx_type()
6126 return -EINVAL; in btf_validate_prog_ctx_type()
6139 return -ENOENT; in btf_translate_to_vmlinux()
6153 kctx_type_id = kctx_member->type; in get_kern_ctx_btf_id()
6157 return -EINVAL; in get_kern_ctx_btf_id()
6173 return ERR_PTR(-ENOENT); in BTF_ID()
6177 err = -ENOMEM; in BTF_ID()
6180 env->btf = btf; in BTF_ID()
6182 btf->data = data; in BTF_ID()
6183 btf->data_size = data_size; in BTF_ID()
6184 btf->kernel_btf = true; in BTF_ID()
6185 snprintf(btf->name, sizeof(btf->name), "%s", name); in BTF_ID()
6191 btf->nohdr_data = btf->data + btf->hdr.hdr_len; in BTF_ID()
6205 refcount_set(&btf->refcnt, 1); in BTF_ID()
6211 kvfree(btf->types); in BTF_ID()
6226 return ERR_PTR(-ENOMEM); in btf_parse_vmlinux()
6228 log = &env->log; in btf_parse_vmlinux()
6229 log->level = BPF_LOG_KERNEL; in btf_parse_vmlinux()
6230 btf = btf_parse_base(env, "vmlinux", __start_BTF, __stop_BTF - __start_BTF); in btf_parse_vmlinux()
6252 if (!btf->base_btf || !btf->base_id_map) in btf_relocate_id()
6254 return btf->base_id_map[id]; in btf_relocate_id()
6272 return ERR_PTR(-EINVAL); in btf_parse_module()
6276 return ERR_PTR(-ENOMEM); in btf_parse_module()
6278 log = &env->log; in btf_parse_module()
6279 log->level = BPF_LOG_KERNEL; in btf_parse_module()
6293 err = -ENOMEM; in btf_parse_module()
6296 env->btf = btf; in btf_parse_module()
6298 btf->base_btf = base_btf; in btf_parse_module()
6299 btf->start_id = base_btf->nr_types; in btf_parse_module()
6300 btf->start_str_off = base_btf->hdr.str_len; in btf_parse_module()
6301 btf->kernel_btf = true; in btf_parse_module()
6302 snprintf(btf->name, sizeof(btf->name), "%s", module_name); in btf_parse_module()
6304 btf->data = kvmemdup(data, data_size, GFP_KERNEL | __GFP_NOWARN); in btf_parse_module()
6305 if (!btf->data) { in btf_parse_module()
6306 err = -ENOMEM; in btf_parse_module()
6309 btf->data_size = data_size; in btf_parse_module()
6315 btf->nohdr_data = btf->data + btf->hdr.hdr_len; in btf_parse_module()
6330 err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map); in btf_parse_module()
6338 refcount_set(&btf->refcnt, 1); in btf_parse_module()
6346 kvfree(btf->data); in btf_parse_module()
6347 kvfree(btf->types); in btf_parse_module()
6357 struct bpf_prog *tgt_prog = prog->aux->dst_prog; in bpf_prog_get_target_btf()
6360 return tgt_prog->aux->btf; in bpf_prog_get_target_btf()
6362 return prog->aux->attach_btf; in bpf_prog_get_target_btf()
6368 t = btf_type_skip_modifiers(btf, t->type, NULL); in is_int_ptr()
6388 offset += btf_type_is_ptr(t) ? 8 : roundup(t->size, 8); in get_ctx_arg_idx()
6393 t = btf_type_skip_modifiers(btf, func_proto->type, NULL); in get_ctx_arg_idx()
6394 offset += btf_type_is_ptr(t) ? 8 : roundup(t->size, 8); in get_ctx_arg_idx()
6403 enum bpf_attach_type atype = prog->expected_attach_type; in prog_args_trusted()
6405 switch (prog->type) { in prog_args_trusted()
6538 const struct btf_type *t = prog->aux->attach_func_proto; in btf_ctx_access()
6539 struct bpf_prog *tgt_prog = prog->aux->dst_prog; in btf_ctx_access()
6541 const char *tname = prog->aux->attach_func_name; in btf_ctx_access()
6542 struct bpf_verifier_log *log = info->log; in btf_ctx_access()
6560 if (prog->aux->attach_btf_trace) { in btf_ctx_access()
6563 nr_args--; in btf_ctx_access()
6567 bpf_log(log, "func '%s' doesn't have %d-th argument\n", in btf_ctx_access()
6573 switch (prog->expected_attach_type) { in btf_ctx_access()
6576 info->is_retval = true; in btf_ctx_access()
6584 * While the LSM programs are BPF_MODIFY_RETURN-like in btf_ctx_access()
6588 * return -EINVAL; in btf_ctx_access()
6595 t = btf_type_by_id(btf, t->type); in btf_ctx_access()
6604 t = btf_type_skip_modifiers(btf, t->type, NULL); in btf_ctx_access()
6613 bpf_log(log, "func '%s' doesn't have %d-th argument\n", in btf_ctx_access()
6626 t = btf_type_by_id(btf, t->type); in btf_ctx_access()
6634 __btf_name_by_offset(btf, t->name_off), in btf_ctx_access()
6646 for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { in btf_ctx_access()
6647 const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; in btf_ctx_access()
6650 type = base_type(ctx_arg_info->reg_type); in btf_ctx_access()
6651 flag = type_flag(ctx_arg_info->reg_type); in btf_ctx_access()
6652 if (ctx_arg_info->offset == off && type == PTR_TO_BUF && in btf_ctx_access()
6654 info->reg_type = ctx_arg_info->reg_type; in btf_ctx_access()
6659 if (t->type == 0) in btf_ctx_access()
6670 for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { in btf_ctx_access()
6671 const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; in btf_ctx_access()
6673 if (ctx_arg_info->offset == off) { in btf_ctx_access()
6674 if (!ctx_arg_info->btf_id) { in btf_ctx_access()
6679 info->reg_type = ctx_arg_info->reg_type; in btf_ctx_access()
6680 info->btf = ctx_arg_info->btf ? : btf_vmlinux; in btf_ctx_access()
6681 info->btf_id = ctx_arg_info->btf_id; in btf_ctx_access()
6686 info->reg_type = PTR_TO_BTF_ID; in btf_ctx_access()
6688 info->reg_type |= PTR_TRUSTED; in btf_ctx_access()
6691 info->reg_type |= PTR_MAYBE_NULL; in btf_ctx_access()
6693 if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { in btf_ctx_access()
6694 struct btf *btf = prog->aux->attach_btf; in btf_ctx_access()
6699 t = btf_type_by_id(btf, prog->aux->attach_btf_id); in btf_ctx_access()
6702 tname = btf_name_by_offset(btf, t->name_off); in btf_ctx_access()
6706 tname += sizeof("btf_trace_") - 1; in btf_ctx_access()
6712 info->reg_type |= PTR_MAYBE_NULL; in btf_ctx_access()
6718 /* If we don't know NULL-ness specification and the tracepoint in btf_ctx_access()
6723 info->reg_type |= PTR_MAYBE_NULL; in btf_ctx_access()
6729 if (tgt_prog->type == BPF_PROG_TYPE_EXT) in btf_ctx_access()
6730 tgt_type = tgt_prog->aux->saved_dst_prog_type; in btf_ctx_access()
6732 tgt_type = tgt_prog->type; in btf_ctx_access()
6736 info->btf = btf_vmlinux; in btf_ctx_access()
6737 info->btf_id = ret; in btf_ctx_access()
6744 info->btf = btf; in btf_ctx_access()
6745 info->btf_id = t->type; in btf_ctx_access()
6746 t = btf_type_by_id(btf, t->type); in btf_ctx_access()
6749 tag_value = __btf_name_by_offset(btf, t->name_off); in btf_ctx_access()
6751 info->reg_type |= MEM_USER; in btf_ctx_access()
6753 info->reg_type |= MEM_PERCPU; in btf_ctx_access()
6758 info->btf_id = t->type; in btf_ctx_access()
6759 t = btf_type_by_id(btf, t->type); in btf_ctx_access()
6768 tname, arg, info->btf_id, btf_type_str(t), in btf_ctx_access()
6769 __btf_name_by_offset(btf, t->name_off)); in btf_ctx_access()
6777 info->reg_type = SCALAR_VALUE; in btf_ctx_access()
6803 t = btf_type_skip_modifiers(btf, t->type, NULL); in btf_struct_walk()
6804 tname = __btf_name_by_offset(btf, t->name_off); in btf_struct_walk()
6807 return -EINVAL; in btf_struct_walk()
6811 if (BTF_INFO_KIND(t->info) == BTF_KIND_UNION && vlen != 1 && !(*flag & PTR_UNTRUSTED)) in btf_struct_walk()
6819 if (off + size > t->size) { in btf_struct_walk()
6828 member = btf_type_member(t) + vlen - 1; in btf_struct_walk()
6829 mtype = btf_type_skip_modifiers(btf, member->type, in btf_struct_walk()
6835 if (array_elem->nelems != 0) in btf_struct_walk()
6843 t = btf_type_skip_modifiers(btf, array_elem->type, in btf_struct_walk()
6852 off = (off - moff) % t->size; in btf_struct_walk()
6858 return -EACCES; in btf_struct_walk()
6904 mid = member->type; in btf_struct_walk()
6905 mtype = btf_type_by_id(btf, member->type); in btf_struct_walk()
6906 mname = __btf_name_by_offset(btf, member->name_off); in btf_struct_walk()
6913 return -EFAULT; in btf_struct_walk()
6925 * linearize a multi-dimensional array. in btf_struct_walk()
6943 * When accessing outer->array[1][0], it moves in btf_struct_walk()
6969 elem_idx = (off - moff) / msize; in btf_struct_walk()
6989 off -= moff; in btf_struct_walk()
7002 return -EACCES; in btf_struct_walk()
7006 t = btf_type_by_id(btf, mtype->type); in btf_struct_walk()
7008 tag_value = __btf_name_by_offset(btf, t->name_off); in btf_struct_walk()
7020 stype = btf_type_skip_modifiers(btf, mtype->type, &id); in btf_struct_walk()
7034 * space. e.g. skb->cb[]. in btf_struct_walk()
7040 return -EACCES; in btf_struct_walk()
7046 return -EINVAL; in btf_struct_walk()
7055 const struct btf *btf = reg->btf; in btf_struct_access()
7058 u32 id = reg->btf_id; in btf_struct_access()
7061 while (type_is_alloc(reg->type)) { in btf_struct_access()
7069 rec = meta->record; in btf_struct_access()
7070 for (i = 0; i < rec->cnt; i++) { in btf_struct_access()
7071 struct btf_field *field = &rec->fields[i]; in btf_struct_access()
7072 u32 offset = field->offset; in btf_struct_access()
7073 if (off < offset + field->size && offset < off + size) { in btf_struct_access()
7076 btf_field_type_name(field->type)); in btf_struct_access()
7077 return -EACCES; in btf_struct_access()
7092 if (type_is_alloc(reg->type)) in btf_struct_access()
7115 return -EINVAL; in btf_struct_access()
7120 return -EINVAL; in btf_struct_access()
7188 t = btf_type_by_id(btf, t->type); in __get_type_size()
7190 return -EINVAL; in __get_type_size()
7196 return t->size; in __get_type_size()
7197 return -EINVAL; in __get_type_size()
7228 m->arg_size[i] = 8; in btf_distill_func_proto()
7229 m->arg_flags[i] = 0; in btf_distill_func_proto()
7231 m->ret_size = 8; in btf_distill_func_proto()
7232 m->ret_flags = 0; in btf_distill_func_proto()
7233 m->nr_args = MAX_BPF_FUNC_REG_ARGS; in btf_distill_func_proto()
7242 return -EINVAL; in btf_distill_func_proto()
7244 ret = __get_type_size(btf, func->type, &t); in btf_distill_func_proto()
7249 return -EINVAL; in btf_distill_func_proto()
7251 m->ret_size = ret; in btf_distill_func_proto()
7252 m->ret_flags = __get_type_fmodel_flags(t); in btf_distill_func_proto()
7255 if (i == nargs - 1 && args[i].type == 0) { in btf_distill_func_proto()
7259 return -EINVAL; in btf_distill_func_proto()
7268 return -EINVAL; in btf_distill_func_proto()
7274 return -EINVAL; in btf_distill_func_proto()
7276 m->arg_size[i] = ret; in btf_distill_func_proto()
7277 m->arg_flags[i] = __get_type_fmodel_flags(t); in btf_distill_func_proto()
7279 m->nr_args = nargs; in btf_distill_func_proto()
7287 * EINVAL - function prototype mismatch
7288 * EFAULT - verifier bug
7289 * 0 - 99% match. The last 1% is validated by the verifier.
7299 fn1 = btf_name_by_offset(btf1, t1->name_off); in btf_check_func_type_match()
7300 fn2 = btf_name_by_offset(btf2, t2->name_off); in btf_check_func_type_match()
7304 return -EINVAL; in btf_check_func_type_match()
7308 return -EINVAL; in btf_check_func_type_match()
7311 t1 = btf_type_by_id(btf1, t1->type); in btf_check_func_type_match()
7313 return -EFAULT; in btf_check_func_type_match()
7314 t2 = btf_type_by_id(btf2, t2->type); in btf_check_func_type_match()
7316 return -EFAULT; in btf_check_func_type_match()
7326 return -EINVAL; in btf_check_func_type_match()
7329 t1 = btf_type_skip_modifiers(btf1, t1->type, NULL); in btf_check_func_type_match()
7330 t2 = btf_type_skip_modifiers(btf2, t2->type, NULL); in btf_check_func_type_match()
7331 if (t1->info != t2->info) { in btf_check_func_type_match()
7336 return -EINVAL; in btf_check_func_type_match()
7343 if (t1->info != t2->info) { in btf_check_func_type_match()
7347 return -EINVAL; in btf_check_func_type_match()
7349 if (btf_type_has_size(t1) && t1->size != t2->size) { in btf_check_func_type_match()
7352 i, fn1, t1->size, in btf_check_func_type_match()
7353 fn2, t2->size); in btf_check_func_type_match()
7354 return -EINVAL; in btf_check_func_type_match()
7367 return -EINVAL; in btf_check_func_type_match()
7369 t1 = btf_type_skip_modifiers(btf1, t1->type, NULL); in btf_check_func_type_match()
7370 t2 = btf_type_skip_modifiers(btf2, t2->type, NULL); in btf_check_func_type_match()
7375 return -EINVAL; in btf_check_func_type_match()
7381 return -EINVAL; in btf_check_func_type_match()
7389 s1 = btf_name_by_offset(btf1, t1->name_off); in btf_check_func_type_match()
7390 s2 = btf_name_by_offset(btf2, t2->name_off); in btf_check_func_type_match()
7395 return -EINVAL; in btf_check_func_type_match()
7405 struct btf *btf1 = prog->aux->btf; in btf_check_type_match()
7409 if (!prog->aux->func_info) { in btf_check_type_match()
7411 return -EINVAL; in btf_check_type_match()
7414 btf_id = prog->aux->func_info[0].type_id; in btf_check_type_match()
7416 return -EFAULT; in btf_check_type_match()
7420 return -EFAULT; in btf_check_type_match()
7429 t = btf_type_by_id(btf, t->type); /* skip PTR */ in btf_is_dynptr_ptr()
7432 t = btf_type_by_id(btf, t->type); in btf_is_dynptr_ptr()
7437 name = btf_str_by_offset(btf, t->name_off); in btf_is_dynptr_ptr()
7472 type_id = t->type; in btf_get_ptr_to_btf_id()
7473 t = btf_type_by_id(btf, t->type); in btf_get_ptr_to_btf_id()
7475 type_id = t->type; in btf_get_ptr_to_btf_id()
7476 t = btf_type_by_id(btf, t->type); in btf_get_ptr_to_btf_id()
7484 arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off), in btf_get_ptr_to_btf_id()
7488 if (cc->cnt != 1) { in btf_get_ptr_to_btf_id()
7490 arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off), in btf_get_ptr_to_btf_id()
7491 cc->cnt == 0 ? "has no matches" : "is ambiguous"); in btf_get_ptr_to_btf_id()
7492 err = cc->cnt == 0 ? -ENOENT : -ESRCH; in btf_get_ptr_to_btf_id()
7495 if (btf_is_module(cc->cands[0].btf)) { in btf_get_ptr_to_btf_id()
7497 arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off)); in btf_get_ptr_to_btf_id()
7498 err = -EOPNOTSUPP; in btf_get_ptr_to_btf_id()
7501 kern_type_id = cc->cands[0].id; in btf_get_ptr_to_btf_id()
7519 /* Process BTF of a function to produce high-level expectation of function
7523 * EFAULT - there is a verifier bug. Abort verification.
7524 * EINVAL - cannot convert BTF.
7525 * 0 - Successfully processed BTF and constructed argument expectations.
7529 bool is_global = subprog_aux(env, subprog)->linkage == BTF_FUNC_GLOBAL; in btf_prepare_func_args()
7531 struct bpf_verifier_log *log = &env->log; in btf_prepare_func_args()
7532 struct bpf_prog *prog = env->prog; in btf_prepare_func_args()
7533 enum bpf_prog_type prog_type = prog->type; in btf_prepare_func_args()
7534 struct btf *btf = prog->aux->btf; in btf_prepare_func_args()
7540 if (sub->args_cached) in btf_prepare_func_args()
7543 if (!prog->aux->func_info) { in btf_prepare_func_args()
7545 return -EFAULT; in btf_prepare_func_args()
7548 btf_id = prog->aux->func_info[subprog].type_id; in btf_prepare_func_args()
7551 return -EINVAL; in btf_prepare_func_args()
7553 return -EFAULT; in btf_prepare_func_args()
7563 return -EFAULT; in btf_prepare_func_args()
7565 tname = btf_name_by_offset(btf, fn_t->name_off); in btf_prepare_func_args()
7567 if (prog->aux->func_info_aux[subprog].unreliable) { in btf_prepare_func_args()
7569 return -EFAULT; in btf_prepare_func_args()
7572 prog_type = prog->aux->dst_prog->type; in btf_prepare_func_args()
7574 t = btf_type_by_id(btf, fn_t->type); in btf_prepare_func_args()
7577 return -EFAULT; in btf_prepare_func_args()
7583 return -EINVAL; in btf_prepare_func_args()
7586 return -EINVAL; in btf_prepare_func_args()
7589 t = btf_type_by_id(btf, t->type); in btf_prepare_func_args()
7591 t = btf_type_by_id(btf, t->type); in btf_prepare_func_args()
7594 return -EINVAL; in btf_prepare_func_args()
7598 return -EINVAL; in btf_prepare_func_args()
7612 const char *tag = __btf_name_by_offset(btf, tag_t->name_off) + 4; in btf_prepare_func_args()
7617 return -EOPNOTSUPP; in btf_prepare_func_args()
7632 return -EOPNOTSUPP; in btf_prepare_func_args()
7635 if (id != -ENOENT) { in btf_prepare_func_args()
7642 t = btf_type_by_id(btf, t->type); in btf_prepare_func_args()
7649 return -EINVAL; in btf_prepare_func_args()
7653 prog->expected_attach_type)) in btf_prepare_func_args()
7654 return -EINVAL; in btf_prepare_func_args()
7655 sub->args[i].arg_type = ARG_PTR_TO_CTX; in btf_prepare_func_args()
7661 return -EINVAL; in btf_prepare_func_args()
7663 sub->args[i].arg_type = ARG_PTR_TO_DYNPTR | MEM_RDONLY; in btf_prepare_func_args()
7671 return -EINVAL; in btf_prepare_func_args()
7678 sub->args[i].arg_type = ARG_PTR_TO_BTF_ID | PTR_TRUSTED; in btf_prepare_func_args()
7680 sub->args[i].arg_type |= PTR_MAYBE_NULL; in btf_prepare_func_args()
7681 sub->args[i].btf_id = kern_type_id; in btf_prepare_func_args()
7687 return -EINVAL; in btf_prepare_func_args()
7689 sub->args[i].arg_type = ARG_PTR_TO_ARENA; in btf_prepare_func_args()
7697 return -EINVAL; in btf_prepare_func_args()
7700 t = btf_type_skip_modifiers(btf, t->type, NULL); in btf_prepare_func_args()
7704 i, btf_type_str(t), btf_name_by_offset(btf, t->name_off), in btf_prepare_func_args()
7706 return -EINVAL; in btf_prepare_func_args()
7709 sub->args[i].arg_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL; in btf_prepare_func_args()
7711 sub->args[i].arg_type &= ~PTR_MAYBE_NULL; in btf_prepare_func_args()
7712 sub->args[i].mem_size = mem_size; in btf_prepare_func_args()
7719 return -EINVAL; in btf_prepare_func_args()
7722 sub->args[i].arg_type = ARG_ANYTHING; in btf_prepare_func_args()
7726 return -EINVAL; in btf_prepare_func_args()
7729 return -EINVAL; in btf_prepare_func_args()
7732 sub->arg_cnt = nargs; in btf_prepare_func_args()
7733 sub->args_cached = true; in btf_prepare_func_args()
7743 show->btf = btf; in btf_type_show()
7744 memset(&show->state, 0, sizeof(show->state)); in btf_type_show()
7745 memset(&show->obj, 0, sizeof(show->obj)); in btf_type_show()
7747 btf_type_ops(t)->show(btf, t, type_id, obj, 0, show); in btf_type_show()
7753 seq_vprintf((struct seq_file *)show->target, fmt, args); in btf_seq_show()
7790 len = vsnprintf(show->target, ssnprintf->len_left, fmt, args); in btf_snprintf_show()
7793 ssnprintf->len_left = 0; in btf_snprintf_show()
7794 ssnprintf->len = len; in btf_snprintf_show()
7795 } else if (len >= ssnprintf->len_left) { in btf_snprintf_show()
7797 ssnprintf->len_left = 0; in btf_snprintf_show()
7798 ssnprintf->len += len; in btf_snprintf_show()
7800 ssnprintf->len_left -= len; in btf_snprintf_show()
7801 ssnprintf->len += len; in btf_snprintf_show()
7802 show->target += len; in btf_snprintf_show()
7830 const struct btf *btf = filp->private_data; in bpf_btf_show_fdinfo()
7832 seq_printf(m, "btf_id:\t%u\n", btf->id); in bpf_btf_show_fdinfo()
7838 btf_put(filp->private_data); in btf_release()
7889 refcount_inc(&btf->refcnt); in btf_get_by_fd()
7906 uinfo = u64_to_user_ptr(attr->info.info); in btf_get_info_by_fd()
7907 uinfo_len = attr->info.info_len; in btf_get_info_by_fd()
7912 return -EFAULT; in btf_get_info_by_fd()
7914 info.id = btf->id; in btf_get_info_by_fd()
7916 btf_copy = min_t(u32, btf->data_size, info.btf_size); in btf_get_info_by_fd()
7917 if (copy_to_user(ubtf, btf->data, btf_copy)) in btf_get_info_by_fd()
7918 return -EFAULT; in btf_get_info_by_fd()
7919 info.btf_size = btf->data_size; in btf_get_info_by_fd()
7921 info.kernel_btf = btf->kernel_btf; in btf_get_info_by_fd()
7926 return -EINVAL; in btf_get_info_by_fd()
7928 name_len = strlen(btf->name); in btf_get_info_by_fd()
7933 if (copy_to_user(uname, btf->name, name_len + 1)) in btf_get_info_by_fd()
7934 return -EFAULT; in btf_get_info_by_fd()
7938 if (copy_to_user(uname, btf->name, uname_len - 1)) in btf_get_info_by_fd()
7939 return -EFAULT; in btf_get_info_by_fd()
7940 if (put_user(zero, uname + uname_len - 1)) in btf_get_info_by_fd()
7941 return -EFAULT; in btf_get_info_by_fd()
7942 /* let user-space know about too short buffer */ in btf_get_info_by_fd()
7943 ret = -ENOSPC; in btf_get_info_by_fd()
7948 put_user(info_copy, &uattr->info.info_len)) in btf_get_info_by_fd()
7949 return -EFAULT; in btf_get_info_by_fd()
7961 if (!btf || !refcount_inc_not_zero(&btf->refcnt)) in btf_get_fd_by_id()
7962 btf = ERR_PTR(-ENOENT); in btf_get_fd_by_id()
7977 return btf->id; in btf_obj_id()
7982 return btf->kernel_btf; in btf_is_kernel()
7987 return btf->kernel_btf && strcmp(btf->name, "vmlinux") != 0; in btf_is_module()
8016 if (mod->btf_data_size == 0 || in btf_module_notify()
8025 err = -ENOMEM; in btf_module_notify()
8028 btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size, in btf_module_notify()
8029 mod->btf_base_data, mod->btf_base_data_size); in btf_module_notify()
8034 mod->name, PTR_ERR(btf)); in btf_module_notify()
8050 btf_mod->module = module; in btf_module_notify()
8051 btf_mod->btf = btf; in btf_module_notify()
8052 list_add(&btf_mod->list, &btf_modules); in btf_module_notify()
8063 attr->attr.name = btf->name; in btf_module_notify()
8064 attr->attr.mode = 0444; in btf_module_notify()
8065 attr->size = btf->data_size; in btf_module_notify()
8066 attr->private = btf->data; in btf_module_notify()
8067 attr->read_new = sysfs_bin_attr_simple_read; in btf_module_notify()
8072 mod->name, err); in btf_module_notify()
8078 btf_mod->sysfs_attr = attr; in btf_module_notify()
8085 if (btf_mod->module != module) in btf_module_notify()
8088 btf_mod->flags |= BTF_MODULE_F_LIVE; in btf_module_notify()
8096 if (btf_mod->module != module) in btf_module_notify()
8099 list_del(&btf_mod->list); in btf_module_notify()
8100 if (btf_mod->sysfs_attr) in btf_module_notify()
8101 sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr); in btf_module_notify()
8102 purge_cand_cache(btf_mod->btf); in btf_module_notify()
8103 btf_put(btf_mod->btf); in btf_module_notify()
8104 kfree(btf_mod->sysfs_attr); in btf_module_notify()
8136 if (btf_mod->btf != btf) in btf_try_get_module()
8144 if ((btf_mod->flags & BTF_MODULE_F_LIVE) && try_module_get(btf_mod->module)) in btf_try_get_module()
8145 res = btf_mod->module; in btf_try_get_module()
8175 if (btf_mod->module != module) in btf_get_module_btf()
8178 btf_get(btf_mod->btf); in btf_get_module_btf()
8179 btf = btf_mod->btf; in btf_get_module_btf()
8192 return -ENOENT; in check_btf_kconfigs()
8206 return -EINVAL; in BPF_CALL_4()
8208 if (name_sz <= 1 || name[name_sz - 1]) in BPF_CALL_4()
8209 return -EINVAL; in BPF_CALL_4()
8240 /* Validate well-formedness of iter argument type. in BTF_ID_LIST_GLOBAL()
8252 return -EINVAL; in BTF_ID_LIST_GLOBAL()
8255 t = btf_type_skip_modifiers(btf, arg->type, NULL); in BTF_ID_LIST_GLOBAL()
8257 return -EINVAL; in BTF_ID_LIST_GLOBAL()
8258 t = btf_type_skip_modifiers(btf, t->type, &btf_id); in BTF_ID_LIST_GLOBAL()
8260 return -EINVAL; in BTF_ID_LIST_GLOBAL()
8262 name = btf_name_by_offset(btf, t->name_off); in BTF_ID_LIST_GLOBAL()
8263 if (!name || strncmp(name, ITER_PREFIX, sizeof(ITER_PREFIX) - 1)) in BTF_ID_LIST_GLOBAL()
8264 return -EINVAL; in BTF_ID_LIST_GLOBAL()
8280 if (!flags || (flags & (flags - 1))) in btf_check_iter_kfuncs()
8281 return -EINVAL; in btf_check_iter_kfuncs()
8286 return -EINVAL; in btf_check_iter_kfuncs()
8296 if (t->size == 0 || (t->size % 8)) in btf_check_iter_kfuncs()
8297 return -EINVAL; in btf_check_iter_kfuncs()
8302 iter_name = btf_name_by_offset(btf, t->name_off) + sizeof(ITER_PREFIX) - 1; in btf_check_iter_kfuncs()
8312 return -EINVAL; in btf_check_iter_kfuncs()
8316 return -EINVAL; in btf_check_iter_kfuncs()
8320 t = btf_type_skip_modifiers(btf, func->type, NULL); in btf_check_iter_kfuncs()
8322 return -EINVAL; in btf_check_iter_kfuncs()
8327 t = btf_type_by_id(btf, func->type); in btf_check_iter_kfuncs()
8329 return -EINVAL; in btf_check_iter_kfuncs()
8341 /* any kfunc should be FUNC -> FUNC_PROTO */ in btf_check_kfunc_protos()
8344 return -EINVAL; in btf_check_kfunc_protos()
8347 func_name = btf_name_by_offset(btf, func->name_off); in btf_check_kfunc_protos()
8349 return -EINVAL; in btf_check_kfunc_protos()
8351 func = btf_type_by_id(btf, func->type); in btf_check_kfunc_protos()
8353 return -EINVAL; in btf_check_kfunc_protos()
8370 struct btf_id_set8 *add_set = kset->set; in btf_populate_kfunc_set()
8372 bool add_filter = !!kset->filter; in btf_populate_kfunc_set()
8379 ret = -EINVAL; in btf_populate_kfunc_set()
8383 if (!add_set->cnt) in btf_populate_kfunc_set()
8386 tab = btf->kfunc_set_tab; in btf_populate_kfunc_set()
8391 hook_filter = &tab->hook_filters[hook]; in btf_populate_kfunc_set()
8392 for (i = 0; i < hook_filter->nr_filters; i++) { in btf_populate_kfunc_set()
8393 if (hook_filter->filters[i] == kset->filter) { in btf_populate_kfunc_set()
8399 if (add_filter && hook_filter->nr_filters == BTF_KFUNC_FILTER_MAX_CNT) { in btf_populate_kfunc_set()
8400 ret = -E2BIG; in btf_populate_kfunc_set()
8408 return -ENOMEM; in btf_populate_kfunc_set()
8409 btf->kfunc_set_tab = tab; in btf_populate_kfunc_set()
8412 set = tab->sets[hook]; in btf_populate_kfunc_set()
8417 ret = -EINVAL; in btf_populate_kfunc_set()
8425 * hence re-sorting the final set again is required to make binary in btf_populate_kfunc_set()
8431 set_cnt = set ? set->cnt : 0; in btf_populate_kfunc_set()
8433 if (set_cnt > U32_MAX - add_set->cnt) { in btf_populate_kfunc_set()
8434 ret = -EOVERFLOW; in btf_populate_kfunc_set()
8438 if (set_cnt + add_set->cnt > BTF_KFUNC_SET_MAX_CNT) { in btf_populate_kfunc_set()
8439 ret = -E2BIG; in btf_populate_kfunc_set()
8444 set = krealloc(tab->sets[hook], in btf_populate_kfunc_set()
8445 offsetof(struct btf_id_set8, pairs[set_cnt + add_set->cnt]), in btf_populate_kfunc_set()
8448 ret = -ENOMEM; in btf_populate_kfunc_set()
8452 /* For newly allocated set, initialize set->cnt to 0 */ in btf_populate_kfunc_set()
8453 if (!tab->sets[hook]) in btf_populate_kfunc_set()
8454 set->cnt = 0; in btf_populate_kfunc_set()
8455 tab->sets[hook] = set; in btf_populate_kfunc_set()
8458 memcpy(set->pairs + set->cnt, add_set->pairs, add_set->cnt * sizeof(set->pairs[0])); in btf_populate_kfunc_set()
8460 for (i = set->cnt; i < set->cnt + add_set->cnt; i++) in btf_populate_kfunc_set()
8461 set->pairs[i].id = btf_relocate_id(btf, set->pairs[i].id); in btf_populate_kfunc_set()
8463 set->cnt += add_set->cnt; in btf_populate_kfunc_set()
8465 sort(set->pairs, set->cnt, sizeof(set->pairs[0]), btf_id_cmp_func, NULL); in btf_populate_kfunc_set()
8468 hook_filter = &tab->hook_filters[hook]; in btf_populate_kfunc_set()
8469 hook_filter->filters[hook_filter->nr_filters++] = kset->filter; in btf_populate_kfunc_set()
8488 if (!btf->kfunc_set_tab) in __btf_kfunc_id_set_contains()
8490 hook_filter = &btf->kfunc_set_tab->hook_filters[hook]; in __btf_kfunc_id_set_contains()
8491 for (i = 0; i < hook_filter->nr_filters; i++) { in __btf_kfunc_id_set_contains()
8492 if (hook_filter->filters[i](prog, kfunc_btf_id)) in __btf_kfunc_id_set_contains()
8495 set = btf->kfunc_set_tab->sets[hook]; in __btf_kfunc_id_set_contains()
8555 * protection for looking up a well-formed btf->kfunc_set_tab.
8585 btf = btf_get_module_btf(kset->owner); in __register_btf_kfunc_id_set()
8587 return check_btf_kconfigs(kset->owner, "kfunc"); in __register_btf_kfunc_id_set()
8591 for (i = 0; i < kset->set->cnt; i++) { in __register_btf_kfunc_id_set()
8592 ret = btf_check_kfunc_protos(btf, btf_relocate_id(btf, kset->set->pairs[i].id), in __register_btf_kfunc_id_set()
8593 kset->set->pairs[i].flags); in __register_btf_kfunc_id_set()
8614 if (!(kset->set->flags & BTF_SET8_KFUNCS)) { in register_btf_kfunc_id_set()
8615 WARN_ON(!kset->owner); in register_btf_kfunc_id_set()
8616 return -EINVAL; in register_btf_kfunc_id_set()
8633 struct btf_id_dtor_kfunc_tab *tab = btf->dtor_kfunc_tab; in btf_find_dtor_kfunc()
8637 return -ENOENT; in btf_find_dtor_kfunc()
8638 /* Even though the size of tab->dtors[0] is > sizeof(u32), we only need in btf_find_dtor_kfunc()
8642 dtor = bsearch(&btf_id, tab->dtors, tab->cnt, sizeof(tab->dtors[0]), btf_id_cmp_func); in btf_find_dtor_kfunc()
8644 return -ENOENT; in btf_find_dtor_kfunc()
8645 return dtor->kfunc_btf_id; in btf_find_dtor_kfunc()
8660 return -EINVAL; in btf_check_dtor_kfuncs()
8662 dtor_func_proto = btf_type_by_id(btf, dtor_func->type); in btf_check_dtor_kfuncs()
8664 return -EINVAL; in btf_check_dtor_kfuncs()
8667 t = btf_type_by_id(btf, dtor_func_proto->type); in btf_check_dtor_kfuncs()
8669 return -EINVAL; in btf_check_dtor_kfuncs()
8673 return -EINVAL; in btf_check_dtor_kfuncs()
8680 return -EINVAL; in btf_check_dtor_kfuncs()
8702 ret = -E2BIG; in register_btf_id_dtor_kfuncs()
8711 tab = btf->dtor_kfunc_tab; in register_btf_id_dtor_kfuncs()
8714 ret = -EINVAL; in register_btf_id_dtor_kfuncs()
8718 tab_cnt = tab ? tab->cnt : 0; in register_btf_id_dtor_kfuncs()
8719 if (tab_cnt > U32_MAX - add_cnt) { in register_btf_id_dtor_kfuncs()
8720 ret = -EOVERFLOW; in register_btf_id_dtor_kfuncs()
8725 ret = -E2BIG; in register_btf_id_dtor_kfuncs()
8729 tab = krealloc(btf->dtor_kfunc_tab, in register_btf_id_dtor_kfuncs()
8733 ret = -ENOMEM; in register_btf_id_dtor_kfuncs()
8737 if (!btf->dtor_kfunc_tab) in register_btf_id_dtor_kfuncs()
8738 tab->cnt = 0; in register_btf_id_dtor_kfuncs()
8739 btf->dtor_kfunc_tab = tab; in register_btf_id_dtor_kfuncs()
8741 memcpy(tab->dtors + tab->cnt, dtors, add_cnt * sizeof(tab->dtors[0])); in register_btf_id_dtor_kfuncs()
8745 tab->dtors[i].btf_id = btf_relocate_id(btf, tab->dtors[i].btf_id); in register_btf_id_dtor_kfuncs()
8746 tab->dtors[i].kfunc_btf_id = btf_relocate_id(btf, tab->dtors[i].kfunc_btf_id); in register_btf_id_dtor_kfuncs()
8749 tab->cnt += add_cnt; in register_btf_id_dtor_kfuncs()
8751 sort(tab->dtors, tab->cnt, sizeof(tab->dtors[0]), btf_id_cmp_func, NULL); in register_btf_id_dtor_kfuncs()
8764 * type-based CO-RE relocations and follow slightly different rules than
8765 * field-based relocations. This function assumes that root types were already
8766 * checked for name match. Beyond that initial root-level name check, names
8768 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs/ENUM64s are considered compatible, but
8771 * - for ENUMs/ENUM64s, the size is ignored;
8772 * - for INT, size and signedness are ignored;
8773 * - for ARRAY, dimensionality is ignored, element types are checked for
8775 * - CONST/VOLATILE/RESTRICT modifiers are ignored;
8776 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
8777 * - FUNC_PROTOs are compatible if they have compatible signature: same
8780 * more experience with using BPF CO-RE relocations.
8811 for (i = n - 5; i >= 0; i--) { in bpf_core_essential_name_len()
8820 if (!cands->cnt) in bpf_free_cands()
8828 kfree(cands->name); in bpf_free_cands_from_cache()
8849 bpf_log(log, "[%d]%s(", i, cc->name); in __print_cand_cache()
8850 for (j = 0; j < cc->cnt; j++) { in __print_cand_cache()
8851 bpf_log(log, "%d", cc->cands[j].id); in __print_cand_cache()
8852 if (j < cc->cnt - 1) in __print_cand_cache()
8872 return jhash(cands->name, cands->name_len, 0); in hash_cands()
8881 if (cc && cc->name_len == cands->name_len && in check_cand_cache()
8882 !strncmp(cc->name, cands->name, cands->name_len)) in check_cand_cache()
8902 new_cands = kmemdup(cands, sizeof_cands(cands->cnt), GFP_KERNEL); in populate_cand_cache()
8905 return ERR_PTR(-ENOMEM); in populate_cand_cache()
8908 * the cands->name points to strings in prog's BTF and the prog can be unloaded. in populate_cand_cache()
8910 new_cands->name = kmemdup_nul(cands->name, cands->name_len, GFP_KERNEL); in populate_cand_cache()
8912 if (!new_cands->name) { in populate_cand_cache()
8914 return ERR_PTR(-ENOMEM); in populate_cand_cache()
8943 for (j = 0; j < cc->cnt; j++) in __purge_cand_cache()
8944 if (cc->cands[j].btf == btf) { in __purge_cand_cache()
8974 if (btf_kind(t) != cands->kind) in bpf_core_add_cands()
8977 targ_name = btf_name_by_offset(targ_btf, t->name_off); in bpf_core_add_cands()
8982 * for non-existing name will have a chance to schedule(). in bpf_core_add_cands()
8986 if (strncmp(cands->name, targ_name, cands->name_len) != 0) in bpf_core_add_cands()
8990 if (targ_essent_len != cands->name_len) in bpf_core_add_cands()
8994 new_cands = kmalloc(sizeof_cands(cands->cnt + 1), GFP_KERNEL); in bpf_core_add_cands()
8997 return ERR_PTR(-ENOMEM); in bpf_core_add_cands()
9000 memcpy(new_cands, cands, sizeof_cands(cands->cnt)); in bpf_core_add_cands()
9003 cands->cands[cands->cnt].btf = targ_btf; in bpf_core_add_cands()
9004 cands->cands[cands->cnt].id = i; in bpf_core_add_cands()
9005 cands->cnt++; in bpf_core_add_cands()
9014 const struct btf *local_btf = ctx->btf; in bpf_core_find_cands()
9026 return ERR_PTR(-EINVAL); in bpf_core_find_cands()
9030 return ERR_PTR(-EINVAL); in bpf_core_find_cands()
9032 name = btf_name_by_offset(local_btf, local_type->name_off); in bpf_core_find_cands()
9034 return ERR_PTR(-EINVAL); in bpf_core_find_cands()
9038 cands->name = name; in bpf_core_find_cands()
9039 cands->kind = btf_kind(local_type); in bpf_core_find_cands()
9040 cands->name_len = local_essent_len; in bpf_core_find_cands()
9045 if (cc->cnt) in bpf_core_find_cands()
9055 /* cands is a pointer to kmalloced memory here if cands->cnt > 0 */ in bpf_core_find_cands()
9057 /* populate cache even when cands->cnt == 0 */ in bpf_core_find_cands()
9063 if (cc->cnt) in bpf_core_find_cands()
9067 /* cands is a pointer to stack here and cands->cnt == 0 */ in bpf_core_find_cands()
9070 /* if cache has it return it even if cc->cnt == 0 */ in bpf_core_find_cands()
9090 /* cands is a pointer to kmalloced memory here if cands->cnt > 0 in bpf_core_find_cands()
9091 * or pointer to stack if cands->cnd == 0. in bpf_core_find_cands()
9092 * Copy it into the cache even when cands->cnt == 0 and in bpf_core_find_cands()
9101 bool need_cands = relo->kind != BPF_CORE_TYPE_ID_LOCAL; in bpf_core_apply()
9113 return -ENOMEM; in bpf_core_apply()
9115 type = btf_type_by_id(ctx->btf, relo->type_id); in bpf_core_apply()
9117 bpf_log(ctx->log, "relo #%u: bad type id %u\n", in bpf_core_apply()
9118 relo_idx, relo->type_id); in bpf_core_apply()
9120 return -EINVAL; in bpf_core_apply()
9128 cc = bpf_core_find_cands(ctx, relo->type_id); in bpf_core_apply()
9130 bpf_log(ctx->log, "target candidate search failed for %d\n", in bpf_core_apply()
9131 relo->type_id); in bpf_core_apply()
9135 if (cc->cnt) { in bpf_core_apply()
9136 cands.cands = kcalloc(cc->cnt, sizeof(*cands.cands), GFP_KERNEL); in bpf_core_apply()
9138 err = -ENOMEM; in bpf_core_apply()
9142 for (i = 0; i < cc->cnt; i++) { in bpf_core_apply()
9143 bpf_log(ctx->log, in bpf_core_apply()
9144 "CO-RE relocating %s %s: found target candidate [%d]\n", in bpf_core_apply()
9145 btf_kind_str[cc->kind], cc->name, cc->cands[i].id); in bpf_core_apply()
9146 cands.cands[i].btf = cc->cands[i].btf; in bpf_core_apply()
9147 cands.cands[i].id = cc->cands[i].id; in bpf_core_apply()
9149 cands.len = cc->cnt; in bpf_core_apply()
9157 err = bpf_core_calc_relo_insn((void *)ctx->log, relo, relo_idx, ctx->btf, &cands, specs, in bpf_core_apply()
9162 err = bpf_core_patch_insn((void *)ctx->log, insn, relo->insn_off / 8, relo, relo_idx, in bpf_core_apply()
9170 if (ctx->log->level & BPF_LOG_LEVEL2) in bpf_core_apply()
9171 print_cand_cache(ctx->log); in bpf_core_apply()
9180 struct btf *btf = reg->btf; in btf_nested_type_is_trusted()
9188 walk_type = btf_type_by_id(btf, reg->btf_id); in btf_nested_type_is_trusted()
9192 tname = btf_name_by_offset(btf, walk_type->name_off); in btf_nested_type_is_trusted()
9198 safe_id = btf_find_by_name_kind(btf, safe_tname, BTF_INFO_KIND(walk_type->info)); in btf_nested_type_is_trusted()
9207 const char *m_name = __btf_name_by_offset(btf, member->name_off); in btf_nested_type_is_trusted()
9208 const struct btf_type *mtype = btf_type_by_id(btf, member->type); in btf_nested_type_is_trusted()
9214 btf_type_skip_modifiers(btf, mtype->type, &id); in btf_nested_type_is_trusted()
9230 size_t pattern_len = sizeof(NOCAST_ALIAS_SUFFIX) - sizeof(char); in btf_type_ids_nocast_alias()
9240 reg_name = btf_name_by_offset(reg_btf, reg_type->name_off); in btf_type_ids_nocast_alias()
9241 arg_name = btf_name_by_offset(arg_btf, arg_type->name_off); in btf_type_ids_nocast_alias()
9247 * if the strings are the same size, they can't possibly be no-cast in btf_type_ids_nocast_alias()
9250 * because they are _not_ no-cast aliases, they are the same type. in btf_type_ids_nocast_alias()
9286 tab = btf->struct_ops_tab; in btf_add_struct_ops()
9291 return -ENOMEM; in btf_add_struct_ops()
9292 tab->capacity = 4; in btf_add_struct_ops()
9293 btf->struct_ops_tab = tab; in btf_add_struct_ops()
9296 for (i = 0; i < tab->cnt; i++) in btf_add_struct_ops()
9297 if (tab->ops[i].st_ops == st_ops) in btf_add_struct_ops()
9298 return -EEXIST; in btf_add_struct_ops()
9300 if (tab->cnt == tab->capacity) { in btf_add_struct_ops()
9303 ops[tab->capacity * 2]), in btf_add_struct_ops()
9306 return -ENOMEM; in btf_add_struct_ops()
9308 tab->capacity *= 2; in btf_add_struct_ops()
9309 btf->struct_ops_tab = tab; in btf_add_struct_ops()
9312 tab->ops[btf->struct_ops_tab->cnt].st_ops = st_ops; in btf_add_struct_ops()
9314 err = bpf_struct_ops_desc_init(&tab->ops[btf->struct_ops_tab->cnt], btf, log); in btf_add_struct_ops()
9318 btf->struct_ops_tab->cnt++; in btf_add_struct_ops()
9332 if (!btf->struct_ops_tab) in bpf_struct_ops_find_value()
9335 cnt = btf->struct_ops_tab->cnt; in bpf_struct_ops_find_value()
9336 st_ops_list = btf->struct_ops_tab->ops; in bpf_struct_ops_find_value()
9354 if (!btf->struct_ops_tab) in bpf_struct_ops_find()
9357 cnt = btf->struct_ops_tab->cnt; in bpf_struct_ops_find()
9358 st_ops_list = btf->struct_ops_tab->ops; in bpf_struct_ops_find()
9373 btf = btf_get_module_btf(st_ops->owner); in __register_bpf_struct_ops()
9375 return check_btf_kconfigs(st_ops->owner, "struct_ops"); in __register_bpf_struct_ops()
9381 err = -ENOMEM; in __register_bpf_struct_ops()
9385 log->level = BPF_LOG_KERNEL; in __register_bpf_struct_ops()
9405 /* In the future, this can be ported to use BTF tagging */ in btf_param_match_suffix()
9406 param_name = btf_name_by_offset(btf, arg->name_off); in btf_param_match_suffix()
9412 param_name += len - suffix_len; in btf_param_match_suffix()