btf.c (4ff5747158f323939e2ce8881ca61f3c646948c4) btf.c (8646db238997df36c6ad71a9d7e0b52ceee221b2)
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2018 Facebook */
3
4#include <uapi/linux/btf.h>
5#include <uapi/linux/bpf.h>
6#include <uapi/linux/bpf_perf_event.h>
7#include <uapi/linux/types.h>
8#include <linux/seq_file.h>

--- 260 unchanged lines hidden (view full) ---

269 struct btf_struct_ops_tab *struct_ops_tab;
270
271 /* split BTF support */
272 struct btf *base_btf;
273 u32 start_id; /* first type ID in this BTF (0 for base BTF) */
274 u32 start_str_off; /* first string offset (0 for base BTF) */
275 char name[MODULE_NAME_LEN];
276 bool kernel_btf;
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2018 Facebook */
3
4#include <uapi/linux/btf.h>
5#include <uapi/linux/bpf.h>
6#include <uapi/linux/bpf_perf_event.h>
7#include <uapi/linux/types.h>
8#include <linux/seq_file.h>

--- 260 unchanged lines hidden (view full) ---

269 struct btf_struct_ops_tab *struct_ops_tab;
270
271 /* split BTF support */
272 struct btf *base_btf;
273 u32 start_id; /* first type ID in this BTF (0 for base BTF) */
274 u32 start_str_off; /* first string offset (0 for base BTF) */
275 char name[MODULE_NAME_LEN];
276 bool kernel_btf;
277 __u32 *base_id_map; /* map from distilled base BTF -> vmlinux BTF ids */
277};
278
279enum verifier_phase {
280 CHECK_META,
281 CHECK_TYPE,
282};
283
284struct resolve_vertex {

--- 240 unchanged lines hidden (view full) ---

525}
526
527static bool btf_type_is_decl_tag_target(const struct btf_type *t)
528{
529 return btf_type_is_func(t) || btf_type_is_struct(t) ||
530 btf_type_is_var(t) || btf_type_is_typedef(t);
531}
532
278};
279
280enum verifier_phase {
281 CHECK_META,
282 CHECK_TYPE,
283};
284
285struct resolve_vertex {

--- 240 unchanged lines hidden (view full) ---

526}
527
528static bool btf_type_is_decl_tag_target(const struct btf_type *t)
529{
530 return btf_type_is_func(t) || btf_type_is_struct(t) ||
531 btf_type_is_var(t) || btf_type_is_typedef(t);
532}
533
534bool btf_is_vmlinux(const struct btf *btf)
535{
536 return btf->kernel_btf && !btf->base_btf;
537}
538
533u32 btf_nr_types(const struct btf *btf)
534{
535 u32 total = 0;
536
537 while (btf) {
538 total += btf->nr_types;
539 btf = btf->base_btf;
540 }

--- 226 unchanged lines hidden (view full) ---

767 if ((first ? !isalpha(c) :
768 !isalnum(c)) &&
769 c != '_' &&
770 c != '.')
771 return false;
772 return true;
773}
774
539u32 btf_nr_types(const struct btf *btf)
540{
541 u32 total = 0;
542
543 while (btf) {
544 total += btf->nr_types;
545 btf = btf->base_btf;
546 }

--- 226 unchanged lines hidden (view full) ---

773 if ((first ? !isalpha(c) :
774 !isalnum(c)) &&
775 c != '_' &&
776 c != '.')
777 return false;
778 return true;
779}
780
775static const char *btf_str_by_offset(const struct btf *btf, u32 offset)
781const char *btf_str_by_offset(const struct btf *btf, u32 offset)
776{
777 while (offset < btf->start_str_off)
778 btf = btf->base_btf;
779
780 offset -= btf->start_str_off;
781 if (offset < btf->hdr.str_len)
782 return &btf->strings[offset];
783

--- 881 unchanged lines hidden (view full) ---

1665
1666static void btf_free_kfunc_set_tab(struct btf *btf)
1667{
1668 struct btf_kfunc_set_tab *tab = btf->kfunc_set_tab;
1669 int hook;
1670
1671 if (!tab)
1672 return;
782{
783 while (offset < btf->start_str_off)
784 btf = btf->base_btf;
785
786 offset -= btf->start_str_off;
787 if (offset < btf->hdr.str_len)
788 return &btf->strings[offset];
789

--- 881 unchanged lines hidden (view full) ---

1671
1672static void btf_free_kfunc_set_tab(struct btf *btf)
1673{
1674 struct btf_kfunc_set_tab *tab = btf->kfunc_set_tab;
1675 int hook;
1676
1677 if (!tab)
1678 return;
1673 /* For module BTF, we directly assign the sets being registered, so
1674 * there is nothing to free except kfunc_set_tab.
1675 */
1676 if (btf_is_module(btf))
1677 goto free_tab;
1678 for (hook = 0; hook < ARRAY_SIZE(tab->sets); hook++)
1679 kfree(tab->sets[hook]);
1679 for (hook = 0; hook < ARRAY_SIZE(tab->sets); hook++)
1680 kfree(tab->sets[hook]);
1680free_tab:
1681 kfree(tab);
1682 btf->kfunc_set_tab = NULL;
1683}
1684
1685static void btf_free_dtor_kfunc_tab(struct btf *btf)
1686{
1687 struct btf_id_dtor_kfunc_tab *tab = btf->dtor_kfunc_tab;
1688

--- 41 unchanged lines hidden (view full) ---

1730{
1731 btf_free_struct_meta_tab(btf);
1732 btf_free_dtor_kfunc_tab(btf);
1733 btf_free_kfunc_set_tab(btf);
1734 btf_free_struct_ops_tab(btf);
1735 kvfree(btf->types);
1736 kvfree(btf->resolved_sizes);
1737 kvfree(btf->resolved_ids);
1681 kfree(tab);
1682 btf->kfunc_set_tab = NULL;
1683}
1684
1685static void btf_free_dtor_kfunc_tab(struct btf *btf)
1686{
1687 struct btf_id_dtor_kfunc_tab *tab = btf->dtor_kfunc_tab;
1688

--- 41 unchanged lines hidden (view full) ---

1730{
1731 btf_free_struct_meta_tab(btf);
1732 btf_free_dtor_kfunc_tab(btf);
1733 btf_free_kfunc_set_tab(btf);
1734 btf_free_struct_ops_tab(btf);
1735 kvfree(btf->types);
1736 kvfree(btf->resolved_sizes);
1737 kvfree(btf->resolved_ids);
1738 kvfree(btf->data);
1738 /* vmlinux does not allocate btf->data, it simply points it at
1739 * __start_BTF.
1740 */
1741 if (!btf_is_vmlinux(btf))
1742 kvfree(btf->data);
1743 kvfree(btf->base_id_map);
1739 kfree(btf);
1740}
1741
1742static void btf_free_rcu(struct rcu_head *rcu)
1743{
1744 struct btf *btf = container_of(rcu, struct btf, rcu);
1745
1746 btf_free(btf);

--- 12 unchanged lines hidden (view full) ---

1759void btf_put(struct btf *btf)
1760{
1761 if (btf && refcount_dec_and_test(&btf->refcnt)) {
1762 btf_free_id(btf);
1763 call_rcu(&btf->rcu, btf_free_rcu);
1764 }
1765}
1766
1744 kfree(btf);
1745}
1746
1747static void btf_free_rcu(struct rcu_head *rcu)
1748{
1749 struct btf *btf = container_of(rcu, struct btf, rcu);
1750
1751 btf_free(btf);

--- 12 unchanged lines hidden (view full) ---

1764void btf_put(struct btf *btf)
1765{
1766 if (btf && refcount_dec_and_test(&btf->refcnt)) {
1767 btf_free_id(btf);
1768 call_rcu(&btf->rcu, btf_free_rcu);
1769 }
1770}
1771
1772struct btf *btf_base_btf(const struct btf *btf)
1773{
1774 return btf->base_btf;
1775}
1776
1777const struct btf_header *btf_header(const struct btf *btf)
1778{
1779 return &btf->hdr;
1780}
1781
1782void btf_set_base_btf(struct btf *btf, const struct btf *base_btf)
1783{
1784 btf->base_btf = (struct btf *)base_btf;
1785 btf->start_id = btf_nr_types(base_btf);
1786 btf->start_str_off = base_btf->hdr.str_len;
1787}
1788
1767static int env_resolve_init(struct btf_verifier_env *env)
1768{
1769 struct btf *btf = env->btf;
1770 u32 nr_types = btf->nr_types;
1771 u32 *resolved_sizes = NULL;
1772 u32 *resolved_ids = NULL;
1773 u8 *visit_states = NULL;
1774

--- 4303 unchanged lines hidden (view full) ---

6078 }
6079
6080 return kctx_type_id;
6081}
6082
6083BTF_ID_LIST(bpf_ctx_convert_btf_id)
6084BTF_ID(struct, bpf_ctx_convert)
6085
1789static int env_resolve_init(struct btf_verifier_env *env)
1790{
1791 struct btf *btf = env->btf;
1792 u32 nr_types = btf->nr_types;
1793 u32 *resolved_sizes = NULL;
1794 u32 *resolved_ids = NULL;
1795 u8 *visit_states = NULL;
1796

--- 4303 unchanged lines hidden (view full) ---

6100 }
6101
6102 return kctx_type_id;
6103}
6104
6105BTF_ID_LIST(bpf_ctx_convert_btf_id)
6106BTF_ID(struct, bpf_ctx_convert)
6107
6086struct btf *btf_parse_vmlinux(void)
6108static struct btf *btf_parse_base(struct btf_verifier_env *env, const char *name,
6109 void *data, unsigned int data_size)
6087{
6110{
6088 struct btf_verifier_env *env = NULL;
6089 struct bpf_verifier_log *log;
6090 struct btf *btf = NULL;
6091 int err;
6092
6093 if (!IS_ENABLED(CONFIG_DEBUG_INFO_BTF))
6094 return ERR_PTR(-ENOENT);
6095
6111 struct btf *btf = NULL;
6112 int err;
6113
6114 if (!IS_ENABLED(CONFIG_DEBUG_INFO_BTF))
6115 return ERR_PTR(-ENOENT);
6116
6096 env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
6097 if (!env)
6098 return ERR_PTR(-ENOMEM);
6099
6100 log = &env->log;
6101 log->level = BPF_LOG_KERNEL;
6102
6103 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
6104 if (!btf) {
6105 err = -ENOMEM;
6106 goto errout;
6107 }
6108 env->btf = btf;
6109
6117 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
6118 if (!btf) {
6119 err = -ENOMEM;
6120 goto errout;
6121 }
6122 env->btf = btf;
6123
6110 btf->data = __start_BTF;
6111 btf->data_size = __stop_BTF - __start_BTF;
6124 btf->data = data;
6125 btf->data_size = data_size;
6112 btf->kernel_btf = true;
6126 btf->kernel_btf = true;
6113 snprintf(btf->name, sizeof(btf->name), "vmlinux");
6127 snprintf(btf->name, sizeof(btf->name), "%s", name);
6114
6115 err = btf_parse_hdr(env);
6116 if (err)
6117 goto errout;
6118
6119 btf->nohdr_data = btf->data + btf->hdr.hdr_len;
6120
6121 err = btf_parse_str_sec(env);
6122 if (err)
6123 goto errout;
6124
6125 err = btf_check_all_metas(env);
6126 if (err)
6127 goto errout;
6128
6129 err = btf_check_type_tags(env, btf, 1);
6130 if (err)
6131 goto errout;
6132
6128
6129 err = btf_parse_hdr(env);
6130 if (err)
6131 goto errout;
6132
6133 btf->nohdr_data = btf->data + btf->hdr.hdr_len;
6134
6135 err = btf_parse_str_sec(env);
6136 if (err)
6137 goto errout;
6138
6139 err = btf_check_all_metas(env);
6140 if (err)
6141 goto errout;
6142
6143 err = btf_check_type_tags(env, btf, 1);
6144 if (err)
6145 goto errout;
6146
6133 /* btf_parse_vmlinux() runs under bpf_verifier_lock */
6134 bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]);
6135
6136 refcount_set(&btf->refcnt, 1);
6137
6147 refcount_set(&btf->refcnt, 1);
6148
6138 err = btf_alloc_id(btf);
6139 if (err)
6140 goto errout;
6141
6142 btf_verifier_env_free(env);
6143 return btf;
6144
6145errout:
6149 return btf;
6150
6151errout:
6146 btf_verifier_env_free(env);
6147 if (btf) {
6148 kvfree(btf->types);
6149 kfree(btf);
6150 }
6151 return ERR_PTR(err);
6152}
6153
6152 if (btf) {
6153 kvfree(btf->types);
6154 kfree(btf);
6155 }
6156 return ERR_PTR(err);
6157}
6158
6159struct btf *btf_parse_vmlinux(void)
6160{
6161 struct btf_verifier_env *env = NULL;
6162 struct bpf_verifier_log *log;
6163 struct btf *btf;
6164 int err;
6165
6166 env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
6167 if (!env)
6168 return ERR_PTR(-ENOMEM);
6169
6170 log = &env->log;
6171 log->level = BPF_LOG_KERNEL;
6172 btf = btf_parse_base(env, "vmlinux", __start_BTF, __stop_BTF - __start_BTF);
6173 if (IS_ERR(btf))
6174 goto err_out;
6175
6176 /* btf_parse_vmlinux() runs under bpf_verifier_lock */
6177 bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]);
6178 err = btf_alloc_id(btf);
6179 if (err) {
6180 btf_free(btf);
6181 btf = ERR_PTR(err);
6182 }
6183err_out:
6184 btf_verifier_env_free(env);
6185 return btf;
6186}
6187
6154#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
6155
6188#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
6189
6156static struct btf *btf_parse_module(const char *module_name, const void *data, unsigned int data_size)
6190/* If .BTF_ids section was created with distilled base BTF, both base and
6191 * split BTF ids will need to be mapped to actual base/split ids for
6192 * BTF now that it has been relocated.
6193 */
6194static __u32 btf_relocate_id(const struct btf *btf, __u32 id)
6157{
6195{
6196 if (!btf->base_btf || !btf->base_id_map)
6197 return id;
6198 return btf->base_id_map[id];
6199}
6200
6201static struct btf *btf_parse_module(const char *module_name, const void *data,
6202 unsigned int data_size, void *base_data,
6203 unsigned int base_data_size)
6204{
6205 struct btf *btf = NULL, *vmlinux_btf, *base_btf = NULL;
6158 struct btf_verifier_env *env = NULL;
6159 struct bpf_verifier_log *log;
6206 struct btf_verifier_env *env = NULL;
6207 struct bpf_verifier_log *log;
6160 struct btf *btf = NULL, *base_btf;
6161 int err;
6208 int err = 0;
6162
6209
6163 base_btf = bpf_get_btf_vmlinux();
6164 if (IS_ERR(base_btf))
6165 return base_btf;
6166 if (!base_btf)
6210 vmlinux_btf = bpf_get_btf_vmlinux();
6211 if (IS_ERR(vmlinux_btf))
6212 return vmlinux_btf;
6213 if (!vmlinux_btf)
6167 return ERR_PTR(-EINVAL);
6168
6169 env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
6170 if (!env)
6171 return ERR_PTR(-ENOMEM);
6172
6173 log = &env->log;
6174 log->level = BPF_LOG_KERNEL;
6175
6214 return ERR_PTR(-EINVAL);
6215
6216 env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
6217 if (!env)
6218 return ERR_PTR(-ENOMEM);
6219
6220 log = &env->log;
6221 log->level = BPF_LOG_KERNEL;
6222
6223 if (base_data) {
6224 base_btf = btf_parse_base(env, ".BTF.base", base_data, base_data_size);
6225 if (IS_ERR(base_btf)) {
6226 err = PTR_ERR(base_btf);
6227 goto errout;
6228 }
6229 } else {
6230 base_btf = vmlinux_btf;
6231 }
6232
6176 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
6177 if (!btf) {
6178 err = -ENOMEM;
6179 goto errout;
6180 }
6181 env->btf = btf;
6182
6183 btf->base_btf = base_btf;

--- 23 unchanged lines hidden (view full) ---

6207 err = btf_check_all_metas(env);
6208 if (err)
6209 goto errout;
6210
6211 err = btf_check_type_tags(env, btf, btf_nr_types(base_btf));
6212 if (err)
6213 goto errout;
6214
6233 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
6234 if (!btf) {
6235 err = -ENOMEM;
6236 goto errout;
6237 }
6238 env->btf = btf;
6239
6240 btf->base_btf = base_btf;

--- 23 unchanged lines hidden (view full) ---

6264 err = btf_check_all_metas(env);
6265 if (err)
6266 goto errout;
6267
6268 err = btf_check_type_tags(env, btf, btf_nr_types(base_btf));
6269 if (err)
6270 goto errout;
6271
6272 if (base_btf != vmlinux_btf) {
6273 err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map);
6274 if (err)
6275 goto errout;
6276 btf_free(base_btf);
6277 base_btf = vmlinux_btf;
6278 }
6279
6215 btf_verifier_env_free(env);
6216 refcount_set(&btf->refcnt, 1);
6217 return btf;
6218
6219errout:
6220 btf_verifier_env_free(env);
6280 btf_verifier_env_free(env);
6281 refcount_set(&btf->refcnt, 1);
6282 return btf;
6283
6284errout:
6285 btf_verifier_env_free(env);
6286 if (base_btf != vmlinux_btf)
6287 btf_free(base_btf);
6221 if (btf) {
6222 kvfree(btf->data);
6223 kvfree(btf->types);
6224 kfree(btf);
6225 }
6226 return ERR_PTR(err);
6227}
6228

--- 1536 unchanged lines hidden (view full) ---

7765
7766 switch (op) {
7767 case MODULE_STATE_COMING:
7768 btf_mod = kzalloc(sizeof(*btf_mod), GFP_KERNEL);
7769 if (!btf_mod) {
7770 err = -ENOMEM;
7771 goto out;
7772 }
6288 if (btf) {
6289 kvfree(btf->data);
6290 kvfree(btf->types);
6291 kfree(btf);
6292 }
6293 return ERR_PTR(err);
6294}
6295

--- 1536 unchanged lines hidden (view full) ---

7832
7833 switch (op) {
7834 case MODULE_STATE_COMING:
7835 btf_mod = kzalloc(sizeof(*btf_mod), GFP_KERNEL);
7836 if (!btf_mod) {
7837 err = -ENOMEM;
7838 goto out;
7839 }
7773 btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size);
7840 btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size,
7841 mod->btf_base_data, mod->btf_base_data_size);
7774 if (IS_ERR(btf)) {
7775 kfree(btf_mod);
7776 if (!IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) {
7777 pr_warn("failed to validate module [%s] BTF: %ld\n",
7778 mod->name, PTR_ERR(btf));
7779 err = PTR_ERR(btf);
7780 } else {
7781 pr_warn_once("Kernel module BTF mismatch detected, BTF debug info may be unavailable for some modules\n");

--- 307 unchanged lines hidden (view full) ---

8089 const struct btf_kfunc_id_set *kset)
8090{
8091 struct btf_kfunc_hook_filter *hook_filter;
8092 struct btf_id_set8 *add_set = kset->set;
8093 bool vmlinux_set = !btf_is_module(btf);
8094 bool add_filter = !!kset->filter;
8095 struct btf_kfunc_set_tab *tab;
8096 struct btf_id_set8 *set;
7842 if (IS_ERR(btf)) {
7843 kfree(btf_mod);
7844 if (!IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) {
7845 pr_warn("failed to validate module [%s] BTF: %ld\n",
7846 mod->name, PTR_ERR(btf));
7847 err = PTR_ERR(btf);
7848 } else {
7849 pr_warn_once("Kernel module BTF mismatch detected, BTF debug info may be unavailable for some modules\n");

--- 307 unchanged lines hidden (view full) ---

8157 const struct btf_kfunc_id_set *kset)
8158{
8159 struct btf_kfunc_hook_filter *hook_filter;
8160 struct btf_id_set8 *add_set = kset->set;
8161 bool vmlinux_set = !btf_is_module(btf);
8162 bool add_filter = !!kset->filter;
8163 struct btf_kfunc_set_tab *tab;
8164 struct btf_id_set8 *set;
8097 u32 set_cnt;
8165 u32 set_cnt, i;
8098 int ret;
8099
8100 if (hook >= BTF_KFUNC_HOOK_MAX) {
8101 ret = -EINVAL;
8102 goto end;
8103 }
8104
8105 if (!add_set->cnt)

--- 29 unchanged lines hidden (view full) ---

8135 /* Warn when register_btf_kfunc_id_set is called twice for the same hook
8136 * for module sets.
8137 */
8138 if (WARN_ON_ONCE(set && !vmlinux_set)) {
8139 ret = -EINVAL;
8140 goto end;
8141 }
8142
8166 int ret;
8167
8168 if (hook >= BTF_KFUNC_HOOK_MAX) {
8169 ret = -EINVAL;
8170 goto end;
8171 }
8172
8173 if (!add_set->cnt)

--- 29 unchanged lines hidden (view full) ---

8203 /* Warn when register_btf_kfunc_id_set is called twice for the same hook
8204 * for module sets.
8205 */
8206 if (WARN_ON_ONCE(set && !vmlinux_set)) {
8207 ret = -EINVAL;
8208 goto end;
8209 }
8210
8143 /* We don't need to allocate, concatenate, and sort module sets, because
8144 * only one is allowed per hook. Hence, we can directly assign the
8145 * pointer and return.
8146 */
8147 if (!vmlinux_set) {
8148 tab->sets[hook] = add_set;
8149 goto do_add_filter;
8150 }
8151
8152 /* In case of vmlinux sets, there may be more than one set being
8153 * registered per hook. To create a unified set, we allocate a new set
8154 * and concatenate all individual sets being registered. While each set
8155 * is individually sorted, they may become unsorted when concatenated,
8156 * hence re-sorting the final set again is required to make binary
8157 * searching the set using btf_id_set8_contains function work.
8211 /* In case of vmlinux sets, there may be more than one set being
8212 * registered per hook. To create a unified set, we allocate a new set
8213 * and concatenate all individual sets being registered. While each set
8214 * is individually sorted, they may become unsorted when concatenated,
8215 * hence re-sorting the final set again is required to make binary
8216 * searching the set using btf_id_set8_contains function work.
8217 *
8218 * For module sets, we need to allocate as we may need to relocate
8219 * BTF ids.
8158 */
8159 set_cnt = set ? set->cnt : 0;
8160
8161 if (set_cnt > U32_MAX - add_set->cnt) {
8162 ret = -EOVERFLOW;
8163 goto end;
8164 }
8165

--- 13 unchanged lines hidden (view full) ---

8179
8180 /* For newly allocated set, initialize set->cnt to 0 */
8181 if (!tab->sets[hook])
8182 set->cnt = 0;
8183 tab->sets[hook] = set;
8184
8185 /* Concatenate the two sets */
8186 memcpy(set->pairs + set->cnt, add_set->pairs, add_set->cnt * sizeof(set->pairs[0]));
8220 */
8221 set_cnt = set ? set->cnt : 0;
8222
8223 if (set_cnt > U32_MAX - add_set->cnt) {
8224 ret = -EOVERFLOW;
8225 goto end;
8226 }
8227

--- 13 unchanged lines hidden (view full) ---

8241
8242 /* For newly allocated set, initialize set->cnt to 0 */
8243 if (!tab->sets[hook])
8244 set->cnt = 0;
8245 tab->sets[hook] = set;
8246
8247 /* Concatenate the two sets */
8248 memcpy(set->pairs + set->cnt, add_set->pairs, add_set->cnt * sizeof(set->pairs[0]));
8249 /* Now that the set is copied, update with relocated BTF ids */
8250 for (i = set->cnt; i < set->cnt + add_set->cnt; i++)
8251 set->pairs[i].id = btf_relocate_id(btf, set->pairs[i].id);
8252
8187 set->cnt += add_set->cnt;
8188
8189 sort(set->pairs, set->cnt, sizeof(set->pairs[0]), btf_id_cmp_func, NULL);
8190
8253 set->cnt += add_set->cnt;
8254
8255 sort(set->pairs, set->cnt, sizeof(set->pairs[0]), btf_id_cmp_func, NULL);
8256
8191do_add_filter:
8192 if (add_filter) {
8193 hook_filter = &tab->hook_filters[hook];
8194 hook_filter->filters[hook_filter->nr_filters++] = kset->filter;
8195 }
8196 return 0;
8197end:
8198 btf_free_kfunc_set_tab(btf);
8199 return ret;

--- 103 unchanged lines hidden (view full) ---

8303
8304 btf = btf_get_module_btf(kset->owner);
8305 if (!btf)
8306 return check_btf_kconfigs(kset->owner, "kfunc");
8307 if (IS_ERR(btf))
8308 return PTR_ERR(btf);
8309
8310 for (i = 0; i < kset->set->cnt; i++) {
8257 if (add_filter) {
8258 hook_filter = &tab->hook_filters[hook];
8259 hook_filter->filters[hook_filter->nr_filters++] = kset->filter;
8260 }
8261 return 0;
8262end:
8263 btf_free_kfunc_set_tab(btf);
8264 return ret;

--- 103 unchanged lines hidden (view full) ---

8368
8369 btf = btf_get_module_btf(kset->owner);
8370 if (!btf)
8371 return check_btf_kconfigs(kset->owner, "kfunc");
8372 if (IS_ERR(btf))
8373 return PTR_ERR(btf);
8374
8375 for (i = 0; i < kset->set->cnt; i++) {
8311 ret = btf_check_kfunc_protos(btf, kset->set->pairs[i].id,
8376 ret = btf_check_kfunc_protos(btf, btf_relocate_id(btf, kset->set->pairs[i].id),
8312 kset->set->pairs[i].flags);
8313 if (ret)
8314 goto err_out;
8315 }
8316
8317 ret = btf_populate_kfunc_set(btf, hook, kset);
8318
8319err_out:

--- 47 unchanged lines hidden (view full) ---

8367static int btf_check_dtor_kfuncs(struct btf *btf, const struct btf_id_dtor_kfunc *dtors, u32 cnt)
8368{
8369 const struct btf_type *dtor_func, *dtor_func_proto, *t;
8370 const struct btf_param *args;
8371 s32 dtor_btf_id;
8372 u32 nr_args, i;
8373
8374 for (i = 0; i < cnt; i++) {
8377 kset->set->pairs[i].flags);
8378 if (ret)
8379 goto err_out;
8380 }
8381
8382 ret = btf_populate_kfunc_set(btf, hook, kset);
8383
8384err_out:

--- 47 unchanged lines hidden (view full) ---

8432static int btf_check_dtor_kfuncs(struct btf *btf, const struct btf_id_dtor_kfunc *dtors, u32 cnt)
8433{
8434 const struct btf_type *dtor_func, *dtor_func_proto, *t;
8435 const struct btf_param *args;
8436 s32 dtor_btf_id;
8437 u32 nr_args, i;
8438
8439 for (i = 0; i < cnt; i++) {
8375 dtor_btf_id = dtors[i].kfunc_btf_id;
8440 dtor_btf_id = btf_relocate_id(btf, dtors[i].kfunc_btf_id);
8376
8377 dtor_func = btf_type_by_id(btf, dtor_btf_id);
8378 if (!dtor_func || !btf_type_is_func(dtor_func))
8379 return -EINVAL;
8380
8381 dtor_func_proto = btf_type_by_id(btf, dtor_func->type);
8382 if (!dtor_func_proto || !btf_type_is_func_proto(dtor_func_proto))
8383 return -EINVAL;

--- 18 unchanged lines hidden (view full) ---

8402}
8403
8404/* This function must be invoked only from initcalls/module init functions */
8405int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dtors, u32 add_cnt,
8406 struct module *owner)
8407{
8408 struct btf_id_dtor_kfunc_tab *tab;
8409 struct btf *btf;
8441
8442 dtor_func = btf_type_by_id(btf, dtor_btf_id);
8443 if (!dtor_func || !btf_type_is_func(dtor_func))
8444 return -EINVAL;
8445
8446 dtor_func_proto = btf_type_by_id(btf, dtor_func->type);
8447 if (!dtor_func_proto || !btf_type_is_func_proto(dtor_func_proto))
8448 return -EINVAL;

--- 18 unchanged lines hidden (view full) ---

8467}
8468
8469/* This function must be invoked only from initcalls/module init functions */
8470int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dtors, u32 add_cnt,
8471 struct module *owner)
8472{
8473 struct btf_id_dtor_kfunc_tab *tab;
8474 struct btf *btf;
8410 u32 tab_cnt;
8475 u32 tab_cnt, i;
8411 int ret;
8412
8413 btf = btf_get_module_btf(owner);
8414 if (!btf)
8415 return check_btf_kconfigs(owner, "dtor kfuncs");
8416 if (IS_ERR(btf))
8417 return PTR_ERR(btf);
8418

--- 34 unchanged lines hidden (view full) ---

8453 goto end;
8454 }
8455
8456 if (!btf->dtor_kfunc_tab)
8457 tab->cnt = 0;
8458 btf->dtor_kfunc_tab = tab;
8459
8460 memcpy(tab->dtors + tab->cnt, dtors, add_cnt * sizeof(tab->dtors[0]));
8476 int ret;
8477
8478 btf = btf_get_module_btf(owner);
8479 if (!btf)
8480 return check_btf_kconfigs(owner, "dtor kfuncs");
8481 if (IS_ERR(btf))
8482 return PTR_ERR(btf);
8483

--- 34 unchanged lines hidden (view full) ---

8518 goto end;
8519 }
8520
8521 if (!btf->dtor_kfunc_tab)
8522 tab->cnt = 0;
8523 btf->dtor_kfunc_tab = tab;
8524
8525 memcpy(tab->dtors + tab->cnt, dtors, add_cnt * sizeof(tab->dtors[0]));
8526
8527 /* remap BTF ids based on BTF relocation (if any) */
8528 for (i = tab_cnt; i < tab_cnt + add_cnt; i++) {
8529 tab->dtors[i].btf_id = btf_relocate_id(btf, tab->dtors[i].btf_id);
8530 tab->dtors[i].kfunc_btf_id = btf_relocate_id(btf, tab->dtors[i].kfunc_btf_id);
8531 }
8532
8461 tab->cnt += add_cnt;
8462
8463 sort(tab->dtors, tab->cnt, sizeof(tab->dtors[0]), btf_id_cmp_func, NULL);
8464
8465end:
8466 if (ret)
8467 btf_free_dtor_kfunc_tab(btf);
8468 btf_put(btf);

--- 649 unchanged lines hidden --- 		8533 tab->cnt += add_cnt;
8534
8535 sort(tab->dtors, tab->cnt, sizeof(tab->dtors[0]), btf_id_cmp_func, NULL);
8536
8537end:
8538 if (ret)
8539 btf_free_dtor_kfunc_tab(btf);
8540 btf_put(btf);

--- 649 unchanged lines hidden ---