Lines Matching +full:debian +full:- +full:testing
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
71 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
306 if (err != -EPERM || geteuid() != 0)
323 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
337 fd = -1; \
348 /* as of v1.0 libbpf_set_strict_mode() is a no-op */
404 /* stored as sec_def->cookie for all libbpf-supported SEC()s */
423 /* BPF program support non-linear XDP buffer */
466 * program. For the entry-point (main) BPF program, this is always
467 * zero. For a sub-program, this gets reset before each of main BPF
469 * whether sub-program was already appended to the main program, and
481 * entry-point BPF programs this includes the size of main program
482 * itself plus all the used sub-programs, appended at the end
535 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
647 /* BTF fd index to be patched in for insn->off, this is
648 * 0 for vmlinux BTF, index in obj->fd_array for module
743 /* Path to the custom BTF to be used for BPF CO-RE relocations as an
747 /* vmlinux BTF override for CO-RE relocations */
803 zclose(prog->fd);
805 zfree(&prog->func_info);
806 zfree(&prog->line_info);
807 zfree(&prog->subprogs);
816 zfree(&prog->name);
817 zfree(&prog->sec_name);
818 zfree(&prog->insns);
819 zfree(&prog->reloc_desc);
821 prog->nr_reloc = 0;
822 prog->insns_cnt = 0;
823 prog->sec_idx = -1;
828 return BPF_CLASS(insn->code) == BPF_JMP &&
829 BPF_OP(insn->code) == BPF_CALL &&
830 BPF_SRC(insn->code) == BPF_K &&
831 insn->src_reg == BPF_PSEUDO_CALL &&
832 insn->dst_reg == 0 &&
833 insn->off == 0;
838 return insn->code == (BPF_JMP | BPF_CALL);
843 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
854 return -EINVAL;
858 prog->obj = obj;
860 prog->sec_idx = sec_idx;
861 prog->sec_insn_off = sec_off / BPF_INSN_SZ;
862 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
864 prog->insns_cnt = prog->sec_insn_cnt;
866 prog->type = BPF_PROG_TYPE_UNSPEC;
867 prog->fd = -1;
868 prog->exception_cb_idx = -1;
875 prog->autoload = false;
879 prog->autoload = true;
882 prog->autoattach = true;
885 prog->log_level = obj->log_level;
887 prog->sec_name = strdup(sec_name);
888 if (!prog->sec_name)
891 prog->name = strdup(name);
892 if (!prog->name)
895 prog->insns = malloc(insn_data_sz);
896 if (!prog->insns)
898 memcpy(prog->insns, insn_data, insn_data_sz);
904 return -ENOMEM;
911 Elf_Data *symbols = obj->efile.symbols;
913 void *data = sec_data->d_buf;
914 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms;
919 progs = obj->programs;
920 nr_progs = obj->nr_programs;
921 nr_syms = symbols->d_size / sizeof(Elf64_Sym);
926 if (sym->st_shndx != sec_idx)
928 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
931 prog_sz = sym->st_size;
932 sec_off = sym->st_value;
934 name = elf_sym_str(obj, sym->st_name);
938 return -LIBBPF_ERRNO__FORMAT;
944 return -LIBBPF_ERRNO__FORMAT;
947 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) {
949 return -ENOTSUP;
958 * In this case the original obj->programs
964 return -ENOMEM;
966 obj->programs = progs;
975 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL)
976 prog->sym_global = true;
983 if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
984 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL))
985 prog->mark_btf_static = true;
988 obj->nr_programs = nr_progs;
996 struct bpf_program *prog = obj->programs;
1000 for (p = 0; p < obj->nr_programs; p++, prog++) {
1001 insn = prog->insns;
1002 for (i = 0; i < prog->insns_cnt; i++, insn++)
1005 pr_debug("converted %zu BPF programs to native byte order\n", obj->nr_programs);
1030 if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
1091 if (kern_data_member->type == kern_type_id)
1097 return -EINVAL;
1111 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
1119 for (i = 0; i < obj->nr_programs; i++) {
1120 if (&obj->programs[i] == prog)
1121 return prog->type == BPF_PROG_TYPE_STRUCT_OPS;
1138 for (i = 0; i < obj->nr_programs; ++i) {
1142 prog = &obj->programs[i];
1143 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
1146 for (j = 0; j < obj->nr_maps; ++j) {
1149 map = &obj->maps[j];
1153 type = btf__type_by_id(obj->btf, map->st_ops->type_id);
1156 slot_prog = map->st_ops->progs[k];
1161 if (map->autocreate)
1166 prog->autoload = should_load;
1178 struct bpf_object *obj = map->obj;
1179 const struct btf *btf = obj->btf;
1187 st_ops = map->st_ops;
1188 type = btf__type_by_id(btf, st_ops->type_id);
1189 tname = btf__name_by_offset(btf, type->name_off);
1197 kern_btf = mod_btf ? mod_btf->btf : obj->btf_vmlinux;
1200 map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
1202 map->mod_btf_fd = mod_btf ? mod_btf->fd : -1;
1203 map->def.value_size = kern_vtype->size;
1204 map->btf_vmlinux_value_type_id = kern_vtype_id;
1206 st_ops->kern_vdata = calloc(1, kern_vtype->size);
1207 if (!st_ops->kern_vdata)
1208 return -ENOMEM;
1210 data = st_ops->data;
1211 kern_data_off = kern_data_member->offset / 8;
1212 kern_data = st_ops->kern_vdata + kern_data_off;
1225 mname = btf__name_by_offset(btf, member->name_off);
1226 moff = member->offset / 8;
1228 msize = btf__resolve_size(btf, member->type);
1231 map->name, mname);
1239 map->name, mname);
1240 return -ENOTSUP;
1243 if (st_ops->progs[i]) {
1251 st_ops->progs[i]->autoload = false;
1252 st_ops->progs[i] = NULL;
1255 /* Skip all-zero/NULL fields if they are not present in the kernel BTF */
1257 map->name, mname);
1261 kern_member_idx = kern_member - btf_members(kern_type);
1265 map->name, mname);
1266 return -ENOTSUP;
1269 kern_moff = kern_member->offset / 8;
1272 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
1273 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
1275 if (BTF_INFO_KIND(mtype->info) !=
1276 BTF_INFO_KIND(kern_mtype->info)) {
1278 map->name, mname, BTF_INFO_KIND(mtype->info),
1279 BTF_INFO_KIND(kern_mtype->info));
1280 return -ENOTSUP;
1289 if (st_ops->progs[i] && st_ops->progs[i] != prog)
1290 st_ops->progs[i]->autoload = false;
1293 st_ops->progs[i] = prog;
1299 map->name, mname);
1300 return -ENOTSUP;
1304 kern_mtype->type,
1307 /* mtype->type must be a func_proto which was
1313 map->name, mname);
1314 return -ENOTSUP;
1318 prog->attach_btf_obj_fd = mod_btf->fd;
1323 if (!prog->attach_btf_id) {
1324 prog->attach_btf_id = kern_type_id;
1325 prog->expected_attach_type = kern_member_idx;
1328 /* struct_ops BPF prog can be re-used between multiple
1333 if (prog->attach_btf_id != kern_type_id) {
1335 map->name, mname, prog->name, prog->sec_name, prog->type,
1336 prog->attach_btf_id, kern_type_id);
1337 return -EINVAL;
1339 if (prog->expected_attach_type != kern_member_idx) {
1341 map->name, mname, prog->name, prog->sec_name, prog->type,
1342 prog->expected_attach_type, kern_member_idx);
1343 return -EINVAL;
1346 st_ops->kern_func_off[i] = kern_data_off + kern_moff;
1349 map->name, mname, prog->name, moff,
1358 map->name, mname, (ssize_t)msize,
1360 return -ENOTSUP;
1364 map->name, mname, (unsigned int)msize,
1378 for (i = 0; i < obj->nr_maps; i++) {
1379 map = &obj->maps[i];
1384 if (!map->autocreate)
1407 if (shndx == -1)
1410 btf = obj->btf;
1416 return -EINVAL;
1422 type = btf__type_by_id(obj->btf, vsi->type);
1423 var_name = btf__name_by_offset(obj->btf, type->name_off);
1425 type_id = btf__resolve_type(obj->btf, vsi->type);
1428 vsi->type, sec_name);
1429 return -EINVAL;
1432 type = btf__type_by_id(obj->btf, type_id);
1433 tname = btf__name_by_offset(obj->btf, type->name_off);
1436 return -ENOTSUP;
1440 return -EINVAL;
1447 map->sec_idx = shndx;
1448 map->sec_offset = vsi->offset;
1449 map->name = strdup(var_name);
1450 if (!map->name)
1451 return -ENOMEM;
1452 map->btf_value_type_id = type_id;
1458 map->autocreate = false;
1463 map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
1464 map->def.key_size = sizeof(int);
1465 map->def.value_size = type->size;
1466 map->def.max_entries = 1;
1467 map->def.map_flags = strcmp(sec_name, STRUCT_OPS_LINK_SEC) == 0 ? BPF_F_LINK : 0;
1468 map->autoattach = true;
1470 map->st_ops = calloc(1, sizeof(*map->st_ops));
1471 if (!map->st_ops)
1472 return -ENOMEM;
1473 st_ops = map->st_ops;
1474 st_ops->data = malloc(type->size);
1475 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
1476 st_ops->kern_func_off = malloc(btf_vlen(type) *
1477 sizeof(*st_ops->kern_func_off));
1478 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
1479 return -ENOMEM;
1481 if (vsi->offset + type->size > data->d_size) {
1484 return -EINVAL;
1487 memcpy(st_ops->data,
1488 data->d_buf + vsi->offset,
1489 type->size);
1490 st_ops->type_id = type_id;
1493 tname, type_id, var_name, vsi->offset);
1504 for (sec_idx = 0; sec_idx < obj->efile.sec_cnt; ++sec_idx) {
1505 struct elf_sec_desc *desc = &obj->efile.secs[sec_idx];
1507 if (desc->sec_type != SEC_ST_OPS)
1512 return -LIBBPF_ERRNO__FORMAT;
1514 err = init_struct_ops_maps(obj, sec_name, sec_idx, desc->data);
1533 return ERR_PTR(-ENOMEM);
1536 strcpy(obj->path, path);
1538 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name));
1541 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name));
1542 end = strchr(obj->name, '.');
1547 obj->efile.fd = -1;
1554 obj->efile.obj_buf = obj_buf;
1555 obj->efile.obj_buf_sz = obj_buf_sz;
1556 obj->efile.btf_maps_shndx = -1;
1557 obj->kconfig_map_idx = -1;
1558 obj->arena_map_idx = -1;
1560 obj->kern_version = get_kernel_version();
1561 obj->state = OBJ_OPEN;
1568 if (!obj->efile.elf)
1571 elf_end(obj->efile.elf);
1572 obj->efile.elf = NULL;
1573 obj->efile.ehdr = NULL;
1574 obj->efile.symbols = NULL;
1575 obj->efile.arena_data = NULL;
1577 zfree(&obj->efile.secs);
1578 obj->efile.sec_cnt = 0;
1579 zclose(obj->efile.fd);
1580 obj->efile.obj_buf = NULL;
1581 obj->efile.obj_buf_sz = 0;
1590 if (obj->efile.elf) {
1592 return -LIBBPF_ERRNO__LIBELF;
1595 if (obj->efile.obj_buf_sz > 0) {
1597 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1599 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
1600 if (obj->efile.fd < 0) {
1601 err = -errno;
1602 pr_warn("elf: failed to open %s: %s\n", obj->path, errstr(err));
1606 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1610 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1611 err = -LIBBPF_ERRNO__LIBELF;
1615 obj->efile.elf = elf;
1618 err = -LIBBPF_ERRNO__FORMAT;
1619 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);
1624 err = -LIBBPF_ERRNO__FORMAT;
1625 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);
1629 obj->efile.ehdr = ehdr = elf64_getehdr(elf);
1630 if (!obj->efile.ehdr) {
1631 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1632 err = -LIBBPF_ERRNO__FORMAT;
1637 if (ehdr->e_ident[EI_DATA] != ELFDATA2LSB &&
1638 ehdr->e_ident[EI_DATA] != ELFDATA2MSB) {
1639 err = -LIBBPF_ERRNO__ENDIAN;
1640 pr_warn("elf: '%s' has unknown byte order\n", obj->path);
1644 obj->byteorder = ehdr->e_ident[EI_DATA];
1646 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
1648 obj->path, elf_errmsg(-1));
1649 err = -LIBBPF_ERRNO__FORMAT;
1654 if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {
1656 obj->path, elf_errmsg(-1));
1657 err = -LIBBPF_ERRNO__FORMAT;
1662 if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {
1663 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1664 err = -LIBBPF_ERRNO__FORMAT;
1677 return obj->byteorder == ELFDATA2LSB;
1679 return obj->byteorder == ELFDATA2MSB;
1689 pr_warn("invalid license section in %s\n", obj->path);
1690 return -LIBBPF_ERRNO__FORMAT;
1692 /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't
1695 libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
1696 pr_debug("license of %s is %s\n", obj->path, obj->license);
1706 pr_warn("invalid kver section in %s\n", obj->path);
1707 return -LIBBPF_ERRNO__FORMAT;
1710 obj->kern_version = kver;
1711 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1729 return -EINVAL;
1734 *size = data->d_size;
1738 return -ENOENT;
1743 Elf_Data *symbols = obj->efile.symbols;
1747 for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {
1750 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)
1753 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&
1754 ELF64_ST_BIND(sym->st_info) != STB_WEAK)
1757 sname = elf_sym_str(obj, sym->st_name);
1760 return ERR_PTR(-EIO);
1766 return ERR_PTR(-ENOENT);
1779 const char *name = "libbpf-placeholder-fd";
1786 return -errno;
1791 return -errno;
1800 err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,
1801 sizeof(*obj->maps), obj->nr_maps + 1);
1805 map = &obj->maps[obj->nr_maps++];
1806 map->obj = obj;
1819 map->fd = create_placeholder_fd();
1820 if (map->fd < 0)
1821 return ERR_PTR(map->fd);
1822 map->inner_map_fd = -1;
1823 map->autocreate = true;
1842 switch (map->def.type) {
1845 return array_map_mmap_sz(map->def.value_size, map->def.max_entries);
1847 return page_sz * map->def.max_entries;
1857 if (!map->mmaped)
1858 return -EINVAL;
1863 mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1865 return -errno;
1867 memcpy(mmaped, map->mmaped, min(old_sz, new_sz));
1868 munmap(map->mmaped, old_sz);
1869 map->mmaped = mmaped;
1904 * '.rodata.abracad' kernel and user-visible name.
1913 sfx_len = BPF_OBJ_NAME_LEN - 1;
1922 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));
1924 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1949 if (!map->btf_value_type_id)
1956 if (map->libbpf_type == LIBBPF_MAP_PERCPU)
1959 t = btf__type_by_id(obj->btf, map->btf_value_type_id);
1965 vt = btf__type_by_id(obj->btf, vsi->type);
1969 if (btf_var(vt)->linkage != BTF_VAR_STATIC)
1990 map->libbpf_type = type;
1991 map->sec_idx = sec_idx;
1992 map->sec_offset = 0;
1993 map->real_name = strdup(real_name);
1994 map->name = internal_map_name(obj, real_name, type);
1995 if (!map->real_name || !map->name) {
1996 zfree(&map->real_name);
1997 zfree(&map->name);
1998 return -ENOMEM;
2001 def = &map->def;
2002 def->type = is_percpu ? BPF_MAP_TYPE_PERCPU_ARRAY : BPF_MAP_TYPE_ARRAY;
2003 def->key_size = sizeof(int);
2004 def->value_size = data_sz;
2005 def->max_entries = 1;
2006 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
2013 def->map_flags |= BPF_F_MMAPABLE;
2016 map->name, is_percpu ? "percpu " : "", map->sec_idx,
2017 map->sec_offset, def->map_flags);
2020 map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
2021 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
2022 if (map->mmaped == MAP_FAILED) {
2023 err = -errno;
2024 map->mmaped = NULL;
2025 pr_warn("failed to alloc map '%s' content buffer: %s\n", map->name, errstr(err));
2026 zfree(&map->real_name);
2027 zfree(&map->name);
2032 memcpy(map->mmaped, data, data_sz);
2034 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
2045 * Populate obj->maps with libbpf internal maps.
2047 for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
2048 sec_desc = &obj->efile.secs[sec_idx];
2051 if (!sec_desc->data || sec_desc->data->d_size == 0)
2054 switch (sec_desc->sec_type) {
2059 sec_desc->data->d_buf,
2060 sec_desc->data->d_size);
2063 obj->has_rodata = true;
2067 sec_desc->data->d_buf,
2068 sec_desc->data->d_size);
2075 sec_desc->data->d_size);
2081 sec_desc->data->d_buf,
2082 sec_desc->data->d_size);
2100 for (i = 0; i < obj->nr_extern; i++) {
2101 if (strcmp(obj->externs[i].name, name) == 0)
2102 return &obj->externs[i];
2113 for (i = 0; i < obj->nr_extern; i++) {
2114 ext_name = obj->externs[i].name;
2116 return &obj->externs[i];
2124 switch (ext->kcfg.type) {
2128 ext->name, value);
2129 return -EINVAL;
2149 ext->name, value);
2150 return -EINVAL;
2152 ext->is_set = true;
2161 if (ext->kcfg.type != KCFG_CHAR_ARR) {
2163 ext->name, value);
2164 return -EINVAL;
2168 if (len < 2 || value[len - 1] != '"') {
2170 ext->name, value);
2171 return -EINVAL;
2175 len -= 2;
2176 if (len >= ext->kcfg.sz) {
2178 ext->name, value, len, ext->kcfg.sz - 1);
2179 len = ext->kcfg.sz - 1;
2183 ext->is_set = true;
2195 err = -errno;
2201 return -EINVAL;
2208 int bit_sz = ext->kcfg.sz * 8;
2210 if (ext->kcfg.sz == 8)
2213 /* Validate that value stored in u64 fits in integer of `ext->sz`
2218 * -2^(Y-1) <= X <= 2^(Y-1) - 1
2219 * 0 <= X + 2^(Y-1) <= 2^Y - 1
2220 * 0 <= X + 2^(Y-1) < 2^Y
2222 * For unsigned target integer, check that all the (64 - Y) bits are
2225 if (ext->kcfg.is_signed)
2226 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
2234 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&
2235 ext->kcfg.type != KCFG_BOOL) {
2237 ext->name, (unsigned long long)value);
2238 return -EINVAL;
2240 if (ext->kcfg.type == KCFG_BOOL && value > 1) {
2242 ext->name, (unsigned long long)value);
2243 return -EINVAL;
2248 ext->name, (unsigned long long)value, ext->kcfg.sz);
2249 return -ERANGE;
2251 switch (ext->kcfg.sz) {
2265 return -EINVAL;
2267 ext->is_set = true;
2286 return -EINVAL;
2291 if (buf[len - 1] == '\n')
2292 buf[len - 1] = '\0';
2298 return -EINVAL;
2302 if (!ext || ext->is_set)
2305 ext_val = data + ext->kcfg.data_off;
2319 pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);
2322 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
2323 pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);
2324 return -EINVAL;
2331 pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);
2343 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
2345 return -EINVAL;
2347 return -ENAMETOOLONG;
2356 return -ENOENT;
2382 err = -errno;
2383 pr_warn("failed to open in-memory Kconfig: %s\n", errstr(err));
2390 pr_warn("error parsing in-memory Kconfig line '%s': %s\n",
2406 for (i = 0; i < obj->nr_extern; i++) {
2407 ext = &obj->externs[i];
2408 if (ext->type == EXT_KCFG)
2415 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
2417 ".kconfig", obj->efile.symbols_shndx,
2422 obj->kconfig_map_idx = obj->nr_maps - 1;
2437 *res_id = t->type;
2438 t = btf__type_by_id(btf, t->type);
2453 t = skip_mods_and_typedefs(btf, t->type, res_id);
2500 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2501 const char *name = btf__name_by_offset(btf, m->name_off);
2511 arr_t = btf__type_by_id(btf, t->type);
2514 map_name, name, t->type);
2523 *res = arr_info->nelems;
2530 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2531 const char *name = btf__name_by_offset(btf, m->name_off);
2558 *res = e->val;
2573 return -EINVAL;
2575 return -ENAMETOOLONG;
2614 const char *name = btf__name_by_offset(btf, m->name_off);
2618 return -EINVAL;
2621 if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
2622 return -EINVAL;
2623 map_def->parts |= MAP_DEF_MAP_TYPE;
2625 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
2626 return -EINVAL;
2627 map_def->parts |= MAP_DEF_MAX_ENTRIES;
2629 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
2630 return -EINVAL;
2631 map_def->parts |= MAP_DEF_MAP_FLAGS;
2633 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
2634 return -EINVAL;
2635 map_def->parts |= MAP_DEF_NUMA_NODE;
2640 return -EINVAL;
2641 if (map_def->key_size && map_def->key_size != sz) {
2643 map_name, map_def->key_size, sz);
2644 return -EINVAL;
2646 map_def->key_size = sz;
2647 map_def->parts |= MAP_DEF_KEY_SIZE;
2651 t = btf__type_by_id(btf, m->type);
2654 map_name, m->type);
2655 return -EINVAL;
2660 return -EINVAL;
2662 sz = btf__resolve_size(btf, t->type);
2665 map_name, t->type, (ssize_t)sz);
2668 if (map_def->key_size && map_def->key_size != sz) {
2670 map_name, map_def->key_size, (ssize_t)sz);
2671 return -EINVAL;
2673 map_def->key_size = sz;
2674 map_def->key_type_id = t->type;
2675 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;
2680 return -EINVAL;
2681 if (map_def->value_size && map_def->value_size != sz) {
2683 map_name, map_def->value_size, sz);
2684 return -EINVAL;
2686 map_def->value_size = sz;
2687 map_def->parts |= MAP_DEF_VALUE_SIZE;
2691 t = btf__type_by_id(btf, m->type);
2694 map_name, m->type);
2695 return -EINVAL;
2700 return -EINVAL;
2702 sz = btf__resolve_size(btf, t->type);
2705 map_name, t->type, (ssize_t)sz);
2708 if (map_def->value_size && map_def->value_size != sz) {
2710 map_name, map_def->value_size, (ssize_t)sz);
2711 return -EINVAL;
2713 map_def->value_size = sz;
2714 map_def->value_type_id = t->type;
2715 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;
2718 bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);
2719 bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;
2720 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2725 pr_warn("map '%s': multi-level inner maps not supported.\n",
2727 return -ENOTSUP;
2729 if (i != vlen - 1) {
2732 return -EINVAL;
2735 pr_warn("map '%s': should be map-in-map or prog-array.\n",
2737 return -ENOTSUP;
2739 if (map_def->value_size && map_def->value_size != 4) {
2741 map_name, map_def->value_size);
2742 return -EINVAL;
2744 map_def->value_size = 4;
2745 t = btf__type_by_id(btf, m->type);
2748 map_name, desc, m->type);
2749 return -EINVAL;
2751 if (!btf_is_array(t) || btf_array(t)->nelems) {
2752 pr_warn("map '%s': %s spec is not a zero-sized array.\n",
2754 return -EINVAL;
2756 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
2760 return -EINVAL;
2762 t = skip_mods_and_typedefs(btf, t->type, NULL);
2765 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
2767 return -EINVAL;
2772 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2774 return -EINVAL;
2782 map_def->parts |= MAP_DEF_INNER_MAP;
2788 return -EINVAL;
2791 return -EINVAL;
2795 return -EINVAL;
2797 map_def->pinning = val;
2798 map_def->parts |= MAP_DEF_PINNING;
2803 return -EINVAL;
2804 map_def->map_extra = map_extra;
2805 map_def->parts |= MAP_DEF_MAP_EXTRA;
2809 return -ENOTSUP;
2815 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
2817 return -EINVAL;
2832 * a power-of-2 multiple of kernel's page size. If user diligently
2839 * user-set size to satisfy both user size request and kernel
2848 * very close to UINT_MAX but is not a power-of-2 multiple of
2856 return map->def.type == BPF_MAP_TYPE_RINGBUF ||
2857 map->def.type == BPF_MAP_TYPE_USER_RINGBUF;
2862 map->def.type = def->map_type;
2863 map->def.key_size = def->key_size;
2864 map->def.value_size = def->value_size;
2865 map->def.max_entries = def->max_entries;
2866 map->def.map_flags = def->map_flags;
2867 map->map_extra = def->map_extra;
2869 map->numa_node = def->numa_node;
2870 map->btf_key_type_id = def->key_type_id;
2871 map->btf_value_type_id = def->value_type_id;
2873 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
2875 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
2877 if (def->parts & MAP_DEF_MAP_TYPE)
2878 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
2880 if (def->parts & MAP_DEF_KEY_TYPE)
2882 map->name, def->key_type_id, def->key_size);
2883 else if (def->parts & MAP_DEF_KEY_SIZE)
2884 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
2886 if (def->parts & MAP_DEF_VALUE_TYPE)
2888 map->name, def->value_type_id, def->value_size);
2889 else if (def->parts & MAP_DEF_VALUE_SIZE)
2890 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
2892 if (def->parts & MAP_DEF_MAX_ENTRIES)
2893 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
2894 if (def->parts & MAP_DEF_MAP_FLAGS)
2895 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);
2896 if (def->parts & MAP_DEF_MAP_EXTRA)
2897 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,
2898 (unsigned long long)def->map_extra);
2899 if (def->parts & MAP_DEF_PINNING)
2900 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
2901 if (def->parts & MAP_DEF_NUMA_NODE)
2902 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
2904 if (def->parts & MAP_DEF_INNER_MAP)
2905 pr_debug("map '%s': found inner map definition.\n", map->name);
2933 var = btf__type_by_id(obj->btf, vi->type);
2935 map_name = btf__name_by_offset(obj->btf, var->name_off);
2939 return -EINVAL;
2941 if ((__u64)vi->offset + vi->size > data->d_size) {
2943 return -EINVAL;
2948 return -EINVAL;
2950 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
2952 map_name, btf_var_linkage_str(var_extra->linkage));
2953 return -EOPNOTSUPP;
2956 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2960 return -EINVAL;
2962 if (def->size > vi->size) {
2964 return -EINVAL;
2970 map->name = strdup(map_name);
2971 if (!map->name) {
2973 return -ENOMEM;
2975 map->libbpf_type = LIBBPF_MAP_UNSPEC;
2976 map->def.type = BPF_MAP_TYPE_UNSPEC;
2977 map->sec_idx = sec_idx;
2978 map->sec_offset = vi->offset;
2979 map->btf_var_idx = var_idx;
2981 map_name, map->sec_idx, map->sec_offset);
2983 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2992 pr_warn("map '%s': couldn't build pin path.\n", map->name);
2998 map->inner_map = calloc(1, sizeof(*map->inner_map));
2999 if (!map->inner_map)
3000 return -ENOMEM;
3001 map->inner_map->fd = create_placeholder_fd();
3002 if (map->inner_map->fd < 0)
3003 return map->inner_map->fd;
3004 map->inner_map->sec_idx = sec_idx;
3005 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
3006 if (!map->inner_map->name)
3007 return -ENOMEM;
3008 sprintf(map->inner_map->name, "%s.inner", map_name);
3010 fill_map_from_def(map->inner_map, &inner_def);
3032 return -E2BIG;
3035 obj->arena_data = malloc(data_sz);
3036 if (!obj->arena_data)
3037 return -ENOMEM;
3038 memcpy(obj->arena_data, data, data_sz);
3039 obj->arena_data_sz = data_sz;
3042 map->mmaped = obj->arena_data;
3057 if (obj->efile.btf_maps_shndx < 0)
3060 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
3064 MAPS_ELF_SEC, obj->path);
3065 return -EINVAL;
3068 nr_types = btf__type_cnt(obj->btf);
3070 t = btf__type_by_id(obj->btf, i);
3073 name = btf__name_by_offset(obj->btf, t->name_off);
3076 obj->efile.btf_maps_sec_btf_id = i;
3083 return -ENOENT;
3089 obj->efile.btf_maps_shndx,
3096 for (i = 0; i < obj->nr_maps; i++) {
3097 struct bpf_map *map = &obj->maps[i];
3099 if (map->def.type != BPF_MAP_TYPE_ARENA)
3102 if (obj->arena_map_idx >= 0) {
3104 map->name, obj->maps[obj->arena_map_idx].name);
3105 return -EINVAL;
3107 obj->arena_map_idx = i;
3109 if (obj->efile.arena_data) {
3110 err = init_arena_map_data(obj, map, ARENA_SEC, obj->efile.arena_data_shndx,
3111 obj->efile.arena_data->d_buf,
3112 obj->efile.arena_data->d_size);
3117 if (obj->efile.arena_data && obj->arena_map_idx < 0) {
3120 return -ENOENT;
3152 return sh->sh_flags & SHF_EXECINSTR;
3200 return ERR_PTR(-ENOMEM);
3201 /* btf_header() gives us endian-safe header info */
3204 if (!has_layout && hdr->hdr_len >= sizeof(struct btf_header) &&
3205 (hdr->layout_len != 0 || hdr->layout_off != 0)) {
3217 return ERR_PTR(-ENOMEM);
3221 new_hdr->layout_off = 0;
3222 new_hdr->layout_len = 0;
3223 new_str_off = hdr->type_off + hdr->type_len;
3225 if (old_hdr->magic != hdr->magic)
3226 new_hdr->str_off = bswap_32(new_str_off);
3228 new_hdr->str_off = new_str_off;
3230 memmove(new_raw_data + hdr->hdr_len + new_str_off,
3231 new_raw_data + hdr->hdr_len + hdr->str_off,
3232 hdr->str_len);
3233 sz = hdr->hdr_len + hdr->type_off + hdr->type_len + hdr->str_len;
3243 /* enforce 8-byte pointers for BPF-targeted BTFs */
3251 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
3257 t->size = 1;
3266 name = (char *)btf__name_by_offset(btf, t->name_off);
3274 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
3277 m->offset = v->offset * 8;
3278 m->type = v->type;
3280 vt = (void *)btf__type_by_id(btf, v->type);
3281 m->name_off = vt->name_off;
3284 starts_with_qmark(btf__name_by_offset(btf, t->name_off))) {
3288 name = (char *)btf__name_by_offset(btf, t->name_off);
3294 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
3295 t->size = sizeof(__u32); /* kernel enforced */
3298 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
3301 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
3303 /* replace FLOAT with an equally-sized empty STRUCT;
3307 t->name_off = 0;
3308 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
3311 t->name_off = 0;
3312 t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
3315 t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
3331 t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
3333 m->type = enum64_placeholder_id;
3334 m->offset = 0;
3344 return obj->efile.btf_maps_shndx >= 0 ||
3345 obj->efile.has_st_ops ||
3346 obj->nr_extern > 0;
3351 return obj->efile.has_st_ops;
3358 int err = -ENOENT;
3361 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
3362 err = libbpf_get_error(obj->btf);
3364 obj->btf = NULL;
3368 /* enforce 8-byte pointers for BPF-targeted BTFs */
3369 btf__set_pointer_size(obj->btf, 8);
3375 if (!obj->btf) {
3380 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
3381 err = libbpf_get_error(obj->btf_ext);
3385 obj->btf_ext = NULL;
3390 ext_segs[0] = &obj->btf_ext->func_info;
3391 ext_segs[1] = &obj->btf_ext->line_info;
3392 ext_segs[2] = &obj->btf_ext->core_relo_info;
3399 if (seg->sec_cnt == 0)
3402 seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));
3403 if (!seg->sec_idxs) {
3404 err = -ENOMEM;
3415 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
3422 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);
3439 return a->offset - b->offset;
3446 const char *sec_name = btf__name_by_offset(btf, t->name_off);
3453 return -ENOENT;
3456 /* Extern-backing datasecs (.ksyms, .kconfig) have their size and
3468 * to be optional. But the STV_HIDDEN handling is non-optional for any
3469 * non-extern DATASEC, so the variable fixup loop below handles both
3470 * functions at the same time, paying the cost of BTF VAR <-> ELF
3473 if (t->size == 0) {
3478 return -ENOENT;
3481 t->size = size;
3491 t_var = btf__type_by_id(btf, vsi->type);
3493 pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name);
3494 return -EINVAL;
3498 if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN)
3501 var_name = btf__name_by_offset(btf, t_var->name_off);
3505 return -ENOENT;
3512 return -ENOENT;
3516 vsi->offset = sym->st_value;
3525 if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
3526 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)
3527 var->linkage = BTF_VAR_STATIC;
3539 if (!obj->btf)
3542 n = btf__type_cnt(obj->btf);
3544 struct btf_type *t = btf_type_by_id(obj->btf, i);
3552 err = btf_fixup_datasec(obj, obj->btf, t);
3563 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
3564 prog->type == BPF_PROG_TYPE_LSM)
3570 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
3587 /* CO-RE relocations need kernel BTF, only when btf_custom_path
3590 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
3594 for (i = 0; i < obj->nr_extern; i++) {
3597 ext = &obj->externs[i];
3598 if (ext->type == EXT_KSYM && ext->ksym.type_id)
3603 if (!prog->autoload)
3622 if (obj->btf_vmlinux || obj->gen_loader)
3628 obj->btf_vmlinux = btf__load_vmlinux_btf();
3629 err = libbpf_get_error(obj->btf_vmlinux);
3632 obj->btf_vmlinux = NULL;
3640 struct btf *kern_btf = obj->btf;
3644 if (!obj->btf)
3649 err = -EOPNOTSUPP;
3664 for (i = 0; i < obj->nr_programs; i++) {
3665 struct bpf_program *prog = &obj->programs[i];
3670 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3673 n = btf__type_cnt(obj->btf);
3675 t = btf_type_by_id(obj->btf, j);
3679 name = btf__str_by_offset(obj->btf, t->name_off);
3680 if (strcmp(name, prog->name) != 0)
3683 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
3690 kern_btf = bpf_object__sanitize_btf(obj, obj->btf);
3695 if (obj->gen_loader) {
3700 return -ENOMEM;
3701 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
3703 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
3708 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,
3709 obj->log_level ? 1 : 0, obj->token_fd);
3714 btf__set_fd(obj->btf, btf__fd(kern_btf));
3715 btf__set_fd(kern_btf, -1);
3738 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
3741 off, obj->path, elf_errmsg(-1));
3752 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
3755 off, obj->path, elf_errmsg(-1));
3766 scn = elf_getscn(obj->efile.elf, idx);
3769 idx, obj->path, elf_errmsg(-1));
3778 Elf *elf = obj->efile.elf;
3804 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3823 name = elf_sec_str(obj, sh->sh_name);
3826 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3844 obj->path, elf_errmsg(-1));
3853 if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))
3856 return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;
3861 if (idx >= data->d_size / sizeof(Elf64_Rel))
3864 return (Elf64_Rel *)data->d_buf + idx;
3876 if (hdr->sh_type == SHT_STRTAB)
3880 if (hdr->sh_type == SHT_LLVM_ADDRSIG)
3884 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
3893 name += sizeof(".rel") - 1;
3912 if (a->sec_idx != b->sec_idx)
3913 return a->sec_idx < b->sec_idx ? -1 : 1;
3916 return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
3922 Elf *elf = obj->efile.elf;
3931 /* ELF section indices are 0-based, but sec #0 is special "invalid"
3936 if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) {
3938 obj->path, elf_errmsg(-1));
3939 return -LIBBPF_ERRNO__FORMAT;
3941 obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));
3942 if (!obj->efile.secs)
3943 return -ENOMEM;
3952 return -LIBBPF_ERRNO__FORMAT;
3954 if (sh->sh_type == SHT_SYMTAB) {
3955 if (obj->efile.symbols) {
3956 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
3957 return -LIBBPF_ERRNO__FORMAT;
3962 return -LIBBPF_ERRNO__FORMAT;
3966 obj->efile.symbols = data;
3967 obj->efile.symbols_shndx = idx;
3968 obj->efile.strtabidx = sh->sh_link;
3972 if (!obj->efile.symbols) {
3974 obj->path);
3975 return -ENOENT;
3981 sec_desc = &obj->efile.secs[idx];
3985 return -LIBBPF_ERRNO__FORMAT;
3987 name = elf_sec_str(obj, sh->sh_name);
3989 return -LIBBPF_ERRNO__FORMAT;
3996 return -LIBBPF_ERRNO__FORMAT;
3999 idx, name, (unsigned long)data->d_size,
4000 (int)sh->sh_link, (unsigned long)sh->sh_flags,
4001 (int)sh->sh_type);
4004 err = bpf_object__init_license(obj, data->d_buf, data->d_size);
4008 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
4013 return -ENOTSUP;
4015 obj->efile.btf_maps_shndx = idx;
4017 if (sh->sh_type != SHT_PROGBITS)
4018 return -LIBBPF_ERRNO__FORMAT;
4021 if (sh->sh_type != SHT_PROGBITS)
4022 return -LIBBPF_ERRNO__FORMAT;
4024 } else if (sh->sh_type == SHT_SYMTAB) {
4026 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {
4027 if (sh->sh_flags & SHF_EXECINSTR) {
4029 obj->efile.text_shndx = idx;
4035 sec_desc->sec_type = SEC_DATA;
4036 sec_desc->shdr = sh;
4037 sec_desc->data = data;
4040 sec_desc->sec_type = SEC_RODATA;
4041 sec_desc->shdr = sh;
4042 sec_desc->data = data;
4045 sec_desc->sec_type = SEC_PERCPU;
4046 sec_desc->shdr = sh;
4047 sec_desc->data = data;
4052 sec_desc->sec_type = SEC_ST_OPS;
4053 sec_desc->shdr = sh;
4054 sec_desc->data = data;
4055 obj->efile.has_st_ops = true;
4057 obj->efile.arena_data = data;
4058 obj->efile.arena_data_shndx = idx;
4060 obj->jumptables_data = malloc(data->d_size);
4061 if (!obj->jumptables_data)
4062 return -ENOMEM;
4063 memcpy(obj->jumptables_data, data->d_buf, data->d_size);
4064 obj->jumptables_data_sz = data->d_size;
4065 obj->efile.jumptables_data_shndx = idx;
4070 } else if (sh->sh_type == SHT_REL) {
4071 int targ_sec_idx = sh->sh_info; /* points to other section */
4073 if (sh->sh_entsize != sizeof(Elf64_Rel) ||
4074 targ_sec_idx >= obj->efile.sec_cnt)
4075 return -LIBBPF_ERRNO__FORMAT;
4090 sec_desc->sec_type = SEC_RELO;
4091 sec_desc->shdr = sh;
4092 sec_desc->data = data;
4093 } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 ||
4095 sec_desc->sec_type = SEC_BSS;
4096 sec_desc->shdr = sh;
4097 sec_desc->data = data;
4100 (size_t)sh->sh_size);
4104 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
4105 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
4106 return -LIBBPF_ERRNO__FORMAT;
4113 /* sort BPF programs by section name and in-section instruction offset
4116 if (obj->nr_programs)
4117 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
4124 int bind = ELF64_ST_BIND(sym->st_info);
4126 return sym->st_shndx == SHN_UNDEF &&
4128 ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;
4133 int bind = ELF64_ST_BIND(sym->st_info);
4134 int type = ELF64_ST_TYPE(sym->st_info);
4137 if (sym->st_shndx != text_shndx)
4155 return -ESRCH;
4164 tname = btf__name_by_offset(btf, t->name_off);
4169 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
4170 return -EINVAL;
4173 return -EINVAL;
4178 return -ENOENT;
4187 return -ESRCH;
4198 if (vs->type == ext_btf_id)
4203 return -ENOENT;
4213 name = btf__name_by_offset(btf, t->name_off);
4222 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
4225 if (t->size == 1)
4227 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
4232 if (t->size != 4)
4242 if (btf_array(t)->nelems == 0)
4244 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
4257 if (a->type != b->type)
4258 return a->type < b->type ? -1 : 1;
4260 if (a->type == EXT_KCFG) {
4262 if (a->kcfg.align != b->kcfg.align)
4263 return a->kcfg.align > b->kcfg.align ? -1 : 1;
4265 if (a->kcfg.sz != b->kcfg.sz)
4266 return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
4270 return strcmp(a->name, b->name);
4308 vt = btf__type_by_id(btf, vs->type);
4339 if (!obj->efile.symbols)
4342 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
4344 if (!sh || sh->sh_entsize != sizeof(Elf64_Sym))
4345 return -LIBBPF_ERRNO__FORMAT;
4347 dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
4351 n = sh->sh_size / sh->sh_entsize;
4358 return -LIBBPF_ERRNO__FORMAT;
4361 ext_name = elf_sym_str(obj, sym->st_name);
4365 ext = obj->externs;
4366 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
4368 return -ENOMEM;
4369 obj->externs = ext;
4370 ext = &ext[obj->nr_extern];
4372 obj->nr_extern++;
4374 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
4375 if (ext->btf_id <= 0) {
4377 ext_name, ext->btf_id);
4378 return ext->btf_id;
4380 t = btf__type_by_id(obj->btf, ext->btf_id);
4381 ext->name = strdup(btf__name_by_offset(obj->btf, t->name_off));
4382 if (!ext->name)
4383 return -ENOMEM;
4384 ext->sym_idx = i;
4385 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;
4387 ext_essent_len = bpf_core_essential_name_len(ext->name);
4388 ext->essent_name = NULL;
4389 if (ext_essent_len != strlen(ext->name)) {
4390 ext->essent_name = strndup(ext->name, ext_essent_len);
4391 if (!ext->essent_name)
4392 return -ENOMEM;
4395 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
4396 if (ext->sec_btf_id <= 0) {
4398 ext_name, ext->btf_id, ext->sec_btf_id);
4399 return ext->sec_btf_id;
4401 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
4402 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
4407 ext->name, KCONFIG_SEC);
4408 return -ENOTSUP;
4411 ext->type = EXT_KCFG;
4412 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
4413 if (ext->kcfg.sz <= 0) {
4415 ext_name, ext->kcfg.sz);
4416 return ext->kcfg.sz;
4418 ext->kcfg.align = btf__align_of(obj->btf, t->type);
4419 if (ext->kcfg.align <= 0) {
4421 ext_name, ext->kcfg.align);
4422 return -EINVAL;
4424 ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
4425 &ext->kcfg.is_signed);
4426 if (ext->kcfg.type == KCFG_UNKNOWN) {
4428 return -ENOTSUP;
4432 ext->type = EXT_KSYM;
4433 skip_mods_and_typedefs(obj->btf, t->type,
4434 &ext->ksym.type_id);
4437 return -ENOTSUP;
4440 pr_debug("collected %d externs total\n", obj->nr_extern);
4442 if (!obj->nr_extern)
4446 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
4450 * pretending that each extern is a 8-byte variable
4453 /* find existing 4-byte integer type in BTF to use for fake
4456 int int_btf_id = find_int_btf_id(obj->btf);
4458 * will be used to replace the vs->type and
4464 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
4465 for (i = 0; i < obj->nr_extern; i++) {
4466 ext = &obj->externs[i];
4467 if (ext->type != EXT_KSYM)
4470 i, ext->sym_idx, ext->name);
4479 vt = (void *)btf__type_by_id(obj->btf, vs->type);
4480 ext_name = btf__name_by_offset(obj->btf, vt->name_off);
4485 return -ESRCH;
4492 func_proto = btf__type_by_id(obj->btf,
4493 vt->type);
4501 dummy_var->name_off;
4502 vs->type = dummy_var_btf_id;
4503 vt->info &= ~0xffff;
4504 vt->info |= BTF_FUNC_GLOBAL;
4506 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4507 vt->type = int_btf_id;
4509 vs->offset = off;
4510 vs->size = sizeof(int);
4512 sec->size = off;
4519 for (i = 0; i < obj->nr_extern; i++) {
4520 ext = &obj->externs[i];
4521 if (ext->type != EXT_KCFG)
4524 ext->kcfg.data_off = roundup(off, ext->kcfg.align);
4525 off = ext->kcfg.data_off + ext->kcfg.sz;
4527 i, ext->sym_idx, ext->kcfg.data_off, ext->name);
4529 sec->size = off;
4534 t = btf__type_by_id(obj->btf, vs->type);
4535 ext_name = btf__name_by_offset(obj->btf, t->name_off);
4540 return -ESRCH;
4542 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4543 vs->offset = ext->kcfg.data_off;
4551 return prog->sec_idx == obj->efile.text_shndx;
4563 if (!strcmp(prog->name, name))
4572 switch (obj->efile.secs[shndx].sec_type) {
4586 return shndx == obj->efile.btf_maps_shndx;
4592 if (shndx == obj->efile.symbols_shndx)
4595 switch (obj->efile.secs[shndx].sec_type) {
4614 purged = calloc(prog->insns_cnt, BPF_INSN_SZ);
4616 return -ENOMEM;
4621 for (i = 0; i < prog->insns_cnt; i++) {
4622 purged[i] = prog->insns[i];
4628 if (i >= prog->insns_cnt ||
4629 prog->insns[i].code != 0 ||
4630 prog->insns[i].dst_reg != 0 ||
4631 prog->insns[i].src_reg != 0 ||
4632 prog->insns[i].off != 0) {
4633 err = -EINVAL;
4636 purged[i] = prog->insns[i];
4640 libbpf_sha256(purged, prog->insns_cnt * sizeof(struct bpf_insn),
4641 prog->hash);
4652 struct bpf_insn *insn = &prog->insns[insn_idx];
4653 size_t map_idx, nr_maps = prog->obj->nr_maps;
4654 struct bpf_object *obj = prog->obj;
4655 __u32 shdr_idx = sym->st_shndx;
4662 prog->name, sym_name, insn_idx, insn->code);
4663 return -LIBBPF_ERRNO__RELOC;
4667 int sym_idx = ELF64_R_SYM(rel->r_info);
4668 int i, n = obj->nr_extern;
4672 ext = &obj->externs[i];
4673 if (ext->sym_idx == sym_idx)
4678 prog->name, sym_name, sym_idx);
4679 return -LIBBPF_ERRNO__RELOC;
4682 prog->name, i, ext->name, ext->sym_idx, insn_idx);
4683 if (insn->code == (BPF_JMP | BPF_CALL))
4684 reloc_desc->type = RELO_EXTERN_CALL;
4686 reloc_desc->type = RELO_EXTERN_LD64;
4687 reloc_desc->insn_idx = insn_idx;
4688 reloc_desc->ext_idx = i;
4692 /* sub-program call relocation */
4694 if (insn->src_reg != BPF_PSEUDO_CALL) {
4695 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
4696 return -LIBBPF_ERRNO__RELOC;
4699 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
4702 prog->name, sym_name, sym_sec_name);
4703 return -LIBBPF_ERRNO__RELOC;
4705 if (sym->st_value % BPF_INSN_SZ) {
4707 prog->name, sym_name, (size_t)sym->st_value);
4708 return -LIBBPF_ERRNO__RELOC;
4710 reloc_desc->type = RELO_CALL;
4711 reloc_desc->insn_idx = insn_idx;
4712 reloc_desc->sym_off = sym->st_value;
4718 prog->name, sym_name, shdr_idx);
4719 return -LIBBPF_ERRNO__RELOC;
4723 if (sym_is_subprog(sym, obj->efile.text_shndx)) {
4724 /* global_func: sym->st_value = offset in the section, insn->imm = 0.
4725 * local_func: sym->st_value = 0, insn->imm = offset in the section.
4727 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
4729 prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4730 return -LIBBPF_ERRNO__RELOC;
4733 reloc_desc->type = RELO_SUBPROG_ADDR;
4734 reloc_desc->insn_idx = insn_idx;
4735 reloc_desc->sym_off = sym->st_value;
4743 if (shdr_idx == obj->efile.arena_data_shndx) {
4744 if (obj->arena_map_idx < 0) {
4746 prog->name, insn_idx);
4747 return -LIBBPF_ERRNO__RELOC;
4749 reloc_desc->type = RELO_DATA;
4750 reloc_desc->insn_idx = insn_idx;
4751 reloc_desc->map_idx = obj->arena_map_idx;
4752 reloc_desc->sym_off = sym->st_value;
4754 map = &obj->maps[obj->arena_map_idx];
4756 prog->name, obj->arena_map_idx, map->name, map->sec_idx,
4757 map->sec_offset, insn_idx);
4762 if (shdr_idx == obj->efile.jumptables_data_shndx) {
4763 reloc_desc->type = RELO_INSN_ARRAY;
4764 reloc_desc->insn_idx = insn_idx;
4765 reloc_desc->map_idx = -1;
4766 reloc_desc->sym_off = sym->st_value;
4767 reloc_desc->sym_size = sym->st_size;
4775 prog->name, sym_name, sym_sec_name);
4776 return -LIBBPF_ERRNO__RELOC;
4779 map = &obj->maps[map_idx];
4780 if (map->libbpf_type != type ||
4781 map->sec_idx != sym->st_shndx ||
4782 map->sec_offset != sym->st_value)
4785 prog->name, map_idx, map->name, map->sec_idx,
4786 map->sec_offset, insn_idx);
4791 prog->name, sym_sec_name, (size_t)sym->st_value);
4792 return -LIBBPF_ERRNO__RELOC;
4794 reloc_desc->type = RELO_LD64;
4795 reloc_desc->insn_idx = insn_idx;
4796 reloc_desc->map_idx = map_idx;
4797 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
4804 prog->name, sym_sec_name);
4805 return -LIBBPF_ERRNO__RELOC;
4808 map = &obj->maps[map_idx];
4809 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)
4812 prog->name, map_idx, map->name, map->sec_idx,
4813 map->sec_offset, insn_idx);
4818 prog->name, sym_sec_name);
4819 return -LIBBPF_ERRNO__RELOC;
4822 reloc_desc->type = RELO_DATA;
4823 reloc_desc->insn_idx = insn_idx;
4824 reloc_desc->map_idx = map_idx;
4825 reloc_desc->sym_off = sym->st_value;
4831 return insn_idx >= prog->sec_insn_off &&
4832 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4838 int l = 0, r = obj->nr_programs - 1, m;
4841 if (!obj->nr_programs)
4845 m = l + (r - l + 1) / 2;
4846 prog = &obj->programs[m];
4848 if (prog->sec_idx < sec_idx ||
4849 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4852 r = m - 1;
4857 prog = &obj->programs[l];
4858 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4867 size_t sec_idx = shdr->sh_info, sym_idx;
4878 if (sec_idx >= obj->efile.sec_cnt)
4879 return -EINVAL;
4884 return -LIBBPF_ERRNO__FORMAT;
4886 relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4889 return -EINVAL;
4893 nrels = shdr->sh_size / shdr->sh_entsize;
4899 return -LIBBPF_ERRNO__FORMAT;
4902 sym_idx = ELF64_R_SYM(rel->r_info);
4907 return -LIBBPF_ERRNO__FORMAT;
4910 if (sym->st_shndx >= obj->efile.sec_cnt) {
4912 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);
4913 return -LIBBPF_ERRNO__FORMAT;
4916 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) {
4918 relo_sec_name, (size_t)rel->r_offset, i);
4919 return -LIBBPF_ERRNO__FORMAT;
4922 insn_idx = rel->r_offset / BPF_INSN_SZ;
4929 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)
4930 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));
4932 sym_name = elf_sym_str(obj, sym->st_name);
4945 relos = libbpf_reallocarray(prog->reloc_desc,
4946 prog->nr_reloc + 1, sizeof(*relos));
4948 return -ENOMEM;
4949 prog->reloc_desc = relos;
4952 insn_idx -= prog->sec_insn_off;
4953 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4958 prog->nr_reloc++;
4967 if (!obj->btf)
4968 return -ENOENT;
4970 /* if it's BTF-defined map, we don't need to search for type IDs.
4974 if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))
4982 return -ENOENT;
4984 id = btf__find_by_name(obj->btf, map->real_name);
4988 map->btf_key_type_id = 0;
4989 map->btf_value_type_id = id;
5005 err = -errno;
5013 info->type = val;
5015 info->key_size = val;
5017 info->value_size = val;
5019 info->max_entries = val;
5021 info->map_flags = val;
5031 return map->obj->state >= OBJ_PREPARED || map->reused;
5036 return map->autocreate;
5042 return libbpf_err(-EBUSY);
5044 map->autocreate = autocreate;
5051 return libbpf_err(-EINVAL);
5053 map->autoattach = autoattach;
5059 return map->autoattach;
5077 if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)
5078 new_name = strdup(map->name);
5083 return libbpf_err(-errno);
5092 err = -errno;
5096 err = reuse_fd(map->fd, new_fd);
5100 free(map->name);
5102 map->name = new_name;
5103 map->def.type = info.type;
5104 map->def.key_size = info.key_size;
5105 map->def.value_size = info.value_size;
5106 map->def.max_entries = info.max_entries;
5107 map->def.map_flags = info.map_flags;
5108 map->btf_key_type_id = info.btf_key_type_id;
5109 map->btf_value_type_id = info.btf_value_type_id;
5110 map->reused = true;
5111 map->map_extra = info.map_extra;
5122 return map->def.max_entries;
5127 if (!bpf_map_type__is_map_in_map(map->def.type))
5130 return map->inner_map;
5136 return libbpf_err(-EBUSY);
5138 map->def.max_entries = max_entries;
5140 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
5142 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
5150 int bpffs_fd = -1, token_fd, err;
5155 if (obj->token_path && obj->token_path[0] == '\0') {
5156 pr_debug("object '%s': token is prevented, skipping...\n", obj->name);
5160 mandatory = obj->token_path != NULL;
5163 bpffs_path = obj->token_path ?: BPF_FS_DEFAULT_PATH;
5166 err = -errno;
5168 obj->name, errstr(err), bpffs_path,
5176 if (!mandatory && token_fd == -ENOENT) {
5178 obj->name, bpffs_path);
5182 obj->name, token_fd, bpffs_path,
5187 obj->feat_cache = calloc(1, sizeof(*obj->feat_cache));
5188 if (!obj->feat_cache) {
5190 return -ENOMEM;
5193 obj->token_fd = token_fd;
5194 obj->feat_cache->token_fd = token_fd;
5208 if (obj->gen_loader || obj->token_fd)
5224 return -ret;
5233 if (obj->gen_loader)
5239 if (obj->feat_cache)
5240 return feat_supported(obj->feat_cache, feat_id);
5245 /* Used in testing to simulate missing features. */
5248 if (obj->feat_cache)
5249 free(obj->feat_cache);
5250 obj->feat_cache = cache;
5276 if (map->def.type == BPF_MAP_TYPE_DEVMAP || map->def.type == BPF_MAP_TYPE_DEVMAP_HASH)
5279 return (map_info.type == map->def.type &&
5280 map_info.key_size == map->def.key_size &&
5281 map_info.value_size == map->def.value_size &&
5282 map_info.max_entries == map->def.max_entries &&
5283 map_info.map_flags == map->def.map_flags &&
5284 map_info.map_extra == map->map_extra);
5292 pin_fd = bpf_obj_get(map->pin_path);
5294 err = -errno;
5295 if (err == -ENOENT) {
5297 map->pin_path);
5302 map->pin_path, errstr(err));
5308 map->pin_path);
5310 return -EINVAL;
5318 map->pinned = true;
5319 pr_debug("reused pinned map at '%s'\n", map->pin_path);
5327 enum libbpf_map_type map_type = map->libbpf_type;
5333 if (obj->gen_loader) {
5334 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
5335 map->mmaped, map->def.value_size, update_flags);
5337 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);
5341 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, update_flags);
5343 err = -errno;
5349 /* Freeze .rodata and .kconfig map as read-only from syscall side. */
5351 err = bpf_map_freeze(map->fd);
5353 err = -errno;
5354 pr_warn("map '%s': failed to freeze as read-only: %s\n",
5360 /* Remap anonymous mmap()-ed "map initialization image" as
5361 * a BPF map-backed mmap()-ed memory, but preserving the same
5370 if (map->def.map_flags & BPF_F_MMAPABLE) {
5374 if (map->def.map_flags & BPF_F_RDONLY_PROG)
5378 mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map->fd, 0);
5380 err = -errno;
5381 pr_warn("map '%s': failed to re-mmap() contents: %s\n",
5385 map->mmaped = mmaped;
5387 if (mprotect(map->mmaped, mmap_sz, PROT_READ)) {
5388 err = -errno;
5393 } else if (map->mmaped) {
5394 munmap(map->mmaped, mmap_sz);
5395 map->mmaped = NULL;
5406 struct bpf_map_def *def = &map->def;
5411 map_name = map->name;
5412 create_attr.map_ifindex = map->map_ifindex;
5413 create_attr.map_flags = def->map_flags;
5414 create_attr.numa_node = map->numa_node;
5415 create_attr.map_extra = map->map_extra;
5416 create_attr.token_fd = obj->token_fd;
5417 if (obj->token_fd)
5419 if (map->excl_prog) {
5420 err = bpf_prog_compute_hash(map->excl_prog);
5424 create_attr.excl_prog_hash = map->excl_prog->hash;
5429 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
5430 if (map->mod_btf_fd >= 0) {
5431 create_attr.value_type_btf_obj_fd = map->mod_btf_fd;
5436 if (obj->btf && btf__fd(obj->btf) >= 0) {
5437 create_attr.btf_fd = btf__fd(obj->btf);
5438 create_attr.btf_key_type_id = map->btf_key_type_id;
5439 create_attr.btf_value_type_id = map->btf_value_type_id;
5442 if (bpf_map_type__is_map_in_map(def->type)) {
5443 if (map->inner_map) {
5444 err = map_set_def_max_entries(map->inner_map);
5447 err = bpf_object__create_map(obj, map->inner_map, true);
5450 map->name, errstr(err));
5453 map->inner_map_fd = map->inner_map->fd;
5455 if (map->inner_map_fd >= 0)
5456 create_attr.inner_map_fd = map->inner_map_fd;
5459 switch (def->type) {
5477 map->btf_key_type_id = 0;
5478 map->btf_value_type_id = 0;
5487 if (obj->gen_loader) {
5488 bpf_gen__map_create(obj->gen_loader, def->type, map_name,
5489 def->key_size, def->value_size, def->max_entries,
5490 &create_attr, is_inner ? -1 : map - obj->maps);
5495 * will be reset to -1 eventually.
5497 map_fd = map->fd;
5499 map_fd = bpf_map_create(def->type, map_name,
5500 def->key_size, def->value_size,
5501 def->max_entries, &create_attr);
5504 err = -errno;
5506 map->name, errstr(err));
5510 map->btf_key_type_id = 0;
5511 map->btf_value_type_id = 0;
5512 map_fd = bpf_map_create(def->type, map_name,
5513 def->key_size, def->value_size,
5514 def->max_entries, &create_attr);
5517 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
5518 if (obj->gen_loader)
5519 map->inner_map->fd = -1;
5520 bpf_map__destroy(map->inner_map);
5521 zfree(&map->inner_map);
5527 /* obj->gen_loader case, prevent reuse_fd() from closing map_fd */
5528 if (map->fd == map_fd)
5534 * map->fd stays valid but now point to what map_fd points to.
5536 return reuse_fd(map->fd, map_fd);
5545 for (i = 0; i < map->init_slots_sz; i++) {
5546 if (!map->init_slots[i])
5549 targ_map = map->init_slots[i];
5550 fd = targ_map->fd;
5552 if (obj->gen_loader) {
5553 bpf_gen__populate_outer_map(obj->gen_loader,
5554 map - obj->maps, i,
5555 targ_map - obj->maps);
5557 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5560 err = -errno;
5562 map->name, i, targ_map->name, fd, errstr(err));
5566 map->name, i, targ_map->name, fd);
5569 zfree(&map->init_slots);
5570 map->init_slots_sz = 0;
5581 if (obj->gen_loader)
5582 return -ENOTSUP;
5584 for (i = 0; i < map->init_slots_sz; i++) {
5585 if (!map->init_slots[i])
5588 targ_prog = map->init_slots[i];
5591 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5593 err = -errno;
5595 map->name, i, targ_prog->name, fd, errstr(err));
5599 map->name, i, targ_prog->name, fd);
5602 zfree(&map->init_slots);
5603 map->init_slots_sz = 0;
5613 for (i = 0; i < obj->nr_maps; i++) {
5614 map = &obj->maps[i];
5616 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY)
5628 if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) {
5634 map->name, nr_cpus);
5637 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
5638 map->def.max_entries = nr_cpus;
5652 for (i = 0; i < obj->nr_maps; i++) {
5653 map = &obj->maps[i];
5657 * loading, if we detect that at least one of the to-be-loaded
5662 * but also it allows to have CO-RE applications that use
5664 * If those global variable-using programs are not loaded at
5671 bool is_percpu = map->libbpf_type == LIBBPF_MAP_PERCPU;
5676 map->autocreate = false;
5679 if (!map->autocreate) {
5680 pr_debug("map '%s': skipped auto-creating...\n", map->name);
5690 if (map->pin_path) {
5694 map->name);
5697 if (retried && map->fd < 0) {
5699 map->name);
5700 err = -ENOENT;
5705 if (map->reused) {
5707 map->name, map->fd);
5714 map->name, map->fd);
5720 } else if (map->def.type == BPF_MAP_TYPE_ARENA) {
5721 map->mmaped = mmap((void *)(long)map->map_extra,
5723 map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED,
5724 map->fd, 0);
5725 if (map->mmaped == MAP_FAILED) {
5726 err = -errno;
5727 map->mmaped = NULL;
5729 map->name, errstr(err));
5732 if (obj->arena_data) {
5733 memcpy(map->mmaped + obj->arena_data_off, obj->arena_data,
5734 obj->arena_data_sz);
5735 zfree(&obj->arena_data);
5738 if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) {
5745 if (map->pin_path && !map->pinned) {
5748 if (!retried && err == -EEXIST) {
5752 pr_warn("map '%s': failed to auto-pin at '%s': %s\n",
5753 map->name, map->pin_path, errstr(err));
5762 pr_warn("map '%s': failed to create: %s\n", map->name, errstr(err));
5765 zclose(obj->maps[j].fd);
5779 * underscore is ignored by BPF CO-RE relocation during relocation matching.
5786 for (i = n - 5; i >= 0; i--) {
5798 free(cands->cands);
5815 local_t = btf__type_by_id(local_cand->btf, local_cand->id);
5816 local_name = btf__str_by_offset(local_cand->btf, local_t->name_off);
5824 targ_name = btf__name_by_offset(targ_btf, t->name_off);
5835 pr_debug("CO-RE relocating [%u] %s %s: found target candidate [%d] %s %s in [%s]\n",
5836 local_cand->id, btf_kind_str(local_t),
5839 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
5840 sizeof(*cands->cands));
5842 return -ENOMEM;
5844 cand = &new_cands[cands->len];
5845 cand->btf = targ_btf;
5846 cand->id = i;
5848 cands->cands = new_cands;
5849 cands->len++;
5863 if (obj->btf_modules_loaded)
5866 if (obj->gen_loader)
5870 obj->btf_modules_loaded = true;
5885 err = -errno;
5894 err = -errno;
5907 err = -errno;
5912 /* ignore non-module BTFs */
5918 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
5926 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
5927 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
5931 mod_btf = &obj->btf_modules[obj->btf_module_cnt];
5933 mod_btf->btf = btf;
5934 mod_btf->id = id;
5935 mod_btf->fd = fd;
5936 mod_btf->name = strdup(name);
5937 if (!mod_btf->name) {
5938 err = -ENOMEM;
5941 obj->btf_module_cnt++;
5966 return ERR_PTR(-EINVAL);
5968 local_name = btf__name_by_offset(local_btf, local_t->name_off);
5970 return ERR_PTR(-EINVAL);
5975 return ERR_PTR(-ENOMEM);
5978 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
5984 if (cands->len)
5988 if (obj->btf_vmlinux_override)
5996 for (i = 0; i < obj->btf_module_cnt; i++) {
5998 obj->btf_modules[i].btf,
5999 obj->btf_modules[i].name,
6000 btf__type_cnt(obj->btf_vmlinux),
6013 * type-based CO-RE relocations and follow slightly different rules than
6014 * field-based relocations. This function assumes that root types were already
6015 * checked for name match. Beyond that initial root-level name check, names
6017 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
6020 * - for ENUMs, the size is ignored;
6021 * - for INT, size and signedness are ignored;
6022 * - for ARRAY, dimensionality is ignored, element types are checked for
6024 * - CONST/VOLATILE/RESTRICT modifiers are ignored;
6025 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
6026 * - FUNC_PROTOs are compatible if they have compatible signature: same
6029 * more experience with using BPF CO-RE relocations.
6058 relos = libbpf_reallocarray(prog->reloc_desc,
6059 prog->nr_reloc + 1, sizeof(*relos));
6061 return -ENOMEM;
6062 relo = &relos[prog->nr_reloc];
6063 relo->type = RELO_CORE;
6064 relo->insn_idx = insn_idx;
6065 relo->core_relo = core_relo;
6066 prog->reloc_desc = relos;
6067 prog->nr_reloc++;
6076 for (i = 0; i < prog->nr_reloc; i++) {
6077 relo = &prog->reloc_desc[i];
6078 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx)
6081 return relo->core_relo;
6096 const char *prog_name = prog->name;
6099 __u32 local_id = relo->type_id;
6104 return -EINVAL;
6106 local_name = btf__name_by_offset(local_btf, local_type->name_off);
6108 return -EINVAL;
6110 if (relo->kind != BPF_CORE_TYPE_ID_LOCAL &&
6112 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
6144 if (obj->btf_ext->core_relo_info.len == 0)
6148 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
6149 err = libbpf_get_error(obj->btf_vmlinux_override);
6162 seg = &obj->btf_ext->core_relo_info;
6165 sec_idx = seg->sec_idxs[sec_num];
6168 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6170 err = -EINVAL;
6174 pr_debug("sec '%s': found %u CO-RE relocations\n", sec_name, sec->num_info);
6177 if (rec->insn_off % BPF_INSN_SZ)
6178 return -EINVAL;
6179 insn_idx = rec->insn_off / BPF_INSN_SZ;
6186 * This is similar to what x86-64 linker does for relocations.
6190 pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subprogram\n",
6194 /* no need to apply CO-RE relocation if the program is
6197 if (!prog->autoload)
6201 * program's frame of reference; (sub-)program code is not yet
6202 * relocated, so it's enough to just subtract in-section offset
6204 insn_idx = insn_idx - prog->sec_insn_off;
6205 if (insn_idx >= prog->insns_cnt)
6206 return -EINVAL;
6207 insn = &prog->insns[insn_idx];
6209 if (is_ldimm64_insn(insn) && (size_t)insn_idx + 1 >= prog->insns_cnt) {
6211 prog->name, i, insn_idx);
6212 err = -EINVAL;
6219 prog->name, i, errstr(err));
6223 if (prog->obj->gen_loader)
6226 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
6229 prog->name, i, errstr(err));
6233 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
6236 prog->name, i, insn_idx, errstr(err));
6243 /* obj->btf_vmlinux and module BTFs are freed after object load */
6244 btf__free(obj->btf_vmlinux_override);
6245 obj->btf_vmlinux_override = NULL;
6249 bpf_core_free_cands(entry->pvalue);
6267 prog->name, relo_idx, insn_idx, map_idx, map->name);
6271 insn->code = BPF_JMP | BPF_CALL;
6272 insn->dst_reg = 0;
6273 insn->src_reg = 0;
6274 insn->off = 0;
6278 * where lower 123 is map index into obj->maps[] array
6280 insn->imm = POISON_LDIMM64_MAP_BASE + map_idx;
6295 prog->name, relo_idx, insn_idx, ext->name);
6298 insn->code = BPF_JMP | BPF_CALL;
6299 insn->dst_reg = 0;
6300 insn->src_reg = 0;
6301 insn->off = 0;
6305 * where lower 123 is extern index into obj->externs[] array
6307 insn->imm = POISON_CALL_KFUNC_BASE + ext_idx;
6314 for (i = 0; i < obj->jumptable_map_cnt; i++) {
6320 if (obj->jumptable_maps[i].sym_off == sym_off &&
6321 obj->jumptable_maps[i].prog == prog)
6322 return obj->jumptable_maps[i].fd;
6325 return -ENOENT;
6330 size_t cnt = obj->jumptable_map_cnt;
6331 size_t size = sizeof(obj->jumptable_maps[0]);
6334 tmp = libbpf_reallocarray(obj->jumptable_maps, cnt + 1, size);
6336 return -ENOMEM;
6338 obj->jumptable_maps = tmp;
6339 obj->jumptable_maps[cnt].prog = prog;
6340 obj->jumptable_maps[cnt].sym_off = sym_off;
6341 obj->jumptable_maps[cnt].fd = map_fd;
6342 obj->jumptable_map_cnt++;
6351 for (i = prog->subprog_cnt - 1; i >= 0; i--) {
6352 if (insn_idx >= prog->subprogs[i].sub_insn_off)
6356 return -1;
6362 unsigned int sym_off = relo->sym_off;
6363 int jt_size = relo->sym_size;
6380 return -EINVAL;
6386 return -EINVAL;
6394 if (!obj->jumptables_data) {
6396 err = -EINVAL;
6399 if (sym_off + jt_size > obj->jumptables_data_sz) {
6401 obj->jumptables_data_sz, sym_off + jt_size);
6402 err = -EINVAL;
6406 subprog_idx = -1; /* main program */
6407 if (relo->insn_idx < 0 || relo->insn_idx >= prog->insns_cnt) {
6408 pr_warn("map '.jumptables': invalid instruction index %d\n", relo->insn_idx);
6409 err = -EINVAL;
6412 if (prog->subprogs)
6413 subprog_idx = find_subprog_idx(prog, relo->insn_idx);
6415 jt = (__u64 *)(obj->jumptables_data + sym_off);
6423 insn_off -= prog->subprogs[subprog_idx].sec_insn_off;
6424 insn_off += prog->subprogs[subprog_idx].sub_insn_off;
6426 insn_off -= prog->sec_insn_off;
6430 * LLVM-generated jump tables contain u64 records, however
6436 err = -EINVAL;
6462 * - map references;
6463 * - global variable references;
6464 * - extern references.
6471 for (i = 0; i < prog->nr_reloc; i++) {
6472 struct reloc_desc *relo = &prog->reloc_desc[i];
6473 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6477 switch (relo->type) {
6479 map = &obj->maps[relo->map_idx];
6480 if (obj->gen_loader) {
6482 insn[0].imm = relo->map_idx;
6483 } else if (map->autocreate) {
6485 insn[0].imm = map->fd;
6487 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6488 relo->map_idx, map);
6492 map = &obj->maps[relo->map_idx];
6493 insn[1].imm = insn[0].imm + relo->sym_off;
6495 if (relo->map_idx == obj->arena_map_idx)
6496 insn[1].imm += obj->arena_data_off;
6498 if (obj->gen_loader) {
6500 insn[0].imm = relo->map_idx;
6501 } else if (map->autocreate) {
6503 insn[0].imm = map->fd;
6505 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6506 relo->map_idx, map);
6510 ext = &obj->externs[relo->ext_idx];
6511 if (ext->type == EXT_KCFG) {
6512 if (obj->gen_loader) {
6514 insn[0].imm = obj->kconfig_map_idx;
6517 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
6519 insn[1].imm = ext->kcfg.data_off;
6521 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */
6523 insn[0].imm = ext->ksym.kernel_btf_id;
6524 insn[1].imm = ext->ksym.kernel_btf_obj_fd;
6526 insn[0].imm = (__u32)ext->ksym.addr;
6527 insn[1].imm = ext->ksym.addr >> 32;
6532 ext = &obj->externs[relo->ext_idx];
6534 if (ext->is_set) {
6535 insn[0].imm = ext->ksym.kernel_btf_id;
6536 insn[0].off = ext->ksym.btf_fd_idx;
6538 poison_kfunc_call(prog, i, relo->insn_idx, insn,
6539 relo->ext_idx, ext);
6545 prog->name, i);
6546 return -EINVAL;
6562 prog->name, i, relo->sym_off);
6566 insn->imm = map_fd;
6567 insn->off = 0;
6572 prog->name, i, relo->type);
6573 return -EINVAL;
6594 sec_idx = ext_info->sec_idxs[sec_num];
6596 if (prog->sec_idx != sec_idx)
6602 if (insn_off < prog->sec_insn_off)
6604 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
6609 copy_end = rec + ext_info->rec_size;
6613 return -ENOENT;
6615 /* append func/line info of a given (sub-)program to the main
6618 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
6619 new_sz = old_sz + (copy_end - copy_start);
6622 return -ENOMEM;
6624 *prog_rec_cnt = new_sz / ext_info->rec_size;
6625 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
6627 /* Kernel instruction offsets are in units of 8-byte
6633 off_adj = prog->sub_insn_off - prog->sec_insn_off;
6636 for (; rec < rec_end; rec += ext_info->rec_size) {
6641 *prog_rec_sz = ext_info->rec_size;
6645 return -ENOENT;
6658 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC))
6664 if (main_prog != prog && !main_prog->func_info)
6667 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6668 &main_prog->func_info,
6669 &main_prog->func_info_cnt,
6670 &main_prog->func_info_rec_size);
6672 if (err != -ENOENT) {
6674 prog->name, errstr(err));
6677 if (main_prog->func_info) {
6682 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6687 prog->name);
6692 if (main_prog != prog && !main_prog->line_info)
6695 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6696 &main_prog->line_info,
6697 &main_prog->line_info_cnt,
6698 &main_prog->line_info_rec_size);
6700 if (err != -ENOENT) {
6702 prog->name, errstr(err));
6705 if (main_prog->line_info) {
6710 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6715 prog->name);
6725 if (insn_idx == relo->insn_idx)
6727 return insn_idx < relo->insn_idx ? -1 : 1;
6732 if (!prog->nr_reloc)
6734 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6735 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6740 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;
6746 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));
6752 return -ENOMEM;
6753 if (subprog->nr_reloc)
6754 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,
6755 sizeof(*relos) * subprog->nr_reloc);
6757 for (i = main_prog->nr_reloc; i < new_cnt; i++)
6758 relos[i].insn_idx += subprog->sub_insn_off;
6762 main_prog->reloc_desc = relos;
6763 main_prog->nr_reloc = new_cnt;
6769 size_t size = sizeof(main_prog->subprogs[0]);
6770 int cnt = main_prog->subprog_cnt;
6773 tmp = libbpf_reallocarray(main_prog->subprogs, cnt + 1, size);
6775 return -ENOMEM;
6777 main_prog->subprogs = tmp;
6778 main_prog->subprogs[cnt].sec_insn_off = subprog->sec_insn_off;
6779 main_prog->subprogs[cnt].sub_insn_off = subprog->sub_insn_off;
6780 main_prog->subprog_cnt++;
6793 subprog->sub_insn_off = main_prog->insns_cnt;
6795 new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6796 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6798 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6799 return -ENOMEM;
6801 main_prog->insns = insns;
6802 main_prog->insns_cnt = new_cnt;
6804 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6805 subprog->insns_cnt * sizeof(*insns));
6807 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6808 main_prog->name, subprog->insns_cnt, subprog->name);
6818 main_prog->name, errstr(err));
6839 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6840 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6845 if (relo && relo->type == RELO_EXTERN_CALL)
6850 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
6852 prog->name, insn_idx, relo->type);
6853 return -LIBBPF_ERRNO__RELOC;
6856 /* sub-program instruction index is a combination of
6859 * call always has imm = -1, but for static functions
6860 * relocation is against STT_SECTION and insn->imm
6863 * for subprog addr relocation, the relo->sym_off + insn->imm is
6866 if (relo->type == RELO_CALL)
6867 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6869 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;
6876 prog->name, insn_idx);
6877 return -LIBBPF_ERRNO__RELOC;
6882 * offset necessary, insns->imm is relative to
6885 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6888 /* we enforce that sub-programs should be in .text section */
6889 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6891 pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6892 prog->name);
6893 return -LIBBPF_ERRNO__RELOC;
6899 * - append it at the end of main program's instructions blog;
6900 * - process is recursively, while current program is put on hold;
6901 * - if that subprogram calls some other not yet processes
6906 if (subprog->sub_insn_off == 0) {
6915 /* main_prog->insns memory could have been re-allocated, so
6918 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6925 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6928 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6935 * Relocate sub-program calls.
6937 * Algorithm operates as follows. Each entry-point BPF program (referred to as
6938 * main prog) is processed separately. For each subprog (non-entry functions,
6947 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6963 * subprog->sub_insn_off as zero at all times and won't be appended to current
6972 * +--------+ +-------+
6974 * +--+---+ +--+-+-+ +---+--+
6976 * +--+---+ +------+ +---+--+
6979 * +---+-------+ +------+----+
6981 * +-----------+ +-----------+
6986 * +-----------+------+
6988 * +-----------+------+
6993 * +-----------+------+------+
6995 * +-----------+------+------+
7004 * +-----------+------+
7006 * +-----------+------+
7009 * +-----------+------+------+
7011 * +-----------+------+------+
7024 for (i = 0; i < obj->nr_programs; i++) {
7025 subprog = &obj->programs[i];
7029 subprog->sub_insn_off = 0;
7046 for (i = 0; i < obj->nr_programs; i++) {
7047 prog = &obj->programs[i];
7048 zfree(&prog->reloc_desc);
7049 prog->nr_reloc = 0;
7058 if (a->insn_idx != b->insn_idx)
7059 return a->insn_idx < b->insn_idx ? -1 : 1;
7062 if (a->type != b->type)
7063 return a->type < b->type ? -1 : 1;
7072 for (i = 0; i < obj->nr_programs; i++) {
7073 struct bpf_program *p = &obj->programs[i];
7075 if (!p->nr_reloc)
7078 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
7088 if (!obj->btf || !kernel_supports(obj, FEAT_BTF_DECL_TAG))
7091 n = btf__type_cnt(obj->btf);
7096 t = btf_type_by_id(obj->btf, i);
7097 if (!btf_is_decl_tag(t) || btf_decl_tag(t)->component_idx != -1)
7100 name = btf__str_by_offset(obj->btf, t->name_off);
7104 t = btf_type_by_id(obj->btf, t->type);
7107 prog->name);
7108 return -EINVAL;
7110 if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)) != 0)
7116 if (prog->exception_cb_idx >= 0) {
7117 prog->exception_cb_idx = -1;
7124 prog->name);
7125 return -EINVAL;
7128 for (j = 0; j < obj->nr_programs; j++) {
7129 struct bpf_program *subprog = &obj->programs[j];
7133 if (strcmp(name, subprog->name) != 0)
7135 /* Enforce non-hidden, as from verifier point of
7139 if (!subprog->sym_global || subprog->mark_btf_static) {
7140 pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n",
7141 prog->name, subprog->name);
7142 return -EINVAL;
7145 if (prog->exception_cb_idx >= 0) {
7147 prog->name, subprog->name);
7148 return -EINVAL;
7150 prog->exception_cb_idx = j;
7154 if (prog->exception_cb_idx >= 0)
7157 pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name);
7158 return -ENOENT;
7196 /* forward declarations for arch-specific underlying types of bpf_user_pt_regs_t typedef,
7198 * with this approach we don't need any extra arch-specific #ifdef guards
7221 t = btf__type_by_id(btf, t->type);
7223 (prog->type == BPF_PROG_TYPE_KPROBE || prog->type == BPF_PROG_TYPE_PERF_EVENT)) {
7224 tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>";
7230 t = skip_mods_and_typedefs(btf, t->type, NULL);
7237 tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>";
7242 switch (prog->type) {
7262 if (btf_is_int(t) && t->size == 8)
7271 prog->name, subprog_name, arg_idx, ctx_name);
7282 /* caller already validated FUNC -> FUNC_PROTO validity */
7284 fn_proto_t = btf_type_by_id(btf, fn_t->type);
7295 fn_name_off = fn_t->name_off; /* we are about to invalidate fn_t */
7297 orig_proto_id = fn_t->type; /* original FUNC_PROTO ID */
7298 ret_type_id = fn_proto_t->type; /* fn_proto_t will be invalidated */
7304 return -EINVAL;
7312 name_off = p->name_off;
7314 err = btf__add_func_param(btf, "", p->type);
7320 p->name_off = name_off; /* use remembered str offset */
7323 /* clone FUNC now, btf__add_func() enforces non-empty name, so use
7327 fn_id = btf__add_func(btf, prog->name, linkage, fn_proto_id);
7329 return -EINVAL;
7332 fn_t->name_off = fn_name_off; /* reuse original string */
7349 struct btf *btf = obj->btf;
7357 if (!obj->btf_ext || !prog->func_info)
7368 if (global_ctx_map[i].prog_type != prog->type)
7377 orig_ids = calloc(prog->func_info_cnt, sizeof(*orig_ids));
7379 return -ENOMEM;
7380 for (i = 0; i < prog->func_info_cnt; i++) {
7381 func_rec = prog->func_info + prog->func_info_rec_size * i;
7382 orig_ids[i] = func_rec->type_id;
7387 * clone and adjust FUNC -> FUNC_PROTO combo
7394 if (strcmp(btf__str_by_offset(btf, t->name_off), ctx_tag) != 0)
7398 orig_fn_id = t->type;
7403 /* sanity check FUNC -> FUNC_PROTO chain, just in case */
7404 fn_proto_t = btf_type_by_id(btf, fn_t->type);
7410 for (rec_idx = 0; rec_idx < prog->func_info_cnt; rec_idx++) {
7411 if (orig_ids[rec_idx] == t->type) {
7412 func_rec = prog->func_info + prog->func_info_rec_size * rec_idx;
7422 arg_idx = btf_decl_tag(t)->component_idx;
7428 fn_name = btf__str_by_offset(btf, fn_t->name_off) ?: "<anon>";
7429 if (!need_func_arg_type_fixup(btf, prog, fn_name, arg_idx, p->type, ctx_name))
7433 if (func_rec->type_id == orig_fn_id) {
7443 func_rec->type_id = fn_id;
7446 /* create PTR -> STRUCT type chain to mark PTR_TO_CTX argument;
7448 * funcs share the same program type, so need only PTR ->
7455 err = -EINVAL;
7461 tag_id = btf__add_decl_tag(btf, ctx_tag, func_rec->type_id, arg_idx);
7463 err = -EINVAL;
7468 fn_t = btf_type_by_id(btf, func_rec->type_id);
7469 fn_proto_t = btf_type_by_id(btf, fn_t->type);
7473 p->type = ptr_id;
7489 if (obj->btf_ext) {
7492 pr_warn("failed to perform CO-RE relocations: %s\n",
7500 if (obj->arena_map_idx >= 0 && kernel_supports(obj, FEAT_LDIMM64_FULL_RANGE_OFF)) {
7501 struct bpf_map *arena_map = &obj->maps[obj->arena_map_idx];
7503 obj->arena_data_off = bpf_map_mmap_sz(arena_map) -
7504 roundup(obj->arena_data_sz, sysconf(_SC_PAGE_SIZE));
7507 /* Before relocating calls pre-process relocations and mark
7514 for (i = 0; i < obj->nr_programs; i++) {
7515 prog = &obj->programs[i];
7516 for (j = 0; j < prog->nr_reloc; j++) {
7517 struct reloc_desc *relo = &prog->reloc_desc[j];
7518 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
7521 if (relo->type == RELO_SUBPROG_ADDR)
7533 for (i = 0; i < obj->nr_programs; i++) {
7534 prog = &obj->programs[i];
7535 /* sub-program's sub-calls are relocated within the context of
7540 if (!prog->autoload)
7546 prog->name, errstr(err));
7554 if (prog->exception_cb_idx >= 0) {
7555 struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx];
7562 if (subprog->sub_insn_off == 0) {
7572 for (i = 0; i < obj->nr_programs; i++) {
7573 prog = &obj->programs[i];
7576 if (!prog->autoload)
7583 prog->name, errstr(err));
7591 prog->name, errstr(err));
7619 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
7620 return -EINVAL;
7621 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
7623 return -EINVAL;
7625 nrels = shdr->sh_size / shdr->sh_entsize;
7630 return -LIBBPF_ERRNO__FORMAT;
7633 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
7636 i, (size_t)ELF64_R_SYM(rel->r_info));
7637 return -LIBBPF_ERRNO__FORMAT;
7639 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
7641 pr_debug(".maps relo #%d: for %zd value %zu rel->r_offset %zu name %u ('%s')\n",
7642 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value,
7643 (size_t)rel->r_offset, sym->st_name, name);
7645 for (j = 0; j < obj->nr_maps; j++) {
7646 map = &obj->maps[j];
7647 if (map->sec_idx != obj->efile.btf_maps_shndx)
7650 vi = btf_var_secinfos(sec) + map->btf_var_idx;
7651 if (vi->offset <= rel->r_offset &&
7652 rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size)
7655 if (j == obj->nr_maps) {
7656 pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n",
7657 i, name, (size_t)rel->r_offset);
7658 return -EINVAL;
7661 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type);
7662 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY;
7665 if (sym->st_shndx != obj->efile.btf_maps_shndx) {
7666 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
7668 return -LIBBPF_ERRNO__RELOC;
7670 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
7671 map->def.key_size != sizeof(int)) {
7672 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
7673 i, map->name, sizeof(int));
7674 return -EINVAL;
7680 return -ESRCH;
7687 return -ESRCH;
7689 if (targ_prog->sec_idx != sym->st_shndx ||
7690 targ_prog->sec_insn_off * 8 != sym->st_value ||
7692 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n",
7694 return -LIBBPF_ERRNO__RELOC;
7697 return -EINVAL;
7700 var = btf__type_by_id(obj->btf, vi->type);
7701 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
7703 return -EINVAL;
7704 member = btf_members(def) + btf_vlen(def) - 1;
7705 mname = btf__name_by_offset(obj->btf, member->name_off);
7707 return -EINVAL;
7709 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
7710 if (rel->r_offset - vi->offset < moff)
7711 return -EINVAL;
7713 moff = rel->r_offset - vi->offset - moff;
7718 return -EINVAL;
7720 if (moff >= map->init_slots_sz) {
7722 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
7724 return -ENOMEM;
7725 map->init_slots = tmp;
7726 memset(map->init_slots + map->init_slots_sz, 0,
7727 (new_sz - map->init_slots_sz) * host_ptr_sz);
7728 map->init_slots_sz = new_sz;
7730 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog;
7733 i, map->name, moff, type, name);
7743 for (i = 0; i < obj->efile.sec_cnt; i++) {
7744 struct elf_sec_desc *sec_desc = &obj->efile.secs[i];
7749 if (sec_desc->sec_type != SEC_RELO)
7752 shdr = sec_desc->shdr;
7753 data = sec_desc->data;
7754 idx = shdr->sh_info;
7756 if (shdr->sh_type != SHT_REL || idx < 0 || idx >= obj->efile.sec_cnt) {
7758 return -LIBBPF_ERRNO__INTERNAL;
7761 if (obj->efile.secs[idx].sec_type == SEC_ST_OPS)
7763 else if (idx == obj->efile.btf_maps_shndx)
7777 if (BPF_CLASS(insn->code) == BPF_JMP &&
7778 BPF_OP(insn->code) == BPF_CALL &&
7779 BPF_SRC(insn->code) == BPF_K &&
7780 insn->src_reg == 0 &&
7781 insn->dst_reg == 0) {
7782 *func_id = insn->imm;
7790 struct bpf_insn *insn = prog->insns;
7794 if (obj->gen_loader)
7797 for (i = 0; i < prog->insns_cnt; i++, insn++) {
7809 insn->imm = BPF_FUNC_probe_read;
7814 insn->imm = BPF_FUNC_probe_read_str;
7839 * - function_pattern or
7840 * - <module>:function_pattern
7845 mod_len = sep - mod_name;
7854 for (i = 0; i < obj->btf_module_cnt; i++) {
7855 const struct module_btf *mod = &obj->btf_modules[i];
7857 if (strncmp(mod->name, mod_name, mod_len) == 0 && mod->name[mod_len] == '\0')
7869 attach_name = strchr(prog->sec_name, '/');
7881 mod = find_attach_module(prog->obj, attach_name);
7883 return -EINVAL;
7885 *btf_obj_fd = mod->fd;
7889 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
7896 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
7897 opts->expected_attach_type = 0;
7900 opts->prog_flags |= BPF_F_SLEEPABLE;
7902 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
7903 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
7906 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) {
7911 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7912 opts->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7915 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
7919 attach_name = strchr(prog->sec_name, '/');
7930 pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n",
7931 prog->name);
7932 return -EINVAL;
7941 prog->attach_btf_obj_fd = btf_obj_fd;
7942 prog->attach_btf_id = btf_type_id;
7945 * prog->attach_btf_obj_fd/prog->attach_btf_id anymore because
7949 opts->attach_btf_obj_fd = btf_obj_fd;
7950 opts->attach_btf_id = btf_type_id;
7953 if (is_tracing_multi(prog->expected_attach_type)) {
7960 prog->attach_btf_obj_fd = btf_obj_fd;
7961 opts->attach_btf_obj_fd = btf_obj_fd;
7978 __u32 log_level = prog->log_level;
7984 switch (prog->type) {
7991 prog->name, prog->sec_name);
7992 return -EINVAL;
7994 if (prog->attach_btf_id == 0) {
7996 prog->name);
7997 return -EINVAL;
8005 return -EINVAL;
8008 prog_name = prog->name;
8009 load_attr.attach_prog_fd = prog->attach_prog_fd;
8010 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
8011 load_attr.attach_btf_id = prog->attach_btf_id;
8013 load_attr.prog_ifindex = prog->prog_ifindex;
8014 load_attr.expected_attach_type = prog->expected_attach_type;
8017 if (obj->btf && btf__fd(obj->btf) >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {
8018 load_attr.prog_btf_fd = btf__fd(obj->btf);
8019 load_attr.func_info = prog->func_info;
8020 load_attr.func_info_rec_size = prog->func_info_rec_size;
8021 load_attr.func_info_cnt = prog->func_info_cnt;
8022 load_attr.line_info = prog->line_info;
8023 load_attr.line_info_rec_size = prog->line_info_rec_size;
8024 load_attr.line_info_cnt = prog->line_info_cnt;
8027 load_attr.prog_flags = prog->prog_flags;
8028 load_attr.fd_array = obj->fd_array;
8030 load_attr.token_fd = obj->token_fd;
8031 if (obj->token_fd)
8035 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
8036 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
8039 prog->name, errstr(err));
8042 insns = prog->insns;
8043 insns_cnt = prog->insns_cnt;
8046 if (obj->gen_loader) {
8047 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
8049 prog - obj->programs);
8050 *prog_fd = -1;
8061 if (prog->log_buf) {
8062 log_buf = prog->log_buf;
8063 log_buf_size = prog->log_size;
8065 } else if (obj->log_buf) {
8066 log_buf = obj->log_buf;
8067 log_buf_size = obj->log_size;
8073 ret = -ENOMEM;
8086 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);
8089 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
8090 prog->name, log_buf);
8093 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) {
8097 for (i = 0; i < obj->nr_maps; i++) {
8098 map = &prog->obj->maps[i];
8099 if (map->libbpf_type != LIBBPF_MAP_RODATA)
8102 if (bpf_prog_bind_map(ret, map->fd, NULL)) {
8104 prog->name, map->real_name, errstr(errno));
8124 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2).
8129 ret = -errno;
8131 /* post-process verifier log to improve error descriptions */
8134 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, errstr(errno));
8138 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
8139 prog->name, log_buf);
8155 p = cur - 1;
8156 while (p - 1 >= buf && *(p - 1) != '\n')
8157 p--;
8165 /* size of the remaining log content to the right from the to-be-replaced part */
8166 size_t rem_sz = (buf + log_sz) - (orig + orig_sz);
8171 * shift log contents by (patch_sz - orig_sz) bytes to the right
8172 * starting from after to-be-replaced part of the log.
8175 * shift log contents by (orig_sz - patch_sz) bytes to the left
8176 * starting from after to-be-replaced part of the log
8185 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1;
8187 } else if (patch_sz - orig_sz > buf_sz - log_sz) {
8189 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz);
8203 /* Expected log for failed and not properly guarded CO-RE relocation:
8204 * line1 -> 123: (85) call unknown#195896080
8205 * line2 -> invalid func unknown#195896080
8206 * line3 -> <anything else or end of buffer>
8209 * instruction index to find corresponding CO-RE relocation and
8211 * failed CO-RE relocation.
8225 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
8231 "%d: <invalid CO-RE relocation>\n"
8232 "failed to resolve CO-RE relocation %s%s\n",
8235 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
8243 * line1 -> 123: (85) call unknown#2001000345
8244 * line2 -> invalid func unknown#2001000345
8245 * line3 -> <anything else or end of buffer>
8248 * "345" in "2001000345" is a map index in obj->maps to fetch map name.
8250 struct bpf_object *obj = prog->obj;
8258 map_idx -= POISON_LDIMM64_MAP_BASE;
8259 if (map_idx < 0 || map_idx >= obj->nr_maps)
8261 map = &obj->maps[map_idx];
8266 insn_idx, map->name);
8268 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
8276 * line1 -> 123: (85) call unknown#2002000345
8277 * line2 -> invalid func unknown#2002000345
8278 * line3 -> <anything else or end of buffer>
8281 * "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name.
8283 struct bpf_object *obj = prog->obj;
8291 ext_idx -= POISON_CALL_KFUNC_BASE;
8292 if (ext_idx < 0 || ext_idx >= obj->nr_extern)
8294 ext = &obj->externs[ext_idx];
8299 insn_idx, ext->name);
8301 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
8316 next_line = buf + log_sz - 1;
8328 /* failed CO-RE relocation case */
8356 struct bpf_object *obj = prog->obj;
8359 for (i = 0; i < prog->nr_reloc; i++) {
8360 struct reloc_desc *relo = &prog->reloc_desc[i];
8361 struct extern_desc *ext = &obj->externs[relo->ext_idx];
8364 switch (relo->type) {
8366 if (ext->type != EXT_KSYM)
8368 kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ?
8370 bpf_gen__record_extern(obj->gen_loader, ext->name,
8371 ext->is_weak, !ext->ksym.type_id,
8372 true, kind, relo->insn_idx);
8375 bpf_gen__record_extern(obj->gen_loader, ext->name,
8376 ext->is_weak, false, false, BTF_KIND_FUNC,
8377 relo->insn_idx);
8381 .insn_off = relo->insn_idx * 8,
8382 .type_id = relo->core_relo->type_id,
8383 .access_str_off = relo->core_relo->access_str_off,
8384 .kind = relo->core_relo->kind,
8387 bpf_gen__record_relo_core(obj->gen_loader, &cr);
8404 for (i = 0; i < obj->nr_programs; i++) {
8405 prog = &obj->programs[i];
8408 if (!prog->autoload) {
8409 pr_debug("prog '%s': skipped loading\n", prog->name);
8412 prog->log_level |= log_level;
8414 if (obj->gen_loader)
8417 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
8418 obj->license, obj->kern_version, &prog->fd);
8420 pr_warn("prog '%s': failed to load: %s\n", prog->name, errstr(err));
8435 for (i = 0; i < obj->nr_programs; i++) {
8436 prog = &obj->programs[i];
8452 prog->sec_def = find_sec_def(prog->sec_name);
8453 if (!prog->sec_def) {
8456 prog->name, prog->sec_name);
8460 prog->type = prog->sec_def->prog_type;
8461 prog->expected_attach_type = prog->sec_def->expected_attach_type;
8466 if (prog->sec_def->prog_setup_fn) {
8467 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
8470 prog->name, errstr(err));
8491 return ERR_PTR(-EINVAL);
8496 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
8500 return ERR_PTR(-EINVAL);
8514 return ERR_PTR(-EINVAL);
8516 return ERR_PTR(-EINVAL);
8526 return ERR_PTR(-ENAMETOOLONG);
8532 obj->log_buf = log_buf;
8533 obj->log_size = log_size;
8534 obj->log_level = log_level;
8537 obj->token_path = strdup(token_path);
8538 if (!obj->token_path) {
8539 err = -ENOMEM;
8547 err = -ENAMETOOLONG;
8550 obj->btf_custom_path = strdup(btf_tmp_path);
8551 if (!obj->btf_custom_path) {
8552 err = -ENOMEM;
8559 obj->kconfig = strdup(kconfig);
8560 if (!obj->kconfig) {
8561 err = -ENOMEM;
8588 return libbpf_err_ptr(-EINVAL);
8605 return libbpf_err_ptr(-EINVAL);
8608 snprintf(tmp_name, sizeof(tmp_name), "%lx-%zx", (unsigned long)obj_buf, obj_buf_sz);
8618 return libbpf_err(-EINVAL);
8620 for (i = 0; i < obj->nr_maps; i++) {
8621 zclose(obj->maps[i].fd);
8622 if (obj->maps[i].st_ops)
8623 zfree(&obj->maps[i].st_ops->kern_vdata);
8626 for (i = 0; i < obj->nr_programs; i++)
8627 bpf_program__unload(&obj->programs[i]);
8640 m->def.map_flags &= ~BPF_F_MMAPABLE;
8658 err = -errno;
8670 err = -EINVAL;
8693 ext = find_extern_by_name_with_len(obj, sym_name, res - sym_name);
8696 if (!ext || ext->type != EXT_KSYM)
8699 t = btf__type_by_id(obj->btf, ext->btf_id);
8703 if (ext->is_set && ext->ksym.addr != sym_addr) {
8705 sym_name, ext->ksym.addr, sym_addr);
8706 return -EINVAL;
8708 if (!ext->is_set) {
8709 ext->is_set = true;
8710 ext->ksym.addr = sym_addr;
8729 btf = obj->btf_vmlinux;
8733 if (id == -ENOENT) {
8738 for (i = 0; i < obj->btf_module_cnt; i++) {
8740 mod_btf = &obj->btf_modules[i];
8741 btf = mod_btf->btf;
8743 if (id != -ENOENT)
8748 return -ESRCH;
8765 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);
8767 if (id == -ESRCH && ext->is_weak)
8770 ext->name);
8775 local_type_id = ext->ksym.type_id;
8779 targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
8780 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
8782 err = bpf_core_types_are_compat(obj->btf, local_type_id,
8788 local_type = btf__type_by_id(obj->btf, local_type_id);
8789 local_name = btf__name_by_offset(obj->btf, local_type->name_off);
8790 targ_name = btf__name_by_offset(btf, targ_type->name_off);
8793 ext->name, local_type_id,
8796 return -EINVAL;
8799 ext->is_set = true;
8800 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
8801 ext->ksym.kernel_btf_id = id;
8803 ext->name, id, btf_kind_str(targ_var), targ_var_name);
8817 local_func_proto_id = ext->ksym.type_id;
8819 kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf,
8822 if (kfunc_id == -ESRCH && ext->is_weak)
8825 ext->name);
8830 kfunc_proto_id = kern_func->type;
8832 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
8835 if (ext->is_weak)
8839 ext->name, local_func_proto_id,
8840 mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id);
8841 return -EINVAL;
8845 if (mod_btf && !mod_btf->fd_array_idx) {
8846 /* insn->off is s16 */
8847 if (obj->fd_array_cnt == INT16_MAX) {
8849 ext->name, mod_btf->fd_array_idx);
8850 return -E2BIG;
8853 if (!obj->fd_array_cnt)
8854 obj->fd_array_cnt = 1;
8856 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),
8857 obj->fd_array_cnt + 1);
8860 mod_btf->fd_array_idx = obj->fd_array_cnt;
8862 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;
8865 ext->is_set = true;
8866 ext->ksym.kernel_btf_id = kfunc_id;
8867 ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0;
8870 * {kernel_btf_id, btf_fd_idx} -> fixup bpf_call.
8871 * {kernel_btf_id, kernel_btf_obj_fd} -> fixup ld_imm64.
8873 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
8875 ext->name, mod_btf ? mod_btf->name : "vmlinux", kfunc_id);
8886 for (i = 0; i < obj->nr_extern; i++) {
8887 ext = &obj->externs[i];
8888 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
8891 if (obj->gen_loader) {
8892 ext->is_set = true;
8893 ext->ksym.kernel_btf_obj_fd = 0;
8894 ext->ksym.kernel_btf_id = 0;
8897 t = btf__type_by_id(obj->btf, ext->btf_id);
8917 if (obj->nr_extern == 0)
8920 if (obj->kconfig_map_idx >= 0)
8921 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
8923 for (i = 0; i < obj->nr_extern; i++) {
8924 ext = &obj->externs[i];
8926 if (ext->type == EXT_KSYM) {
8927 if (ext->ksym.type_id)
8932 } else if (ext->type == EXT_KCFG) {
8933 void *ext_ptr = kcfg_data + ext->kcfg.data_off;
8937 if (str_has_pfx(ext->name, "CONFIG_")) {
8943 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
8946 pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name);
8947 return -EINVAL;
8949 } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) {
8951 } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) {
8953 } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) {
8961 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name);
8962 return -EINVAL;
8969 ext->name, (unsigned long long)value);
8971 pr_warn("extern '%s': unrecognized extern kind\n", ext->name);
8972 return -EINVAL;
8978 return -EINVAL;
8980 for (i = 0; i < obj->nr_extern; i++) {
8981 ext = &obj->externs[i];
8982 if (ext->type == EXT_KCFG && !ext->is_set) {
8991 return -EINVAL;
8996 return -EINVAL;
9001 return -EINVAL;
9003 for (i = 0; i < obj->nr_extern; i++) {
9004 ext = &obj->externs[i];
9006 if (!ext->is_set && !ext->is_weak) {
9007 pr_warn("extern '%s' (strong): not resolved\n", ext->name);
9008 return -ESRCH;
9009 } else if (!ext->is_set) {
9011 ext->name);
9024 st_ops = map->st_ops;
9025 type = btf__type_by_id(map->obj->btf, st_ops->type_id);
9027 struct bpf_program *prog = st_ops->progs[i];
9035 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
9045 for (i = 0; i < obj->nr_maps; i++) {
9046 map = &obj->maps[i];
9051 if (!map->autocreate)
9064 /* unpin any maps that were auto-pinned during load */
9065 for (i = 0; i < obj->nr_maps; i++)
9066 if (obj->maps[i].pinned && !obj->maps[i].reused)
9067 bpf_map__unpin(&obj->maps[i], NULL);
9075 for (i = 0; i < obj->btf_module_cnt; i++) {
9076 close(obj->btf_modules[i].fd);
9077 btf__free(obj->btf_modules[i].btf);
9078 free(obj->btf_modules[i].name);
9080 obj->btf_module_cnt = 0;
9081 obj->btf_module_cap = 0;
9082 obj->btf_modules_loaded = false;
9083 zfree(&obj->btf_modules);
9086 btf__free(obj->btf_vmlinux);
9087 obj->btf_vmlinux = NULL;
9093 zfree(&obj->fd_array);
9103 if (obj->state >= OBJ_PREPARED) {
9104 pr_warn("object '%s': prepare loading can't be attempted twice\n", obj->name);
9105 return -EINVAL;
9111 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
9115 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
9123 obj->state = OBJ_LOADED;
9127 obj->state = OBJ_PREPARED;
9136 return libbpf_err(-EINVAL);
9138 if (obj->state >= OBJ_LOADED) {
9139 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
9140 return libbpf_err(-EINVAL);
9143 /* Disallow kernel loading programs of non-native endianness but
9144 * permit cross-endian creation of "light skeleton".
9146 if (obj->gen_loader) {
9147 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);
9149 pr_warn("object '%s': loading non-native endianness is unsupported\n", obj->name);
9150 return libbpf_err(-LIBBPF_ERRNO__ENDIAN);
9153 if (obj->state < OBJ_PREPARED) {
9162 if (obj->gen_loader) {
9164 if (obj->btf)
9165 btf__set_fd(obj->btf, -1);
9167 err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
9171 obj->state = OBJ_LOADED; /* doesn't matter if successfully or not */
9176 pr_warn("failed to load object '%s'\n", obj->path);
9200 return -ENOMEM;
9204 err = -errno;
9220 return -EINVAL;
9224 return -ENOMEM;
9229 err = -errno;
9235 err = -EINVAL;
9245 if (prog->fd < 0) {
9246 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
9247 return libbpf_err(-EINVAL);
9258 if (bpf_obj_pin(prog->fd, path)) {
9259 err = -errno;
9260 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, errstr(err));
9264 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
9272 if (prog->fd < 0) {
9273 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
9274 return libbpf_err(-EINVAL);
9283 return libbpf_err(-errno);
9285 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
9295 return libbpf_err(-EINVAL);
9298 if (map->fd < 0) {
9299 pr_warn("map '%s': can't pin BPF map without FD (was it created?)\n", map->name);
9300 return libbpf_err(-EINVAL);
9303 if (map->pin_path) {
9304 if (path && strcmp(path, map->pin_path)) {
9306 bpf_map__name(map), map->pin_path, path);
9307 return libbpf_err(-EINVAL);
9308 } else if (map->pinned) {
9309 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
9310 bpf_map__name(map), map->pin_path);
9317 return libbpf_err(-EINVAL);
9318 } else if (map->pinned) {
9320 return libbpf_err(-EEXIST);
9323 map->pin_path = strdup(path);
9324 if (!map->pin_path) {
9325 err = -errno;
9330 err = make_parent_dir(map->pin_path);
9334 err = check_path(map->pin_path);
9338 if (bpf_obj_pin(map->fd, map->pin_path)) {
9339 err = -errno;
9343 map->pinned = true;
9344 pr_debug("pinned map '%s'\n", map->pin_path);
9359 return libbpf_err(-EINVAL);
9362 if (map->pin_path) {
9363 if (path && strcmp(path, map->pin_path)) {
9365 bpf_map__name(map), map->pin_path, path);
9366 return libbpf_err(-EINVAL);
9368 path = map->pin_path;
9372 return libbpf_err(-EINVAL);
9381 return libbpf_err(-errno);
9383 map->pinned = false;
9396 return libbpf_err(-errno);
9399 free(map->pin_path);
9400 map->pin_path = new;
9409 return map->pin_path;
9414 return map->pinned;
9433 return libbpf_err(-ENOENT);
9435 if (obj->state < OBJ_PREPARED) {
9437 return libbpf_err(-ENOENT);
9444 if (!map->autocreate)
9453 } else if (!map->pin_path) {
9466 if (!map->pin_path)
9481 return libbpf_err(-ENOENT);
9493 } else if (!map->pin_path) {
9512 return libbpf_err(-ENOENT);
9514 if (obj->state < OBJ_LOADED) {
9516 return libbpf_err(-ENOENT);
9520 err = pathname_concat(buf, sizeof(buf), path, prog->name);
9533 if (pathname_concat(buf, sizeof(buf), path, prog->name))
9548 return libbpf_err(-ENOENT);
9553 err = pathname_concat(buf, sizeof(buf), path, prog->name);
9599 if (map->inner_map) {
9600 bpf_map__destroy(map->inner_map);
9601 zfree(&map->inner_map);
9604 zfree(&map->init_slots);
9605 map->init_slots_sz = 0;
9607 if (map->mmaped && map->mmaped != map->obj->arena_data)
9608 munmap(map->mmaped, bpf_map_mmap_sz(map));
9609 map->mmaped = NULL;
9611 if (map->st_ops) {
9612 zfree(&map->st_ops->data);
9613 zfree(&map->st_ops->progs);
9614 zfree(&map->st_ops->kern_func_off);
9615 zfree(&map->st_ops);
9618 zfree(&map->name);
9619 zfree(&map->real_name);
9620 zfree(&map->pin_path);
9622 if (map->fd >= 0)
9623 zclose(map->fd);
9640 usdt_manager_free(obj->usdt_man);
9641 obj->usdt_man = NULL;
9643 bpf_gen__free(obj->gen_loader);
9646 btf__free(obj->btf);
9647 btf__free(obj->btf_vmlinux);
9648 btf_ext__free(obj->btf_ext);
9650 for (i = 0; i < obj->nr_maps; i++)
9651 bpf_map__destroy(&obj->maps[i]);
9653 zfree(&obj->btf_custom_path);
9654 zfree(&obj->kconfig);
9656 for (i = 0; i < obj->nr_extern; i++) {
9657 zfree(&obj->externs[i].name);
9658 zfree(&obj->externs[i].essent_name);
9661 zfree(&obj->externs);
9662 obj->nr_extern = 0;
9664 zfree(&obj->maps);
9665 obj->nr_maps = 0;
9667 if (obj->programs && obj->nr_programs) {
9668 for (i = 0; i < obj->nr_programs; i++)
9669 bpf_program__exit(&obj->programs[i]);
9671 zfree(&obj->programs);
9673 zfree(&obj->feat_cache);
9674 zfree(&obj->token_path);
9675 if (obj->token_fd > 0)
9676 close(obj->token_fd);
9678 zfree(&obj->arena_data);
9680 zfree(&obj->jumptables_data);
9681 obj->jumptables_data_sz = 0;
9683 for (i = 0; i < obj->jumptable_map_cnt; i++)
9684 close(obj->jumptable_maps[i].fd);
9685 zfree(&obj->jumptable_maps);
9692 return obj ? obj->name : libbpf_err_ptr(-EINVAL);
9697 return obj ? obj->kern_version : 0;
9702 return obj->token_fd ?: -1;
9707 return obj ? obj->btf : NULL;
9712 return obj->btf ? btf__fd(obj->btf) : -1;
9717 if (obj->state >= OBJ_LOADED)
9718 return libbpf_err(-EINVAL);
9720 obj->kern_version = kern_version;
9730 return libbpf_err(-EFAULT);
9732 return libbpf_err(-EINVAL);
9735 return libbpf_err(-ENOMEM);
9736 gen->opts = opts;
9737 gen->swapped_endian = !is_native_endianness(obj);
9738 obj->gen_loader = gen;
9746 size_t nr_programs = obj->nr_programs;
9754 return forward ? &obj->programs[0] :
9755 &obj->programs[nr_programs - 1];
9757 if (p->obj != obj) {
9762 idx = (p - obj->programs) + (forward ? 1 : -1);
9763 if (idx >= obj->nr_programs || idx < 0)
9765 return &obj->programs[idx];
9794 prog->prog_ifindex = ifindex;
9799 return prog->name;
9804 return prog->sec_name;
9809 return prog->autoload;
9814 if (prog->obj->state >= OBJ_LOADED)
9815 return libbpf_err(-EINVAL);
9817 prog->autoload = autoload;
9823 return prog->autoattach;
9828 prog->autoattach = autoattach;
9833 return prog->insns;
9838 return prog->insns_cnt;
9846 if (prog->obj->state >= OBJ_LOADED)
9847 return libbpf_err(-EBUSY);
9849 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
9852 pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
9853 return libbpf_err(-ENOMEM);
9857 prog->insns = insns;
9858 prog->insns_cnt = new_insn_cnt;
9865 return libbpf_err(-EINVAL);
9867 if (prog->fd < 0)
9868 return libbpf_err(-ENOENT);
9870 return prog->fd;
9878 return prog->type;
9889 if (prog->obj->state >= OBJ_LOADED)
9890 return libbpf_err(-EBUSY);
9893 if (prog->type == type)
9896 prog->type = type;
9900 * fallback handler, which by definition is program type-agnostic and
9901 * is a catch-all custom handler, optionally set by the application,
9904 if (prog->sec_def != &custom_fallback_def)
9905 prog->sec_def = NULL;
9914 return prog->expected_attach_type;
9920 if (prog->obj->state >= OBJ_LOADED)
9921 return libbpf_err(-EBUSY);
9923 prog->expected_attach_type = type;
9929 return prog->prog_flags;
9934 if (prog->obj->state >= OBJ_LOADED)
9935 return libbpf_err(-EBUSY);
9937 prog->prog_flags = flags;
9943 return prog->log_level;
9948 if (prog->obj->state >= OBJ_LOADED)
9949 return libbpf_err(-EBUSY);
9951 prog->log_level = log_level;
9957 *log_size = prog->log_size;
9958 return prog->log_buf;
9964 return libbpf_err(-EINVAL);
9965 if (prog->log_size > UINT_MAX)
9966 return libbpf_err(-EINVAL);
9967 if (prog->obj->state >= OBJ_LOADED)
9968 return libbpf_err(-EBUSY);
9970 prog->log_buf = log_buf;
9971 prog->log_size = log_size;
9977 if (prog->func_info_rec_size != sizeof(struct bpf_func_info))
9978 return libbpf_err_ptr(-EOPNOTSUPP);
9979 return prog->func_info;
9984 return prog->func_info_cnt;
9989 if (prog->line_info_rec_size != sizeof(struct bpf_line_info))
9990 return libbpf_err_ptr(-EOPNOTSUPP);
9991 return prog->line_info;
9996 return prog->line_info_cnt;
10008 return libbpf_err(-EINVAL);
10011 return libbpf_err(-EINVAL);
10013 obj = prog->obj;
10014 if (obj->state < OBJ_PREPARED)
10015 return libbpf_err(-EINVAL);
10018 * Caller-provided opts take priority; fall back to
10021 attr.attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0) ?: prog->attach_prog_fd;
10022 attr.prog_flags = OPTS_GET(opts, prog_flags, 0) ?: prog->prog_flags;
10023 attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0) ?: prog->prog_ifindex;
10024 attr.kern_version = OPTS_GET(opts, kern_version, 0) ?: obj->kern_version;
10025 attr.fd_array = OPTS_GET(opts, fd_array, NULL) ?: obj->fd_array;
10026 attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0) ?: obj->fd_array_cnt;
10027 attr.token_fd = OPTS_GET(opts, token_fd, 0) ?: obj->token_fd;
10032 if (!prog_btf_fd && obj->btf)
10033 prog_btf_fd = btf__fd(obj->btf);
10039 /* func_info/line_info triples: all-or-nothing from caller */
10045 prog->name);
10046 return libbpf_err(-EINVAL);
10048 attr.func_info = info ?: prog->func_info;
10049 attr.func_info_cnt = info ? info_cnt : prog->func_info_cnt;
10050 attr.func_info_rec_size = info ? info_rec_size : prog->func_info_rec_size;
10057 prog->name);
10058 return libbpf_err(-EINVAL);
10060 attr.line_info = info ?: prog->line_info;
10061 attr.line_info_cnt = info ? info_cnt : prog->line_info_cnt;
10062 attr.line_info_rec_size = info ? info_rec_size : prog->line_info_rec_size;
10065 /* Logging is caller-controlled; no fallback to prog/obj log settings */
10073 * Later override with caller-provided opts.
10075 attr.expected_attach_type = prog->expected_attach_type;
10076 attr.attach_btf_id = prog->attach_btf_id;
10077 attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
10079 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
10080 err = prog->sec_def->prog_prepare_load_fn(prog, &attr, prog->sec_def->cookie);
10085 /* Re-apply caller overrides for output fields */
10098 fd = bpf_prog_load(prog->type, prog->name, obj->license, prog->insns, prog->insns_cnt,
10254 return libbpf_err(-EINVAL);
10257 return libbpf_err(-E2BIG);
10263 return libbpf_err(-ENOMEM);
10269 return libbpf_err(-EBUSY);
10274 sec_def->sec = sec ? strdup(sec) : NULL;
10275 if (sec && !sec_def->sec)
10276 return libbpf_err(-ENOMEM);
10278 sec_def->prog_type = prog_type;
10279 sec_def->expected_attach_type = exp_attach_type;
10280 sec_def->cookie = OPTS_GET(opts, cookie, 0);
10282 sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL);
10283 sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL);
10284 sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL);
10286 sec_def->handler_id = ++last_custom_sec_def_handler_id;
10293 return sec_def->handler_id;
10302 return libbpf_err(-EINVAL);
10316 return libbpf_err(-ENOENT);
10320 custom_sec_defs[i - 1] = custom_sec_defs[i];
10321 custom_sec_def_cnt--;
10337 size_t len = strlen(sec_def->sec);
10340 if (sec_def->sec[len - 1] == '/') {
10341 if (str_has_pfx(sec_name, sec_def->sec))
10347 * well-formed SEC("type/extras") with proper '/' separator
10349 if (sec_def->sec[len - 1] == '+') {
10350 len--;
10352 if (strncmp(sec_name, sec_def->sec, len) != 0)
10360 return strcmp(sec_name, sec_def->sec) == 0;
10405 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
10408 if (!(sec_def->cookie & SEC_ATTACHABLE))
10430 return libbpf_err(-EINVAL);
10434 *prog_type = sec_def->prog_type;
10435 *expected_attach_type = sec_def->expected_attach_type;
10446 return libbpf_err(-ESRCH);
10488 for (i = 0; i < obj->nr_maps; i++) {
10489 map = &obj->maps[i];
10492 if (map->sec_idx == sec_idx &&
10493 map->sec_offset <= offset &&
10494 offset - map->sec_offset < map->def.value_size)
10501 /* Collect the reloc from ELF, populate the st_ops->progs[], and update
10502 * st_ops->data for shadow type.
10521 btf = obj->btf;
10522 nrels = shdr->sh_size / shdr->sh_entsize;
10527 return -LIBBPF_ERRNO__FORMAT;
10530 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
10533 (size_t)ELF64_R_SYM(rel->r_info));
10534 return -LIBBPF_ERRNO__FORMAT;
10537 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
10538 map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset);
10540 pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n",
10541 (size_t)rel->r_offset);
10542 return -EINVAL;
10545 moff = rel->r_offset - map->sec_offset;
10546 shdr_idx = sym->st_shndx;
10547 st_ops = map->st_ops;
10548 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %u (\'%s\')\n",
10549 map->name,
10550 (long long)(rel->r_info >> 32),
10551 (long long)sym->st_value,
10552 shdr_idx, (size_t)rel->r_offset,
10553 map->sec_offset, sym->st_name, name);
10556 pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n",
10557 map->name, (size_t)rel->r_offset, shdr_idx);
10558 return -LIBBPF_ERRNO__RELOC;
10560 if (sym->st_value % BPF_INSN_SZ) {
10562 map->name, (unsigned long long)sym->st_value);
10563 return -LIBBPF_ERRNO__FORMAT;
10565 insn_idx = sym->st_value / BPF_INSN_SZ;
10567 type = btf__type_by_id(btf, st_ops->type_id);
10571 map->name, moff);
10572 return -EINVAL;
10574 member_idx = member - btf_members(type);
10575 name = btf__name_by_offset(btf, member->name_off);
10577 if (!resolve_func_ptr(btf, member->type, NULL)) {
10579 map->name, name);
10580 return -EINVAL;
10586 map->name, shdr_idx, name);
10587 return -EINVAL;
10591 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
10593 map->name, prog->name);
10594 return -EINVAL;
10597 st_ops->progs[member_idx] = prog;
10599 /* st_ops->data will be exposed to users, being returned by
10605 *((struct bpf_program **)(st_ops->data + moff)) = prog;
10652 return -ENAMETOOLONG;
10702 err = -EINVAL;
10733 mod_len = fn_name - mod_name;
10738 ret = find_attach_btf_id(obj->btf_vmlinux,
10746 if (ret != -ENOENT)
10754 for (i = 0; i < obj->btf_module_cnt; i++) {
10755 const struct module_btf *mod = &obj->btf_modules[i];
10757 if (mod_name && strncmp(mod->name, mod_name, mod_len) != 0)
10760 ret = find_attach_btf_id(mod->btf,
10764 *btf_obj_fd = mod->fd;
10768 if (ret == -ENOENT)
10774 return -ESRCH;
10780 enum bpf_attach_type attach_type = prog->expected_attach_type;
10781 __u32 attach_prog_fd = prog->attach_prog_fd;
10785 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {
10787 pr_warn("prog '%s': attach program FD is not set\n", prog->name);
10788 return -EINVAL;
10790 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd, prog->obj->token_fd);
10793 prog->name, attach_prog_fd, attach_name, errstr(err));
10802 if (prog->obj->gen_loader) {
10803 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
10807 err = find_kernel_btf_id(prog->obj, attach_name,
10813 prog->name, attach_name, errstr(err));
10826 return libbpf_err(-EINVAL);
10837 return libbpf_err(-EINVAL);
10840 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
10841 return libbpf_err(-EINVAL);
10842 if (!(sec_def->cookie & SEC_ATTACHABLE))
10843 return libbpf_err(-EINVAL);
10845 *attach_type = sec_def->expected_attach_type;
10852 return libbpf_err(-EINVAL);
10854 return -1;
10855 return map->fd;
10861 * their user-visible name differs from kernel-visible name. Users see
10866 * BPF_OBJ_NAME_LEN-1 chars in internal_map_name(). Hence, percpu data
10867 * maps must use real name for their user-visible name.
10869 if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0)
10871 if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0)
10873 if (map->libbpf_type == LIBBPF_MAP_PERCPU)
10884 return map->real_name;
10886 return map->name;
10891 return map->def.type;
10897 return libbpf_err(-EBUSY);
10898 map->def.type = type;
10904 return map->def.map_flags;
10910 return libbpf_err(-EBUSY);
10911 map->def.map_flags = flags;
10917 return map->map_extra;
10923 return libbpf_err(-EBUSY);
10924 map->map_extra = map_extra;
10930 return map->numa_node;
10936 return libbpf_err(-EBUSY);
10937 map->numa_node = numa_node;
10943 return map->def.key_size;
10949 return libbpf_err(-EBUSY);
10950 map->def.key_size = size;
10956 return map->def.value_size;
10970 btf = bpf_object__btf(map->obj);
10972 return -ENOENT;
10979 return -EINVAL;
10987 return -EINVAL;
10991 var = &btf_var_secinfos(datasec_type)[vlen - 1];
10992 var_type = btf_type_by_id(btf, var->type);
10993 array_type = skip_mods_and_typedefs(btf, var_type->type, NULL);
10997 return -EINVAL;
11002 element_sz = btf__resolve_size(btf, array->type);
11003 if (element_sz <= 0 || (size - var->offset) % element_sz != 0) {
11006 return -EINVAL;
11010 nr_elements = (size - var->offset) / element_sz;
11011 new_array_id = btf__add_array(btf, array->index_type, array->type, nr_elements);
11018 datasec_type = btf_type_by_id(btf, map->btf_value_type_id);
11019 var = &btf_var_secinfos(datasec_type)[vlen - 1];
11020 var_type = btf_type_by_id(btf, var->type);
11023 datasec_type->size = size;
11024 var->size = size - var->offset;
11025 var_type->type = new_array_id;
11033 return libbpf_err(-EBUSY);
11035 if (map->mmaped) {
11039 if (map->def.type != BPF_MAP_TYPE_ARRAY &&
11040 map->def.type != BPF_MAP_TYPE_PERCPU_ARRAY)
11041 return libbpf_err(-EOPNOTSUPP);
11044 mmap_new_sz = array_map_mmap_sz(size, map->def.max_entries);
11047 pr_warn("map '%s': failed to resize memory-mapped region: %s\n",
11052 if (err && err != -ENOENT) {
11055 map->btf_value_type_id = 0;
11056 map->btf_key_type_id = 0;
11060 map->def.value_size = size;
11066 return map ? map->btf_key_type_id : 0;
11071 return map ? map->btf_value_type_id : 0;
11080 return libbpf_err(-EBUSY);
11082 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG)
11083 return libbpf_err(-EINVAL);
11085 if (map->def.type == BPF_MAP_TYPE_ARENA)
11086 actual_sz = map->obj->arena_data_sz;
11088 actual_sz = map->def.value_size;
11090 return libbpf_err(-EINVAL);
11092 memcpy(map->mmaped, data, size);
11100 *psize = map->def.value_size;
11101 return map->st_ops->data;
11104 if (!map->mmaped)
11107 if (map->def.type == BPF_MAP_TYPE_ARENA)
11108 *psize = map->obj->arena_data_sz;
11110 *psize = map->def.value_size;
11112 return map->mmaped;
11117 return map->libbpf_type != LIBBPF_MAP_UNSPEC;
11122 return map->map_ifindex;
11128 return libbpf_err(-EBUSY);
11129 map->map_ifindex = ifindex;
11135 if (!bpf_map_type__is_map_in_map(map->def.type)) {
11137 return libbpf_err(-EINVAL);
11139 if (map->inner_map_fd != -1) {
11141 return libbpf_err(-EINVAL);
11143 if (map->inner_map) {
11144 bpf_map__destroy(map->inner_map);
11145 zfree(&map->inner_map);
11147 map->inner_map_fd = fd;
11155 return libbpf_err(-EINVAL);
11158 if (map->obj != prog->obj) {
11160 return libbpf_err(-EINVAL);
11163 map->excl_prog = prog;
11169 return map->excl_prog;
11178 if (!obj || !obj->maps)
11181 s = obj->maps;
11182 e = obj->maps + obj->nr_maps;
11190 idx = (m - obj->maps) + i;
11191 if (idx >= obj->nr_maps || idx < 0)
11193 return &obj->maps[idx];
11200 return obj->maps;
11209 if (!obj->nr_maps)
11211 return obj->maps + obj->nr_maps - 1;
11214 return __bpf_map__iter(next, obj, -1);
11228 if (pos->real_name && strcmp(pos->real_name, name) == 0)
11234 if (strcmp(pos->real_name, name) == 0)
11238 if (strcmp(pos->name, name) == 0)
11254 return -ENOENT;
11256 if (map->def.key_size != key_sz) {
11258 map->name, key_sz, map->def.key_size);
11259 return -EINVAL;
11262 if (map->fd < 0) {
11263 pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
11264 return -EINVAL;
11270 switch (map->def.type) {
11276 size_t elem_sz = roundup(map->def.value_size, 8);
11281 map->name);
11282 return -EINVAL;
11284 if (map->def.value_size != value_sz) {
11286 map->name, value_sz, map->def.value_size);
11287 return -EINVAL;
11293 pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zu\n",
11294 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);
11295 return -EINVAL;
11300 if (map->def.value_size != value_sz) {
11302 map->name, value_sz, map->def.value_size);
11303 return -EINVAL;
11320 return bpf_map_lookup_elem_flags(map->fd, key, value, flags);
11333 return bpf_map_update_elem(map->fd, key, value, flags);
11345 return bpf_map_delete_elem_flags(map->fd, key, flags);
11358 return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags);
11370 return bpf_map_get_next_key(map->fd, cur_key, next_key);
11379 errno = -PTR_ERR(ptr);
11386 return -errno;
11397 prog->name);
11398 return libbpf_err(-EINVAL);
11417 link->disconnected = true;
11427 if (!link->disconnected && link->detach)
11428 err = link->detach(link);
11429 if (link->pin_path)
11430 free(link->pin_path);
11431 if (link->dealloc)
11432 link->dealloc(link);
11441 return link->fd;
11446 return link->pin_path;
11451 return libbpf_err_errno(close(link->fd));
11461 fd = -errno;
11469 return libbpf_err_ptr(-ENOMEM);
11471 link->detach = &bpf_link__detach_fd;
11472 link->fd = fd;
11474 link->pin_path = strdup(path);
11475 if (!link->pin_path) {
11477 return libbpf_err_ptr(-ENOMEM);
11485 return bpf_link_detach(link->fd) ? -errno : 0;
11492 if (link->pin_path)
11493 return libbpf_err(-EBUSY);
11501 link->pin_path = strdup(path);
11502 if (!link->pin_path)
11503 return libbpf_err(-ENOMEM);
11505 if (bpf_obj_pin(link->fd, link->pin_path)) {
11506 err = -errno;
11507 zfree(&link->pin_path);
11511 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
11519 if (!link->pin_path)
11520 return libbpf_err(-EINVAL);
11522 err = unlink(link->pin_path);
11524 return -errno;
11526 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
11527 zfree(&link->pin_path);
11548 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0)
11549 err = -errno;
11551 if (perf_link->perf_event_fd != link->fd)
11552 close(perf_link->perf_event_fd);
11553 close(link->fd);
11556 if (perf_link->legacy_probe_name) {
11557 if (perf_link->legacy_is_kprobe) {
11558 err = remove_kprobe_event_legacy(perf_link->legacy_probe_name,
11559 perf_link->legacy_is_retprobe);
11561 err = remove_uprobe_event_legacy(perf_link->legacy_probe_name,
11562 perf_link->legacy_is_retprobe);
11573 free(perf_link->legacy_probe_name);
11581 int prog_fd, link_fd = -1, err;
11585 return libbpf_err_ptr(-EINVAL);
11589 prog->name, pfd);
11590 return libbpf_err_ptr(-EINVAL);
11595 prog->name);
11596 return libbpf_err_ptr(-EINVAL);
11601 return libbpf_err_ptr(-ENOMEM);
11602 link->link.detach = &bpf_link_perf_detach;
11603 link->link.dealloc = &bpf_link_perf_dealloc;
11604 link->perf_event_fd = pfd;
11607 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) {
11613 err = -errno;
11615 prog->name, pfd, errstr(err));
11618 link->link.fd = link_fd;
11621 pr_warn("prog '%s': user context value is not supported\n", prog->name);
11622 err = -EOPNOTSUPP;
11627 err = -errno;
11629 prog->name, pfd, errstr(err));
11630 if (err == -EPROTO)
11632 prog->name, pfd);
11635 link->link.fd = pfd;
11640 err = -errno;
11642 prog->name, pfd, errstr(err));
11647 return &link->link;
11661 * this function is expected to parse integer in the range of [0, 2^31-1] from
11672 err = -errno;
11678 err = err == EOF ? -EIO : -errno;
11726 return -EINVAL;
11758 pid < 0 ? -1 : pid /* pid */,
11759 pid == -1 ? 0 : -1 /* cpu */,
11760 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11761 return pfd >= 0 ? pfd : -errno;
11775 return -EINVAL;
11779 return -errno;
11782 err = -errno;
11793 static int has_debugfs = -1;
11855 return append_to_file(tracefs_kprobe_events(), "-:%s/%s",
11898 pid < 0 ? -1 : pid, /* pid */
11899 pid == -1 ? 0 : -1, /* cpu */
11900 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11902 err = -errno;
11985 return libbpf_err_ptr(-EINVAL);
12000 return libbpf_err_ptr(-ENOTSUP);
12004 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
12005 return libbpf_err_ptr(-ENOTSUP);
12010 return libbpf_err_ptr(-EINVAL);
12013 return libbpf_err_ptr(-EOPNOTSUPP);
12018 -1 /* pid */, 0 /* ref_ctr_off */);
12027 return libbpf_err_ptr(-ENOMEM);
12030 offset, -1 /* pid */);
12035 prog->name, retprobe ? "kretprobe" : "kprobe",
12045 prog->name, retprobe ? "kretprobe" : "kprobe",
12053 perf_link->legacy_probe_name = legacy_probe;
12054 perf_link->legacy_is_kprobe = true;
12055 perf_link->legacy_is_retprobe = retprobe;
12087 return libbpf_err_ptr(-EINVAL);
12089 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
12156 struct kprobe_multi_resolve *res = data->res;
12159 if (!glob_match(sym_name, res->pattern))
12162 if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) {
12167 * make multi-kprobe usability a bit better: if no match is
12182 snprintf(sym_trim, sizeof(sym_trim), "%.*s", (int)(sym_sfx - sym_name), sym_name);
12183 if (!bsearch(&psym_trim, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp))
12187 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1);
12191 res->addrs[res->cnt++] = (unsigned long)sym_addr;
12207 err = -errno;
12221 err = -EINVAL;
12225 if (!glob_match(sym_name, res->pattern))
12234 err = -errno;
12243 err = -ENOENT;
12255 if (res->cnt == 0)
12256 err = -ENOENT;
12269 return access(tracefs_available_filter_functions_addrs(), R_OK) != -1;
12282 err = -errno;
12295 err = -EINVAL;
12299 if (!glob_match(sym_name, res->pattern))
12302 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap,
12303 sizeof(*res->addrs), res->cnt + 1);
12307 res->addrs[res->cnt++] = (unsigned long)sym_addr;
12310 if (res->cnt == 0)
12311 err = -ENOENT;
12337 return libbpf_err_ptr(-EINVAL);
12342 prog->name);
12343 return libbpf_err_ptr(-EINVAL);
12353 return libbpf_err_ptr(-EINVAL);
12355 return libbpf_err_ptr(-EINVAL);
12357 return libbpf_err_ptr(-EINVAL);
12359 return libbpf_err_ptr(-EINVAL);
12361 return libbpf_err_ptr(-EINVAL);
12382 prog->name, pattern, res.cnt);
12383 err = -EINVAL;
12395 return libbpf_err_ptr(-EINVAL);
12407 err = -ENOMEM;
12410 link->detach = &bpf_link__detach_fd;
12414 err = -errno;
12421 if (err == -ESRCH)
12422 err = -ENOENT;
12424 prog->name, errstr(err));
12427 link->fd = link_fd;
12447 /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */
12448 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)
12451 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");
12453 func_name = prog->sec_name + sizeof("kretprobe/") - 1;
12455 func_name = prog->sec_name + sizeof("kprobe/") - 1;
12457 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);
12460 return -EINVAL;
12465 pr_warn("kprobe offset must be a non-negative integer: %li\n", offset);
12466 return -EINVAL;
12472 return -EINVAL;
12488 /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */
12489 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)
12492 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
12494 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
12496 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
12499 return *link ? 0 : -errno;
12511 /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */
12512 if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||
12513 strcmp(prog->sec_name, "kretprobe.multi") == 0)
12516 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");
12518 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;
12520 spec = prog->sec_name + sizeof("kprobe.multi/") - 1;
12522 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
12525 return -EINVAL;
12543 /* no auto-attach for SEC("kprobe.session") */
12544 if (strcmp(prog->sec_name, "kprobe.session") == 0)
12547 spec = prog->sec_name + sizeof("kprobe.session/") - 1;
12548 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
12551 return -EINVAL;
12556 return *link ? 0 : -errno;
12563 int n, ret = -EINVAL;
12567 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
12571 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
12578 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);
12582 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
12583 prog->sec_name);
12596 switch (BTF_INFO_KIND(t->info)) {
12621 t = btf__type_by_id(btf, t->type);
12623 return -EINVAL;
12628 return t->size;
12629 return -EINVAL;
12641 proto = btf__type_by_id(btf, t->type);
12651 ret = btf_get_type_size(btf, proto->type, &t);
12657 if (i == nargs - 1 && args[i].type == 0)
12678 return -EINVAL;
12687 name = btf__name_by_offset(btf, t->name_off);
12699 return -ENOMEM;
12709 struct bpf_object *obj = prog->obj;
12724 err = -EINVAL;
12727 btf = mod->btf;
12731 if (prog->attach_btf_obj_fd) {
12732 err = -EINVAL;
12735 btf = obj->btf_vmlinux;
12757 return libbpf_err_ptr(-EINVAL);
12762 prog->name);
12763 return libbpf_err_ptr(-EINVAL);
12771 return libbpf_err_ptr(-EINVAL);
12773 return libbpf_err_ptr(-EINVAL);
12775 return libbpf_err_ptr(-EINVAL);
12782 return libbpf_err_ptr(-EINVAL);
12792 err = -ENOMEM;
12795 link->detach = &bpf_link__detach_fd;
12797 link_fd = bpf_link_create(prog_fd, 0, prog->expected_attach_type, &lopts);
12799 err = -errno;
12800 pr_warn("prog '%s': failed to attach: %s\n", prog->name, errstr(err));
12803 link->fd = link_fd;
12833 if (!str_has_pfx(prog->sec_name, prefixes[i]))
12837 /* no auto-attach case of, e.g., SEC("fentry.multi") */
12838 if (prog->sec_name[pfx_len] == '\0')
12841 if (prog->sec_name[pfx_len] != '/')
12844 spec = prog->sec_name + pfx_len + 1;
12850 prog->name, prog->sec_name);
12851 return -EINVAL;
12854 n = sscanf(spec, "%m[a-zA-Z0-9_.*?:]", &pattern);
12857 return -EINVAL;
12876 return append_to_file(tracefs_uprobe_events(), "-:%s/%s",
12917 pid < 0 ? -1 : pid, /* pid */
12918 pid == -1 ? 0 : -1, /* cpu */
12919 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
12921 err = -errno;
12969 ret = -LIBBPF_ERRNO__FORMAT;
12976 elf_errmsg(-1));
12977 ret = -LIBBPF_ERRNO__LIBELF;
12998 * Based on https://packages.debian.org/sid/libc6.
13004 return "/lib/x86_64-linux-gnu";
13006 return "/lib/i386-linux-gnu";
13008 return "/lib/s390x-linux-gnu";
13010 return "/lib/arm-linux-gnueabi";
13012 return "/lib/arm-linux-gnueabihf";
13014 return "/lib/aarch64-linux-gnu";
13016 return "/lib/mips64el-linux-gnuabi64";
13018 return "/lib/mipsel-linux-gnu";
13020 return "/lib/powerpc64le-linux-gnu";
13022 return "/lib/sparc64-linux-gnu";
13024 return "/lib/riscv64-linux-gnu";
13060 seg_len = next_path ? next_path - s : strlen(s);
13071 return -ENOENT;
13094 return libbpf_err_ptr(-EINVAL);
13099 prog->name);
13100 return libbpf_err_ptr(-EINVAL);
13119 * - syms and offsets are mutually exclusive
13120 * - ref_ctr_offsets and cookies are optional
13126 return libbpf_err_ptr(-EINVAL);
13128 return libbpf_err_ptr(-EINVAL);
13132 return libbpf_err_ptr(-EINVAL);
13135 return libbpf_err_ptr(-EINVAL);
13139 return libbpf_err_ptr(-EINVAL);
13146 prog->name, path, errstr(err));
13180 err = -ENOMEM;
13183 link->detach = &bpf_link__detach_fd;
13187 err = -errno;
13188 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",
13189 prog->name, errstr(err));
13192 link->fd = link_fd;
13219 return libbpf_err_ptr(-EINVAL);
13227 return libbpf_err_ptr(-EINVAL);
13234 min(sizeof(full_path), (size_t)(archive_sep - binary_path + 1)));
13241 prog->name, binary_path, errstr(err));
13270 return libbpf_err_ptr(-ENOTSUP);
13274 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
13275 return libbpf_err_ptr(-ENOTSUP);
13280 return libbpf_err_ptr(-EINVAL);
13290 return libbpf_err_ptr(-EINVAL);
13298 return libbpf_err_ptr(-ENOMEM);
13306 prog->name, retprobe ? "uretprobe" : "uprobe",
13317 prog->name, retprobe ? "uretprobe" : "uprobe",
13325 perf_link->legacy_probe_name = legacy_probe;
13326 perf_link->legacy_is_kprobe = false;
13327 perf_link->legacy_is_retprobe = retprobe;
13339 /* Format of u[ret]probe section definition supporting auto-attach:
13346 * specified (and auto-attach is not possible) or the above format is specified for
13347 * auto-attach.
13353 int n, c, ret = -EINVAL;
13358 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
13362 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
13367 prog->name, prog->sec_name);
13385 prog->name);
13389 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);
13393 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
13394 prog->sec_name);
13420 struct bpf_object *obj = prog->obj;
13426 return libbpf_err_ptr(-EINVAL);
13430 prog->name);
13431 return libbpf_err_ptr(-EINVAL);
13435 return libbpf_err_ptr(-EINVAL);
13441 prog->name, binary_path, errstr(err));
13450 if (IS_ERR(obj->usdt_man))
13451 return libbpf_ptr(obj->usdt_man);
13452 if (!obj->usdt_man) {
13453 obj->usdt_man = usdt_manager_new(obj);
13454 if (IS_ERR(obj->usdt_man))
13455 return libbpf_ptr(obj->usdt_man);
13459 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
13475 /* no auto-attach for just SEC("usdt") */
13484 err = -EINVAL;
13486 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path,
13505 return -errno;
13509 return -E2BIG;
13534 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
13535 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
13537 err = -errno;
13556 return libbpf_err_ptr(-EINVAL);
13563 prog->name, tp_category, tp_name,
13572 prog->name, tp_category, tp_name,
13628 match = sec_name_match_prefix(prog->sec_name, prefixes, ARRAY_SIZE(prefixes));
13630 pr_warn("prog '%s': invalid section name '%s'\n", prog->name, prog->sec_name);
13631 return -EINVAL;
13636 sec_name = strdup(prog->sec_name);
13638 return -ENOMEM;
13640 tp_cat = sec_name + (match - prog->sec_name);
13644 return -EINVAL;
13664 return libbpf_err_ptr(-EINVAL);
13668 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13669 return libbpf_err_ptr(-EINVAL);
13674 return libbpf_err_ptr(-ENOMEM);
13675 link->detach = &bpf_link__detach_fd;
13681 pfd = -errno;
13684 prog->name, tp_name, errstr(pfd));
13687 link->fd = pfd;
13711 match = sec_name_match_prefix(prog->sec_name, prefixes, ARRAY_SIZE(prefixes));
13713 pr_warn("prog '%s': invalid section name '%s'\n", prog->name, prog->sec_name);
13714 return -EINVAL;
13732 return libbpf_err_ptr(-EINVAL);
13736 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13737 return libbpf_err_ptr(-EINVAL);
13742 return libbpf_err_ptr(-ENOMEM);
13743 link->detach = &bpf_link__detach_fd;
13749 pfd = -errno;
13752 prog->name, errstr(pfd));
13755 link->fd = pfd;
13798 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13799 return libbpf_err_ptr(-EINVAL);
13804 return libbpf_err_ptr(-ENOMEM);
13805 link->detach = &bpf_link__detach_fd;
13810 link_fd = -errno;
13813 prog->name, target_name,
13817 link->fd = link_fd;
13854 return libbpf_err_ptr(-EINVAL);
13861 prog->name);
13862 return libbpf_err_ptr(-EINVAL);
13882 return libbpf_err_ptr(-EINVAL);
13887 /* validate we don't have unexpected combinations of non-zero fields */
13890 prog->name);
13891 return libbpf_err_ptr(-EINVAL);
13895 prog->name);
13896 return libbpf_err_ptr(-EINVAL);
13917 return libbpf_err_ptr(-EINVAL);
13922 /* validate we don't have unexpected combinations of non-zero fields */
13925 prog->name);
13926 return libbpf_err_ptr(-EINVAL);
13930 prog->name);
13931 return libbpf_err_ptr(-EINVAL);
13950 prog->name);
13951 return libbpf_err_ptr(-EINVAL);
13954 if (prog->type != BPF_PROG_TYPE_EXT) {
13956 prog->name);
13957 return libbpf_err_ptr(-EINVAL);
13963 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd, prog->obj->token_fd);
13989 return libbpf_err_ptr(-EINVAL);
13996 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13997 return libbpf_err_ptr(-EINVAL);
14002 return libbpf_err_ptr(-ENOMEM);
14003 link->detach = &bpf_link__detach_fd;
14008 link_fd = -errno;
14011 prog->name, errstr(link_fd));
14014 link->fd = link_fd;
14032 return libbpf_err_ptr(-EINVAL);
14036 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
14037 return libbpf_err_ptr(-EINVAL);
14042 return libbpf_err_ptr(-ENOMEM);
14044 link->detach = &bpf_link__detach_fd;
14053 link_fd = -errno;
14056 prog->name, errstr(link_fd));
14059 link->fd = link_fd;
14069 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
14070 return libbpf_err_ptr(-EOPNOTSUPP);
14074 prog->name);
14075 return libbpf_err_ptr(-EINVAL);
14078 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
14082 /* When calling bpf_program__attach() explicitly, auto-attach support
14088 return libbpf_err_ptr(-EOPNOTSUPP);
14105 if (st_link->map_fd < 0)
14107 return bpf_map_delete_elem(link->fd, &zero);
14109 return close(link->fd);
14119 pr_warn("map '%s': can't attach non-struct_ops map\n", map->name);
14120 return libbpf_err_ptr(-EINVAL);
14123 if (map->fd < 0) {
14124 pr_warn("map '%s': can't attach BPF map without FD (was it created?)\n", map->name);
14125 return libbpf_err_ptr(-EINVAL);
14130 return libbpf_err_ptr(-EINVAL);
14133 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
14139 if (err && (!(map->def.map_flags & BPF_F_LINK) || err != -EBUSY)) {
14144 link->link.detach = bpf_link__detach_struct_ops;
14146 if (!(map->def.map_flags & BPF_F_LINK)) {
14148 link->link.fd = map->fd;
14149 link->map_fd = -1;
14150 return &link->link;
14153 fd = bpf_link_create(map->fd, 0, BPF_STRUCT_OPS, NULL);
14159 link->link.fd = fd;
14160 link->map_fd = map->fd;
14162 return &link->link;
14175 return libbpf_err(-EINVAL);
14177 if (map->fd < 0) {
14178 pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
14179 return libbpf_err(-EINVAL);
14184 if (st_ops_link->map_fd < 0)
14185 return libbpf_err(-EINVAL);
14187 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
14193 if (err && err != -EBUSY)
14196 err = bpf_link_update(link->fd, map->fd, NULL);
14200 st_ops_link->map_fd = map->fd;
14215 __u64 data_tail = header->data_tail;
14222 ehdr = base + (data_tail & (mmap_size - 1));
14223 ehdr_size = ehdr->size;
14227 size_t len_first = base + mmap_size - copy_start;
14228 size_t len_second = ehdr_size - len_first;
14262 /* sample_cb and lost_cb are higher-level common-case callbacks */
14301 if (cpu_buf->base &&
14302 munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
14303 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
14304 if (cpu_buf->fd >= 0) {
14305 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
14306 close(cpu_buf->fd);
14308 free(cpu_buf->buf);
14318 if (pb->cpu_bufs) {
14319 for (i = 0; i < pb->cpu_cnt; i++) {
14320 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
14325 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
14328 free(pb->cpu_bufs);
14330 if (pb->epoll_fd >= 0)
14331 close(pb->epoll_fd);
14332 free(pb->events);
14345 return ERR_PTR(-ENOMEM);
14347 cpu_buf->pb = pb;
14348 cpu_buf->cpu = cpu;
14349 cpu_buf->map_key = map_key;
14351 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
14352 -1, PERF_FLAG_FD_CLOEXEC);
14353 if (cpu_buf->fd < 0) {
14354 err = -errno;
14360 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
14362 cpu_buf->fd, 0);
14363 if (cpu_buf->base == MAP_FAILED) {
14364 cpu_buf->base = NULL;
14365 err = -errno;
14371 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
14372 err = -errno;
14400 return libbpf_err_ptr(-EINVAL);
14429 return libbpf_err_ptr(-EINVAL);
14432 return libbpf_err_ptr(-EINVAL);
14454 if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) {
14457 return ERR_PTR(-EINVAL);
14460 /* best-effort sanity checks */
14465 err = -errno;
14467 * -EBADFD, -EFAULT, or -E2BIG on real error
14469 if (err != -EINVAL) {
14480 return ERR_PTR(-EINVAL);
14486 return ERR_PTR(-ENOMEM);
14488 pb->event_cb = p->event_cb;
14489 pb->sample_cb = p->sample_cb;
14490 pb->lost_cb = p->lost_cb;
14491 pb->ctx = p->ctx;
14493 pb->page_size = getpagesize();
14494 pb->mmap_size = pb->page_size * page_cnt;
14495 pb->map_fd = map_fd;
14497 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
14498 if (pb->epoll_fd < 0) {
14499 err = -errno;
14505 if (p->cpu_cnt > 0) {
14506 pb->cpu_cnt = p->cpu_cnt;
14508 pb->cpu_cnt = libbpf_num_possible_cpus();
14509 if (pb->cpu_cnt < 0) {
14510 err = pb->cpu_cnt;
14513 if (map.max_entries && map.max_entries < pb->cpu_cnt)
14514 pb->cpu_cnt = map.max_entries;
14517 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
14518 if (!pb->events) {
14519 err = -ENOMEM;
14523 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
14524 if (!pb->cpu_bufs) {
14525 err = -ENOMEM;
14536 for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
14540 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
14541 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
14546 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
14549 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
14555 pb->cpu_bufs[j] = cpu_buf;
14557 err = bpf_map_update_elem(pb->map_fd, &map_key,
14558 &cpu_buf->fd, 0);
14560 err = -errno;
14561 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
14562 cpu, map_key, cpu_buf->fd,
14567 pb->events[j].events = EPOLLIN;
14568 pb->events[j].data.ptr = cpu_buf;
14569 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
14570 &pb->events[j]) < 0) {
14571 err = -errno;
14573 cpu, cpu_buf->fd,
14579 pb->cpu_cnt = j;
14608 struct perf_buffer *pb = cpu_buf->pb;
14612 if (pb->event_cb)
14613 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
14615 switch (e->type) {
14619 if (pb->sample_cb)
14620 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
14626 if (pb->lost_cb)
14627 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
14631 pr_warn("unknown perf sample type %u\n", e->type);
14642 ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size,
14643 pb->page_size, &cpu_buf->buf,
14644 &cpu_buf->buf_size,
14653 return pb->epoll_fd;
14660 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
14662 return -errno;
14665 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
14681 return pb->cpu_cnt;
14693 if (buf_idx >= pb->cpu_cnt)
14694 return libbpf_err(-EINVAL);
14696 cpu_buf = pb->cpu_bufs[buf_idx];
14698 return libbpf_err(-ENOENT);
14700 return cpu_buf->fd;
14707 if (buf_idx >= pb->cpu_cnt)
14708 return libbpf_err(-EINVAL);
14710 cpu_buf = pb->cpu_bufs[buf_idx];
14712 return libbpf_err(-ENOENT);
14714 *buf = cpu_buf->base;
14715 *buf_size = pb->mmap_size;
14724 * - 0 on success;
14725 * - <0 on failure.
14731 if (buf_idx >= pb->cpu_cnt)
14732 return libbpf_err(-EINVAL);
14734 cpu_buf = pb->cpu_bufs[buf_idx];
14736 return libbpf_err(-ENOENT);
14745 for (i = 0; i < pb->cpu_cnt; i++) {
14746 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
14768 return libbpf_err(-EINVAL);
14770 if (prog->obj->state >= OBJ_LOADED)
14771 return libbpf_err(-EINVAL);
14777 prog->attach_prog_fd = attach_prog_fd;
14783 attach_prog_fd, prog->obj->token_fd);
14788 return libbpf_err(-EINVAL);
14791 err = bpf_object__load_vmlinux_btf(prog->obj, true);
14794 err = find_kernel_btf_id(prog->obj, attach_func_name,
14795 prog->expected_attach_type,
14801 prog->attach_btf_id = btf_id;
14802 prog->attach_btf_obj_fd = btf_obj_fd;
14803 prog->attach_prog_fd = attach_prog_fd;
14815 prog->name);
14816 return libbpf_err(-EINVAL);
14819 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) {
14820 pr_warn("prog '%s': can't associate struct_ops program\n", prog->name);
14821 return libbpf_err(-EINVAL);
14826 pr_warn("map '%s': can't associate BPF map without FD (was it created?)\n", map->name);
14827 return libbpf_err(-EINVAL);
14831 pr_warn("map '%s': can't associate non-struct_ops map\n", map->name);
14832 return libbpf_err(-EINVAL);
14840 int err = 0, n, len, start, end = -1;
14846 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
14852 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
14855 err = -EINVAL;
14863 err = -EINVAL;
14868 err = -ENOMEM;
14872 memset(tmp + *mask_sz, 0, start - *mask_sz);
14873 memset(tmp + start, 1, end - start + 1);
14879 return -EINVAL;
14895 err = -errno;
14902 err = len ? -errno : -EINVAL;
14908 return -E2BIG;
14949 struct bpf_map **map = map_skel->map;
14950 const char *name = map_skel->name;
14951 void **mmaped = map_skel->mmaped;
14956 return -ESRCH;
14959 /* externs shouldn't be pre-setup from user code */
14960 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
14961 *mmaped = (*map)->mmaped;
14974 struct bpf_program **prog = prog_skel->prog;
14975 const char *name = prog_skel->name;
14980 return -ESRCH;
14992 obj = bpf_object_open(NULL, s->data, s->data_sz, s->name, opts);
14996 s->name, errstr(err));
15000 *s->obj = obj;
15001 err = populate_skeleton_maps(obj, s->maps, s->map_cnt, s->map_skel_sz);
15003 pr_warn("failed to populate skeleton maps for '%s': %s\n", s->name, errstr(err));
15007 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt, s->prog_skel_sz);
15009 pr_warn("failed to populate skeleton progs for '%s': %s\n", s->name, errstr(err));
15027 if (!s->obj)
15028 return libbpf_err(-EINVAL);
15030 btf = bpf_object__btf(s->obj);
15033 bpf_object__name(s->obj));
15034 return libbpf_err(-errno);
15037 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt, s->map_skel_sz);
15043 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt, s->prog_skel_sz);
15049 for (var_idx = 0; var_idx < s->var_cnt; var_idx++) {
15050 var_skel = (void *)s->vars + var_idx * s->var_skel_sz;
15051 map = *var_skel->map;
15059 return libbpf_err(-EINVAL);
15065 var_type = btf__type_by_id(btf, var->type);
15066 var_name = btf__name_by_offset(btf, var_type->name_off);
15067 if (strcmp(var_name, var_skel->name) == 0) {
15068 *var_skel->addr = map->mmaped + var->offset;
15080 free(s->maps);
15081 free(s->progs);
15082 free(s->vars);
15090 err = bpf_object__load(*s->obj);
15092 pr_warn("failed to load BPF skeleton '%s': %s\n", s->name, errstr(err));
15096 for (i = 0; i < s->map_cnt; i++) {
15097 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;
15098 struct bpf_map *map = *map_skel->map;
15100 if (!map_skel->mmaped)
15103 if (map->def.type == BPF_MAP_TYPE_ARENA)
15104 *map_skel->mmaped = map->mmaped + map->obj->arena_data_off;
15106 *map_skel->mmaped = map->mmaped;
15116 for (i = 0; i < s->prog_cnt; i++) {
15117 struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz;
15118 struct bpf_program *prog = *prog_skel->prog;
15119 struct bpf_link **link = prog_skel->link;
15121 if (!prog->autoload || !prog->autoattach)
15124 /* auto-attaching not supported for this program */
15125 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
15128 /* if user already set the link manually, don't attempt auto-attach */
15132 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
15134 pr_warn("prog '%s': failed to auto-attach: %s\n",
15139 /* It's possible that for some SEC() definitions auto-attach
15144 * auto-attached. But if not, it shouldn't trigger skeleton's
15152 for (i = 0; i < s->map_cnt; i++) {
15153 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;
15154 struct bpf_map *map = *map_skel->map;
15157 if (!map->autocreate || !map->autoattach)
15165 if (s->map_skel_sz < offsetofend(struct bpf_map_skeleton, link)) {
15166 pr_warn("map '%s': BPF skeleton version is old, skipping map auto-attachment...\n",
15171 link = map_skel->link;
15183 err = -errno;
15184 pr_warn("map '%s': failed to auto-attach: %s\n",
15197 for (i = 0; i < s->prog_cnt; i++) {
15198 struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz;
15199 struct bpf_link **link = prog_skel->link;
15205 if (s->map_skel_sz < sizeof(struct bpf_map_skeleton))
15208 for (i = 0; i < s->map_cnt; i++) {
15209 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;
15210 struct bpf_link **link = map_skel->link;
15225 if (s->obj)
15226 bpf_object__close(*s->obj);
15227 free(s->maps);
15228 free(s->progs);