Lines Matching +full:phase +full:- +full:aligns
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
71 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
301 if (err != -EPERM || geteuid() != 0)
318 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
332 fd = -1; \
343 /* as of v1.0 libbpf_set_strict_mode() is a no-op */
399 /* stored as sec_def->cookie for all libbpf-supported SEC()s */
418 /* BPF program support non-linear XDP buffer */
461 * program. For the entry-point (main) BPF program, this is always
462 * zero. For a sub-program, this gets reset before each of main BPF
464 * whether sub-program was already appended to the main program, and
476 * entry-point BPF programs this includes the size of main program
477 * itself plus all the used sub-programs, appended at the end
530 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
640 /* BTF fd index to be patched in for insn->off, this is
641 * 0 for vmlinux BTF, index in obj->fd_array for module
735 /* Path to the custom BTF to be used for BPF CO-RE relocations as an
739 /* vmlinux BTF override for CO-RE relocations */
795 zclose(prog->fd);
797 zfree(&prog->func_info);
798 zfree(&prog->line_info);
799 zfree(&prog->subprogs);
808 zfree(&prog->name);
809 zfree(&prog->sec_name);
810 zfree(&prog->insns);
811 zfree(&prog->reloc_desc);
813 prog->nr_reloc = 0;
814 prog->insns_cnt = 0;
815 prog->sec_idx = -1;
820 return BPF_CLASS(insn->code) == BPF_JMP &&
821 BPF_OP(insn->code) == BPF_CALL &&
822 BPF_SRC(insn->code) == BPF_K &&
823 insn->src_reg == BPF_PSEUDO_CALL &&
824 insn->dst_reg == 0 &&
825 insn->off == 0;
830 return insn->code == (BPF_JMP | BPF_CALL);
835 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
846 return -EINVAL;
850 prog->obj = obj;
852 prog->sec_idx = sec_idx;
853 prog->sec_insn_off = sec_off / BPF_INSN_SZ;
854 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
856 prog->insns_cnt = prog->sec_insn_cnt;
858 prog->type = BPF_PROG_TYPE_UNSPEC;
859 prog->fd = -1;
860 prog->exception_cb_idx = -1;
867 prog->autoload = false;
871 prog->autoload = true;
874 prog->autoattach = true;
877 prog->log_level = obj->log_level;
879 prog->sec_name = strdup(sec_name);
880 if (!prog->sec_name)
883 prog->name = strdup(name);
884 if (!prog->name)
887 prog->insns = malloc(insn_data_sz);
888 if (!prog->insns)
890 memcpy(prog->insns, insn_data, insn_data_sz);
896 return -ENOMEM;
903 Elf_Data *symbols = obj->efile.symbols;
905 void *data = sec_data->d_buf;
906 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms;
911 progs = obj->programs;
912 nr_progs = obj->nr_programs;
913 nr_syms = symbols->d_size / sizeof(Elf64_Sym);
918 if (sym->st_shndx != sec_idx)
920 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
923 prog_sz = sym->st_size;
924 sec_off = sym->st_value;
926 name = elf_sym_str(obj, sym->st_name);
930 return -LIBBPF_ERRNO__FORMAT;
936 return -LIBBPF_ERRNO__FORMAT;
939 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) {
941 return -ENOTSUP;
950 * In this case the original obj->programs
956 return -ENOMEM;
958 obj->programs = progs;
967 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL)
968 prog->sym_global = true;
975 if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
976 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL))
977 prog->mark_btf_static = true;
980 obj->nr_programs = nr_progs;
988 struct bpf_program *prog = obj->programs;
992 for (p = 0; p < obj->nr_programs; p++, prog++) {
993 insn = prog->insns;
994 for (i = 0; i < prog->insns_cnt; i++, insn++)
997 pr_debug("converted %zu BPF programs to native byte order\n", obj->nr_programs);
1022 if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
1083 if (kern_data_member->type == kern_type_id)
1089 return -EINVAL;
1103 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
1111 for (i = 0; i < obj->nr_programs; i++) {
1112 if (&obj->programs[i] == prog)
1113 return prog->type == BPF_PROG_TYPE_STRUCT_OPS;
1130 for (i = 0; i < obj->nr_programs; ++i) {
1134 prog = &obj->programs[i];
1135 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
1138 for (j = 0; j < obj->nr_maps; ++j) {
1141 map = &obj->maps[j];
1145 type = btf__type_by_id(obj->btf, map->st_ops->type_id);
1148 slot_prog = map->st_ops->progs[k];
1153 if (map->autocreate)
1158 prog->autoload = should_load;
1170 struct bpf_object *obj = map->obj;
1171 const struct btf *btf = obj->btf;
1179 st_ops = map->st_ops;
1180 type = btf__type_by_id(btf, st_ops->type_id);
1181 tname = btf__name_by_offset(btf, type->name_off);
1189 kern_btf = mod_btf ? mod_btf->btf : obj->btf_vmlinux;
1192 map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
1194 map->mod_btf_fd = mod_btf ? mod_btf->fd : -1;
1195 map->def.value_size = kern_vtype->size;
1196 map->btf_vmlinux_value_type_id = kern_vtype_id;
1198 st_ops->kern_vdata = calloc(1, kern_vtype->size);
1199 if (!st_ops->kern_vdata)
1200 return -ENOMEM;
1202 data = st_ops->data;
1203 kern_data_off = kern_data_member->offset / 8;
1204 kern_data = st_ops->kern_vdata + kern_data_off;
1217 mname = btf__name_by_offset(btf, member->name_off);
1218 moff = member->offset / 8;
1220 msize = btf__resolve_size(btf, member->type);
1223 map->name, mname);
1231 map->name, mname);
1232 return -ENOTSUP;
1235 if (st_ops->progs[i]) {
1243 st_ops->progs[i]->autoload = false;
1244 st_ops->progs[i] = NULL;
1247 /* Skip all-zero/NULL fields if they are not present in the kernel BTF */
1249 map->name, mname);
1253 kern_member_idx = kern_member - btf_members(kern_type);
1257 map->name, mname);
1258 return -ENOTSUP;
1261 kern_moff = kern_member->offset / 8;
1264 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
1265 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
1267 if (BTF_INFO_KIND(mtype->info) !=
1268 BTF_INFO_KIND(kern_mtype->info)) {
1270 map->name, mname, BTF_INFO_KIND(mtype->info),
1271 BTF_INFO_KIND(kern_mtype->info));
1272 return -ENOTSUP;
1281 if (st_ops->progs[i] && st_ops->progs[i] != prog)
1282 st_ops->progs[i]->autoload = false;
1285 st_ops->progs[i] = prog;
1291 map->name, mname);
1292 return -ENOTSUP;
1296 kern_mtype->type,
1299 /* mtype->type must be a func_proto which was
1305 map->name, mname);
1306 return -ENOTSUP;
1310 prog->attach_btf_obj_fd = mod_btf->fd;
1315 if (!prog->attach_btf_id) {
1316 prog->attach_btf_id = kern_type_id;
1317 prog->expected_attach_type = kern_member_idx;
1320 /* struct_ops BPF prog can be re-used between multiple
1325 if (prog->attach_btf_id != kern_type_id) {
1327 map->name, mname, prog->name, prog->sec_name, prog->type,
1328 prog->attach_btf_id, kern_type_id);
1329 return -EINVAL;
1331 if (prog->expected_attach_type != kern_member_idx) {
1333 map->name, mname, prog->name, prog->sec_name, prog->type,
1334 prog->expected_attach_type, kern_member_idx);
1335 return -EINVAL;
1338 st_ops->kern_func_off[i] = kern_data_off + kern_moff;
1341 map->name, mname, prog->name, moff,
1350 map->name, mname, (ssize_t)msize,
1352 return -ENOTSUP;
1356 map->name, mname, (unsigned int)msize,
1370 for (i = 0; i < obj->nr_maps; i++) {
1371 map = &obj->maps[i];
1376 if (!map->autocreate)
1399 if (shndx == -1)
1402 btf = obj->btf;
1408 return -EINVAL;
1414 type = btf__type_by_id(obj->btf, vsi->type);
1415 var_name = btf__name_by_offset(obj->btf, type->name_off);
1417 type_id = btf__resolve_type(obj->btf, vsi->type);
1420 vsi->type, sec_name);
1421 return -EINVAL;
1424 type = btf__type_by_id(obj->btf, type_id);
1425 tname = btf__name_by_offset(obj->btf, type->name_off);
1428 return -ENOTSUP;
1432 return -EINVAL;
1439 map->sec_idx = shndx;
1440 map->sec_offset = vsi->offset;
1441 map->name = strdup(var_name);
1442 if (!map->name)
1443 return -ENOMEM;
1444 map->btf_value_type_id = type_id;
1450 map->autocreate = false;
1455 map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
1456 map->def.key_size = sizeof(int);
1457 map->def.value_size = type->size;
1458 map->def.max_entries = 1;
1459 map->def.map_flags = strcmp(sec_name, STRUCT_OPS_LINK_SEC) == 0 ? BPF_F_LINK : 0;
1460 map->autoattach = true;
1462 map->st_ops = calloc(1, sizeof(*map->st_ops));
1463 if (!map->st_ops)
1464 return -ENOMEM;
1465 st_ops = map->st_ops;
1466 st_ops->data = malloc(type->size);
1467 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
1468 st_ops->kern_func_off = malloc(btf_vlen(type) *
1469 sizeof(*st_ops->kern_func_off));
1470 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
1471 return -ENOMEM;
1473 if (vsi->offset + type->size > data->d_size) {
1476 return -EINVAL;
1479 memcpy(st_ops->data,
1480 data->d_buf + vsi->offset,
1481 type->size);
1482 st_ops->type_id = type_id;
1485 tname, type_id, var_name, vsi->offset);
1496 for (sec_idx = 0; sec_idx < obj->efile.sec_cnt; ++sec_idx) {
1497 struct elf_sec_desc *desc = &obj->efile.secs[sec_idx];
1499 if (desc->sec_type != SEC_ST_OPS)
1504 return -LIBBPF_ERRNO__FORMAT;
1506 err = init_struct_ops_maps(obj, sec_name, sec_idx, desc->data);
1525 return ERR_PTR(-ENOMEM);
1528 strcpy(obj->path, path);
1530 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name));
1533 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name));
1534 end = strchr(obj->name, '.');
1539 obj->efile.fd = -1;
1546 obj->efile.obj_buf = obj_buf;
1547 obj->efile.obj_buf_sz = obj_buf_sz;
1548 obj->efile.btf_maps_shndx = -1;
1549 obj->kconfig_map_idx = -1;
1550 obj->arena_map_idx = -1;
1552 obj->kern_version = get_kernel_version();
1553 obj->state = OBJ_OPEN;
1560 if (!obj->efile.elf)
1563 elf_end(obj->efile.elf);
1564 obj->efile.elf = NULL;
1565 obj->efile.ehdr = NULL;
1566 obj->efile.symbols = NULL;
1567 obj->efile.arena_data = NULL;
1569 zfree(&obj->efile.secs);
1570 obj->efile.sec_cnt = 0;
1571 zclose(obj->efile.fd);
1572 obj->efile.obj_buf = NULL;
1573 obj->efile.obj_buf_sz = 0;
1582 if (obj->efile.elf) {
1584 return -LIBBPF_ERRNO__LIBELF;
1587 if (obj->efile.obj_buf_sz > 0) {
1589 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1591 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
1592 if (obj->efile.fd < 0) {
1593 err = -errno;
1594 pr_warn("elf: failed to open %s: %s\n", obj->path, errstr(err));
1598 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1602 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1603 err = -LIBBPF_ERRNO__LIBELF;
1607 obj->efile.elf = elf;
1610 err = -LIBBPF_ERRNO__FORMAT;
1611 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);
1616 err = -LIBBPF_ERRNO__FORMAT;
1617 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);
1621 obj->efile.ehdr = ehdr = elf64_getehdr(elf);
1622 if (!obj->efile.ehdr) {
1623 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1624 err = -LIBBPF_ERRNO__FORMAT;
1629 if (ehdr->e_ident[EI_DATA] != ELFDATA2LSB &&
1630 ehdr->e_ident[EI_DATA] != ELFDATA2MSB) {
1631 err = -LIBBPF_ERRNO__ENDIAN;
1632 pr_warn("elf: '%s' has unknown byte order\n", obj->path);
1636 obj->byteorder = ehdr->e_ident[EI_DATA];
1638 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
1640 obj->path, elf_errmsg(-1));
1641 err = -LIBBPF_ERRNO__FORMAT;
1646 if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {
1648 obj->path, elf_errmsg(-1));
1649 err = -LIBBPF_ERRNO__FORMAT;
1654 if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {
1655 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1656 err = -LIBBPF_ERRNO__FORMAT;
1669 return obj->byteorder == ELFDATA2LSB;
1671 return obj->byteorder == ELFDATA2MSB;
1681 pr_warn("invalid license section in %s\n", obj->path);
1682 return -LIBBPF_ERRNO__FORMAT;
1684 /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't
1687 libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
1688 pr_debug("license of %s is %s\n", obj->path, obj->license);
1698 pr_warn("invalid kver section in %s\n", obj->path);
1699 return -LIBBPF_ERRNO__FORMAT;
1702 obj->kern_version = kver;
1703 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1721 return -EINVAL;
1726 *size = data->d_size;
1730 return -ENOENT;
1735 Elf_Data *symbols = obj->efile.symbols;
1739 for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {
1742 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)
1745 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&
1746 ELF64_ST_BIND(sym->st_info) != STB_WEAK)
1749 sname = elf_sym_str(obj, sym->st_name);
1752 return ERR_PTR(-EIO);
1758 return ERR_PTR(-ENOENT);
1771 const char *name = "libbpf-placeholder-fd";
1778 return -errno;
1783 return -errno;
1792 err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,
1793 sizeof(*obj->maps), obj->nr_maps + 1);
1797 map = &obj->maps[obj->nr_maps++];
1798 map->obj = obj;
1805 * to finalize and load BTF very late in BPF object's loading phase,
1811 map->fd = create_placeholder_fd();
1812 if (map->fd < 0)
1813 return ERR_PTR(map->fd);
1814 map->inner_map_fd = -1;
1815 map->autocreate = true;
1834 switch (map->def.type) {
1836 return array_map_mmap_sz(map->def.value_size, map->def.max_entries);
1838 return page_sz * map->def.max_entries;
1848 if (!map->mmaped)
1849 return -EINVAL;
1854 mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1856 return -errno;
1858 memcpy(mmaped, map->mmaped, min(old_sz, new_sz));
1859 munmap(map->mmaped, old_sz);
1860 map->mmaped = mmaped;
1894 * '.rodata.abracad' kernel and user-visible name.
1903 sfx_len = BPF_OBJ_NAME_LEN - 1;
1909 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));
1911 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1936 if (!map->btf_value_type_id)
1939 t = btf__type_by_id(obj->btf, map->btf_value_type_id);
1945 vt = btf__type_by_id(obj->btf, vsi->type);
1949 if (btf_var(vt)->linkage != BTF_VAR_STATIC)
1969 map->libbpf_type = type;
1970 map->sec_idx = sec_idx;
1971 map->sec_offset = 0;
1972 map->real_name = strdup(real_name);
1973 map->name = internal_map_name(obj, real_name);
1974 if (!map->real_name || !map->name) {
1975 zfree(&map->real_name);
1976 zfree(&map->name);
1977 return -ENOMEM;
1980 def = &map->def;
1981 def->type = BPF_MAP_TYPE_ARRAY;
1982 def->key_size = sizeof(int);
1983 def->value_size = data_sz;
1984 def->max_entries = 1;
1985 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1992 def->map_flags |= BPF_F_MMAPABLE;
1995 map->name, map->sec_idx, map->sec_offset, def->map_flags);
1998 map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
1999 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
2000 if (map->mmaped == MAP_FAILED) {
2001 err = -errno;
2002 map->mmaped = NULL;
2003 pr_warn("failed to alloc map '%s' content buffer: %s\n", map->name, errstr(err));
2004 zfree(&map->real_name);
2005 zfree(&map->name);
2010 memcpy(map->mmaped, data, data_sz);
2012 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
2023 * Populate obj->maps with libbpf internal maps.
2025 for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
2026 sec_desc = &obj->efile.secs[sec_idx];
2029 if (!sec_desc->data || sec_desc->data->d_size == 0)
2032 switch (sec_desc->sec_type) {
2037 sec_desc->data->d_buf,
2038 sec_desc->data->d_size);
2041 obj->has_rodata = true;
2045 sec_desc->data->d_buf,
2046 sec_desc->data->d_size);
2053 sec_desc->data->d_size);
2071 for (i = 0; i < obj->nr_extern; i++) {
2072 if (strcmp(obj->externs[i].name, name) == 0)
2073 return &obj->externs[i];
2084 for (i = 0; i < obj->nr_extern; i++) {
2085 ext_name = obj->externs[i].name;
2087 return &obj->externs[i];
2095 switch (ext->kcfg.type) {
2099 ext->name, value);
2100 return -EINVAL;
2120 ext->name, value);
2121 return -EINVAL;
2123 ext->is_set = true;
2132 if (ext->kcfg.type != KCFG_CHAR_ARR) {
2134 ext->name, value);
2135 return -EINVAL;
2139 if (len < 2 || value[len - 1] != '"') {
2141 ext->name, value);
2142 return -EINVAL;
2146 len -= 2;
2147 if (len >= ext->kcfg.sz) {
2149 ext->name, value, len, ext->kcfg.sz - 1);
2150 len = ext->kcfg.sz - 1;
2154 ext->is_set = true;
2166 err = -errno;
2172 return -EINVAL;
2179 int bit_sz = ext->kcfg.sz * 8;
2181 if (ext->kcfg.sz == 8)
2184 /* Validate that value stored in u64 fits in integer of `ext->sz`
2189 * -2^(Y-1) <= X <= 2^(Y-1) - 1
2190 * 0 <= X + 2^(Y-1) <= 2^Y - 1
2191 * 0 <= X + 2^(Y-1) < 2^Y
2193 * For unsigned target integer, check that all the (64 - Y) bits are
2196 if (ext->kcfg.is_signed)
2197 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
2205 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&
2206 ext->kcfg.type != KCFG_BOOL) {
2208 ext->name, (unsigned long long)value);
2209 return -EINVAL;
2211 if (ext->kcfg.type == KCFG_BOOL && value > 1) {
2213 ext->name, (unsigned long long)value);
2214 return -EINVAL;
2219 ext->name, (unsigned long long)value, ext->kcfg.sz);
2220 return -ERANGE;
2222 switch (ext->kcfg.sz) {
2236 return -EINVAL;
2238 ext->is_set = true;
2257 return -EINVAL;
2262 if (buf[len - 1] == '\n')
2263 buf[len - 1] = '\0';
2269 return -EINVAL;
2273 if (!ext || ext->is_set)
2276 ext_val = data + ext->kcfg.data_off;
2290 pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);
2293 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
2294 pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);
2295 return -EINVAL;
2302 pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);
2314 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
2316 return -EINVAL;
2318 return -ENAMETOOLONG;
2327 return -ENOENT;
2353 err = -errno;
2354 pr_warn("failed to open in-memory Kconfig: %s\n", errstr(err));
2361 pr_warn("error parsing in-memory Kconfig line '%s': %s\n",
2377 for (i = 0; i < obj->nr_extern; i++) {
2378 ext = &obj->externs[i];
2379 if (ext->type == EXT_KCFG)
2386 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
2388 ".kconfig", obj->efile.symbols_shndx,
2393 obj->kconfig_map_idx = obj->nr_maps - 1;
2408 *res_id = t->type;
2409 t = btf__type_by_id(btf, t->type);
2424 t = skip_mods_and_typedefs(btf, t->type, res_id);
2471 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2472 const char *name = btf__name_by_offset(btf, m->name_off);
2482 arr_t = btf__type_by_id(btf, t->type);
2485 map_name, name, t->type);
2494 *res = arr_info->nelems;
2501 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2502 const char *name = btf__name_by_offset(btf, m->name_off);
2529 *res = e->val;
2544 return -EINVAL;
2546 return -ENAMETOOLONG;
2585 const char *name = btf__name_by_offset(btf, m->name_off);
2589 return -EINVAL;
2592 if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
2593 return -EINVAL;
2594 map_def->parts |= MAP_DEF_MAP_TYPE;
2596 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
2597 return -EINVAL;
2598 map_def->parts |= MAP_DEF_MAX_ENTRIES;
2600 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
2601 return -EINVAL;
2602 map_def->parts |= MAP_DEF_MAP_FLAGS;
2604 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
2605 return -EINVAL;
2606 map_def->parts |= MAP_DEF_NUMA_NODE;
2611 return -EINVAL;
2612 if (map_def->key_size && map_def->key_size != sz) {
2614 map_name, map_def->key_size, sz);
2615 return -EINVAL;
2617 map_def->key_size = sz;
2618 map_def->parts |= MAP_DEF_KEY_SIZE;
2622 t = btf__type_by_id(btf, m->type);
2625 map_name, m->type);
2626 return -EINVAL;
2631 return -EINVAL;
2633 sz = btf__resolve_size(btf, t->type);
2636 map_name, t->type, (ssize_t)sz);
2639 if (map_def->key_size && map_def->key_size != sz) {
2641 map_name, map_def->key_size, (ssize_t)sz);
2642 return -EINVAL;
2644 map_def->key_size = sz;
2645 map_def->key_type_id = t->type;
2646 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;
2651 return -EINVAL;
2652 if (map_def->value_size && map_def->value_size != sz) {
2654 map_name, map_def->value_size, sz);
2655 return -EINVAL;
2657 map_def->value_size = sz;
2658 map_def->parts |= MAP_DEF_VALUE_SIZE;
2662 t = btf__type_by_id(btf, m->type);
2665 map_name, m->type);
2666 return -EINVAL;
2671 return -EINVAL;
2673 sz = btf__resolve_size(btf, t->type);
2676 map_name, t->type, (ssize_t)sz);
2679 if (map_def->value_size && map_def->value_size != sz) {
2681 map_name, map_def->value_size, (ssize_t)sz);
2682 return -EINVAL;
2684 map_def->value_size = sz;
2685 map_def->value_type_id = t->type;
2686 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;
2689 bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);
2690 bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;
2691 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2696 pr_warn("map '%s': multi-level inner maps not supported.\n",
2698 return -ENOTSUP;
2700 if (i != vlen - 1) {
2703 return -EINVAL;
2706 pr_warn("map '%s': should be map-in-map or prog-array.\n",
2708 return -ENOTSUP;
2710 if (map_def->value_size && map_def->value_size != 4) {
2712 map_name, map_def->value_size);
2713 return -EINVAL;
2715 map_def->value_size = 4;
2716 t = btf__type_by_id(btf, m->type);
2719 map_name, desc, m->type);
2720 return -EINVAL;
2722 if (!btf_is_array(t) || btf_array(t)->nelems) {
2723 pr_warn("map '%s': %s spec is not a zero-sized array.\n",
2725 return -EINVAL;
2727 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
2731 return -EINVAL;
2733 t = skip_mods_and_typedefs(btf, t->type, NULL);
2736 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
2738 return -EINVAL;
2743 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2745 return -EINVAL;
2753 map_def->parts |= MAP_DEF_INNER_MAP;
2759 return -EINVAL;
2762 return -EINVAL;
2766 return -EINVAL;
2768 map_def->pinning = val;
2769 map_def->parts |= MAP_DEF_PINNING;
2774 return -EINVAL;
2775 map_def->map_extra = map_extra;
2776 map_def->parts |= MAP_DEF_MAP_EXTRA;
2780 return -ENOTSUP;
2786 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
2788 return -EINVAL;
2803 * a power-of-2 multiple of kernel's page size. If user diligently
2810 * user-set size to satisfy both user size request and kernel
2819 * very close to UINT_MAX but is not a power-of-2 multiple of
2827 return map->def.type == BPF_MAP_TYPE_RINGBUF ||
2828 map->def.type == BPF_MAP_TYPE_USER_RINGBUF;
2833 map->def.type = def->map_type;
2834 map->def.key_size = def->key_size;
2835 map->def.value_size = def->value_size;
2836 map->def.max_entries = def->max_entries;
2837 map->def.map_flags = def->map_flags;
2838 map->map_extra = def->map_extra;
2840 map->numa_node = def->numa_node;
2841 map->btf_key_type_id = def->key_type_id;
2842 map->btf_value_type_id = def->value_type_id;
2844 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
2846 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
2848 if (def->parts & MAP_DEF_MAP_TYPE)
2849 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
2851 if (def->parts & MAP_DEF_KEY_TYPE)
2853 map->name, def->key_type_id, def->key_size);
2854 else if (def->parts & MAP_DEF_KEY_SIZE)
2855 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
2857 if (def->parts & MAP_DEF_VALUE_TYPE)
2859 map->name, def->value_type_id, def->value_size);
2860 else if (def->parts & MAP_DEF_VALUE_SIZE)
2861 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
2863 if (def->parts & MAP_DEF_MAX_ENTRIES)
2864 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
2865 if (def->parts & MAP_DEF_MAP_FLAGS)
2866 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);
2867 if (def->parts & MAP_DEF_MAP_EXTRA)
2868 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,
2869 (unsigned long long)def->map_extra);
2870 if (def->parts & MAP_DEF_PINNING)
2871 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
2872 if (def->parts & MAP_DEF_NUMA_NODE)
2873 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
2875 if (def->parts & MAP_DEF_INNER_MAP)
2876 pr_debug("map '%s': found inner map definition.\n", map->name);
2904 var = btf__type_by_id(obj->btf, vi->type);
2906 map_name = btf__name_by_offset(obj->btf, var->name_off);
2910 return -EINVAL;
2912 if ((__u64)vi->offset + vi->size > data->d_size) {
2914 return -EINVAL;
2919 return -EINVAL;
2921 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
2923 map_name, btf_var_linkage_str(var_extra->linkage));
2924 return -EOPNOTSUPP;
2927 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2931 return -EINVAL;
2933 if (def->size > vi->size) {
2935 return -EINVAL;
2941 map->name = strdup(map_name);
2942 if (!map->name) {
2944 return -ENOMEM;
2946 map->libbpf_type = LIBBPF_MAP_UNSPEC;
2947 map->def.type = BPF_MAP_TYPE_UNSPEC;
2948 map->sec_idx = sec_idx;
2949 map->sec_offset = vi->offset;
2950 map->btf_var_idx = var_idx;
2952 map_name, map->sec_idx, map->sec_offset);
2954 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2963 pr_warn("map '%s': couldn't build pin path.\n", map->name);
2969 map->inner_map = calloc(1, sizeof(*map->inner_map));
2970 if (!map->inner_map)
2971 return -ENOMEM;
2972 map->inner_map->fd = create_placeholder_fd();
2973 if (map->inner_map->fd < 0)
2974 return map->inner_map->fd;
2975 map->inner_map->sec_idx = sec_idx;
2976 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
2977 if (!map->inner_map->name)
2978 return -ENOMEM;
2979 sprintf(map->inner_map->name, "%s.inner", map_name);
2981 fill_map_from_def(map->inner_map, &inner_def);
3003 return -E2BIG;
3006 obj->arena_data = malloc(data_sz);
3007 if (!obj->arena_data)
3008 return -ENOMEM;
3009 memcpy(obj->arena_data, data, data_sz);
3010 obj->arena_data_sz = data_sz;
3013 map->mmaped = obj->arena_data;
3028 if (obj->efile.btf_maps_shndx < 0)
3031 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
3035 MAPS_ELF_SEC, obj->path);
3036 return -EINVAL;
3039 nr_types = btf__type_cnt(obj->btf);
3041 t = btf__type_by_id(obj->btf, i);
3044 name = btf__name_by_offset(obj->btf, t->name_off);
3047 obj->efile.btf_maps_sec_btf_id = i;
3054 return -ENOENT;
3060 obj->efile.btf_maps_shndx,
3067 for (i = 0; i < obj->nr_maps; i++) {
3068 struct bpf_map *map = &obj->maps[i];
3070 if (map->def.type != BPF_MAP_TYPE_ARENA)
3073 if (obj->arena_map_idx >= 0) {
3075 map->name, obj->maps[obj->arena_map_idx].name);
3076 return -EINVAL;
3078 obj->arena_map_idx = i;
3080 if (obj->efile.arena_data) {
3081 err = init_arena_map_data(obj, map, ARENA_SEC, obj->efile.arena_data_shndx,
3082 obj->efile.arena_data->d_buf,
3083 obj->efile.arena_data->d_size);
3088 if (obj->efile.arena_data && obj->arena_map_idx < 0) {
3091 return -ENOENT;
3123 return sh->sh_flags & SHF_EXECINSTR;
3171 return ERR_PTR(-ENOMEM);
3172 /* btf_header() gives us endian-safe header info */
3175 if (!has_layout && hdr->hdr_len >= sizeof(struct btf_header) &&
3176 (hdr->layout_len != 0 || hdr->layout_off != 0)) {
3188 return ERR_PTR(-ENOMEM);
3192 new_hdr->layout_off = 0;
3193 new_hdr->layout_len = 0;
3194 new_str_off = hdr->type_off + hdr->type_len;
3196 if (old_hdr->magic != hdr->magic)
3197 new_hdr->str_off = bswap_32(new_str_off);
3199 new_hdr->str_off = new_str_off;
3201 memmove(new_raw_data + hdr->hdr_len + new_str_off,
3202 new_raw_data + hdr->hdr_len + hdr->str_off,
3203 hdr->str_len);
3204 sz = hdr->hdr_len + hdr->type_off + hdr->type_len + hdr->str_len;
3214 /* enforce 8-byte pointers for BPF-targeted BTFs */
3222 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
3228 t->size = 1;
3237 name = (char *)btf__name_by_offset(btf, t->name_off);
3245 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
3248 m->offset = v->offset * 8;
3249 m->type = v->type;
3251 vt = (void *)btf__type_by_id(btf, v->type);
3252 m->name_off = vt->name_off;
3255 starts_with_qmark(btf__name_by_offset(btf, t->name_off))) {
3259 name = (char *)btf__name_by_offset(btf, t->name_off);
3265 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
3266 t->size = sizeof(__u32); /* kernel enforced */
3269 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
3272 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
3274 /* replace FLOAT with an equally-sized empty STRUCT;
3278 t->name_off = 0;
3279 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
3282 t->name_off = 0;
3283 t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
3286 t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
3302 t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
3304 m->type = enum64_placeholder_id;
3305 m->offset = 0;
3315 return obj->efile.btf_maps_shndx >= 0 ||
3316 obj->efile.has_st_ops ||
3317 obj->nr_extern > 0;
3322 return obj->efile.has_st_ops;
3329 int err = -ENOENT;
3332 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
3333 err = libbpf_get_error(obj->btf);
3335 obj->btf = NULL;
3339 /* enforce 8-byte pointers for BPF-targeted BTFs */
3340 btf__set_pointer_size(obj->btf, 8);
3346 if (!obj->btf) {
3351 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
3352 err = libbpf_get_error(obj->btf_ext);
3356 obj->btf_ext = NULL;
3361 ext_segs[0] = &obj->btf_ext->func_info;
3362 ext_segs[1] = &obj->btf_ext->line_info;
3363 ext_segs[2] = &obj->btf_ext->core_relo_info;
3370 if (seg->sec_cnt == 0)
3373 seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));
3374 if (!seg->sec_idxs) {
3375 err = -ENOMEM;
3386 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
3393 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);
3410 return a->offset - b->offset;
3417 const char *sec_name = btf__name_by_offset(btf, t->name_off);
3424 return -ENOENT;
3427 /* Extern-backing datasecs (.ksyms, .kconfig) have their size and
3439 * to be optional. But the STV_HIDDEN handling is non-optional for any
3440 * non-extern DATASEC, so the variable fixup loop below handles both
3441 * functions at the same time, paying the cost of BTF VAR <-> ELF
3444 if (t->size == 0) {
3449 return -ENOENT;
3452 t->size = size;
3462 t_var = btf__type_by_id(btf, vsi->type);
3464 pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name);
3465 return -EINVAL;
3469 if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN)
3472 var_name = btf__name_by_offset(btf, t_var->name_off);
3476 return -ENOENT;
3483 return -ENOENT;
3487 vsi->offset = sym->st_value;
3496 if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
3497 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)
3498 var->linkage = BTF_VAR_STATIC;
3510 if (!obj->btf)
3513 n = btf__type_cnt(obj->btf);
3515 struct btf_type *t = btf_type_by_id(obj->btf, i);
3523 err = btf_fixup_datasec(obj, obj->btf, t);
3534 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
3535 prog->type == BPF_PROG_TYPE_LSM)
3541 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
3558 /* CO-RE relocations need kernel BTF, only when btf_custom_path
3561 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
3565 for (i = 0; i < obj->nr_extern; i++) {
3568 ext = &obj->externs[i];
3569 if (ext->type == EXT_KSYM && ext->ksym.type_id)
3574 if (!prog->autoload)
3593 if (obj->btf_vmlinux || obj->gen_loader)
3599 obj->btf_vmlinux = btf__load_vmlinux_btf();
3600 err = libbpf_get_error(obj->btf_vmlinux);
3603 obj->btf_vmlinux = NULL;
3611 struct btf *kern_btf = obj->btf;
3615 if (!obj->btf)
3620 err = -EOPNOTSUPP;
3635 for (i = 0; i < obj->nr_programs; i++) {
3636 struct bpf_program *prog = &obj->programs[i];
3641 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3644 n = btf__type_cnt(obj->btf);
3646 t = btf_type_by_id(obj->btf, j);
3650 name = btf__str_by_offset(obj->btf, t->name_off);
3651 if (strcmp(name, prog->name) != 0)
3654 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
3661 kern_btf = bpf_object__sanitize_btf(obj, obj->btf);
3666 if (obj->gen_loader) {
3671 return -ENOMEM;
3672 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
3674 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
3679 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,
3680 obj->log_level ? 1 : 0, obj->token_fd);
3685 btf__set_fd(obj->btf, btf__fd(kern_btf));
3686 btf__set_fd(kern_btf, -1);
3709 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
3712 off, obj->path, elf_errmsg(-1));
3723 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
3726 off, obj->path, elf_errmsg(-1));
3737 scn = elf_getscn(obj->efile.elf, idx);
3740 idx, obj->path, elf_errmsg(-1));
3749 Elf *elf = obj->efile.elf;
3775 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3794 name = elf_sec_str(obj, sh->sh_name);
3797 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3815 obj->path, elf_errmsg(-1));
3824 if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))
3827 return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;
3832 if (idx >= data->d_size / sizeof(Elf64_Rel))
3835 return (Elf64_Rel *)data->d_buf + idx;
3847 if (hdr->sh_type == SHT_STRTAB)
3851 if (hdr->sh_type == SHT_LLVM_ADDRSIG)
3855 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
3864 name += sizeof(".rel") - 1;
3883 if (a->sec_idx != b->sec_idx)
3884 return a->sec_idx < b->sec_idx ? -1 : 1;
3887 return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
3893 Elf *elf = obj->efile.elf;
3902 /* ELF section indices are 0-based, but sec #0 is special "invalid"
3907 if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) {
3909 obj->path, elf_errmsg(-1));
3910 return -LIBBPF_ERRNO__FORMAT;
3912 obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));
3913 if (!obj->efile.secs)
3914 return -ENOMEM;
3923 return -LIBBPF_ERRNO__FORMAT;
3925 if (sh->sh_type == SHT_SYMTAB) {
3926 if (obj->efile.symbols) {
3927 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
3928 return -LIBBPF_ERRNO__FORMAT;
3933 return -LIBBPF_ERRNO__FORMAT;
3937 obj->efile.symbols = data;
3938 obj->efile.symbols_shndx = idx;
3939 obj->efile.strtabidx = sh->sh_link;
3943 if (!obj->efile.symbols) {
3945 obj->path);
3946 return -ENOENT;
3952 sec_desc = &obj->efile.secs[idx];
3956 return -LIBBPF_ERRNO__FORMAT;
3958 name = elf_sec_str(obj, sh->sh_name);
3960 return -LIBBPF_ERRNO__FORMAT;
3967 return -LIBBPF_ERRNO__FORMAT;
3970 idx, name, (unsigned long)data->d_size,
3971 (int)sh->sh_link, (unsigned long)sh->sh_flags,
3972 (int)sh->sh_type);
3975 err = bpf_object__init_license(obj, data->d_buf, data->d_size);
3979 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
3984 return -ENOTSUP;
3986 obj->efile.btf_maps_shndx = idx;
3988 if (sh->sh_type != SHT_PROGBITS)
3989 return -LIBBPF_ERRNO__FORMAT;
3992 if (sh->sh_type != SHT_PROGBITS)
3993 return -LIBBPF_ERRNO__FORMAT;
3995 } else if (sh->sh_type == SHT_SYMTAB) {
3997 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {
3998 if (sh->sh_flags & SHF_EXECINSTR) {
4000 obj->efile.text_shndx = idx;
4006 sec_desc->sec_type = SEC_DATA;
4007 sec_desc->shdr = sh;
4008 sec_desc->data = data;
4011 sec_desc->sec_type = SEC_RODATA;
4012 sec_desc->shdr = sh;
4013 sec_desc->data = data;
4018 sec_desc->sec_type = SEC_ST_OPS;
4019 sec_desc->shdr = sh;
4020 sec_desc->data = data;
4021 obj->efile.has_st_ops = true;
4023 obj->efile.arena_data = data;
4024 obj->efile.arena_data_shndx = idx;
4026 obj->jumptables_data = malloc(data->d_size);
4027 if (!obj->jumptables_data)
4028 return -ENOMEM;
4029 memcpy(obj->jumptables_data, data->d_buf, data->d_size);
4030 obj->jumptables_data_sz = data->d_size;
4031 obj->efile.jumptables_data_shndx = idx;
4036 } else if (sh->sh_type == SHT_REL) {
4037 int targ_sec_idx = sh->sh_info; /* points to other section */
4039 if (sh->sh_entsize != sizeof(Elf64_Rel) ||
4040 targ_sec_idx >= obj->efile.sec_cnt)
4041 return -LIBBPF_ERRNO__FORMAT;
4056 sec_desc->sec_type = SEC_RELO;
4057 sec_desc->shdr = sh;
4058 sec_desc->data = data;
4059 } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 ||
4061 sec_desc->sec_type = SEC_BSS;
4062 sec_desc->shdr = sh;
4063 sec_desc->data = data;
4066 (size_t)sh->sh_size);
4070 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
4071 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
4072 return -LIBBPF_ERRNO__FORMAT;
4079 /* sort BPF programs by section name and in-section instruction offset
4082 if (obj->nr_programs)
4083 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
4090 int bind = ELF64_ST_BIND(sym->st_info);
4092 return sym->st_shndx == SHN_UNDEF &&
4094 ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;
4099 int bind = ELF64_ST_BIND(sym->st_info);
4100 int type = ELF64_ST_TYPE(sym->st_info);
4103 if (sym->st_shndx != text_shndx)
4121 return -ESRCH;
4130 tname = btf__name_by_offset(btf, t->name_off);
4135 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
4136 return -EINVAL;
4139 return -EINVAL;
4144 return -ENOENT;
4153 return -ESRCH;
4164 if (vs->type == ext_btf_id)
4169 return -ENOENT;
4179 name = btf__name_by_offset(btf, t->name_off);
4188 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
4191 if (t->size == 1)
4193 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
4198 if (t->size != 4)
4208 if (btf_array(t)->nelems == 0)
4210 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
4223 if (a->type != b->type)
4224 return a->type < b->type ? -1 : 1;
4226 if (a->type == EXT_KCFG) {
4228 if (a->kcfg.align != b->kcfg.align)
4229 return a->kcfg.align > b->kcfg.align ? -1 : 1;
4231 if (a->kcfg.sz != b->kcfg.sz)
4232 return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
4236 return strcmp(a->name, b->name);
4274 vt = btf__type_by_id(btf, vs->type);
4305 if (!obj->efile.symbols)
4308 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
4310 if (!sh || sh->sh_entsize != sizeof(Elf64_Sym))
4311 return -LIBBPF_ERRNO__FORMAT;
4313 dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
4317 n = sh->sh_size / sh->sh_entsize;
4324 return -LIBBPF_ERRNO__FORMAT;
4327 ext_name = elf_sym_str(obj, sym->st_name);
4331 ext = obj->externs;
4332 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
4334 return -ENOMEM;
4335 obj->externs = ext;
4336 ext = &ext[obj->nr_extern];
4338 obj->nr_extern++;
4340 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
4341 if (ext->btf_id <= 0) {
4343 ext_name, ext->btf_id);
4344 return ext->btf_id;
4346 t = btf__type_by_id(obj->btf, ext->btf_id);
4347 ext->name = strdup(btf__name_by_offset(obj->btf, t->name_off));
4348 if (!ext->name)
4349 return -ENOMEM;
4350 ext->sym_idx = i;
4351 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;
4353 ext_essent_len = bpf_core_essential_name_len(ext->name);
4354 ext->essent_name = NULL;
4355 if (ext_essent_len != strlen(ext->name)) {
4356 ext->essent_name = strndup(ext->name, ext_essent_len);
4357 if (!ext->essent_name)
4358 return -ENOMEM;
4361 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
4362 if (ext->sec_btf_id <= 0) {
4364 ext_name, ext->btf_id, ext->sec_btf_id);
4365 return ext->sec_btf_id;
4367 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
4368 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
4373 ext->name, KCONFIG_SEC);
4374 return -ENOTSUP;
4377 ext->type = EXT_KCFG;
4378 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
4379 if (ext->kcfg.sz <= 0) {
4381 ext_name, ext->kcfg.sz);
4382 return ext->kcfg.sz;
4384 ext->kcfg.align = btf__align_of(obj->btf, t->type);
4385 if (ext->kcfg.align <= 0) {
4387 ext_name, ext->kcfg.align);
4388 return -EINVAL;
4390 ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
4391 &ext->kcfg.is_signed);
4392 if (ext->kcfg.type == KCFG_UNKNOWN) {
4394 return -ENOTSUP;
4398 ext->type = EXT_KSYM;
4399 skip_mods_and_typedefs(obj->btf, t->type,
4400 &ext->ksym.type_id);
4403 return -ENOTSUP;
4406 pr_debug("collected %d externs total\n", obj->nr_extern);
4408 if (!obj->nr_extern)
4412 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
4416 * pretending that each extern is a 8-byte variable
4419 /* find existing 4-byte integer type in BTF to use for fake
4422 int int_btf_id = find_int_btf_id(obj->btf);
4424 * will be used to replace the vs->type and
4430 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
4431 for (i = 0; i < obj->nr_extern; i++) {
4432 ext = &obj->externs[i];
4433 if (ext->type != EXT_KSYM)
4436 i, ext->sym_idx, ext->name);
4445 vt = (void *)btf__type_by_id(obj->btf, vs->type);
4446 ext_name = btf__name_by_offset(obj->btf, vt->name_off);
4451 return -ESRCH;
4458 func_proto = btf__type_by_id(obj->btf,
4459 vt->type);
4467 dummy_var->name_off;
4468 vs->type = dummy_var_btf_id;
4469 vt->info &= ~0xffff;
4470 vt->info |= BTF_FUNC_GLOBAL;
4472 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4473 vt->type = int_btf_id;
4475 vs->offset = off;
4476 vs->size = sizeof(int);
4478 sec->size = off;
4485 for (i = 0; i < obj->nr_extern; i++) {
4486 ext = &obj->externs[i];
4487 if (ext->type != EXT_KCFG)
4490 ext->kcfg.data_off = roundup(off, ext->kcfg.align);
4491 off = ext->kcfg.data_off + ext->kcfg.sz;
4493 i, ext->sym_idx, ext->kcfg.data_off, ext->name);
4495 sec->size = off;
4500 t = btf__type_by_id(obj->btf, vs->type);
4501 ext_name = btf__name_by_offset(obj->btf, t->name_off);
4506 return -ESRCH;
4508 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4509 vs->offset = ext->kcfg.data_off;
4517 return prog->sec_idx == obj->efile.text_shndx;
4529 if (!strcmp(prog->name, name))
4538 switch (obj->efile.secs[shndx].sec_type) {
4551 return shndx == obj->efile.btf_maps_shndx;
4557 if (shndx == obj->efile.symbols_shndx)
4560 switch (obj->efile.secs[shndx].sec_type) {
4577 purged = calloc(prog->insns_cnt, BPF_INSN_SZ);
4579 return -ENOMEM;
4584 for (i = 0; i < prog->insns_cnt; i++) {
4585 purged[i] = prog->insns[i];
4591 if (i >= prog->insns_cnt ||
4592 prog->insns[i].code != 0 ||
4593 prog->insns[i].dst_reg != 0 ||
4594 prog->insns[i].src_reg != 0 ||
4595 prog->insns[i].off != 0) {
4596 err = -EINVAL;
4599 purged[i] = prog->insns[i];
4603 libbpf_sha256(purged, prog->insns_cnt * sizeof(struct bpf_insn),
4604 prog->hash);
4615 struct bpf_insn *insn = &prog->insns[insn_idx];
4616 size_t map_idx, nr_maps = prog->obj->nr_maps;
4617 struct bpf_object *obj = prog->obj;
4618 __u32 shdr_idx = sym->st_shndx;
4625 prog->name, sym_name, insn_idx, insn->code);
4626 return -LIBBPF_ERRNO__RELOC;
4630 int sym_idx = ELF64_R_SYM(rel->r_info);
4631 int i, n = obj->nr_extern;
4635 ext = &obj->externs[i];
4636 if (ext->sym_idx == sym_idx)
4641 prog->name, sym_name, sym_idx);
4642 return -LIBBPF_ERRNO__RELOC;
4645 prog->name, i, ext->name, ext->sym_idx, insn_idx);
4646 if (insn->code == (BPF_JMP | BPF_CALL))
4647 reloc_desc->type = RELO_EXTERN_CALL;
4649 reloc_desc->type = RELO_EXTERN_LD64;
4650 reloc_desc->insn_idx = insn_idx;
4651 reloc_desc->ext_idx = i;
4655 /* sub-program call relocation */
4657 if (insn->src_reg != BPF_PSEUDO_CALL) {
4658 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
4659 return -LIBBPF_ERRNO__RELOC;
4662 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
4665 prog->name, sym_name, sym_sec_name);
4666 return -LIBBPF_ERRNO__RELOC;
4668 if (sym->st_value % BPF_INSN_SZ) {
4670 prog->name, sym_name, (size_t)sym->st_value);
4671 return -LIBBPF_ERRNO__RELOC;
4673 reloc_desc->type = RELO_CALL;
4674 reloc_desc->insn_idx = insn_idx;
4675 reloc_desc->sym_off = sym->st_value;
4681 prog->name, sym_name, shdr_idx);
4682 return -LIBBPF_ERRNO__RELOC;
4686 if (sym_is_subprog(sym, obj->efile.text_shndx)) {
4687 /* global_func: sym->st_value = offset in the section, insn->imm = 0.
4688 * local_func: sym->st_value = 0, insn->imm = offset in the section.
4690 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
4692 prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4693 return -LIBBPF_ERRNO__RELOC;
4696 reloc_desc->type = RELO_SUBPROG_ADDR;
4697 reloc_desc->insn_idx = insn_idx;
4698 reloc_desc->sym_off = sym->st_value;
4706 if (shdr_idx == obj->efile.arena_data_shndx) {
4707 if (obj->arena_map_idx < 0) {
4709 prog->name, insn_idx);
4710 return -LIBBPF_ERRNO__RELOC;
4712 reloc_desc->type = RELO_DATA;
4713 reloc_desc->insn_idx = insn_idx;
4714 reloc_desc->map_idx = obj->arena_map_idx;
4715 reloc_desc->sym_off = sym->st_value;
4717 map = &obj->maps[obj->arena_map_idx];
4719 prog->name, obj->arena_map_idx, map->name, map->sec_idx,
4720 map->sec_offset, insn_idx);
4725 if (shdr_idx == obj->efile.jumptables_data_shndx) {
4726 reloc_desc->type = RELO_INSN_ARRAY;
4727 reloc_desc->insn_idx = insn_idx;
4728 reloc_desc->map_idx = -1;
4729 reloc_desc->sym_off = sym->st_value;
4730 reloc_desc->sym_size = sym->st_size;
4738 prog->name, sym_name, sym_sec_name);
4739 return -LIBBPF_ERRNO__RELOC;
4742 map = &obj->maps[map_idx];
4743 if (map->libbpf_type != type ||
4744 map->sec_idx != sym->st_shndx ||
4745 map->sec_offset != sym->st_value)
4748 prog->name, map_idx, map->name, map->sec_idx,
4749 map->sec_offset, insn_idx);
4754 prog->name, sym_sec_name, (size_t)sym->st_value);
4755 return -LIBBPF_ERRNO__RELOC;
4757 reloc_desc->type = RELO_LD64;
4758 reloc_desc->insn_idx = insn_idx;
4759 reloc_desc->map_idx = map_idx;
4760 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
4767 prog->name, sym_sec_name);
4768 return -LIBBPF_ERRNO__RELOC;
4771 map = &obj->maps[map_idx];
4772 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)
4775 prog->name, map_idx, map->name, map->sec_idx,
4776 map->sec_offset, insn_idx);
4781 prog->name, sym_sec_name);
4782 return -LIBBPF_ERRNO__RELOC;
4785 reloc_desc->type = RELO_DATA;
4786 reloc_desc->insn_idx = insn_idx;
4787 reloc_desc->map_idx = map_idx;
4788 reloc_desc->sym_off = sym->st_value;
4794 return insn_idx >= prog->sec_insn_off &&
4795 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4801 int l = 0, r = obj->nr_programs - 1, m;
4804 if (!obj->nr_programs)
4808 m = l + (r - l + 1) / 2;
4809 prog = &obj->programs[m];
4811 if (prog->sec_idx < sec_idx ||
4812 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4815 r = m - 1;
4820 prog = &obj->programs[l];
4821 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4830 size_t sec_idx = shdr->sh_info, sym_idx;
4841 if (sec_idx >= obj->efile.sec_cnt)
4842 return -EINVAL;
4847 return -LIBBPF_ERRNO__FORMAT;
4849 relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4852 return -EINVAL;
4856 nrels = shdr->sh_size / shdr->sh_entsize;
4862 return -LIBBPF_ERRNO__FORMAT;
4865 sym_idx = ELF64_R_SYM(rel->r_info);
4870 return -LIBBPF_ERRNO__FORMAT;
4873 if (sym->st_shndx >= obj->efile.sec_cnt) {
4875 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);
4876 return -LIBBPF_ERRNO__FORMAT;
4879 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) {
4881 relo_sec_name, (size_t)rel->r_offset, i);
4882 return -LIBBPF_ERRNO__FORMAT;
4885 insn_idx = rel->r_offset / BPF_INSN_SZ;
4892 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)
4893 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));
4895 sym_name = elf_sym_str(obj, sym->st_name);
4908 relos = libbpf_reallocarray(prog->reloc_desc,
4909 prog->nr_reloc + 1, sizeof(*relos));
4911 return -ENOMEM;
4912 prog->reloc_desc = relos;
4915 insn_idx -= prog->sec_insn_off;
4916 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4921 prog->nr_reloc++;
4930 if (!obj->btf)
4931 return -ENOENT;
4933 /* if it's BTF-defined map, we don't need to search for type IDs.
4937 if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))
4945 return -ENOENT;
4947 id = btf__find_by_name(obj->btf, map->real_name);
4951 map->btf_key_type_id = 0;
4952 map->btf_value_type_id = id;
4968 err = -errno;
4976 info->type = val;
4978 info->key_size = val;
4980 info->value_size = val;
4982 info->max_entries = val;
4984 info->map_flags = val;
4994 return map->obj->state >= OBJ_PREPARED || map->reused;
4999 return map->autocreate;
5005 return libbpf_err(-EBUSY);
5007 map->autocreate = autocreate;
5014 return libbpf_err(-EINVAL);
5016 map->autoattach = autoattach;
5022 return map->autoattach;
5040 if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)
5041 new_name = strdup(map->name);
5046 return libbpf_err(-errno);
5055 err = -errno;
5059 err = reuse_fd(map->fd, new_fd);
5063 free(map->name);
5065 map->name = new_name;
5066 map->def.type = info.type;
5067 map->def.key_size = info.key_size;
5068 map->def.value_size = info.value_size;
5069 map->def.max_entries = info.max_entries;
5070 map->def.map_flags = info.map_flags;
5071 map->btf_key_type_id = info.btf_key_type_id;
5072 map->btf_value_type_id = info.btf_value_type_id;
5073 map->reused = true;
5074 map->map_extra = info.map_extra;
5085 return map->def.max_entries;
5090 if (!bpf_map_type__is_map_in_map(map->def.type))
5093 return map->inner_map;
5099 return libbpf_err(-EBUSY);
5101 map->def.max_entries = max_entries;
5103 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
5105 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
5113 int bpffs_fd = -1, token_fd, err;
5118 if (obj->token_path && obj->token_path[0] == '\0') {
5119 pr_debug("object '%s': token is prevented, skipping...\n", obj->name);
5123 mandatory = obj->token_path != NULL;
5126 bpffs_path = obj->token_path ?: BPF_FS_DEFAULT_PATH;
5129 err = -errno;
5131 obj->name, errstr(err), bpffs_path,
5139 if (!mandatory && token_fd == -ENOENT) {
5141 obj->name, bpffs_path);
5145 obj->name, token_fd, bpffs_path,
5150 obj->feat_cache = calloc(1, sizeof(*obj->feat_cache));
5151 if (!obj->feat_cache) {
5153 return -ENOMEM;
5156 obj->token_fd = token_fd;
5157 obj->feat_cache->token_fd = token_fd;
5171 .token_fd = obj->token_fd,
5172 .prog_flags = obj->token_fd ? BPF_F_TOKEN_FD : 0,
5175 if (obj->gen_loader)
5191 return -ret;
5200 if (obj->gen_loader)
5206 if (obj->feat_cache)
5207 return feat_supported(obj->feat_cache, feat_id);
5215 if (obj->feat_cache)
5216 free(obj->feat_cache);
5217 obj->feat_cache = cache;
5243 if (map->def.type == BPF_MAP_TYPE_DEVMAP || map->def.type == BPF_MAP_TYPE_DEVMAP_HASH)
5246 return (map_info.type == map->def.type &&
5247 map_info.key_size == map->def.key_size &&
5248 map_info.value_size == map->def.value_size &&
5249 map_info.max_entries == map->def.max_entries &&
5250 map_info.map_flags == map->def.map_flags &&
5251 map_info.map_extra == map->map_extra);
5259 pin_fd = bpf_obj_get(map->pin_path);
5261 err = -errno;
5262 if (err == -ENOENT) {
5264 map->pin_path);
5269 map->pin_path, errstr(err));
5275 map->pin_path);
5277 return -EINVAL;
5285 map->pinned = true;
5286 pr_debug("reused pinned map at '%s'\n", map->pin_path);
5294 enum libbpf_map_type map_type = map->libbpf_type;
5298 if (obj->gen_loader) {
5299 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
5300 map->mmaped, map->def.value_size);
5302 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);
5306 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
5308 err = -errno;
5314 /* Freeze .rodata and .kconfig map as read-only from syscall side. */
5316 err = bpf_map_freeze(map->fd);
5318 err = -errno;
5319 pr_warn("map '%s': failed to freeze as read-only: %s\n",
5325 /* Remap anonymous mmap()-ed "map initialization image" as
5326 * a BPF map-backed mmap()-ed memory, but preserving the same
5335 if (map->def.map_flags & BPF_F_MMAPABLE) {
5339 if (map->def.map_flags & BPF_F_RDONLY_PROG)
5343 mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map->fd, 0);
5345 err = -errno;
5346 pr_warn("map '%s': failed to re-mmap() contents: %s\n",
5350 map->mmaped = mmaped;
5351 } else if (map->mmaped) {
5352 munmap(map->mmaped, mmap_sz);
5353 map->mmaped = NULL;
5364 struct bpf_map_def *def = &map->def;
5369 map_name = map->name;
5370 create_attr.map_ifindex = map->map_ifindex;
5371 create_attr.map_flags = def->map_flags;
5372 create_attr.numa_node = map->numa_node;
5373 create_attr.map_extra = map->map_extra;
5374 create_attr.token_fd = obj->token_fd;
5375 if (obj->token_fd)
5377 if (map->excl_prog) {
5378 err = bpf_prog_compute_hash(map->excl_prog);
5382 create_attr.excl_prog_hash = map->excl_prog->hash;
5387 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
5388 if (map->mod_btf_fd >= 0) {
5389 create_attr.value_type_btf_obj_fd = map->mod_btf_fd;
5394 if (obj->btf && btf__fd(obj->btf) >= 0) {
5395 create_attr.btf_fd = btf__fd(obj->btf);
5396 create_attr.btf_key_type_id = map->btf_key_type_id;
5397 create_attr.btf_value_type_id = map->btf_value_type_id;
5400 if (bpf_map_type__is_map_in_map(def->type)) {
5401 if (map->inner_map) {
5402 err = map_set_def_max_entries(map->inner_map);
5405 err = bpf_object__create_map(obj, map->inner_map, true);
5408 map->name, errstr(err));
5411 map->inner_map_fd = map->inner_map->fd;
5413 if (map->inner_map_fd >= 0)
5414 create_attr.inner_map_fd = map->inner_map_fd;
5417 switch (def->type) {
5435 map->btf_key_type_id = 0;
5436 map->btf_value_type_id = 0;
5445 if (obj->gen_loader) {
5446 bpf_gen__map_create(obj->gen_loader, def->type, map_name,
5447 def->key_size, def->value_size, def->max_entries,
5448 &create_attr, is_inner ? -1 : map - obj->maps);
5453 * will be reset to -1 eventually.
5455 map_fd = map->fd;
5457 map_fd = bpf_map_create(def->type, map_name,
5458 def->key_size, def->value_size,
5459 def->max_entries, &create_attr);
5462 err = -errno;
5464 map->name, errstr(err));
5468 map->btf_key_type_id = 0;
5469 map->btf_value_type_id = 0;
5470 map_fd = bpf_map_create(def->type, map_name,
5471 def->key_size, def->value_size,
5472 def->max_entries, &create_attr);
5475 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
5476 if (obj->gen_loader)
5477 map->inner_map->fd = -1;
5478 bpf_map__destroy(map->inner_map);
5479 zfree(&map->inner_map);
5485 /* obj->gen_loader case, prevent reuse_fd() from closing map_fd */
5486 if (map->fd == map_fd)
5492 * map->fd stays valid but now point to what map_fd points to.
5494 return reuse_fd(map->fd, map_fd);
5503 for (i = 0; i < map->init_slots_sz; i++) {
5504 if (!map->init_slots[i])
5507 targ_map = map->init_slots[i];
5508 fd = targ_map->fd;
5510 if (obj->gen_loader) {
5511 bpf_gen__populate_outer_map(obj->gen_loader,
5512 map - obj->maps, i,
5513 targ_map - obj->maps);
5515 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5518 err = -errno;
5520 map->name, i, targ_map->name, fd, errstr(err));
5524 map->name, i, targ_map->name, fd);
5527 zfree(&map->init_slots);
5528 map->init_slots_sz = 0;
5539 if (obj->gen_loader)
5540 return -ENOTSUP;
5542 for (i = 0; i < map->init_slots_sz; i++) {
5543 if (!map->init_slots[i])
5546 targ_prog = map->init_slots[i];
5549 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5551 err = -errno;
5553 map->name, i, targ_prog->name, fd, errstr(err));
5557 map->name, i, targ_prog->name, fd);
5560 zfree(&map->init_slots);
5561 map->init_slots_sz = 0;
5571 for (i = 0; i < obj->nr_maps; i++) {
5572 map = &obj->maps[i];
5574 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY)
5586 if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) {
5592 map->name, nr_cpus);
5595 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
5596 map->def.max_entries = nr_cpus;
5610 for (i = 0; i < obj->nr_maps; i++) {
5611 map = &obj->maps[i];
5615 * loading, if we detect that at least one of the to-be-loaded
5620 * but also it allows to have CO-RE applications that use
5622 * If those global variable-using programs are not loaded at
5628 map->autocreate = false;
5630 if (!map->autocreate) {
5631 pr_debug("map '%s': skipped auto-creating...\n", map->name);
5641 if (map->pin_path) {
5645 map->name);
5648 if (retried && map->fd < 0) {
5650 map->name);
5651 err = -ENOENT;
5656 if (map->reused) {
5658 map->name, map->fd);
5665 map->name, map->fd);
5671 } else if (map->def.type == BPF_MAP_TYPE_ARENA) {
5672 map->mmaped = mmap((void *)(long)map->map_extra,
5674 map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED,
5675 map->fd, 0);
5676 if (map->mmaped == MAP_FAILED) {
5677 err = -errno;
5678 map->mmaped = NULL;
5680 map->name, errstr(err));
5683 if (obj->arena_data) {
5684 memcpy(map->mmaped + obj->arena_data_off, obj->arena_data,
5685 obj->arena_data_sz);
5686 zfree(&obj->arena_data);
5689 if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) {
5696 if (map->pin_path && !map->pinned) {
5699 if (!retried && err == -EEXIST) {
5703 pr_warn("map '%s': failed to auto-pin at '%s': %s\n",
5704 map->name, map->pin_path, errstr(err));
5713 pr_warn("map '%s': failed to create: %s\n", map->name, errstr(err));
5716 zclose(obj->maps[j].fd);
5730 * underscore is ignored by BPF CO-RE relocation during relocation matching.
5737 for (i = n - 5; i >= 0; i--) {
5749 free(cands->cands);
5766 local_t = btf__type_by_id(local_cand->btf, local_cand->id);
5767 local_name = btf__str_by_offset(local_cand->btf, local_t->name_off);
5775 targ_name = btf__name_by_offset(targ_btf, t->name_off);
5786 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
5787 local_cand->id, btf_kind_str(local_t),
5790 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
5791 sizeof(*cands->cands));
5793 return -ENOMEM;
5795 cand = &new_cands[cands->len];
5796 cand->btf = targ_btf;
5797 cand->id = i;
5799 cands->cands = new_cands;
5800 cands->len++;
5814 if (obj->btf_modules_loaded)
5817 if (obj->gen_loader)
5821 obj->btf_modules_loaded = true;
5836 err = -errno;
5845 err = -errno;
5858 err = -errno;
5863 /* ignore non-module BTFs */
5869 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
5877 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
5878 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
5882 mod_btf = &obj->btf_modules[obj->btf_module_cnt];
5884 mod_btf->btf = btf;
5885 mod_btf->id = id;
5886 mod_btf->fd = fd;
5887 mod_btf->name = strdup(name);
5888 if (!mod_btf->name) {
5889 err = -ENOMEM;
5892 obj->btf_module_cnt++;
5917 return ERR_PTR(-EINVAL);
5919 local_name = btf__name_by_offset(local_btf, local_t->name_off);
5921 return ERR_PTR(-EINVAL);
5926 return ERR_PTR(-ENOMEM);
5929 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
5935 if (cands->len)
5939 if (obj->btf_vmlinux_override)
5947 for (i = 0; i < obj->btf_module_cnt; i++) {
5949 obj->btf_modules[i].btf,
5950 obj->btf_modules[i].name,
5951 btf__type_cnt(obj->btf_vmlinux),
5964 * type-based CO-RE relocations and follow slightly different rules than
5965 * field-based relocations. This function assumes that root types were already
5966 * checked for name match. Beyond that initial root-level name check, names
5968 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
5971 * - for ENUMs, the size is ignored;
5972 * - for INT, size and signedness are ignored;
5973 * - for ARRAY, dimensionality is ignored, element types are checked for
5975 * - CONST/VOLATILE/RESTRICT modifiers are ignored;
5976 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
5977 * - FUNC_PROTOs are compatible if they have compatible signature: same
5980 * more experience with using BPF CO-RE relocations.
6009 relos = libbpf_reallocarray(prog->reloc_desc,
6010 prog->nr_reloc + 1, sizeof(*relos));
6012 return -ENOMEM;
6013 relo = &relos[prog->nr_reloc];
6014 relo->type = RELO_CORE;
6015 relo->insn_idx = insn_idx;
6016 relo->core_relo = core_relo;
6017 prog->reloc_desc = relos;
6018 prog->nr_reloc++;
6027 for (i = 0; i < prog->nr_reloc; i++) {
6028 relo = &prog->reloc_desc[i];
6029 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx)
6032 return relo->core_relo;
6047 const char *prog_name = prog->name;
6050 __u32 local_id = relo->type_id;
6055 return -EINVAL;
6057 local_name = btf__name_by_offset(local_btf, local_type->name_off);
6059 return -EINVAL;
6061 if (relo->kind != BPF_CORE_TYPE_ID_LOCAL &&
6063 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
6095 if (obj->btf_ext->core_relo_info.len == 0)
6099 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
6100 err = libbpf_get_error(obj->btf_vmlinux_override);
6113 seg = &obj->btf_ext->core_relo_info;
6116 sec_idx = seg->sec_idxs[sec_num];
6119 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6121 err = -EINVAL;
6125 pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info);
6128 if (rec->insn_off % BPF_INSN_SZ)
6129 return -EINVAL;
6130 insn_idx = rec->insn_off / BPF_INSN_SZ;
6137 * This is similar to what x86-64 linker does for relocations.
6141 pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subprogram\n",
6145 /* no need to apply CO-RE relocation if the program is
6148 if (!prog->autoload)
6152 * program's frame of reference; (sub-)program code is not yet
6153 * relocated, so it's enough to just subtract in-section offset
6155 insn_idx = insn_idx - prog->sec_insn_off;
6156 if (insn_idx >= prog->insns_cnt)
6157 return -EINVAL;
6158 insn = &prog->insns[insn_idx];
6163 prog->name, i, errstr(err));
6167 if (prog->obj->gen_loader)
6170 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
6173 prog->name, i, errstr(err));
6177 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
6180 prog->name, i, insn_idx, errstr(err));
6187 /* obj->btf_vmlinux and module BTFs are freed after object load */
6188 btf__free(obj->btf_vmlinux_override);
6189 obj->btf_vmlinux_override = NULL;
6193 bpf_core_free_cands(entry->pvalue);
6211 prog->name, relo_idx, insn_idx, map_idx, map->name);
6215 insn->code = BPF_JMP | BPF_CALL;
6216 insn->dst_reg = 0;
6217 insn->src_reg = 0;
6218 insn->off = 0;
6222 * where lower 123 is map index into obj->maps[] array
6224 insn->imm = POISON_LDIMM64_MAP_BASE + map_idx;
6239 prog->name, relo_idx, insn_idx, ext->name);
6242 insn->code = BPF_JMP | BPF_CALL;
6243 insn->dst_reg = 0;
6244 insn->src_reg = 0;
6245 insn->off = 0;
6249 * where lower 123 is extern index into obj->externs[] array
6251 insn->imm = POISON_CALL_KFUNC_BASE + ext_idx;
6258 for (i = 0; i < obj->jumptable_map_cnt; i++) {
6264 if (obj->jumptable_maps[i].sym_off == sym_off &&
6265 obj->jumptable_maps[i].prog == prog)
6266 return obj->jumptable_maps[i].fd;
6269 return -ENOENT;
6274 size_t cnt = obj->jumptable_map_cnt;
6275 size_t size = sizeof(obj->jumptable_maps[0]);
6278 tmp = libbpf_reallocarray(obj->jumptable_maps, cnt + 1, size);
6280 return -ENOMEM;
6282 obj->jumptable_maps = tmp;
6283 obj->jumptable_maps[cnt].prog = prog;
6284 obj->jumptable_maps[cnt].sym_off = sym_off;
6285 obj->jumptable_maps[cnt].fd = map_fd;
6286 obj->jumptable_map_cnt++;
6295 for (i = prog->subprog_cnt - 1; i >= 0; i--) {
6296 if (insn_idx >= prog->subprogs[i].sub_insn_off)
6300 return -1;
6306 unsigned int sym_off = relo->sym_off;
6307 int jt_size = relo->sym_size;
6324 return -EINVAL;
6330 return -EINVAL;
6338 if (!obj->jumptables_data) {
6340 err = -EINVAL;
6343 if (sym_off + jt_size > obj->jumptables_data_sz) {
6345 obj->jumptables_data_sz, sym_off + jt_size);
6346 err = -EINVAL;
6350 subprog_idx = -1; /* main program */
6351 if (relo->insn_idx < 0 || relo->insn_idx >= prog->insns_cnt) {
6352 pr_warn("map '.jumptables': invalid instruction index %d\n", relo->insn_idx);
6353 err = -EINVAL;
6356 if (prog->subprogs)
6357 subprog_idx = find_subprog_idx(prog, relo->insn_idx);
6359 jt = (__u64 *)(obj->jumptables_data + sym_off);
6367 insn_off -= prog->subprogs[subprog_idx].sec_insn_off;
6368 insn_off += prog->subprogs[subprog_idx].sub_insn_off;
6370 insn_off -= prog->sec_insn_off;
6374 * LLVM-generated jump tables contain u64 records, however
6380 err = -EINVAL;
6406 * - map references;
6407 * - global variable references;
6408 * - extern references.
6415 for (i = 0; i < prog->nr_reloc; i++) {
6416 struct reloc_desc *relo = &prog->reloc_desc[i];
6417 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6421 switch (relo->type) {
6423 map = &obj->maps[relo->map_idx];
6424 if (obj->gen_loader) {
6426 insn[0].imm = relo->map_idx;
6427 } else if (map->autocreate) {
6429 insn[0].imm = map->fd;
6431 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6432 relo->map_idx, map);
6436 map = &obj->maps[relo->map_idx];
6437 insn[1].imm = insn[0].imm + relo->sym_off;
6439 if (relo->map_idx == obj->arena_map_idx)
6440 insn[1].imm += obj->arena_data_off;
6442 if (obj->gen_loader) {
6444 insn[0].imm = relo->map_idx;
6445 } else if (map->autocreate) {
6447 insn[0].imm = map->fd;
6449 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6450 relo->map_idx, map);
6454 ext = &obj->externs[relo->ext_idx];
6455 if (ext->type == EXT_KCFG) {
6456 if (obj->gen_loader) {
6458 insn[0].imm = obj->kconfig_map_idx;
6461 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
6463 insn[1].imm = ext->kcfg.data_off;
6465 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */
6467 insn[0].imm = ext->ksym.kernel_btf_id;
6468 insn[1].imm = ext->ksym.kernel_btf_obj_fd;
6470 insn[0].imm = (__u32)ext->ksym.addr;
6471 insn[1].imm = ext->ksym.addr >> 32;
6476 ext = &obj->externs[relo->ext_idx];
6478 if (ext->is_set) {
6479 insn[0].imm = ext->ksym.kernel_btf_id;
6480 insn[0].off = ext->ksym.btf_fd_idx;
6482 poison_kfunc_call(prog, i, relo->insn_idx, insn,
6483 relo->ext_idx, ext);
6489 prog->name, i);
6490 return -EINVAL;
6506 prog->name, i, relo->sym_off);
6510 insn->imm = map_fd;
6511 insn->off = 0;
6516 prog->name, i, relo->type);
6517 return -EINVAL;
6538 sec_idx = ext_info->sec_idxs[sec_num];
6540 if (prog->sec_idx != sec_idx)
6546 if (insn_off < prog->sec_insn_off)
6548 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
6553 copy_end = rec + ext_info->rec_size;
6557 return -ENOENT;
6559 /* append func/line info of a given (sub-)program to the main
6562 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
6563 new_sz = old_sz + (copy_end - copy_start);
6566 return -ENOMEM;
6568 *prog_rec_cnt = new_sz / ext_info->rec_size;
6569 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
6571 /* Kernel instruction offsets are in units of 8-byte
6577 off_adj = prog->sub_insn_off - prog->sec_insn_off;
6580 for (; rec < rec_end; rec += ext_info->rec_size) {
6585 *prog_rec_sz = ext_info->rec_size;
6589 return -ENOENT;
6602 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC))
6608 if (main_prog != prog && !main_prog->func_info)
6611 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6612 &main_prog->func_info,
6613 &main_prog->func_info_cnt,
6614 &main_prog->func_info_rec_size);
6616 if (err != -ENOENT) {
6618 prog->name, errstr(err));
6621 if (main_prog->func_info) {
6626 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6631 prog->name);
6636 if (main_prog != prog && !main_prog->line_info)
6639 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6640 &main_prog->line_info,
6641 &main_prog->line_info_cnt,
6642 &main_prog->line_info_rec_size);
6644 if (err != -ENOENT) {
6646 prog->name, errstr(err));
6649 if (main_prog->line_info) {
6654 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6659 prog->name);
6669 if (insn_idx == relo->insn_idx)
6671 return insn_idx < relo->insn_idx ? -1 : 1;
6676 if (!prog->nr_reloc)
6678 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6679 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6684 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;
6690 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));
6696 return -ENOMEM;
6697 if (subprog->nr_reloc)
6698 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,
6699 sizeof(*relos) * subprog->nr_reloc);
6701 for (i = main_prog->nr_reloc; i < new_cnt; i++)
6702 relos[i].insn_idx += subprog->sub_insn_off;
6706 main_prog->reloc_desc = relos;
6707 main_prog->nr_reloc = new_cnt;
6713 size_t size = sizeof(main_prog->subprogs[0]);
6714 int cnt = main_prog->subprog_cnt;
6717 tmp = libbpf_reallocarray(main_prog->subprogs, cnt + 1, size);
6719 return -ENOMEM;
6721 main_prog->subprogs = tmp;
6722 main_prog->subprogs[cnt].sec_insn_off = subprog->sec_insn_off;
6723 main_prog->subprogs[cnt].sub_insn_off = subprog->sub_insn_off;
6724 main_prog->subprog_cnt++;
6737 subprog->sub_insn_off = main_prog->insns_cnt;
6739 new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6740 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6742 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6743 return -ENOMEM;
6745 main_prog->insns = insns;
6746 main_prog->insns_cnt = new_cnt;
6748 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6749 subprog->insns_cnt * sizeof(*insns));
6751 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6752 main_prog->name, subprog->insns_cnt, subprog->name);
6762 main_prog->name, errstr(err));
6783 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6784 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6789 if (relo && relo->type == RELO_EXTERN_CALL)
6794 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
6796 prog->name, insn_idx, relo->type);
6797 return -LIBBPF_ERRNO__RELOC;
6800 /* sub-program instruction index is a combination of
6803 * call always has imm = -1, but for static functions
6804 * relocation is against STT_SECTION and insn->imm
6807 * for subprog addr relocation, the relo->sym_off + insn->imm is
6810 if (relo->type == RELO_CALL)
6811 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6813 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;
6820 prog->name, insn_idx);
6821 return -LIBBPF_ERRNO__RELOC;
6826 * offset necessary, insns->imm is relative to
6829 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6832 /* we enforce that sub-programs should be in .text section */
6833 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6835 pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6836 prog->name);
6837 return -LIBBPF_ERRNO__RELOC;
6843 * - append it at the end of main program's instructions blog;
6844 * - process is recursively, while current program is put on hold;
6845 * - if that subprogram calls some other not yet processes
6850 if (subprog->sub_insn_off == 0) {
6859 /* main_prog->insns memory could have been re-allocated, so
6862 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6869 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6872 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6879 * Relocate sub-program calls.
6881 * Algorithm operates as follows. Each entry-point BPF program (referred to as
6882 * main prog) is processed separately. For each subprog (non-entry functions,
6891 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6907 * subprog->sub_insn_off as zero at all times and won't be appended to current
6916 * +--------+ +-------+
6918 * +--+---+ +--+-+-+ +---+--+
6920 * +--+---+ +------+ +---+--+
6923 * +---+-------+ +------+----+
6925 * +-----------+ +-----------+
6930 * +-----------+------+
6932 * +-----------+------+
6937 * +-----------+------+------+
6939 * +-----------+------+------+
6948 * +-----------+------+
6950 * +-----------+------+
6953 * +-----------+------+------+
6955 * +-----------+------+------+
6968 for (i = 0; i < obj->nr_programs; i++) {
6969 subprog = &obj->programs[i];
6973 subprog->sub_insn_off = 0;
6990 for (i = 0; i < obj->nr_programs; i++) {
6991 prog = &obj->programs[i];
6992 zfree(&prog->reloc_desc);
6993 prog->nr_reloc = 0;
7002 if (a->insn_idx != b->insn_idx)
7003 return a->insn_idx < b->insn_idx ? -1 : 1;
7006 if (a->type != b->type)
7007 return a->type < b->type ? -1 : 1;
7016 for (i = 0; i < obj->nr_programs; i++) {
7017 struct bpf_program *p = &obj->programs[i];
7019 if (!p->nr_reloc)
7022 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
7032 if (!obj->btf || !kernel_supports(obj, FEAT_BTF_DECL_TAG))
7035 n = btf__type_cnt(obj->btf);
7040 t = btf_type_by_id(obj->btf, i);
7041 if (!btf_is_decl_tag(t) || btf_decl_tag(t)->component_idx != -1)
7044 name = btf__str_by_offset(obj->btf, t->name_off);
7048 t = btf_type_by_id(obj->btf, t->type);
7051 prog->name);
7052 return -EINVAL;
7054 if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)) != 0)
7060 if (prog->exception_cb_idx >= 0) {
7061 prog->exception_cb_idx = -1;
7068 prog->name);
7069 return -EINVAL;
7072 for (j = 0; j < obj->nr_programs; j++) {
7073 struct bpf_program *subprog = &obj->programs[j];
7077 if (strcmp(name, subprog->name) != 0)
7079 /* Enforce non-hidden, as from verifier point of
7083 if (!subprog->sym_global || subprog->mark_btf_static) {
7084 pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n",
7085 prog->name, subprog->name);
7086 return -EINVAL;
7089 if (prog->exception_cb_idx >= 0) {
7091 prog->name, subprog->name);
7092 return -EINVAL;
7094 prog->exception_cb_idx = j;
7098 if (prog->exception_cb_idx >= 0)
7101 pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name);
7102 return -ENOENT;
7140 /* forward declarations for arch-specific underlying types of bpf_user_pt_regs_t typedef,
7142 * with this approach we don't need any extra arch-specific #ifdef guards
7165 t = btf__type_by_id(btf, t->type);
7167 (prog->type == BPF_PROG_TYPE_KPROBE || prog->type == BPF_PROG_TYPE_PERF_EVENT)) {
7168 tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>";
7174 t = skip_mods_and_typedefs(btf, t->type, NULL);
7181 tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>";
7186 switch (prog->type) {
7206 if (btf_is_int(t) && t->size == 8)
7215 prog->name, subprog_name, arg_idx, ctx_name);
7226 /* caller already validated FUNC -> FUNC_PROTO validity */
7228 fn_proto_t = btf_type_by_id(btf, fn_t->type);
7239 fn_name_off = fn_t->name_off; /* we are about to invalidate fn_t */
7241 orig_proto_id = fn_t->type; /* original FUNC_PROTO ID */
7242 ret_type_id = fn_proto_t->type; /* fn_proto_t will be invalidated */
7248 return -EINVAL;
7256 name_off = p->name_off;
7258 err = btf__add_func_param(btf, "", p->type);
7264 p->name_off = name_off; /* use remembered str offset */
7267 /* clone FUNC now, btf__add_func() enforces non-empty name, so use
7271 fn_id = btf__add_func(btf, prog->name, linkage, fn_proto_id);
7273 return -EINVAL;
7276 fn_t->name_off = fn_name_off; /* reuse original string */
7293 struct btf *btf = obj->btf;
7301 if (!obj->btf_ext || !prog->func_info)
7312 if (global_ctx_map[i].prog_type != prog->type)
7321 orig_ids = calloc(prog->func_info_cnt, sizeof(*orig_ids));
7323 return -ENOMEM;
7324 for (i = 0; i < prog->func_info_cnt; i++) {
7325 func_rec = prog->func_info + prog->func_info_rec_size * i;
7326 orig_ids[i] = func_rec->type_id;
7331 * clone and adjust FUNC -> FUNC_PROTO combo
7338 if (strcmp(btf__str_by_offset(btf, t->name_off), ctx_tag) != 0)
7342 orig_fn_id = t->type;
7347 /* sanity check FUNC -> FUNC_PROTO chain, just in case */
7348 fn_proto_t = btf_type_by_id(btf, fn_t->type);
7354 for (rec_idx = 0; rec_idx < prog->func_info_cnt; rec_idx++) {
7355 if (orig_ids[rec_idx] == t->type) {
7356 func_rec = prog->func_info + prog->func_info_rec_size * rec_idx;
7366 arg_idx = btf_decl_tag(t)->component_idx;
7372 fn_name = btf__str_by_offset(btf, fn_t->name_off) ?: "<anon>";
7373 if (!need_func_arg_type_fixup(btf, prog, fn_name, arg_idx, p->type, ctx_name))
7377 if (func_rec->type_id == orig_fn_id) {
7387 func_rec->type_id = fn_id;
7390 /* create PTR -> STRUCT type chain to mark PTR_TO_CTX argument;
7392 * funcs share the same program type, so need only PTR ->
7399 err = -EINVAL;
7405 tag_id = btf__add_decl_tag(btf, ctx_tag, func_rec->type_id, arg_idx);
7407 err = -EINVAL;
7412 fn_t = btf_type_by_id(btf, func_rec->type_id);
7413 fn_proto_t = btf_type_by_id(btf, fn_t->type);
7417 p->type = ptr_id;
7433 if (obj->btf_ext) {
7436 pr_warn("failed to perform CO-RE relocations: %s\n",
7444 if (obj->arena_map_idx >= 0 && kernel_supports(obj, FEAT_LDIMM64_FULL_RANGE_OFF)) {
7445 struct bpf_map *arena_map = &obj->maps[obj->arena_map_idx];
7447 obj->arena_data_off = bpf_map_mmap_sz(arena_map) -
7448 roundup(obj->arena_data_sz, sysconf(_SC_PAGE_SIZE));
7451 /* Before relocating calls pre-process relocations and mark
7458 for (i = 0; i < obj->nr_programs; i++) {
7459 prog = &obj->programs[i];
7460 for (j = 0; j < prog->nr_reloc; j++) {
7461 struct reloc_desc *relo = &prog->reloc_desc[j];
7462 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
7465 if (relo->type == RELO_SUBPROG_ADDR)
7477 for (i = 0; i < obj->nr_programs; i++) {
7478 prog = &obj->programs[i];
7479 /* sub-program's sub-calls are relocated within the context of
7484 if (!prog->autoload)
7490 prog->name, errstr(err));
7498 if (prog->exception_cb_idx >= 0) {
7499 struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx];
7506 if (subprog->sub_insn_off == 0) {
7516 for (i = 0; i < obj->nr_programs; i++) {
7517 prog = &obj->programs[i];
7520 if (!prog->autoload)
7527 prog->name, errstr(err));
7535 prog->name, errstr(err));
7563 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
7564 return -EINVAL;
7565 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
7567 return -EINVAL;
7569 nrels = shdr->sh_size / shdr->sh_entsize;
7574 return -LIBBPF_ERRNO__FORMAT;
7577 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
7580 i, (size_t)ELF64_R_SYM(rel->r_info));
7581 return -LIBBPF_ERRNO__FORMAT;
7583 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
7585 pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n",
7586 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value,
7587 (size_t)rel->r_offset, sym->st_name, name);
7589 for (j = 0; j < obj->nr_maps; j++) {
7590 map = &obj->maps[j];
7591 if (map->sec_idx != obj->efile.btf_maps_shndx)
7594 vi = btf_var_secinfos(sec) + map->btf_var_idx;
7595 if (vi->offset <= rel->r_offset &&
7596 rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size)
7599 if (j == obj->nr_maps) {
7600 pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n",
7601 i, name, (size_t)rel->r_offset);
7602 return -EINVAL;
7605 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type);
7606 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY;
7609 if (sym->st_shndx != obj->efile.btf_maps_shndx) {
7610 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
7612 return -LIBBPF_ERRNO__RELOC;
7614 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
7615 map->def.key_size != sizeof(int)) {
7616 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
7617 i, map->name, sizeof(int));
7618 return -EINVAL;
7624 return -ESRCH;
7631 return -ESRCH;
7633 if (targ_prog->sec_idx != sym->st_shndx ||
7634 targ_prog->sec_insn_off * 8 != sym->st_value ||
7636 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n",
7638 return -LIBBPF_ERRNO__RELOC;
7641 return -EINVAL;
7644 var = btf__type_by_id(obj->btf, vi->type);
7645 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
7647 return -EINVAL;
7648 member = btf_members(def) + btf_vlen(def) - 1;
7649 mname = btf__name_by_offset(obj->btf, member->name_off);
7651 return -EINVAL;
7653 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
7654 if (rel->r_offset - vi->offset < moff)
7655 return -EINVAL;
7657 moff = rel->r_offset - vi->offset - moff;
7662 return -EINVAL;
7664 if (moff >= map->init_slots_sz) {
7666 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
7668 return -ENOMEM;
7669 map->init_slots = tmp;
7670 memset(map->init_slots + map->init_slots_sz, 0,
7671 (new_sz - map->init_slots_sz) * host_ptr_sz);
7672 map->init_slots_sz = new_sz;
7674 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog;
7677 i, map->name, moff, type, name);
7687 for (i = 0; i < obj->efile.sec_cnt; i++) {
7688 struct elf_sec_desc *sec_desc = &obj->efile.secs[i];
7693 if (sec_desc->sec_type != SEC_RELO)
7696 shdr = sec_desc->shdr;
7697 data = sec_desc->data;
7698 idx = shdr->sh_info;
7700 if (shdr->sh_type != SHT_REL || idx < 0 || idx >= obj->efile.sec_cnt) {
7702 return -LIBBPF_ERRNO__INTERNAL;
7705 if (obj->efile.secs[idx].sec_type == SEC_ST_OPS)
7707 else if (idx == obj->efile.btf_maps_shndx)
7721 if (BPF_CLASS(insn->code) == BPF_JMP &&
7722 BPF_OP(insn->code) == BPF_CALL &&
7723 BPF_SRC(insn->code) == BPF_K &&
7724 insn->src_reg == 0 &&
7725 insn->dst_reg == 0) {
7726 *func_id = insn->imm;
7734 struct bpf_insn *insn = prog->insns;
7738 if (obj->gen_loader)
7741 for (i = 0; i < prog->insns_cnt; i++, insn++) {
7753 insn->imm = BPF_FUNC_probe_read;
7758 insn->imm = BPF_FUNC_probe_read_str;
7770 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
7777 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
7778 opts->expected_attach_type = 0;
7781 opts->prog_flags |= BPF_F_SLEEPABLE;
7783 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
7784 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
7787 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) {
7792 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7793 opts->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7796 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
7800 attach_name = strchr(prog->sec_name, '/');
7811 pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n",
7812 prog->name);
7813 return -EINVAL;
7822 prog->attach_btf_obj_fd = btf_obj_fd;
7823 prog->attach_btf_id = btf_type_id;
7826 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because
7830 opts->attach_btf_obj_fd = btf_obj_fd;
7831 opts->attach_btf_id = btf_type_id;
7847 __u32 log_level = prog->log_level;
7853 switch (prog->type) {
7860 prog->name, prog->sec_name);
7861 return -EINVAL;
7863 if (prog->attach_btf_id == 0) {
7865 prog->name);
7866 return -EINVAL;
7874 return -EINVAL;
7877 prog_name = prog->name;
7878 load_attr.attach_prog_fd = prog->attach_prog_fd;
7879 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
7880 load_attr.attach_btf_id = prog->attach_btf_id;
7882 load_attr.prog_ifindex = prog->prog_ifindex;
7883 load_attr.expected_attach_type = prog->expected_attach_type;
7886 if (obj->btf && btf__fd(obj->btf) >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {
7887 load_attr.prog_btf_fd = btf__fd(obj->btf);
7888 load_attr.func_info = prog->func_info;
7889 load_attr.func_info_rec_size = prog->func_info_rec_size;
7890 load_attr.func_info_cnt = prog->func_info_cnt;
7891 load_attr.line_info = prog->line_info;
7892 load_attr.line_info_rec_size = prog->line_info_rec_size;
7893 load_attr.line_info_cnt = prog->line_info_cnt;
7896 load_attr.prog_flags = prog->prog_flags;
7897 load_attr.fd_array = obj->fd_array;
7899 load_attr.token_fd = obj->token_fd;
7900 if (obj->token_fd)
7904 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
7905 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
7908 prog->name, errstr(err));
7911 insns = prog->insns;
7912 insns_cnt = prog->insns_cnt;
7915 if (obj->gen_loader) {
7916 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
7918 prog - obj->programs);
7919 *prog_fd = -1;
7930 if (prog->log_buf) {
7931 log_buf = prog->log_buf;
7932 log_buf_size = prog->log_size;
7934 } else if (obj->log_buf) {
7935 log_buf = obj->log_buf;
7936 log_buf_size = obj->log_size;
7942 ret = -ENOMEM;
7955 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);
7958 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
7959 prog->name, log_buf);
7962 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) {
7966 for (i = 0; i < obj->nr_maps; i++) {
7967 map = &prog->obj->maps[i];
7968 if (map->libbpf_type != LIBBPF_MAP_RODATA)
7971 if (bpf_prog_bind_map(ret, map->fd, NULL)) {
7973 prog->name, map->real_name, errstr(errno));
7993 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2).
7998 ret = -errno;
8000 /* post-process verifier log to improve error descriptions */
8003 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, errstr(errno));
8007 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
8008 prog->name, log_buf);
8024 p = cur - 1;
8025 while (p - 1 >= buf && *(p - 1) != '\n')
8026 p--;
8034 /* size of the remaining log content to the right from the to-be-replaced part */
8035 size_t rem_sz = (buf + log_sz) - (orig + orig_sz);
8040 * shift log contents by (patch_sz - orig_sz) bytes to the right
8041 * starting from after to-be-replaced part of the log.
8044 * shift log contents by (orig_sz - patch_sz) bytes to the left
8045 * starting from after to-be-replaced part of the log
8054 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1;
8056 } else if (patch_sz - orig_sz > buf_sz - log_sz) {
8058 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz);
8072 /* Expected log for failed and not properly guarded CO-RE relocation:
8073 * line1 -> 123: (85) call unknown#195896080
8074 * line2 -> invalid func unknown#195896080
8075 * line3 -> <anything else or end of buffer>
8078 * instruction index to find corresponding CO-RE relocation and
8080 * failed CO-RE relocation.
8094 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
8100 "%d: <invalid CO-RE relocation>\n"
8101 "failed to resolve CO-RE relocation %s%s\n",
8104 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
8112 * line1 -> 123: (85) call unknown#2001000345
8113 * line2 -> invalid func unknown#2001000345
8114 * line3 -> <anything else or end of buffer>
8117 * "345" in "2001000345" is a map index in obj->maps to fetch map name.
8119 struct bpf_object *obj = prog->obj;
8127 map_idx -= POISON_LDIMM64_MAP_BASE;
8128 if (map_idx < 0 || map_idx >= obj->nr_maps)
8130 map = &obj->maps[map_idx];
8135 insn_idx, map->name);
8137 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
8145 * line1 -> 123: (85) call unknown#2002000345
8146 * line2 -> invalid func unknown#2002000345
8147 * line3 -> <anything else or end of buffer>
8150 * "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name.
8152 struct bpf_object *obj = prog->obj;
8160 ext_idx -= POISON_CALL_KFUNC_BASE;
8161 if (ext_idx < 0 || ext_idx >= obj->nr_extern)
8163 ext = &obj->externs[ext_idx];
8168 insn_idx, ext->name);
8170 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
8185 next_line = buf + log_sz - 1;
8197 /* failed CO-RE relocation case */
8225 struct bpf_object *obj = prog->obj;
8228 for (i = 0; i < prog->nr_reloc; i++) {
8229 struct reloc_desc *relo = &prog->reloc_desc[i];
8230 struct extern_desc *ext = &obj->externs[relo->ext_idx];
8233 switch (relo->type) {
8235 if (ext->type != EXT_KSYM)
8237 kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ?
8239 bpf_gen__record_extern(obj->gen_loader, ext->name,
8240 ext->is_weak, !ext->ksym.type_id,
8241 true, kind, relo->insn_idx);
8244 bpf_gen__record_extern(obj->gen_loader, ext->name,
8245 ext->is_weak, false, false, BTF_KIND_FUNC,
8246 relo->insn_idx);
8250 .insn_off = relo->insn_idx * 8,
8251 .type_id = relo->core_relo->type_id,
8252 .access_str_off = relo->core_relo->access_str_off,
8253 .kind = relo->core_relo->kind,
8256 bpf_gen__record_relo_core(obj->gen_loader, &cr);
8273 for (i = 0; i < obj->nr_programs; i++) {
8274 prog = &obj->programs[i];
8277 if (!prog->autoload) {
8278 pr_debug("prog '%s': skipped loading\n", prog->name);
8281 prog->log_level |= log_level;
8283 if (obj->gen_loader)
8286 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
8287 obj->license, obj->kern_version, &prog->fd);
8289 pr_warn("prog '%s': failed to load: %s\n", prog->name, errstr(err));
8304 for (i = 0; i < obj->nr_programs; i++) {
8305 prog = &obj->programs[i];
8321 prog->sec_def = find_sec_def(prog->sec_name);
8322 if (!prog->sec_def) {
8325 prog->name, prog->sec_name);
8329 prog->type = prog->sec_def->prog_type;
8330 prog->expected_attach_type = prog->sec_def->expected_attach_type;
8335 if (prog->sec_def->prog_setup_fn) {
8336 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
8339 prog->name, errstr(err));
8360 return ERR_PTR(-EINVAL);
8365 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
8369 return ERR_PTR(-EINVAL);
8383 return ERR_PTR(-EINVAL);
8385 return ERR_PTR(-EINVAL);
8395 return ERR_PTR(-ENAMETOOLONG);
8401 obj->log_buf = log_buf;
8402 obj->log_size = log_size;
8403 obj->log_level = log_level;
8406 obj->token_path = strdup(token_path);
8407 if (!obj->token_path) {
8408 err = -ENOMEM;
8416 err = -ENAMETOOLONG;
8419 obj->btf_custom_path = strdup(btf_tmp_path);
8420 if (!obj->btf_custom_path) {
8421 err = -ENOMEM;
8428 obj->kconfig = strdup(kconfig);
8429 if (!obj->kconfig) {
8430 err = -ENOMEM;
8457 return libbpf_err_ptr(-EINVAL);
8474 return libbpf_err_ptr(-EINVAL);
8477 snprintf(tmp_name, sizeof(tmp_name), "%lx-%zx", (unsigned long)obj_buf, obj_buf_sz);
8487 return libbpf_err(-EINVAL);
8489 for (i = 0; i < obj->nr_maps; i++) {
8490 zclose(obj->maps[i].fd);
8491 if (obj->maps[i].st_ops)
8492 zfree(&obj->maps[i].st_ops->kern_vdata);
8495 for (i = 0; i < obj->nr_programs; i++)
8496 bpf_program__unload(&obj->programs[i]);
8509 m->def.map_flags &= ~BPF_F_MMAPABLE;
8527 err = -errno;
8539 err = -EINVAL;
8562 ext = find_extern_by_name_with_len(obj, sym_name, res - sym_name);
8565 if (!ext || ext->type != EXT_KSYM)
8568 t = btf__type_by_id(obj->btf, ext->btf_id);
8572 if (ext->is_set && ext->ksym.addr != sym_addr) {
8574 sym_name, ext->ksym.addr, sym_addr);
8575 return -EINVAL;
8577 if (!ext->is_set) {
8578 ext->is_set = true;
8579 ext->ksym.addr = sym_addr;
8598 btf = obj->btf_vmlinux;
8602 if (id == -ENOENT) {
8607 for (i = 0; i < obj->btf_module_cnt; i++) {
8609 mod_btf = &obj->btf_modules[i];
8610 btf = mod_btf->btf;
8612 if (id != -ENOENT)
8617 return -ESRCH;
8634 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);
8636 if (id == -ESRCH && ext->is_weak)
8639 ext->name);
8644 local_type_id = ext->ksym.type_id;
8648 targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
8649 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
8651 err = bpf_core_types_are_compat(obj->btf, local_type_id,
8657 local_type = btf__type_by_id(obj->btf, local_type_id);
8658 local_name = btf__name_by_offset(obj->btf, local_type->name_off);
8659 targ_name = btf__name_by_offset(btf, targ_type->name_off);
8662 ext->name, local_type_id,
8665 return -EINVAL;
8668 ext->is_set = true;
8669 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
8670 ext->ksym.kernel_btf_id = id;
8672 ext->name, id, btf_kind_str(targ_var), targ_var_name);
8686 local_func_proto_id = ext->ksym.type_id;
8688 kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf,
8691 if (kfunc_id == -ESRCH && ext->is_weak)
8694 ext->name);
8699 kfunc_proto_id = kern_func->type;
8701 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
8704 if (ext->is_weak)
8708 ext->name, local_func_proto_id,
8709 mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id);
8710 return -EINVAL;
8714 if (mod_btf && !mod_btf->fd_array_idx) {
8715 /* insn->off is s16 */
8716 if (obj->fd_array_cnt == INT16_MAX) {
8718 ext->name, mod_btf->fd_array_idx);
8719 return -E2BIG;
8722 if (!obj->fd_array_cnt)
8723 obj->fd_array_cnt = 1;
8725 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),
8726 obj->fd_array_cnt + 1);
8729 mod_btf->fd_array_idx = obj->fd_array_cnt;
8731 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;
8734 ext->is_set = true;
8735 ext->ksym.kernel_btf_id = kfunc_id;
8736 ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0;
8739 * {kernel_btf_id, btf_fd_idx} -> fixup bpf_call.
8740 * {kernel_btf_id, kernel_btf_obj_fd} -> fixup ld_imm64.
8742 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
8744 ext->name, mod_btf ? mod_btf->name : "vmlinux", kfunc_id);
8755 for (i = 0; i < obj->nr_extern; i++) {
8756 ext = &obj->externs[i];
8757 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
8760 if (obj->gen_loader) {
8761 ext->is_set = true;
8762 ext->ksym.kernel_btf_obj_fd = 0;
8763 ext->ksym.kernel_btf_id = 0;
8766 t = btf__type_by_id(obj->btf, ext->btf_id);
8786 if (obj->nr_extern == 0)
8789 if (obj->kconfig_map_idx >= 0)
8790 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
8792 for (i = 0; i < obj->nr_extern; i++) {
8793 ext = &obj->externs[i];
8795 if (ext->type == EXT_KSYM) {
8796 if (ext->ksym.type_id)
8801 } else if (ext->type == EXT_KCFG) {
8802 void *ext_ptr = kcfg_data + ext->kcfg.data_off;
8806 if (str_has_pfx(ext->name, "CONFIG_")) {
8812 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
8815 pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name);
8816 return -EINVAL;
8818 } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) {
8820 } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) {
8822 } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) {
8830 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name);
8831 return -EINVAL;
8838 ext->name, (long long)value);
8840 pr_warn("extern '%s': unrecognized extern kind\n", ext->name);
8841 return -EINVAL;
8847 return -EINVAL;
8849 for (i = 0; i < obj->nr_extern; i++) {
8850 ext = &obj->externs[i];
8851 if (ext->type == EXT_KCFG && !ext->is_set) {
8860 return -EINVAL;
8865 return -EINVAL;
8870 return -EINVAL;
8872 for (i = 0; i < obj->nr_extern; i++) {
8873 ext = &obj->externs[i];
8875 if (!ext->is_set && !ext->is_weak) {
8876 pr_warn("extern '%s' (strong): not resolved\n", ext->name);
8877 return -ESRCH;
8878 } else if (!ext->is_set) {
8880 ext->name);
8893 st_ops = map->st_ops;
8894 type = btf__type_by_id(map->obj->btf, st_ops->type_id);
8896 struct bpf_program *prog = st_ops->progs[i];
8904 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
8914 for (i = 0; i < obj->nr_maps; i++) {
8915 map = &obj->maps[i];
8920 if (!map->autocreate)
8933 /* unpin any maps that were auto-pinned during load */
8934 for (i = 0; i < obj->nr_maps; i++)
8935 if (obj->maps[i].pinned && !obj->maps[i].reused)
8936 bpf_map__unpin(&obj->maps[i], NULL);
8944 zfree(&obj->fd_array);
8947 for (i = 0; i < obj->btf_module_cnt; i++) {
8948 close(obj->btf_modules[i].fd);
8949 btf__free(obj->btf_modules[i].btf);
8950 free(obj->btf_modules[i].name);
8952 obj->btf_module_cnt = 0;
8953 zfree(&obj->btf_modules);
8956 btf__free(obj->btf_vmlinux);
8957 obj->btf_vmlinux = NULL;
8964 if (obj->state >= OBJ_PREPARED) {
8965 pr_warn("object '%s': prepare loading can't be attempted twice\n", obj->name);
8966 return -EINVAL;
8972 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
8976 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
8984 obj->state = OBJ_LOADED;
8988 obj->state = OBJ_PREPARED;
8997 return libbpf_err(-EINVAL);
8999 if (obj->state >= OBJ_LOADED) {
9000 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
9001 return libbpf_err(-EINVAL);
9004 /* Disallow kernel loading programs of non-native endianness but
9005 * permit cross-endian creation of "light skeleton".
9007 if (obj->gen_loader) {
9008 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);
9010 pr_warn("object '%s': loading non-native endianness is unsupported\n", obj->name);
9011 return libbpf_err(-LIBBPF_ERRNO__ENDIAN);
9014 if (obj->state < OBJ_PREPARED) {
9023 if (obj->gen_loader) {
9025 if (obj->btf)
9026 btf__set_fd(obj->btf, -1);
9028 err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
9032 obj->state = OBJ_LOADED; /* doesn't matter if successfully or not */
9037 pr_warn("failed to load object '%s'\n", obj->path);
9061 return -ENOMEM;
9065 err = -errno;
9081 return -EINVAL;
9085 return -ENOMEM;
9090 err = -errno;
9096 err = -EINVAL;
9106 if (prog->fd < 0) {
9107 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
9108 return libbpf_err(-EINVAL);
9119 if (bpf_obj_pin(prog->fd, path)) {
9120 err = -errno;
9121 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, errstr(err));
9125 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
9133 if (prog->fd < 0) {
9134 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
9135 return libbpf_err(-EINVAL);
9144 return libbpf_err(-errno);
9146 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
9156 return libbpf_err(-EINVAL);
9159 if (map->fd < 0) {
9160 pr_warn("map '%s': can't pin BPF map without FD (was it created?)\n", map->name);
9161 return libbpf_err(-EINVAL);
9164 if (map->pin_path) {
9165 if (path && strcmp(path, map->pin_path)) {
9167 bpf_map__name(map), map->pin_path, path);
9168 return libbpf_err(-EINVAL);
9169 } else if (map->pinned) {
9170 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
9171 bpf_map__name(map), map->pin_path);
9178 return libbpf_err(-EINVAL);
9179 } else if (map->pinned) {
9181 return libbpf_err(-EEXIST);
9184 map->pin_path = strdup(path);
9185 if (!map->pin_path) {
9186 err = -errno;
9191 err = make_parent_dir(map->pin_path);
9195 err = check_path(map->pin_path);
9199 if (bpf_obj_pin(map->fd, map->pin_path)) {
9200 err = -errno;
9204 map->pinned = true;
9205 pr_debug("pinned map '%s'\n", map->pin_path);
9220 return libbpf_err(-EINVAL);
9223 if (map->pin_path) {
9224 if (path && strcmp(path, map->pin_path)) {
9226 bpf_map__name(map), map->pin_path, path);
9227 return libbpf_err(-EINVAL);
9229 path = map->pin_path;
9233 return libbpf_err(-EINVAL);
9242 return libbpf_err(-errno);
9244 map->pinned = false;
9257 return libbpf_err(-errno);
9260 free(map->pin_path);
9261 map->pin_path = new;
9270 return map->pin_path;
9275 return map->pinned;
9294 return libbpf_err(-ENOENT);
9296 if (obj->state < OBJ_PREPARED) {
9298 return libbpf_err(-ENOENT);
9305 if (!map->autocreate)
9314 } else if (!map->pin_path) {
9327 if (!map->pin_path)
9342 return libbpf_err(-ENOENT);
9354 } else if (!map->pin_path) {
9373 return libbpf_err(-ENOENT);
9375 if (obj->state < OBJ_LOADED) {
9377 return libbpf_err(-ENOENT);
9381 err = pathname_concat(buf, sizeof(buf), path, prog->name);
9394 if (pathname_concat(buf, sizeof(buf), path, prog->name))
9409 return libbpf_err(-ENOENT);
9414 err = pathname_concat(buf, sizeof(buf), path, prog->name);
9460 if (map->inner_map) {
9461 bpf_map__destroy(map->inner_map);
9462 zfree(&map->inner_map);
9465 zfree(&map->init_slots);
9466 map->init_slots_sz = 0;
9468 if (map->mmaped && map->mmaped != map->obj->arena_data)
9469 munmap(map->mmaped, bpf_map_mmap_sz(map));
9470 map->mmaped = NULL;
9472 if (map->st_ops) {
9473 zfree(&map->st_ops->data);
9474 zfree(&map->st_ops->progs);
9475 zfree(&map->st_ops->kern_func_off);
9476 zfree(&map->st_ops);
9479 zfree(&map->name);
9480 zfree(&map->real_name);
9481 zfree(&map->pin_path);
9483 if (map->fd >= 0)
9484 zclose(map->fd);
9501 usdt_manager_free(obj->usdt_man);
9502 obj->usdt_man = NULL;
9504 bpf_gen__free(obj->gen_loader);
9507 btf__free(obj->btf);
9508 btf__free(obj->btf_vmlinux);
9509 btf_ext__free(obj->btf_ext);
9511 for (i = 0; i < obj->nr_maps; i++)
9512 bpf_map__destroy(&obj->maps[i]);
9514 zfree(&obj->btf_custom_path);
9515 zfree(&obj->kconfig);
9517 for (i = 0; i < obj->nr_extern; i++) {
9518 zfree(&obj->externs[i].name);
9519 zfree(&obj->externs[i].essent_name);
9522 zfree(&obj->externs);
9523 obj->nr_extern = 0;
9525 zfree(&obj->maps);
9526 obj->nr_maps = 0;
9528 if (obj->programs && obj->nr_programs) {
9529 for (i = 0; i < obj->nr_programs; i++)
9530 bpf_program__exit(&obj->programs[i]);
9532 zfree(&obj->programs);
9534 zfree(&obj->feat_cache);
9535 zfree(&obj->token_path);
9536 if (obj->token_fd > 0)
9537 close(obj->token_fd);
9539 zfree(&obj->arena_data);
9541 zfree(&obj->jumptables_data);
9542 obj->jumptables_data_sz = 0;
9544 for (i = 0; i < obj->jumptable_map_cnt; i++)
9545 close(obj->jumptable_maps[i].fd);
9546 zfree(&obj->jumptable_maps);
9553 return obj ? obj->name : libbpf_err_ptr(-EINVAL);
9558 return obj ? obj->kern_version : 0;
9563 return obj->token_fd ?: -1;
9568 return obj ? obj->btf : NULL;
9573 return obj->btf ? btf__fd(obj->btf) : -1;
9578 if (obj->state >= OBJ_LOADED)
9579 return libbpf_err(-EINVAL);
9581 obj->kern_version = kern_version;
9591 return libbpf_err(-EFAULT);
9593 return libbpf_err(-EINVAL);
9596 return libbpf_err(-ENOMEM);
9597 gen->opts = opts;
9598 gen->swapped_endian = !is_native_endianness(obj);
9599 obj->gen_loader = gen;
9607 size_t nr_programs = obj->nr_programs;
9615 return forward ? &obj->programs[0] :
9616 &obj->programs[nr_programs - 1];
9618 if (p->obj != obj) {
9623 idx = (p - obj->programs) + (forward ? 1 : -1);
9624 if (idx >= obj->nr_programs || idx < 0)
9626 return &obj->programs[idx];
9655 prog->prog_ifindex = ifindex;
9660 return prog->name;
9665 return prog->sec_name;
9670 return prog->autoload;
9675 if (prog->obj->state >= OBJ_LOADED)
9676 return libbpf_err(-EINVAL);
9678 prog->autoload = autoload;
9684 return prog->autoattach;
9689 prog->autoattach = autoattach;
9694 return prog->insns;
9699 return prog->insns_cnt;
9707 if (prog->obj->state >= OBJ_LOADED)
9708 return libbpf_err(-EBUSY);
9710 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
9713 pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
9714 return libbpf_err(-ENOMEM);
9718 prog->insns = insns;
9719 prog->insns_cnt = new_insn_cnt;
9726 return libbpf_err(-EINVAL);
9728 if (prog->fd < 0)
9729 return libbpf_err(-ENOENT);
9731 return prog->fd;
9739 return prog->type;
9750 if (prog->obj->state >= OBJ_LOADED)
9751 return libbpf_err(-EBUSY);
9754 if (prog->type == type)
9757 prog->type = type;
9761 * fallback handler, which by definition is program type-agnostic and
9762 * is a catch-all custom handler, optionally set by the application,
9765 if (prog->sec_def != &custom_fallback_def)
9766 prog->sec_def = NULL;
9775 return prog->expected_attach_type;
9781 if (prog->obj->state >= OBJ_LOADED)
9782 return libbpf_err(-EBUSY);
9784 prog->expected_attach_type = type;
9790 return prog->prog_flags;
9795 if (prog->obj->state >= OBJ_LOADED)
9796 return libbpf_err(-EBUSY);
9798 prog->prog_flags = flags;
9804 return prog->log_level;
9809 if (prog->obj->state >= OBJ_LOADED)
9810 return libbpf_err(-EBUSY);
9812 prog->log_level = log_level;
9818 *log_size = prog->log_size;
9819 return prog->log_buf;
9825 return libbpf_err(-EINVAL);
9826 if (prog->log_size > UINT_MAX)
9827 return libbpf_err(-EINVAL);
9828 if (prog->obj->state >= OBJ_LOADED)
9829 return libbpf_err(-EBUSY);
9831 prog->log_buf = log_buf;
9832 prog->log_size = log_size;
9838 if (prog->func_info_rec_size != sizeof(struct bpf_func_info))
9839 return libbpf_err_ptr(-EOPNOTSUPP);
9840 return prog->func_info;
9845 return prog->func_info_cnt;
9850 if (prog->line_info_rec_size != sizeof(struct bpf_line_info))
9851 return libbpf_err_ptr(-EOPNOTSUPP);
9852 return prog->line_info;
9857 return prog->line_info_cnt;
9869 return libbpf_err(-EINVAL);
9872 return libbpf_err(-EINVAL);
9874 obj = prog->obj;
9875 if (obj->state < OBJ_PREPARED)
9876 return libbpf_err(-EINVAL);
9879 * Caller-provided opts take priority; fall back to
9882 attr.attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0) ?: prog->attach_prog_fd;
9883 attr.prog_flags = OPTS_GET(opts, prog_flags, 0) ?: prog->prog_flags;
9884 attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0) ?: prog->prog_ifindex;
9885 attr.kern_version = OPTS_GET(opts, kern_version, 0) ?: obj->kern_version;
9886 attr.fd_array = OPTS_GET(opts, fd_array, NULL) ?: obj->fd_array;
9887 attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0) ?: obj->fd_array_cnt;
9888 attr.token_fd = OPTS_GET(opts, token_fd, 0) ?: obj->token_fd;
9893 if (!prog_btf_fd && obj->btf)
9894 prog_btf_fd = btf__fd(obj->btf);
9900 /* func_info/line_info triples: all-or-nothing from caller */
9906 prog->name);
9907 return libbpf_err(-EINVAL);
9909 attr.func_info = info ?: prog->func_info;
9910 attr.func_info_cnt = info ? info_cnt : prog->func_info_cnt;
9911 attr.func_info_rec_size = info ? info_rec_size : prog->func_info_rec_size;
9918 prog->name);
9919 return libbpf_err(-EINVAL);
9921 attr.line_info = info ?: prog->line_info;
9922 attr.line_info_cnt = info ? info_cnt : prog->line_info_cnt;
9923 attr.line_info_rec_size = info ? info_rec_size : prog->line_info_rec_size;
9926 /* Logging is caller-controlled; no fallback to prog/obj log settings */
9934 * Later override with caller-provided opts.
9936 attr.expected_attach_type = prog->expected_attach_type;
9937 attr.attach_btf_id = prog->attach_btf_id;
9938 attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
9940 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
9941 err = prog->sec_def->prog_prepare_load_fn(prog, &attr, prog->sec_def->cookie);
9946 /* Re-apply caller overrides for output fields */
9959 fd = bpf_prog_load(prog->type, prog->name, obj->license, prog->insns, prog->insns_cnt,
10103 return libbpf_err(-EINVAL);
10106 return libbpf_err(-E2BIG);
10112 return libbpf_err(-ENOMEM);
10118 return libbpf_err(-EBUSY);
10123 sec_def->sec = sec ? strdup(sec) : NULL;
10124 if (sec && !sec_def->sec)
10125 return libbpf_err(-ENOMEM);
10127 sec_def->prog_type = prog_type;
10128 sec_def->expected_attach_type = exp_attach_type;
10129 sec_def->cookie = OPTS_GET(opts, cookie, 0);
10131 sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL);
10132 sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL);
10133 sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL);
10135 sec_def->handler_id = ++last_custom_sec_def_handler_id;
10142 return sec_def->handler_id;
10151 return libbpf_err(-EINVAL);
10165 return libbpf_err(-ENOENT);
10169 custom_sec_defs[i - 1] = custom_sec_defs[i];
10170 custom_sec_def_cnt--;
10186 size_t len = strlen(sec_def->sec);
10189 if (sec_def->sec[len - 1] == '/') {
10190 if (str_has_pfx(sec_name, sec_def->sec))
10196 * well-formed SEC("type/extras") with proper '/' separator
10198 if (sec_def->sec[len - 1] == '+') {
10199 len--;
10201 if (strncmp(sec_name, sec_def->sec, len) != 0)
10209 return strcmp(sec_name, sec_def->sec) == 0;
10254 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
10257 if (!(sec_def->cookie & SEC_ATTACHABLE))
10279 return libbpf_err(-EINVAL);
10283 *prog_type = sec_def->prog_type;
10284 *expected_attach_type = sec_def->expected_attach_type;
10295 return libbpf_err(-ESRCH);
10337 for (i = 0; i < obj->nr_maps; i++) {
10338 map = &obj->maps[i];
10341 if (map->sec_idx == sec_idx &&
10342 map->sec_offset <= offset &&
10343 offset - map->sec_offset < map->def.value_size)
10350 /* Collect the reloc from ELF, populate the st_ops->progs[], and update
10351 * st_ops->data for shadow type.
10370 btf = obj->btf;
10371 nrels = shdr->sh_size / shdr->sh_entsize;
10376 return -LIBBPF_ERRNO__FORMAT;
10379 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
10382 (size_t)ELF64_R_SYM(rel->r_info));
10383 return -LIBBPF_ERRNO__FORMAT;
10386 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
10387 map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset);
10389 pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n",
10390 (size_t)rel->r_offset);
10391 return -EINVAL;
10394 moff = rel->r_offset - map->sec_offset;
10395 shdr_idx = sym->st_shndx;
10396 st_ops = map->st_ops;
10397 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
10398 map->name,
10399 (long long)(rel->r_info >> 32),
10400 (long long)sym->st_value,
10401 shdr_idx, (size_t)rel->r_offset,
10402 map->sec_offset, sym->st_name, name);
10405 pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n",
10406 map->name, (size_t)rel->r_offset, shdr_idx);
10407 return -LIBBPF_ERRNO__RELOC;
10409 if (sym->st_value % BPF_INSN_SZ) {
10411 map->name, (unsigned long long)sym->st_value);
10412 return -LIBBPF_ERRNO__FORMAT;
10414 insn_idx = sym->st_value / BPF_INSN_SZ;
10416 type = btf__type_by_id(btf, st_ops->type_id);
10420 map->name, moff);
10421 return -EINVAL;
10423 member_idx = member - btf_members(type);
10424 name = btf__name_by_offset(btf, member->name_off);
10426 if (!resolve_func_ptr(btf, member->type, NULL)) {
10428 map->name, name);
10429 return -EINVAL;
10435 map->name, shdr_idx, name);
10436 return -EINVAL;
10440 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
10442 map->name, prog->name);
10443 return -EINVAL;
10446 st_ops->progs[member_idx] = prog;
10448 /* st_ops->data will be exposed to users, being returned by
10454 *((struct bpf_program **)(st_ops->data + moff)) = prog;
10501 return -ENAMETOOLONG;
10551 err = -EINVAL;
10582 mod_len = fn_name - mod_name;
10587 ret = find_attach_btf_id(obj->btf_vmlinux,
10595 if (ret != -ENOENT)
10603 for (i = 0; i < obj->btf_module_cnt; i++) {
10604 const struct module_btf *mod = &obj->btf_modules[i];
10606 if (mod_name && strncmp(mod->name, mod_name, mod_len) != 0)
10609 ret = find_attach_btf_id(mod->btf,
10613 *btf_obj_fd = mod->fd;
10617 if (ret == -ENOENT)
10623 return -ESRCH;
10629 enum bpf_attach_type attach_type = prog->expected_attach_type;
10630 __u32 attach_prog_fd = prog->attach_prog_fd;
10634 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {
10636 pr_warn("prog '%s': attach program FD is not set\n", prog->name);
10637 return -EINVAL;
10639 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd, prog->obj->token_fd);
10642 prog->name, attach_prog_fd, attach_name, errstr(err));
10651 if (prog->obj->gen_loader) {
10652 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
10656 err = find_kernel_btf_id(prog->obj, attach_name,
10662 prog->name, attach_name, errstr(err));
10675 return libbpf_err(-EINVAL);
10686 return libbpf_err(-EINVAL);
10689 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
10690 return libbpf_err(-EINVAL);
10691 if (!(sec_def->cookie & SEC_ATTACHABLE))
10692 return libbpf_err(-EINVAL);
10694 *attach_type = sec_def->expected_attach_type;
10701 return libbpf_err(-EINVAL);
10703 return -1;
10704 return map->fd;
10710 * their user-visible name differs from kernel-visible name. Users see
10715 if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0)
10717 if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0)
10728 return map->real_name;
10730 return map->name;
10735 return map->def.type;
10741 return libbpf_err(-EBUSY);
10742 map->def.type = type;
10748 return map->def.map_flags;
10754 return libbpf_err(-EBUSY);
10755 map->def.map_flags = flags;
10761 return map->map_extra;
10767 return libbpf_err(-EBUSY);
10768 map->map_extra = map_extra;
10774 return map->numa_node;
10780 return libbpf_err(-EBUSY);
10781 map->numa_node = numa_node;
10787 return map->def.key_size;
10793 return libbpf_err(-EBUSY);
10794 map->def.key_size = size;
10800 return map->def.value_size;
10814 btf = bpf_object__btf(map->obj);
10816 return -ENOENT;
10823 return -EINVAL;
10831 return -EINVAL;
10835 var = &btf_var_secinfos(datasec_type)[vlen - 1];
10836 var_type = btf_type_by_id(btf, var->type);
10837 array_type = skip_mods_and_typedefs(btf, var_type->type, NULL);
10841 return -EINVAL;
10844 /* verify request size aligns with array */
10846 element_sz = btf__resolve_size(btf, array->type);
10847 if (element_sz <= 0 || (size - var->offset) % element_sz != 0) {
10850 return -EINVAL;
10854 nr_elements = (size - var->offset) / element_sz;
10855 new_array_id = btf__add_array(btf, array->index_type, array->type, nr_elements);
10862 datasec_type = btf_type_by_id(btf, map->btf_value_type_id);
10863 var = &btf_var_secinfos(datasec_type)[vlen - 1];
10864 var_type = btf_type_by_id(btf, var->type);
10867 datasec_type->size = size;
10868 var->size = size - var->offset;
10869 var_type->type = new_array_id;
10877 return libbpf_err(-EBUSY);
10879 if (map->mmaped) {
10883 if (map->def.type != BPF_MAP_TYPE_ARRAY)
10884 return libbpf_err(-EOPNOTSUPP);
10887 mmap_new_sz = array_map_mmap_sz(size, map->def.max_entries);
10890 pr_warn("map '%s': failed to resize memory-mapped region: %s\n",
10895 if (err && err != -ENOENT) {
10898 map->btf_value_type_id = 0;
10899 map->btf_key_type_id = 0;
10903 map->def.value_size = size;
10909 return map ? map->btf_key_type_id : 0;
10914 return map ? map->btf_value_type_id : 0;
10923 return libbpf_err(-EBUSY);
10925 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG)
10926 return libbpf_err(-EINVAL);
10928 if (map->def.type == BPF_MAP_TYPE_ARENA)
10929 actual_sz = map->obj->arena_data_sz;
10931 actual_sz = map->def.value_size;
10933 return libbpf_err(-EINVAL);
10935 memcpy(map->mmaped, data, size);
10943 *psize = map->def.value_size;
10944 return map->st_ops->data;
10947 if (!map->mmaped)
10950 if (map->def.type == BPF_MAP_TYPE_ARENA)
10951 *psize = map->obj->arena_data_sz;
10953 *psize = map->def.value_size;
10955 return map->mmaped;
10960 return map->libbpf_type != LIBBPF_MAP_UNSPEC;
10965 return map->map_ifindex;
10971 return libbpf_err(-EBUSY);
10972 map->map_ifindex = ifindex;
10978 if (!bpf_map_type__is_map_in_map(map->def.type)) {
10980 return libbpf_err(-EINVAL);
10982 if (map->inner_map_fd != -1) {
10984 return libbpf_err(-EINVAL);
10986 if (map->inner_map) {
10987 bpf_map__destroy(map->inner_map);
10988 zfree(&map->inner_map);
10990 map->inner_map_fd = fd;
10998 return libbpf_err(-EINVAL);
11001 if (map->obj != prog->obj) {
11003 return libbpf_err(-EINVAL);
11006 map->excl_prog = prog;
11012 return map->excl_prog;
11021 if (!obj || !obj->maps)
11024 s = obj->maps;
11025 e = obj->maps + obj->nr_maps;
11033 idx = (m - obj->maps) + i;
11034 if (idx >= obj->nr_maps || idx < 0)
11036 return &obj->maps[idx];
11043 return obj->maps;
11052 if (!obj->nr_maps)
11054 return obj->maps + obj->nr_maps - 1;
11057 return __bpf_map__iter(next, obj, -1);
11071 if (pos->real_name && strcmp(pos->real_name, name) == 0)
11077 if (strcmp(pos->real_name, name) == 0)
11081 if (strcmp(pos->name, name) == 0)
11097 return -ENOENT;
11099 if (map->def.key_size != key_sz) {
11101 map->name, key_sz, map->def.key_size);
11102 return -EINVAL;
11105 if (map->fd < 0) {
11106 pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
11107 return -EINVAL;
11113 switch (map->def.type) {
11119 size_t elem_sz = roundup(map->def.value_size, 8);
11124 map->name);
11125 return -EINVAL;
11127 if (map->def.value_size != value_sz) {
11129 map->name, value_sz, map->def.value_size);
11130 return -EINVAL;
11136 pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n",
11137 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);
11138 return -EINVAL;
11143 if (map->def.value_size != value_sz) {
11145 map->name, value_sz, map->def.value_size);
11146 return -EINVAL;
11163 return bpf_map_lookup_elem_flags(map->fd, key, value, flags);
11176 return bpf_map_update_elem(map->fd, key, value, flags);
11188 return bpf_map_delete_elem_flags(map->fd, key, flags);
11201 return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags);
11213 return bpf_map_get_next_key(map->fd, cur_key, next_key);
11222 errno = -PTR_ERR(ptr);
11229 return -errno;
11240 prog->name);
11241 return libbpf_err(-EINVAL);
11260 link->disconnected = true;
11270 if (!link->disconnected && link->detach)
11271 err = link->detach(link);
11272 if (link->pin_path)
11273 free(link->pin_path);
11274 if (link->dealloc)
11275 link->dealloc(link);
11284 return link->fd;
11289 return link->pin_path;
11294 return libbpf_err_errno(close(link->fd));
11304 fd = -errno;
11312 return libbpf_err_ptr(-ENOMEM);
11314 link->detach = &bpf_link__detach_fd;
11315 link->fd = fd;
11317 link->pin_path = strdup(path);
11318 if (!link->pin_path) {
11320 return libbpf_err_ptr(-ENOMEM);
11328 return bpf_link_detach(link->fd) ? -errno : 0;
11335 if (link->pin_path)
11336 return libbpf_err(-EBUSY);
11344 link->pin_path = strdup(path);
11345 if (!link->pin_path)
11346 return libbpf_err(-ENOMEM);
11348 if (bpf_obj_pin(link->fd, link->pin_path)) {
11349 err = -errno;
11350 zfree(&link->pin_path);
11354 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
11362 if (!link->pin_path)
11363 return libbpf_err(-EINVAL);
11365 err = unlink(link->pin_path);
11367 return -errno;
11369 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
11370 zfree(&link->pin_path);
11391 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0)
11392 err = -errno;
11394 if (perf_link->perf_event_fd != link->fd)
11395 close(perf_link->perf_event_fd);
11396 close(link->fd);
11399 if (perf_link->legacy_probe_name) {
11400 if (perf_link->legacy_is_kprobe) {
11401 err = remove_kprobe_event_legacy(perf_link->legacy_probe_name,
11402 perf_link->legacy_is_retprobe);
11404 err = remove_uprobe_event_legacy(perf_link->legacy_probe_name,
11405 perf_link->legacy_is_retprobe);
11416 free(perf_link->legacy_probe_name);
11424 int prog_fd, link_fd = -1, err;
11428 return libbpf_err_ptr(-EINVAL);
11432 prog->name, pfd);
11433 return libbpf_err_ptr(-EINVAL);
11438 prog->name);
11439 return libbpf_err_ptr(-EINVAL);
11444 return libbpf_err_ptr(-ENOMEM);
11445 link->link.detach = &bpf_link_perf_detach;
11446 link->link.dealloc = &bpf_link_perf_dealloc;
11447 link->perf_event_fd = pfd;
11450 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) {
11456 err = -errno;
11458 prog->name, pfd, errstr(err));
11461 link->link.fd = link_fd;
11464 pr_warn("prog '%s': user context value is not supported\n", prog->name);
11465 err = -EOPNOTSUPP;
11470 err = -errno;
11472 prog->name, pfd, errstr(err));
11473 if (err == -EPROTO)
11475 prog->name, pfd);
11478 link->link.fd = pfd;
11483 err = -errno;
11485 prog->name, pfd, errstr(err));
11490 return &link->link;
11504 * this function is expected to parse integer in the range of [0, 2^31-1] from
11515 err = -errno;
11521 err = err == EOF ? -EIO : -errno;
11569 return -EINVAL;
11601 pid < 0 ? -1 : pid /* pid */,
11602 pid == -1 ? 0 : -1 /* cpu */,
11603 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11604 return pfd >= 0 ? pfd : -errno;
11618 return -EINVAL;
11622 return -errno;
11625 err = -errno;
11636 static int has_debugfs = -1;
11698 return append_to_file(tracefs_kprobe_events(), "-:%s/%s",
11741 pid < 0 ? -1 : pid, /* pid */
11742 pid == -1 ? 0 : -1, /* cpu */
11743 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11745 err = -errno;
11828 return libbpf_err_ptr(-EINVAL);
11843 return libbpf_err_ptr(-ENOTSUP);
11847 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
11848 return libbpf_err_ptr(-ENOTSUP);
11853 return libbpf_err_ptr(-EINVAL);
11856 return libbpf_err_ptr(-EOPNOTSUPP);
11861 -1 /* pid */, 0 /* ref_ctr_off */);
11870 return libbpf_err_ptr(-ENOMEM);
11873 offset, -1 /* pid */);
11878 prog->name, retprobe ? "kretprobe" : "kprobe",
11888 prog->name, retprobe ? "kretprobe" : "kprobe",
11896 perf_link->legacy_probe_name = legacy_probe;
11897 perf_link->legacy_is_kprobe = true;
11898 perf_link->legacy_is_retprobe = retprobe;
11930 return libbpf_err_ptr(-EINVAL);
11932 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
11999 struct kprobe_multi_resolve *res = data->res;
12002 if (!glob_match(sym_name, res->pattern))
12005 if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) {
12010 * make multi-kprobe usability a bit better: if no match is
12025 snprintf(sym_trim, sizeof(sym_trim), "%.*s", (int)(sym_sfx - sym_name), sym_name);
12026 if (!bsearch(&psym_trim, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp))
12030 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1);
12034 res->addrs[res->cnt++] = (unsigned long)sym_addr;
12050 err = -errno;
12064 err = -EINVAL;
12068 if (!glob_match(sym_name, res->pattern))
12077 err = -errno;
12086 err = -ENOENT;
12098 if (res->cnt == 0)
12099 err = -ENOENT;
12112 return access(tracefs_available_filter_functions_addrs(), R_OK) != -1;
12125 err = -errno;
12138 err = -EINVAL;
12142 if (!glob_match(sym_name, res->pattern))
12145 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap,
12146 sizeof(*res->addrs), res->cnt + 1);
12150 res->addrs[res->cnt++] = (unsigned long)sym_addr;
12153 if (res->cnt == 0)
12154 err = -ENOENT;
12180 return libbpf_err_ptr(-EINVAL);
12185 prog->name);
12186 return libbpf_err_ptr(-EINVAL);
12196 return libbpf_err_ptr(-EINVAL);
12198 return libbpf_err_ptr(-EINVAL);
12200 return libbpf_err_ptr(-EINVAL);
12202 return libbpf_err_ptr(-EINVAL);
12204 return libbpf_err_ptr(-EINVAL);
12225 prog->name, pattern, res.cnt);
12226 err = -EINVAL;
12238 return libbpf_err_ptr(-EINVAL);
12250 err = -ENOMEM;
12253 link->detach = &bpf_link__detach_fd;
12257 err = -errno;
12264 if (err == -ESRCH)
12265 err = -ENOENT;
12267 prog->name, errstr(err));
12270 link->fd = link_fd;
12290 /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */
12291 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)
12294 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");
12296 func_name = prog->sec_name + sizeof("kretprobe/") - 1;
12298 func_name = prog->sec_name + sizeof("kprobe/") - 1;
12300 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);
12303 return -EINVAL;
12308 return -EINVAL;
12324 /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */
12325 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)
12328 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
12330 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
12332 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
12335 return *link ? 0 : -errno;
12347 /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */
12348 if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||
12349 strcmp(prog->sec_name, "kretprobe.multi") == 0)
12352 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");
12354 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;
12356 spec = prog->sec_name + sizeof("kprobe.multi/") - 1;
12358 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
12361 return -EINVAL;
12379 /* no auto-attach for SEC("kprobe.session") */
12380 if (strcmp(prog->sec_name, "kprobe.session") == 0)
12383 spec = prog->sec_name + sizeof("kprobe.session/") - 1;
12384 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
12387 return -EINVAL;
12392 return *link ? 0 : -errno;
12399 int n, ret = -EINVAL;
12403 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
12407 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
12414 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);
12418 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
12419 prog->sec_name);
12439 return append_to_file(tracefs_uprobe_events(), "-:%s/%s",
12480 pid < 0 ? -1 : pid, /* pid */
12481 pid == -1 ? 0 : -1, /* cpu */
12482 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
12484 err = -errno;
12532 ret = -LIBBPF_ERRNO__FORMAT;
12539 elf_errmsg(-1));
12540 ret = -LIBBPF_ERRNO__LIBELF;
12567 return "/lib/x86_64-linux-gnu";
12569 return "/lib/i386-linux-gnu";
12571 return "/lib/s390x-linux-gnu";
12573 return "/lib/arm-linux-gnueabi";
12575 return "/lib/arm-linux-gnueabihf";
12577 return "/lib/aarch64-linux-gnu";
12579 return "/lib/mips64el-linux-gnuabi64";
12581 return "/lib/mipsel-linux-gnu";
12583 return "/lib/powerpc64le-linux-gnu";
12585 return "/lib/sparc64-linux-gnu";
12587 return "/lib/riscv64-linux-gnu";
12622 seg_len = next_path ? next_path - s : strlen(s);
12633 return -ENOENT;
12656 return libbpf_err_ptr(-EINVAL);
12661 prog->name);
12662 return libbpf_err_ptr(-EINVAL);
12681 * - syms and offsets are mutually exclusive
12682 * - ref_ctr_offsets and cookies are optional
12688 return libbpf_err_ptr(-EINVAL);
12690 return libbpf_err_ptr(-EINVAL);
12694 return libbpf_err_ptr(-EINVAL);
12697 return libbpf_err_ptr(-EINVAL);
12701 return libbpf_err_ptr(-EINVAL);
12708 prog->name, path, errstr(err));
12742 err = -ENOMEM;
12745 link->detach = &bpf_link__detach_fd;
12749 err = -errno;
12750 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",
12751 prog->name, errstr(err));
12754 link->fd = link_fd;
12781 return libbpf_err_ptr(-EINVAL);
12789 return libbpf_err_ptr(-EINVAL);
12796 min(sizeof(full_path), (size_t)(archive_sep - binary_path + 1)));
12803 prog->name, binary_path, errstr(err));
12832 return libbpf_err_ptr(-ENOTSUP);
12836 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
12837 return libbpf_err_ptr(-ENOTSUP);
12842 return libbpf_err_ptr(-EINVAL);
12852 return libbpf_err_ptr(-EINVAL);
12860 return libbpf_err_ptr(-ENOMEM);
12868 prog->name, retprobe ? "uretprobe" : "uprobe",
12879 prog->name, retprobe ? "uretprobe" : "uprobe",
12887 perf_link->legacy_probe_name = legacy_probe;
12888 perf_link->legacy_is_kprobe = false;
12889 perf_link->legacy_is_retprobe = retprobe;
12901 /* Format of u[ret]probe section definition supporting auto-attach:
12908 * specified (and auto-attach is not possible) or the above format is specified for
12909 * auto-attach.
12915 int n, c, ret = -EINVAL;
12920 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
12924 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
12929 prog->name, prog->sec_name);
12947 prog->name);
12951 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);
12955 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
12956 prog->sec_name);
12982 struct bpf_object *obj = prog->obj;
12988 return libbpf_err_ptr(-EINVAL);
12992 prog->name);
12993 return libbpf_err_ptr(-EINVAL);
12997 return libbpf_err_ptr(-EINVAL);
13003 prog->name, binary_path, errstr(err));
13012 if (IS_ERR(obj->usdt_man))
13013 return libbpf_ptr(obj->usdt_man);
13014 if (!obj->usdt_man) {
13015 obj->usdt_man = usdt_manager_new(obj);
13016 if (IS_ERR(obj->usdt_man))
13017 return libbpf_ptr(obj->usdt_man);
13021 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
13037 /* no auto-attach for just SEC("usdt") */
13046 err = -EINVAL;
13048 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path,
13067 return -errno;
13071 return -E2BIG;
13096 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
13097 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
13099 err = -errno;
13118 return libbpf_err_ptr(-EINVAL);
13125 prog->name, tp_category, tp_name,
13134 prog->name, tp_category, tp_name,
13154 /* no auto-attach for SEC("tp") or SEC("tracepoint") */
13155 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0)
13158 sec_name = strdup(prog->sec_name);
13160 return -ENOMEM;
13163 if (str_has_pfx(prog->sec_name, "tp/"))
13164 tp_cat = sec_name + sizeof("tp/") - 1;
13166 tp_cat = sec_name + sizeof("tracepoint/") - 1;
13170 return -EINVAL;
13190 return libbpf_err_ptr(-EINVAL);
13194 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13195 return libbpf_err_ptr(-EINVAL);
13200 return libbpf_err_ptr(-ENOMEM);
13201 link->detach = &bpf_link__detach_fd;
13207 pfd = -errno;
13210 prog->name, tp_name, errstr(pfd));
13213 link->fd = pfd;
13239 if (!str_has_pfx(prog->sec_name, prefixes[i]))
13243 /* no auto-attach case of, e.g., SEC("raw_tp") */
13244 if (prog->sec_name[pfx_len] == '\0')
13247 if (prog->sec_name[pfx_len] != '/')
13250 tp_name = prog->sec_name + pfx_len + 1;
13256 prog->name, prog->sec_name);
13257 return -EINVAL;
13273 return libbpf_err_ptr(-EINVAL);
13277 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13278 return libbpf_err_ptr(-EINVAL);
13283 return libbpf_err_ptr(-ENOMEM);
13284 link->detach = &bpf_link__detach_fd;
13290 pfd = -errno;
13293 prog->name, errstr(pfd));
13296 link->fd = pfd;
13339 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13340 return libbpf_err_ptr(-EINVAL);
13345 return libbpf_err_ptr(-ENOMEM);
13346 link->detach = &bpf_link__detach_fd;
13351 link_fd = -errno;
13354 prog->name, target_name,
13358 link->fd = link_fd;
13395 return libbpf_err_ptr(-EINVAL);
13402 prog->name);
13403 return libbpf_err_ptr(-EINVAL);
13423 return libbpf_err_ptr(-EINVAL);
13428 /* validate we don't have unexpected combinations of non-zero fields */
13431 prog->name);
13432 return libbpf_err_ptr(-EINVAL);
13436 prog->name);
13437 return libbpf_err_ptr(-EINVAL);
13458 return libbpf_err_ptr(-EINVAL);
13463 /* validate we don't have unexpected combinations of non-zero fields */
13466 prog->name);
13467 return libbpf_err_ptr(-EINVAL);
13471 prog->name);
13472 return libbpf_err_ptr(-EINVAL);
13491 prog->name);
13492 return libbpf_err_ptr(-EINVAL);
13495 if (prog->type != BPF_PROG_TYPE_EXT) {
13497 prog->name);
13498 return libbpf_err_ptr(-EINVAL);
13504 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd, prog->obj->token_fd);
13530 return libbpf_err_ptr(-EINVAL);
13537 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13538 return libbpf_err_ptr(-EINVAL);
13543 return libbpf_err_ptr(-ENOMEM);
13544 link->detach = &bpf_link__detach_fd;
13549 link_fd = -errno;
13552 prog->name, errstr(link_fd));
13555 link->fd = link_fd;
13573 return libbpf_err_ptr(-EINVAL);
13577 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13578 return libbpf_err_ptr(-EINVAL);
13583 return libbpf_err_ptr(-ENOMEM);
13585 link->detach = &bpf_link__detach_fd;
13594 link_fd = -errno;
13597 prog->name, errstr(link_fd));
13600 link->fd = link_fd;
13610 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
13611 return libbpf_err_ptr(-EOPNOTSUPP);
13615 prog->name);
13616 return libbpf_err_ptr(-EINVAL);
13619 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
13623 /* When calling bpf_program__attach() explicitly, auto-attach support
13629 return libbpf_err_ptr(-EOPNOTSUPP);
13646 if (st_link->map_fd < 0)
13648 return bpf_map_delete_elem(link->fd, &zero);
13650 return close(link->fd);
13660 pr_warn("map '%s': can't attach non-struct_ops map\n", map->name);
13661 return libbpf_err_ptr(-EINVAL);
13664 if (map->fd < 0) {
13665 pr_warn("map '%s': can't attach BPF map without FD (was it created?)\n", map->name);
13666 return libbpf_err_ptr(-EINVAL);
13671 return libbpf_err_ptr(-EINVAL);
13673 /* kern_vdata should be prepared during the loading phase. */
13674 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
13680 if (err && (!(map->def.map_flags & BPF_F_LINK) || err != -EBUSY)) {
13685 link->link.detach = bpf_link__detach_struct_ops;
13687 if (!(map->def.map_flags & BPF_F_LINK)) {
13689 link->link.fd = map->fd;
13690 link->map_fd = -1;
13691 return &link->link;
13694 fd = bpf_link_create(map->fd, 0, BPF_STRUCT_OPS, NULL);
13700 link->link.fd = fd;
13701 link->map_fd = map->fd;
13703 return &link->link;
13716 return libbpf_err(-EINVAL);
13718 if (map->fd < 0) {
13719 pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
13720 return libbpf_err(-EINVAL);
13725 if (st_ops_link->map_fd < 0)
13726 return libbpf_err(-EINVAL);
13728 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
13734 if (err && err != -EBUSY)
13737 err = bpf_link_update(link->fd, map->fd, NULL);
13741 st_ops_link->map_fd = map->fd;
13756 __u64 data_tail = header->data_tail;
13763 ehdr = base + (data_tail & (mmap_size - 1));
13764 ehdr_size = ehdr->size;
13768 size_t len_first = base + mmap_size - copy_start;
13769 size_t len_secnd = ehdr_size - len_first;
13803 /* sample_cb and lost_cb are higher-level common-case callbacks */
13842 if (cpu_buf->base &&
13843 munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
13844 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
13845 if (cpu_buf->fd >= 0) {
13846 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
13847 close(cpu_buf->fd);
13849 free(cpu_buf->buf);
13859 if (pb->cpu_bufs) {
13860 for (i = 0; i < pb->cpu_cnt; i++) {
13861 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
13866 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
13869 free(pb->cpu_bufs);
13871 if (pb->epoll_fd >= 0)
13872 close(pb->epoll_fd);
13873 free(pb->events);
13886 return ERR_PTR(-ENOMEM);
13888 cpu_buf->pb = pb;
13889 cpu_buf->cpu = cpu;
13890 cpu_buf->map_key = map_key;
13892 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
13893 -1, PERF_FLAG_FD_CLOEXEC);
13894 if (cpu_buf->fd < 0) {
13895 err = -errno;
13901 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
13903 cpu_buf->fd, 0);
13904 if (cpu_buf->base == MAP_FAILED) {
13905 cpu_buf->base = NULL;
13906 err = -errno;
13912 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
13913 err = -errno;
13941 return libbpf_err_ptr(-EINVAL);
13970 return libbpf_err_ptr(-EINVAL);
13973 return libbpf_err_ptr(-EINVAL);
13995 if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) {
13998 return ERR_PTR(-EINVAL);
14001 /* best-effort sanity checks */
14006 err = -errno;
14008 * -EBADFD, -EFAULT, or -E2BIG on real error
14010 if (err != -EINVAL) {
14021 return ERR_PTR(-EINVAL);
14027 return ERR_PTR(-ENOMEM);
14029 pb->event_cb = p->event_cb;
14030 pb->sample_cb = p->sample_cb;
14031 pb->lost_cb = p->lost_cb;
14032 pb->ctx = p->ctx;
14034 pb->page_size = getpagesize();
14035 pb->mmap_size = pb->page_size * page_cnt;
14036 pb->map_fd = map_fd;
14038 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
14039 if (pb->epoll_fd < 0) {
14040 err = -errno;
14046 if (p->cpu_cnt > 0) {
14047 pb->cpu_cnt = p->cpu_cnt;
14049 pb->cpu_cnt = libbpf_num_possible_cpus();
14050 if (pb->cpu_cnt < 0) {
14051 err = pb->cpu_cnt;
14054 if (map.max_entries && map.max_entries < pb->cpu_cnt)
14055 pb->cpu_cnt = map.max_entries;
14058 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
14059 if (!pb->events) {
14060 err = -ENOMEM;
14064 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
14065 if (!pb->cpu_bufs) {
14066 err = -ENOMEM;
14077 for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
14081 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
14082 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
14087 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
14090 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
14096 pb->cpu_bufs[j] = cpu_buf;
14098 err = bpf_map_update_elem(pb->map_fd, &map_key,
14099 &cpu_buf->fd, 0);
14101 err = -errno;
14102 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
14103 cpu, map_key, cpu_buf->fd,
14108 pb->events[j].events = EPOLLIN;
14109 pb->events[j].data.ptr = cpu_buf;
14110 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
14111 &pb->events[j]) < 0) {
14112 err = -errno;
14114 cpu, cpu_buf->fd,
14120 pb->cpu_cnt = j;
14149 struct perf_buffer *pb = cpu_buf->pb;
14153 if (pb->event_cb)
14154 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
14156 switch (e->type) {
14160 if (pb->sample_cb)
14161 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
14167 if (pb->lost_cb)
14168 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
14172 pr_warn("unknown perf sample type %d\n", e->type);
14183 ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size,
14184 pb->page_size, &cpu_buf->buf,
14185 &cpu_buf->buf_size,
14194 return pb->epoll_fd;
14201 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
14203 return -errno;
14206 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
14222 return pb->cpu_cnt;
14234 if (buf_idx >= pb->cpu_cnt)
14235 return libbpf_err(-EINVAL);
14237 cpu_buf = pb->cpu_bufs[buf_idx];
14239 return libbpf_err(-ENOENT);
14241 return cpu_buf->fd;
14248 if (buf_idx >= pb->cpu_cnt)
14249 return libbpf_err(-EINVAL);
14251 cpu_buf = pb->cpu_bufs[buf_idx];
14253 return libbpf_err(-ENOENT);
14255 *buf = cpu_buf->base;
14256 *buf_size = pb->mmap_size;
14265 * - 0 on success;
14266 * - <0 on failure.
14272 if (buf_idx >= pb->cpu_cnt)
14273 return libbpf_err(-EINVAL);
14275 cpu_buf = pb->cpu_bufs[buf_idx];
14277 return libbpf_err(-ENOENT);
14286 for (i = 0; i < pb->cpu_cnt; i++) {
14287 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
14309 return libbpf_err(-EINVAL);
14311 if (prog->obj->state >= OBJ_LOADED)
14312 return libbpf_err(-EINVAL);
14316 * the normal object/program load phase.
14318 prog->attach_prog_fd = attach_prog_fd;
14324 attach_prog_fd, prog->obj->token_fd);
14329 return libbpf_err(-EINVAL);
14332 err = bpf_object__load_vmlinux_btf(prog->obj, true);
14335 err = find_kernel_btf_id(prog->obj, attach_func_name,
14336 prog->expected_attach_type,
14342 prog->attach_btf_id = btf_id;
14343 prog->attach_btf_obj_fd = btf_obj_fd;
14344 prog->attach_prog_fd = attach_prog_fd;
14356 prog->name);
14357 return libbpf_err(-EINVAL);
14360 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) {
14361 pr_warn("prog '%s': can't associate struct_ops program\n", prog->name);
14362 return libbpf_err(-EINVAL);
14367 pr_warn("map '%s': can't associate BPF map without FD (was it created?)\n", map->name);
14368 return libbpf_err(-EINVAL);
14372 pr_warn("map '%s': can't associate non-struct_ops map\n", map->name);
14373 return libbpf_err(-EINVAL);
14381 int err = 0, n, len, start, end = -1;
14387 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
14393 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
14396 err = -EINVAL;
14404 err = -EINVAL;
14409 err = -ENOMEM;
14413 memset(tmp + *mask_sz, 0, start - *mask_sz);
14414 memset(tmp + start, 1, end - start + 1);
14420 return -EINVAL;
14436 err = -errno;
14443 err = len ? -errno : -EINVAL;
14449 return -E2BIG;
14490 struct bpf_map **map = map_skel->map;
14491 const char *name = map_skel->name;
14492 void **mmaped = map_skel->mmaped;
14497 return -ESRCH;
14500 /* externs shouldn't be pre-setup from user code */
14501 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
14502 *mmaped = (*map)->mmaped;
14515 struct bpf_program **prog = prog_skel->prog;
14516 const char *name = prog_skel->name;
14521 return -ESRCH;
14533 obj = bpf_object_open(NULL, s->data, s->data_sz, s->name, opts);
14537 s->name, errstr(err));
14541 *s->obj = obj;
14542 err = populate_skeleton_maps(obj, s->maps, s->map_cnt, s->map_skel_sz);
14544 pr_warn("failed to populate skeleton maps for '%s': %s\n", s->name, errstr(err));
14548 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt, s->prog_skel_sz);
14550 pr_warn("failed to populate skeleton progs for '%s': %s\n", s->name, errstr(err));
14568 if (!s->obj)
14569 return libbpf_err(-EINVAL);
14571 btf = bpf_object__btf(s->obj);
14574 bpf_object__name(s->obj));
14575 return libbpf_err(-errno);
14578 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt, s->map_skel_sz);
14584 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt, s->prog_skel_sz);
14590 for (var_idx = 0; var_idx < s->var_cnt; var_idx++) {
14591 var_skel = (void *)s->vars + var_idx * s->var_skel_sz;
14592 map = *var_skel->map;
14600 return libbpf_err(-EINVAL);
14606 var_type = btf__type_by_id(btf, var->type);
14607 var_name = btf__name_by_offset(btf, var_type->name_off);
14608 if (strcmp(var_name, var_skel->name) == 0) {
14609 *var_skel->addr = map->mmaped + var->offset;
14621 free(s->maps);
14622 free(s->progs);
14623 free(s->vars);
14631 err = bpf_object__load(*s->obj);
14633 pr_warn("failed to load BPF skeleton '%s': %s\n", s->name, errstr(err));
14637 for (i = 0; i < s->map_cnt; i++) {
14638 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;
14639 struct bpf_map *map = *map_skel->map;
14641 if (!map_skel->mmaped)
14644 if (map->def.type == BPF_MAP_TYPE_ARENA)
14645 *map_skel->mmaped = map->mmaped + map->obj->arena_data_off;
14647 *map_skel->mmaped = map->mmaped;
14657 for (i = 0; i < s->prog_cnt; i++) {
14658 struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz;
14659 struct bpf_program *prog = *prog_skel->prog;
14660 struct bpf_link **link = prog_skel->link;
14662 if (!prog->autoload || !prog->autoattach)
14665 /* auto-attaching not supported for this program */
14666 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
14669 /* if user already set the link manually, don't attempt auto-attach */
14673 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
14675 pr_warn("prog '%s': failed to auto-attach: %s\n",
14680 /* It's possible that for some SEC() definitions auto-attach
14685 * auto-attached. But if not, it shouldn't trigger skeleton's
14693 for (i = 0; i < s->map_cnt; i++) {
14694 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;
14695 struct bpf_map *map = *map_skel->map;
14698 if (!map->autocreate || !map->autoattach)
14706 if (s->map_skel_sz < offsetofend(struct bpf_map_skeleton, link)) {
14707 pr_warn("map '%s': BPF skeleton version is old, skipping map auto-attachment...\n",
14712 link = map_skel->link;
14724 err = -errno;
14725 pr_warn("map '%s': failed to auto-attach: %s\n",
14738 for (i = 0; i < s->prog_cnt; i++) {
14739 struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz;
14740 struct bpf_link **link = prog_skel->link;
14746 if (s->map_skel_sz < sizeof(struct bpf_map_skeleton))
14749 for (i = 0; i < s->map_cnt; i++) {
14750 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;
14751 struct bpf_link **link = map_skel->link;
14766 if (s->obj)
14767 bpf_object__close(*s->obj);
14768 free(s->maps);
14769 free(s->progs);