Lines Matching refs:prog
76 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
767 struct bpf_program *prog;
790 void bpf_program__unload(struct bpf_program *prog)
792 if (!prog)
795 zclose(prog->fd);
797 zfree(&prog->func_info);
798 zfree(&prog->line_info);
799 zfree(&prog->subprogs);
802 static void bpf_program__exit(struct bpf_program *prog)
804 if (!prog)
807 bpf_program__unload(prog);
808 zfree(&prog->name);
809 zfree(&prog->sec_name);
810 zfree(&prog->insns);
811 zfree(&prog->reloc_desc);
813 prog->nr_reloc = 0;
814 prog->insns_cnt = 0;
815 prog->sec_idx = -1;
839 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
849 memset(prog, 0, sizeof(*prog));
850 prog->obj = obj;
852 prog->sec_idx = sec_idx;
853 prog->sec_insn_off = sec_off / BPF_INSN_SZ;
854 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
856 prog->insns_cnt = prog->sec_insn_cnt;
858 prog->type = BPF_PROG_TYPE_UNSPEC;
859 prog->fd = -1;
860 prog->exception_cb_idx = -1;
867 prog->autoload = false;
871 prog->autoload = true;
874 prog->autoattach = true;
877 prog->log_level = obj->log_level;
879 prog->sec_name = strdup(sec_name);
880 if (!prog->sec_name)
883 prog->name = strdup(name);
884 if (!prog->name)
887 prog->insns = malloc(insn_data_sz);
888 if (!prog->insns)
890 memcpy(prog->insns, insn_data, insn_data_sz);
894 pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
895 bpf_program__exit(prog);
904 struct bpf_program *prog, *progs;
960 prog = &progs[nr_progs];
962 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
968 prog->sym_global = true;
975 if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
977 prog->mark_btf_static = true;
988 struct bpf_program *prog = obj->programs;
992 for (p = 0; p < obj->nr_programs; p++, prog++) {
993 insn = prog->insns;
994 for (i = 0; i < prog->insns_cnt; i++, insn++)
1107 const struct bpf_program *prog)
1112 if (&obj->programs[i] == prog)
1113 return prog->type == BPF_PROG_TYPE_STRUCT_OPS;
1126 struct bpf_program *prog, *slot_prog;
1134 prog = &obj->programs[i];
1135 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
1149 if (prog != slot_prog)
1158 prog->autoload = should_load;
1211 struct bpf_program *prog;
1276 prog = *(void **)mdata;
1281 if (st_ops->progs[i] && st_ops->progs[i] != prog)
1285 st_ops->progs[i] = prog;
1286 if (!prog)
1289 if (!is_valid_st_ops_program(obj, prog)) {
1310 prog->attach_btf_obj_fd = mod_btf->fd;
1315 if (!prog->attach_btf_id) {
1316 prog->attach_btf_id = kern_type_id;
1317 prog->expected_attach_type = kern_member_idx;
1320 /* struct_ops BPF prog can be re-used between multiple
1325 if (prog->attach_btf_id != kern_type_id) {
1326 pr_warn("struct_ops init_kern %s func ptr %s: invalid reuse of prog %s in sec %s with type %u: attach_btf_id %u != kern_type_id %u\n",
1327 map->name, mname, prog->name, prog->sec_name, prog->type,
1328 prog->attach_btf_id, kern_type_id);
1331 if (prog->expected_attach_type != kern_member_idx) {
1332 pr_warn("struct_ops init_kern %s func ptr %s: invalid reuse of prog %s in sec %s with type %u: expected_attach_type %u != kern_member_idx %u\n",
1333 map->name, mname, prog->name, prog->sec_name, prog->type,
1334 prog->expected_attach_type, kern_member_idx);
1340 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
1341 map->name, mname, prog->name, moff,
2691 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2706 pr_warn("map '%s': should be map-in-map or prog-array.\n",
2736 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
3532 static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
3534 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
3535 prog->type == BPF_PROG_TYPE_LSM)
3541 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
3554 struct bpf_program *prog;
3573 bpf_object__for_each_program(prog, obj) {
3574 if (!prog->autoload)
3576 if (prog_needs_vmlinux_btf(prog))
3636 struct bpf_program *prog = &obj->programs[i];
3641 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3651 if (strcmp(name, prog->name) != 0)
4515 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)
4517 return prog->sec_idx == obj->efile.text_shndx;
4524 struct bpf_program *prog;
4526 bpf_object__for_each_program(prog, obj) {
4527 if (prog_is_subprog(obj, prog))
4529 if (!strcmp(prog->name, name))
4530 return prog;
4572 static int bpf_prog_compute_hash(struct bpf_program *prog)
4577 purged = calloc(prog->insns_cnt, BPF_INSN_SZ);
4584 for (i = 0; i < prog->insns_cnt; i++) {
4585 purged[i] = prog->insns[i];
4591 if (i >= prog->insns_cnt ||
4592 prog->insns[i].code != 0 ||
4593 prog->insns[i].dst_reg != 0 ||
4594 prog->insns[i].src_reg != 0 ||
4595 prog->insns[i].off != 0) {
4599 purged[i] = prog->insns[i];
4603 libbpf_sha256(purged, prog->insns_cnt * sizeof(struct bpf_insn),
4604 prog->hash);
4610 static int bpf_program__record_reloc(struct bpf_program *prog,
4615 struct bpf_insn *insn = &prog->insns[insn_idx];
4616 size_t map_idx, nr_maps = prog->obj->nr_maps;
4617 struct bpf_object *obj = prog->obj;
4624 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
4625 prog->name, sym_name, insn_idx, insn->code);
4640 pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
4641 prog->name, sym_name, sym_idx);
4644 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
4645 prog->name, i, ext->name, ext->sym_idx, insn_idx);
4658 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
4664 pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
4665 prog->name, sym_name, sym_sec_name);
4669 pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
4670 prog->name, sym_name, (size_t)sym->st_value);
4680 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
4681 prog->name, sym_name, shdr_idx);
4691 pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n",
4692 prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4708 pr_warn("prog '%s': bad arena data relocation at insn %u, no arena maps defined\n",
4709 prog->name, insn_idx);
4718 pr_debug("prog '%s': found arena map %d (%s, sec %d, off %zu) for insn %u\n",
4719 prog->name, obj->arena_map_idx, map->name, map->sec_idx,
4737 pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
4738 prog->name, sym_name, sym_sec_name);
4747 pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
4748 prog->name, map_idx, map->name, map->sec_idx,
4753 pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
4754 prog->name, sym_sec_name, (size_t)sym->st_value);
4766 pr_warn("prog '%s': bad data relo against section '%s'\n",
4767 prog->name, sym_sec_name);
4774 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
4775 prog->name, map_idx, map->name, map->sec_idx,
4780 pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
4781 prog->name, sym_sec_name);
4792 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
4794 return insn_idx >= prog->sec_insn_off &&
4795 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4802 struct bpf_program *prog;
4809 prog = &obj->programs[m];
4811 if (prog->sec_idx < sec_idx ||
4812 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4820 prog = &obj->programs[l];
4821 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4822 return prog;
4831 struct bpf_program *prog;
4901 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
4902 if (!prog) {
4908 relos = libbpf_reallocarray(prog->reloc_desc,
4909 prog->nr_reloc + 1, sizeof(*relos));
4912 prog->reloc_desc = relos;
4915 insn_idx -= prog->sec_insn_off;
4916 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4921 prog->nr_reloc++;
5552 pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %s\n",
5556 pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n",
5623 * runtime due to bpf_program__set_autoload(prog, false),
6004 static int record_relo_core(struct bpf_program *prog,
6009 relos = libbpf_reallocarray(prog->reloc_desc,
6010 prog->nr_reloc + 1, sizeof(*relos));
6013 relo = &relos[prog->nr_reloc];
6017 prog->reloc_desc = relos;
6018 prog->nr_reloc++;
6022 static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx)
6027 for (i = 0; i < prog->nr_reloc; i++) {
6028 relo = &prog->reloc_desc[i];
6038 static int bpf_core_resolve_relo(struct bpf_program *prog,
6047 const char *prog_name = prog->name;
6063 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
6065 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
6090 struct bpf_program *prog;
6131 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
6132 if (!prog) {
6148 if (!prog->autoload)
6155 insn_idx = insn_idx - prog->sec_insn_off;
6156 if (insn_idx >= prog->insns_cnt)
6158 insn = &prog->insns[insn_idx];
6160 err = record_relo_core(prog, rec, insn_idx);
6162 pr_warn("prog '%s': relo #%d: failed to record relocation: %s\n",
6163 prog->name, i, errstr(err));
6167 if (prog->obj->gen_loader)
6170 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
6172 pr_warn("prog '%s': relo #%d: failed to relocate: %s\n",
6173 prog->name, i, errstr(err));
6177 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
6179 pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %s\n",
6180 prog->name, i, insn_idx, errstr(err));
6204 static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx,
6210 pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n",
6211 prog->name, relo_idx, insn_idx, map_idx, map->name);
6234 static void poison_kfunc_call(struct bpf_program *prog, int relo_idx,
6238 pr_debug("prog '%s': relo #%d: poisoning insn #%d that calls kfunc '%s'\n",
6239 prog->name, relo_idx, insn_idx, ext->name);
6254 static int find_jt_map(struct bpf_object *obj, struct bpf_program *prog, unsigned int sym_off)
6265 obj->jumptable_maps[i].prog == prog)
6272 static int add_jt_map(struct bpf_object *obj, struct bpf_program *prog, unsigned int sym_off, int map_fd)
6283 obj->jumptable_maps[cnt].prog = prog;
6291 static int find_subprog_idx(struct bpf_program *prog, int insn_idx)
6295 for (i = prog->subprog_cnt - 1; i >= 0; i--) {
6296 if (insn_idx >= prog->subprogs[i].sub_insn_off)
6303 static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struct reloc_desc *relo)
6317 map_fd = find_jt_map(obj, prog, sym_off);
6351 if (relo->insn_idx < 0 || relo->insn_idx >= prog->insns_cnt) {
6356 if (prog->subprogs)
6357 subprog_idx = find_subprog_idx(prog, relo->insn_idx);
6367 insn_off -= prog->subprogs[subprog_idx].sec_insn_off;
6368 insn_off += prog->subprogs[subprog_idx].sub_insn_off;
6370 insn_off -= prog->sec_insn_off;
6394 err = add_jt_map(obj, prog, sym_off, map_fd);
6411 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
6415 for (i = 0; i < prog->nr_reloc; i++) {
6416 struct reloc_desc *relo = &prog->reloc_desc[i];
6417 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6431 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6449 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6482 poison_kfunc_call(prog, i, relo->insn_idx, insn,
6488 pr_warn("prog '%s': relo #%d: bad insn\n",
6489 prog->name, i);
6503 map_fd = create_jt_map(obj, prog, relo);
6505 pr_warn("prog '%s': relo #%d: can't create jump table: sym_off %u\n",
6506 prog->name, i, relo->sym_off);
6515 pr_warn("prog '%s': relo #%d: bad relo type %d\n",
6516 prog->name, i, relo->type);
6525 const struct bpf_program *prog,
6540 if (prog->sec_idx != sec_idx)
6546 if (insn_off < prog->sec_insn_off)
6548 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
6577 off_adj = prog->sub_insn_off - prog->sec_insn_off;
6595 const struct bpf_program *prog)
6608 if (main_prog != prog && !main_prog->func_info)
6611 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6617 pr_warn("prog '%s': error relocating .BTF.ext function info: %s\n",
6618 prog->name, errstr(err));
6626 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6630 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",
6631 prog->name);
6636 if (main_prog != prog && !main_prog->line_info)
6639 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6645 pr_warn("prog '%s': error relocating .BTF.ext line info: %s\n",
6646 prog->name, errstr(err));
6654 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6658 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",
6659 prog->name);
6674 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx)
6676 if (!prog->nr_reloc)
6678 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6679 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6742 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6751 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6761 pr_warn("prog '%s': failed to add subprog offsets: %s\n",
6771 struct bpf_program *prog)
6779 err = reloc_prog_func_and_line_info(obj, main_prog, prog);
6783 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6784 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6788 relo = find_prog_insn_relo(prog, insn_idx);
6795 pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
6796 prog->name, insn_idx, relo->type);
6819 pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n",
6820 prog->name, insn_idx);
6829 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6835 pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6836 prog->name);
6862 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6864 * prog; each main prog can have a different set of
6869 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6871 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",
6872 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6882 * main prog) is processed separately. For each subprog (non-entry functions,
6885 * hasn't been yet appended and relocated within current main prog. Once its
6886 * relocated, sub_insn_off will point at the position within current main prog
6896 * the *main* prog's instructions (subprog's instructions are always left
6898 * and subprog instructions are always sent only as part of a main prog) and
6900 * prog or any of its subprogs are appended (and relocated), all their
6906 * main prog and any of its called/used subprogs. Those will keep their
6908 * main prog and won't be relocated within the context of current main prog.
6960 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)
6976 err = bpf_object__reloc_code(obj, prog, prog);
6986 struct bpf_program *prog;
6991 prog = &obj->programs[i];
6992 zfree(&prog->reloc_desc);
6993 prog->nr_reloc = 0;
7026 static int bpf_prog_assign_exc_cb(struct bpf_object *obj, struct bpf_program *prog)
7050 pr_warn("prog '%s': exception_callback:<value> decl tag not applied to the main program\n",
7051 prog->name);
7054 if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)) != 0)
7056 /* Multiple callbacks are specified for the same prog,
7060 if (prog->exception_cb_idx >= 0) {
7061 prog->exception_cb_idx = -1;
7067 pr_warn("prog '%s': exception_callback:<value> decl tag contains empty value\n",
7068 prog->name);
7084 pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n",
7085 prog->name, subprog->name);
7089 if (prog->exception_cb_idx >= 0) {
7090 pr_warn("prog '%s': multiple subprogs with same name as exception callback '%s'\n",
7091 prog->name, subprog->name);
7094 prog->exception_cb_idx = j;
7098 if (prog->exception_cb_idx >= 0)
7101 pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name);
7148 static bool need_func_arg_type_fixup(const struct btf *btf, const struct bpf_program *prog,
7167 (prog->type == BPF_PROG_TYPE_KPROBE || prog->type == BPF_PROG_TYPE_PERF_EVENT)) {
7186 switch (prog->type) {
7214 pr_warn("prog '%s': subprog '%s' arg#%d is expected to be of `struct %s *` type\n",
7215 prog->name, subprog_name, arg_idx, ctx_name);
7219 static int clone_func_btf_info(struct btf *btf, int orig_fn_id, struct bpf_program *prog)
7271 fn_id = btf__add_func(btf, prog->name, linkage, fn_proto_id);
7288 static int bpf_program_fixup_func_info(struct bpf_object *obj, struct bpf_program *prog)
7301 if (!obj->btf_ext || !prog->func_info)
7312 if (global_ctx_map[i].prog_type != prog->type)
7321 orig_ids = calloc(prog->func_info_cnt, sizeof(*orig_ids));
7324 for (i = 0; i < prog->func_info_cnt; i++) {
7325 func_rec = prog->func_info + prog->func_info_rec_size * i;
7354 for (rec_idx = 0; rec_idx < prog->func_info_cnt; rec_idx++) {
7356 func_rec = prog->func_info + prog->func_info_rec_size * rec_idx;
7373 if (!need_func_arg_type_fixup(btf, prog, fn_name, arg_idx, p->type, ctx_name))
7380 fn_id = clone_func_btf_info(btf, orig_fn_id, prog);
7429 struct bpf_program *prog;
7459 prog = &obj->programs[i];
7460 for (j = 0; j < prog->nr_reloc; j++) {
7461 struct reloc_desc *relo = &prog->reloc_desc[j];
7462 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
7478 prog = &obj->programs[i];
7482 if (prog_is_subprog(obj, prog))
7484 if (!prog->autoload)
7487 err = bpf_object__relocate_calls(obj, prog);
7489 pr_warn("prog '%s': failed to relocate calls: %s\n",
7490 prog->name, errstr(err));
7494 err = bpf_prog_assign_exc_cb(obj, prog);
7498 if (prog->exception_cb_idx >= 0) {
7499 struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx];
7507 err = bpf_object__append_subprog_code(obj, prog, subprog);
7510 err = bpf_object__reloc_code(obj, prog, subprog);
7517 prog = &obj->programs[i];
7518 if (prog_is_subprog(obj, prog))
7520 if (!prog->autoload)
7524 err = bpf_object__relocate_data(obj, prog);
7526 pr_warn("prog '%s': failed to relocate data references: %s\n",
7527 prog->name, errstr(err));
7532 err = bpf_program_fixup_func_info(obj, prog);
7534 pr_warn("prog '%s': failed to perform .BTF.ext fix ups: %s\n",
7535 prog->name, errstr(err));
7607 type = is_map_in_map ? "map" : "prog";
7732 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog)
7734 struct bpf_insn *insn = prog->insns;
7741 for (i = 0; i < prog->insns_cnt; i++, insn++) {
7767 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
7770 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
7771 static int libbpf_prepare_prog_load(struct bpf_program *prog,
7777 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
7783 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
7787 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) {
7789 * in prog, and expected_attach_type we set in kernel is from opts, so we
7792 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7796 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
7800 attach_name = strchr(prog->sec_name, '/');
7811 pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n",
7812 prog->name);
7817 err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id);
7821 /* cache resolved BTF FD and BTF type ID in the prog */
7822 prog->attach_btf_obj_fd = btf_obj_fd;
7823 prog->attach_btf_id = btf_type_id;
7826 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because
7836 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz);
7838 static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog,
7847 __u32 log_level = prog->log_level;
7853 switch (prog->type) {
7859 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
7860 prog->name, prog->sec_name);
7863 if (prog->attach_btf_id == 0) {
7864 pr_warn("prog '%s': SEC(\"struct_ops\") program isn't referenced anywhere, did you forget to use it?\n",
7865 prog->name);
7877 prog_name = prog->name;
7878 load_attr.attach_prog_fd = prog->attach_prog_fd;
7879 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
7880 load_attr.attach_btf_id = prog->attach_btf_id;
7882 load_attr.prog_ifindex = prog->prog_ifindex;
7883 load_attr.expected_attach_type = prog->expected_attach_type;
7888 load_attr.func_info = prog->func_info;
7889 load_attr.func_info_rec_size = prog->func_info_rec_size;
7890 load_attr.func_info_cnt = prog->func_info_cnt;
7891 load_attr.line_info = prog->line_info;
7892 load_attr.line_info_rec_size = prog->line_info_rec_size;
7893 load_attr.line_info_cnt = prog->line_info_cnt;
7896 load_attr.prog_flags = prog->prog_flags;
7904 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
7905 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
7907 pr_warn("prog '%s': failed to prepare load attributes: %s\n",
7908 prog->name, errstr(err));
7911 insns = prog->insns;
7912 insns_cnt = prog->insns_cnt;
7916 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
7918 prog - obj->programs);
7930 if (prog->log_buf) {
7931 log_buf = prog->log_buf;
7932 log_buf_size = prog->log_size;
7955 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);
7958 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
7959 prog->name, log_buf);
7967 map = &prog->obj->maps[i];
7972 pr_warn("prog '%s': failed to bind map '%s': %s\n",
7973 prog->name, map->real_name, errstr(errno));
8001 fixup_verifier_log(prog, log_buf, log_buf_size);
8003 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, errstr(errno));
8007 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
8008 prog->name, log_buf);
8068 static void fixup_log_failed_core_relo(struct bpf_program *prog,
8090 relo = find_relo_core(prog, insn_idx);
8094 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
8107 static void fixup_log_missing_map_load(struct bpf_program *prog,
8119 struct bpf_object *obj = prog->obj;
8140 static void fixup_log_missing_kfunc_call(struct bpf_program *prog,
8152 struct bpf_object *obj = prog->obj;
8173 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz)
8198 fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz,
8207 fixup_log_missing_map_load(prog, buf, buf_sz, log_sz,
8216 fixup_log_missing_kfunc_call(prog, buf, buf_sz, log_sz,
8223 static int bpf_program_record_relos(struct bpf_program *prog)
8225 struct bpf_object *obj = prog->obj;
8228 for (i = 0; i < prog->nr_reloc; i++) {
8229 struct reloc_desc *relo = &prog->reloc_desc[i];
8269 struct bpf_program *prog;
8274 prog = &obj->programs[i];
8275 if (prog_is_subprog(obj, prog))
8277 if (!prog->autoload) {
8278 pr_debug("prog '%s': skipped loading\n", prog->name);
8281 prog->log_level |= log_level;
8284 bpf_program_record_relos(prog);
8286 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
8287 obj->license, obj->kern_version, &prog->fd);
8289 pr_warn("prog '%s': failed to load: %s\n", prog->name, errstr(err));
8300 struct bpf_program *prog;
8305 prog = &obj->programs[i];
8306 err = bpf_object__sanitize_prog(obj, prog);
8317 struct bpf_program *prog;
8320 bpf_object__for_each_program(prog, obj) {
8321 prog->sec_def = find_sec_def(prog->sec_name);
8322 if (!prog->sec_def) {
8324 pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
8325 prog->name, prog->sec_name);
8329 prog->type = prog->sec_def->prog_type;
8330 prog->expected_attach_type = prog->sec_def->expected_attach_type;
8335 if (prog->sec_def->prog_setup_fn) {
8336 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
8338 pr_warn("prog '%s': failed to initialize: %s\n",
8339 prog->name, errstr(err));
8896 struct bpf_program *prog = st_ops->progs[i];
8900 if (!prog)
8903 prog_fd = bpf_program__fd(prog);
9102 int bpf_program__pin(struct bpf_program *prog, const char *path)
9106 if (prog->fd < 0) {
9107 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
9119 if (bpf_obj_pin(prog->fd, path)) {
9121 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, errstr(err));
9125 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
9129 int bpf_program__unpin(struct bpf_program *prog, const char *path)
9133 if (prog->fd < 0) {
9134 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
9146 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
9368 struct bpf_program *prog;
9380 bpf_object__for_each_program(prog, obj) {
9381 err = pathname_concat(buf, sizeof(buf), path, prog->name);
9385 err = bpf_program__pin(prog, buf);
9393 while ((prog = bpf_object__prev_program(obj, prog))) {
9394 if (pathname_concat(buf, sizeof(buf), path, prog->name))
9397 bpf_program__unpin(prog, buf);
9405 struct bpf_program *prog;
9411 bpf_object__for_each_program(prog, obj) {
9414 err = pathname_concat(buf, sizeof(buf), path, prog->name);
9418 err = bpf_program__unpin(prog, buf);
9632 struct bpf_program *prog = prev;
9635 prog = __bpf_program__iter(prog, obj, true);
9636 } while (prog && prog_is_subprog(obj, prog));
9638 return prog;
9644 struct bpf_program *prog = next;
9647 prog = __bpf_program__iter(prog, obj, false);
9648 } while (prog && prog_is_subprog(obj, prog));
9650 return prog;
9653 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
9655 prog->prog_ifindex = ifindex;
9658 const char *bpf_program__name(const struct bpf_program *prog)
9660 return prog->name;
9663 const char *bpf_program__section_name(const struct bpf_program *prog)
9665 return prog->sec_name;
9668 bool bpf_program__autoload(const struct bpf_program *prog)
9670 return prog->autoload;
9673 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
9675 if (prog->obj->state >= OBJ_LOADED)
9678 prog->autoload = autoload;
9682 bool bpf_program__autoattach(const struct bpf_program *prog)
9684 return prog->autoattach;
9687 void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach)
9689 prog->autoattach = autoattach;
9692 const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog)
9694 return prog->insns;
9697 size_t bpf_program__insn_cnt(const struct bpf_program *prog)
9699 return prog->insns_cnt;
9702 int bpf_program__set_insns(struct bpf_program *prog,
9707 if (prog->obj->state >= OBJ_LOADED)
9710 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
9713 pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
9718 prog->insns = insns;
9719 prog->insns_cnt = new_insn_cnt;
9723 int bpf_program__fd(const struct bpf_program *prog)
9725 if (!prog)
9728 if (prog->fd < 0)
9731 return prog->fd;
9735 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog);
9737 enum bpf_prog_type bpf_program__type(const struct bpf_program *prog)
9739 return prog->type;
9748 int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
9750 if (prog->obj->state >= OBJ_LOADED)
9754 if (prog->type == type)
9757 prog->type = type;
9765 if (prog->sec_def != &custom_fallback_def)
9766 prog->sec_def = NULL;
9771 enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog);
9773 enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog)
9775 return prog->expected_attach_type;
9778 int bpf_program__set_expected_attach_type(struct bpf_program *prog,
9781 if (prog->obj->state >= OBJ_LOADED)
9784 prog->expected_attach_type = type;
9788 __u32 bpf_program__flags(const struct bpf_program *prog)
9790 return prog->prog_flags;
9793 int bpf_program__set_flags(struct bpf_program *prog, __u32 flags)
9795 if (prog->obj->state >= OBJ_LOADED)
9798 prog->prog_flags = flags;
9802 __u32 bpf_program__log_level(const struct bpf_program *prog)
9804 return prog->log_level;
9807 int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level)
9809 if (prog->obj->state >= OBJ_LOADED)
9812 prog->log_level = log_level;
9816 const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size)
9818 *log_size = prog->log_size;
9819 return prog->log_buf;
9822 int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size)
9826 if (prog->log_size > UINT_MAX)
9828 if (prog->obj->state >= OBJ_LOADED)
9831 prog->log_buf = log_buf;
9832 prog->log_size = log_size;
9836 struct bpf_func_info *bpf_program__func_info(const struct bpf_program *prog)
9838 if (prog->func_info_rec_size != sizeof(struct bpf_func_info))
9840 return prog->func_info;
9843 __u32 bpf_program__func_info_cnt(const struct bpf_program *prog)
9845 return prog->func_info_cnt;
9848 struct bpf_line_info *bpf_program__line_info(const struct bpf_program *prog)
9850 if (prog->line_info_rec_size != sizeof(struct bpf_line_info))
9852 return prog->line_info;
9855 __u32 bpf_program__line_info_cnt(const struct bpf_program *prog)
9857 return prog->line_info_cnt;
9860 int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts *opts)
9868 if (!prog)
9874 obj = prog->obj;
9880 * prog/object defaults when the caller leaves them zero.
9882 attr.attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0) ?: prog->attach_prog_fd;
9883 attr.prog_flags = OPTS_GET(opts, prog_flags, 0) ?: prog->prog_flags;
9884 attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0) ?: prog->prog_ifindex;
9905 pr_warn("prog '%s': func_info, func_info_cnt, and func_info_rec_size must all be specified or all omitted\n",
9906 prog->name);
9909 attr.func_info = info ?: prog->func_info;
9910 attr.func_info_cnt = info ? info_cnt : prog->func_info_cnt;
9911 attr.func_info_rec_size = info ? info_rec_size : prog->func_info_rec_size;
9917 pr_warn("prog '%s': line_info, line_info_cnt, and line_info_rec_size must all be specified or all omitted\n",
9918 prog->name);
9921 attr.line_info = info ?: prog->line_info;
9922 attr.line_info_cnt = info ? info_cnt : prog->line_info_cnt;
9923 attr.line_info_rec_size = info ? info_rec_size : prog->line_info_rec_size;
9926 /* Logging is caller-controlled; no fallback to prog/obj log settings */
9933 * Seed them from prog/obj defaults here;
9936 attr.expected_attach_type = prog->expected_attach_type;
9937 attr.attach_btf_id = prog->attach_btf_id;
9938 attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
9940 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
9941 err = prog->sec_def->prog_prepare_load_fn(prog, &attr, prog->sec_def->cookie);
9959 fd = bpf_prog_load(prog->type, prog->name, obj->license, prog->insns, prog->insns_cnt,
9974 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9975 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9976 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9977 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9978 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9979 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9980 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9981 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9982 static int attach_kprobe_session(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9983 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9984 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9985 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link);
10359 struct bpf_program *prog;
10432 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
10433 if (!prog) {
10434 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
10439 /* prevent the use of BPF prog with invalid type */
10440 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
10441 pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n",
10442 map->name, prog->name);
10446 st_ops->progs[member_idx] = prog;
10454 *((struct bpf_program **)(st_ops->data + moff)) = prog;
10565 pr_warn("%s is not found in prog's BTF\n", name);
10626 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
10629 enum bpf_attach_type attach_type = prog->expected_attach_type;
10630 __u32 attach_prog_fd = prog->attach_prog_fd;
10634 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {
10636 pr_warn("prog '%s': attach program FD is not set\n", prog->name);
10639 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd, prog->obj->token_fd);
10641 pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %s\n",
10642 prog->name, attach_prog_fd, attach_name, errstr(err));
10651 if (prog->obj->gen_loader) {
10652 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
10656 err = find_kernel_btf_id(prog->obj, attach_name,
10661 pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %s\n",
10662 prog->name, attach_name, errstr(err));
10994 int bpf_map__set_exclusive_program(struct bpf_map *map, struct bpf_program *prog)
11001 if (map->obj != prog->obj) {
11006 map->excl_prog = prog;
11233 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
11236 int prog_fd = bpf_program__fd(prog);
11239 pr_warn("prog '%s': can't use BPF program without FD (was it loaded?)\n",
11240 prog->name);
11420 struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd,
11431 pr_warn("prog '%s': invalid perf event FD %d\n",
11432 prog->name, pfd);
11435 prog_fd = bpf_program__fd(prog);
11437 pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
11438 prog->name);
11450 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) {
11457 pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %s\n",
11458 prog->name, pfd, errstr(err));
11464 pr_warn("prog '%s': user context value is not supported\n", prog->name);
11471 pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n",
11472 prog->name, pfd, errstr(err));
11474 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
11475 prog->name, pfd);
11484 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n",
11485 prog->name, pfd, errstr(err));
11498 struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd)
11500 return bpf_program__attach_perf_event_opts(prog, pfd, NULL);
11815 bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
11847 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
11877 pr_warn("prog '%s': failed to create %s '%s%s0x%zx' perf event: %s\n",
11878 prog->name, retprobe ? "kretprobe" : "kprobe",
11883 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
11887 pr_warn("prog '%s': failed to attach to %s '%s%s0x%zx': %s\n",
11888 prog->name, retprobe ? "kretprobe" : "kprobe",
11911 struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog,
11919 return bpf_program__attach_kprobe_opts(prog, func_name, &opts);
11922 struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog,
11932 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
11947 return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts);
12162 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
12182 prog_fd = bpf_program__fd(prog);
12184 pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
12185 prog->name);
12224 pr_warn("prog '%s': failed to find a unique match for '%s' (%zu matches)\n",
12225 prog->name, pattern, res.cnt);
12266 pr_warn("prog '%s': failed to attach: %s\n",
12267 prog->name, errstr(err));
12280 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12291 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)
12294 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");
12296 func_name = prog->sec_name + sizeof("kretprobe/") - 1;
12298 func_name = prog->sec_name + sizeof("kprobe/") - 1;
12312 *link = bpf_program__attach_kprobe_opts(prog, func, &opts);
12317 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12325 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)
12328 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
12330 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
12332 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
12334 *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts);
12338 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12348 if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||
12349 strcmp(prog->sec_name, "kretprobe.multi") == 0)
12352 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");
12354 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;
12356 spec = prog->sec_name + sizeof("kprobe.multi/") - 1;
12364 *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts);
12369 static int attach_kprobe_session(const struct bpf_program *prog, long cookie,
12380 if (strcmp(prog->sec_name, "kprobe.session") == 0)
12383 spec = prog->sec_name + sizeof("kprobe.session/") - 1;
12390 *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts);
12395 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12403 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
12414 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);
12418 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
12419 prog->sec_name);
12637 bpf_program__attach_uprobe_multi(const struct bpf_program *prog,
12658 prog_fd = bpf_program__fd(prog);
12660 pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
12661 prog->name);
12707 pr_warn("prog '%s': failed to resolve full path for '%s': %s\n",
12708 prog->name, path, errstr(err));
12750 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",
12751 prog->name, errstr(err));
12765 bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
12802 pr_warn("prog '%s': failed to resolve full path for '%s': %s\n",
12803 prog->name, binary_path, errstr(err));
12836 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
12867 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
12868 prog->name, retprobe ? "uretprobe" : "uprobe",
12874 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
12878 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
12879 prog->name, retprobe ? "uretprobe" : "uprobe",
12911 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12920 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
12928 pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n",
12929 prog->name, prog->sec_name);
12946 pr_warn("prog '%s': uretprobes do not support offset specification\n",
12947 prog->name);
12951 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);
12955 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
12956 prog->sec_name);
12966 struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog,
12973 return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts);
12976 struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog,
12982 struct bpf_object *obj = prog->obj;
12990 if (bpf_program__fd(prog) < 0) {
12991 pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
12992 prog->name);
13002 pr_warn("prog '%s': failed to resolve full path for '%s': %s\n",
13003 prog->name, binary_path, errstr(err));
13021 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
13029 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link)
13035 sec_name = bpf_program__section_name(prog);
13048 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path,
13108 struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,
13124 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
13125 prog->name, tp_category, tp_name,
13129 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
13133 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
13134 prog->name, tp_category, tp_name,
13141 struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog,
13145 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL);
13148 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
13155 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0)
13158 sec_name = strdup(prog->sec_name);
13163 if (str_has_pfx(prog->sec_name, "tp/"))
13175 *link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
13181 bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog,
13192 prog_fd = bpf_program__fd(prog);
13194 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13209 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
13210 prog->name, tp_name, errstr(pfd));
13217 struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
13220 return bpf_program__attach_raw_tracepoint_opts(prog, tp_name, NULL);
13223 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
13239 if (!str_has_pfx(prog->sec_name, prefixes[i]))
13244 if (prog->sec_name[pfx_len] == '\0')
13247 if (prog->sec_name[pfx_len] != '/')
13250 tp_name = prog->sec_name + pfx_len + 1;
13255 pr_warn("prog '%s': invalid section name '%s'\n",
13256 prog->name, prog->sec_name);
13260 *link = bpf_program__attach_raw_tracepoint(prog, tp_name);
13265 static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog,
13275 prog_fd = bpf_program__fd(prog);
13277 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13288 pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts);
13292 pr_warn("prog '%s': failed to attach: %s\n",
13293 prog->name, errstr(pfd));
13300 struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)
13302 return bpf_program__attach_btf_id(prog, NULL);
13305 struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog,
13308 return bpf_program__attach_btf_id(prog, opts);
13311 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog)
13313 return bpf_program__attach_btf_id(prog, NULL);
13316 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link)
13318 *link = bpf_program__attach_trace(prog);
13322 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link)
13324 *link = bpf_program__attach_lsm(prog);
13329 bpf_program_attach_fd(const struct bpf_program *prog,
13337 prog_fd = bpf_program__fd(prog);
13339 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13348 attach_type = bpf_program__expected_attach_type(prog);
13353 pr_warn("prog '%s': failed to attach to %s: %s\n",
13354 prog->name, target_name,
13363 bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd)
13365 return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", NULL);
13369 bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd)
13371 return bpf_program_attach_fd(prog, netns_fd, "netns", NULL);
13375 bpf_program__attach_sockmap(const struct bpf_program *prog, int map_fd)
13377 return bpf_program_attach_fd(prog, map_fd, "sockmap", NULL);
13380 struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex)
13383 return bpf_program_attach_fd(prog, ifindex, "xdp", NULL);
13387 bpf_program__attach_cgroup_opts(const struct bpf_program *prog, int cgroup_fd,
13401 pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n",
13402 prog->name);
13411 return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", &link_create_opts);
13415 bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex,
13430 pr_warn("prog '%s': target netdevice ifindex cannot be zero\n",
13431 prog->name);
13435 pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n",
13436 prog->name);
13446 return bpf_program_attach_fd(prog, ifindex, "tcx", &link_create_opts);
13450 bpf_program__attach_netkit(const struct bpf_program *prog, int ifindex,
13465 pr_warn("prog '%s': target netdevice ifindex cannot be zero\n",
13466 prog->name);
13470 pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n",
13471 prog->name);
13480 return bpf_program_attach_fd(prog, ifindex, "netkit", &link_create_opts);
13483 struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog,
13490 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
13491 prog->name);
13495 if (prog->type != BPF_PROG_TYPE_EXT) {
13496 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace\n",
13497 prog->name);
13504 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd, prog->obj->token_fd);
13510 return bpf_program_attach_fd(prog, target_fd, "freplace",
13516 return bpf_program__attach_trace(prog);
13521 bpf_program__attach_iter(const struct bpf_program *prog,
13535 prog_fd = bpf_program__fd(prog);
13537 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13551 pr_warn("prog '%s': failed to attach to iterator: %s\n",
13552 prog->name, errstr(link_fd));
13559 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link)
13561 *link = bpf_program__attach_iter(prog, NULL);
13565 struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog,
13575 prog_fd = bpf_program__fd(prog);
13577 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13596 pr_warn("prog '%s': failed to attach to netfilter: %s\n",
13597 prog->name, errstr(link_fd));
13605 struct bpf_link *bpf_program__attach(const struct bpf_program *prog)
13610 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
13613 if (bpf_program__fd(prog) < 0) {
13614 pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
13615 prog->name);
13619 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
14302 int bpf_program__set_attach_target(struct bpf_program *prog,
14308 if (!prog || attach_prog_fd < 0)
14311 if (prog->obj->state >= OBJ_LOADED)
14318 prog->attach_prog_fd = attach_prog_fd;
14324 attach_prog_fd, prog->obj->token_fd);
14332 err = bpf_object__load_vmlinux_btf(prog->obj, true);
14335 err = find_kernel_btf_id(prog->obj, attach_func_name,
14336 prog->expected_attach_type,
14342 prog->attach_btf_id = btf_id;
14343 prog->attach_btf_obj_fd = btf_obj_fd;
14344 prog->attach_prog_fd = attach_prog_fd;
14348 int bpf_program__assoc_struct_ops(struct bpf_program *prog, struct bpf_map *map,
14353 prog_fd = bpf_program__fd(prog);
14355 pr_warn("prog '%s': can't associate BPF program without FD (was it loaded?)\n",
14356 prog->name);
14360 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) {
14361 pr_warn("prog '%s': can't associate struct_ops program\n", prog->name);
14515 struct bpf_program **prog = prog_skel->prog;
14518 *prog = bpf_object__find_program_by_name(obj, name);
14519 if (!*prog) {
14659 struct bpf_program *prog = *prog_skel->prog;
14662 if (!prog->autoload || !prog->autoattach)
14666 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
14673 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
14675 pr_warn("prog '%s': failed to auto-attach: %s\n",
14676 bpf_program__name(prog), errstr(err));