Lines Matching defs:aux
102 struct bpf_prog_aux *aux;
110 aux = kzalloc(sizeof(*aux), bpf_memcg_flags(GFP_KERNEL | gfp_extra_flags));
111 if (aux == NULL) {
118 kfree(aux);
123 fp->aux = aux;
124 fp->aux->main_prog_aux = aux;
125 fp->aux->prog = fp;
129 aux->cgroup_atype = CGROUP_BPF_ATTACH_TYPE_INVALID;
132 INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode);
134 INIT_LIST_HEAD_RCU(&fp->aux->ksym_prefix.lnode);
136 mutex_init(&fp->aux->used_maps_mutex);
137 mutex_init(&fp->aux->ext_mutex);
138 mutex_init(&fp->aux->dst_mutex);
160 kfree(prog->aux);
177 if (!prog->aux->nr_linfo || !prog->jit_requested)
180 prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo,
181 sizeof(*prog->aux->jited_linfo),
183 if (!prog->aux->jited_linfo)
191 if (prog->aux->jited_linfo &&
192 (!prog->jited || !prog->aux->jited_linfo[0])) {
193 kvfree(prog->aux->jited_linfo);
194 prog->aux->jited_linfo = NULL;
197 kfree(prog->aux->kfunc_tab);
198 prog->aux->kfunc_tab = NULL;
216 * The prog's idx to prog->aux->linfo and jited_linfo
232 if (!prog->aux->jited_linfo || prog->aux->func_idx > prog->aux->func_cnt)
236 linfo_idx = prog->aux->linfo_idx;
237 linfo = &prog->aux->linfo[linfo_idx];
241 jited_linfo = &prog->aux->jited_linfo[linfo_idx];
244 nr_linfo = prog->aux->nr_linfo - linfo_idx;
270 fp->aux->prog = fp;
272 /* We keep fp->aux from fp_old around in the new
275 fp_old->aux = NULL;
286 if (fp->aux) {
287 mutex_destroy(&fp->aux->used_maps_mutex);
288 mutex_destroy(&fp->aux->dst_mutex);
289 kfree(fp->aux->poke_tab);
290 kfree(fp->aux);
436 nr_linfo = prog->aux->nr_linfo;
440 linfo = prog->aux->linfo;
532 for (i = 0; i < fp->aux->real_func_cnt; i++)
533 bpf_prog_kallsyms_del(fp->aux->func[i]);
555 prog->aux->ksym.start = (unsigned long) prog->bpf_func;
556 prog->aux->ksym.end = prog->aux->ksym.start + prog->jited_len;
562 char *sym = prog->aux->ksym.name;
576 sizeof(prog->aux->name) > KSYM_NAME_LEN);
581 /* prog->aux->name will be ignored if full btf name is available */
582 if (prog->aux->func_info_cnt && prog->aux->func_idx < prog->aux->func_info_cnt) {
583 type = btf_type_by_id(prog->aux->btf,
584 prog->aux->func_info[prog->aux->func_idx].type_id);
585 func_name = btf_name_by_offset(prog->aux->btf, type->name_off);
590 if (prog->aux->name[0])
591 snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name);
668 !bpf_token_capable(fp->aux->token, CAP_BPF))
673 fp->aux->ksym.prog = true;
675 bpf_ksym_add(&fp->aux->ksym);
685 snprintf(fp->aux->ksym_prefix.name, KSYM_NAME_LEN,
686 "__cfi_%s", fp->aux->ksym.name);
688 fp->aux->ksym_prefix.start = (unsigned long) fp->bpf_func - 16;
689 fp->aux->ksym_prefix.end = (unsigned long) fp->bpf_func;
691 bpf_ksym_add(&fp->aux->ksym_prefix);
700 bpf_ksym_del(&fp->aux->ksym);
704 bpf_ksym_del(&fp->aux->ksym_prefix);
772 if (!prog->aux->num_exentries)
775 e = search_extable(prog->aux->extable, prog->aux->num_exentries, addr);
812 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;
814 u32 slot = prog->aux->size_poke_tab;
837 prog->aux->size_poke_tab = size;
838 prog->aux->poke_tab = tab;
1241 * can pick them up from prog->aux.
1245 else if (prog->aux->func &&
1246 off >= 0 && off < prog->aux->real_func_cnt)
1247 addr = (u8 *)prog->aux->func[off]->bpf_func;
1270 if (prog->aux->ksym.prog)
1271 return prog->aux->ksym.name;
1272 return prog->aux->name;
1276 const struct bpf_insn *aux,
1385 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[1].imm);
1388 *to++ = BPF_ALU64_REG(BPF_MOV, aux[0].dst_reg, BPF_REG_AX);
1391 *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[0].imm);
1395 *to++ = BPF_ALU64_REG(BPF_OR, aux[0].dst_reg, BPF_REG_AX);
1419 /* aux->prog still points to the fp_other one, so
1431 /* aux was stolen by the other clone, so we cannot free
1438 fp->aux = NULL;
1446 /* We have to repoint aux->prog to self, as we don't
1449 fp->aux->prog = fp;
1455 struct bpf_insn insn_buff[16], aux[2];
1488 memcpy(aux, insn, sizeof(aux));
1490 rewritten = bpf_jit_blind_insn(insn, aux, insn_buff,
1491 clone->aux->verifier_zext);
1497 /* Patching may have repointed aux->prog during
2347 struct bpf_prog_aux *aux = fp->aux;
2363 map->owner->xdp_has_frags = aux->xdp_has_frags;
2365 map->owner->attach_func_proto = aux->attach_func_proto;
2368 aux->cgroup_storage[i] ?
2369 aux->cgroup_storage[i]->cookie : 0;
2375 map->owner->xdp_has_frags == aux->xdp_has_frags;
2383 cookie = aux->cgroup_storage[i] ?
2384 aux->cgroup_storage[i]->cookie : 0;
2389 map->owner->attach_func_proto != aux->attach_func_proto) {
2414 if (bpf_prog_is_dev_bound(fp->aux))
2422 struct bpf_prog_aux *aux = fp->aux;
2425 mutex_lock(&aux->used_maps_mutex);
2426 for (i = 0; i < aux->used_map_cnt; i++) {
2427 struct bpf_map *map = aux->used_maps[i];
2439 mutex_unlock(&aux->used_maps_mutex);
2447 u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1);
2500 if (!bpf_prog_is_offloaded(fp->aux)) {
2623 prog_ids[i] = item->prog->aux->id;
2817 void __bpf_free_used_maps(struct bpf_prog_aux *aux,
2824 sleepable = aux->prog->sleepable;
2828 map->ops->map_poke_untrack(map, aux);
2835 static void bpf_free_used_maps(struct bpf_prog_aux *aux)
2837 __bpf_free_used_maps(aux, aux->used_maps, aux->used_map_cnt);
2838 kfree(aux->used_maps);
2856 static void bpf_free_used_btfs(struct bpf_prog_aux *aux)
2858 __bpf_free_used_btfs(aux->used_btfs, aux->used_btf_cnt);
2859 kfree(aux->used_btfs);
2864 struct bpf_prog_aux *aux;
2867 aux = container_of(work, struct bpf_prog_aux, work);
2869 bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab);
2870 bpf_prog_stream_free(aux->prog);
2873 if (aux->cgroup_atype != CGROUP_BPF_ATTACH_TYPE_INVALID)
2874 bpf_cgroup_atype_put(aux->cgroup_atype);
2876 bpf_free_used_maps(aux);
2877 bpf_free_used_btfs(aux);
2878 if (bpf_prog_is_dev_bound(aux))
2879 bpf_prog_dev_bound_destroy(aux->prog);
2881 if (aux->prog->has_callchain_buf)
2884 if (aux->dst_trampoline)
2885 bpf_trampoline_put(aux->dst_trampoline);
2886 for (i = 0; i < aux->real_func_cnt; i++) {
2891 aux->func[i]->aux->poke_tab = NULL;
2892 bpf_jit_free(aux->func[i]);
2894 if (aux->real_func_cnt) {
2895 kfree(aux->func);
2896 bpf_prog_unlock_free(aux->prog);
2898 bpf_jit_free(aux->prog);
2904 struct bpf_prog_aux *aux = fp->aux;
2906 if (aux->dst_prog)
2907 bpf_prog_put(aux->dst_prog);
2908 bpf_token_put(aux->token);
2909 INIT_WORK(&aux->work, bpf_prog_free_deferred);
2910 schedule_work(&aux->work);
3250 btf = prog->aux->btf;
3251 linfo = prog->aux->linfo;
3252 jited_linfo = prog->aux->jited_linfo;
3256 len = prog->aux->func ? prog->aux->func[prog->aux->func_idx]->len : prog->len;
3258 linfo = &prog->aux->linfo[prog->aux->linfo_idx];
3259 jited_linfo = &prog->aux->jited_linfo[prog->aux->linfo_idx];
3263 nr_linfo = prog->aux->nr_linfo - prog->aux->linfo_idx;
3306 ctxp->prog = prog->aux->main_prog_aux->prog;