Lines Matching refs:aux
81 tab = prog->aux->kfunc_tab; in bpf_jit_find_kfunc_model()
112 tab = env->prog->aux->kfunc_tab; in sort_kfunc_descs_by_imm_off()
235 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; in adjust_poke_descs()
236 int i, sz = prog->aux->size_poke_tab; in adjust_poke_descs()
337 struct bpf_prog_aux *aux = env->prog->aux; in adjust_subprog_starts_after_remove() local
349 if (aux->func_info) { in adjust_subprog_starts_after_remove()
350 move = aux->func_info_cnt - j; in adjust_subprog_starts_after_remove()
352 memmove(aux->func_info + i, in adjust_subprog_starts_after_remove()
353 aux->func_info + j, in adjust_subprog_starts_after_remove()
354 sizeof(*aux->func_info) * move); in adjust_subprog_starts_after_remove()
355 aux->func_info_cnt -= j - i; in adjust_subprog_starts_after_remove()
380 nr_linfo = prog->aux->nr_linfo; in bpf_adj_linfo_after_remove()
384 linfo = prog->aux->linfo; in bpf_adj_linfo_after_remove()
414 prog->aux->nr_linfo -= l_cnt; in bpf_adj_linfo_after_remove()
415 nr_linfo = prog->aux->nr_linfo; in bpf_adj_linfo_after_remove()
464 if (bpf_prog_is_offloaded(env->prog->aux)) in verifier_remove_insns()
526 if (bpf_prog_is_offloaded(env->prog->aux)) in bpf_opt_hard_wire_dead_code_branches()
589 struct bpf_insn_aux_data *aux = env->insn_aux_data; in bpf_opt_subreg_zext_lo32_rnd_hi32() local
607 if (!aux[adj_idx].zext_dst) { in bpf_opt_subreg_zext_lo32_rnd_hi32()
632 aux[adj_idx].ptr_type == PTR_TO_CTX) in bpf_opt_subreg_zext_lo32_rnd_hi32()
676 aux = env->insn_aux_data; in bpf_opt_subreg_zext_lo32_rnd_hi32()
755 if (bpf_prog_is_offloaded(env->prog->aux)) in bpf_convert_ctx_accesses()
809 env->prog->aux->num_exentries++; in bpf_convert_ctx_accesses()
887 env->prog->aux->num_exentries++; in bpf_convert_ctx_accesses()
900 env->prog->aux->num_exentries++; in bpf_convert_ctx_accesses()
1105 func[i]->aux->func_idx = i; in jit_subprogs()
1107 func[i]->aux->btf = prog->aux->btf; in jit_subprogs()
1108 func[i]->aux->subprog_start = subprog_start; in jit_subprogs()
1109 func[i]->aux->func_info = prog->aux->func_info; in jit_subprogs()
1110 func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; in jit_subprogs()
1111 func[i]->aux->poke_tab = prog->aux->poke_tab; in jit_subprogs()
1112 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; in jit_subprogs()
1113 func[i]->aux->main_prog_aux = prog->aux; in jit_subprogs()
1115 for (j = 0; j < prog->aux->size_poke_tab; j++) { in jit_subprogs()
1118 poke = &prog->aux->poke_tab[j]; in jit_subprogs()
1121 poke->aux = func[i]->aux; in jit_subprogs()
1124 func[i]->aux->name[0] = 'F'; in jit_subprogs()
1125 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; in jit_subprogs()
1127 func[i]->aux->jits_use_priv_stack = true; in jit_subprogs()
1131 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; in jit_subprogs()
1132 func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab; in jit_subprogs()
1133 func[i]->aux->linfo = prog->aux->linfo; in jit_subprogs()
1134 func[i]->aux->nr_linfo = prog->aux->nr_linfo; in jit_subprogs()
1135 func[i]->aux->jited_linfo = prog->aux->jited_linfo; in jit_subprogs()
1136 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; in jit_subprogs()
1137 func[i]->aux->arena = prog->aux->arena; in jit_subprogs()
1138 func[i]->aux->used_maps = env->used_maps; in jit_subprogs()
1139 func[i]->aux->used_map_cnt = env->used_map_cnt; in jit_subprogs()
1157 func[i]->aux->num_exentries = num_exentries; in jit_subprogs()
1158 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; in jit_subprogs()
1159 func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; in jit_subprogs()
1160 func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data; in jit_subprogs()
1161 func[i]->aux->might_sleep = env->subprog_info[i].might_sleep; in jit_subprogs()
1162 func[i]->aux->token = prog->aux->token; in jit_subprogs()
1164 func[i]->aux->exception_boundary = env->seen_exception; in jit_subprogs()
1203 func[i]->aux->func = func; in jit_subprogs()
1204 func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; in jit_subprogs()
1205 func[i]->aux->real_func_cnt = env->subprog_cnt; in jit_subprogs()
1223 func[i]->aux->used_maps = NULL; in jit_subprogs()
1224 func[i]->aux->used_map_cnt = 0; in jit_subprogs()
1261 prog->aux->extable = func[0]->aux->extable; in jit_subprogs()
1262 prog->aux->num_exentries = func[0]->aux->num_exentries; in jit_subprogs()
1263 prog->aux->func = func; in jit_subprogs()
1264 prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; in jit_subprogs()
1265 prog->aux->real_func_cnt = env->subprog_cnt; in jit_subprogs()
1266 prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func; in jit_subprogs()
1267 prog->aux->exception_boundary = func[0]->aux->exception_boundary; in jit_subprogs()
1275 for (i = 0; i < prog->aux->size_poke_tab; i++) { in jit_subprogs()
1276 map_ptr = prog->aux->poke_tab[i].tail_call.map; in jit_subprogs()
1277 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); in jit_subprogs()
1286 func[i]->aux->poke_tab = NULL; in jit_subprogs()
1386 !bpf_prog_is_offloaded(env->prog->aux)) { in bpf_fixup_call_args()
1398 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { in bpf_fixup_call_args()
1471 struct bpf_insn_aux_data *aux; in bpf_do_misc_fixups() local
1500 (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) { in bpf_do_misc_fixups()
1693 aux = &env->insn_aux_data[i + delta]; in bpf_do_misc_fixups()
1694 if (!aux->alu_state || in bpf_do_misc_fixups()
1695 aux->alu_state == BPF_ALU_NON_POINTER) in bpf_do_misc_fixups()
1698 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; in bpf_do_misc_fixups()
1699 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == in bpf_do_misc_fixups()
1701 isimm = aux->alu_state & BPF_ALU_IMMEDIATE; in bpf_do_misc_fixups()
1705 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); in bpf_do_misc_fixups()
1709 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); in bpf_do_misc_fixups()
1842 prog->aux->stack_depth = MAX_BPF_STACK; in bpf_do_misc_fixups()
1843 prog->aux->max_pkt_offset = MAX_PACKET_OFF; in bpf_do_misc_fixups()
1853 aux = &env->insn_aux_data[i + delta]; in bpf_do_misc_fixups()
1856 !bpf_map_key_poisoned(aux) && in bpf_do_misc_fixups()
1857 !bpf_map_ptr_poisoned(aux) && in bpf_do_misc_fixups()
1858 !bpf_map_ptr_unpriv(aux)) { in bpf_do_misc_fixups()
1861 .tail_call.map = aux->map_ptr_state.map_ptr, in bpf_do_misc_fixups()
1862 .tail_call.key = bpf_map_key_immediate(aux), in bpf_do_misc_fixups()
1876 if (!bpf_map_ptr_unpriv(aux)) in bpf_do_misc_fixups()
1885 if (bpf_map_ptr_poisoned(aux)) { in bpf_do_misc_fixups()
1890 map_ptr = aux->map_ptr_state.map_ptr; in bpf_do_misc_fixups()
1924 BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), in bpf_do_misc_fixups()
1975 aux = &env->insn_aux_data[i + delta]; in bpf_do_misc_fixups()
1976 if (bpf_map_ptr_poisoned(aux)) in bpf_do_misc_fixups()
1979 map_ptr = aux->map_ptr_state.map_ptr; in bpf_do_misc_fixups()
2135 int nr_args = btf_type_vlen(prog->aux->attach_func_proto); in bpf_do_misc_fixups()
2199 int nr_args = btf_type_vlen(prog->aux->attach_func_proto); in bpf_do_misc_fixups()
2340 env->prog->aux->stack_depth = subprogs[0].stack_depth; in bpf_do_misc_fixups()
2382 for (i = 0; i < prog->aux->size_poke_tab; i++) { in bpf_do_misc_fixups()
2383 map_ptr = prog->aux->poke_tab[i].tail_call.map; in bpf_do_misc_fixups()
2391 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); in bpf_do_misc_fixups()
2537 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; in bpf_optimize_bpf_loop()
2548 struct bpf_insn_aux_data *aux = env->insn_aux_data; in bpf_remove_fastcall_spills_fills() local
2556 if (aux[i].fastcall_spills_num > 0) { in bpf_remove_fastcall_spills_fills()
2557 spills_num = aux[i].fastcall_spills_num; in bpf_remove_fastcall_spills_fills()