31 	struct bpf_map *arena = (struct bpf_map *)prog->aux->arena;  in is_addr_space_cast32()
142 	tab = prog->aux->kfunc_tab;  in bpf_jit_find_kfunc_model()
173 	tab = env->prog->aux->kfunc_tab;  in sort_kfunc_descs_by_imm_off()
229 	/* aux info at OFF always needs adjustment, no matter fast path  in adjust_insn_aux_data()
297 	struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;  in adjust_poke_descs()
298 	int i, sz = prog->aux->size_poke_tab;  in adjust_poke_descs()
399 		struct bpf_prog_aux *aux = env->prog->aux;  in adjust_subprog_starts_after_remove()  local
410 		/* remove func_info and its aux */  in adjust_subprog_starts_after_remove()
411 		if (aux->func_info) {  in adjust_subprog_starts_after_remove()
412 			move = aux->func_info_cnt - j;  in adjust_subprog_starts_after_remove()
414 			memmove(aux->func_info + i,  in adjust_subprog_starts_after_remove()
415 				aux->func_info + j,  in adjust_subprog_starts_after_remove()
416 				sizeof(*aux->func_info) * move);  in adjust_subprog_starts_after_remove()
417 			if (aux->func_info_aux)  in adjust_subprog_starts_after_remove()
418 				memmove(aux->func_info_aux + i,  in adjust_subprog_starts_after_remove()
419 					aux->func_info_aux + j,  in adjust_subprog_starts_after_remove()
420 					sizeof(*aux->func_info_aux) * move);  in adjust_subprog_starts_after_remove()
421 			aux->func_info_cnt -= j - i;  in adjust_subprog_starts_after_remove()
446 	nr_linfo = prog->aux->nr_linfo;  in bpf_adj_linfo_after_remove()
450 	linfo = prog->aux->linfo;  in bpf_adj_linfo_after_remove()
480 		prog->aux->nr_linfo -= l_cnt;  in bpf_adj_linfo_after_remove()
481 		nr_linfo = prog->aux->nr_linfo;  in bpf_adj_linfo_after_remove()
504  * Clean up dynamically allocated fields of aux data for instructions [start, ...]
526 	if (bpf_prog_is_offloaded(env->prog->aux))  in verifier_remove_insns()
588 		if (bpf_prog_is_offloaded(env->prog->aux))  in bpf_opt_hard_wire_dead_code_branches()
651 	struct bpf_insn_aux_data *aux = env->insn_aux_data;  in bpf_opt_subreg_zext_lo32_rnd_hi32()  local
669 		if (!aux[adj_idx].zext_dst) {  in bpf_opt_subreg_zext_lo32_rnd_hi32()
690 			    aux[adj_idx].ptr_type == PTR_TO_CTX)  in bpf_opt_subreg_zext_lo32_rnd_hi32()
734 		aux = env->insn_aux_data;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
813 	if (bpf_prog_is_offloaded(env->prog->aux))  in bpf_convert_ctx_accesses()
868 			env->prog->aux->num_exentries++;  in bpf_convert_ctx_accesses()
940 			env->prog->aux->num_exentries++;  in bpf_convert_ctx_accesses()
960 				env->prog->aux->num_exentries++;  in bpf_convert_ctx_accesses()
1148 		func[i]->aux->func_idx = i;  in jit_subprogs()
1149 		/* Below members will be freed only at prog->aux */  in jit_subprogs()
1150 		func[i]->aux->btf = prog->aux->btf;  in jit_subprogs()
1151 		func[i]->aux->subprog_start = subprog_start;  in jit_subprogs()
1152 		func[i]->aux->func_info = prog->aux->func_info;  in jit_subprogs()
1153 		func[i]->aux->func_info_cnt = prog->aux->func_info_cnt;  in jit_subprogs()
1154 		func[i]->aux->poke_tab = prog->aux->poke_tab;  in jit_subprogs()
1155 		func[i]->aux->size_poke_tab = prog->aux->size_poke_tab;  in jit_subprogs()
1156 		func[i]->aux->main_prog_aux = prog->aux;  in jit_subprogs()
1158 		for (j = 0; j < prog->aux->size_poke_tab; j++) {  in jit_subprogs()
1161 			poke = &prog->aux->poke_tab[j];  in jit_subprogs()
1164 				poke->aux = func[i]->aux;  in jit_subprogs()
1167 		func[i]->aux->name[0] = 'F';  in jit_subprogs()
1168 		func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;  in jit_subprogs()
1170 			func[i]->aux->jits_use_priv_stack = true;  in jit_subprogs()
1174 		func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;  in jit_subprogs()
1175 		func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab;  in jit_subprogs()
1176 		func[i]->aux->linfo = prog->aux->linfo;  in jit_subprogs()
1177 		func[i]->aux->nr_linfo = prog->aux->nr_linfo;  in jit_subprogs()
1178 		func[i]->aux->jited_linfo = prog->aux->jited_linfo;  in jit_subprogs()
1179 		func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;  in jit_subprogs()
1180 		func[i]->aux->arena = prog->aux->arena;  in jit_subprogs()
1181 		func[i]->aux->used_maps = env->used_maps;  in jit_subprogs()
1182 		func[i]->aux->used_map_cnt = env->used_map_cnt;  in jit_subprogs()
1200 		func[i]->aux->num_exentries = num_exentries;  in jit_subprogs()
1201 		func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;  in jit_subprogs()
1202 		func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;  in jit_subprogs()
1203 		func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;  in jit_subprogs()
1204 		func[i]->aux->might_sleep = env->subprog_info[i].might_sleep;  in jit_subprogs()
1205 		func[i]->aux->token = prog->aux->token;  in jit_subprogs()
1207 			func[i]->aux->exception_boundary = env->seen_exception;  in jit_subprogs()
1235 		/* we use the aux data to keep a list of the start addresses  in jit_subprogs()
1246 		func[i]->aux->func = func;  in jit_subprogs()
1247 		func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;  in jit_subprogs()
1248 		func[i]->aux->real_func_cnt = env->subprog_cnt;  in jit_subprogs()
1262 	 * Cleanup func[i]->aux fields which aren't required  in jit_subprogs()
1266 		func[i]->aux->used_maps = NULL;  in jit_subprogs()
1267 		func[i]->aux->used_map_cnt = 0;  in jit_subprogs()
1304 	prog->aux->extable = func[0]->aux->extable;  in jit_subprogs()
1305 	prog->aux->num_exentries = func[0]->aux->num_exentries;  in jit_subprogs()
1306 	prog->aux->func = func;  in jit_subprogs()
1307 	prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;  in jit_subprogs()
1308 	prog->aux->real_func_cnt = env->subprog_cnt;  in jit_subprogs()
1309 	prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func;  in jit_subprogs()
1310 	prog->aux->exception_boundary = func[0]->aux->exception_boundary;  in jit_subprogs()
1311 	prog->aux->stack_arg_sp_adjust = func[0]->aux->stack_arg_sp_adjust;  in jit_subprogs()
1319 	for (i = 0; i < prog->aux->size_poke_tab; i++) {  in jit_subprogs()
1320 		map_ptr = prog->aux->poke_tab[i].tail_call.map;  in jit_subprogs()
1321 		map_ptr->ops->map_poke_untrack(map_ptr, prog->aux);  in jit_subprogs()
1330 		func[i]->aux->poke_tab = NULL;  in jit_subprogs()
1432 	    !bpf_prog_is_offloaded(env->prog->aux)) {  in bpf_fixup_call_args()
1450 	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {  in bpf_fixup_call_args()
1522 	struct bpf_insn_aux_data *aux;  in bpf_do_misc_fixups()  local
1741 			aux = &env->insn_aux_data[i + delta];  in bpf_do_misc_fixups()
1742 			if (!aux->alu_state ||  in bpf_do_misc_fixups()
1743 			    aux->alu_state == BPF_ALU_NON_POINTER)  in bpf_do_misc_fixups()
1746 			isneg = aux->alu_state & BPF_ALU_NEG_VALUE;  in bpf_do_misc_fixups()
1747 			issrc = (aux->alu_state & BPF_ALU_SANITIZE) ==  in bpf_do_misc_fixups()
1749 			isimm = aux->alu_state & BPF_ALU_IMMEDIATE;  in bpf_do_misc_fixups()
1753 				*patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);  in bpf_do_misc_fixups()
1757 				*patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);  in bpf_do_misc_fixups()
1857 			aux = &env->insn_aux_data[i + delta];  in bpf_do_misc_fixups()
1858 			map = env->used_maps[aux->map_index];  in bpf_do_misc_fixups()
1929 				prog->aux->stack_depth = MAX_BPF_STACK;  in bpf_do_misc_fixups()
1930 			prog->aux->max_pkt_offset = MAX_PACKET_OFF;  in bpf_do_misc_fixups()
1940 			aux = &env->insn_aux_data[i + delta];  in bpf_do_misc_fixups()
1943 			    !bpf_map_key_poisoned(aux) &&  in bpf_do_misc_fixups()
1944 			    !bpf_map_ptr_poisoned(aux) &&  in bpf_do_misc_fixups()
1945 			    !bpf_map_ptr_unpriv(aux)) {  in bpf_do_misc_fixups()
1948 					.tail_call.map = aux->map_ptr_state.map_ptr,  in bpf_do_misc_fixups()
1949 					.tail_call.key = bpf_map_key_immediate(aux),  in bpf_do_misc_fixups()
1963 			if (!bpf_map_ptr_unpriv(aux))  in bpf_do_misc_fixups()
1972 			if (bpf_map_ptr_poisoned(aux)) {  in bpf_do_misc_fixups()
1977 			map_ptr = aux->map_ptr_state.map_ptr;  in bpf_do_misc_fixups()
2011 				BPF_LD_IMM64(BPF_REG_3, (long)prog->aux),  in bpf_do_misc_fixups()
2062 			aux = &env->insn_aux_data[i + delta];  in bpf_do_misc_fixups()
2063 			if (bpf_map_ptr_poisoned(aux))  in bpf_do_misc_fixups()
2066 			map_ptr = aux->map_ptr_state.map_ptr;  in bpf_do_misc_fixups()
2227 				int nr_args = btf_type_vlen(prog->aux->attach_func_proto);  in bpf_do_misc_fixups()
2293 				int nr_args = btf_type_vlen(prog->aux->attach_func_proto);  in bpf_do_misc_fixups()
2434 	env->prog->aux->stack_depth = subprogs[0].stack_depth;  in bpf_do_misc_fixups()
2475 	/* Since poke tab is now finalized, publish aux to tracker. */  in bpf_do_misc_fixups()
2476 	for (i = 0; i < prog->aux->size_poke_tab; i++) {  in bpf_do_misc_fixups()
2477 		map_ptr = prog->aux->poke_tab[i].tail_call.map;  in bpf_do_misc_fixups()
2485 		ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux);  in bpf_do_misc_fixups()
2631 	env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;  in bpf_optimize_bpf_loop()
2642 	struct bpf_insn_aux_data *aux = env->insn_aux_data;  in bpf_remove_fastcall_spills_fills()  local
2650 		if (aux[i].fastcall_spills_num > 0) {  in bpf_remove_fastcall_spills_fills()
2651 			spills_num = aux[i].fastcall_spills_num;  in bpf_remove_fastcall_spills_fills()