29 static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *insn)  in is_addr_space_cast32()  argument
31 	struct bpf_map *arena = (struct bpf_map *)prog->aux->arena;  in is_addr_space_cast32()
68 static bool bpf_is_reg64(struct bpf_prog *prog, struct bpf_insn *insn)  in bpf_is_reg64()  argument
85 	if (is_addr_space_cast32(prog, insn))  in bpf_is_reg64()
109 int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn)  in bpf_insn_def32()  argument
113 	if (dst_reg < 0 || bpf_is_reg64(prog, insn))  in bpf_insn_def32()
132 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,  in bpf_jit_find_kfunc_model()  argument
142 	tab = prog->aux->kfunc_tab;  in bpf_jit_find_kfunc_model()
173 	tab = env->prog->aux->kfunc_tab;  in sort_kfunc_descs_by_imm_off()
295 static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len)  in adjust_poke_descs()  argument
297 	struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;  in adjust_poke_descs()
298 	int i, sz = prog->aux->size_poke_tab;  in adjust_poke_descs()
317 				    array_size(env->prog->len + len - 1,  in bpf_patch_insn_data()
326 	new_prog = bpf_patch_insn_single(env->prog, off, patch, len);  in bpf_patch_insn_data()
345 static int adjust_jmp_off(struct bpf_prog *prog, u32 tgt_idx, u32 delta)  in adjust_jmp_off()  argument
347 	struct bpf_insn *insn = prog->insnsi;  in adjust_jmp_off()
348 	u32 insn_cnt = prog->len, i;  in adjust_jmp_off()
399 		struct bpf_prog_aux *aux = env->prog->aux;  in adjust_subprog_starts_after_remove()
442 	struct bpf_prog *prog = env->prog;  in bpf_adj_linfo_after_remove()  local
446 	nr_linfo = prog->aux->nr_linfo;  in bpf_adj_linfo_after_remove()
450 	linfo = prog->aux->linfo;  in bpf_adj_linfo_after_remove()
469 	if (prog->len != off && l_cnt &&  in bpf_adj_linfo_after_remove()
480 		prog->aux->nr_linfo -= l_cnt;  in bpf_adj_linfo_after_remove()
481 		nr_linfo = prog->aux->nr_linfo;  in bpf_adj_linfo_after_remove()
523 	unsigned int orig_prog_len = env->prog->len;  in verifier_remove_insns()
526 	if (bpf_prog_is_offloaded(env->prog->aux))  in verifier_remove_insns()
531 	err = bpf_remove_insns(env->prog, off, cnt);  in verifier_remove_insns()
573 	struct bpf_insn *insn = env->prog->insnsi;  in bpf_opt_hard_wire_dead_code_branches()
574 	const int insn_cnt = env->prog->len;  in bpf_opt_hard_wire_dead_code_branches()
588 		if (bpf_prog_is_offloaded(env->prog->aux))  in bpf_opt_hard_wire_dead_code_branches()
598 	int insn_cnt = env->prog->len;  in bpf_opt_remove_dead_code()
613 		insn_cnt = env->prog->len;  in bpf_opt_remove_dead_code()
621 	struct bpf_insn *insn = env->prog->insnsi;  in bpf_opt_remove_nops()
622 	int insn_cnt = env->prog->len;  in bpf_opt_remove_nops()
652 	int i, patch_len, delta = 0, len = env->prog->len;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
653 	struct bpf_insn *insns = env->prog->insnsi;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
681 			if (bpf_is_reg64(env->prog, &insn)) {  in bpf_opt_subreg_zext_lo32_rnd_hi32()
732 		env->prog = new_prog;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
751 	const int insn_cnt = env->prog->len;  in bpf_convert_ctx_accesses()
762 		epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog,  in bpf_convert_ctx_accesses()
773 			insn_buf[cnt++] = env->prog->insnsi[0];  in bpf_convert_ctx_accesses()
777 			env->prog = new_prog;  in bpf_convert_ctx_accesses()
792 					env->prog);  in bpf_convert_ctx_accesses()
801 			env->prog = new_prog;  in bpf_convert_ctx_accesses()
811 		WARN_ON(adjust_jmp_off(env->prog, 0, delta));  in bpf_convert_ctx_accesses()
813 	if (bpf_prog_is_offloaded(env->prog->aux))  in bpf_convert_ctx_accesses()
816 	insn = env->prog->insnsi + delta;  in bpf_convert_ctx_accesses()
835 			env->prog = new_prog;  in bpf_convert_ctx_accesses()
868 			env->prog->aux->num_exentries++;  in bpf_convert_ctx_accesses()
908 			env->prog = new_prog;  in bpf_convert_ctx_accesses()
940 			env->prog->aux->num_exentries++;  in bpf_convert_ctx_accesses()
960 				env->prog->aux->num_exentries++;  in bpf_convert_ctx_accesses()
1000 		cnt = convert_ctx_access(type, insn, insn_buf, env->prog,  in bpf_convert_ctx_accesses()
1044 		env->prog = new_prog;  in bpf_convert_ctx_accesses()
1068 	env->subprog_info[env->subprog_cnt].start = env->prog->len;  in bpf_restore_subprog_starts()
1073 	struct bpf_prog *prog = env->prog, **func, *tmp;  in jit_subprogs()  local
1080 	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {  in jit_subprogs()
1117 	err = bpf_prog_alloc_jited_linfo(prog);  in jit_subprogs()
1122 	func = kzalloc_objs(prog, env->subprog_cnt);  in jit_subprogs()
1139 		memcpy(func[i]->insnsi, &prog->insnsi[subprog_start],  in jit_subprogs()
1141 		func[i]->type = prog->type;  in jit_subprogs()
1146 		func[i]->sleepable = prog->sleepable;  in jit_subprogs()
1147 		func[i]->blinded = prog->blinded;  in jit_subprogs()
1150 		func[i]->aux->btf = prog->aux->btf;  in jit_subprogs()
1152 		func[i]->aux->func_info = prog->aux->func_info;  in jit_subprogs()
1153 		func[i]->aux->func_info_cnt = prog->aux->func_info_cnt;  in jit_subprogs()
1154 		func[i]->aux->poke_tab = prog->aux->poke_tab;  in jit_subprogs()
1155 		func[i]->aux->size_poke_tab = prog->aux->size_poke_tab;  in jit_subprogs()
1156 		func[i]->aux->main_prog_aux = prog->aux;  in jit_subprogs()
1158 		for (j = 0; j < prog->aux->size_poke_tab; j++) {  in jit_subprogs()
1161 			poke = &prog->aux->poke_tab[j];  in jit_subprogs()
1173 		func[i]->blinding_requested = prog->blinding_requested;  in jit_subprogs()
1174 		func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;  in jit_subprogs()
1175 		func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab;  in jit_subprogs()
1176 		func[i]->aux->linfo = prog->aux->linfo;  in jit_subprogs()
1177 		func[i]->aux->nr_linfo = prog->aux->nr_linfo;  in jit_subprogs()
1178 		func[i]->aux->jited_linfo = prog->aux->jited_linfo;  in jit_subprogs()
1180 		func[i]->aux->arena = prog->aux->arena;  in jit_subprogs()
1205 		func[i]->aux->token = prog->aux->token;  in jit_subprogs()
1287 	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {  in jit_subprogs()
1301 	prog->jited = 1;  in jit_subprogs()
1302 	prog->bpf_func = func[0]->bpf_func;  in jit_subprogs()
1303 	prog->jited_len = func[0]->jited_len;  in jit_subprogs()
1304 	prog->aux->extable = func[0]->aux->extable;  in jit_subprogs()
1305 	prog->aux->num_exentries = func[0]->aux->num_exentries;  in jit_subprogs()
1306 	prog->aux->func = func;  in jit_subprogs()
1307 	prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;  in jit_subprogs()
1308 	prog->aux->real_func_cnt = env->subprog_cnt;  in jit_subprogs()
1309 	prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func;  in jit_subprogs()
1310 	prog->aux->exception_boundary = func[0]->aux->exception_boundary;  in jit_subprogs()
1311 	prog->aux->stack_arg_sp_adjust = func[0]->aux->stack_arg_sp_adjust;  in jit_subprogs()
1312 	bpf_prog_jit_attempt_done(prog);  in jit_subprogs()
1319 	for (i = 0; i < prog->aux->size_poke_tab; i++) {  in jit_subprogs()
1320 		map_ptr = prog->aux->poke_tab[i].tail_call.map;  in jit_subprogs()
1321 		map_ptr->ops->map_poke_untrack(map_ptr, prog->aux);  in jit_subprogs()
1335 	bpf_prog_jit_attempt_done(prog);  in jit_subprogs()
1344 	struct bpf_prog *prog, *orig_prog;  in bpf_jit_subprogs()  local
1350 	prog = orig_prog = env->prog;  in bpf_jit_subprogs()
1351 	if (bpf_prog_need_blind(prog)) {  in bpf_jit_subprogs()
1357 		prog = bpf_jit_blind_constants(env, prog);  in bpf_jit_subprogs()
1358 		if (IS_ERR(prog)) {  in bpf_jit_subprogs()
1360 			prog = orig_prog;  in bpf_jit_subprogs()
1371 		bpf_jit_prog_release_other(prog, orig_prog);  in bpf_jit_subprogs()
1379 		bpf_jit_prog_release_other(orig_prog, prog);  in bpf_jit_subprogs()
1381 		prog = env->prog = orig_prog;  in bpf_jit_subprogs()
1390 			for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {  in bpf_jit_subprogs()
1405 	prog->jit_requested = 0;  in bpf_jit_subprogs()
1406 	prog->blinding_requested = 0;  in bpf_jit_subprogs()
1413 	struct bpf_prog *prog = env->prog;  in bpf_fixup_call_args()  local
1414 	struct bpf_insn *insn = prog->insnsi;  in bpf_fixup_call_args()
1431 	if (env->prog->jit_requested &&  in bpf_fixup_call_args()
1432 	    !bpf_prog_is_offloaded(env->prog->aux)) {  in bpf_fixup_call_args()
1440 	if (prog->jit_required) {  in bpf_fixup_call_args()
1450 	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {  in bpf_fixup_call_args()
1457 	for (i = 0; i < prog->len; i++, insn++) {  in bpf_fixup_call_args()
1488 	struct bpf_prog *prog;  in add_hidden_subprog()  local
1499 	prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len);  in add_hidden_subprog()
1500 	if (!prog)  in add_hidden_subprog()
1502 	env->prog = prog;  in add_hidden_subprog()
1504 	info[cnt].start = prog->len - len + 1;  in add_hidden_subprog()
1515 	struct bpf_prog *prog = env->prog;  in bpf_do_misc_fixups()  local
1516 	enum bpf_attach_type eatype = prog->expected_attach_type;  in bpf_do_misc_fixups()
1517 	enum bpf_prog_type prog_type = resolve_prog_type(prog);  in bpf_do_misc_fixups()
1518 	struct bpf_insn *insn = prog->insnsi;  in bpf_do_misc_fixups()
1520 	const int insn_cnt = prog->len;  in bpf_do_misc_fixups()
1534 		*patch++ = env->prog->insnsi[insn_cnt - 1];  in bpf_do_misc_fixups()
1540 		prog = env->prog;  in bpf_do_misc_fixups()
1541 		insn = prog->insnsi;  in bpf_do_misc_fixups()
1549 		if (is_addr_space_cast32(env->prog, insn)) {  in bpf_do_misc_fixups()
1586 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1677 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1707 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1727 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1780 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1825 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1846 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1862 			prog->jit_required = true;  in bpf_do_misc_fixups()
1883 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1904 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1911 			prog->jit_required = 1;  in bpf_do_misc_fixups()
1916 			prog->dst_needed = 1;  in bpf_do_misc_fixups()
1920 			prog->kprobe_override = 1;  in bpf_do_misc_fixups()
1927 			prog->cb_access = 1;  in bpf_do_misc_fixups()
1929 				prog->aux->stack_depth = MAX_BPF_STACK;  in bpf_do_misc_fixups()
1930 			prog->aux->max_pkt_offset = MAX_PACKET_OFF;  in bpf_do_misc_fixups()
1941 			if (env->bpf_capable && !prog->blinding_requested &&  in bpf_do_misc_fixups()
1942 			    prog->jit_requested &&  in bpf_do_misc_fixups()
1953 				ret = bpf_jit_add_poke_descriptor(prog, &desc);  in bpf_do_misc_fixups()
1991 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2011 				BPF_LD_IMM64(BPF_REG_3, (long)prog->aux),  in bpf_do_misc_fixups()
2024 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2043 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2052 		if (prog->jit_requested && BITS_PER_LONG == 64 &&  in bpf_do_misc_fixups()
2079 					prog->jit_required = true;  in bpf_do_misc_fixups()
2087 				env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2151 		if (prog->jit_requested && BITS_PER_LONG == 64 &&  in bpf_do_misc_fixups()
2170 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2185 			prog->jit_required = true;  in bpf_do_misc_fixups()
2199 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2207 			prog->jit_required = true;  in bpf_do_misc_fixups()
2218 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2227 				int nr_args = btf_type_vlen(prog->aux->attach_func_proto);  in bpf_do_misc_fixups()
2252 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2284 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2293 				int nr_args = btf_type_vlen(prog->aux->attach_func_proto);  in bpf_do_misc_fixups()
2310 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2325 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2332 		    prog->jit_requested && BITS_PER_LONG == 64 &&  in bpf_do_misc_fixups()
2381 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2387 		if (prog->jit_requested && BITS_PER_LONG == 64 &&  in bpf_do_misc_fixups()
2399 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2404 		fn = env->ops->get_func_proto(insn->imm, env->prog);  in bpf_do_misc_fixups()
2421 			if (stack_depth > MAX_BPF_STACK && !prog->jit_requested) {  in bpf_do_misc_fixups()
2434 	env->prog->aux->stack_depth = subprogs[0].stack_depth;  in bpf_do_misc_fixups()
2460 		insn_buf[cnt++] = env->prog->insnsi[subprog_start];  in bpf_do_misc_fixups()
2465 		env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2472 		WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta));  in bpf_do_misc_fixups()
2476 	for (i = 0; i < prog->aux->size_poke_tab; i++) {  in bpf_do_misc_fixups()
2477 		map_ptr = prog->aux->poke_tab[i].tail_call.map;  in bpf_do_misc_fixups()
2485 		ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux);  in bpf_do_misc_fixups()
2595 	struct bpf_insn *insn = env->prog->insnsi;  in bpf_optimize_bpf_loop()
2596 	int insn_cnt = env->prog->len;  in bpf_optimize_bpf_loop()
2618 			env->prog  = new_prog;  in bpf_optimize_bpf_loop()
2631 	env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;  in bpf_optimize_bpf_loop()
2643 	struct bpf_insn *insn = env->prog->insnsi;  in bpf_remove_fastcall_spills_fills()
2644 	int insn_cnt = env->prog->len;  in bpf_remove_fastcall_spills_fills()