14 #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)  argument
88 static int set_kfunc_desc_imm(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc)  in set_kfunc_desc_imm()  argument
98 			verbose(env, "address of kernel func_id %u is out of range\n",  in set_kfunc_desc_imm()
107 static int sort_kfunc_descs_by_imm_off(struct bpf_verifier_env *env)  in sort_kfunc_descs_by_imm_off()  argument
112 	tab = env->prog->aux->kfunc_tab;  in sort_kfunc_descs_by_imm_off()
117 		err = set_kfunc_desc_imm(env, &tab->descs[i]);  in sort_kfunc_descs_by_imm_off()
127 static int add_kfunc_in_insns(struct bpf_verifier_env *env,  in add_kfunc_in_insns()  argument
134 			ret = bpf_add_kfunc_call(env, insn->imm, insn->off);  in add_kfunc_in_insns()
143 static int get_callee_stack_depth(struct bpf_verifier_env *env,  in get_callee_stack_depth()  argument
148 	subprog = bpf_find_subprog(env, start);  in get_callee_stack_depth()
149 	if (verifier_bug_if(subprog < 0, env, "get stack depth: no program at insn %d", start))  in get_callee_stack_depth()
151 	return env->subprog_info[subprog].stack_depth;  in get_callee_stack_depth()
159 static void adjust_insn_aux_data(struct bpf_verifier_env *env,  in adjust_insn_aux_data()  argument
162 	struct bpf_insn_aux_data *data = env->insn_aux_data;  in adjust_insn_aux_data()
200 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)  in adjust_subprog_starts()  argument
207 	for (i = 0; i <= env->subprog_cnt; i++) {  in adjust_subprog_starts()
208 		if (env->subprog_info[i].start <= off)  in adjust_subprog_starts()
210 		env->subprog_info[i].start += len - 1;  in adjust_subprog_starts()
214 static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len)  in adjust_insn_arrays()  argument
221 	for (i = 0; i < env->insn_array_map_cnt; i++)  in adjust_insn_arrays()
222 		bpf_insn_array_adjust(env->insn_array_maps[i], off, len);  in adjust_insn_arrays()
225 static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len)  in adjust_insn_arrays_after_remove()  argument
229 	for (i = 0; i < env->insn_array_map_cnt; i++)  in adjust_insn_arrays_after_remove()
230 		bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len);  in adjust_insn_arrays_after_remove()
247 struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,  in bpf_patch_insn_data()  argument
254 		new_data = vrealloc(env->insn_aux_data,  in bpf_patch_insn_data()
255 				    array_size(env->prog->len + len - 1,  in bpf_patch_insn_data()
261 		env->insn_aux_data = new_data;  in bpf_patch_insn_data()
264 	new_prog = bpf_patch_insn_single(env->prog, off, patch, len);  in bpf_patch_insn_data()
267 			verbose(env,  in bpf_patch_insn_data()
269 				env->insn_aux_data[off].orig_idx);  in bpf_patch_insn_data()
272 	adjust_insn_aux_data(env, new_prog, off, len);  in bpf_patch_insn_data()
273 	adjust_subprog_starts(env, off, len);  in bpf_patch_insn_data()
274 	adjust_insn_arrays(env, off, len);  in bpf_patch_insn_data()
317 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,  in adjust_subprog_starts_after_remove()  argument
323 	for (i = 0; i < env->subprog_cnt; i++)  in adjust_subprog_starts_after_remove()
324 		if (env->subprog_info[i].start >= off)  in adjust_subprog_starts_after_remove()
327 	for (j = i; j < env->subprog_cnt; j++)  in adjust_subprog_starts_after_remove()
328 		if (env->subprog_info[j].start >= off + cnt)  in adjust_subprog_starts_after_remove()
333 	if (env->subprog_info[j].start != off + cnt)  in adjust_subprog_starts_after_remove()
337 		struct bpf_prog_aux *aux = env->prog->aux;  in adjust_subprog_starts_after_remove()
341 		move = env->subprog_cnt + 1 - j;  in adjust_subprog_starts_after_remove()
343 		memmove(env->subprog_info + i,  in adjust_subprog_starts_after_remove()
344 			env->subprog_info + j,  in adjust_subprog_starts_after_remove()
345 			sizeof(*env->subprog_info) * move);  in adjust_subprog_starts_after_remove()
346 		env->subprog_cnt -= j - i;  in adjust_subprog_starts_after_remove()
362 		if (env->subprog_info[i].start == off)  in adjust_subprog_starts_after_remove()
367 	for (; i <= env->subprog_cnt; i++)  in adjust_subprog_starts_after_remove()
368 		env->subprog_info[i].start -= cnt;  in adjust_subprog_starts_after_remove()
373 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,  in bpf_adj_linfo_after_remove()  argument
376 	struct bpf_prog *prog = env->prog;  in bpf_adj_linfo_after_remove()
423 	for (i = 0; i <= env->subprog_cnt; i++)  in bpf_adj_linfo_after_remove()
424 		if (env->subprog_info[i].linfo_idx > l_off) {  in bpf_adj_linfo_after_remove()
428 			if (env->subprog_info[i].linfo_idx >= l_off + l_cnt)  in bpf_adj_linfo_after_remove()
429 				env->subprog_info[i].linfo_idx -= l_cnt;  in bpf_adj_linfo_after_remove()
431 				env->subprog_info[i].linfo_idx = l_off;  in bpf_adj_linfo_after_remove()
440 void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len)  in bpf_clear_insn_aux_data()  argument
442 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;  in bpf_clear_insn_aux_data()
443 	struct bpf_insn *insns = env->prog->insnsi;  in bpf_clear_insn_aux_data()
458 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)  in verifier_remove_insns()  argument
460 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;  in verifier_remove_insns()
461 	unsigned int orig_prog_len = env->prog->len;  in verifier_remove_insns()
464 	if (bpf_prog_is_offloaded(env->prog->aux))  in verifier_remove_insns()
465 		bpf_prog_offload_remove_insns(env, off, cnt);  in verifier_remove_insns()
468 	bpf_clear_insn_aux_data(env, off, cnt);  in verifier_remove_insns()
470 	err = bpf_remove_insns(env->prog, off, cnt);  in verifier_remove_insns()
474 	err = adjust_subprog_starts_after_remove(env, off, cnt);  in verifier_remove_insns()
478 	err = bpf_adj_linfo_after_remove(env, off, cnt);  in verifier_remove_insns()
482 	adjust_insn_arrays_after_remove(env, off, cnt);  in verifier_remove_insns()
507 void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)  in bpf_opt_hard_wire_dead_code_branches()  argument
509 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;  in bpf_opt_hard_wire_dead_code_branches()
511 	struct bpf_insn *insn = env->prog->insnsi;  in bpf_opt_hard_wire_dead_code_branches()
512 	const int insn_cnt = env->prog->len;  in bpf_opt_hard_wire_dead_code_branches()
526 		if (bpf_prog_is_offloaded(env->prog->aux))  in bpf_opt_hard_wire_dead_code_branches()
527 			bpf_prog_offload_replace_insn(env, i, &ja);  in bpf_opt_hard_wire_dead_code_branches()
533 int bpf_opt_remove_dead_code(struct bpf_verifier_env *env)  in bpf_opt_remove_dead_code()  argument
535 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;  in bpf_opt_remove_dead_code()
536 	int insn_cnt = env->prog->len;  in bpf_opt_remove_dead_code()
548 		err = verifier_remove_insns(env, i, j);  in bpf_opt_remove_dead_code()
551 		insn_cnt = env->prog->len;  in bpf_opt_remove_dead_code()
557 int bpf_opt_remove_nops(struct bpf_verifier_env *env)  in bpf_opt_remove_nops()  argument
559 	struct bpf_insn *insn = env->prog->insnsi;  in bpf_opt_remove_nops()
560 	int insn_cnt = env->prog->len;  in bpf_opt_remove_nops()
571 		err = verifier_remove_insns(env, i, 1);  in bpf_opt_remove_nops()
582 int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,  in bpf_opt_subreg_zext_lo32_rnd_hi32()  argument
587 	struct bpf_insn *zext_patch = env->insn_buf;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
588 	struct bpf_insn *rnd_hi32_patch = &env->insn_buf[2];  in bpf_opt_subreg_zext_lo32_rnd_hi32()
589 	struct bpf_insn_aux_data *aux = env->insn_aux_data;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
590 	int i, patch_len, delta = 0, len = env->prog->len;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
591 	struct bpf_insn *insns = env->prog->insnsi;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
661 		if (verifier_bug_if(load_reg == -1, env,  in bpf_opt_subreg_zext_lo32_rnd_hi32()
671 		new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len);  in bpf_opt_subreg_zext_lo32_rnd_hi32()
674 		env->prog = new_prog;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
676 		aux = env->insn_aux_data;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
688 int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)  in bpf_convert_ctx_accesses()  argument
690 	struct bpf_subprog_info *subprogs = env->subprog_info;  in bpf_convert_ctx_accesses()
691 	const struct bpf_verifier_ops *ops = env->ops;  in bpf_convert_ctx_accesses()
693 	const int insn_cnt = env->prog->len;  in bpf_convert_ctx_accesses()
694 	struct bpf_insn *epilogue_buf = env->epilogue_buf;  in bpf_convert_ctx_accesses()
695 	struct bpf_insn *insn_buf = env->insn_buf;  in bpf_convert_ctx_accesses()
704 		epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog,  in bpf_convert_ctx_accesses()
707 			verifier_bug(env, "epilogue is too long");  in bpf_convert_ctx_accesses()
715 			insn_buf[cnt++] = env->prog->insnsi[0];  in bpf_convert_ctx_accesses()
716 			new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);  in bpf_convert_ctx_accesses()
719 			env->prog = new_prog;  in bpf_convert_ctx_accesses()
722 			ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1);  in bpf_convert_ctx_accesses()
728 	if (ops->gen_prologue || env->seen_direct_write) {  in bpf_convert_ctx_accesses()
730 			verifier_bug(env, "gen_prologue is null");  in bpf_convert_ctx_accesses()
733 		cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,  in bpf_convert_ctx_accesses()
734 					env->prog);  in bpf_convert_ctx_accesses()
736 			verifier_bug(env, "prologue is too long");  in bpf_convert_ctx_accesses()
739 			new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);  in bpf_convert_ctx_accesses()
743 			env->prog = new_prog;  in bpf_convert_ctx_accesses()
746 			ret = add_kfunc_in_insns(env, insn_buf, cnt - 1);  in bpf_convert_ctx_accesses()
753 		WARN_ON(adjust_jmp_off(env->prog, 0, delta));  in bpf_convert_ctx_accesses()
755 	if (bpf_prog_is_offloaded(env->prog->aux))  in bpf_convert_ctx_accesses()
758 	insn = env->prog->insnsi + delta;  in bpf_convert_ctx_accesses()
764 		if (env->insn_aux_data[i + delta].nospec) {  in bpf_convert_ctx_accesses()
765 			WARN_ON_ONCE(env->insn_aux_data[i + delta].alu_state);  in bpf_convert_ctx_accesses()
771 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_convert_ctx_accesses()
776 			env->prog = new_prog;  in bpf_convert_ctx_accesses()
807 			   env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) {  in bpf_convert_ctx_accesses()
809 			env->prog->aux->num_exentries++;  in bpf_convert_ctx_accesses()
835 		    env->insn_aux_data[i + delta].nospec_result) {  in bpf_convert_ctx_accesses()
844 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_convert_ctx_accesses()
849 			env->prog = new_prog;  in bpf_convert_ctx_accesses()
854 		switch ((int)env->insn_aux_data[i + delta].ptr_type) {  in bpf_convert_ctx_accesses()
887 				env->prog->aux->num_exentries++;  in bpf_convert_ctx_accesses()
893 					verbose(env, "sign extending loads from arena are not supported yet\n");  in bpf_convert_ctx_accesses()
900 			env->prog->aux->num_exentries++;  in bpf_convert_ctx_accesses()
906 		ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;  in bpf_convert_ctx_accesses()
922 				verifier_bug(env, "narrow ctx access misconfigured");  in bpf_convert_ctx_accesses()
937 		cnt = convert_ctx_access(type, insn, insn_buf, env->prog,  in bpf_convert_ctx_accesses()
941 			verifier_bug(env, "error during ctx access conversion (%d)", cnt);  in bpf_convert_ctx_accesses()
949 				verifier_bug(env, "narrow ctx load misconfigured");  in bpf_convert_ctx_accesses()
974 		new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_convert_ctx_accesses()
981 		env->prog = new_prog;  in bpf_convert_ctx_accesses()
988 static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env)  in bpf_dup_subprog_starts()  argument
992 	starts = kvmalloc_objs(u32, env->subprog_cnt, GFP_KERNEL_ACCOUNT);  in bpf_dup_subprog_starts()
994 		for (int i = 0; i < env->subprog_cnt; i++)  in bpf_dup_subprog_starts()
995 			starts[i] = env->subprog_info[i].start;  in bpf_dup_subprog_starts()
1000 static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts)  in bpf_restore_subprog_starts()  argument
1002 	for (int i = 0; i < env->subprog_cnt; i++)  in bpf_restore_subprog_starts()
1003 		env->subprog_info[i].start = orig_starts[i];  in bpf_restore_subprog_starts()
1005 	env->subprog_info[env->subprog_cnt].start = env->prog->len;  in bpf_restore_subprog_starts()
1008 struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env)  in bpf_dup_insn_aux_data()  argument
1013 	size = array_size(sizeof(struct bpf_insn_aux_data), env->prog->len);  in bpf_dup_insn_aux_data()
1016 		memcpy(new_aux, env->insn_aux_data, size);  in bpf_dup_insn_aux_data()
1020 void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,  in bpf_restore_insn_aux_data()  argument
1024 	vfree(env->insn_aux_data);  in bpf_restore_insn_aux_data()
1025 	env->insn_aux_data = orig_insn_aux;  in bpf_restore_insn_aux_data()
1028 static int jit_subprogs(struct bpf_verifier_env *env)  in jit_subprogs()  argument
1030 	struct bpf_prog *prog = env->prog, **func, *tmp;  in jit_subprogs()
1045 		subprog = bpf_find_subprog(env, i + insn->imm + 1);  in jit_subprogs()
1046 		if (verifier_bug_if(subprog < 0, env, "No program to jit at insn %d",  in jit_subprogs()
1056 		env->insn_aux_data[i].call_imm = insn->imm;  in jit_subprogs()
1079 	func = kzalloc_objs(prog, env->subprog_cnt);  in jit_subprogs()
1083 	for (i = 0; i < env->subprog_cnt; i++) {  in jit_subprogs()
1085 		subprog_end = env->subprog_info[i + 1].start;  in jit_subprogs()
1125 		func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;  in jit_subprogs()
1126 		if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE)  in jit_subprogs()
1136 		func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;  in jit_subprogs()
1138 		func[i]->aux->used_maps = env->used_maps;  in jit_subprogs()
1139 		func[i]->aux->used_map_cnt = env->used_map_cnt;  in jit_subprogs()
1158 		func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;  in jit_subprogs()
1159 		func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;  in jit_subprogs()
1160 		func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;  in jit_subprogs()
1161 		func[i]->aux->might_sleep = env->subprog_info[i].might_sleep;  in jit_subprogs()
1164 			func[i]->aux->exception_boundary = env->seen_exception;  in jit_subprogs()
1165 		func[i] = bpf_int_jit_compile(env, func[i]);  in jit_subprogs()
1177 	for (i = 0; i < env->subprog_cnt; i++) {  in jit_subprogs()
1204 		func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;  in jit_subprogs()
1205 		func[i]->aux->real_func_cnt = env->subprog_cnt;  in jit_subprogs()
1207 	for (i = 0; i < env->subprog_cnt; i++) {  in jit_subprogs()
1209 		tmp = bpf_int_jit_compile(env, func[i]);  in jit_subprogs()
1211 			verbose(env, "JIT doesn't support bpf-to-bpf calls\n");  in jit_subprogs()
1222 	for (i = 0; i < env->subprog_cnt; i++) {  in jit_subprogs()
1231 	for (i = 1; i < env->subprog_cnt; i++) {  in jit_subprogs()
1237 	for (i = 1; i < env->subprog_cnt; i++)  in jit_subprogs()
1246 			insn[0].imm = env->insn_aux_data[i].call_imm;  in jit_subprogs()
1253 		insn->imm = env->insn_aux_data[i].call_imm;  in jit_subprogs()
1254 		subprog = bpf_find_subprog(env, i + insn->imm + 1);  in jit_subprogs()
1264 	prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;  in jit_subprogs()
1265 	prog->aux->real_func_cnt = env->subprog_cnt;  in jit_subprogs()
1266 	prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func;  in jit_subprogs()
1283 	for (i = 0; i < env->subprog_cnt; i++) {  in jit_subprogs()
1295 int bpf_jit_subprogs(struct bpf_verifier_env *env)  in bpf_jit_subprogs()  argument
1304 	if (env->subprog_cnt <= 1)  in bpf_jit_subprogs()
1307 	prog = orig_prog = env->prog;  in bpf_jit_subprogs()
1309 		orig_insn_aux = bpf_dup_insn_aux_data(env);  in bpf_jit_subprogs()
1314 		orig_subprog_starts = bpf_dup_subprog_starts(env);  in bpf_jit_subprogs()
1320 		prog = bpf_jit_blind_constants(env, prog);  in bpf_jit_subprogs()
1329 	err = jit_subprogs(env);  in bpf_jit_subprogs()
1345 		prog = env->prog = orig_prog;  in bpf_jit_subprogs()
1358 				insn->imm = env->insn_aux_data[i].call_imm;  in bpf_jit_subprogs()
1365 	bpf_restore_subprog_starts(env, orig_subprog_starts);  in bpf_jit_subprogs()
1366 	bpf_restore_insn_aux_data(env, orig_insn_aux);  in bpf_jit_subprogs()
1375 int bpf_fixup_call_args(struct bpf_verifier_env *env)  in bpf_fixup_call_args()  argument
1378 	struct bpf_prog *prog = env->prog;  in bpf_fixup_call_args()
1385 	if (env->prog->jit_requested &&  in bpf_fixup_call_args()
1386 	    !bpf_prog_is_offloaded(env->prog->aux)) {  in bpf_fixup_call_args()
1387 		err = bpf_jit_subprogs(env);  in bpf_fixup_call_args()
1395 		verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");  in bpf_fixup_call_args()
1398 	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {  in bpf_fixup_call_args()
1402 		verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");  in bpf_fixup_call_args()
1410 			verbose(env, "callbacks are not allowed in non-JITed programs\n");  in bpf_fixup_call_args()
1416 		depth = get_callee_stack_depth(env, insn, i);  in bpf_fixup_call_args()
1421 			verbose(env, "stack depth %d exceeds interpreter stack depth limit\n",  in bpf_fixup_call_args()
1433 static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len)  in add_hidden_subprog()  argument
1435 	struct bpf_subprog_info *info = env->subprog_info;  in add_hidden_subprog()
1436 	int cnt = env->subprog_cnt;  in add_hidden_subprog()
1440 	if (env->hidden_subprog_cnt) {  in add_hidden_subprog()
1441 		verifier_bug(env, "only one hidden subprog supported");  in add_hidden_subprog()
1448 	prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len);  in add_hidden_subprog()
1451 	env->prog = prog;  in add_hidden_subprog()
1454 	env->subprog_cnt++;  in add_hidden_subprog()
1455 	env->hidden_subprog_cnt++;  in add_hidden_subprog()
1462 int bpf_do_misc_fixups(struct bpf_verifier_env *env)  in bpf_do_misc_fixups()  argument
1464 	struct bpf_prog *prog = env->prog;  in bpf_do_misc_fixups()
1472 	struct bpf_insn *insn_buf = env->insn_buf;  in bpf_do_misc_fixups()
1476 	struct bpf_subprog_info *subprogs = env->subprog_info;  in bpf_do_misc_fixups()
1480 	if (env->seen_exception && !env->exception_callback_subprog) {  in bpf_do_misc_fixups()
1483 		*patch++ = env->prog->insnsi[insn_cnt - 1];  in bpf_do_misc_fixups()
1486 		ret = add_hidden_subprog(env, insn_buf, patch - insn_buf);  in bpf_do_misc_fixups()
1489 		prog = env->prog;  in bpf_do_misc_fixups()
1492 		env->exception_callback_subprog = env->subprog_cnt - 1;  in bpf_do_misc_fixups()
1494 		bpf_mark_subprog_exc_cb(env, env->exception_callback_subprog);  in bpf_do_misc_fixups()
1500 			    (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) {  in bpf_do_misc_fixups()
1510 		if (env->insn_aux_data[i + delta].needs_zext)  in bpf_do_misc_fixups()
1533 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1538 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1624 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1629 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1654 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1659 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1668 			cnt = env->ops->gen_ld_abs(insn, insn_buf);  in bpf_do_misc_fixups()
1670 				verifier_bug(env, "%d insns generated for ld_abs", cnt);  in bpf_do_misc_fixups()
1674 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1679 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1693 			aux = &env->insn_aux_data[i + delta];  in bpf_do_misc_fixups()
1727 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1732 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1772 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1777 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1793 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1798 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1808 			ret = bpf_fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt);  in bpf_do_misc_fixups()
1814 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1819 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1841 			if (!bpf_allow_tail_call_in_subprogs(env))  in bpf_do_misc_fixups()
1853 			aux = &env->insn_aux_data[i + delta];  in bpf_do_misc_fixups()
1854 			if (env->bpf_capable && !prog->blinding_requested &&  in bpf_do_misc_fixups()
1868 					verbose(env, "adding tail call poke descriptor failed\n");  in bpf_do_misc_fixups()
1886 				verbose(env, "tail_call abusing map_ptr\n");  in bpf_do_misc_fixups()
1899 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1904 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1932 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1937 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1943 		if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) {  in bpf_do_misc_fixups()
1951 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1956 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1975 			aux = &env->insn_aux_data[i + delta];  in bpf_do_misc_fixups()
1987 					verifier_bug(env, "%d insns generated for map lookup", cnt);  in bpf_do_misc_fixups()
1991 				new_prog = bpf_patch_insn_data(env, i + delta,  in bpf_do_misc_fixups()
1997 				env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2074 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,  in bpf_do_misc_fixups()
2080 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2088 		    bpf_verifier_inlines_helper_call(env, insn->imm)) {  in bpf_do_misc_fixups()
2103 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
2108 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2115 		    bpf_verifier_inlines_helper_call(env, insn->imm)) {  in bpf_do_misc_fixups()
2121 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
2126 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2155 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
2160 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2185 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
2190 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2211 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
2216 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2227 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);  in bpf_do_misc_fixups()
2231 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2282 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
2287 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2300 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
2305 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2310 		fn = env->ops->get_func_proto(insn->imm, env->prog);  in bpf_do_misc_fixups()
2315 			verifier_bug(env,  in bpf_do_misc_fixups()
2328 				verbose(env, "stack size %d(extra %d) is too large\n",  in bpf_do_misc_fixups()
2340 	env->prog->aux->stack_depth = subprogs[0].stack_depth;  in bpf_do_misc_fixups()
2341 	for (i = 0; i < env->subprog_cnt; i++) {  in bpf_do_misc_fixups()
2351 			verifier_bug(env, "stack_slots supports may_goto only");  in bpf_do_misc_fixups()
2366 		insn_buf[cnt++] = env->prog->insnsi[subprog_start];  in bpf_do_misc_fixups()
2368 		new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt);  in bpf_do_misc_fixups()
2371 		env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2378 		WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta));  in bpf_do_misc_fixups()
2387 			verifier_bug(env, "poke tab is misconfigured");  in bpf_do_misc_fixups()
2393 			verbose(env, "tracking tail call prog failed\n");  in bpf_do_misc_fixups()
2398 	ret = sort_kfunc_descs_by_imm_off(env);  in bpf_do_misc_fixups()
2405 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env,  in inline_bpf_loop()  argument
2418 	struct bpf_insn *insn_buf = env->insn_buf;  in inline_bpf_loop()
2467 	new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt);  in inline_bpf_loop()
2472 	callback_start = env->subprog_info[callback_subprogno].start;  in inline_bpf_loop()
2497 int bpf_optimize_bpf_loop(struct bpf_verifier_env *env)  in bpf_optimize_bpf_loop()  argument
2499 	struct bpf_subprog_info *subprogs = env->subprog_info;  in bpf_optimize_bpf_loop()
2501 	struct bpf_insn *insn = env->prog->insnsi;  in bpf_optimize_bpf_loop()
2502 	int insn_cnt = env->prog->len;  in bpf_optimize_bpf_loop()
2509 			&env->insn_aux_data[i + delta].loop_inline_state;  in bpf_optimize_bpf_loop()
2515 			new_prog = inline_bpf_loop(env,  in bpf_optimize_bpf_loop()
2524 			env->prog  = new_prog;  in bpf_optimize_bpf_loop()
2537 	env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;  in bpf_optimize_bpf_loop()
2545 int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env)  in bpf_remove_fastcall_spills_fills()  argument
2547 	struct bpf_subprog_info *subprog = env->subprog_info;  in bpf_remove_fastcall_spills_fills()
2548 	struct bpf_insn_aux_data *aux = env->insn_aux_data;  in bpf_remove_fastcall_spills_fills()
2549 	struct bpf_insn *insn = env->prog->insnsi;  in bpf_remove_fastcall_spills_fills()
2550 	int insn_cnt = env->prog->len;  in bpf_remove_fastcall_spills_fills()