340 static int adjust_jmp_off(struct bpf_prog *prog, u32 tgt_idx, u32 delta)  in adjust_jmp_off()  argument
350 		if (tgt_idx <= i && i < tgt_idx + delta)  in adjust_jmp_off()
360 			if (check_add_overflow(insn->imm, delta, &imm))  in adjust_jmp_off()
366 			if (check_add_overflow(insn->off, delta, &off))  in adjust_jmp_off()
647 	int i, patch_len, delta = 0, len = env->prog->len;  in bpf_opt_subreg_zext_lo32_rnd_hi32()  local
658 		int adj_idx = i + delta;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
730 		delta += patch_len - 1;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
745 	int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0;  in bpf_convert_ctx_accesses()  local
773 			delta += cnt - 1;  in bpf_convert_ctx_accesses()
797 			delta += cnt - 1;  in bpf_convert_ctx_accesses()
805 	if (delta)  in bpf_convert_ctx_accesses()
806 		WARN_ON(adjust_jmp_off(env->prog, 0, delta));  in bpf_convert_ctx_accesses()
811 	insn = env->prog->insnsi + delta;  in bpf_convert_ctx_accesses()
818 		if (env->insn_aux_data[i + delta].nospec) {  in bpf_convert_ctx_accesses()
819 			WARN_ON_ONCE(env->insn_aux_data[i + delta].alu_state);  in bpf_convert_ctx_accesses()
825 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_convert_ctx_accesses()
829 			delta    += cnt - 1;  in bpf_convert_ctx_accesses()
831 			insn      = new_prog->insnsi + i + delta;  in bpf_convert_ctx_accesses()
861 			   env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) {  in bpf_convert_ctx_accesses()
867 			   i + delta < subprogs[1].start) {  in bpf_convert_ctx_accesses()
871 				insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - delta - 1);  in bpf_convert_ctx_accesses()
881 				epilogue_idx = i + delta;  in bpf_convert_ctx_accesses()
889 		    env->insn_aux_data[i + delta].nospec_result) {  in bpf_convert_ctx_accesses()
898 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_convert_ctx_accesses()
902 			delta    += cnt - 1;  in bpf_convert_ctx_accesses()
904 			insn      = new_prog->insnsi + i + delta;  in bpf_convert_ctx_accesses()
908 		ptr_type = env->insn_aux_data[i + delta].ptr_type;  in bpf_convert_ctx_accesses()
964 		ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;  in bpf_convert_ctx_accesses()
1032 		new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_convert_ctx_accesses()
1036 		delta += cnt - 1;  in bpf_convert_ctx_accesses()
1040 		insn      = new_prog->insnsi + i + delta;  in bpf_convert_ctx_accesses()
1521 	int i, ret, cnt, delta = 0, cur_subprog = 0;  in bpf_do_misc_fixups()  local
1553 		if (env->insn_aux_data[i + delta].needs_zext)  in bpf_do_misc_fixups()
1576 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1580 			delta    += cnt - 1;  in bpf_do_misc_fixups()
1582 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
1667 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1671 			delta    += cnt - 1;  in bpf_do_misc_fixups()
1673 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
1697 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1701 			delta    += cnt - 1;  in bpf_do_misc_fixups()
1703 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
1717 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1721 			delta    += cnt - 1;  in bpf_do_misc_fixups()
1723 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
1736 			aux = &env->insn_aux_data[i + delta];  in bpf_do_misc_fixups()
1770 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1774 			delta    += cnt - 1;  in bpf_do_misc_fixups()
1776 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
1815 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1819 			delta += cnt - 1;  in bpf_do_misc_fixups()
1821 			insn = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
1836 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1840 			delta += cnt - 1;  in bpf_do_misc_fixups()
1842 			insn = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
1852 			aux = &env->insn_aux_data[i + delta];  in bpf_do_misc_fixups()
1873 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1877 			delta    += cnt - 1;  in bpf_do_misc_fixups()
1879 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
1888 			ret = bpf_fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt);  in bpf_do_misc_fixups()
1894 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1898 			delta	 += cnt - 1;  in bpf_do_misc_fixups()
1900 			insn	  = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
1935 			aux = &env->insn_aux_data[i + delta];  in bpf_do_misc_fixups()
1945 					.insn_idx = i + delta,  in bpf_do_misc_fixups()
1981 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
1985 			delta    += cnt - 1;  in bpf_do_misc_fixups()
1987 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
2014 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
2018 			delta    += cnt - 1;  in bpf_do_misc_fixups()
2020 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
2025 		if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) {  in bpf_do_misc_fixups()
2033 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
2037 			delta += cnt - 1;  in bpf_do_misc_fixups()
2039 			insn = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
2057 			aux = &env->insn_aux_data[i + delta];  in bpf_do_misc_fixups()
2076 				new_prog = bpf_patch_insn_data(env, i + delta,  in bpf_do_misc_fixups()
2081 				delta    += cnt - 1;  in bpf_do_misc_fixups()
2083 				insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
2159 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,  in bpf_do_misc_fixups()
2164 			delta    += cnt - 1;  in bpf_do_misc_fixups()
2166 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
2189 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
2193 			delta    += cnt - 1;  in bpf_do_misc_fixups()
2195 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
2208 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
2212 			delta    += cnt - 1;  in bpf_do_misc_fixups()
2214 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
2242 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
2246 			delta    += cnt - 1;  in bpf_do_misc_fixups()
2248 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
2274 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
2278 			delta    += cnt - 1;  in bpf_do_misc_fixups()
2280 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
2300 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
2304 			delta    += cnt - 1;  in bpf_do_misc_fixups()
2306 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
2316 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);  in bpf_do_misc_fixups()
2321 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
2371 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
2375 			delta    += cnt - 1;  in bpf_do_misc_fixups()
2377 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
2389 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in bpf_do_misc_fixups()
2393 			delta    += cnt - 1;  in bpf_do_misc_fixups()
2395 			insn      = new_prog->insnsi + i + delta;  in bpf_do_misc_fixups()
2411 		if (subprogs[cur_subprog + 1].start == i + delta + 1) {  in bpf_do_misc_fixups()
2431 		int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1;  in bpf_do_misc_fixups()  local
2434 		int slots = delta, cnt = 0;  in bpf_do_misc_fixups()
2467 		WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta));  in bpf_do_misc_fixups()
2589 	int i, cur_subprog = 0, cnt, delta = 0;  in bpf_optimize_bpf_loop()  local
2598 			&env->insn_aux_data[i + delta].loop_inline_state;  in bpf_optimize_bpf_loop()
2605 						   i + delta,  in bpf_optimize_bpf_loop()
2612 			delta     += cnt - 1;  in bpf_optimize_bpf_loop()
2614 			insn       = new_prog->insnsi + i + delta;  in bpf_optimize_bpf_loop()
2617 		if (subprogs[cur_subprog + 1].start == i + delta + 1) {  in bpf_optimize_bpf_loop()