71 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,  in bpf_jit_find_kfunc_model()  argument
81 	tab = prog->aux->kfunc_tab;  in bpf_jit_find_kfunc_model()
112 	tab = env->prog->aux->kfunc_tab;  in sort_kfunc_descs_by_imm_off()
233 static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len)  in adjust_poke_descs()  argument
235 	struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;  in adjust_poke_descs()
236 	int i, sz = prog->aux->size_poke_tab;  in adjust_poke_descs()
255 				    array_size(env->prog->len + len - 1,  in bpf_patch_insn_data()
264 	new_prog = bpf_patch_insn_single(env->prog, off, patch, len);  in bpf_patch_insn_data()
283 static int adjust_jmp_off(struct bpf_prog *prog, u32 tgt_idx, u32 delta)  in adjust_jmp_off()  argument
285 	struct bpf_insn *insn = prog->insnsi;  in adjust_jmp_off()
286 	u32 insn_cnt = prog->len, i;  in adjust_jmp_off()
337 		struct bpf_prog_aux *aux = env->prog->aux;  in adjust_subprog_starts_after_remove()
376 	struct bpf_prog *prog = env->prog;  in bpf_adj_linfo_after_remove()  local
380 	nr_linfo = prog->aux->nr_linfo;  in bpf_adj_linfo_after_remove()
384 	linfo = prog->aux->linfo;  in bpf_adj_linfo_after_remove()
403 	if (prog->len != off && l_cnt &&  in bpf_adj_linfo_after_remove()
414 		prog->aux->nr_linfo -= l_cnt;  in bpf_adj_linfo_after_remove()
415 		nr_linfo = prog->aux->nr_linfo;  in bpf_adj_linfo_after_remove()
443 	struct bpf_insn *insns = env->prog->insnsi;  in bpf_clear_insn_aux_data()
461 	unsigned int orig_prog_len = env->prog->len;  in verifier_remove_insns()
464 	if (bpf_prog_is_offloaded(env->prog->aux))  in verifier_remove_insns()
470 	err = bpf_remove_insns(env->prog, off, cnt);  in verifier_remove_insns()
511 	struct bpf_insn *insn = env->prog->insnsi;  in bpf_opt_hard_wire_dead_code_branches()
512 	const int insn_cnt = env->prog->len;  in bpf_opt_hard_wire_dead_code_branches()
526 		if (bpf_prog_is_offloaded(env->prog->aux))  in bpf_opt_hard_wire_dead_code_branches()
536 	int insn_cnt = env->prog->len;  in bpf_opt_remove_dead_code()
551 		insn_cnt = env->prog->len;  in bpf_opt_remove_dead_code()
559 	struct bpf_insn *insn = env->prog->insnsi;  in bpf_opt_remove_nops()
560 	int insn_cnt = env->prog->len;  in bpf_opt_remove_nops()
590 	int i, patch_len, delta = 0, len = env->prog->len;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
591 	struct bpf_insn *insns = env->prog->insnsi;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
674 		env->prog = new_prog;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
693 	const int insn_cnt = env->prog->len;  in bpf_convert_ctx_accesses()
704 		epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog,  in bpf_convert_ctx_accesses()
715 			insn_buf[cnt++] = env->prog->insnsi[0];  in bpf_convert_ctx_accesses()
719 			env->prog = new_prog;  in bpf_convert_ctx_accesses()
734 					env->prog);  in bpf_convert_ctx_accesses()
743 			env->prog = new_prog;  in bpf_convert_ctx_accesses()
753 		WARN_ON(adjust_jmp_off(env->prog, 0, delta));  in bpf_convert_ctx_accesses()
755 	if (bpf_prog_is_offloaded(env->prog->aux))  in bpf_convert_ctx_accesses()
758 	insn = env->prog->insnsi + delta;  in bpf_convert_ctx_accesses()
776 			env->prog = new_prog;  in bpf_convert_ctx_accesses()
809 			env->prog->aux->num_exentries++;  in bpf_convert_ctx_accesses()
849 			env->prog = new_prog;  in bpf_convert_ctx_accesses()
887 				env->prog->aux->num_exentries++;  in bpf_convert_ctx_accesses()
900 			env->prog->aux->num_exentries++;  in bpf_convert_ctx_accesses()
937 		cnt = convert_ctx_access(type, insn, insn_buf, env->prog,  in bpf_convert_ctx_accesses()
981 		env->prog = new_prog;  in bpf_convert_ctx_accesses()
1005 	env->subprog_info[env->subprog_cnt].start = env->prog->len;  in bpf_restore_subprog_starts()
1013 	size = array_size(sizeof(struct bpf_insn_aux_data), env->prog->len);  in bpf_dup_insn_aux_data()
1030 	struct bpf_prog *prog = env->prog, **func, *tmp;  in jit_subprogs()  local
1037 	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {  in jit_subprogs()
1074 	err = bpf_prog_alloc_jited_linfo(prog);  in jit_subprogs()
1079 	func = kzalloc_objs(prog, env->subprog_cnt);  in jit_subprogs()
1096 		memcpy(func[i]->insnsi, &prog->insnsi[subprog_start],  in jit_subprogs()
1098 		func[i]->type = prog->type;  in jit_subprogs()
1103 		func[i]->sleepable = prog->sleepable;  in jit_subprogs()
1104 		func[i]->blinded = prog->blinded;  in jit_subprogs()
1107 		func[i]->aux->btf = prog->aux->btf;  in jit_subprogs()
1109 		func[i]->aux->func_info = prog->aux->func_info;  in jit_subprogs()
1110 		func[i]->aux->func_info_cnt = prog->aux->func_info_cnt;  in jit_subprogs()
1111 		func[i]->aux->poke_tab = prog->aux->poke_tab;  in jit_subprogs()
1112 		func[i]->aux->size_poke_tab = prog->aux->size_poke_tab;  in jit_subprogs()
1113 		func[i]->aux->main_prog_aux = prog->aux;  in jit_subprogs()
1115 		for (j = 0; j < prog->aux->size_poke_tab; j++) {  in jit_subprogs()
1118 			poke = &prog->aux->poke_tab[j];  in jit_subprogs()
1130 		func[i]->blinding_requested = prog->blinding_requested;  in jit_subprogs()
1131 		func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;  in jit_subprogs()
1132 		func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab;  in jit_subprogs()
1133 		func[i]->aux->linfo = prog->aux->linfo;  in jit_subprogs()
1134 		func[i]->aux->nr_linfo = prog->aux->nr_linfo;  in jit_subprogs()
1135 		func[i]->aux->jited_linfo = prog->aux->jited_linfo;  in jit_subprogs()
1137 		func[i]->aux->arena = prog->aux->arena;  in jit_subprogs()
1162 		func[i]->aux->token = prog->aux->token;  in jit_subprogs()
1244 	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {  in jit_subprogs()
1258 	prog->jited = 1;  in jit_subprogs()
1259 	prog->bpf_func = func[0]->bpf_func;  in jit_subprogs()
1260 	prog->jited_len = func[0]->jited_len;  in jit_subprogs()
1261 	prog->aux->extable = func[0]->aux->extable;  in jit_subprogs()
1262 	prog->aux->num_exentries = func[0]->aux->num_exentries;  in jit_subprogs()
1263 	prog->aux->func = func;  in jit_subprogs()
1264 	prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;  in jit_subprogs()
1265 	prog->aux->real_func_cnt = env->subprog_cnt;  in jit_subprogs()
1266 	prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func;  in jit_subprogs()
1267 	prog->aux->exception_boundary = func[0]->aux->exception_boundary;  in jit_subprogs()
1268 	bpf_prog_jit_attempt_done(prog);  in jit_subprogs()
1275 	for (i = 0; i < prog->aux->size_poke_tab; i++) {  in jit_subprogs()
1276 		map_ptr = prog->aux->poke_tab[i].tail_call.map;  in jit_subprogs()
1277 		map_ptr->ops->map_poke_untrack(map_ptr, prog->aux);  in jit_subprogs()
1291 	bpf_prog_jit_attempt_done(prog);  in jit_subprogs()
1300 	struct bpf_prog *prog, *orig_prog;  in bpf_jit_subprogs()  local
1307 	prog = orig_prog = env->prog;  in bpf_jit_subprogs()
1308 	if (bpf_prog_need_blind(prog)) {  in bpf_jit_subprogs()
1320 		prog = bpf_jit_blind_constants(env, prog);  in bpf_jit_subprogs()
1321 		if (IS_ERR(prog)) {  in bpf_jit_subprogs()
1323 			prog = orig_prog;  in bpf_jit_subprogs()
1334 		bpf_jit_prog_release_other(prog, orig_prog);  in bpf_jit_subprogs()
1343 		bpf_jit_prog_release_other(orig_prog, prog);  in bpf_jit_subprogs()
1345 		prog = env->prog = orig_prog;  in bpf_jit_subprogs()
1354 			for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {  in bpf_jit_subprogs()
1370 	prog->jit_requested = 0;  in bpf_jit_subprogs()
1371 	prog->blinding_requested = 0;  in bpf_jit_subprogs()
1378 	struct bpf_prog *prog = env->prog;  in bpf_fixup_call_args()  local
1379 	struct bpf_insn *insn = prog->insnsi;  in bpf_fixup_call_args()
1380 	bool has_kfunc_call = bpf_prog_has_kfunc_call(prog);  in bpf_fixup_call_args()
1385 	if (env->prog->jit_requested &&  in bpf_fixup_call_args()
1386 	    !bpf_prog_is_offloaded(env->prog->aux)) {  in bpf_fixup_call_args()
1398 	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {  in bpf_fixup_call_args()
1405 	for (i = 0; i < prog->len; i++, insn++) {  in bpf_fixup_call_args()
1437 	struct bpf_prog *prog;  in add_hidden_subprog()  local
1448 	prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len);  in add_hidden_subprog()
1449 	if (!prog)  in add_hidden_subprog()
1451 	env->prog = prog;  in add_hidden_subprog()
1453 	info[cnt].start = prog->len - len + 1;  in add_hidden_subprog()
1464 	struct bpf_prog *prog = env->prog;  in bpf_do_misc_fixups()  local
1465 	enum bpf_attach_type eatype = prog->expected_attach_type;  in bpf_do_misc_fixups()
1466 	enum bpf_prog_type prog_type = resolve_prog_type(prog);  in bpf_do_misc_fixups()
1467 	struct bpf_insn *insn = prog->insnsi;  in bpf_do_misc_fixups()
1469 	const int insn_cnt = prog->len;  in bpf_do_misc_fixups()
1483 		*patch++ = env->prog->insnsi[insn_cnt - 1];  in bpf_do_misc_fixups()
1489 		prog = env->prog;  in bpf_do_misc_fixups()
1490 		insn = prog->insnsi;  in bpf_do_misc_fixups()
1500 			    (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) {  in bpf_do_misc_fixups()
1538 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1629 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1659 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1679 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1732 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1777 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1798 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1819 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1829 			prog->dst_needed = 1;  in bpf_do_misc_fixups()
1833 			prog->kprobe_override = 1;  in bpf_do_misc_fixups()
1840 			prog->cb_access = 1;  in bpf_do_misc_fixups()
1842 				prog->aux->stack_depth = MAX_BPF_STACK;  in bpf_do_misc_fixups()
1843 			prog->aux->max_pkt_offset = MAX_PACKET_OFF;  in bpf_do_misc_fixups()
1854 			if (env->bpf_capable && !prog->blinding_requested &&  in bpf_do_misc_fixups()
1855 			    prog->jit_requested &&  in bpf_do_misc_fixups()
1866 				ret = bpf_jit_add_poke_descriptor(prog, &desc);  in bpf_do_misc_fixups()
1904 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1924 				BPF_LD_IMM64(BPF_REG_3, (long)prog->aux),  in bpf_do_misc_fixups()
1937 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1956 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
1965 		if (prog->jit_requested && BITS_PER_LONG == 64 &&  in bpf_do_misc_fixups()
1997 				env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2061 		if (prog->jit_requested && BITS_PER_LONG == 64 &&  in bpf_do_misc_fixups()
2080 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2108 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2126 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2135 				int nr_args = btf_type_vlen(prog->aux->attach_func_proto);  in bpf_do_misc_fixups()
2160 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2190 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2199 				int nr_args = btf_type_vlen(prog->aux->attach_func_proto);  in bpf_do_misc_fixups()
2216 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2231 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2238 		    prog->jit_requested && BITS_PER_LONG == 64 &&  in bpf_do_misc_fixups()
2287 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2293 		if (prog->jit_requested && BITS_PER_LONG == 64 &&  in bpf_do_misc_fixups()
2305 			env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2310 		fn = env->ops->get_func_proto(insn->imm, env->prog);  in bpf_do_misc_fixups()
2327 			if (stack_depth > MAX_BPF_STACK && !prog->jit_requested) {  in bpf_do_misc_fixups()
2340 	env->prog->aux->stack_depth = subprogs[0].stack_depth;  in bpf_do_misc_fixups()
2366 		insn_buf[cnt++] = env->prog->insnsi[subprog_start];  in bpf_do_misc_fixups()
2371 		env->prog = prog = new_prog;  in bpf_do_misc_fixups()
2378 		WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta));  in bpf_do_misc_fixups()
2382 	for (i = 0; i < prog->aux->size_poke_tab; i++) {  in bpf_do_misc_fixups()
2383 		map_ptr = prog->aux->poke_tab[i].tail_call.map;  in bpf_do_misc_fixups()
2391 		ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux);  in bpf_do_misc_fixups()
2501 	struct bpf_insn *insn = env->prog->insnsi;  in bpf_optimize_bpf_loop()
2502 	int insn_cnt = env->prog->len;  in bpf_optimize_bpf_loop()
2524 			env->prog  = new_prog;  in bpf_optimize_bpf_loop()
2537 	env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;  in bpf_optimize_bpf_loop()
2549 	struct bpf_insn *insn = env->prog->insnsi;  in bpf_remove_fastcall_spills_fills()
2550 	int insn_cnt = env->prog->len;  in bpf_remove_fastcall_spills_fills()