52 void bpf_jit_build_fentry_stubs(u32 *image, struct codegen_context *ctx)  in bpf_jit_build_fentry_stubs()  argument
82 	if (image)  in bpf_jit_build_fentry_stubs()
83 		*((unsigned long *)&image[ctx->idx]) = (unsigned long)dummy_tramp;  in bpf_jit_build_fentry_stubs()
100 int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)  in bpf_jit_emit_exit_insn()  argument
110 		bpf_jit_build_epilogue(image, ctx);  in bpf_jit_emit_exit_insn()
169 	u8 *image = NULL;  in bpf_int_jit_compile()  local
228 		image = (void *)hdr + ((void *)fimage - (void *)fhdr);  in bpf_int_jit_compile()
297 	fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,  in bpf_int_jit_compile()
306 	code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);  in bpf_int_jit_compile()
337 	((u64 *)image)[0] = (u64)fcode_base;  in bpf_int_jit_compile()
338 	((u64 *)image)[1] = local_paca->kernel_toc;  in bpf_int_jit_compile()
360 		if (!image && priv_stack_ptr) {  in bpf_int_jit_compile()
392 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass,  in bpf_add_extable_entry()  argument
415 	pc = (unsigned long)&image[insn_idx];  in bpf_add_extable_entry()
416 	ex = (void *)fp->aux->extable - (void *)fimage + (void *)image;  in bpf_add_extable_entry()
589 void arch_free_bpf_trampoline(void *image, unsigned int size)  in arch_free_bpf_trampoline()  argument
591 	bpf_prog_pack_free(image, size);  in arch_free_bpf_trampoline()
594 int arch_protect_bpf_trampoline(void *image, unsigned int size)  in arch_protect_bpf_trampoline()  argument
599 static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ctx,  in invoke_bpf_prog()  argument
626 	ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,  in invoke_bpf_prog()
649 	if (image && !create_branch(&branch_insn, (u32 *)&ro_image[ctx->idx],  in invoke_bpf_prog()
652 		image[ctx->idx] = ppc_inst_val(branch_insn);  in invoke_bpf_prog()
664 	if (image) {  in invoke_bpf_prog()
665 		if (create_cond_branch(&branch_insn, &image[jmp_idx],  in invoke_bpf_prog()
666 				       (unsigned long)&image[ctx->idx], COND_EQ << 16))  in invoke_bpf_prog()
668 		image[jmp_idx] = ppc_inst_val(branch_insn);  in invoke_bpf_prog()
675 	ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,  in invoke_bpf_prog()
681 static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context *ctx,  in invoke_bpf_mod_ret()  argument
694 		if (invoke_bpf_prog(image, ro_image, ctx, tl->links[i], regs_off, retval_off,  in invoke_bpf_mod_ret()
725 static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_context *ctx,  in bpf_trampoline_setup_tail_call_info()  argument
753 static void bpf_trampoline_restore_tail_call_cnt(u32 *image, struct codegen_context *ctx,  in bpf_trampoline_restore_tail_call_cnt()  argument
765 static void bpf_trampoline_save_args(u32 *image, struct codegen_context *ctx,  in bpf_trampoline_save_args()  argument
784 static void bpf_trampoline_restore_args_regs(u32 *image, struct codegen_context *ctx,  in bpf_trampoline_restore_args_regs()  argument
792 static void bpf_trampoline_restore_args_stack(u32 *image, struct codegen_context *ctx,  in bpf_trampoline_restore_args_stack()  argument
804 	bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off);  in bpf_trampoline_restore_args_stack()
823 	u32 *image = (u32 *)rw_image;  in __arch_prepare_bpf_trampoline()  local
968 	bpf_trampoline_save_args(image, ctx, bpf_frame_size, nr_regs, regs_off);  in __arch_prepare_bpf_trampoline()
1009 	store_func_meta(image, ctx, func_meta, func_meta_off);  in __arch_prepare_bpf_trampoline()
1017 		ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,  in __arch_prepare_bpf_trampoline()
1028 		prepare_for_fsession_fentry(image, ctx, cookie_cnt, cookie_off, retval_off);  in __arch_prepare_bpf_trampoline()
1037 			store_func_meta(image, ctx, meta, func_meta_off);  in __arch_prepare_bpf_trampoline()
1041 		if (invoke_bpf_prog(image, ro_image, ctx, fentry->links[i], regs_off, retval_off,  in __arch_prepare_bpf_trampoline()
1051 		if (invoke_bpf_mod_ret(image, ro_image, ctx, fmod_ret, regs_off, retval_off,  in __arch_prepare_bpf_trampoline()
1070 			bpf_trampoline_setup_tail_call_info(image, ctx, bpf_frame_size, r4_off);  in __arch_prepare_bpf_trampoline()
1073 		bpf_trampoline_restore_args_stack(image, ctx, bpf_frame_size, nr_regs, regs_off);  in __arch_prepare_bpf_trampoline()
1087 			bpf_trampoline_restore_tail_call_cnt(image, ctx, bpf_frame_size, r4_off);  in __arch_prepare_bpf_trampoline()
1096 	for (i = 0; i < fmod_ret->nr_links && image; i++) {  in __arch_prepare_bpf_trampoline()
1097 		if (create_cond_branch(&branch_insn, &image[branches[i]],  in __arch_prepare_bpf_trampoline()
1098 				       (unsigned long)&image[ctx->idx], COND_NE << 16)) {  in __arch_prepare_bpf_trampoline()
1103 		image[branches[i]] = ppc_inst_val(branch_insn);  in __arch_prepare_bpf_trampoline()
1109 		store_func_meta(image, ctx, func_meta, func_meta_off);  in __arch_prepare_bpf_trampoline()
1117 			store_func_meta(image, ctx, meta, func_meta_off);  in __arch_prepare_bpf_trampoline()
1121 		if (invoke_bpf_prog(image, ro_image, ctx, fexit->links[i], regs_off, retval_off,  in __arch_prepare_bpf_trampoline()
1132 		ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,  in __arch_prepare_bpf_trampoline()
1139 		bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off);  in __arch_prepare_bpf_trampoline()
1176 	if (image && WARN_ON_ONCE(&image[ctx->idx] > (u32 *)rw_image_end - BPF_INSN_SAFETY)) {  in __arch_prepare_bpf_trampoline()
1197 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,  in arch_prepare_bpf_trampoline()  argument
1202 	u32 size = image_end - image;  in arch_prepare_bpf_trampoline()
1214 	ret = __arch_prepare_bpf_trampoline(im, rw_image, rw_image + size, image, m,  in arch_prepare_bpf_trampoline()
1222 	tmp = bpf_arch_text_copy(image, rw_image, size);  in arch_prepare_bpf_trampoline()