xref: /linux/arch/powerpc/net/bpf_jit_comp.c (revision d5a821f66047c2effd06147b84775f9b2ce0f4aa)

// SPDX-License-Identifier: GPL-2.0-only
/*
 * eBPF JIT compiler
 *
 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
 *		  IBM Corporation
 *
 * Based on the powerpc classic BPF JIT compiler by Matt Evans
 */
#include <linux/moduleloader.h>
#include <asm/cacheflush.h>
#include <asm/asm-compat.h>
#include <linux/netdevice.h>
#include <linux/filter.h>
#include <linux/if_vlan.h>
#include <linux/kernel.h>
#include <linux/memory.h>
#include <linux/bpf.h>

#include <asm/kprobes.h>
#include <asm/text-patching.h>

#include "bpf_jit.h"

/* These offsets are from bpf prog end and stay the same across progs */
static int bpf_jit_ool_stub, bpf_jit_long_branch_stub;

static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
{
	memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
}

void dummy_tramp(void);

asm (
"	.pushsection .text, \"ax\", @progbits	;"
"	.global dummy_tramp			;"
"	.type dummy_tramp, @function		;"
"dummy_tramp:					;"
#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
"	blr					;"
#else
/* LR is always in r11, so we don't need a 'mflr r11' here */
"	mtctr	11				;"
"	mtlr	0				;"
"	bctr					;"
#endif
"	.size dummy_tramp, .-dummy_tramp	;"
"	.popsection				;"
);

void bpf_jit_build_fentry_stubs(u32 *image, struct codegen_context *ctx)
{
	int ool_stub_idx, long_branch_stub_idx;

	/*
	 * Out-of-line stub:
	 *	mflr	r0
	 *	[b|bl]	tramp
	 *	mtlr	r0 // only with CONFIG_PPC_FTRACE_OUT_OF_LINE
	 *	b	bpf_func + 4
	 */
	ool_stub_idx = ctx->idx;
	EMIT(PPC_RAW_MFLR(_R0));
	EMIT(PPC_RAW_NOP());
	if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
		EMIT(PPC_RAW_MTLR(_R0));
	WARN_ON_ONCE(!is_offset_in_branch_range(4 - (long)ctx->idx * 4));
	EMIT(PPC_RAW_BRANCH(4 - (long)ctx->idx * 4));

	/*
	 * Long branch stub:
	 *	.long	<dummy_tramp_addr>
	 *	mflr	r11
	 *	bcl	20,31,$+4
	 *	mflr	r12
	 *	ld	r12, -8-SZL(r12)
	 *	mtctr	r12
	 *	mtlr	r11 // needed to retain ftrace ABI
	 *	bctr
	 */
	if (image)
		*((unsigned long *)&image[ctx->idx]) = (unsigned long)dummy_tramp;
	ctx->idx += SZL / 4;
	long_branch_stub_idx = ctx->idx;
	EMIT(PPC_RAW_MFLR(_R11));
	EMIT(PPC_RAW_BCL4());
	EMIT(PPC_RAW_MFLR(_R12));
	EMIT(PPC_RAW_LL(_R12, _R12, -8-SZL));
	EMIT(PPC_RAW_MTCTR(_R12));
	EMIT(PPC_RAW_MTLR(_R11));
	EMIT(PPC_RAW_BCTR());

	if (!bpf_jit_ool_stub) {
		bpf_jit_ool_stub = (ctx->idx - ool_stub_idx) * 4;
		bpf_jit_long_branch_stub = (ctx->idx - long_branch_stub_idx) * 4;
	}
}

int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)
{
	if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) {
		PPC_JMP(exit_addr);
	} else if (ctx->alt_exit_addr) {
		if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4))))
			return -1;
		PPC_JMP(ctx->alt_exit_addr);
	} else {
		ctx->alt_exit_addr = ctx->idx * 4;
		bpf_jit_build_epilogue(image, ctx);
	}

	return 0;
}

struct powerpc_jit_data {
	/* address of rw header */
	struct bpf_binary_header *hdr;
	/* address of ro final header */
	struct bpf_binary_header *fhdr;
	u32 *addrs;
	u8 *fimage;
	u32 proglen;
	struct codegen_context ctx;
};

bool bpf_jit_needs_zext(void)
{
	return true;
}

static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size)
{
	int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
	u64 *stack_ptr;

	for_each_possible_cpu(cpu) {
		stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu);
		stack_ptr[0] = PRIV_STACK_GUARD_VAL;
		stack_ptr[1] = PRIV_STACK_GUARD_VAL;
		stack_ptr[underflow_idx] = PRIV_STACK_GUARD_VAL;
		stack_ptr[underflow_idx + 1] = PRIV_STACK_GUARD_VAL;
	}
}

static void priv_stack_check_guard(void __percpu *priv_stack_ptr, int alloc_size,
								struct bpf_prog *fp)
{
	int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
	u64 *stack_ptr;

	for_each_possible_cpu(cpu) {
		stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu);
		if (stack_ptr[0] != PRIV_STACK_GUARD_VAL ||
			stack_ptr[1] != PRIV_STACK_GUARD_VAL ||
			stack_ptr[underflow_idx] != PRIV_STACK_GUARD_VAL ||
			stack_ptr[underflow_idx + 1] != PRIV_STACK_GUARD_VAL) {
			pr_err("BPF private stack overflow/underflow detected for prog %s\n",
			bpf_jit_get_prog_name(fp));
			break;
		}
	}
}

struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp)
{
	u32 proglen;
	u32 alloclen;
	u8 *image = NULL;
	u32 *code_base = NULL;
	u32 *addrs = NULL;
	struct powerpc_jit_data *jit_data = NULL;
	struct codegen_context cgctx;
	int pass;
	int flen;
	int priv_stack_alloc_size;
	void __percpu *priv_stack_ptr = NULL;
	struct bpf_binary_header *fhdr = NULL;
	struct bpf_binary_header *hdr = NULL;
	bool extra_pass = false;
	u8 *fimage = NULL;
	u32 *fcode_base = NULL;
	u32 extable_len;
	u32 fixup_len;

	if (!fp->jit_requested)
		return fp;

	jit_data = fp->aux->jit_data;
	if (!jit_data) {
		jit_data = kzalloc_obj(*jit_data);
		if (!jit_data)
			return fp;
		fp->aux->jit_data = jit_data;
	}

	priv_stack_ptr = fp->aux->priv_stack_ptr;
	if (!priv_stack_ptr && fp->aux->jits_use_priv_stack) {
		/*
		 * Allocate private stack of size equivalent to
		 * verifier-calculated stack size plus two memory
		 * guard regions to detect private stack overflow
		 * and underflow.
		 */
		priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) +
							2 * PRIV_STACK_GUARD_SZ;
		priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_size, 16, GFP_KERNEL);
		if (!priv_stack_ptr)
			goto out_priv_stack;

		priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_size);
		fp->aux->priv_stack_ptr = priv_stack_ptr;
	}

	flen = fp->len;
	addrs = jit_data->addrs;
	if (addrs) {
		cgctx = jit_data->ctx;
		/*
		 * JIT compiled to a writable location (image/code_base) first.
		 * It is then moved to the readonly final location (fimage/fcode_base)
		 * using instruction patching.
		 */
		fimage = jit_data->fimage;
		fhdr = jit_data->fhdr;
		proglen = jit_data->proglen;
		hdr = jit_data->hdr;
		image = (void *)hdr + ((void *)fimage - (void *)fhdr);
		extra_pass = true;
		/* During extra pass, ensure index is reset before repopulating extable entries */
		cgctx.exentry_idx = 0;
		goto skip_init_ctx;
	}

	addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
	if (addrs == NULL)
		goto out_err;

	memset(&cgctx, 0, sizeof(struct codegen_context));
	bpf_jit_init_reg_mapping(&cgctx);

	/* Make sure that the stack is quadword aligned. */
	cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
	cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena);
	cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena);
	cgctx.is_subprog = bpf_is_subprog(fp);
	cgctx.exception_boundary = fp->aux->exception_boundary;
	cgctx.exception_cb = fp->aux->exception_cb;
	cgctx.priv_sp = priv_stack_ptr;
	cgctx.priv_stack_size = 0;
	if (priv_stack_ptr) {
		/*
		 * priv_stack_size required for setting bpf FP inside
		 * percpu allocation.
		 * stack_size is marked 0 to prevent allocation on
		 * general stack and offset calculation don't go for
		 * a toss in bpf_jit_stack_offsetof() & bpf_jit_stack_local()
		 */
		cgctx.priv_stack_size = cgctx.stack_size;
		cgctx.stack_size = 0;
	}

	/* Scouting faux-generate pass 0 */
	if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false))
		/* We hit something illegal or unsupported. */
		goto out_err;

	/*
	 * If we have seen a tail call, we need a second pass.
	 * This is because bpf_jit_emit_common_epilogue() is called
	 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
	 * We also need a second pass if we ended up with too large
	 * a program so as to ensure BPF_EXIT branches are in range.
	 */
	if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {
		cgctx.idx = 0;
		if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false))
			goto out_err;
	}

	bpf_jit_realloc_regs(&cgctx);
	/*
	 * Pretend to build prologue, given the features we've seen.  This will
	 * update ctgtx.idx as it pretends to output instructions, then we can
	 * calculate total size from idx.
	 */
	bpf_jit_build_prologue(NULL, &cgctx);
	addrs[fp->len] = cgctx.idx * 4;
	bpf_jit_build_epilogue(NULL, &cgctx);

	fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;
	extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry);

	proglen = cgctx.idx * 4;
	alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;

	fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,
					 bpf_jit_fill_ill_insns, bpf_prog_was_classic(fp));
	if (!fhdr)
		goto out_err;

	if (extable_len)
		fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len;

skip_init_ctx:
	code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
	fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE);

	/* Code generation passes 1-2 */
	for (pass = 1; pass < 3; pass++) {
		/* Now build the prologue, body code & epilogue for real. */
		cgctx.idx = 0;
		cgctx.alt_exit_addr = 0;
		bpf_jit_build_prologue(code_base, &cgctx);
		if (bpf_jit_build_body(fp, code_base, fcode_base, &cgctx, addrs, pass,
				       extra_pass)) {
			bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size));
			bpf_jit_binary_pack_free(fhdr, hdr);
			goto out_err;
		}
		bpf_jit_build_epilogue(code_base, &cgctx);

		if (bpf_jit_enable > 1)
			pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
				proglen - (cgctx.idx * 4), cgctx.seen);
	}

	if (bpf_jit_enable > 1)
		/*
		 * Note that we output the base address of the code_base
		 * rather than image, since opcodes are in code_base.
		 */
		bpf_jit_dump(flen, proglen, pass, code_base);

#ifdef CONFIG_PPC64_ELF_ABI_V1
	/* Function descriptor nastiness: Address + TOC */
	((u64 *)image)[0] = (u64)fcode_base;
	((u64 *)image)[1] = local_paca->kernel_toc;
#endif

	if (!fp->is_func || extra_pass) {
		if (bpf_jit_binary_pack_finalize(fhdr, hdr))
			goto out_err;
	}

	fp->bpf_func = (void *)fimage;
	fp->jited = 1;
	fp->jited_len = cgctx.idx * 4 + FUNCTION_DESCR_SIZE;

	if (!fp->is_func || extra_pass) {
		bpf_prog_fill_jited_linfo(fp, addrs);
		/*
		 * On ABI V1, executable code starts after the function
		 * descriptor, so adjust base accordingly.
		 */
		bpf_prog_update_insn_ptrs(fp, addrs,
				(void *)fimage + FUNCTION_DESCR_SIZE);

out_addrs:
		if (!image && priv_stack_ptr) {
			fp->aux->priv_stack_ptr = NULL;
			free_percpu(priv_stack_ptr);
		}
out_priv_stack:
		kfree(addrs);
		kfree(jit_data);
		fp->aux->jit_data = NULL;
	} else {
		jit_data->addrs = addrs;
		jit_data->ctx = cgctx;
		jit_data->proglen = proglen;
		jit_data->fimage = fimage;
		jit_data->fhdr = fhdr;
		jit_data->hdr = hdr;
	}

	return fp;

out_err:
	if (extra_pass) {
		fp->bpf_func = NULL;
		fp->jited = 0;
		fp->jited_len = 0;
	}
	goto out_addrs;
}

/*
 * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling
 * this function, as this only applies to BPF_PROBE_MEM, for now.
 */
int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass,
			  struct codegen_context *ctx, int insn_idx, int jmp_off,
			  int dst_reg, u32 code)
{
	off_t offset;
	unsigned long pc;
	struct exception_table_entry *ex, *ex_entry;
	u32 *fixup;

	/* Populate extable entries only in the last pass */
	if (pass != 2)
		return 0;

	if (!fp->aux->extable ||
	    WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))
		return -EINVAL;

	/*
	 * Program is first written to image before copying to the
	 * final location (fimage). Accordingly, update in the image first.
	 * As all offsets used are relative, copying as is to the
	 * final location should be alright.
	 */
	pc = (unsigned long)&image[insn_idx];
	ex = (void *)fp->aux->extable - (void *)fimage + (void *)image;

	fixup = (void *)ex -
		(fp->aux->num_exentries * BPF_FIXUP_LEN * 4) +
		(ctx->exentry_idx * BPF_FIXUP_LEN * 4);

	fixup[0] = PPC_RAW_LI(dst_reg, 0);
	if (BPF_CLASS(code) == BPF_ST || BPF_CLASS(code) == BPF_STX)
		fixup[0] = PPC_RAW_NOP();

	if (IS_ENABLED(CONFIG_PPC32))
		fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */

	fixup[BPF_FIXUP_LEN - 1] =
		PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]);

	ex_entry = &ex[ctx->exentry_idx];

	offset = pc - (long)&ex_entry->insn;
	if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
		return -ERANGE;
	ex_entry->insn = offset;

	offset = (long)fixup - (long)&ex_entry->fixup;
	if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
		return -ERANGE;
	ex_entry->fixup = offset;

	ctx->exentry_idx++;
	return 0;
}

void *bpf_arch_text_copy(void *dst, void *src, size_t len)
{
	int err;

	if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
		return ERR_PTR(-EINVAL);

	mutex_lock(&text_mutex);
	err = patch_instructions(dst, src, len, false);
	mutex_unlock(&text_mutex);

	return err ? ERR_PTR(err) : dst;
}

int bpf_arch_text_invalidate(void *dst, size_t len)
{
	u32 insn = BREAKPOINT_INSTRUCTION;
	int ret;

	if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
		return -EINVAL;

	mutex_lock(&text_mutex);
	ret = patch_instructions(dst, &insn, len, true);
	mutex_unlock(&text_mutex);

	return ret;
}

void bpf_jit_free(struct bpf_prog *fp)
{
	if (fp->jited) {
		struct powerpc_jit_data *jit_data = fp->aux->jit_data;
		struct bpf_binary_header *hdr;
		void __percpu *priv_stack_ptr;
		int priv_stack_alloc_size;

		/*
		 * If we fail the final pass of JIT (from jit_subprogs),
		 * the program may not be finalized yet. Call finalize here
		 * before freeing it.
		 */
		if (jit_data) {
			bpf_jit_binary_pack_finalize(jit_data->fhdr, jit_data->hdr);
			kvfree(jit_data->addrs);
			kfree(jit_data);
		}
		hdr = bpf_jit_binary_pack_hdr(fp);
		bpf_jit_binary_pack_free(hdr, NULL);
		priv_stack_ptr = fp->aux->priv_stack_ptr;
		if (priv_stack_ptr) {
			priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) +
							2 * PRIV_STACK_GUARD_SZ;
			priv_stack_check_guard(priv_stack_ptr, priv_stack_alloc_size, fp);
			free_percpu(priv_stack_ptr);
		}
		WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp));
	}

	bpf_prog_unlock_free(fp);
}

bool bpf_jit_supports_exceptions(void)
{
	return IS_ENABLED(CONFIG_PPC64);
}

bool bpf_jit_supports_subprog_tailcalls(void)
{
	return IS_ENABLED(CONFIG_PPC64);
}

bool bpf_jit_supports_timed_may_goto(void)
{
	return IS_ENABLED(CONFIG_PPC64);
}

bool bpf_jit_supports_kfunc_call(void)
{
	return IS_ENABLED(CONFIG_PPC64);
}

bool bpf_jit_supports_private_stack(void)
{
	return IS_ENABLED(CONFIG_PPC64);
}

bool bpf_jit_supports_fsession(void)
{
	/*
	 * TODO: Remove after validating support
	 * for fsession and trampoline on ppc32.
	 */
	if (IS_ENABLED(CONFIG_PPC32))
		return -EOPNOTSUPP;
	return true;
}

bool bpf_jit_supports_arena(void)
{
	return IS_ENABLED(CONFIG_PPC64);
}

bool bpf_jit_supports_far_kfunc_call(void)
{
	return IS_ENABLED(CONFIG_PPC64);
}

bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
{
	if (!in_arena)
		return true;
	switch (insn->code) {
	case BPF_STX | BPF_ATOMIC | BPF_H:
	case BPF_STX | BPF_ATOMIC | BPF_B:
	case BPF_STX | BPF_ATOMIC | BPF_W:
	case BPF_STX | BPF_ATOMIC | BPF_DW:
		if (bpf_atomic_is_load_store(insn))
			return false;
		return IS_ENABLED(CONFIG_PPC64);
	}
	return true;
}

bool bpf_jit_supports_percpu_insn(void)
{
	return IS_ENABLED(CONFIG_PPC64);
}

bool bpf_jit_inlines_helper_call(s32 imm)
{
	switch (imm) {
	case BPF_FUNC_get_smp_processor_id:
	case BPF_FUNC_get_current_task:
	case BPF_FUNC_get_current_task_btf:
		return true;
	default:
		return false;
	}
}

void *arch_alloc_bpf_trampoline(unsigned int size)
{
	return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns, false);
}

void arch_free_bpf_trampoline(void *image, unsigned int size)
{
	bpf_prog_pack_free(image, size);
}

int arch_protect_bpf_trampoline(void *image, unsigned int size)
{
	return 0;
}

static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ctx,
			   struct bpf_tramp_image *im, struct bpf_tramp_node *n,
			   int regs_off, int retval_off, int run_ctx_off, bool save_ret)
{
	struct bpf_prog *p = n->link->prog;
	ppc_inst_t branch_insn;
	u32 jmp_idx, skip_idx;
	int ret = 0;

	/* nop, patched to skip this prog when it is detached */
	skip_idx = ctx->idx;
	EMIT(PPC_RAW_NOP());

	/* Save cookie */
	if (IS_ENABLED(CONFIG_PPC64)) {
		PPC_LI64(_R3, n->cookie);
		EMIT(PPC_RAW_STD(_R3, _R1, run_ctx_off + offsetof(struct bpf_tramp_run_ctx,
				 bpf_cookie)));
	} else {
		PPC_LI32(_R3, n->cookie >> 32);
		PPC_LI32(_R4, n->cookie);
		EMIT(PPC_RAW_STW(_R3, _R1,
				 run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie)));
		EMIT(PPC_RAW_STW(_R4, _R1,
				 run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie) + 4));
	}

	/* __bpf_prog_enter(p, &bpf_tramp_run_ctx) */
	PPC_LI_ADDR(_R3, p);
	EMIT(PPC_RAW_MR(_R25, _R3));
	EMIT(PPC_RAW_ADDI(_R4, _R1, run_ctx_off));
	ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
					 (unsigned long)bpf_trampoline_enter(p));
	if (ret)
		return ret;

	/* Remember prog start time returned by __bpf_prog_enter */
	EMIT(PPC_RAW_MR(_R26, _R3));

	/*
	 * if (__bpf_prog_enter(p) == 0)
	 *	goto skip_exec_of_prog;
	 *
	 * Emit a nop to be later patched with conditional branch, once offset is known
	 */
	EMIT(PPC_RAW_CMPLI(_R3, 0));
	jmp_idx = ctx->idx;
	EMIT(PPC_RAW_NOP());

	/* p->bpf_func(ctx) */
	EMIT(PPC_RAW_ADDI(_R3, _R1, regs_off));
	if (!p->jited)
		PPC_LI_ADDR(_R4, (unsigned long)p->insnsi);
	/* Account for max possible instructions during dummy pass for size calculation */
	if (image && !create_branch(&branch_insn, (u32 *)&ro_image[ctx->idx],
				    (unsigned long)p->bpf_func,
				    BRANCH_SET_LINK)) {
		image[ctx->idx] = ppc_inst_val(branch_insn);
		ctx->idx++;
	} else {
		EMIT(PPC_RAW_LL(_R12, _R25, offsetof(struct bpf_prog, bpf_func)));
		EMIT(PPC_RAW_MTCTR(_R12));
		EMIT(PPC_RAW_BCTRL());
	}

	if (save_ret)
		EMIT(PPC_RAW_STL(_R3, _R1, retval_off));

	/* Fix up branch */
	if (image) {
		if (create_cond_branch(&branch_insn, &image[jmp_idx],
				       (unsigned long)&image[ctx->idx], COND_EQ << 16))
			return -EINVAL;
		image[jmp_idx] = ppc_inst_val(branch_insn);
	}

	/* __bpf_prog_exit(p, start_time, &bpf_tramp_run_ctx) */
	EMIT(PPC_RAW_MR(_R3, _R25));
	EMIT(PPC_RAW_MR(_R4, _R26));
	EMIT(PPC_RAW_ADDI(_R5, _R1, run_ctx_off));
	ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
					 (unsigned long)bpf_trampoline_exit(p));
	if (ret)
		return ret;

	if (ro_image) /* image is NULL for dummy pass */
		bpf_tramp_image_add_skip(im, p, &ro_image[skip_idx], &ro_image[ctx->idx]);
	return 0;
}

static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context *ctx,
			      struct bpf_tramp_image *im, struct bpf_tramp_nodes *tn,
			      int regs_off, int retval_off, int run_ctx_off, u32 *branches)
{
	int i;

	/*
	 * The first fmod_ret program will receive a garbage return value.
	 * Set this to 0 to avoid confusing the program.
	 */
	EMIT(PPC_RAW_LI(_R3, 0));
	EMIT(PPC_RAW_STL(_R3, _R1, retval_off));
	for (i = 0; i < tn->nr_nodes; i++) {
		if (invoke_bpf_prog(image, ro_image, ctx, im, tn->nodes[i], regs_off,
				    retval_off, run_ctx_off, true))
			return -EINVAL;

		/*
		 * mod_ret prog stored return value after prog ctx. Emit:
		 * if (*(u64 *)(ret_val) !=  0)
		 *	goto do_fexit;
		 */
		EMIT(PPC_RAW_LL(_R3, _R1, retval_off));
		EMIT(PPC_RAW_CMPLI(_R3, 0));

		/*
		 * Save the location of the branch and generate a nop, which is
		 * replaced with a conditional jump once do_fexit (i.e. the
		 * start of the fexit invocation) is finalized.
		 */
		branches[i] = ctx->idx;
		EMIT(PPC_RAW_NOP());
	}

	return 0;
}

/*
 * Refer __arch_prepare_bpf_trampoline() for stack component details.
 *
 * The tailcall count/reference is present in caller's stack frame. The
 * tail_call_info is saved at the same offset on the trampoline frame
 * for the traced function (BPF subprog/callee) to fetch it.
 */
static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_context *ctx,
						int bpf_frame_size, int r4_off)
{
	if (IS_ENABLED(CONFIG_PPC64)) {
		EMIT(PPC_RAW_LD(_R4, _R1, bpf_frame_size));
		/* Refer to trampoline's Generated stack layout */
		EMIT(PPC_RAW_LD(_R3, _R4, -BPF_PPC_TAILCALL));

		/*
		 * Setting the tail_call_info in trampoline's frame
		 * depending on if previous frame had value or reference.
		 */
		EMIT(PPC_RAW_CMPLWI(_R3, MAX_TAIL_CALL_CNT));
		PPC_BCC_CONST_SHORT(COND_GT, 8);
		EMIT(PPC_RAW_ADDI(_R3, _R4, -BPF_PPC_TAILCALL));

		/*
		 * Trampoline's tail_call_info is at the same offset, as that of
		 * any bpf program, with reference to previous frame. Update the
		 * address of main's tail_call_info in trampoline frame.
		 */
		EMIT(PPC_RAW_STL(_R3, _R1, bpf_frame_size - BPF_PPC_TAILCALL));
	} else {
		/* See bpf_jit_stack_offsetof() and BPF_PPC_TC */
		EMIT(PPC_RAW_LL(_R4, _R1, r4_off));
	}
}

static void bpf_trampoline_restore_tail_call_cnt(u32 *image, struct codegen_context *ctx,
						 int bpf_frame_size, int r4_off)
{
	if (IS_ENABLED(CONFIG_PPC32)) {
		/*
		 * Restore tailcall for 32-bit powerpc
		 * See bpf_jit_stack_offsetof() and BPF_PPC_TC
		 */
		EMIT(PPC_RAW_STL(_R4, _R1, r4_off));
	}
}

static void bpf_trampoline_save_args(u32 *image, struct codegen_context *ctx,
				     int bpf_frame_size, int nr_regs, int regs_off)
{
	int param_save_area_offset;

	param_save_area_offset = bpf_frame_size;
	param_save_area_offset += STACK_FRAME_MIN_SIZE; /* param save area is past frame header */

	for (int i = 0; i < nr_regs; i++) {
		if (i < 8) {
			EMIT(PPC_RAW_STL(_R3 + i, _R1, regs_off + i * SZL));
		} else {
			EMIT(PPC_RAW_LL(_R3, _R1, param_save_area_offset + i * SZL));
			EMIT(PPC_RAW_STL(_R3, _R1, regs_off + i * SZL));
		}
	}
}

/* Used when restoring just the register parameters when returning back */
static void bpf_trampoline_restore_args_regs(u32 *image, struct codegen_context *ctx,
					     int nr_regs, int regs_off)
{
	for (int i = 0; i < nr_regs && i < 8; i++)
		EMIT(PPC_RAW_LL(_R3 + i, _R1, regs_off + i * SZL));
}

/* Used when we call into the traced function. Replicate parameter save area */
static void bpf_trampoline_restore_args_stack(u32 *image, struct codegen_context *ctx,
					      int bpf_frame_size, int nr_regs, int regs_off)
{
	int param_save_area_offset;

	param_save_area_offset = bpf_frame_size;
	param_save_area_offset += STACK_FRAME_MIN_SIZE; /* param save area is past frame header */

	for (int i = 8; i < nr_regs; i++) {
		EMIT(PPC_RAW_LL(_R3, _R1, param_save_area_offset + i * SZL));
		EMIT(PPC_RAW_STL(_R3, _R1, STACK_FRAME_MIN_SIZE + i * SZL));
	}
	bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off);
}

static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_image,
					 void *rw_image_end, void *ro_image,
					 const struct btf_func_model *m, u32 flags,
					 struct bpf_tramp_nodes *tnodes,
					 void *func_addr)
{
	int regs_off, func_meta_off, ip_off, run_ctx_off, retval_off;
	int nvr_off, alt_lr_off, r4_off = 0;
	struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN];
	struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY];
	struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT];
	int i, ret, nr_regs, retaddr_off, bpf_frame_size = 0;
	struct codegen_context codegen_ctx, *ctx;
	int cookie_off, cookie_cnt, cookie_ctx_off;
	int fsession_cnt = bpf_fsession_cnt(tnodes);
	u64 func_meta;
	u32 *image = (u32 *)rw_image;
	ppc_inst_t branch_insn;
	u32 *branches = NULL;
	bool save_ret;

	if (IS_ENABLED(CONFIG_PPC32))
		return -EOPNOTSUPP;

	nr_regs = m->nr_args;
	/* Extra registers for struct arguments */
	for (i = 0; i < m->nr_args; i++)
		if (m->arg_size[i] > SZL)
			nr_regs += round_up(m->arg_size[i], SZL) / SZL - 1;

	if (nr_regs > MAX_BPF_FUNC_ARGS)
		return -EOPNOTSUPP;

	ctx = &codegen_ctx;
	memset(ctx, 0, sizeof(*ctx));

	/*
	 * Generated stack layout:
	 *
	 * func prev back chain         [ back chain        ]
	 *                              [ tail_call_info    ] optional - 64-bit powerpc
	 *                              [ padding           ] align stack frame
	 *       r4_off                 [ r4 (tailcallcnt)  ] optional - 32-bit powerpc
	 *       alt_lr_off             [ real lr (ool stub)] optional - actual lr
	 *       retaddr_off            [ return address    ]
	 *                              [ r26               ]
	 *       nvr_off                [ r25               ] nvr save area
	 *       retval_off             [ return value      ]
	 *                              [ reg argN          ]
	 *                              [ ...               ]
	 *       regs_off               [ reg_arg1          ] prog_ctx
	 *       func_meta_off          [ args count        ] ((u64 *)prog_ctx)[-1]
	 *       ip_off                 [ traced function   ] ((u64 *)prog_ctx)[-2]
	 *                              [ stack cookieN     ]
	 *                              [ ...               ]
	 *       cookie_off             [ stack cookie1     ]
	 *       run_ctx_off            [ bpf_tramp_run_ctx ]
	 *                              [ reg argN          ]
	 *                              [ ...               ]
	 *       param_save_area        [ reg_arg1          ] min 8 doublewords, per ABI
	 *                              [ TOC save (64-bit) ] --
	 *                              [ LR save (64-bit)  ]   | header
	 *                              [ LR save (32-bit)  ]   |
	 * bpf trampoline frame	        [ back chain 2      ] --
	 *
	 */

	/* Minimum stack frame header */
	bpf_frame_size = STACK_FRAME_MIN_SIZE;

	/*
	 * Room for parameter save area.
	 *
	 * As per the ABI, this is required if we call into the traced
	 * function (BPF_TRAMP_F_CALL_ORIG):
	 * - if the function takes more than 8 arguments for the rest to spill onto the stack
	 * - or, if the function has variadic arguments
	 * - or, if this functions's prototype was not available to the caller
	 *
	 * Reserve space for at least 8 registers for now. This can be optimized later.
	 */
	bpf_frame_size += (nr_regs > 8 ? nr_regs : 8) * SZL;

	/* Room for struct bpf_tramp_run_ctx */
	run_ctx_off = bpf_frame_size;
	bpf_frame_size += round_up(sizeof(struct bpf_tramp_run_ctx), SZL);

	/* room for session cookies */
	cookie_off = bpf_frame_size;
	cookie_cnt = bpf_fsession_cookie_cnt(tnodes);
	bpf_frame_size += cookie_cnt * 8;

	/* Room for IP address argument */
	ip_off = bpf_frame_size;
	if (flags & BPF_TRAMP_F_IP_ARG)
		bpf_frame_size += SZL;

	/* Room for function metadata, arg regs count */
	func_meta_off = bpf_frame_size;
	bpf_frame_size += SZL;

	/* Room for arg regs */
	regs_off = bpf_frame_size;
	bpf_frame_size += nr_regs * SZL;

	/* Room for return value of func_addr or fentry prog */
	retval_off = bpf_frame_size;
	save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
	if (save_ret)
		bpf_frame_size += SZL;

	/* Room for nvr save area */
	nvr_off = bpf_frame_size;
	bpf_frame_size += 2 * SZL;

	/* Save area for return address */
	retaddr_off = bpf_frame_size;
	bpf_frame_size += SZL;

	/* Optional save area for actual LR in case of ool ftrace */
	if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
		alt_lr_off = bpf_frame_size;
		bpf_frame_size += SZL;
	}

	if (IS_ENABLED(CONFIG_PPC32)) {
		if (nr_regs < 2) {
			r4_off = bpf_frame_size;
			bpf_frame_size += SZL;
		} else {
			r4_off = regs_off + SZL;
		}
	}

	/*
	 * Save tailcall count pointer at the same offset on the
	 * stack where subprogs expect it
	 */
	if ((flags & BPF_TRAMP_F_CALL_ORIG) &&
		(flags & BPF_TRAMP_F_TAIL_CALL_CTX))
		bpf_frame_size += BPF_PPC_TAILCALL;

	/* Padding to align stack frame, if any */
	bpf_frame_size = round_up(bpf_frame_size, SZL * 2);

	/*  Store original return value */
	EMIT(PPC_RAW_STL(_R0, _R1, PPC_LR_STKOFF));

	/* Create our stack frame */
	EMIT(PPC_RAW_STLU(_R1, _R1, -bpf_frame_size));

	/* 64-bit: Save TOC and load kernel TOC */
	if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) {
		EMIT(PPC_RAW_STD(_R2, _R1, 24));
		PPC64_LOAD_PACA();
	}

	/* 32-bit: save tail call count in r4 */
	if (IS_ENABLED(CONFIG_PPC32) && nr_regs < 2)
		EMIT(PPC_RAW_STL(_R4, _R1, r4_off));

	bpf_trampoline_save_args(image, ctx, bpf_frame_size, nr_regs, regs_off);

	/* Save our LR/return address */
	EMIT(PPC_RAW_MFLR(_R3));
	if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
		EMIT(PPC_RAW_STL(_R3, _R1, alt_lr_off));
	else
		EMIT(PPC_RAW_STL(_R3, _R1, retaddr_off));

	/*
	 * Derive IP address of the traced function.
	 * In case of CONFIG_PPC_FTRACE_OUT_OF_LINE or BPF program, LR points to the instruction
	 * after the 'bl' instruction in the OOL stub. Refer to ftrace_init_ool_stub() and
	 * bpf_arch_text_poke() for OOL stub of kernel functions and bpf programs respectively.
	 * Relevant stub sequence:
	 *
	 *               bl <tramp>
	 *   LR (R3) =>  mtlr r0
	 *               b <func_addr+4>
	 *
	 * Recover kernel function/bpf program address from the unconditional
	 * branch instruction at the end of OOL stub.
	 */
	if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) || flags & BPF_TRAMP_F_IP_ARG) {
		EMIT(PPC_RAW_LWZ(_R4, _R3, 4));
		EMIT(PPC_RAW_SLWI(_R4, _R4, 6));
		EMIT(PPC_RAW_SRAWI(_R4, _R4, 6));
		EMIT(PPC_RAW_ADD(_R3, _R3, _R4));
	}

	if (flags & BPF_TRAMP_F_IP_ARG)
		EMIT(PPC_RAW_STL(_R3, _R1, ip_off));

	if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
		/* Fake our LR for BPF_TRAMP_F_CALL_ORIG case */
		EMIT(PPC_RAW_ADDI(_R3, _R3, 4));
		EMIT(PPC_RAW_STL(_R3, _R1, retaddr_off));
	}

	/* Save function arg regs count -- see bpf_get_func_arg_cnt() */
	func_meta = nr_regs;
	store_func_meta(image, ctx, func_meta, func_meta_off);

	/* Save nv regs */
	EMIT(PPC_RAW_STL(_R25, _R1, nvr_off));
	EMIT(PPC_RAW_STL(_R26, _R1, nvr_off + SZL));

	if (flags & BPF_TRAMP_F_CALL_ORIG) {
		PPC_LI_ADDR(_R3, (unsigned long)im);
		ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
						 (unsigned long)__bpf_tramp_enter);
		if (ret)
			return ret;
	}

	if (fsession_cnt) {
		/*
		 * Clear all the session cookies' values
		 * Clear the return value to make sure fentry always get 0
		 */
		prepare_for_fsession_fentry(image, ctx, cookie_cnt, cookie_off, retval_off);
	}

	cookie_ctx_off = (regs_off - cookie_off) / 8;

	for (i = 0; i < fentry->nr_nodes; i++) {
		if (bpf_prog_calls_session_cookie(fentry->nodes[i])) {
			u64 meta = func_meta | (cookie_ctx_off << BPF_TRAMP_COOKIE_INDEX_SHIFT);

			store_func_meta(image, ctx, meta, func_meta_off);
			cookie_ctx_off--;
		}

		if (invoke_bpf_prog(image, ro_image, ctx, im, fentry->nodes[i], regs_off,
				    retval_off, run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET))
			return -EINVAL;
	}

	if (fmod_ret->nr_nodes) {
		branches = kcalloc(fmod_ret->nr_nodes, sizeof(u32), GFP_KERNEL);
		if (!branches)
			return -ENOMEM;

		if (invoke_bpf_mod_ret(image, ro_image, ctx, im, fmod_ret, regs_off, retval_off,
				       run_ctx_off, branches)) {
			ret = -EINVAL;
			goto cleanup;
		}
	}

	/* Call the traced function */
	if (flags & BPF_TRAMP_F_CALL_ORIG) {
		/*
		 * retaddr on trampoline stack points to the correct point in the original function
		 * with both PPC_FTRACE_OUT_OF_LINE as well as with traditional ftrace instruction
		 * sequence
		 */
		EMIT(PPC_RAW_LL(_R3, _R1, retaddr_off));
		EMIT(PPC_RAW_MTCTR(_R3));

		/* Replicate tail_call_cnt before calling the original BPF prog */
		if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
			bpf_trampoline_setup_tail_call_info(image, ctx, bpf_frame_size, r4_off);

		/* Restore args */
		bpf_trampoline_restore_args_stack(image, ctx, bpf_frame_size, nr_regs, regs_off);

		/* Restore TOC for 64-bit */
		if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
			EMIT(PPC_RAW_LD(_R2, _R1, 24));
		EMIT(PPC_RAW_BCTRL());
		if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
			PPC64_LOAD_PACA();

		/* Store return value for bpf prog to access */
		EMIT(PPC_RAW_STL(_R3, _R1, retval_off));

		/* Restore updated tail_call_cnt */
		if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
			bpf_trampoline_restore_tail_call_cnt(image, ctx, bpf_frame_size, r4_off);
	}

	/* Update branches saved in invoke_bpf_mod_ret with address of do_fexit */
	for (i = 0; i < fmod_ret->nr_nodes && image; i++) {
		if (create_cond_branch(&branch_insn, &image[branches[i]],
				       (unsigned long)&image[ctx->idx], COND_NE << 16)) {
			ret = -EINVAL;
			goto cleanup;
		}

		image[branches[i]] = ppc_inst_val(branch_insn);
	}

	/* set the "is_return" flag for fsession */
	func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT);
	if (fsession_cnt)
		store_func_meta(image, ctx, func_meta, func_meta_off);

	cookie_ctx_off = (regs_off - cookie_off) / 8;

	for (i = 0; i < fexit->nr_nodes; i++) {
		if (bpf_prog_calls_session_cookie(fexit->nodes[i])) {
			u64 meta = func_meta | (cookie_ctx_off << BPF_TRAMP_COOKIE_INDEX_SHIFT);

			store_func_meta(image, ctx, meta, func_meta_off);
			cookie_ctx_off--;
		}

		if (invoke_bpf_prog(image, ro_image, ctx, im, fexit->nodes[i], regs_off,
				    retval_off, run_ctx_off, false)) {
			ret = -EINVAL;
			goto cleanup;
		}
	}

	if (flags & BPF_TRAMP_F_CALL_ORIG) {
		PPC_LI_ADDR(_R3, im);
		ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
						 (unsigned long)__bpf_tramp_exit);
		if (ret)
			goto cleanup;
	}

	if (flags & BPF_TRAMP_F_RESTORE_REGS)
		bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off);

	/* Restore return value of func_addr or fentry prog */
	if (save_ret)
		EMIT(PPC_RAW_LL(_R3, _R1, retval_off));

	/* Restore nv regs */
	EMIT(PPC_RAW_LL(_R26, _R1, nvr_off + SZL));
	EMIT(PPC_RAW_LL(_R25, _R1, nvr_off));

	/* Epilogue */
	if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
		EMIT(PPC_RAW_LD(_R2, _R1, 24));
	if (flags & BPF_TRAMP_F_SKIP_FRAME) {
		/* Skip the traced function and return to parent */
		EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size));
		EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF));
		EMIT(PPC_RAW_MTLR(_R0));
		EMIT(PPC_RAW_BLR());
	} else {
		if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
			EMIT(PPC_RAW_LL(_R0, _R1, alt_lr_off));
			EMIT(PPC_RAW_MTLR(_R0));
			EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size));
			EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF));
			EMIT(PPC_RAW_BLR());
		} else {
			EMIT(PPC_RAW_LL(_R0, _R1, retaddr_off));
			EMIT(PPC_RAW_MTCTR(_R0));
			EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size));
			EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF));
			EMIT(PPC_RAW_MTLR(_R0));
			EMIT(PPC_RAW_BCTR());
		}
	}

	/* Make sure the trampoline generation logic doesn't overflow */
	if (image && WARN_ON_ONCE(&image[ctx->idx] > (u32 *)rw_image_end - BPF_INSN_SAFETY)) {
		ret = -EFAULT;
		goto cleanup;
	}
	ret = ctx->idx * 4 + BPF_INSN_SAFETY * 4;

cleanup:
	kfree(branches);
	return ret;
}

int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
			     struct bpf_tramp_nodes *tnodes, void *func_addr)
{
	struct bpf_tramp_image im = {};
	int ret;

	ret = __arch_prepare_bpf_trampoline(&im, NULL, NULL, NULL, m, flags, tnodes, func_addr);
	return ret;
}

int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
				const struct btf_func_model *m, u32 flags,
				struct bpf_tramp_nodes *tnodes,
				void *func_addr)
{
	u32 size = image_end - image;
	void *rw_image, *tmp;
	int ret;

	/*
	 * rw_image doesn't need to be in module memory range, so we can
	 * use kvmalloc.
	 */
	rw_image = kvmalloc(size, GFP_KERNEL);
	if (!rw_image)
		return -ENOMEM;

	ret = __arch_prepare_bpf_trampoline(im, rw_image, rw_image + size, image, m,
					    flags, tnodes, func_addr);
	if (ret < 0)
		goto out;

	if (bpf_jit_enable > 1)
		bpf_jit_dump(1, ret - BPF_INSN_SAFETY * 4, 1, rw_image);

	tmp = bpf_arch_text_copy(image, rw_image, size);
	if (IS_ERR(tmp))
		ret = PTR_ERR(tmp);

out:
	kvfree(rw_image);
	return ret;
}

static int bpf_modify_inst(void *ip, ppc_inst_t old_inst, ppc_inst_t new_inst)
{
	ppc_inst_t org_inst;

	if (copy_inst_from_kernel_nofault(&org_inst, ip)) {
		pr_err("0x%lx: fetching instruction failed\n", (unsigned long)ip);
		return -EFAULT;
	}

	if (!ppc_inst_equal(org_inst, old_inst)) {
		pr_err("0x%lx: expected (%08lx) != found (%08lx)\n",
		       (unsigned long)ip, ppc_inst_as_ulong(old_inst), ppc_inst_as_ulong(org_inst));
		return -EINVAL;
	}

	if (ppc_inst_equal(old_inst, new_inst))
		return 0;

	return patch_instruction(ip, new_inst);
}

static void do_isync(void *info __maybe_unused)
{
	isync();
}

/*
 * A 3-step process for bpf prog entry:
 * 1. At bpf prog entry, a single nop/b:
 * bpf_func:
 *	[nop|b]	ool_stub
 * 2. Out-of-line stub:
 * ool_stub:
 *	mflr	r0
 *	[b|bl]	<bpf_prog>/<long_branch_stub>
 *	mtlr	r0 // CONFIG_PPC_FTRACE_OUT_OF_LINE only
 *	b	bpf_func + 4
 * 3. Long branch stub:
 * long_branch_stub:
 *	.long	<branch_addr>/<dummy_tramp>
 *	mflr	r11
 *	bcl	20,31,$+4
 *	mflr	r12
 *	ld	r12, -16(r12)
 *	mtctr	r12
 *	mtlr	r11 // needed to retain ftrace ABI
 *	bctr
 *
 * dummy_tramp is used to reduce synchronization requirements.
 *
 * When attaching a bpf trampoline to a bpf prog, we do not need any
 * synchronization here since we always have a valid branch target regardless
 * of the order in which the above stores are seen. dummy_tramp ensures that
 * the long_branch stub goes to a valid destination on other cpus, even when
 * the branch to the long_branch stub is seen before the updated trampoline
 * address.
 *
 * However, when detaching a bpf trampoline from a bpf prog, or if changing
 * the bpf trampoline address, we need synchronization to ensure that other
 * cpus can no longer branch into the older trampoline so that it can be
 * safely freed. bpf_tramp_image_put() uses rcu_tasks to ensure all cpus
 * make forward progress, but we still need to ensure that other cpus
 * execute isync (or some CSI) so that they don't go back into the
 * trampoline again.
 */
int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
		       enum bpf_text_poke_type new_t, void *old_addr,
		       void *new_addr)
{
	unsigned long bpf_func, bpf_func_end, size, offset;
	ppc_inst_t old_inst, new_inst;
	int ret = 0, branch_flags;
	char name[KSYM_NAME_LEN];

	if (IS_ENABLED(CONFIG_PPC32))
		return -EOPNOTSUPP;

	bpf_func = (unsigned long)ip;

	/* We currently only support poking bpf programs */
	if (!bpf_address_lookup(bpf_func, &size, &offset, name)) {
		pr_err("%s (0x%lx): kernel/modules are not supported\n", __func__, bpf_func);
		return -EOPNOTSUPP;
	}

	/*
	 * If we are not poking at bpf prog entry, then we are simply patching in/out
	 * an unconditional branch instruction in a trampoline image
	 */
	if (offset) {
		if (old_t == BPF_MOD_CALL || new_t == BPF_MOD_CALL) {
			pr_err("%s (0x%lx): calls are not supported in bpf prog body\n", __func__,
			       bpf_func);
			return -EOPNOTSUPP;
		}
		old_inst = ppc_inst(PPC_RAW_NOP());
		if (old_addr)
			if (create_branch(&old_inst, ip, (unsigned long)old_addr, 0))
				return -ERANGE;
		new_inst = ppc_inst(PPC_RAW_NOP());
		if (new_addr)
			if (create_branch(&new_inst, ip, (unsigned long)new_addr, 0))
				return -ERANGE;
		mutex_lock(&text_mutex);
		ret = bpf_modify_inst(ip, old_inst, new_inst);
		mutex_unlock(&text_mutex);

		/* Make sure all cpus see the new instruction */
		smp_call_function(do_isync, NULL, 1);
		return ret;
	}

	bpf_func_end = bpf_func + size;

	/* Address of the jmp/call instruction in the out-of-line stub */
	ip = (void *)(bpf_func_end - bpf_jit_ool_stub + 4);

	if (!is_offset_in_branch_range((long)ip - 4 - bpf_func)) {
		pr_err("%s (0x%lx): bpf prog too large, ool stub out of branch range\n", __func__,
		       bpf_func);
		return -ERANGE;
	}

	old_inst = ppc_inst(PPC_RAW_NOP());
	branch_flags = old_t == BPF_MOD_CALL ? BRANCH_SET_LINK : 0;
	if (old_addr) {
		if (is_offset_in_branch_range(ip - old_addr))
			create_branch(&old_inst, ip, (unsigned long)old_addr, branch_flags);
		else
			create_branch(&old_inst, ip, bpf_func_end - bpf_jit_long_branch_stub,
				      branch_flags);
	}
	new_inst = ppc_inst(PPC_RAW_NOP());
	branch_flags = new_t == BPF_MOD_CALL ? BRANCH_SET_LINK : 0;
	if (new_addr) {
		if (is_offset_in_branch_range(ip - new_addr))
			create_branch(&new_inst, ip, (unsigned long)new_addr, branch_flags);
		else
			create_branch(&new_inst, ip, bpf_func_end - bpf_jit_long_branch_stub,
				      branch_flags);
	}

	mutex_lock(&text_mutex);

	/*
	 * 1. Update the address in the long branch stub:
	 * If new_addr is out of range, we will have to use the long branch stub, so patch new_addr
	 * here. Otherwise, revert to dummy_tramp, but only if we had patched old_addr here.
	 */
	if ((new_addr && !is_offset_in_branch_range(new_addr - ip)) ||
	    (old_addr && !is_offset_in_branch_range(old_addr - ip)))
		ret = patch_ulong((void *)(bpf_func_end - bpf_jit_long_branch_stub - SZL),
				  (new_addr && !is_offset_in_branch_range(new_addr - ip)) ?
				  (unsigned long)new_addr : (unsigned long)dummy_tramp);
	if (ret)
		goto out;

	/* 2. Update the branch/call in the out-of-line stub */
	ret = bpf_modify_inst(ip, old_inst, new_inst);
	if (ret)
		goto out;

	/* 3. Update instruction at bpf prog entry */
	ip = (void *)bpf_func;
	if (!old_addr || !new_addr) {
		if (!old_addr) {
			old_inst = ppc_inst(PPC_RAW_NOP());
			create_branch(&new_inst, ip, bpf_func_end - bpf_jit_ool_stub, 0);
		} else {
			new_inst = ppc_inst(PPC_RAW_NOP());
			create_branch(&old_inst, ip, bpf_func_end - bpf_jit_ool_stub, 0);
		}
		ret = bpf_modify_inst(ip, old_inst, new_inst);
	}

out:
	mutex_unlock(&text_mutex);

	/*
	 * Sync only if we are not attaching a trampoline to a bpf prog so the older
	 * trampoline can be freed safely.
	 */
	if (old_addr)
		smp_call_function(do_isync, NULL, 1);

	return ret;
}