// SPDX-License-Identifier: GPL-2.0
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include "../../../include/linux/filter.h"
#include "bpf_misc.h"

/*
 * llvm lowers unwinding to two insns:
 *   invoke foo unwind pad    call foo; unwind_goto pad
 *   resume                   unwind_exit
 * The tests spell them out as raw insns, so they don't depend on llvm version.
 * The argument of unwind_goto() is the number of insns to skip.
 */
#define unwind_goto(off) ".8byte %[insn_unwind_goto_" #off "];"
#define unwind_exit ".8byte %[insn_unwind_exit];"

#define __imm_unwind_goto(off) \
	__imm_insn(insn_unwind_goto_##off, \
		   BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, BPF_UNWIND_GOTO, off, 0))
#define __imm_unwind_exit \
	__imm_insn(insn_unwind_exit, \
		   BPF_RAW_INSN(BPF_JMP | BPF_EXIT, 0, BPF_UNWIND_EXIT, 0, 0))

extern void bpf_rcu_read_lock(void) __ksym;
extern void bpf_rcu_read_unlock(void) __ksym;
extern void bpf_throw(__u64 cookie) __ksym;

struct {
	__uint(type, BPF_MAP_TYPE_PROG_ARRAY);
	__uint(max_entries, 1);
	__uint(key_size, sizeof(int));
	__uint(value_size, sizeof(int));
} prog_array SEC(".maps");

struct timer_elem {
	struct bpf_timer t;
};

struct {
	__uint(type, BPF_MAP_TYPE_ARRAY);
	__uint(max_entries, 1);
	__type(key, int);
	__type(value, struct timer_elem);
} timer_map SEC(".maps");

/* The verifier doesn't know what is in there */
unsigned long zero;
unsigned long one = 1;

/*
 * The verifier follows a static subprog, so it knows which way it leaves when
 * it knows the argument. To have both ways at run time the tests pass skb->len,
 * which is 64, or skb->len - 64.
 *
 * r1 == 0: returns 7, otherwise unwinds with r0 = 5
 */
__naked __noinline __used
static unsigned long may_unwind(void)
{
	asm volatile (
		"if r1 != 0 goto 1f;"
		"r0 = 7;"
		"exit;"
	"1:"
		"r0 = 5;"
		unwind_exit
		:
		: __imm_unwind_exit
	);
}

/* owns nothing: its pad only carries on */
__naked __noinline __used
static unsigned long passes_on(void)
{
	asm volatile (
		"call may_unwind;"
		unwind_goto(2)
		"r0 += 1;"
		"exit;"
		unwind_exit
		:
		: __imm_unwind_goto(2),
		  __imm_unwind_exit
	);
}

__naked __noinline __used
static unsigned long always_unwinds(void)
{
	asm volatile (
		"r0 = 5;"
		unwind_exit
		:
		: __imm_unwind_exit
	);
}

__naked __noinline __used
static unsigned long never_unwinds(void)
{
	asm volatile (
		"r0 = 7;"
		"exit;"
	);
}

/* Both can unwind. This one returns 7. */
__naked __noinline __used
unsigned long global_returns(void)
{
	asm volatile (
		"r1 = %[zero] ll;"
		"r1 = *(u64 *)(r1 + 0);"
		"if r1 != 0 goto 1f;"
		"r0 = 7;"
		"exit;"
	"1:"
		"r0 = 5;"
		unwind_exit
		:
		: __imm_addr(zero),
		  __imm_unwind_exit
	);
}

/* This one unwinds with r0 = 5 */
__naked __noinline __used
unsigned long global_unwinds(void)
{
	asm volatile (
		"r1 = %[one] ll;"
		"r1 = *(u64 *)(r1 + 0);"
		"if r1 != 0 goto 1f;"
		"r0 = 7;"
		"exit;"
	"1:"
		"r0 = 5;"
		unwind_exit
		:
		: __imm_addr(one),
		  __imm_unwind_exit
	);
}

SEC("tc")
__description("unwind: the callee returns")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__naked int callee_returns(void)
{
	asm volatile (
		"r1 = *(u32 *)(r1 + %[len]);"
		"r1 -= 64;"
		"call may_unwind;"
		unwind_goto(1)
		"exit;"
		"r0 = 42;"
		"exit;"
		:
		: __imm_const(len, offsetof(struct __sk_buff, len)),
		  __imm_unwind_goto(1)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: the callee unwinds")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(42)
__naked int callee_unwinds(void)
{
	asm volatile (
		"r1 = *(u32 *)(r1 + %[len]);"
		"call may_unwind;"
		unwind_goto(1)
		"exit;"
		"r0 = 42;"
		"exit;"
		:
		: __imm_const(len, offsetof(struct __sk_buff, len)),
		  __imm_unwind_goto(1)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: return through two frames")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(8)
__naked int two_frames_return(void)
{
	asm volatile (
		"r1 = *(u32 *)(r1 + %[len]);"
		"r1 -= 64;"
		"call passes_on;"
		unwind_goto(1)
		"exit;"
		"r0 = 42;"
		"exit;"
		:
		: __imm_const(len, offsetof(struct __sk_buff, len)),
		  __imm_unwind_goto(1)
		: __clobber_all
	);
}

/* unwind_exit returns r0 like exit does */
SEC("tc")
__description("unwind: unwind through two frames, r0 reaches the pad")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(15)
__naked int two_frames_unwind(void)
{
	asm volatile (
		"r1 = *(u32 *)(r1 + %[len]);"
		"call passes_on;"
		unwind_goto(1)
		"exit;"
		"r0 += 10;"
		"exit;"
		:
		: __imm_const(len, offsetof(struct __sk_buff, len)),
		  __imm_unwind_goto(1)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: r6-r9 and the stack are intact in the pad")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(1111)
__naked int pad_sees_callee_saved(void)
{
	asm volatile (
		"r1 = *(u32 *)(r1 + %[len]);"
		"r6 = 1;"
		"r7 = 10;"
		"r8 = 100;"
		"r9 = 400;"
		"*(u64 *)(r10 - 8) = 600;"
		"call passes_on;"
		unwind_goto(2)
		"r0 = 0;"
		"exit;"
		"r0 = r6;"
		"r0 += r7;"
		"r0 += r8;"
		"r0 += r9;"
		"r1 = *(u64 *)(r10 - 8);"
		"r0 += r1;"
		"exit;"
		:
		: __imm_const(len, offsetof(struct __sk_buff, len)),
		  __imm_unwind_goto(2)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: r1-r5 are not readable in the pad")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("R3 !read_ok")
__naked int pad_reads_r3(void)
{
	asm volatile (
		"r3 = 0;"
		"call always_unwinds;"
		unwind_goto(1)
		"exit;"
		"r0 = r3;"
		"exit;"
		:
		: __imm_unwind_goto(1)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: unwind_exit of the main prog is exit")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(3)
__naked int main_prog_unwinds(void)
{
	asm volatile (
		"r0 = 3;"
		unwind_exit
		:
		: __imm_unwind_exit
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: unwind_exit of the main prog needs r0")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("R0 !read_ok")
__naked int main_prog_unwinds_no_r0(void)
{
	asm volatile (
		unwind_exit
		:
		: __imm_unwind_exit
		: __clobber_all
	);
}

/*
 * What follows unwind_goto is dead code. unwind_exit still has to find the pad,
 * and the cpu predicts that the callee returns: there is a trap for that.
 */
SEC("tc")
__description("unwind: the callee never returns")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(42)
__xlated("0: call pc+4")
__xlated("1: unwind_goto pc+1")
__xlated("2: goto pc-1")
__xlated("3: r0 = 42")
__xlated("4: exit")
__naked int callee_never_returns(void)
{
	asm volatile (
		"call always_unwinds;"
		unwind_goto(2)
		"r0 = 1;"
		"exit;"
		"r0 = 42;"
		"exit;"
		:
		: __imm_unwind_goto(2)
		: __clobber_all
	);
}

/* The pad is dead code and goes away together with unwind_goto */
SEC("tc")
__description("unwind: the callee never unwinds")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__xlated("0: call pc+1")
__xlated("1: exit")
__xlated("2: r0 = 7")
__xlated("3: exit")
__naked int callee_never_unwinds(void)
{
	asm volatile (
		"call never_unwinds;"
		unwind_goto(1)
		"exit;"
		"r0 = *(u64 *)(r0 + 0);"
		"exit;"
		:
		: __imm_unwind_goto(1)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: the pad drops what the frame owns")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(42)
__naked int pad_unlocks(void)
{
	asm volatile (
		"r6 = *(u32 *)(r1 + %[len]);"
		"call %[bpf_rcu_read_lock];"
		"r1 = r6;"
		"call may_unwind;"
		unwind_goto(3)
		"call %[bpf_rcu_read_unlock];"
		"r0 = 0;"
		"exit;"
		"call %[bpf_rcu_read_unlock];"
		"r0 = 42;"
		"exit;"
		:
		: __imm_const(len, offsetof(struct __sk_buff, len)),
		  __imm(bpf_rcu_read_lock),
		  __imm(bpf_rcu_read_unlock),
		  __imm_unwind_goto(3)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: the pad is checked like any other code")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("in main prog cannot be used inside bpf_rcu_read_lock-ed region")
__naked int pad_does_not_unlock(void)
{
	asm volatile (
		"r6 = *(u32 *)(r1 + %[len]);"
		"call %[bpf_rcu_read_lock];"
		"r1 = r6;"
		"call may_unwind;"
		unwind_goto(3)
		"call %[bpf_rcu_read_unlock];"
		"r0 = 0;"
		"exit;"
		"r0 = 42;"
		"exit;"
		:
		: __imm_const(len, offsetof(struct __sk_buff, len)),
		  __imm(bpf_rcu_read_lock),
		  __imm(bpf_rcu_read_unlock),
		  __imm_unwind_goto(3)
		: __clobber_all
	);
}

/* Leaves either way with the same registers */
__naked __noinline __used
static unsigned long same_regs(void)
{
	asm volatile (
		"r0 = 0;"
		"if r1 != 0 goto 1f;"
		"exit;"
	"1:"
		unwind_exit
		:
		: __imm_unwind_exit
	);
}

/*
 * After the call the two states differ only in the way the callee left.
 * The second one must not be pruned, or nobody looks at the pad.
 */
SEC("tc")
__description("unwind: the way the callee left is a part of the state")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("R0 invalid mem access 'scalar'")
__flag(BPF_F_TEST_STATE_FREQ)
__naked int no_pruning_after_the_call(void)
{
	asm volatile (
		"r1 = *(u32 *)(r1 + %[len]);"
		"call same_regs;"
		unwind_goto(1)
		"exit;"
		"r0 = *(u64 *)(r0 + 0);"
		"exit;"
		:
		: __imm_const(len, offsetof(struct __sk_buff, len)),
		  __imm_unwind_goto(1)
		: __clobber_all
	);
}

/* Returns ctx, or unwinds with a number in r0 */
__naked __noinline __used
static unsigned long ctx_or_unwind(void)
{
	asm volatile (
		"r2 = *(u32 *)(r1 + %[len]);"
		"if r2 > 100 goto 1f;"
		"r0 = r1;"
		"exit;"
	"1:"
		"r0 = 0;"
		unwind_exit
		:
		: __imm_const(len, offsetof(struct __sk_buff, len)),
		  __imm_unwind_exit
	);
}

/*
 * unwind_exit changes its return address, so the cpu mispredicts the return and
 * runs what follows unwind_goto with r0 of unwind_exit.
 */
SEC("socket")
__description("unwind: what follows unwind_goto runs speculatively")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(50) /* 64 without the ethernet header */
__success_unpriv __retval_unpriv(50)
__caps_unpriv(CAP_BPF)
#ifdef SPEC_V1
__xlated_unpriv("unwind_goto pc+{{.*}}")
__xlated_unpriv("nospec") /* inserted to prevent `R0 invalid mem access 'scalar'` */
__xlated_unpriv("r0 = *(u32 *)(r0 +{{.*}})")
#endif
__naked int speculative_fall_through(void)
{
	asm volatile (
		"call ctx_or_unwind;"
		unwind_goto(2)
		"r0 = *(u32 *)(r0 + %[len]);"
		"exit;"
		"r0 = 42;"
		"exit;"
		:
		: __imm_const(len, offsetof(struct __sk_buff, len)),
		  __imm_unwind_goto(2)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: a global subprog returns")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__naked int calls_global_returns(void)
{
	asm volatile (
		"call global_returns;"
		unwind_goto(1)
		"exit;"
		"r0 = 42;"
		"exit;"
		:
		: __imm_unwind_goto(1)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: a global subprog unwinds")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(15)
__naked int calls_global_unwinds(void)
{
	asm volatile (
		"call global_unwinds;"
		unwind_goto(1)
		"exit;"
		"r0 += 10;"
		"exit;"
		:
		: __imm_unwind_goto(1)
		: __clobber_all
	);
}

/* The verifier doesn't look inside */
SEC("tc")
__description("unwind: the pad of a call of a global subprog is always checked")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("R0 invalid mem access 'scalar'")
__naked int global_pad_is_checked(void)
{
	asm volatile (
		"call global_returns;"
		unwind_goto(1)
		"exit;"
		"r0 = *(u64 *)(r0 + 0);"
		"exit;"
		:
		: __imm_unwind_goto(1)
		: __clobber_all
	);
}

__naked __noinline __used
unsigned long global_never_unwinds(void)
{
	asm volatile (
		"r0 = 7;"
		"exit;"
	);
}

SEC("tc")
__description("unwind: a global subprog that cannot unwind")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__naked int calls_global_never_unwinds(void)
{
	asm volatile (
		"call global_never_unwinds;"
		"exit;"
		::: __clobber_all
	);
}

/*
 * The verifier gets to the pad from the first call only, with r6 = 1. The second
 * unwind_goto stays, since the pad is not dead code.
 */
SEC("tc")
__description("unwind: a pad shared with a call of a subprog that cannot unwind")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__naked int shared_pad(void)
{
	asm volatile (
		"r6 = 1;"
		"call global_returns;"
		unwind_goto(4)
		"r6 = 2;"
		"call global_never_unwinds;"
		unwind_goto(1)
		"exit;"
		"r0 = r6;"
		"exit;"
		:
		: __imm_unwind_goto(4),
		  __imm_unwind_goto(1)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: no unwind_goto after a call of a global subprog")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("no unwind_goto after the call of global_returns()")
__naked int global_no_unwind_goto(void)
{
	asm volatile (
		"call global_returns;"
		"exit;"
		::: __clobber_all
	);
}

SEC("tc")
__description("unwind: no unwind_goto after a call of a static subprog")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("unwind_exit, but no unwind_goto after the call at insn 1")
__naked int static_no_unwind_goto(void)
{
	asm volatile (
		"r1 = *(u32 *)(r1 + %[len]);"
		"call may_unwind;"
		"exit;"
		:
		: __imm_const(len, offsetof(struct __sk_buff, len))
		: __clobber_all
	);
}

/* The verifier sees that it doesn't get to unwind_exit */
SEC("tc")
__description("unwind: no unwind_goto is fine when the callee doesn't unwind")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__naked int static_does_not_get_there(void)
{
	asm volatile (
		"r1 = 0;"
		"call may_unwind;"
		"exit;"
		::: __clobber_all
	);
}

SEC("tc")
__description("unwind: callx, the callee returns")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__naked int callx_returns(void)
{
	asm volatile (
		"r1 = *(u32 *)(r1 + %[len]);"
		"r1 -= 64;"
		"r6 = %[may_unwind] ll;"
		"callx r6;"
		unwind_goto(1)
		"exit;"
		"r0 = 42;"
		"exit;"
		:
		: __imm_const(len, offsetof(struct __sk_buff, len)),
		  __imm_addr(may_unwind),
		  __imm_unwind_goto(1)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: callx, the callee unwinds")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(42)
__naked int callx_unwinds(void)
{
	asm volatile (
		"r1 = *(u32 *)(r1 + %[len]);"
		"r6 = %[may_unwind] ll;"
		"callx r6;"
		unwind_goto(1)
		"exit;"
		"r0 = 42;"
		"exit;"
		:
		: __imm_const(len, offsetof(struct __sk_buff, len)),
		  __imm_addr(may_unwind),
		  __imm_unwind_goto(1)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: unwind_goto after a plain insn")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("unwind_goto at insn 1 doesn't follow a call of a subprog")
__naked int unwind_goto_after_mov(void)
{
	asm volatile (
		"r0 = 0;"
		unwind_goto(0)
		"exit;"
		:
		: __imm_unwind_goto(0)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: unwind_goto is the first insn")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("unwind_goto at insn 0 doesn't follow a call of a subprog")
__naked int unwind_goto_first(void)
{
	asm volatile (
		unwind_goto(0)
		"r0 = 0;"
		"exit;"
		:
		: __imm_unwind_goto(0)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: unwind_goto after a call of a helper")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("unwind_goto at insn 1 doesn't follow a call of a subprog")
__naked int unwind_goto_after_helper(void)
{
	asm volatile (
		"call %[bpf_get_prandom_u32];"
		unwind_goto(0)
		"exit;"
		:
		: __imm(bpf_get_prandom_u32),
		  __imm_unwind_goto(0)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: unwind_goto after a call of a kfunc")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("unwind_goto at insn 1 doesn't follow a call of a subprog")
__naked int unwind_goto_after_kfunc(void)
{
	asm volatile (
		"call %[bpf_rcu_read_lock];"
		unwind_goto(0)
		"call %[bpf_rcu_read_unlock];"
		"r0 = 0;"
		"exit;"
		:
		: __imm(bpf_rcu_read_lock),
		  __imm(bpf_rcu_read_unlock),
		  __imm_unwind_goto(0)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: jump to unwind_goto")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("jump to unwind_goto at insn 3")
__naked int jump_to_unwind_goto(void)
{
	asm volatile (
		"r0 = 0;"
		"if r1 != 0 goto 1f;"
		"call never_unwinds;"
	"1:"
		unwind_goto(0)
		"exit;"
		:
		: __imm_unwind_goto(0)
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: reserved fields of unwind_goto")
__failure __msg("invalid may_goto imm 1")
__naked int unwind_goto_reserved(void)
{
	asm volatile (
		"call never_unwinds;"
		".8byte %[bad];"
		"exit;"
		:
		: __imm_insn(bad, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, BPF_UNWIND_GOTO, 0, 1))
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: unknown kind of jcond")
__failure __msg("invalid may_goto imm 0")
__naked int jcond_unknown(void)
{
	asm volatile (
		"call never_unwinds;"
		".8byte %[bad];"
		"exit;"
		:
		: __imm_insn(bad, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, BPF_UNWIND_GOTO + 1, 0, 0))
		: __clobber_all
	);
}

SEC("tc")
__description("unwind: unknown kind of exit")
__failure __msg("BPF_EXIT uses reserved fields")
__naked int exit_unknown(void)
{
	asm volatile (
		"r0 = 0;"
		".8byte %[bad];"
		:
		: __imm_insn(bad, BPF_RAW_INSN(BPF_JMP | BPF_EXIT, 0, BPF_UNWIND_EXIT + 1, 0, 0))
		: __clobber_all
	);
}

/* r1 is the index: unwinds in the second iteration */
__naked __noinline __used
static unsigned long cb_unwinds(void)
{
	asm volatile (
		"r0 = 0;"
		"if r1 != 0 goto 1f;"
		"exit;"
	"1:"
		unwind_exit
		:
		: __imm_unwind_exit
	);
}

SEC("tc")
__description("unwind: callback unwinds")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("callback cannot unwind")
__naked int callback_unwinds(void)
{
	asm volatile (
		"r1 = 3;"
		"r2 = %[cb_unwinds];"
		"r3 = 0;"
		"r4 = 0;"
		"call %[bpf_loop];"
		"exit;"
		:
		: __imm_ptr(cb_unwinds),
		  __imm(bpf_loop)
		: __clobber_common
	);
}

__naked __noinline __used
static unsigned long async_cb_unwinds(void)
{
	asm volatile (
		"r0 = 0;"
		unwind_exit
		:
		: __imm_unwind_exit
	);
}

/* The kernel calls it. It's the top frame like the main prog: unwind_exit is exit. */
SEC("tc")
__description("unwind: async callback unwinds")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(0)
int async_callback_unwinds(void *ctx)
{
	struct timer_elem *val;
	int key = 0;

	val = bpf_map_lookup_elem(&timer_map, &key);
	if (!val)
		return 0;

	bpf_timer_init(&val->t, &timer_map, 0);
	bpf_timer_set_callback(&val->t, (void *)async_cb_unwinds);
	bpf_timer_start(&val->t, 0, 0);
	return 0;
}

/* Not naked: the argument has to have a name in BTF */
__noinline int exception_cb_unwinds(__u64 cookie)
{
	if (cookie)
		asm volatile ("r0 = 7;" unwind_exit :: __imm_unwind_exit : "r0");
	return 0;
}

SEC("tc")
__description("unwind: exception callback unwinds")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__attribute__((btf_decl_tag("exception_callback:exception_cb_unwinds")))
__success __retval(7)
int exception_callback_unwinds(struct __sk_buff *skb)
{
	bpf_throw(skb->len);
	return 0;
}

/* What it calls can unwind as long as it doesn't let it out */
__naked __noinline __used
static unsigned long cb_catches(void)
{
	asm volatile (
		"call may_unwind;"
		unwind_goto(0)
		"r0 = 0;"
		"exit;"
		:
		: __imm_unwind_goto(0)
	);
}

SEC("tc")
__description("unwind: callback catches")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(3)
__naked int callback_catches(void)
{
	asm volatile (
		"r1 = 3;"
		"r2 = %[cb_catches];"
		"r3 = 0;"
		"r4 = 0;"
		"call %[bpf_loop];"
		"exit;"
		:
		: __imm_ptr(cb_catches),
		  __imm(bpf_loop)
		: __clobber_common
	);
}

/* Both leave the subprog like exit does */
__naked __noinline __used
static unsigned long does_tail_call(void)
{
	asm volatile (
		"r6 = *(u32 *)(r1 + %[len]);"
		"r2 = %[prog_array] ll;"
		"r3 = 0;"
		"call %[bpf_tail_call];"
		"r0 = 7;"
		"if r6 == 64 goto 1f;"
		unwind_exit
	"1:"
		"exit;"
		:
		: __imm_const(len, offsetof(struct __sk_buff, len)),
		  __imm(bpf_tail_call),
		  __imm_addr(prog_array),
		  __imm_unwind_exit
	);
}

SEC("tc")
__description("unwind: tail call in a subprog that can unwind")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__naked int tail_call_in_subprog(void)
{
	asm volatile (
		"call does_tail_call;"
		unwind_goto(1)
		"exit;"
		"r0 = 42;"
		"exit;"
		:
		: __imm_unwind_goto(1)
		: __clobber_all
	);
}

/* The packet is shorter than that: ld_abs leaves the subprog with r0 = 0 */
__naked __noinline __used
static unsigned long does_ld_abs(void)
{
	asm volatile (
		"r6 = r1;"
		"r0 = *(u8 *)skb[1000];"
		"r0 = 5;"
		unwind_exit
		:
		: __imm_unwind_exit
	);
}

SEC("tc")
__description("unwind: ld_abs in a subprog that can unwind")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__naked int ld_abs_in_subprog(void)
{
	asm volatile (
		"call does_ld_abs;"
		unwind_goto(2)
		"r0 += 7;"
		"exit;"
		"r0 = 42;"
		"exit;"
		:
		: __imm_unwind_goto(2)
		: __clobber_all
	);
}

/*
 * JITs don't emit anything for unwind_goto. It's an extable entry, where
 * unwind_exit looks its return address up.
 * With KASAN x86-64 JIT wraps a load in pushes, a call and pops. 'r2 = 0x1234'
 * is where the patterns start, so that they don't take that call for the one
 * of the subprog.
 */
SEC("tc")
__description("unwind: jit")
__success __retval(42)
__load_if_JITed()
__arch_x86_64
__jited("	movl	$0x1234, %esi")
__jited("...") /* call depth accounting */
__jited("	callq	0x{{.*}}")
__jited("	movl	$0x7, %eax")
__jited("...")
__jited("	movl	$0x5, %eax")
__jited("	pushq	%rax")
__jited("	pushq	%rsi")
__jited("	movq	0x8(%rbp), %rdi")
__jited("...") /* call depth accounting */
__jited("	callq	0x{{.*}}")
__jited("	movq	%rax, 0x8(%rbp)")
__jited("	popq	%rsi")
__jited("	popq	%rax")
__arch_arm64
__jited("	mov	x1, #0x1234")
__jited("	bl	{{.*}}")
__jited("	mov	x8, x0")
__jited("	mov	x8, #0x7")
__jited("...")
__jited("	mov	x8, #0x5")
__jited("	stp	x8, x1, [sp, #-0x10]!")
__jited("	ldr	x0, [x29, #0x8]")
__jited("...")
__jited("	mov	x10, x0")
__jited("	ldp	x8, x1, [sp], #0x10")
__jited("...")
__jited("	add	x30, x30, x10")
__jited("	ret")
__naked int jit(void)
{
	asm volatile (
		"r1 = *(u32 *)(r1 + %[len]);"
		"r2 = 0x1234;"
		"call may_unwind;"
		unwind_goto(2)
		"r0 = 7;"
		"exit;"
		"r0 = 42;"
		"exit;"
		:
		: __imm_const(len, offsetof(struct __sk_buff, len)),
		  __imm_unwind_goto(2)
		: __clobber_all
	);
}

/* r1 == 0: returns 7, otherwise unwinds with r0 = 5 */
__naked __noinline __used
static unsigned long may_unwind_big_stack(void)
{
	asm volatile (
		"*(u64 *)(r10 - 200) = r1;"
		"if r1 != 0 goto 1f;"
		"r0 = 7;"
		"exit;"
	"1:"
		"r0 = 5;"
		unwind_exit
		:
		: __imm_unwind_exit
	);
}

/*
 * x86-64 keeps the pointer to the private stack in r9 and saves it around a call.
 * The value that is read back in the pad is there only when r9 is restored.
 */
SEC("raw_tp")
__description("unwind: private stack, the callee unwinds")
__success __retval(47)
__load_if_JITed()
__arch_x86_64
__jited("	movl	$0x1234, %esi") /* past the load, see above */
__jited("	pushq	%r9")
__jited("...") /* call depth accounting */
__jited("	callq	0x{{.*}}")
__jited("	popq	%r9")
__jited("	jmp	L{{.*}}")
__jited("	popq	%r9")
__jited("	jmp	L{{.*}}")
__arch_arm64
__naked int private_stack_unwinds(void)
{
	asm volatile (
		"*(u64 *)(r10 - 200) = 42;"
		"r1 = %[one] ll;"
		"r1 = *(u64 *)(r1 + 0);"
		"r2 = 0x1234;"
		"call may_unwind_big_stack;"
		unwind_goto(1)
		"exit;"
		"r1 = *(u64 *)(r10 - 200);"
		"r0 += r1;"
		"exit;"
		:
		: __imm_addr(one),
		  __imm_unwind_goto(1)
		: __clobber_all
	);
}

SEC("raw_tp")
__description("unwind: private stack, the callee returns")
__success __retval(49)
__load_if_JITed()
__arch_x86_64
__arch_arm64
__naked int private_stack_returns(void)
{
	asm volatile (
		"*(u64 *)(r10 - 200) = 42;"
		"r1 = %[zero] ll;"
		"r1 = *(u64 *)(r1 + 0);"
		"call may_unwind_big_stack;"
		unwind_goto(3)
		"r1 = *(u64 *)(r10 - 200);"
		"r0 += r1;"
		"exit;"
		"r0 = 0;"
		"exit;"
		:
		: __imm_addr(zero),
		  __imm_unwind_goto(3)
		: __clobber_all
	);
}

/*
 * BTF FUNC records are not generated for kfuncs referenced
 * from inline assembly. These records are necessary for
 * libbpf to link the program. The function below is a hack
 * to ensure that BTF FUNC records are generated.
 */
void __kfunc_btf_root(void)
{
	bpf_rcu_read_lock();
	bpf_rcu_read_unlock();
}

char _license[] SEC("license") = "GPL";