// SPDX-License-Identifier: GPL-2.0
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include "../../../include/linux/filter.h"
#include "bpf_misc.h"
/*
* llvm lowers unwinding to two insns:
* invoke foo unwind pad call foo; unwind_goto pad
* resume unwind_exit
* The tests spell them out as raw insns, so they don't depend on llvm version.
* The argument of unwind_goto() is the number of insns to skip.
*/
#define unwind_goto(off) ".8byte %[insn_unwind_goto_" #off "];"
#define unwind_exit ".8byte %[insn_unwind_exit];"
#define __imm_unwind_goto(off) \
__imm_insn(insn_unwind_goto_##off, \
BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, BPF_UNWIND_GOTO, off, 0))
#define __imm_unwind_exit \
__imm_insn(insn_unwind_exit, \
BPF_RAW_INSN(BPF_JMP | BPF_EXIT, 0, BPF_UNWIND_EXIT, 0, 0))
extern void bpf_rcu_read_lock(void) __ksym;
extern void bpf_rcu_read_unlock(void) __ksym;
extern void bpf_throw(__u64 cookie) __ksym;
struct {
__uint(type, BPF_MAP_TYPE_PROG_ARRAY);
__uint(max_entries, 1);
__uint(key_size, sizeof(int));
__uint(value_size, sizeof(int));
} prog_array SEC(".maps");
struct timer_elem {
struct bpf_timer t;
};
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__uint(max_entries, 1);
__type(key, int);
__type(value, struct timer_elem);
} timer_map SEC(".maps");
/* The verifier doesn't know what is in there */
unsigned long zero;
unsigned long one = 1;
/*
* The verifier follows a static subprog, so it knows which way it leaves when
* it knows the argument. To have both ways at run time the tests pass skb->len,
* which is 64, or skb->len - 64.
*
* r1 == 0: returns 7, otherwise unwinds with r0 = 5
*/
__naked __noinline __used
static unsigned long may_unwind(void)
{
asm volatile (
"if r1 != 0 goto 1f;"
"r0 = 7;"
"exit;"
"1:"
"r0 = 5;"
unwind_exit
:
: __imm_unwind_exit
);
}
/* owns nothing: its pad only carries on */
__naked __noinline __used
static unsigned long passes_on(void)
{
asm volatile (
"call may_unwind;"
unwind_goto(2)
"r0 += 1;"
"exit;"
unwind_exit
:
: __imm_unwind_goto(2),
__imm_unwind_exit
);
}
__naked __noinline __used
static unsigned long always_unwinds(void)
{
asm volatile (
"r0 = 5;"
unwind_exit
:
: __imm_unwind_exit
);
}
__naked __noinline __used
static unsigned long never_unwinds(void)
{
asm volatile (
"r0 = 7;"
"exit;"
);
}
/* Both can unwind. This one returns 7. */
__naked __noinline __used
unsigned long global_returns(void)
{
asm volatile (
"r1 = %[zero] ll;"
"r1 = *(u64 *)(r1 + 0);"
"if r1 != 0 goto 1f;"
"r0 = 7;"
"exit;"
"1:"
"r0 = 5;"
unwind_exit
:
: __imm_addr(zero),
__imm_unwind_exit
);
}
/* This one unwinds with r0 = 5 */
__naked __noinline __used
unsigned long global_unwinds(void)
{
asm volatile (
"r1 = %[one] ll;"
"r1 = *(u64 *)(r1 + 0);"
"if r1 != 0 goto 1f;"
"r0 = 7;"
"exit;"
"1:"
"r0 = 5;"
unwind_exit
:
: __imm_addr(one),
__imm_unwind_exit
);
}
SEC("tc")
__description("unwind: the callee returns")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__naked int callee_returns(void)
{
asm volatile (
"r1 = *(u32 *)(r1 + %[len]);"
"r1 -= 64;"
"call may_unwind;"
unwind_goto(1)
"exit;"
"r0 = 42;"
"exit;"
:
: __imm_const(len, offsetof(struct __sk_buff, len)),
__imm_unwind_goto(1)
: __clobber_all
);
}
SEC("tc")
__description("unwind: the callee unwinds")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(42)
__naked int callee_unwinds(void)
{
asm volatile (
"r1 = *(u32 *)(r1 + %[len]);"
"call may_unwind;"
unwind_goto(1)
"exit;"
"r0 = 42;"
"exit;"
:
: __imm_const(len, offsetof(struct __sk_buff, len)),
__imm_unwind_goto(1)
: __clobber_all
);
}
SEC("tc")
__description("unwind: return through two frames")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(8)
__naked int two_frames_return(void)
{
asm volatile (
"r1 = *(u32 *)(r1 + %[len]);"
"r1 -= 64;"
"call passes_on;"
unwind_goto(1)
"exit;"
"r0 = 42;"
"exit;"
:
: __imm_const(len, offsetof(struct __sk_buff, len)),
__imm_unwind_goto(1)
: __clobber_all
);
}
/* unwind_exit returns r0 like exit does */
SEC("tc")
__description("unwind: unwind through two frames, r0 reaches the pad")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(15)
__naked int two_frames_unwind(void)
{
asm volatile (
"r1 = *(u32 *)(r1 + %[len]);"
"call passes_on;"
unwind_goto(1)
"exit;"
"r0 += 10;"
"exit;"
:
: __imm_const(len, offsetof(struct __sk_buff, len)),
__imm_unwind_goto(1)
: __clobber_all
);
}
SEC("tc")
__description("unwind: r6-r9 and the stack are intact in the pad")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(1111)
__naked int pad_sees_callee_saved(void)
{
asm volatile (
"r1 = *(u32 *)(r1 + %[len]);"
"r6 = 1;"
"r7 = 10;"
"r8 = 100;"
"r9 = 400;"
"*(u64 *)(r10 - 8) = 600;"
"call passes_on;"
unwind_goto(2)
"r0 = 0;"
"exit;"
"r0 = r6;"
"r0 += r7;"
"r0 += r8;"
"r0 += r9;"
"r1 = *(u64 *)(r10 - 8);"
"r0 += r1;"
"exit;"
:
: __imm_const(len, offsetof(struct __sk_buff, len)),
__imm_unwind_goto(2)
: __clobber_all
);
}
SEC("tc")
__description("unwind: r1-r5 are not readable in the pad")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("R3 !read_ok")
__naked int pad_reads_r3(void)
{
asm volatile (
"r3 = 0;"
"call always_unwinds;"
unwind_goto(1)
"exit;"
"r0 = r3;"
"exit;"
:
: __imm_unwind_goto(1)
: __clobber_all
);
}
SEC("tc")
__description("unwind: unwind_exit of the main prog is exit")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(3)
__naked int main_prog_unwinds(void)
{
asm volatile (
"r0 = 3;"
unwind_exit
:
: __imm_unwind_exit
: __clobber_all
);
}
SEC("tc")
__description("unwind: unwind_exit of the main prog needs r0")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("R0 !read_ok")
__naked int main_prog_unwinds_no_r0(void)
{
asm volatile (
unwind_exit
:
: __imm_unwind_exit
: __clobber_all
);
}
/*
* What follows unwind_goto is dead code. unwind_exit still has to find the pad,
* and the cpu predicts that the callee returns: there is a trap for that.
*/
SEC("tc")
__description("unwind: the callee never returns")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(42)
__xlated("0: call pc+4")
__xlated("1: unwind_goto pc+1")
__xlated("2: goto pc-1")
__xlated("3: r0 = 42")
__xlated("4: exit")
__naked int callee_never_returns(void)
{
asm volatile (
"call always_unwinds;"
unwind_goto(2)
"r0 = 1;"
"exit;"
"r0 = 42;"
"exit;"
:
: __imm_unwind_goto(2)
: __clobber_all
);
}
/* The pad is dead code and goes away together with unwind_goto */
SEC("tc")
__description("unwind: the callee never unwinds")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__xlated("0: call pc+1")
__xlated("1: exit")
__xlated("2: r0 = 7")
__xlated("3: exit")
__naked int callee_never_unwinds(void)
{
asm volatile (
"call never_unwinds;"
unwind_goto(1)
"exit;"
"r0 = *(u64 *)(r0 + 0);"
"exit;"
:
: __imm_unwind_goto(1)
: __clobber_all
);
}
SEC("tc")
__description("unwind: the pad drops what the frame owns")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(42)
__naked int pad_unlocks(void)
{
asm volatile (
"r6 = *(u32 *)(r1 + %[len]);"
"call %[bpf_rcu_read_lock];"
"r1 = r6;"
"call may_unwind;"
unwind_goto(3)
"call %[bpf_rcu_read_unlock];"
"r0 = 0;"
"exit;"
"call %[bpf_rcu_read_unlock];"
"r0 = 42;"
"exit;"
:
: __imm_const(len, offsetof(struct __sk_buff, len)),
__imm(bpf_rcu_read_lock),
__imm(bpf_rcu_read_unlock),
__imm_unwind_goto(3)
: __clobber_all
);
}
SEC("tc")
__description("unwind: the pad is checked like any other code")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("in main prog cannot be used inside bpf_rcu_read_lock-ed region")
__naked int pad_does_not_unlock(void)
{
asm volatile (
"r6 = *(u32 *)(r1 + %[len]);"
"call %[bpf_rcu_read_lock];"
"r1 = r6;"
"call may_unwind;"
unwind_goto(3)
"call %[bpf_rcu_read_unlock];"
"r0 = 0;"
"exit;"
"r0 = 42;"
"exit;"
:
: __imm_const(len, offsetof(struct __sk_buff, len)),
__imm(bpf_rcu_read_lock),
__imm(bpf_rcu_read_unlock),
__imm_unwind_goto(3)
: __clobber_all
);
}
/* Leaves either way with the same registers */
__naked __noinline __used
static unsigned long same_regs(void)
{
asm volatile (
"r0 = 0;"
"if r1 != 0 goto 1f;"
"exit;"
"1:"
unwind_exit
:
: __imm_unwind_exit
);
}
/*
* After the call the two states differ only in the way the callee left.
* The second one must not be pruned, or nobody looks at the pad.
*/
SEC("tc")
__description("unwind: the way the callee left is a part of the state")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("R0 invalid mem access 'scalar'")
__flag(BPF_F_TEST_STATE_FREQ)
__naked int no_pruning_after_the_call(void)
{
asm volatile (
"r1 = *(u32 *)(r1 + %[len]);"
"call same_regs;"
unwind_goto(1)
"exit;"
"r0 = *(u64 *)(r0 + 0);"
"exit;"
:
: __imm_const(len, offsetof(struct __sk_buff, len)),
__imm_unwind_goto(1)
: __clobber_all
);
}
/* Returns ctx, or unwinds with a number in r0 */
__naked __noinline __used
static unsigned long ctx_or_unwind(void)
{
asm volatile (
"r2 = *(u32 *)(r1 + %[len]);"
"if r2 > 100 goto 1f;"
"r0 = r1;"
"exit;"
"1:"
"r0 = 0;"
unwind_exit
:
: __imm_const(len, offsetof(struct __sk_buff, len)),
__imm_unwind_exit
);
}
/*
* unwind_exit changes its return address, so the cpu mispredicts the return and
* runs what follows unwind_goto with r0 of unwind_exit.
*/
SEC("socket")
__description("unwind: what follows unwind_goto runs speculatively")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(50) /* 64 without the ethernet header */
__success_unpriv __retval_unpriv(50)
__caps_unpriv(CAP_BPF)
#ifdef SPEC_V1
__xlated_unpriv("unwind_goto pc+{{.*}}")
__xlated_unpriv("nospec") /* inserted to prevent `R0 invalid mem access 'scalar'` */
__xlated_unpriv("r0 = *(u32 *)(r0 +{{.*}})")
#endif
__naked int speculative_fall_through(void)
{
asm volatile (
"call ctx_or_unwind;"
unwind_goto(2)
"r0 = *(u32 *)(r0 + %[len]);"
"exit;"
"r0 = 42;"
"exit;"
:
: __imm_const(len, offsetof(struct __sk_buff, len)),
__imm_unwind_goto(2)
: __clobber_all
);
}
SEC("tc")
__description("unwind: a global subprog returns")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__naked int calls_global_returns(void)
{
asm volatile (
"call global_returns;"
unwind_goto(1)
"exit;"
"r0 = 42;"
"exit;"
:
: __imm_unwind_goto(1)
: __clobber_all
);
}
SEC("tc")
__description("unwind: a global subprog unwinds")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(15)
__naked int calls_global_unwinds(void)
{
asm volatile (
"call global_unwinds;"
unwind_goto(1)
"exit;"
"r0 += 10;"
"exit;"
:
: __imm_unwind_goto(1)
: __clobber_all
);
}
/* The verifier doesn't look inside */
SEC("tc")
__description("unwind: the pad of a call of a global subprog is always checked")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("R0 invalid mem access 'scalar'")
__naked int global_pad_is_checked(void)
{
asm volatile (
"call global_returns;"
unwind_goto(1)
"exit;"
"r0 = *(u64 *)(r0 + 0);"
"exit;"
:
: __imm_unwind_goto(1)
: __clobber_all
);
}
__naked __noinline __used
unsigned long global_never_unwinds(void)
{
asm volatile (
"r0 = 7;"
"exit;"
);
}
SEC("tc")
__description("unwind: a global subprog that cannot unwind")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__naked int calls_global_never_unwinds(void)
{
asm volatile (
"call global_never_unwinds;"
"exit;"
::: __clobber_all
);
}
/*
* The verifier gets to the pad from the first call only, with r6 = 1. The second
* unwind_goto stays, since the pad is not dead code.
*/
SEC("tc")
__description("unwind: a pad shared with a call of a subprog that cannot unwind")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__naked int shared_pad(void)
{
asm volatile (
"r6 = 1;"
"call global_returns;"
unwind_goto(4)
"r6 = 2;"
"call global_never_unwinds;"
unwind_goto(1)
"exit;"
"r0 = r6;"
"exit;"
:
: __imm_unwind_goto(4),
__imm_unwind_goto(1)
: __clobber_all
);
}
SEC("tc")
__description("unwind: no unwind_goto after a call of a global subprog")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("no unwind_goto after the call of global_returns()")
__naked int global_no_unwind_goto(void)
{
asm volatile (
"call global_returns;"
"exit;"
::: __clobber_all
);
}
SEC("tc")
__description("unwind: no unwind_goto after a call of a static subprog")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("unwind_exit, but no unwind_goto after the call at insn 1")
__naked int static_no_unwind_goto(void)
{
asm volatile (
"r1 = *(u32 *)(r1 + %[len]);"
"call may_unwind;"
"exit;"
:
: __imm_const(len, offsetof(struct __sk_buff, len))
: __clobber_all
);
}
/* The verifier sees that it doesn't get to unwind_exit */
SEC("tc")
__description("unwind: no unwind_goto is fine when the callee doesn't unwind")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__naked int static_does_not_get_there(void)
{
asm volatile (
"r1 = 0;"
"call may_unwind;"
"exit;"
::: __clobber_all
);
}
SEC("tc")
__description("unwind: callx, the callee returns")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__naked int callx_returns(void)
{
asm volatile (
"r1 = *(u32 *)(r1 + %[len]);"
"r1 -= 64;"
"r6 = %[may_unwind] ll;"
"callx r6;"
unwind_goto(1)
"exit;"
"r0 = 42;"
"exit;"
:
: __imm_const(len, offsetof(struct __sk_buff, len)),
__imm_addr(may_unwind),
__imm_unwind_goto(1)
: __clobber_all
);
}
SEC("tc")
__description("unwind: callx, the callee unwinds")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(42)
__naked int callx_unwinds(void)
{
asm volatile (
"r1 = *(u32 *)(r1 + %[len]);"
"r6 = %[may_unwind] ll;"
"callx r6;"
unwind_goto(1)
"exit;"
"r0 = 42;"
"exit;"
:
: __imm_const(len, offsetof(struct __sk_buff, len)),
__imm_addr(may_unwind),
__imm_unwind_goto(1)
: __clobber_all
);
}
SEC("tc")
__description("unwind: unwind_goto after a plain insn")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("unwind_goto at insn 1 doesn't follow a call of a subprog")
__naked int unwind_goto_after_mov(void)
{
asm volatile (
"r0 = 0;"
unwind_goto(0)
"exit;"
:
: __imm_unwind_goto(0)
: __clobber_all
);
}
SEC("tc")
__description("unwind: unwind_goto is the first insn")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("unwind_goto at insn 0 doesn't follow a call of a subprog")
__naked int unwind_goto_first(void)
{
asm volatile (
unwind_goto(0)
"r0 = 0;"
"exit;"
:
: __imm_unwind_goto(0)
: __clobber_all
);
}
SEC("tc")
__description("unwind: unwind_goto after a call of a helper")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("unwind_goto at insn 1 doesn't follow a call of a subprog")
__naked int unwind_goto_after_helper(void)
{
asm volatile (
"call %[bpf_get_prandom_u32];"
unwind_goto(0)
"exit;"
:
: __imm(bpf_get_prandom_u32),
__imm_unwind_goto(0)
: __clobber_all
);
}
SEC("tc")
__description("unwind: unwind_goto after a call of a kfunc")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("unwind_goto at insn 1 doesn't follow a call of a subprog")
__naked int unwind_goto_after_kfunc(void)
{
asm volatile (
"call %[bpf_rcu_read_lock];"
unwind_goto(0)
"call %[bpf_rcu_read_unlock];"
"r0 = 0;"
"exit;"
:
: __imm(bpf_rcu_read_lock),
__imm(bpf_rcu_read_unlock),
__imm_unwind_goto(0)
: __clobber_all
);
}
SEC("tc")
__description("unwind: jump to unwind_goto")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("jump to unwind_goto at insn 3")
__naked int jump_to_unwind_goto(void)
{
asm volatile (
"r0 = 0;"
"if r1 != 0 goto 1f;"
"call never_unwinds;"
"1:"
unwind_goto(0)
"exit;"
:
: __imm_unwind_goto(0)
: __clobber_all
);
}
SEC("tc")
__description("unwind: reserved fields of unwind_goto")
__failure __msg("invalid may_goto imm 1")
__naked int unwind_goto_reserved(void)
{
asm volatile (
"call never_unwinds;"
".8byte %[bad];"
"exit;"
:
: __imm_insn(bad, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, BPF_UNWIND_GOTO, 0, 1))
: __clobber_all
);
}
SEC("tc")
__description("unwind: unknown kind of jcond")
__failure __msg("invalid may_goto imm 0")
__naked int jcond_unknown(void)
{
asm volatile (
"call never_unwinds;"
".8byte %[bad];"
"exit;"
:
: __imm_insn(bad, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, BPF_UNWIND_GOTO + 1, 0, 0))
: __clobber_all
);
}
SEC("tc")
__description("unwind: unknown kind of exit")
__failure __msg("BPF_EXIT uses reserved fields")
__naked int exit_unknown(void)
{
asm volatile (
"r0 = 0;"
".8byte %[bad];"
:
: __imm_insn(bad, BPF_RAW_INSN(BPF_JMP | BPF_EXIT, 0, BPF_UNWIND_EXIT + 1, 0, 0))
: __clobber_all
);
}
/* r1 is the index: unwinds in the second iteration */
__naked __noinline __used
static unsigned long cb_unwinds(void)
{
asm volatile (
"r0 = 0;"
"if r1 != 0 goto 1f;"
"exit;"
"1:"
unwind_exit
:
: __imm_unwind_exit
);
}
SEC("tc")
__description("unwind: callback unwinds")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__failure __msg("callback cannot unwind")
__naked int callback_unwinds(void)
{
asm volatile (
"r1 = 3;"
"r2 = %[cb_unwinds];"
"r3 = 0;"
"r4 = 0;"
"call %[bpf_loop];"
"exit;"
:
: __imm_ptr(cb_unwinds),
__imm(bpf_loop)
: __clobber_common
);
}
__naked __noinline __used
static unsigned long async_cb_unwinds(void)
{
asm volatile (
"r0 = 0;"
unwind_exit
:
: __imm_unwind_exit
);
}
/* The kernel calls it. It's the top frame like the main prog: unwind_exit is exit. */
SEC("tc")
__description("unwind: async callback unwinds")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(0)
int async_callback_unwinds(void *ctx)
{
struct timer_elem *val;
int key = 0;
val = bpf_map_lookup_elem(&timer_map, &key);
if (!val)
return 0;
bpf_timer_init(&val->t, &timer_map, 0);
bpf_timer_set_callback(&val->t, (void *)async_cb_unwinds);
bpf_timer_start(&val->t, 0, 0);
return 0;
}
/* Not naked: the argument has to have a name in BTF */
__noinline int exception_cb_unwinds(__u64 cookie)
{
if (cookie)
asm volatile ("r0 = 7;" unwind_exit :: __imm_unwind_exit : "r0");
return 0;
}
SEC("tc")
__description("unwind: exception callback unwinds")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__attribute__((btf_decl_tag("exception_callback:exception_cb_unwinds")))
__success __retval(7)
int exception_callback_unwinds(struct __sk_buff *skb)
{
bpf_throw(skb->len);
return 0;
}
/* What it calls can unwind as long as it doesn't let it out */
__naked __noinline __used
static unsigned long cb_catches(void)
{
asm volatile (
"call may_unwind;"
unwind_goto(0)
"r0 = 0;"
"exit;"
:
: __imm_unwind_goto(0)
);
}
SEC("tc")
__description("unwind: callback catches")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(3)
__naked int callback_catches(void)
{
asm volatile (
"r1 = 3;"
"r2 = %[cb_catches];"
"r3 = 0;"
"r4 = 0;"
"call %[bpf_loop];"
"exit;"
:
: __imm_ptr(cb_catches),
__imm(bpf_loop)
: __clobber_common
);
}
/* Both leave the subprog like exit does */
__naked __noinline __used
static unsigned long does_tail_call(void)
{
asm volatile (
"r6 = *(u32 *)(r1 + %[len]);"
"r2 = %[prog_array] ll;"
"r3 = 0;"
"call %[bpf_tail_call];"
"r0 = 7;"
"if r6 == 64 goto 1f;"
unwind_exit
"1:"
"exit;"
:
: __imm_const(len, offsetof(struct __sk_buff, len)),
__imm(bpf_tail_call),
__imm_addr(prog_array),
__imm_unwind_exit
);
}
SEC("tc")
__description("unwind: tail call in a subprog that can unwind")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__naked int tail_call_in_subprog(void)
{
asm volatile (
"call does_tail_call;"
unwind_goto(1)
"exit;"
"r0 = 42;"
"exit;"
:
: __imm_unwind_goto(1)
: __clobber_all
);
}
/* The packet is shorter than that: ld_abs leaves the subprog with r0 = 0 */
__naked __noinline __used
static unsigned long does_ld_abs(void)
{
asm volatile (
"r6 = r1;"
"r0 = *(u8 *)skb[1000];"
"r0 = 5;"
unwind_exit
:
: __imm_unwind_exit
);
}
SEC("tc")
__description("unwind: ld_abs in a subprog that can unwind")
__arch_x86_64
__arch_arm64
__load_if_JITed()
__success __retval(7)
__naked int ld_abs_in_subprog(void)
{
asm volatile (
"call does_ld_abs;"
unwind_goto(2)
"r0 += 7;"
"exit;"
"r0 = 42;"
"exit;"
:
: __imm_unwind_goto(2)
: __clobber_all
);
}
/*
* JITs don't emit anything for unwind_goto. It's an extable entry, where
* unwind_exit looks its return address up.
* With KASAN x86-64 JIT wraps a load in pushes, a call and pops. 'r2 = 0x1234'
* is where the patterns start, so that they don't take that call for the one
* of the subprog.
*/
SEC("tc")
__description("unwind: jit")
__success __retval(42)
__load_if_JITed()
__arch_x86_64
__jited(" movl $0x1234, %esi")
__jited("...") /* call depth accounting */
__jited(" callq 0x{{.*}}")
__jited(" movl $0x7, %eax")
__jited("...")
__jited(" movl $0x5, %eax")
__jited(" pushq %rax")
__jited(" pushq %rsi")
__jited(" movq 0x8(%rbp), %rdi")
__jited("...") /* call depth accounting */
__jited(" callq 0x{{.*}}")
__jited(" movq %rax, 0x8(%rbp)")
__jited(" popq %rsi")
__jited(" popq %rax")
__arch_arm64
__jited(" mov x1, #0x1234")
__jited(" bl {{.*}}")
__jited(" mov x8, x0")
__jited(" mov x8, #0x7")
__jited("...")
__jited(" mov x8, #0x5")
__jited(" stp x8, x1, [sp, #-0x10]!")
__jited(" ldr x0, [x29, #0x8]")
__jited("...")
__jited(" mov x10, x0")
__jited(" ldp x8, x1, [sp], #0x10")
__jited("...")
__jited(" add x30, x30, x10")
__jited(" ret")
__naked int jit(void)
{
asm volatile (
"r1 = *(u32 *)(r1 + %[len]);"
"r2 = 0x1234;"
"call may_unwind;"
unwind_goto(2)
"r0 = 7;"
"exit;"
"r0 = 42;"
"exit;"
:
: __imm_const(len, offsetof(struct __sk_buff, len)),
__imm_unwind_goto(2)
: __clobber_all
);
}
/* r1 == 0: returns 7, otherwise unwinds with r0 = 5 */
__naked __noinline __used
static unsigned long may_unwind_big_stack(void)
{
asm volatile (
"*(u64 *)(r10 - 200) = r1;"
"if r1 != 0 goto 1f;"
"r0 = 7;"
"exit;"
"1:"
"r0 = 5;"
unwind_exit
:
: __imm_unwind_exit
);
}
/*
* x86-64 keeps the pointer to the private stack in r9 and saves it around a call.
* The value that is read back in the pad is there only when r9 is restored.
*/
SEC("raw_tp")
__description("unwind: private stack, the callee unwinds")
__success __retval(47)
__load_if_JITed()
__arch_x86_64
__jited(" movl $0x1234, %esi") /* past the load, see above */
__jited(" pushq %r9")
__jited("...") /* call depth accounting */
__jited(" callq 0x{{.*}}")
__jited(" popq %r9")
__jited(" jmp L{{.*}}")
__jited(" popq %r9")
__jited(" jmp L{{.*}}")
__arch_arm64
__naked int private_stack_unwinds(void)
{
asm volatile (
"*(u64 *)(r10 - 200) = 42;"
"r1 = %[one] ll;"
"r1 = *(u64 *)(r1 + 0);"
"r2 = 0x1234;"
"call may_unwind_big_stack;"
unwind_goto(1)
"exit;"
"r1 = *(u64 *)(r10 - 200);"
"r0 += r1;"
"exit;"
:
: __imm_addr(one),
__imm_unwind_goto(1)
: __clobber_all
);
}
SEC("raw_tp")
__description("unwind: private stack, the callee returns")
__success __retval(49)
__load_if_JITed()
__arch_x86_64
__arch_arm64
__naked int private_stack_returns(void)
{
asm volatile (
"*(u64 *)(r10 - 200) = 42;"
"r1 = %[zero] ll;"
"r1 = *(u64 *)(r1 + 0);"
"call may_unwind_big_stack;"
unwind_goto(3)
"r1 = *(u64 *)(r10 - 200);"
"r0 += r1;"
"exit;"
"r0 = 0;"
"exit;"
:
: __imm_addr(zero),
__imm_unwind_goto(3)
: __clobber_all
);
}
/*
* BTF FUNC records are not generated for kfuncs referenced
* from inline assembly. These records are necessary for
* libbpf to link the program. The function below is a hack
* to ensure that BTF FUNC records are generated.
*/
void __kfunc_btf_root(void)
{
bpf_rcu_read_lock();
bpf_rcu_read_unlock();
}
char _license[] SEC("license") = "GPL";