// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#include <test_progs.h>
#include "call_rcu.skel.h"
#include "call_rcu_fail.skel.h"
struct elem {
__u64 pad;
struct bpf_rcu_head rh;
__u64 val;
};
/*
* Force the grace period, then poll: the callback still has to be invoked
* afterwards, and call_rcu() is lazy on a CONFIG_RCU_LAZY kernel.
*/
static bool wait_for_callbacks(struct call_rcu *skel, int expected)
{
int i, got = 0;
kern_sync_rcu();
for (i = 0; i < 3000; i++) {
got = __atomic_load_n(&skel->bss->callbacks, __ATOMIC_ACQUIRE);
if (got >= expected)
return true;
usleep(10000);
}
return ASSERT_GE(got, expected, "callbacks");
}
static void test_call_rcu_run(bool trace)
{
LIBBPF_OPTS(bpf_test_run_opts, opts);
struct bpf_program *prog;
struct call_rcu *skel;
struct elem elem;
__u32 key = 1;
int err;
skel = call_rcu__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
return;
prog = trace ? skel->progs.arm_trace : skel->progs.arm;
err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts);
if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(opts.retval, 0, "retval"))
goto out;
ASSERT_EQ(skel->bss->arm_err, 0, "arm_err");
ASSERT_EQ(skel->bss->busy_err, -EBUSY, "busy_err");
if (!wait_for_callbacks(skel, 1))
goto out;
ASSERT_EQ(skel->bss->cb_key, key, "cb_key");
ASSERT_EQ(skel->bss->cb_val, 0xdeadbeef, "cb_val");
ASSERT_EQ(skel->bss->cb_max_entries, bpf_map__max_entries(skel->maps.arr), "cb_map");
err = bpf_map__lookup_elem(skel->maps.arr, &key, sizeof(key), &elem, sizeof(elem), 0);
if (ASSERT_OK(err, "lookup"))
ASSERT_EQ(elem.val, 0, "value_cleared");
/* The head is disarmed before the callback runs, so it can be reused. */
err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts);
if (!ASSERT_OK(err, "test_run_again"))
goto out;
ASSERT_EQ(skel->bss->arm_err, 0, "rearm_err");
wait_for_callbacks(skel, 2);
out:
call_rcu__destroy(skel);
}
static void test_call_rcu_chain(void)
{
LIBBPF_OPTS(bpf_test_run_opts, opts);
struct call_rcu *skel;
skel = call_rcu__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
return;
skel->bss->chain = 1;
if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm), &opts), "test_run"))
goto out;
if (!wait_for_callbacks(skel, 1))
goto out;
ASSERT_EQ(skel->bss->chain_err, 0, "chain_err");
wait_for_callbacks(skel, 2);
out:
call_rcu__destroy(skel);
}
/*
* A callback that keeps re-arming must stop once the map loses its last user
* reference, or it pins the program for good. Hold an independent fd on .bss
* so the refusal is still readable after the skeleton is gone.
*/
static void test_call_rcu_teardown(void)
{
LIBBPF_OPTS(bpf_test_run_opts, opts);
int i, err, fd = 0, bss_fd = -1;
struct bpf_prog_info pinfo = {};
struct bpf_map_info minfo = {};
__u32 len, prog_id, zero = 0;
struct call_rcu *skel;
char *buf = NULL;
size_t off, vsz;
skel = call_rcu__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
return;
len = sizeof(pinfo);
if (!ASSERT_OK(bpf_prog_get_info_by_fd(bpf_program__fd(skel->progs.arm), &pinfo, &len),
"prog_info"))
goto out;
prog_id = pinfo.id;
len = sizeof(minfo);
if (!ASSERT_OK(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.bss), &minfo, &len),
"bss_info"))
goto out;
bss_fd = bpf_map_get_fd_by_id(minfo.id);
if (!ASSERT_GE(bss_fd, 0, "bss_fd"))
goto out;
vsz = bpf_map__value_size(skel->maps.bss);
off = (char *)&skel->bss->chain_err - (char *)skel->bss;
buf = malloc(vsz);
if (!ASSERT_OK_PTR(buf, "buf"))
goto out;
skel->bss->chain = INT_MAX;
err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm), &opts);
if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(opts.retval, 0, "retval"))
goto out;
if (!ASSERT_EQ(skel->bss->arm_err, 0, "arm_err"))
goto out;
/* The chain has to be running before the map reference goes away. */
if (!wait_for_callbacks(skel, 2))
goto out;
call_rcu__destroy(skel);
skel = NULL;
for (i = 0; i < 3000; i++) {
fd = bpf_prog_get_fd_by_id(prog_id);
if (fd < 0)
break;
close(fd);
usleep(10000);
}
if (!ASSERT_EQ(fd, -ENOENT, "prog_freed"))
goto out;
if (ASSERT_OK(bpf_map_lookup_elem(bss_fd, &zero, buf), "bss_lookup"))
ASSERT_EQ(*(int *)(buf + off), -EPERM, "chain_refused");
out:
free(buf);
if (bss_fd >= 0)
close(bss_fd);
call_rcu__destroy(skel);
}
static void test_call_rcu_bad_map(void)
{
LIBBPF_OPTS(bpf_map_create_opts, opts);
struct call_rcu *skel;
int fd;
skel = call_rcu__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
return;
opts.btf_fd = bpf_object__btf_fd(skel->obj);
opts.btf_key_type_id = bpf_map__btf_key_type_id(skel->maps.arr);
opts.btf_value_type_id = bpf_map__btf_value_type_id(skel->maps.arr);
fd = bpf_map_create(BPF_MAP_TYPE_HASH, "rcu_hash", sizeof(__u32),
bpf_map__value_size(skel->maps.arr), 1, &opts);
ASSERT_EQ(fd, -EOPNOTSUPP, "hash_rejected");
if (fd >= 0)
close(fd);
call_rcu__destroy(skel);
}
/* Iterating would hand the program a writable pointer to the head. */
static void test_call_rcu_iter(void)
{
LIBBPF_OPTS(bpf_iter_attach_opts, opts);
union bpf_iter_link_info linfo = {};
struct bpf_link *link;
struct call_rcu *skel;
skel = call_rcu__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
return;
linfo.map.map_fd = bpf_map__fd(skel->maps.arr);
opts.link_info = &linfo;
opts.link_info_len = sizeof(linfo);
link = bpf_program__attach_iter(skel->progs.dump, &opts);
if (!ASSERT_ERR_PTR(link, "iter_rejected"))
bpf_link__destroy(link);
else
ASSERT_EQ(libbpf_get_error(link), -EOPNOTSUPP, "iter_errno");
call_rcu__destroy(skel);
}
static void test_call_rcu_inner_map(void)
{
LIBBPF_OPTS(bpf_map_create_opts, opts);
struct call_rcu *skel;
int fd;
skel = call_rcu__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
return;
opts.inner_map_fd = bpf_map__fd(skel->maps.arr);
fd = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "rcu_outer",
sizeof(__u32), sizeof(__u32), 1, &opts);
ASSERT_EQ(fd, -EOPNOTSUPP, "inner_map_rejected");
if (fd >= 0)
close(fd);
call_rcu__destroy(skel);
}
/* Two heads in the same map must be independent. */
static void test_call_rcu_two_heads(void)
{
LIBBPF_OPTS(bpf_test_run_opts, opts);
struct call_rcu *skel;
skel = call_rcu__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
return;
if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm_both), &opts),
"test_run"))
goto out;
if (!ASSERT_EQ(skel->bss->arm_err, 0, "arm_err"))
goto out;
if (!wait_for_callbacks(skel, 2))
goto out;
ASSERT_EQ(skel->bss->cb_keys, 0x3, "both_keys");
out:
call_rcu__destroy(skel);
}
void test_call_rcu(void)
{
if (test__start_subtest("run"))
test_call_rcu_run(false);
if (test__start_subtest("run_tasks_trace"))
test_call_rcu_run(true);
if (test__start_subtest("chain"))
test_call_rcu_chain();
if (test__start_subtest("two_heads"))
test_call_rcu_two_heads();
if (test__start_subtest("teardown"))
test_call_rcu_teardown();
if (test__start_subtest("bad_map"))
test_call_rcu_bad_map();
if (test__start_subtest("iter"))
test_call_rcu_iter();
if (test__start_subtest("inner_map"))
test_call_rcu_inner_map();
RUN_TESTS(call_rcu_fail);
}