// SPDX-License-Identifier: GPL-2.0
#define BPF_NO_KFUNC_PROTOTYPES
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include "bpf_experimental.h"
#include <bpf_arena_common.h>
const volatile __u64 old_marker = 0x1111222233334444ULL;
const volatile __u64 new_marker = 0x5555666677778888ULL;
struct {
__uint(type, BPF_MAP_TYPE_ARENA);
__uint(map_flags, BPF_F_MMAPABLE);
__uint(max_entries, 1024);
#ifdef __TARGET_ARCH_arm64
__ulong(map_extra, 0x1ull << 32);
#else
__ulong(map_extra, 0x1ull << 44);
#endif
} arena SEC(".maps");
bool skip;
int page_idx;
void __arena *ptr;
void __arena *realloc_ptr;
bool realloc_after_free;
__u64 current_marker;
__u64 marker_seq;
int target_tid;
int pause_on_zap;
int zap_entered;
int release;
int timed_out;
__u64 trigger_pid_tgid;
long trigger_syscall;
int deferred_free;
int worker_armed;
int worker_exited;
static __always_inline void wait_for_release(void)
{
while (!*(volatile int *)&release && can_loop)
;
if (!*(volatile int *)&release)
timed_out = 1;
}
SEC("syscall")
int alloc_old(void *ctx)
{
#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM)
__u64 __arena *p;
char __arena *base = arena_base(&arena);
realloc_ptr = NULL;
ptr = bpf_arena_alloc_pages(&arena, base + (page_idx + 1) * __PAGE_SIZE, 1,
NUMA_NO_NODE, 0);
if (!ptr)
return 1;
p = (__u64 __arena *)ptr;
*p = old_marker;
current_marker = old_marker;
#else
skip = true;
#endif
return 0;
}
SEC("syscall")
int free_page(void *ctx)
{
#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM)
__u64 __arena *p;
__u64 marker;
if (!ptr)
return 1;
bpf_arena_free_pages(&arena, ptr, 1);
if (!realloc_after_free)
return 0;
marker = new_marker + ++marker_seq;
realloc_ptr = bpf_arena_alloc_pages(&arena, ptr, 1, NUMA_NO_NODE, 0);
if (!realloc_ptr)
return 0;
ptr = realloc_ptr;
p = (__u64 __arena *)ptr;
*p = marker;
current_marker = marker;
#endif
return 0;
}
SEC("syscall")
int try_realloc(void *ctx)
{
#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM)
__u64 __arena *p;
realloc_ptr = bpf_arena_alloc_pages(&arena, ptr, 1, NUMA_NO_NODE, 0);
if (realloc_ptr) {
ptr = realloc_ptr;
p = (__u64 __arena *)realloc_ptr;
*p = new_marker;
current_marker = new_marker;
}
#endif
return 0;
}
SEC("tp_btf/sys_enter")
int BPF_PROG(deferred_free_prog, struct pt_regs *regs, long id)
{
#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM)
if (!deferred_free || bpf_get_current_pid_tgid() != trigger_pid_tgid ||
id != trigger_syscall)
return 0;
deferred_free = 0;
/* The worker can run on another CPU before the kfunc returns. */
worker_armed = 1;
bpf_arena_free_pages(&arena, ptr, 1);
#endif
return 0;
}
SEC("fentry/arena_free_worker")
int BPF_PROG(trace_free_worker, struct work_struct *work)
{
if (worker_armed && !target_tid)
target_tid = (__u32)bpf_get_current_pid_tgid();
return 0;
}
SEC("fexit/arena_free_worker")
int BPF_PROG(trace_free_worker_ret, struct work_struct *work)
{
if ((__u32)bpf_get_current_pid_tgid() == target_tid)
worker_exited = 1;
return 0;
}
/*
* Pause a free while it zaps user mappings: after it cleared the kernel PTEs
* and flushed the TLB, but before it releases the range. zap_vma_range() is
* generic MM code, so it is an fentry target even where
* flush_tlb_kernel_range() is inline (arm64).
*/
SEC("?fentry/zap_vma_range")
int BPF_PROG(trace_zap, struct vm_area_struct *vma, unsigned long address,
unsigned long size)
{
if ((__u32)bpf_get_current_pid_tgid() != target_tid)
return 0;
zap_entered = 1;
if (pause_on_zap)
wait_for_release();
return 0;
}
char _license[] SEC("license") = "GPL";