// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2025 Google LLC
* Author: Vincent Donnefort <vdonnefort@google.com>
*/
#include <nvhe/alloc.h>
#include <nvhe/clock.h>
#include <nvhe/mem_protect.h>
#include <nvhe/mm.h>
#include <nvhe/trace.h>
#include <asm/percpu.h>
#include <asm/kvm_mmu.h>
#include <asm/local.h>
#include "simple_ring_buffer.c"
static DEFINE_PER_CPU(struct simple_rb_per_cpu, __simple_rbs);
static struct hyp_trace_buffer {
struct simple_rb_per_cpu __percpu *simple_rbs;
void *bpages_backing_start;
size_t bpages_backing_size;
hyp_spinlock_t lock;
} trace_buffer = {
.simple_rbs = &__simple_rbs,
.lock = __HYP_SPIN_LOCK_UNLOCKED,
};
static bool hyp_trace_buffer_loaded(struct hyp_trace_buffer *trace_buffer)
{
return trace_buffer->bpages_backing_size > 0;
}
void *tracing_reserve_entry(unsigned long length)
{
return simple_ring_buffer_reserve(this_cpu_ptr(trace_buffer.simple_rbs), length,
trace_hyp_clock());
}
void tracing_commit_entry(void)
{
simple_ring_buffer_commit(this_cpu_ptr(trace_buffer.simple_rbs));
}
static int __admit_host_mem(void *start, u64 size)
{
if (!PAGE_ALIGNED(start) || !PAGE_ALIGNED(size) || !size)
return -EINVAL;
if (!is_protected_kvm_enabled())
return 0;
return __pkvm_host_donate_hyp(hyp_virt_to_pfn(start), size >> PAGE_SHIFT);
}
static void __release_host_mem(void *start, u64 size)
{
if (!is_protected_kvm_enabled())
return;
WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(start), size >> PAGE_SHIFT));
}
static int hyp_trace_buffer_alloc_bpages(struct hyp_trace_buffer *trace_buffer,
struct hyp_trace_desc *desc)
{
void *start;
size_t size;
int ret;
if (is_protected_kvm_enabled()) {
struct ring_buffer_desc *rb_desc;
int cpu;
size = 0;
for_each_ring_buffer_desc(rb_desc, cpu, &desc->trace_buffer_desc)
size += rb_desc->nr_page_va * sizeof(struct simple_buffer_page);
start = hyp_alloc(size);
if (!start)
return hyp_alloc_errno();
} else {
start = (void *)kern_hyp_va(desc->bpages_backing_start);
size = desc->bpages_backing_size;
ret = __admit_host_mem(start, size);
if (ret)
return ret;
memset(start, 0, size);
}
trace_buffer->bpages_backing_start = start;
trace_buffer->bpages_backing_size = size;
return 0;
}
static void hyp_trace_buffer_free_bpages(struct hyp_trace_buffer *trace_buffer)
{
void *start = trace_buffer->bpages_backing_start;
size_t size = trace_buffer->bpages_backing_size;
if (!size)
return;
if (is_protected_kvm_enabled()) {
hyp_free(start);
} else {
memset(start, 0, size);
__release_host_mem(start, size);
}
trace_buffer->bpages_backing_start = NULL;
trace_buffer->bpages_backing_size = 0;
}
static void *__pin_shared_page(unsigned long kern_va)
{
void *va = kern_hyp_va((void *)kern_va);
if (!is_protected_kvm_enabled())
return va;
return hyp_pin_shared_mem(va, va + PAGE_SIZE) ? NULL : va;
}
static void __unpin_shared_page(void *va)
{
if (!is_protected_kvm_enabled())
return;
hyp_unpin_shared_mem(va, va + PAGE_SIZE);
}
static void hyp_trace_buffer_unload(struct hyp_trace_buffer *trace_buffer)
{
int cpu;
hyp_assert_lock_held(&trace_buffer->lock);
if (!hyp_trace_buffer_loaded(trace_buffer))
return;
for (cpu = 0; cpu < hyp_nr_cpus; cpu++)
simple_ring_buffer_unload_mm(per_cpu_ptr(trace_buffer->simple_rbs, cpu),
__unpin_shared_page);
hyp_trace_buffer_free_bpages(trace_buffer);
}
static int hyp_trace_buffer_load(struct hyp_trace_buffer *trace_buffer,
struct hyp_trace_desc *desc)
{
struct simple_buffer_page *bpages;
struct ring_buffer_desc *rb_desc;
int ret, cpu;
hyp_assert_lock_held(&trace_buffer->lock);
if (hyp_trace_buffer_loaded(trace_buffer))
return -EINVAL;
ret = hyp_trace_buffer_alloc_bpages(trace_buffer, desc);
if (ret)
return ret;
bpages = trace_buffer->bpages_backing_start;
for_each_ring_buffer_desc(rb_desc, cpu, &desc->trace_buffer_desc) {
ret = simple_ring_buffer_init_mm(per_cpu_ptr(trace_buffer->simple_rbs, cpu),
bpages, rb_desc, __pin_shared_page,
__unpin_shared_page);
if (ret)
break;
bpages += rb_desc->nr_page_va;
}
if (ret)
hyp_trace_buffer_unload(trace_buffer);
return ret;
}
static bool hyp_trace_desc_is_valid(struct hyp_trace_desc *desc, size_t desc_size)
{
struct ring_buffer_desc *rb_desc;
unsigned int cpu;
void *desc_end;
if (!is_protected_kvm_enabled())
return true;
/*
* desc_size is an untrusted host-provided value. We rely on
* __pkvm_host_donate_hyp() to enforce its validity.
*/
desc_end = (void *)desc + desc_size;
if (desc->trace_buffer_desc.nr_cpus != hyp_nr_cpus)
return false;
for_each_ring_buffer_desc(rb_desc, cpu, &desc->trace_buffer_desc) {
/* Can we read nr_page_va? */
if ((void *)rb_desc + struct_size(rb_desc, page_va, 0) > desc_end)
return false;
/* simple_ring_buffer_init_mm() expects at least 3 pages */
if (rb_desc->nr_page_va < 3)
return false;
/* Overflow desc? */
if ((void *)rb_desc + struct_size(rb_desc, page_va, rb_desc->nr_page_va) > desc_end)
return false;
if (cpu != rb_desc->cpu)
return false;
}
return true;
}
int __tracing_load(void __kern *desc_hva, size_t desc_size)
{
struct hyp_trace_desc *desc = kern_hyp_va_host(desc_hva);
int ret;
ret = __admit_host_mem(desc, desc_size);
if (ret)
return ret;
if (!hyp_trace_desc_is_valid(desc, desc_size)) {
ret = -EINVAL;
goto err_release_desc;
}
hyp_spin_lock(&trace_buffer.lock);
ret = hyp_trace_buffer_load(&trace_buffer, desc);
hyp_spin_unlock(&trace_buffer.lock);
err_release_desc:
__release_host_mem(desc, desc_size);
return ret;
}
void __tracing_unload(void)
{
hyp_spin_lock(&trace_buffer.lock);
hyp_trace_buffer_unload(&trace_buffer);
hyp_spin_unlock(&trace_buffer.lock);
}
int __tracing_enable(bool enable)
{
int cpu, ret = enable ? -EINVAL : 0;
hyp_spin_lock(&trace_buffer.lock);
if (!hyp_trace_buffer_loaded(&trace_buffer))
goto unlock;
for (cpu = 0; cpu < hyp_nr_cpus; cpu++)
simple_ring_buffer_enable_tracing(per_cpu_ptr(trace_buffer.simple_rbs, cpu),
enable);
ret = 0;
unlock:
hyp_spin_unlock(&trace_buffer.lock);
return ret;
}
int __tracing_swap_reader(unsigned int cpu)
{
int ret = -ENODEV;
if (cpu >= hyp_nr_cpus)
return -EINVAL;
hyp_spin_lock(&trace_buffer.lock);
if (hyp_trace_buffer_loaded(&trace_buffer))
ret = simple_ring_buffer_swap_reader_page(
per_cpu_ptr(trace_buffer.simple_rbs, cpu));
hyp_spin_unlock(&trace_buffer.lock);
return ret;
}
void __tracing_update_clock(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc)
{
int cpu;
/* After this loop, all CPUs are observing the new bank... */
for (cpu = 0; cpu < hyp_nr_cpus; cpu++) {
struct simple_rb_per_cpu *simple_rb = per_cpu_ptr(trace_buffer.simple_rbs, cpu);
while (READ_ONCE(simple_rb->status) == SIMPLE_RB_WRITING)
;
}
/* ...we can now override the old one and swap. */
trace_hyp_clock_update(mult, shift, epoch_ns, epoch_cyc);
}
int __tracing_reset(unsigned int cpu)
{
int ret = -ENODEV;
if (cpu >= hyp_nr_cpus)
return -EINVAL;
hyp_spin_lock(&trace_buffer.lock);
if (hyp_trace_buffer_loaded(&trace_buffer))
ret = simple_ring_buffer_reset(per_cpu_ptr(trace_buffer.simple_rbs, cpu));
hyp_spin_unlock(&trace_buffer.lock);
return ret;
}