// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2024, Google LLC.
*/
#include <pthread.h>
#include <sys/mman.h>
#include <time.h>
#include <unistd.h>
#include <linux/align.h>
#include <linux/kvm.h>
#include <linux/sizes.h>
#include "kvm_util.h"
#include "kvm_syscalls.h"
#include "kselftest_harness.h"
#include "test_util.h"
#include "ucall_common.h"
FIXTURE(gmem_conversions) {
struct kvm_vcpu *vcpu;
int gmem_fd;
/* HVA of the first byte of the memory mmap()-ed from gmem_fd. */
char *mem;
};
typedef FIXTURE_DATA(gmem_conversions) test_data_t;
FIXTURE_SETUP(gmem_conversions) { }
static size_t page_size;
static void guest_do_rmw(void);
#define GUEST_MEMFD_SHARING_TEST_GVA 0x90000000ULL
/*
* Defer setup until the individual test is invoked so that tests can specify
* the number of pages and flags for the guest_memfd instance.
*/
static void gmem_conversions_do_setup(test_data_t *t, int nr_pages,
int gmem_flags)
{
const struct vm_shape shape = {
.mode = VM_MODE_DEFAULT,
.type = KVM_X86_SW_PROTECTED_VM,
};
/*
* Use high GPA above APIC_DEFAULT_PHYS_BASE to avoid clashing with
* APIC_DEFAULT_PHYS_BASE.
*/
const gpa_t gpa = SZ_4G;
const u32 slot = 1;
struct kvm_vm *vm;
vm = __vm_create_shape_with_one_vcpu(shape, &t->vcpu, nr_pages, guest_do_rmw);
vm_mem_add(vm, VM_MEM_SRC_SHMEM, gpa, slot, nr_pages,
KVM_MEM_GUEST_MEMFD, -1, 0, gmem_flags);
t->gmem_fd = kvm_slot_to_fd(vm, slot);
t->mem = addr_gpa2hva(vm, gpa);
virt_map(vm, GUEST_MEMFD_SHARING_TEST_GVA, gpa, nr_pages);
}
static void gmem_conversions_do_teardown(test_data_t *t)
{
/* Use NULL to avoid second free in FIXTURE_TEARDOWN (multipage tests). */
if (!t->vcpu)
return;
/* No need to close gmem_fd, it's owned by the VM structure. */
kvm_vm_free(t->vcpu->vm);
t->vcpu = NULL;
}
FIXTURE_TEARDOWN(gmem_conversions)
{
gmem_conversions_do_teardown(self);
}
/*
* In these test definition macros, __nr_pages and nr_pages is used to set up
* the total number of pages in the guest_memfd under test. This will be
* available in the test definitions as nr_pages.
*/
#define __GMEM_CONVERSION_TEST(test, __nr_pages, flags) \
static void __gmem_conversions_##test(test_data_t *t, int nr_pages); \
\
TEST_F(gmem_conversions, test) \
{ \
const int nr = (__nr_pages); \
\
gmem_conversions_do_setup(self, nr, flags); \
__gmem_conversions_##test(self, nr); \
} \
static void __gmem_conversions_##test(test_data_t *t, int nr_pages) \
#define GMEM_CONVERSION_TEST(test, __nr_pages, flags) \
__GMEM_CONVERSION_TEST(test, __nr_pages, (flags) | GUEST_MEMFD_FLAG_MMAP)
#define __GMEM_CONVERSION_TEST_INIT_PRIVATE(test, __nr_pages) \
GMEM_CONVERSION_TEST(test, __nr_pages, 0)
#define GMEM_CONVERSION_TEST_INIT_PRIVATE(test) \
__GMEM_CONVERSION_TEST_INIT_PRIVATE(test, 1)
#define __GMEM_CONVERSION_TEST_INIT_SHARED(test, __nr_pages) \
GMEM_CONVERSION_TEST(test, __nr_pages, GUEST_MEMFD_FLAG_INIT_SHARED)
#define GMEM_CONVERSION_TEST_INIT_SHARED(test) \
__GMEM_CONVERSION_TEST_INIT_SHARED(test, 1)
/*
* Repeats test over nr_pages in a guest_memfd of size nr_pages, providing each
* test iteration with test_page, the index of the page under test in
* guest_memfd. test_page takes values 0..(nr_pages - 1) inclusive.
*/
#define GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(test, __nr_pages) \
static void __gmem_conversions_multipage_##test(test_data_t *t, int nr_pages, \
const int test_page); \
\
TEST_F(gmem_conversions, test) \
{ \
const u64 flags = GUEST_MEMFD_FLAG_MMAP | GUEST_MEMFD_FLAG_INIT_SHARED; \
const int nr = (__nr_pages); \
int i; \
\
for (i = 0; i < nr; ++i) { \
gmem_conversions_do_setup(self, nr, flags); \
__gmem_conversions_multipage_##test(self, nr, i); \
gmem_conversions_do_teardown(self); \
} \
} \
static void __gmem_conversions_multipage_##test(test_data_t *t, int nr_pages, \
const int test_page)
struct guest_check_data {
void *mem;
char expected_val;
char write_val;
};
static struct guest_check_data guest_data;
static void guest_do_rmw(void)
{
for (;;) {
char *mem = READ_ONCE(guest_data.mem);
GUEST_ASSERT_EQ(READ_ONCE(*mem), READ_ONCE(guest_data.expected_val));
WRITE_ONCE(*mem, READ_ONCE(guest_data.write_val));
GUEST_SYNC(0);
}
}
static void run_guest_do_rmw(struct kvm_vcpu *vcpu, u64 pgoff,
char expected_val, char write_val)
{
struct ucall uc;
int r;
guest_data.mem = (void *)GUEST_MEMFD_SHARING_TEST_GVA + pgoff * page_size;
guest_data.expected_val = expected_val;
guest_data.write_val = write_val;
sync_global_to_guest(vcpu->vm, guest_data);
do {
r = __vcpu_run(vcpu);
} while (r == -1 && errno == EINTR);
TEST_ASSERT_EQ(r, 0);
switch (get_ucall(vcpu, &uc)) {
case UCALL_ABORT:
REPORT_GUEST_ASSERT(uc);
case UCALL_SYNC:
break;
default:
TEST_FAIL("Unexpected ucall %lu", uc.cmd);
}
}
static void host_do_rmw(char *mem, u64 pgoff, char expected_val,
char write_val)
{
TEST_ASSERT_EQ(READ_ONCE(mem[pgoff * page_size]), expected_val);
WRITE_ONCE(mem[pgoff * page_size], write_val);
}
static void test_private(test_data_t *t, u64 pgoff, char starting_val,
char write_val)
{
TEST_EXPECT_SIGBUS(WRITE_ONCE(t->mem[pgoff * page_size], write_val));
run_guest_do_rmw(t->vcpu, pgoff, starting_val, write_val);
TEST_EXPECT_SIGBUS(READ_ONCE(t->mem[pgoff * page_size]));
}
static void test_convert_to_private(test_data_t *t, u64 pgoff,
char starting_val, char write_val)
{
gmem_set_private(t->gmem_fd, pgoff * page_size, page_size);
test_private(t, pgoff, starting_val, write_val);
}
static void test_shared(test_data_t *t, u64 pgoff, char starting_val,
char host_write_val, char write_val)
{
host_do_rmw(t->mem, pgoff, starting_val, host_write_val);
run_guest_do_rmw(t->vcpu, pgoff, host_write_val, write_val);
TEST_ASSERT_EQ(READ_ONCE(t->mem[pgoff * page_size]), write_val);
}
static void test_convert_to_shared(test_data_t *t, u64 pgoff,
char starting_val, char host_write_val,
char write_val)
{
gmem_set_shared(t->gmem_fd, pgoff * page_size, page_size);
test_shared(t, pgoff, starting_val, host_write_val, write_val);
}
GMEM_CONVERSION_TEST_INIT_PRIVATE(init_private)
{
test_private(t, 0, 0, 'A');
test_convert_to_shared(t, 0, 'A', 'B', 'C');
test_convert_to_private(t, 0, 'C', 'E');
}
GMEM_CONVERSION_TEST_INIT_SHARED(init_shared)
{
test_shared(t, 0, 0, 'A', 'B');
test_convert_to_private(t, 0, 'B', 'C');
test_convert_to_shared(t, 0, 'C', 'D', 'E');
}
GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(indexing, 4)
{
int i;
/* Get a char that varies with both i and n. */
#define combine(x, n) (((x) << 4) + (n))
#define i_(n) (combine(i, n))
#define t_(n) (combine(test_page, n))
/*
* Start with the highest index, to catch any errors when, perhaps, the
* first page is returned even for the last index.
*/
for (i = nr_pages - 1; i >= 0; --i)
test_shared(t, i, 0, i_(0), i_(2));
test_convert_to_private(t, test_page, t_(2), t_(3));
for (i = 0; i < nr_pages; ++i) {
if (i == test_page)
test_private(t, test_page, t_(3), t_(4));
else
test_shared(t, i, i_(2), i_(3), i_(4));
}
test_convert_to_shared(t, test_page, t_(4), t_(5), t_(6));
for (i = 0; i < nr_pages; ++i) {
char expected = i == test_page ? t_(6) : i_(4);
test_shared(t, i, expected, i_(7), i_(8));
}
#undef t_
#undef i_
#undef combine
}
/*
* Test that even if there are no folios yet, conversion requests are recorded
* in guest_memfd.
*/
GMEM_CONVERSION_TEST_INIT_SHARED(before_allocation_shared)
{
test_convert_to_private(t, 0, 0, 'A');
}
GMEM_CONVERSION_TEST_INIT_PRIVATE(before_allocation_private)
{
test_convert_to_shared(t, 0, 0, 'A', 'B');
}
/*
* Test that when some of the folios in the conversion range are allocated,
* conversion requests are handled correctly in guest_memfd. Vary the ranges
* allocated before conversion, using test_page, to cover various layouts of
* contiguous allocated and unallocated regions.
*/
GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(unallocated_folios, 8)
{
const int second_page_to_fault = 4;
int i;
/*
* Fault 2 of the pages to test filemap range operations except when
* test_page == second_page_to_fault.
*/
host_do_rmw(t->mem, test_page, 0, 'A');
if (test_page != second_page_to_fault)
host_do_rmw(t->mem, second_page_to_fault, 0, 'A');
gmem_set_private(t->gmem_fd, 0, nr_pages * page_size);
for (i = 0; i < nr_pages; ++i) {
char expected = (i == test_page || i == second_page_to_fault) ? 'A' : 0;
test_private(t, i, expected, 'B');
}
for (i = 0; i < nr_pages; ++i)
test_convert_to_shared(t, i, 'B', 'C', 'D');
}
/* Truncation should not affect shared/private status. */
GMEM_CONVERSION_TEST_INIT_SHARED(truncate)
{
host_do_rmw(t->mem, 0, 0, 'A');
kvm_fallocate(t->gmem_fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, page_size);
host_do_rmw(t->mem, 0, 0, 'A');
test_convert_to_private(t, 0, 'A', 'B');
kvm_fallocate(t->gmem_fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, page_size);
test_private(t, 0, 0, 'A');
}
/* Test that shared/private memory protections work and are seen from any process. */
GMEM_CONVERSION_TEST_INIT_SHARED(forked_accesses)
{
enum test_state {
STATE_INIT,
STATE_CHECK_SHARED,
STATE_DONE_CHECKING_SHARED,
STATE_CHECK_PRIVATE,
STATE_DONE_CHECKING_PRIVATE,
};
struct sync_state {
pthread_mutex_t mutex;
pthread_cond_t cond;
enum test_state step;
} *sync;
pthread_mutexattr_t mattr;
pthread_condattr_t cattr;
pid_t child_pid, parent_pid;
int status;
sync = kvm_mmap(sizeof(*sync), PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1);
pthread_mutexattr_init(&mattr);
pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED);
pthread_mutex_init(&sync->mutex, &mattr);
pthread_mutexattr_destroy(&mattr);
pthread_condattr_init(&cattr);
pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED);
pthread_cond_init(&sync->cond, &cattr);
pthread_condattr_destroy(&cattr);
sync->step = STATE_INIT;
#define TEST_STATE_AWAIT(__state) \
do { \
pthread_mutex_lock(&sync->mutex); \
while (sync->step != (__state)) { \
struct timespec ts, stop; \
int ret; \
\
clock_gettime(CLOCK_REALTIME, &ts); \
stop = timespec_add_ns(ts, 100 * 1000000UL); \
\
ret = pthread_cond_timedwait(&sync->cond, &sync->mutex, &stop); \
if (ret == ETIMEDOUT) { \
bool alive = (child_pid == 0) ? \
(getppid() == parent_pid) : \
(waitpid(child_pid, NULL, WNOHANG) == 0); \
TEST_ASSERT(alive, "Other process exited prematurely"); \
} else { \
TEST_ASSERT(!ret, "pthread_cond_timedwait failed"); \
} \
} \
pthread_mutex_unlock(&sync->mutex); \
} while (0)
#define TEST_STATE_SET(__state) \
do { \
pthread_mutex_lock(&sync->mutex); \
sync->step = (__state); \
pthread_cond_broadcast(&sync->cond); \
pthread_mutex_unlock(&sync->mutex); \
} while (0)
parent_pid = getpid();
child_pid = fork();
TEST_ASSERT(child_pid != -1, "fork failed");
if (child_pid == 0) {
const char inconsequential = 0xdd;
TEST_STATE_AWAIT(STATE_CHECK_SHARED);
/*
* This maps the pages into the child process as well, and tests
* that the conversion process will unmap the guest_memfd memory
* from all processes.
*/
host_do_rmw(t->mem, 0, 0xB, 0xC);
TEST_STATE_SET(STATE_DONE_CHECKING_SHARED);
TEST_STATE_AWAIT(STATE_CHECK_PRIVATE);
TEST_EXPECT_SIGBUS(READ_ONCE(t->mem[0]));
TEST_EXPECT_SIGBUS(WRITE_ONCE(t->mem[0], inconsequential));
TEST_STATE_SET(STATE_DONE_CHECKING_PRIVATE);
exit(0);
}
test_shared(t, 0, 0, 0xA, 0xB);
TEST_STATE_SET(STATE_CHECK_SHARED);
TEST_STATE_AWAIT(STATE_DONE_CHECKING_SHARED);
test_convert_to_private(t, 0, 0xC, 0xD);
TEST_STATE_SET(STATE_CHECK_PRIVATE);
TEST_STATE_AWAIT(STATE_DONE_CHECKING_PRIVATE);
TEST_ASSERT_EQ(waitpid(child_pid, &status, 0), child_pid);
TEST_ASSERT(WIFEXITED(status) && WEXITSTATUS(status) == 0,
"Child exited with unexpected status");
pthread_mutex_destroy(&sync->mutex);
pthread_cond_destroy(&sync->cond);
kvm_munmap(sync, sizeof(*sync));
#undef TEST_STATE_SET
#undef TEST_STATE_AWAIT
}
static void test_convert_to_private_fails(test_data_t *t, u64 pgoff,
size_t nr_pages,
u64 expected_error_offset)
{
/* +1 to make it anything but expected_error_offset. */
u64 error_offset = expected_error_offset + 1;
u64 offset = pgoff * page_size;
int ret;
do {
ret = __gmem_set_private(t->gmem_fd, offset,
nr_pages * page_size, &error_offset);
} while (ret == -1 && errno == EINTR);
TEST_ASSERT(ret == -1 && errno == EAGAIN,
"Wanted EAGAIN on page %lu, got %d (ret = %d)", pgoff,
errno, ret);
TEST_ASSERT_EQ(error_offset, expected_error_offset);
}
GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(elevated_refcount, 4)
{
int i;
pin_pages(t->mem + test_page * page_size, page_size);
for (i = 0; i < nr_pages; i++)
test_shared(t, i, 0, 'A', 'B');
/*
* Converting in bulk should fail as long any page in the range has
* unexpected refcounts.
*/
test_convert_to_private_fails(t, 0, nr_pages, test_page * page_size);
for (i = 0; i < nr_pages; i++) {
/*
* Converting page-wise should also fail as long any page in the
* range has unexpected refcounts.
*/
if (i == test_page)
test_convert_to_private_fails(t, i, 1, test_page * page_size);
else
test_convert_to_private(t, i, 'B', 'C');
}
unpin_pages();
gmem_set_private(t->gmem_fd, 0, nr_pages * page_size);
for (i = 0; i < nr_pages; i++) {
char expected = i == test_page ? 'B' : 'C';
test_private(t, i, expected, 'D');
}
}
int main(int argc, char *argv[])
{
TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM));
TEST_REQUIRE(kvm_check_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES) &
KVM_MEMORY_ATTRIBUTE_PRIVATE);
page_size = getpagesize();
return test_harness_run(argc, argv);
}