// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* This test case verifies that nested TSC scaling behaves as expected when
* both L1 and L2 are scaled using different ratios.
*/
#include <linux/math64.h>
#include <time.h>
#include "kvm_util.h"
#include "vmx.h"
#include "svm_util.h"
#include "kselftest.h"
/* L2's TSC multiplier and offset, relative to L1. */
static u64 l2_multiplier;
static u64 l2_offset;
enum ucall_cmd { UCALL_SLEEP, UCALL_CHECK_L1, UCALL_CHECK_L2 };
/*
* This function checks whether the "actual" TSC frequency of a guest matches
* its expected frequency. In order to account for delays in taking the TSC
* measurements, a difference of 1% between the actual and the expected value
* is tolerated.
*/
static void host_check_tsc_freq(int level, u64 actual, u64 expected)
{
u64 tolerance, thresh_low, thresh_high;
tolerance = max(expected / 100, (u64)1);
thresh_low = expected - tolerance;
thresh_high = expected + tolerance;
TEST_ASSERT(thresh_low <= actual && thresh_high >= actual,
"L%u TSC freq is '%lu', expected to be between %lu and %lu",
level, actual, thresh_low, thresh_high);
}
static void guest_check_tsc_freq(enum ucall_cmd check_level)
{
u64 tsc_start, tsc_end, tsc_freq;
/*
* Reading the TSC twice with about a second's difference should give
* us an approximation of the TSC frequency from the guest's
* perspective. Now, this won't be completely accurate, but it should
* be good enough for the purposes of this test.
*/
tsc_start = rdmsr(MSR_IA32_TSC);
GUEST_SYNC2(UCALL_SLEEP, 1);
tsc_end = rdmsr(MSR_IA32_TSC);
tsc_freq = tsc_end - tsc_start;
GUEST_SYNC2(check_level, tsc_freq);
}
static void l2_guest_code(void)
{
guest_check_tsc_freq(UCALL_CHECK_L2);
/* exit to L1 */
__asm__ __volatile__("vmcall");
}
static void l1_svm_code(struct svm_test_data *svm)
{
/* check that L1's frequency looks alright before launching L2 */
guest_check_tsc_freq(UCALL_CHECK_L1);
generic_svm_setup(svm, l2_guest_code);
svm->vmcb->control.tsc_offset = l2_offset;
/* enable TSC scaling for L2 */
wrmsr(MSR_AMD64_TSC_RATIO, l2_multiplier);
/* launch L2 */
run_guest(svm->vmcb, svm->vmcb_gpa);
GUEST_ASSERT(svm->vmcb->control.exit_code == SVM_EXIT_VMMCALL);
/* check that L1's frequency still looks good */
guest_check_tsc_freq(UCALL_CHECK_L1);
GUEST_DONE();
}
static void l1_vmx_code(struct vmx_pages *vmx_pages)
{
u32 control;
/* check that L1's frequency looks alright before launching L2 */
guest_check_tsc_freq(UCALL_CHECK_L1);
prepare_for_vmx_operation(vmx_pages);
load_vmcs(vmx_pages);
/* prepare the VMCS for L2 execution */
prepare_vmcs(vmx_pages, l2_guest_code);
/* enable TSC offsetting and TSC scaling for L2 */
control = vmread(CPU_BASED_VM_EXEC_CONTROL);
control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_USE_TSC_OFFSETTING;
vmwrite(CPU_BASED_VM_EXEC_CONTROL, control);
control = vmread(SECONDARY_VM_EXEC_CONTROL);
control |= SECONDARY_EXEC_TSC_SCALING;
vmwrite(SECONDARY_VM_EXEC_CONTROL, control);
vmwrite(TSC_OFFSET, l2_offset);
vmwrite(TSC_MULTIPLIER, l2_multiplier);
/* launch L2 */
vmlaunch();
GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
/* check that L1's frequency still looks good */
guest_check_tsc_freq(UCALL_CHECK_L1);
GUEST_DONE();
}
static void l1_guest_code(void *data)
{
if (this_cpu_has(X86_FEATURE_VMX))
l1_vmx_code(data);
else
l1_svm_code(data);
}
static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_tsc_freq, u64 l2_tsc_freq,
u64 __l2_multiplier)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
gva_t guest_gva;
printf("Testing L0 freq = %lu, L1 freq = %lu, L2 freq = %lu, L2 mult = 0x%lx\n",
l0_tsc_freq, l1_tsc_freq, l2_tsc_freq, __l2_multiplier);
vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code);
l2_offset = kvm_random_u64(&kvm_rng);
sync_global_to_guest(vm, l2_offset);
l2_multiplier = __l2_multiplier;
sync_global_to_guest(vm, l2_multiplier);
if (kvm_cpu_has(X86_FEATURE_VMX))
vcpu_alloc_vmx(vm, &guest_gva);
else
vcpu_alloc_svm(vm, &guest_gva);
vcpu_args_set(vcpu, 1, guest_gva);
vcpu_ioctl(vcpu, KVM_SET_TSC_KHZ, (void *)(l1_tsc_freq / 1000));
for (;;) {
struct ucall uc;
vcpu_run(vcpu);
TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
switch (get_ucall(vcpu, &uc)) {
case UCALL_ABORT:
REPORT_GUEST_ASSERT(uc);
case UCALL_SYNC:
switch (uc.args[0]) {
case UCALL_SLEEP:
sleep(uc.args[1]);
break;
case UCALL_CHECK_L1:
printf("L1's observed TSC frequency: %lu\n", uc.args[1]);
host_check_tsc_freq(1, uc.args[1], l1_tsc_freq);
break;
case UCALL_CHECK_L2:
printf("L2's observed TSC frequency: %lu\n", uc.args[1]);
host_check_tsc_freq(2, uc.args[1], l2_tsc_freq);
break;
}
break;
case UCALL_DONE:
goto done;
default:
TEST_FAIL("Unknown ucall %lu", uc.cmd);
}
}
done:
kvm_vm_free(vm);
}
int main(int argc, char *argv[])
{
u64 min_freq, min_multiplier, max_freq, max_multiplier, l1_max_freq, l1_min_freq;
u64 l0_tsc_freq, tsc_start, tsc_end, l1_scale, l2_scale;
u8 frac_bits = kvm_cpu_has(X86_FEATURE_VMX) ? 48 : 32;
struct kvm_vm *vm;
TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) ||
kvm_cpu_has(X86_FEATURE_SVM));
TEST_REQUIRE(kvm_has_cap(KVM_CAP_TSC_CONTROL));
TEST_REQUIRE(sys_clocksource_is_based_on_tsc());
/*
* Create a dummy VM to get KVM's default TSC frequency. All CPUs that
* support TSC scaling should have a constant TSC, i.e. there's no need
* to calibrate the "real" TSC. But do sanity check that the observed
* TSC is within range of KVM's reported TSC frequency.
*/
vm = vm_create_barebones();
l0_tsc_freq = (u64)__vm_ioctl(vm, KVM_GET_TSC_KHZ, NULL) * 1000;
TEST_ASSERT(l0_tsc_freq, "vcpu ioctl KVM_GET_TSC_KHZ failed");
kvm_vm_free(vm);
printf("L0 TSC frequency is: %lu\n", l0_tsc_freq);
tsc_start = rdtsc();
sleep(1);
tsc_end = rdtsc();
host_check_tsc_freq(0, tsc_end - tsc_start, l0_tsc_freq);
/* Sanity check the frequency reported by KVM_GET_TSC_KHZ. */
test_tsc_scaling(l0_tsc_freq, l0_tsc_freq, l0_tsc_freq, BIT_ULL(frac_bits));
/*
* Scale L1 and L2 up and down at random factors from 2 to 10. Current
* CPUs have two full orders of magnitude of "breathing room" before an
* ultrafast L0 TSC frequency multiplied by 10x will encounter KVM's
* signed 32-bit limit on L1's frequency, and it's highly unlikely a
* CPU that supports TSC scaling will show up running at 100MHz, i.e.
* underflowing KVM's minimum 1KHz frequency is extremely unlikely.
*
* For L2, the frequency is limited only by what hardware can support,
* not by KVM's limits. SVM provides 8 bits of integer scale up, and
* 32 bits of fractional scale down, i.e. can scale up 255x and down a
* comical amount, so scaling up 100x and down 1/100 is well within
* hardware's capabilities (VMX provides 16 bits of "up" and 48 bits of
* "down").
*/
l1_scale = (kvm_random_u32(&kvm_rng) % 9) + 2;
l2_scale = (kvm_random_u32(&kvm_rng) % 9) + 2;
test_tsc_scaling(l0_tsc_freq, l0_tsc_freq / l1_scale,
mul_u64_u64_div64(l0_tsc_freq, l2_scale, l1_scale),
(l2_scale << frac_bits));
test_tsc_scaling(l0_tsc_freq, l0_tsc_freq * l1_scale,
mul_u64_u64_div64(l0_tsc_freq, l1_scale, l2_scale),
(1ull << frac_bits) / l2_scale);
test_tsc_scaling(l0_tsc_freq, l0_tsc_freq / l1_scale,
l0_tsc_freq / l1_scale / l2_scale,
(1ull << frac_bits) / l2_scale);
test_tsc_scaling(l0_tsc_freq, l0_tsc_freq * l1_scale,
l0_tsc_freq * l1_scale * l2_scale,
(l2_scale << frac_bits));
/*
* Test that KVM saturates L2's frequency on both ends if the resulting
* L2 TSC frequency would be below or above what hardware can support.
* Because L2 = L0 * (L1_mult >> frac) * (L2_mult >> frac) needs to be
* distilled down to a single multiplier, very small/large multipliers
* will underflow/overflow the minimum/maximum multiplier supported by
* hardware when L1 and L2 multipliers are combined. KVM's behavior is
* saturate on {under,over}flow, i.e. to run at the min/max frequency.
*
* Note, userspace can only program L1's frequency in KHz, i.e. can't
* specify an exact multiplier. As a result, the minimum and maximum
* frequencies are different for L1 vs L2, because L1 is constrained by
* hardware *and* KVM, whereas L2 is constrained only by hardware.
*/
min_multiplier = 1;
min_freq = mul_u64_u64_div64(l0_tsc_freq, min_multiplier, BIT_ULL(frac_bits));
min_freq = max(min_freq, (u64)1);
l1_min_freq = max(min_freq, (u64)1 * 1000);
test_tsc_scaling(l0_tsc_freq, l1_min_freq, min_freq, 1);
/*
* SVM takes a 40-bit value (right shifted by 32), while VMX takes a
* 64-bit value (right shifted by 48). mul_u64_u64_shr() isn't (yet)
* available in selftests, but mul_u64_u64_div64() does nicely since,
* albeit more slowly (performance is obviously not a concern). Note,
* because KVM_GET_TSC_KHZ returns a signed 32-bit integer, KVM limits
* KVM_SET_TSC_KHZ to INT_MAX, even though hardware (both SVM and VMX)
* supports much higher frequencies.
*/
max_multiplier = kvm_cpu_has(X86_FEATURE_VMX) ? -1ull : GENMASK_U64(39, 0);
max_freq = mul_u64_u64_div64(l0_tsc_freq, max_multiplier, BIT_ULL(frac_bits));
l1_max_freq = min(max_freq, (u64)INT32_MAX * 1000);
test_tsc_scaling(l0_tsc_freq, l1_max_freq, max_freq, max_multiplier);
return 0;
}