// SPDX-License-Identifier: GPL-2.0
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/miscdevice.h>
#include <linux/kvm.h>
#include <linux/kvm_types.h>
#include <linux/kvm_host.h>
#include <linux/cleanup.h>
#include <linux/fpu.h>
#include <asm/access-regs.h>
#include <arm64/kvm_emulate.h>
#include <arm64/sysreg.h>
#include <gmap.h>
#include <kvm_mmu.h>
#include "arm.h"
#include "handle_exit.h"
#define CREATE_TRACE_POINTS
#include "trace.h"
static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu);
#define __INCL_GEN_ARM_FILE
#include "generated/arm.inc"
#undef __INCL_GEN_ARM_FILE
#define KVM_PHYS_SHIFT (40)
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
{
int ret;
switch (ext) {
case KVM_CAP_NR_VCPUS:
case KVM_CAP_MAX_VCPUS:
case KVM_CAP_MAX_VCPU_ID:
ret = KVM_MAX_VCPUS;
break;
case KVM_CAP_ARM_VM_IPA_SIZE:
ret = get_kvm_ipa_limit();
break;
case KVM_CAP_IOEVENTFD:
case KVM_CAP_PRE_FAULT_MEMORY:
ret = 1;
break;
default:
ret = 0;
}
return ret;
}
static u64 kvm_max_guest_address(void)
{
u64 max_addr;
max_addr = min_t(u64, TASK_SIZE_MAX, sclp.hamax);
max_addr = max_t(u64, max_addr, SZ_1G - 1);
return ALIGN_DOWN(max_addr + 1, SZ_1G) - 1;
}
static int kvm_gmap_init(struct kvm *kvm)
{
struct crst_table *table;
kvm->arch.gmap = gmap_new(kvm, gpa_to_gfn(kvm->arch.guest_phys_size));
if (!kvm->arch.gmap)
return -ENOMEM;
/* arm64 (on s390) do not have pfault */
clear_bit(GMAP_FLAG_PFAULT_ENABLED, &kvm->arch.gmap->flags);
set_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags);
table = dereference_asce(kvm->arch.gmap->asce);
crst_table_init((void *)table, _CRSTE_HOLE(table->crstes[0].h.tt).val);
return 0;
}
static int kvm_vm_type_ipa_size_shift(unsigned long type)
{
int phys_shift;
phys_shift = KVM_VM_TYPE_ARM_IPA_SIZE(type);
if (phys_shift) {
if (phys_shift > get_kvm_ipa_limit() ||
phys_shift < ARM64_MIN_PARANGE_BITS)
return -EINVAL;
} else {
phys_shift = KVM_PHYS_SHIFT;
if (phys_shift > get_kvm_ipa_limit()) {
pr_warn_once("%s using unsupported default IPA limit\n",
current->comm);
return -EINVAL;
}
}
return phys_shift;
}
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
{
char debug_name[32];
int ret;
if (type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK)
return -EINVAL;
ret = kvm_vm_type_ipa_size_shift(type);
if (ret < 0)
return ret;
kvm->arch.guest_phys_size = 1UL << ret;
mutex_init(&kvm->arch.config_lock);
bitmap_zero(kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES);
snprintf(debug_name, sizeof(debug_name), "kvm-arm64-%u-%u",
current->pid, get_random_u32());
kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
if (!kvm->arch.dbf)
return -ENOMEM;
debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
ret = kvm_gmap_init(kvm);
if (ret)
goto out_err;
kvm->arch.mem_limit = kvm->arch.guest_phys_size;
VM_EVENT(kvm, 3, "vm created with type %lu", type);
return 0;
out_err:
debug_unregister(kvm->arch.dbf);
return ret;
}
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
{
return VM_FAULT_SIGBUS;
}
long kvm_arch_dev_ioctl(struct file *filp,
unsigned int ioctl, unsigned long arg)
{
return -EINVAL;
}
void kvm_arch_destroy_vm(struct kvm *kvm)
{
kvm_destroy_vcpus(kvm);
debug_unregister(kvm->arch.dbf);
kvm->arch.gmap = gmap_put(kvm->arch.gmap);
}
u32 get_kvm_ipa_limit(void)
{
return fls64(kvm_max_guest_address() + 1) - 1;
}
int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
{
return 0;
}
int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
{
struct kvm_sae_block *sae_block = &vcpu->arch.sae_block;
struct kvm_sae_save_area *save_area = &vcpu->arch.save_area;
spin_lock_init(&vcpu->arch.mp_state_lock);
/* Force users to call KVM_ARM_VCPU_INIT */
vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
vcpu->arch.mc = kvm_s390_new_mmu_cache();
if (!vcpu->arch.mc)
return -ENOMEM;
save_area->sdo = virt_to_phys(sae_block);
sae_block->hbasce = vcpu->kvm->arch.gmap->asce.val;
sae_block->mso = 0L;
sae_block->msl = vcpu->kvm->arch.mem_limit - 1;
sae_block->save_area = virt_to_phys(save_area);
VM_EVENT(vcpu->kvm, 3,
"create cpu %d at 0x%p, sae block at 0x%p, satellite at 0x%p",
vcpu->vcpu_id, vcpu, &vcpu->arch.sae_block,
&vcpu->arch.save_area);
return 0;
}
void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
{
}
void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
{
kvm_s390_free_mmu_cache(vcpu->arch.mc);
VCPU_EVENT(vcpu, 3, "%s", "free cpu");
}
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
save_access_regs(&vcpu->arch.host_acrs[0]);
vcpu->cpu = cpu;
lasrm(&vcpu->arch.save_area);
}
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
{
stiasrm(&vcpu->arch.save_area);
vcpu->cpu = -1;
restore_access_regs(&vcpu->arch.host_acrs[0]);
}
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
struct kvm_mp_state *mp_state)
{
*mp_state = READ_ONCE(vcpu->arch.mp_state);
return 0;
}
int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
struct kvm_mp_state *mp_state)
{
return -EINVAL;
}
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
{
return 0;
}
static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu)
{
unsigned long features = KVM_S390_ARM64_IMPL_FEATURES;
return features;
}
bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
{
return vcpu_mode_priv(vcpu);
}
int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
{
struct kvm *kvm = vcpu->kvm;
if (!kvm_vcpu_initialized(vcpu))
return -ENOEXEC;
if (!kvm_arm_vcpu_is_finalized(vcpu))
return -EPERM;
if (likely(READ_ONCE(vcpu->pid)))
return 0;
scoped_guard(mutex, &kvm->arch.config_lock)
set_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags);
return 0;
}
/**
* check_vcpu_requests - check and handle pending vCPU requests
* @vcpu: the VCPU pointer
*
* Return: 1 if we should enter the guest
* 0 if we should exit to userspace
* < 0 if we should exit to userspace, where the return value indicates
* an error
*/
static int check_vcpu_requests(struct kvm_vcpu *vcpu)
{
if (kvm_request_pending(vcpu)) {
if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu))
return -EIO;
if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
kvm_reset_vcpu(vcpu);
/*
* Clear IRQ_PENDING requests that were made to guarantee
* that a VCPU sees new virtual interrupts.
*/
kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu);
if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
asce_flush_tlb(vcpu->kvm->arch.gmap->asce);
}
return 1;
}
static int kvm_vcpu_initialize(struct kvm_vcpu *vcpu,
const struct kvm_vcpu_init *init)
{
unsigned long features = init->features[0];
struct kvm *kvm = vcpu->kvm;
guard(mutex)(&kvm->arch.config_lock);
if (test_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags) &&
kvm_vcpu_init_changed(vcpu, init))
return -EINVAL;
bitmap_copy(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES);
kvm_reset_vcpu(vcpu);
set_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags);
vcpu_set_flag(vcpu, VCPU_INITIALIZED);
return 0;
}
static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
const struct kvm_vcpu_init *init)
{
int ret;
if (init->target != KVM_ARM_TARGET_GENERIC_V8)
return -EINVAL;
ret = kvm_vcpu_init_check_features(vcpu, init);
if (ret)
return ret;
if (!kvm_vcpu_initialized(vcpu))
return kvm_vcpu_initialize(vcpu, init);
if (kvm_vcpu_init_changed(vcpu, init))
return -EINVAL;
kvm_reset_vcpu(vcpu);
return 0;
}
static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
struct kvm_vcpu_init *init)
{
struct kvm_sae_save_area *save_area = &vcpu->arch.save_area;
struct kvm_sae_block *sae_block = &vcpu->arch.sae_block;
int ret;
sae_block->save_area = virt_to_phys(save_area);
save_area->sdo = virt_to_phys(sae_block);
ret = kvm_vcpu_set_target(vcpu, init);
if (ret)
return ret;
scoped_guard(spinlock, &vcpu->arch.mp_state_lock) {
WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);
}
return 0;
}
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
bool line_status)
{
/* stub for now*/
return -EINVAL;
}
static void adjust_pc(struct kvm_vcpu *vcpu)
{
if (vcpu_get_flag(vcpu, INCREMENT_PC)) {
kvm_skip_instr(vcpu);
vcpu_clear_flag(vcpu, INCREMENT_PC);
}
}
static noinstr void arm_vcpu_enter_exit(phys_addr_t sae_block)
{
guest_timing_enter_irqoff();
guest_state_enter_irqoff();
/*
* External interrupts may occur while the SAE instruction is executing
* and must preempt guest execution to maintain responsiveness of the
* host.
*
* As a temporary workaround, run SAE with interrupts enabled so that any
* interrupt is taken via the normal host interrupt path rather than
* being subsumed inside the SAE window.
*/
raw_local_irq_enable();
sae(sae_block);
raw_local_irq_disable();
guest_state_exit_irqoff();
guest_timing_exit_irqoff();
}
/** kvm_arch_vcpu_ioctl_run() - run arm64 vCPU
*
* Execute arm64 guest instructions using SAE.
*
* Returns:
* 1 enter the guest (should not be observed by userspace)
* 0 exit to userspace
* < 0 exit to userspace, where the return value indicates an error
*
*
*/
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
{
DECLARE_KERNEL_FPU_ONSTACK32(fpu_save);
struct kvm_run *kvm_run = vcpu->run;
int ret;
if (kvm_run->exit_reason == KVM_EXIT_MMIO) {
ret = kvm_handle_mmio_return(vcpu);
if (ret <= 0)
return ret;
}
vcpu_load(vcpu);
kernel_fpu_begin(&fpu_save, KERNEL_FPC | KERNEL_VXR);
load_vx_regs((vcpu->arch.ctxt.vregs));
if (!vcpu->wants_to_run) {
ret = -EINTR;
goto out;
}
kvm_sigset_activate(vcpu);
might_fault();
ret = 1;
kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
while (ret > 0) {
ret = kvm_xfer_to_guest_mode_handle_work(vcpu);
if (!ret)
ret = 1;
if (ret > 0)
ret = check_vcpu_requests(vcpu);
vcpu->arch.sae_block.icptr = 0;
local_irq_disable();
xfer_to_guest_mode_prepare();
/*
* The full memory barrier in smp_store_mb() orders the mode
* store against the kvm_request_pending() check below, so
* any pending kick is visible before we decide to enter the
* guest.
*/
smp_store_mb(vcpu->mode, IN_GUEST_MODE);
if (ret <= 0 || kvm_request_pending(vcpu) ||
xfer_to_guest_mode_work_pending()) {
vcpu->mode = OUTSIDE_GUEST_MODE;
local_irq_enable();
continue;
}
trace_kvm_entry(*vcpu_pc(vcpu));
adjust_pc(vcpu);
arm_vcpu_enter_exit(virt_to_phys(&vcpu->arch.sae_block));
vcpu->mode = OUTSIDE_GUEST_MODE;
vcpu->stat.exits++;
local_irq_enable();
trace_kvm_exit(kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
ret = handle_exit(vcpu);
}
kvm_sigset_deactivate(vcpu);
out:
if (unlikely(vcpu_get_flag(vcpu, INCREMENT_PC)))
adjust_pc(vcpu);
save_vx_regs(vcpu->arch.ctxt.vregs);
kernel_fpu_end(&fpu_save, KERNEL_FPC | KERNEL_VXR);
vcpu_put(vcpu);
return ret;
}
static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu,
struct kvm_device_attr *attr)
{
int ret;
switch (attr->group) {
default:
ret = -ENXIO;
break;
}
return ret;
}
static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu,
struct kvm_device_attr *attr)
{
int ret;
switch (attr->group) {
default:
ret = -ENXIO;
break;
}
return ret;
}
static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu,
struct kvm_device_attr *attr)
{
int ret;
switch (attr->group) {
default:
ret = -ENXIO;
break;
}
return ret;
}
long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
{
struct kvm_vcpu *vcpu = filp->private_data;
void __user *argp = (void __user *)arg;
struct kvm_device_attr attr;
int ret;
switch (ioctl) {
case KVM_ARM_VCPU_INIT: {
struct kvm_vcpu_init init;
ret = -EFAULT;
if (copy_from_user(&init, argp, sizeof(init)))
break;
ret = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
break;
}
case KVM_SET_ONE_REG:
case KVM_GET_ONE_REG: {
struct kvm_one_reg reg;
ret = -ENOEXEC;
if (unlikely(!kvm_vcpu_initialized(vcpu)))
break;
ret = -EFAULT;
if (copy_from_user(®, argp, sizeof(reg)))
break;
if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
kvm_reset_vcpu(vcpu);
if (ioctl == KVM_SET_ONE_REG)
ret = kvm_arm_set_reg(vcpu, ®);
else
ret = kvm_arm_get_reg(vcpu, ®);
break;
}
case KVM_GET_REG_LIST: {
struct kvm_reg_list __user *user_list = argp;
struct kvm_reg_list reg_list;
unsigned int n;
ret = -ENOEXEC;
if (unlikely(!kvm_vcpu_initialized(vcpu)))
break;
ret = -EPERM;
if (!kvm_arm_vcpu_is_finalized(vcpu))
break;
ret = -EFAULT;
if (copy_from_user(®_list, user_list, sizeof(reg_list)))
break;
n = reg_list.n;
reg_list.n = kvm_arm_num_regs(vcpu);
if (copy_to_user(user_list, ®_list, sizeof(reg_list)))
break;
ret = -E2BIG;
if (n < reg_list.n)
break;
ret = kvm_arm_copy_reg_indices(vcpu, user_list->reg);
break;
}
case KVM_ARM_VCPU_FINALIZE: {
int what;
if (!kvm_vcpu_initialized(vcpu))
return -ENOEXEC;
if (get_user(what, (const int __user *)argp))
return -EFAULT;
ret = kvm_arm_vcpu_finalize(vcpu, what);
break;
}
case KVM_SET_DEVICE_ATTR: {
ret = -EFAULT;
if (copy_from_user(&attr, argp, sizeof(attr)))
break;
ret = kvm_arm_vcpu_set_attr(vcpu, &attr);
break;
}
case KVM_GET_DEVICE_ATTR: {
ret = -EFAULT;
if (copy_from_user(&attr, argp, sizeof(attr)))
break;
ret = kvm_arm_vcpu_get_attr(vcpu, &attr);
break;
}
case KVM_HAS_DEVICE_ATTR: {
ret = -EFAULT;
if (copy_from_user(&attr, argp, sizeof(attr)))
break;
ret = kvm_arm_vcpu_has_attr(vcpu, &attr);
break;
}
default:
ret = -EINVAL;
}
return ret;
}
int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
struct kvm_dirty_log *log)
{
return s390_kvm_mmu_get_dirty_log(kvm, log);
}
bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
{
scoped_guard(read_lock, &kvm->mmu_lock)
return gmap_age_gfn(kvm->arch.gmap, range->start, range->end);
}
void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
{
gfn_t last_gfn = memslot->base_gfn + memslot->npages;
scoped_guard(read_lock, &kvm->mmu_lock)
gmap_sync_dirty_log(kvm->arch.gmap, memslot->base_gfn, last_gfn);
}
int kvm_arch_prepare_memory_region(struct kvm *kvm,
const struct kvm_memory_slot *old,
struct kvm_memory_slot *new,
enum kvm_mr_change change)
{
return s390_kvm_mmu_prepare_memory_region(kvm, old, new, change);
}
void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old,
const struct kvm_memory_slot *new,
enum kvm_mr_change change)
{
}
bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
{
return gmap_unmap_gfn_range(kvm->arch.gmap, range->slot, range->start, range->end);
}
bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
{
scoped_guard(read_lock, &kvm->mmu_lock)
return dat_test_age_gfn(kvm->arch.gmap->asce, range->start, range->end);
}
int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
{
void __user *argp = (void __user *)arg;
switch (ioctl) {
case KVM_ARM_PREFERRED_TARGET: {
struct kvm_vcpu_init init = {
.target = KVM_ARM_TARGET_GENERIC_V8,
};
if (copy_to_user(argp, &init, sizeof(init)))
return -EFAULT;
return 0;
}
default:
return -EINVAL;
}
}
bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
{
return false;
}
int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
int irq_source_id, int level, bool line_status)
{
return -EINVAL;
}
int kvm_set_routing_entry(struct kvm *kvm,
struct kvm_kernel_irq_routing_entry *e,
const struct kvm_irq_routing_entry *ue)
{
return -EINVAL;
}
int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
{
return 0;
}
long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl,
unsigned long arg)
{
return -ENOIOCTLCMD;
}
static int __init kvm_s390_arm64_init(void)
{
if (!sclp.has_aef)
return -ENODEV;
return kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
}
static __exit void kvm_s390_arm64_exit(void)
{
kvm_exit();
}
module_init(kvm_s390_arm64_init);
module_exit(kvm_s390_arm64_exit);