// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2025, 2026 Arm Ltd.
*/
#include <kvm/arm_vgic.h>
#include <linux/bitops.h>
#include <linux/irqchip/arm-vgic-info.h>
#include <linux/irqdomain.h>
#include <linux/kvm_host.h>
#include <linux/uaccess.h>
#include "vgic-v5-tables.h"
#include "vgic.h"
#define ppi_caps kvm_vgic_global_state.vgic_v5_ppi_caps
#define irs_caps kvm_vgic_global_state.vgic_v5_irs_caps
static int vgic_v5_irs_assign_vmt(bool two_level, u8 vm_id_bits, phys_addr_t vmt_base);
static int vgic_v5_irs_clear_vmt(void);
/*
* Not all PPIs are guaranteed to be implemented for GICv5. Deterermine which
* ones are, and generate a mask.
*/
static void vgic_v5_get_implemented_ppis(void)
{
/*
* If we have KVM, we have EL2, which means that we have support for the
* EL1 and EL2 Physical & Virtual timers.
*/
__set_bit(GICV5_ARCH_PPI_CNTHP, ppi_caps.impl_ppi_mask);
__set_bit(GICV5_ARCH_PPI_CNTV, ppi_caps.impl_ppi_mask);
__set_bit(GICV5_ARCH_PPI_CNTHV, ppi_caps.impl_ppi_mask);
__set_bit(GICV5_ARCH_PPI_CNTP, ppi_caps.impl_ppi_mask);
/* The SW_PPI should be available */
__set_bit(GICV5_ARCH_PPI_SW_PPI, ppi_caps.impl_ppi_mask);
/* The PMUIRQ is available if we have the PMU */
__assign_bit(GICV5_ARCH_PPI_PMUIRQ, ppi_caps.impl_ppi_mask, system_supports_pmuv3());
}
static u32 irs_readl_relaxed(const u32 reg_offset)
{
return readl_relaxed(irs_caps.irs_base + reg_offset);
}
static void irs_writel_relaxed(const u32 val, const u32 reg_offset)
{
writel_relaxed(val, irs_caps.irs_base + reg_offset);
}
static u64 irs_readq_relaxed(const u32 reg_offset)
{
return readq_relaxed(irs_caps.irs_base + reg_offset);
}
static void irs_writeq_relaxed(const u64 val, const u32 reg_offset)
{
writeq_relaxed(val, irs_caps.irs_base + reg_offset);
}
static void vgic_v5_irs_cache_id_regs(const struct gic_kvm_info *info)
{
irs_caps.irs_base = info->gicv5_irs.base;
irs_caps.non_coherent = info->gicv5_irs.non_coherent;
irs_caps.idr2 = irs_readl_relaxed(GICV5_IRS_IDR2);
irs_caps.idr3 = irs_readl_relaxed(GICV5_IRS_IDR3);
irs_caps.idr4 = irs_readl_relaxed(GICV5_IRS_IDR4);
}
/*
* Probe for a vGICv5 compatible interrupt controller, returning 0 on success.
*/
int vgic_v5_probe(const struct gic_kvm_info *info)
{
bool v5_registered = false;
int ret;
kvm_vgic_global_state.type = VGIC_V5;
kvm_vgic_global_state.max_gic_vcpus = 0;
kvm_vgic_global_state.max_gicv5_vcpus = 0;
kvm_vgic_global_state.vcpu_base = 0;
kvm_vgic_global_state.vctrl_base = NULL;
kvm_vgic_global_state.can_emulate_gicv2 = false;
kvm_vgic_global_state.has_gicv4 = false;
kvm_vgic_global_state.has_gicv4_1 = false;
/*
* GICv5 is currently not supported in Protected mode. Skip the
* registration of GICv5 completely to make sure no guests can create a
* GICv5-based guest.
*/
if (is_protected_kvm_enabled()) {
kvm_info("GICv5-based guests are not supported with pKVM\n");
goto skip_v5;
}
if (!gicv5_global_data.lpi_domain) {
kvm_err("GICv5 LPI domain unavailable\n");
return -ENODEV;
}
vgic_v5_irs_cache_id_regs(info);
vgic_v5_get_implemented_ppis();
/*
* Even if the HW supports more per-VM vCPUs, artificially cap as we
* can't use them all.
*/
kvm_vgic_global_state.max_gicv5_vcpus = min(vgic_v5_irs_max_vpes(&irs_caps),
VGIC_V5_MAX_CPUS);
/*
* GICv5 requires a set of tables to be allocated in order to manage
* VMs. We allocate them in advance here, which alas means that we
* already have to make a decisions regarding the maximum number of VMs
* we want to run. For now, we match the maximum number offered by the
* hardware, but this might not be a wise choice in the long term.
*/
ret = vgic_v5_vmt_allocate(kvm_vgic_global_state.max_gicv5_vcpus);
if (ret) {
kvm_err("Failed to allocate the GICv5 VM tables; no GICv5 support\n");
return -ENODEV;
}
/*
* We've now allocated the VM table, but the host's IRS doesn't know
* about it yet. Provide the base address of the VMT to the IRS, as well
* as the number of ID bits that it covers and the structure used
* (linear/two-level).
*/
ret = vgic_v5_irs_assign_vmt(vgic_v5_irs_two_level_vmt_support(&irs_caps),
ilog2(vgic_v5_irs_max_vms(&irs_caps)),
vgic_v5_get_vmt_base());
if (ret) {
kvm_err("Failed to assign the GICv5 VM tables to the IRS; no GICv5 support\n");
if (!vgic_v5_irs_clear_vmt())
vgic_v5_vmt_free();
return -ENODEV;
}
ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V5);
if (ret) {
kvm_err("Cannot register GICv5 KVM device.\n");
/*
* Don't free the VMT itself if the hardware still has a valid
* pointer to it.
*/
if (!vgic_v5_irs_clear_vmt())
vgic_v5_vmt_free();
return -ENODEV;
}
v5_registered = true;
kvm_vgic_global_state.max_gic_vcpus =
kvm_vgic_global_state.max_gicv5_vcpus;
kvm_info("GCIE system register CPU interface\n");
skip_v5:
/* If we don't support the GICv3 compat mode we're done. */
if (!cpus_have_final_cap(ARM64_HAS_GICV5_LEGACY)) {
if (!v5_registered)
return -ENODEV;
return 0;
}
kvm_vgic_global_state.has_gcie_v3_compat = true;
/*
* The ListRegs field is 5 bits, but there is an architectural
* maximum of 16 list registers. Just ignore bit 4...
*/
kvm_vgic_global_state.nr_lr = (vgic_ich_vtr() & 0xf) + 1;
ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V3);
if (ret) {
kvm_err("Cannot register GICv3-legacy KVM device.\n");
/* vGICv5 should still work */
return v5_registered ? 0 : ret;
}
kvm_vgic_global_state.max_gic_vcpus = max(kvm_vgic_global_state.max_gic_vcpus,
VGIC_V3_MAX_CPUS);
static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif);
kvm_info("GCIE legacy system register CPU interface\n");
vgic_v3_enable_cpuif_traps();
return 0;
}
/*
* Wait for completion of a change in any of IRS_VMT_BASER, IRS_VMAP_L2_VMTR,
* IRS_VMAP_VMR, IRS_VMAP_VPER, IRS_VMAP_VISTR, IRS_VMAP_L2_VISTR.
*/
static int vgic_v5_irs_wait_for_vm_op(void)
{
return gicv5_wait_for_op_atomic(irs_caps.irs_base,
GICV5_IRS_VMT_STATUSR,
GICV5_IRS_VMT_STATUSR_IDLE,
NULL);
}
/*
* Wait for completion of a change in any of IRS_VPE_SELR, IRS_VPE_DBR,
* IRS_VPE_CR0.
*/
static int vgic_v5_irs_wait_for_vpe_op(void)
{
return gicv5_wait_for_op_atomic(irs_caps.irs_base,
GICV5_IRS_VPE_STATUSR,
GICV5_IRS_VPE_STATUSR_IDLE,
NULL);
}
/*
* Wait for a write to IRS_SAVE_VMR to complete.
*/
static int vgic_v5_irs_wait_for_save_vm_op(u32 *statusr)
{
return gicv5_wait_for_op_atomic(irs_caps.irs_base,
GICV5_IRS_SAVE_VM_STATUSR,
GICV5_IRS_SAVE_VM_STATUSR_IDLE,
statusr);
}
static int vgic_v5_irs_write_vm_mmio_reg(u64 val, u32 offset)
{
int ret;
lockdep_assert_held(&vgic_v5_irs_lock);
/* Make sure that we are idle to begin with */
ret = vgic_v5_irs_wait_for_vm_op();
if (ret)
return ret;
irs_writeq_relaxed(val, offset);
return vgic_v5_irs_wait_for_vm_op();
}
static int vgic_v5_irs_assign_vmt(bool two_level, u8 vm_id_bits,
phys_addr_t vmt_base)
{
u64 vmt_baser;
u32 vmt_cfgr;
int ret;
guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock);
ret = vgic_v5_irs_wait_for_vm_op();
if (ret)
return ret;
vmt_baser = irs_readq_relaxed(GICV5_IRS_VMT_BASER);
if (!!FIELD_GET(GICV5_IRS_VMT_BASER_VALID, vmt_baser))
return -EBUSY;
vmt_cfgr = FIELD_PREP(GICV5_IRS_VMT_CFGR_VM_ID_BITS, vm_id_bits);
if (two_level)
vmt_cfgr |= FIELD_PREP(GICV5_IRS_VMT_CFGR_STRUCTURE,
GICV5_IRS_VMT_CFGR_STRUCTURE_TWO_LEVEL);
irs_writel_relaxed(vmt_cfgr, GICV5_IRS_VMT_CFGR);
/* The base address is intentionally only masked and not shifted */
vmt_baser = FIELD_PREP(GICV5_IRS_VMT_BASER_VALID, true) |
(vmt_base & GICV5_IRS_VMT_BASER_ADDR);
irs_writeq_relaxed(vmt_baser, GICV5_IRS_VMT_BASER);
return vgic_v5_irs_wait_for_vm_op();
}
static int vgic_v5_irs_clear_vmt(void)
{
guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock);
return vgic_v5_irs_write_vm_mmio_reg(0, GICV5_IRS_VMT_BASER);
}
static int vgic_v5_irs_vmap_l2_vmt(u16 vm_id)
{
u64 val = FIELD_PREP(GICV5_IRS_VMAP_L2_VMTR_VM_ID, vm_id) |
GICV5_IRS_VMAP_L2_VMTR_M;
return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_L2_VMTR);
}
static int __vgic_v5_irs_vmap_vm(u16 vm_id, bool unmap)
{
u64 val = FIELD_PREP(GICV5_IRS_VMAP_VMR_VM_ID, vm_id) |
FIELD_PREP(GICV5_IRS_VMAP_VMR_U, unmap) |
GICV5_IRS_VMAP_VMR_M;
return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_VMR);
}
static int vgic_v5_irs_set_vm_valid(u16 vm_id)
{
return __vgic_v5_irs_vmap_vm(vm_id, false);
}
static int vgic_v5_irs_set_vm_invalid(u16 vm_id)
{
return __vgic_v5_irs_vmap_vm(vm_id, true);
}
static int __vgic_v5_irs_update_vist_validity(u16 vm_id, bool spi_ist, bool unmap)
{
u8 type = spi_ist ? 0b011 : 0b010;
u64 val = FIELD_PREP(GICV5_IRS_VMAP_VISTR_TYPE, type) |
FIELD_PREP(GICV5_IRS_VMAP_VISTR_VM_ID, vm_id) |
FIELD_PREP(GICV5_IRS_VMAP_VISTR_U, unmap) |
GICV5_IRS_VMAP_VISTR_M;
return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_VISTR);
}
static int vgic_v5_irs_set_vist_valid(u16 vm_id, bool spi_ist)
{
return __vgic_v5_irs_update_vist_validity(vm_id, spi_ist, false);
}
/*
* LPI ISTs can be invalidated explicitly. SPI ISTs are invalidated by making
* the VMTE invalid during teardown.
*/
static int vgic_v5_irs_set_vist_invalid(u16 vm_id, bool spi_ist)
{
return __vgic_v5_irs_update_vist_validity(vm_id, spi_ist, true);
}
static int vgic_v5_irs_set_up_vpe(u16 vm_id, u16 vpe_id,
irq_hw_number_t db_hwirq)
{
u64 vmap_vper, dbr, selr;
u32 statusr, cr0;
int ret;
lockdep_assert_held(&vgic_v5_irs_lock);
/* Make sure that we are idle to begin with */
ret = vgic_v5_irs_wait_for_vm_op();
if (ret)
return ret;
/* Mark the VPE as valid */
vmap_vper = FIELD_PREP(GICV5_IRS_VMAP_VPER_VPE_ID, vpe_id) |
FIELD_PREP(GICV5_IRS_VMAP_VPER_VM_ID, vm_id) |
GICV5_IRS_VMAP_VPER_M;
irs_writeq_relaxed(vmap_vper, GICV5_IRS_VMAP_VPER);
/* Wait for the VPE to be marked valid in the VPET */
ret = vgic_v5_irs_wait_for_vm_op();
if (ret)
return ret;
selr = FIELD_PREP(GICV5_IRS_VPE_SELR_VPE_ID, vpe_id) |
FIELD_PREP(GICV5_IRS_VPE_SELR_VM_ID, vm_id) |
GICV5_IRS_VPE_SELR_S;
irs_writeq_relaxed(selr, GICV5_IRS_VPE_SELR);
ret = vgic_v5_irs_wait_for_vpe_op();
if (ret)
return ret;
statusr = irs_readl_relaxed(GICV5_IRS_VPE_STATUSR);
if (!FIELD_GET(GICV5_IRS_VPE_STATUSR_V, statusr))
return -EINVAL;
/* Set targeted only routing (disable 1ofN vPE selection) */
cr0 = GICV5_IRS_VPE_CR0_DPS;
irs_writel_relaxed(cr0, GICV5_IRS_VPE_CR0);
ret = vgic_v5_irs_wait_for_vpe_op();
if (ret)
return ret;
/*
* The VPE has not yet run. Therefore, make sure that all interrupts
* will generate a doorbell.
*/
dbr = FIELD_PREP(GICV5_IRS_VPE_DBR_INTID, db_hwirq) |
GICV5_IRS_VPE_DBR_DBV;
irs_writeq_relaxed(dbr, GICV5_IRS_VPE_DBR);
ret = vgic_v5_irs_wait_for_vpe_op();
if (ret)
return ret;
return 0;
}
static int vgic_v5_irs_save_vm_op(u16 vm_id, bool save, u32 *statusr)
{
u64 save_vmr;
int ret;
save_vmr = FIELD_PREP(GICV5_IRS_SAVE_VMR_VM_ID, vm_id);
save_vmr |= GICV5_IRS_SAVE_VMR_Q;
save_vmr |= FIELD_PREP(GICV5_IRS_SAVE_VMR_S, save);
guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock);
/* Make sure that we are idle to begin with. */
ret = vgic_v5_irs_wait_for_save_vm_op(NULL);
if (ret)
return ret;
irs_writeq_relaxed(save_vmr, GICV5_IRS_SAVE_VMR);
return vgic_v5_irs_wait_for_save_vm_op(statusr);
}
static irqreturn_t db_handler(int irq, void *data)
{
struct kvm_vcpu *vcpu = data;
WRITE_ONCE(vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db_fired, true);
kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
kvm_vcpu_kick(vcpu);
return IRQ_HANDLED;
}
static int vgic_v5_send_command(struct kvm_vcpu *vcpu, enum gicv5_vcpu_cmd cmd)
{
int irq = vgic_v5_vpe_db(vcpu);
if (!irq)
return -ENXIO;
return irq_set_vcpu_affinity(irq, &cmd);
}
static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info)
{
struct vgic_v5_vm *vm = data->domain->host_data;
enum gicv5_vcpu_cmd *cmd = vcpu_info;
unsigned int vcpu_idx = data->hwirq;
struct kvm_vcpu *vcpu;
u16 vpe_id;
guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock);
switch (*cmd) {
case VMT_L2_MAP:
return vgic_v5_irs_vmap_l2_vmt(vm->vm_id);
case VMTE_MAKE_VALID:
return vgic_v5_irs_set_vm_valid(vm->vm_id);
case VMTE_MAKE_INVALID:
return vgic_v5_irs_set_vm_invalid(vm->vm_id);
case VPE_MAKE_VALID:
/*
* The index in the doorbell domain aligns with our flat
* vcpu_idx index. However, we need the actual VPE ID, which
* means we first need to resolve the actual vcpu.
*/
vcpu = kvm_get_vcpu(vm->kvm, vcpu_idx);
if (!vcpu)
return -EINVAL;
vpe_id = vgic_v5_vpe_id(vcpu);
/*
* We need the actual LPI ID which lives in the top-most parent
* domain. This hwirq won't include the type (LPI) but that's
* not required for the IRS_VPE_DBR.
*/
while (data->parent_data)
data = data->parent_data;
return vgic_v5_irs_set_up_vpe(vm->vm_id, vpe_id, data->hwirq);
case SPI_VIST_MAKE_VALID:
return vgic_v5_irs_set_vist_valid(vm->vm_id, true);
case LPI_VIST_MAKE_VALID:
return vgic_v5_irs_set_vist_valid(vm->vm_id, false);
case LPI_VIST_MAKE_INVALID:
return vgic_v5_irs_set_vist_invalid(vm->vm_id, false);
default:
return -EINVAL;
}
}
/*
* This set of irq_chip functions is specific for doorbells.
*/
static const struct irq_chip vgic_v5_db_irq_chip = {
.name = "GICv5-DB",
.irq_mask = irq_chip_mask_parent,
.irq_unmask = irq_chip_unmask_parent,
.irq_eoi = irq_chip_eoi_parent,
.irq_set_affinity = irq_chip_set_affinity_parent,
.irq_get_irqchip_state = irq_chip_get_parent_state,
.irq_set_irqchip_state = irq_chip_set_parent_state,
.irq_set_vcpu_affinity = vgic_v5_db_set_vcpu_affinity,
.flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_SKIP_SET_WAKE |
IRQCHIP_MASK_ON_SUSPEND,
};
static void vgic_v5_irq_db_domain_free(struct irq_domain *domain,
unsigned int virq, unsigned int nr_irqs)
{
int i;
for (i = 0; i < nr_irqs; i++) {
struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
irq_set_handler(virq + i, NULL);
irq_domain_reset_irq_data(d);
}
irq_domain_free_irqs_parent(domain, virq, nr_irqs);
}
static int vgic_v5_irq_db_domain_alloc(struct irq_domain *domain,
unsigned int virq, unsigned int nr_irqs,
void *arg)
{
const struct irq_chip *chip = &vgic_v5_db_irq_chip;
struct irq_data *irqd;
int ret;
ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, NULL);
if (ret)
return ret;
for (int i = 0; i < nr_irqs; i++) {
irq_domain_set_hwirq_and_chip(domain, virq + i, i, chip,
domain->host_data);
irqd = irq_desc_get_irq_data(irq_to_desc(virq + i));
irqd_set_single_target(irqd);
}
return 0;
}
static const struct irq_domain_ops vgic_v5_irq_db_domain_ops = {
.alloc = vgic_v5_irq_db_domain_alloc,
.free = vgic_v5_irq_db_domain_free,
};
static int vgic_v5_create_per_vm_domain(struct kvm *kvm)
{
struct vgic_v5_vm *vm = &kvm->arch.vgic.gicv5_vm;
int nr_vcpus = atomic_read(&kvm->online_vcpus);
int id = task_pid_nr(current);
int ret, db_virq = 0;
vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv5-vpe-db", id);
if (!vm->fwnode)
return -ENOMEM;
vm->domain = irq_domain_create_hierarchy(gicv5_global_data.lpi_domain,
0, nr_vcpus, vm->fwnode,
&vgic_v5_irq_db_domain_ops, vm);
if (!vm->domain) {
ret = -ENOMEM;
goto err;
}
db_virq = irq_domain_alloc_irqs(vm->domain, nr_vcpus, NUMA_NO_NODE, vm);
if (db_virq <= 0) {
ret = db_virq;
goto err;
}
kvm->arch.vgic.gicv5_vm.vpe_db_base = db_virq;
return 0;
err:
if (vm->domain)
irq_domain_remove(vm->domain);
if (vm->fwnode)
irq_domain_free_fwnode(vm->fwnode);
kvm->arch.vgic.gicv5_vm.vpe_db_base = 0;
vm->domain = NULL;
vm->fwnode = NULL;
return ret;
}
static void vgic_v5_teardown_per_vm_domain(struct vgic_v5_vm *vm)
{
if (!vm->domain)
return;
irq_domain_remove(vm->domain);
irq_domain_free_fwnode(vm->fwnode);
vm->domain = NULL;
vm->fwnode = NULL;
}
void vgic_v5_reset(struct kvm_vcpu *vcpu)
{
/*
* We always present 16-bits of ID space to the guest, irrespective of
* the host allowing more.
*/
vcpu->arch.vgic_cpu.num_id_bits = ICC_IDR0_EL1_ID_BITS_16BITS;
/*
* The GICv5 architeture only supports 5-bits of priority in the
* CPUIF (but potentially fewer in the IRS).
*/
vcpu->arch.vgic_cpu.num_pri_bits = 5;
/* Make the VPE valid in the VPET */
if (WARN_ON(vgic_v5_send_command(vcpu, VPE_MAKE_VALID)))
return;
}
static void vgic_v5_free_doorbells(struct kvm *kvm, unsigned int nr_dbs)
{
struct vgic_v5_vm *vm = &kvm->arch.vgic.gicv5_vm;
struct kvm_vcpu *vcpu;
unsigned long i;
int db;
for (i = 0; i < nr_dbs; i++) {
vcpu = kvm_get_vcpu(kvm, i);
db = vgic_v5_vpe_db(vcpu);
if (!db)
continue;
free_irq(db, vcpu);
vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db = 0;
}
if (vm->vpe_db_base) {
irq_domain_free_irqs(vm->vpe_db_base,
atomic_read(&kvm->online_vcpus));
vm->vpe_db_base = 0;
}
}
void vgic_v5_teardown(struct kvm *kvm)
{
struct vgic_dist *dist = &kvm->arch.vgic;
struct kvm_vcpu *vcpu, *vcpu0;
bool release_vm_id = true;
unsigned long i;
int rc;
lockdep_assert_held(&kvm->arch.config_lock);
/*
* If the VM's ID isn't valid, then we either failed init very early or
* we've been called a second time. Nothing to do here in either case.
*/
if (kvm->arch.vgic.gicv5_vm.vm_id == VGIC_V5_VM_ID_INVAL)
return;
if (kvm->arch.vgic.gicv5_vm.vmte_allocated) {
/* Make the VM invalid */
vcpu0 = kvm_get_vcpu(kvm, 0);
rc = vgic_v5_send_command(vcpu0, VMTE_MAKE_INVALID);
if (rc) {
kvm_err("could not make VMTE invalid\n");
release_vm_id = false;
goto out_free_doorbells;
}
kvm_for_each_vcpu(i, vcpu, kvm) {
if (vgic_v5_vmte_free_vpe(vcpu)) {
kvm_err("Failed to free VPE\n");
release_vm_id = false;
goto out_free_doorbells;
}
}
if (vgic_v5_vmte_release(kvm)) {
kvm_err("Failed to release VM 0x%x\n", dist->gicv5_vm.vm_id);
release_vm_id = false;
}
}
out_free_doorbells:
vgic_v5_free_doorbells(kvm, atomic_read(&kvm->online_vcpus));
vgic_v5_teardown_per_vm_domain(&kvm->arch.vgic.gicv5_vm);
/*
* We only release the VM ID itself if we didn't fail earlier. It does
* mean that we might lose the VM ID (and associated VMTE, etc), but
* given that we've failed to tear them down correctly there's no way to
* safely reuse them. The VM ID allocating IDA will make sure we don't
* accidentally reuse this partially torn down state.
*/
if (release_vm_id)
vgic_v5_release_vm_id(kvm);
}
/*
* Claim and populate a VMTE (optionally making a new L2 VMT valid), create VPE
* doorbells, allocate VPET and populate for each VPE. Finally, we also init the
* vIRS, which means allocating and making the virtual SPI IST valid.
*
* Note: We do need to put the cart before the horse here. The VPE doorbells are
* our conduit for communication with the IRS, which means we need to have those
* before making the VMTE valid.
*
* On failure, we clean up in the teardown path (vgic_v5_teardown()).
*/
int vgic_v5_init(struct kvm *kvm)
{
struct kvm_vcpu *vcpu, *vcpu0;
int nr_vcpus, ret = 0;
unsigned int db_virq;
unsigned long i;
lockdep_assert_held(&kvm->arch.config_lock);
nr_vcpus = atomic_read(&kvm->online_vcpus);
if (nr_vcpus == 0)
return -ENODEV;
kvm_for_each_vcpu(i, vcpu, kvm) {
if (vcpu_has_nv(vcpu)) {
kvm_err("Nested GICv5 VMs are currently unsupported\n");
return -EINVAL;
}
}
/* We only allow userspace to drive the SW_PPI, if it is implemented. */
bitmap_zero(kvm->arch.vgic.gicv5_vm.userspace_ppis,
VGIC_V5_NR_PRIVATE_IRQS);
__set_bit(GICV5_ARCH_PPI_SW_PPI, kvm->arch.vgic.gicv5_vm.userspace_ppis);
bitmap_and(kvm->arch.vgic.gicv5_vm.userspace_ppis,
kvm->arch.vgic.gicv5_vm.userspace_ppis,
ppi_caps.impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS);
ret = vgic_v5_allocate_vm_id(kvm);
if (ret)
return ret;
/*
* Stash a backpointer to struct kvm. It is required to resolve the VPE
* ID from the doorbell index, which matches the flat vcpu_idx and not
* the vcpu_id.
*/
kvm->arch.vgic.gicv5_vm.kvm = kvm;
ret = vgic_v5_create_per_vm_domain(kvm);
if (ret)
goto err;
db_virq = kvm->arch.vgic.gicv5_vm.vpe_db_base;
kvm_for_each_vcpu(i, vcpu, kvm) {
ret = request_irq(db_virq + i, db_handler, 0, "vcpu", vcpu);
if (ret)
goto err;
/* Stash it with the VCPU for easy retrieval */
vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db = db_virq + i;
}
/* Populate VMTE (with VPET and VM descriptor) */
ret = vgic_v5_vmte_init(kvm);
if (ret)
goto err;
/* We pick the first vcpu to make the VMTE valid - any would do */
vcpu0 = kvm_get_vcpu(kvm, 0);
ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_VALID);
if (ret)
goto err;
/* Populate the VPETE for each VPE. */
kvm_for_each_vcpu(i, vcpu, kvm) {
ret = vgic_v5_vmte_alloc_vpe(vcpu);
if (ret)
goto err;
}
/* Init IRS (and alloc SPI IST) */
ret = kvm_vgic_v5_irs_init(kvm, kvm->arch.vgic.nr_spis);
if (ret)
goto err;
return 0;
err:
/*
* Explicitly tear everything down on failure. The teardown function is
* written to handle any partial state we might have, so we don't need
* to do any clean-up first. Teardown will be called a second time on VM
* destruction, but that's fine - it is better to leave things in a
* clean state now, and doubly so because userspace could actually go
* and retry init.
*/
vgic_v5_teardown(kvm);
return ret;
}
int vgic_v5_map_resources(struct kvm *kvm)
{
if (!vgic_initialized(kvm))
return -EBUSY;
if (kvm->arch.vgic.vgic_v5_irs_data->lpi_ist_restore_pending)
return -EINVAL;
return 0;
}
int vgic_v5_finalize_ppi_state(struct kvm_vcpu *vcpu)
{
struct kvm *kvm = vcpu->kvm;
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
int i;
if (!vgic_is_v5(kvm))
return 0;
guard(mutex)(&kvm->arch.config_lock);
/*
* Discover the set of PPIs that are exposed to the guest once per VM.
* Once known, apply that mask to each VCPU's restored PPI state as the
* VCPUs are first run.
*/
if (!test_bit(GICV5_ARCH_PPI_SW_PPI, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask)) {
bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_mask,
VGIC_V5_NR_PRIVATE_IRQS);
bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr,
VGIC_V5_NR_PRIVATE_IRQS);
for_each_set_bit(i, ppi_caps.impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS) {
const u32 intid = vgic_v5_make_ppi(i);
struct vgic_irq *irq;
irq = vgic_get_vcpu_irq(vcpu, intid);
/* Expose PPIs with an owner or the SW_PPI, only */
scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
if (irq->owner || i == GICV5_ARCH_PPI_SW_PPI) {
__set_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask);
__assign_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr,
irq->config == VGIC_CONFIG_LEVEL);
}
}
vgic_put_irq(kvm, irq);
}
}
/*
* Apply the mask to Enable, Active. Skip pending as that's calculated
* on guest entry.
*/
bitmap_and(cpu_if->vgic_ppi_enabler, cpu_if->vgic_ppi_enabler,
kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS);
bitmap_and(cpu_if->vgic_ppi_activer, cpu_if->vgic_ppi_activer,
kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS);
/* Also update the vgic_irqs */
for (i = 0; i < VGIC_V5_NR_PRIVATE_IRQS; i++) {
bool visible = test_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask);
const u32 intid = vgic_v5_make_ppi(i);
struct vgic_irq *irq;
irq = vgic_get_vcpu_irq(vcpu, intid);
scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
if (!visible) {
irq->enabled = false;
irq->active = false;
irq->pending_latch = false;
irq->line_level = false;
} else {
irq->enabled = test_bit(i, cpu_if->vgic_ppi_enabler);
irq->active = test_bit(i, cpu_if->vgic_ppi_activer);
}
}
vgic_put_irq(kvm, irq);
}
return 0;
}
static u32 vgic_v5_get_effective_priority_mask(struct kvm_vcpu *vcpu)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
u32 highest_ap, priority_mask, apr;
/*
* If the guest's CPU has not opted to receive interrupts, then the
* effective running priority is the highest priority. Just return 0
* (the highest priority).
*/
if (!FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_EN, cpu_if->vgic_vmcr))
return 0;
/*
* Counting the number of trailing zeros gives the current active
* priority. Explicitly use the 32-bit version here as we have 32
* priorities. 32 then means that there are no active priorities.
*/
apr = cpu_if->vgic_apr;
highest_ap = apr ? __builtin_ctz(apr) : 32;
/*
* An interrupt is of sufficient priority if it is equal to or
* greater than the priority mask. Add 1 to the priority mask
* (i.e., lower priority) to match the APR logic before taking
* the min. This gives us the lowest priority that is masked.
*/
priority_mask = FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_VPMR, cpu_if->vgic_vmcr);
return min(highest_ap, priority_mask + 1);
}
/*
* For GICv5, the PPIs are mostly directly managed by the hardware. We (the
* hypervisor) handle the pending, active, enable state save/restore, but
* don't need the PPIs to be queued on a per-VCPU AP list. Therefore,
* unlock, kick the vcpu and return.
*/
bool vgic_v5_ppi_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,
unsigned long flags)
__releases(&irq->irq_lock)
{
struct kvm_vcpu *vcpu;
lockdep_assert_held(&irq->irq_lock);
vcpu = irq->target_vcpu;
raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
/* Directly kick the target VCPU to make sure it sees the IRQ */
kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
kvm_vcpu_kick(vcpu);
return true;
}
/*
* Sets/clears the corresponding bit in the ICH_PPI_DVIR register.
*/
void vgic_v5_set_ppi_dvi(struct kvm_vcpu *vcpu, struct vgic_irq *irq, bool dvi)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
u32 ppi;
lockdep_assert_held(&irq->irq_lock);
ppi = vgic_v5_get_hwirq_id(irq->intid);
assign_bit(ppi, cpu_if->vgic_ppi_dvir, dvi);
}
static const struct irq_ops vgic_v5_ppi_irq_ops = {
.queue_irq_unlock = vgic_v5_ppi_queue_irq_unlock,
.set_direct_injection = vgic_v5_set_ppi_dvi,
};
void vgic_v5_set_ppi_ops(struct kvm_vcpu *vcpu, u32 vintid)
{
kvm_vgic_set_irq_ops(vcpu, vintid, &vgic_v5_ppi_irq_ops);
}
/*
* Sync back the PPI priorities to the vgic_irq shadow state for any interrupts
* exposed to the guest (skipping all others).
*/
static void vgic_v5_sync_ppi_priorities(struct kvm_vcpu *vcpu)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
u64 priorityr;
int i;
/*
* We have up to 16 PPI Priority regs, but only have a few interrupts
* that the guest is allowed to use. Limit our sync of PPI priorities to
* those actually exposed to the guest by first iterating over the mask
* of exposed PPIs.
*/
for_each_visible_v5_ppi(i, vcpu->kvm) {
u32 intid = vgic_v5_make_ppi(i);
struct vgic_irq *irq;
int pri_idx, pri_reg, pri_bit;
u8 priority;
/*
* Determine which priority register and the field within it to
* extract.
*/
pri_reg = i / 8;
pri_idx = i % 8;
pri_bit = pri_idx * 8;
priorityr = cpu_if->vgic_ppi_priorityr[pri_reg];
priority = field_get(GENMASK(pri_bit + 4, pri_bit), priorityr);
irq = vgic_get_vcpu_irq(vcpu, intid);
scoped_guard(raw_spinlock_irqsave, &irq->irq_lock)
irq->priority = priority;
vgic_put_irq(vcpu->kvm, irq);
}
}
bool vgic_v5_has_pending_ppi(struct kvm_vcpu *vcpu)
{
unsigned int priority_mask;
int i;
priority_mask = vgic_v5_get_effective_priority_mask(vcpu);
/*
* If the combined priority mask is 0, nothing can be signalled! In the
* case where the guest has disabled interrupt delivery for the vcpu
* (via ICV_CR0_EL1.EN->ICH_VMCR_EL2.EN), we calculate the priority mask
* as 0 too (the highest possible priority).
*/
if (!priority_mask)
return false;
for_each_visible_v5_ppi(i, vcpu->kvm) {
u32 intid = vgic_v5_make_ppi(i);
bool has_pending = false;
struct vgic_irq *irq;
irq = vgic_get_vcpu_irq(vcpu, intid);
scoped_guard(raw_spinlock_irqsave, &irq->irq_lock)
if (irq->enabled && irq->priority < priority_mask)
has_pending = irq->hw ? vgic_get_phys_line_level(irq) : irq_is_pending(irq);
vgic_put_irq(vcpu->kvm, irq);
if (has_pending)
return true;
}
return false;
}
/*
* Detect any PPIs state changes, and propagate the state with KVM's
* shadow structures.
*/
void vgic_v5_fold_ppi_state(struct kvm_vcpu *vcpu)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
unsigned long *activer, *pendr;
int i;
activer = host_data_ptr(vgic_v5_ppi_state)->activer_exit;
pendr = host_data_ptr(vgic_v5_ppi_state)->pendr;
for_each_visible_v5_ppi(i, vcpu->kvm) {
u32 intid = vgic_v5_make_ppi(i);
struct vgic_irq *irq;
irq = vgic_get_vcpu_irq(vcpu, intid);
scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
irq->active = test_bit(i, activer);
/* This is an OR to avoid losing incoming edges! */
if (irq->config == VGIC_CONFIG_EDGE)
irq->pending_latch |= test_bit(i, pendr);
}
vgic_put_irq(vcpu->kvm, irq);
}
/*
* Re-inject the exit state as entry state next time!
*
* Note that the write of the Enable state is trapped, and hence there
* is nothing to explcitly sync back here as we already have the latest
* copy by definition.
*/
bitmap_copy(cpu_if->vgic_ppi_activer, activer, VGIC_V5_NR_PRIVATE_IRQS);
}
void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu)
{
DECLARE_BITMAP(pendr, VGIC_V5_NR_PRIVATE_IRQS);
int i;
/*
* Time to enter the guest - we first need to build the guest's
* ICC_PPI_PENDRx_EL1, however.
*/
bitmap_zero(pendr, VGIC_V5_NR_PRIVATE_IRQS);
for_each_visible_v5_ppi(i, vcpu->kvm) {
u32 intid = vgic_v5_make_ppi(i);
struct vgic_irq *irq;
irq = vgic_get_vcpu_irq(vcpu, intid);
scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
__assign_bit(i, pendr, irq_is_pending(irq));
if (irq->config == VGIC_CONFIG_EDGE)
irq->pending_latch = false;
}
vgic_put_irq(vcpu->kvm, irq);
}
/*
* Copy the shadow state to the pending reg that will be written to the
* ICH_PPI_PENDRx_EL2 regs. While the guest is running we track any
* incoming changes to the pending state in the vgic_irq structures. The
* incoming changes are merged with the outgoing changes on the return
* path.
*/
bitmap_copy(host_data_ptr(vgic_v5_ppi_state)->pendr, pendr,
VGIC_V5_NR_PRIVATE_IRQS);
}
static bool vgic_v5_set_spi_pending_state(struct kvm_vcpu *vcpu,
struct vgic_irq *irq)
{
vgic_v5_set_irq_pend(irq->target_vcpu, irq);
return true;
}
bool vgic_v5_spi_queue_irq_unlock(struct kvm *kvm,
struct vgic_irq *irq,
unsigned long flags)
__releases(&irq->irq_lock)
{
lockdep_assert_held(&irq->irq_lock);
raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
return true;
}
static const struct irq_ops vgic_v5_spi_irq_ops = {
.set_pending_state = vgic_v5_set_spi_pending_state,
.queue_irq_unlock = vgic_v5_spi_queue_irq_unlock,
};
void vgic_v5_set_spi_ops(struct vgic_irq *irq)
{
if (WARN_ON(!irq) || WARN_ON(irq->ops))
return;
irq->ops = &vgic_v5_spi_irq_ops;
}
/* Set the pending state for GICv5 SPIs and LPIs */
void vgic_v5_set_irq_pend(struct kvm_vcpu *vcpu, struct vgic_irq *irq)
{
if (WARN_ON(__irq_is_ppi(KVM_DEV_TYPE_ARM_VGIC_V5, irq->intid)))
return;
kvm_call_hyp(__vgic_v5_vdpend, irq->intid, irq_is_pending(irq),
vcpu->kvm->arch.vgic.gicv5_vm.vm_id);
}
void vgic_v5_load(struct kvm_vcpu *vcpu)
{
bool irichppidis = !READ_ONCE(vcpu->kvm->arch.vgic.enabled);
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
u16 vm = vgic_v5_vm_id(vcpu->kvm);
u16 vpe = vgic_v5_vpe_id(vcpu);
/*
* On the WFI path, vgic_load is called a second time. The first is when
* scheduling in the vcpu thread again, and the second is when leaving
* WFI. Skip the second instance as it serves no purpose and just
* restores the same state again.
*/
if (cpu_if->gicv5_vpe.resident)
return;
kvm_call_hyp(__vgic_v5_restore_vmcr_apr, cpu_if);
cpu_if->vgic_contextr = FIELD_PREP(ICH_CONTEXTR_EL2_V, true) |
FIELD_PREP(ICH_CONTEXTR_EL2_IRICHPPIDIS, irichppidis) |
FIELD_PREP(ICH_CONTEXTR_EL2_VPE, vpe) |
FIELD_PREP(ICH_CONTEXTR_EL2_VM, vm);
kvm_call_hyp(__vgic_v5_make_resident, cpu_if);
/* Failed to make the VPE resident? Bang! */
if (WARN_ON(!!FIELD_GET(ICH_CONTEXTR_EL2_F, cpu_if->vgic_contextr)))
kvm_vm_dead(vcpu->kvm);
}
void vgic_v5_put(struct kvm_vcpu *vcpu)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
/*
* Do nothing if we're not resident. This can happen in the WFI path
* where we do a vgic_put in the WFI path and again later when
* descheduling the thread. We risk losing VMCR state if we sync it
* twice, so instead return early in this case.
*/
if (!cpu_if->gicv5_vpe.resident)
return;
kvm_call_hyp(__vgic_v5_save_apr, cpu_if);
cpu_if->vgic_contextr = 0;
/* Request a doorbell if entering WFI, unless the IRS is disabled */
if (vcpu_get_flag(vcpu, IN_WFI) &&
READ_ONCE(vcpu->kvm->arch.vgic.enabled)) {
u32 priority_mask;
int dbpm;
/*
* Find the virtual running priority and use this to calculate
* the doorbell priority mask. We combine the highest active
* priority and the CPU's priority mask. The guest can't handle
* interrupts with priorities less than or equal to the virtual
* running priority, so there's literally no point in waking the
* guest for these.
*
* The priority needs to be higher than the mask to signal, so
* pick the next higher priority (subtract 1).
*/
priority_mask = vgic_v5_get_effective_priority_mask(vcpu);
/*
* Request a doorbell *unless* the priority is 0, indicating
* that no interrupt can wake the CPU up.
*/
if (priority_mask) {
int db_irq = vgic_v5_vpe_db(vcpu);
struct irq_data *d = irq_get_irq_data(db_irq);
const struct cpumask *aff = irq_data_get_effective_affinity_mask(d);
int cpu = smp_processor_id();
dbpm = priority_mask - 1;
cpu_if->vgic_contextr = FIELD_PREP(ICH_CONTEXTR_EL2_DB, 1) |
FIELD_PREP(ICH_CONTEXTR_EL2_DBPM, dbpm);
/*
* Make the doorbell affine to this CPU, if it isn't
* already. Actively check the cpumask first as it is
* cheaper than changing the affinity every time.
*/
if (!cpumask_test_cpu(cpu, aff)) {
int ret;
ret = irq_set_affinity(db_irq, cpumask_of(cpu));
if (ret)
kvm_debug_ratelimited("Failed to move doorbell IRQ %d to CPU %d: %d\n",
db_irq, cpu, ret);
}
}
}
kvm_call_hyp(__vgic_v5_make_non_resident, cpu_if);
/* The shadow priority is only updated on entering WFI */
if (vcpu_get_flag(vcpu, IN_WFI))
vgic_v5_sync_ppi_priorities(vcpu);
}
void vgic_v5_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
u64 vmcr = cpu_if->vgic_vmcr;
vmcrp->en = FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_EN, vmcr);
vmcrp->pmr = FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_VPMR, vmcr);
}
void vgic_v5_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
u64 vmcr;
vmcr = FIELD_PREP(FEAT_GCIE_ICH_VMCR_EL2_VPMR, vmcrp->pmr) |
FIELD_PREP(FEAT_GCIE_ICH_VMCR_EL2_EN, vmcrp->en);
cpu_if->vgic_vmcr = vmcr;
}
void vgic_v5_restore_state(struct kvm_vcpu *vcpu)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
__vgic_v5_restore_state(cpu_if);
__vgic_v5_restore_ppi_state(cpu_if);
dsb(sy);
}
void vgic_v5_save_state(struct kvm_vcpu *vcpu)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
__vgic_v5_save_state(cpu_if);
__vgic_v5_save_ppi_state(cpu_if);
dsb(sy);
}
static int vgic_v5_irs_status_is_quiesced(u32 statusr)
{
if (statusr & GICV5_IRS_SAVE_VM_STATUSR_Q)
return 0;
return -EBUSY;
}
static int vgic_v5_irs_is_quiesced(u16 vm_id)
{
u32 statusr;
int ret;
/*
* Ensure that the IST snapshot is complete before asking the IRS
* whether the VM remained quiescent while it was copied.
*/
mb();
ret = vgic_v5_irs_save_vm_op(vm_id, false, &statusr);
if (ret)
return ret;
return vgic_v5_irs_status_is_quiesced(statusr);
}
static int vgic_v5_copy_ist_attr(struct kvm_device_attr *attr,
struct kvm_vgic_v5_ist *ist_attr)
{
void __user *uaddr = (void __user *)(unsigned long)attr->addr;
if (!uaddr)
return -EINVAL;
if (copy_from_user(ist_attr, uaddr, sizeof(*ist_attr)))
return -EFAULT;
return 0;
}
static int vgic_v5_validate_ist_user_buffer(__u64 addr, __u64 size,
size_t expected)
{
if (!addr || size != expected)
return -EINVAL;
return 0;
}
static int vgic_v5_validate_ist_attr(struct kvm *kvm,
const struct kvm_vgic_v5_ist *ist_attr)
{
unsigned int id_bits;
int ret;
/* We always have SPIs to save */
ret = vgic_v5_validate_ist_user_buffer(ist_attr->spi_ist_addr,
ist_attr->spi_ist_size,
kvm->arch.vgic.nr_spis * sizeof(__u32));
if (ret)
return ret;
/* We don't always have LPIs to save */
ret = vgic_v5_irs_lpi_ist_id_bits(kvm, &id_bits);
if (ret < 0)
return ret;
/* No LPI IST */
if (!ret) {
if (ist_attr->lpi_ist_addr || ist_attr->lpi_ist_size)
return -EINVAL;
return 0;
}
return vgic_v5_validate_ist_user_buffer(ist_attr->lpi_ist_addr,
ist_attr->lpi_ist_size,
BIT(id_bits) * sizeof(__u32));
}
int vgic_v5_irs_save_ists(struct kvm *kvm, struct kvm_device_attr *attr)
{
struct kvm_vgic_v5_ist ist_attr;
u16 vm_id = vgic_v5_vm_id(kvm);
u32 statusr;
int ret = 0;
mutex_lock(&kvm->lock);
if (kvm_trylock_all_vcpus(kvm)) {
mutex_unlock(&kvm->lock);
return -EBUSY;
}
mutex_lock(&kvm->arch.config_lock);
if (!vgic_initialized(kvm)) {
ret = -EBUSY;
goto out_unlock;
}
ret = vgic_v5_copy_ist_attr(attr, &ist_attr);
if (ret)
goto out_unlock;
ret = vgic_v5_validate_ist_attr(kvm, &ist_attr);
if (ret)
goto out_unlock;
ret = vgic_v5_irs_save_vm_op(vm_id, true, &statusr);
if (ret) {
kvm_err("Failed to save GICv5 IRS VM state: %d\n", ret);
goto out_unlock;
}
ret = vgic_v5_irs_status_is_quiesced(statusr);
if (ret)
goto out_unlock;
/* Save the SPI IST to the userspace buffer. */
ret = vgic_v5_save_spi_ist(kvm, &ist_attr);
if (ret)
goto out_unlock;
ret = vgic_v5_irs_is_quiesced(vm_id);
if (ret)
goto out_unlock;
/* Save the LPI IST to the userspace buffer. */
ret = vgic_v5_save_lpi_ist(kvm, &ist_attr);
if (ret)
goto out_unlock;
ret = vgic_v5_irs_is_quiesced(vm_id);
if (ret)
goto out_unlock;
out_unlock:
mutex_unlock(&kvm->arch.config_lock);
kvm_unlock_all_vcpus(kvm);
mutex_unlock(&kvm->lock);
return ret;
}
/* Allocate the LPI IST to restore into */
static int vgic_v5_restore_lpi_ist_alloc(struct kvm *kvm, bool *allocated)
{
unsigned int id_bits;
int ret;
*allocated = false;
ret = vgic_v5_irs_lpi_ist_id_bits(kvm, &id_bits);
if (ret <= 0)
return ret;
ret = vgic_v5_lpi_ist_alloc(kvm, id_bits);
if (ret)
return ret;
*allocated = true;
return 0;
}
/*
* Clean up the LPI IST if we allocated it, and restore the VMTE to the
* original, valid state.
*/
static void vgic_v5_restore_cleanup(struct kvm *kvm,
struct kvm_vcpu *vcpu,
bool lpi_ist_allocated)
{
/*
* We are on the restore failure path, so we do a best-effort
* cleanup. These commands might fail, but at this stage this is the
* best we can realistically do.
*/
if (lpi_ist_allocated) {
if (!vgic_v5_send_command(vcpu, VMTE_MAKE_INVALID))
vgic_v5_lpi_ist_free(kvm);
}
vgic_v5_send_command(vcpu, VMTE_MAKE_VALID);
}
int vgic_v5_irs_restore_ists(struct kvm *kvm, struct kvm_device_attr *attr)
{
bool lpi_ist_allocated = false, vmte_invalid = false;
struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
struct kvm_vgic_v5_ist ist_attr;
int ret = 0;
mutex_lock(&kvm->lock);
if (kvm_trylock_all_vcpus(kvm)) {
mutex_unlock(&kvm->lock);
return -EBUSY;
}
mutex_lock(&kvm->arch.config_lock);
if (!vgic_initialized(kvm)) {
ret = -EBUSY;
goto out_unlock;
}
if (kvm_vm_has_ran_once(kvm)) {
ret = -EBUSY;
goto out_unlock;
}
ret = vgic_v5_copy_ist_attr(attr, &ist_attr);
if (ret)
goto out_unlock;
ret = vgic_v5_validate_ist_attr(kvm, &ist_attr);
if (ret)
goto out_unlock;
ret = vgic_v5_lpi_ist_exists(kvm);
if (ret) {
if (ret > 0)
ret = -EBUSY;
goto out_unlock;
}
/*
* If the guest has previously allocated an IST (which we check based on
* the IRS_IST_BASER), extract the number of LPI ID bits from the
* IRS_IST_CFGR. Else, do nothing.
*
* We do this before making the VMTE invalid as we rely on
* IRS_VMAP_VISTR to mark the IST as valid in the VMTE. This can only
* happen while the VMTE is valid.
*/
ret = vgic_v5_restore_lpi_ist_alloc(kvm, &lpi_ist_allocated);
if (ret)
goto out_unlock;
/*
* Host ISTs are updated while the VMTE is invalid, so the GIC cannot
* observe partially restored state.
*/
ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_INVALID);
if (ret) {
/*
* If invalidation fails, the restore cannot safely update host
* IST state.
*/
goto out_unlock;
}
vmte_invalid = true;
/* Restore the SPI IST from the userspace buffer. */
ret = vgic_v5_restore_spi_ist(kvm, &ist_attr);
if (ret)
goto out_unlock;
/* Restore the LPI IST from the userspace buffer. */
if (lpi_ist_allocated) {
ret = vgic_v5_restore_lpi_ist(kvm, &ist_attr);
if (ret)
goto out_unlock;
}
/* And make the VM Valid again */
ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_VALID);
if (ret)
goto out_unlock;
vmte_invalid = false;
/*
* As part of restoring the ISTs, and previously pending interrupts have
* been tracked and made non-pending. Now that the ISTs have been
* restored, and the VM is valid again, restore the pending interrupts.
*/
ret = vgic_v5_restore_pending_irqs(kvm);
if (ret)
goto out_unlock;
kvm->arch.vgic.vgic_v5_irs_data->lpi_ist_restore_pending = false;
out_unlock:
if (ret && (vmte_invalid || lpi_ist_allocated)) {
vgic_v5_discard_pending_irqs(kvm);
vgic_v5_restore_cleanup(kvm, vcpu0, lpi_ist_allocated);
}
mutex_unlock(&kvm->arch.config_lock);
kvm_unlock_all_vcpus(kvm);
mutex_unlock(&kvm->lock);
return ret;
}