// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2025, 2026 Arm Ltd.
 */

#include <kvm/arm_vgic.h>
#include <linux/dma-mapping.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/mutex.h>
#include <linux/sizes.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/xarray.h>
#include <asm/kvm_mmu.h>

#include "vgic.h"
#include "vgic-v5-tables.h"

#define irs_caps	kvm_vgic_global_state.vgic_v5_irs_caps

static struct vgic_v5_vmt *vmt_info;

/* Serialises IRS MMIO interaction (commands) and accesses to IRS tables. */
DEFINE_RAW_SPINLOCK(vgic_v5_irs_lock);

/* Serialises lazy installation of shared second-level VMTs. */
static DEFINE_MUTEX(vmt_l2_lock);

static DEFINE_XARRAY(vm_info);

/* Level 1 Virtual Machine Table Entry */
#define GICV5_VMTEL1E_VALID		BIT_ULL(0)
/* Note that there is no shift for the address by design */
#define GICV5_VMTEL1E_L2_ADDR		GENMASK(51, 12)

#define GICV5_VMTEL2E_SIZE		32ULL
/* An L2 table (two-level VMT) is ALWAYS 4kB! */
#define GICV5_VMT_L2_TABLE_SIZE		4096ULL
#define GICV5_VMT_L2_TABLE_ENTRIES	(GICV5_VMT_L2_TABLE_SIZE / GICV5_VMTEL2E_SIZE)

/*
 * As the L2 VMTE is a large data structure, we are splitting it into 4 parts.
 * We only mask and shift WITHIN each part for simplicity.
 */
/* First 64-bit chunk */
#define GICV5_VMTEL2E_VALID		BIT_ULL(0)
#define GICV5_VMTEL2E_VMD_ADDR_SHIFT	3ULL
#define GICV5_VMTEL2E_VMD_ADDR		GENMASK_ULL(55, 3)
/* Second 64-bit chunk */
#define GICV5_VMTEL2E_VPET_ADDR_SHIFT	3ULL
#define GICV5_VMTEL2E_VPET_ADDR		GENMASK_ULL(55, 3)
#define GICV5_VMTEL2E_VPE_ID_BITS	GENMASK_ULL(63, 59)
/* Third & fourth 64-bit chunks (the encodings are the same for each) */
#define GICV5_VMTEL2E_IST_VALID		BIT_ULL(0)
#define GICV5_VMTEL2E_IST_L2SZ		GENMASK_ULL(2, 1)
#define GICV5_VMTEL2E_IST_ADDR_SHIFT	6ULL
#define GICV5_VMTEL2E_IST_ADDR		GENMASK_ULL(55, 6)
#define GICV5_VMTEL2E_IST_ISTSZ		GENMASK_ULL(57, 56)
#define GICV5_VMTEL2E_IST_STRUCTURE	BIT_ULL(58)
#define GICV5_VMTEL2E_IST_ID_BITS	GENMASK_ULL(63, 59)

/* Virtual PE Table Entry */
#define GICV5_VPE_VALID			BIT_ULL(0)
/* Note that there is no shift for the address by design. */
#define GICV5_VPED_ADDR			GENMASK_ULL(55, 3)

/* L2 Interrupt State Table Entry */
#define GICV5_ISTL2E_PENDING		BIT(0)
#define GICV5_ISTL2E_ACTIVE		BIT(1)
#define GICV5_ISTL2E_HM			BIT(2)
#define GICV5_ISTL2E_ENABLE		BIT(3)
#define GICV5_ISTL2E_IRM		BIT(4)
#define GICV5_ISTL2E_HWU		GENMASK(10, 9)
#define GICV5_ISTL2E_PRIORITY		GENMASK(15, 11)
#define GICV5_ISTL2E_IAFFID		GENMASK(31, 16)

#define GICV5_ISTE_SIZE(istsz)		BIT((istsz) + 2)
#define GICV5_LINEAR_IST_SIZE(id_bits, istsz)	\
	(BIT(id_bits) * GICV5_ISTE_SIZE(istsz))

/*
 * The LPI and SPI configuration is stored in the 2nd and 3rd 64-bit chunks of
 * the VMTE (0-based). We call this a section here in an attempt to simplify the
 * code.
 */
#define GICV5_VMTEL2_LPI_SECTION	2
#define GICV5_VMTEL2_SPI_SECTION	3

struct vgic_v5_ist_desc {
	struct vgic_v5_vm_info	*vmi;
	void			*base;
	unsigned int		id_bits;
	unsigned int		istsz;
	unsigned int		l2sz;
	size_t			iste_size;
	bool			present;
};

struct vgic_v5_two_level_ist_shape {
	size_t	l1_entries;
	size_t	l2_entries;
};

struct vgic_v5_pending_irq {
	u32			irq;
	struct list_head	next;
};

static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist,
				    unsigned int id_bits,
				    unsigned int istsz);
static int vgic_v5_alloc_l1_ist(struct kvm *kvm, unsigned int id_bits,
				unsigned int istsz, unsigned int l2_split);
static int vgic_v5_alloc_l2_ists(struct kvm *kvm, unsigned int id_bits,
				 unsigned int istsz, unsigned int l2_split);
static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm,
					   unsigned int id_bits,
					   unsigned int istsz,
					   unsigned int l2_split);
static int vgic_v5_linear_ist_free(struct kvm *kvm, bool spi);
static int vgic_v5_two_level_ist_free(struct kvm *kvm, bool spi);
static int vgic_v5_spi_ist_free(struct kvm *kvm);

/*
 * Our IRS might be coherent or non-coherent. If coherent, we can just emit a
 * DSB to ensure that we're in sync. However, when non-coherent, we need to
 * manage our cached data explicitly.
 *
 * This helper is used to handle both coherent and non-coherent IRSes, and
 * handles all combinations of cleaning and invalidating to the PoC.
 */
static void vgic_v5_clean_inval(void *va, size_t size)
{
	unsigned long base = (unsigned long)va;

	dsb(ishst);

	if (kvm_vgic_global_state.vgic_v5_irs_caps.non_coherent)
		dcache_clean_inval_poc(base, base + size);
}

static void vgic_v5_drain_pending_irqs(struct kvm *kvm,
				       struct vgic_v5_vm_info *vmi,
				       bool reinject)
{
	struct vgic_v5_pending_irq *pirq, *tmp;

	list_for_each_entry_safe(pirq, tmp, &vmi->pending_irqs, next) {
		if (reinject)
			kvm_call_hyp(__vgic_v5_vdpend, pirq->irq, true,
				     vgic_v5_vm_id(kvm));

		list_del(&pirq->next);
		kfree(pirq);
	}
}

/*
 * Create a linear VM Table. Directly using the number of entries supplied as
 * the size of an L2 VMTE (32 bytes) guarantees that our allocation is aligned per
 * the GICv5 requirements for the IRS_VMT_BASER.
 */
static int vgic_v5_alloc_vmt_linear(unsigned int num_entries)
{
	vmt_info->linear.vmt_base = kzalloc_objs(*vmt_info->linear.vmt_base,
						 num_entries);
	if (!vmt_info->linear.vmt_base)
		return -ENOMEM;

	vgic_v5_clean_inval(vmt_info->linear.vmt_base,
			    num_entries * sizeof(struct vmtl2_entry));

	return 0;
}

/*
 * Allocate the first level of a two-level VM table. The second-level VM tables
 * are allocated on demand (by vgic_v5_alloc_l2_vmt()).
 */
static int vgic_v5_alloc_vmt_two_level(unsigned int num_entries)
{
	/*
	 * Each L2 VMT array is always 4k-sized (covering 128 VMs). This is
	 * mandated by the GICv5 specification (GICv5 EAC0 Specification rule
	 * D_LSPBK). Hence, round up the number of entries to be at least 128
	 * (or the next highest power of two as we give the HW the number of VM
	 * ID bits).
	 */
	if (num_entries < GICV5_VMT_L2_TABLE_ENTRIES)
		num_entries = GICV5_VMT_L2_TABLE_ENTRIES;
	num_entries = roundup_pow_of_two(num_entries);

	vmt_info->l2.num_l1_ents = (num_entries / GICV5_VMT_L2_TABLE_ENTRIES);
	vmt_info->l2.vmt_base = kzalloc_objs(*vmt_info->l2.vmt_base,
					     vmt_info->l2.num_l1_ents);
	if (!vmt_info->l2.vmt_base)
		return -ENOMEM;

	vmt_info->l2.l2ptrs = kzalloc_objs(*vmt_info->l2.l2ptrs,
					   vmt_info->l2.num_l1_ents);
	if (!vmt_info->l2.l2ptrs) {
		kfree(vmt_info->l2.vmt_base);
		return -ENOMEM;
	}

	vgic_v5_clean_inval(vmt_info->l2.vmt_base,
			    vmt_info->l2.num_l1_ents * sizeof(vmtl1_entry));

	return 0;
}

/*
 * Allocate a second level VMT, if required. This can be called eagerly, and
 * will only perform the allocation if required.
 */
static int vgic_v5_alloc_l2_vmt(struct kvm *kvm)
{
	struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
	u32 vm_id = vgic_v5_vm_id(kvm);
	enum gicv5_vcpu_cmd cmd = VMT_L2_MAP;
	struct vmtl2_entry *l2_table;
	unsigned int l1_index;
	int ret;

	/* Nothing to do if we have linear tables! */
	if (!vmt_info->two_level)
		return 0;

	if (vm_id == VGIC_V5_VM_ID_INVAL)
		return -EINVAL;

	/*
	 * We have 4k-sized L2 tables - this is mandated by the spec for
	 * two-level VMTs (GICv5 EAC0 Specification rule D_LSPBK). This means
	 * that we have 128 entries per L1 VMTE.
	 */
	l1_index = vm_id / GICV5_VMT_L2_TABLE_ENTRIES;

	guard(mutex)(&vmt_l2_lock);

	/* Already valid? Great! */
	if (vmt_info->l2.l2ptrs[l1_index])
		return 0;

	l2_table = kzalloc_objs(*l2_table, GICV5_VMT_L2_TABLE_ENTRIES);
	if (!l2_table)
		return -ENOMEM;

	/* The VMT is shared between all VMs. */
	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
		vgic_v5_clean_inval(l2_table, GICV5_VMT_L2_TABLE_SIZE);
		vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index,
				    sizeof(vmtl1_entry));

		WRITE_ONCE(vmt_info->l2.vmt_base[l1_index],
			   cpu_to_le64(virt_to_phys(l2_table)));

		vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index,
				    sizeof(vmtl1_entry));

	}

	/*
	 * VMAP in the L2 VMT via the IRS. We use any of the VM's CPUs as a
	 * conduit for interacting with the host's IRS. In the current case,
	 * this lets us resolve the VM ID to pass to the hardware.
	 */
	ret = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu0), &cmd);

	/* We've failed to make the L2 VMT valid - things are very broken! */
	if (ret) {
		scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
			/* Remove the pointer from L1 table */
			WRITE_ONCE(vmt_info->l2.vmt_base[l1_index], 0);

			vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index,
					    sizeof(vmtl1_entry));
		}

		kfree(l2_table);
		return ret;
	}

	vmt_info->l2.l2ptrs[l1_index] = l2_table;

	return 0;
}

/*
 * Allocate the top-level VMT. This can either be linear or two-level.
 */
int vgic_v5_vmt_allocate(unsigned int max_vpes)
{
	int ret;

	/* Allocate the tracking structure */
	vmt_info = kzalloc_obj(*vmt_info);
	if (!vmt_info)
		return -ENOMEM;

	ida_init(&vmt_info->vm_id_ida);
	vmt_info->max_vpes = max_vpes;
	vmt_info->vmd_size = vgic_v5_irs_vmd_size(&irs_caps);
	vmt_info->vped_size = vgic_v5_irs_vped_size(&irs_caps);
	vmt_info->two_level = vgic_v5_irs_two_level_vmt_support(&irs_caps);
	vmt_info->num_entries = vgic_v5_irs_max_vms(&irs_caps);

	if (vmt_info->two_level)
		ret = vgic_v5_alloc_vmt_two_level(vmt_info->num_entries);
	else
		ret = vgic_v5_alloc_vmt_linear(vmt_info->num_entries);

	/* If anything failed, free our tracking structure before returning */
	if (ret) {
		kfree(vmt_info);
		vmt_info = NULL;
	}

	return ret;
}

/*
 * Free the VMT and associated tracking structures. This isn't strictly expected
 * to be called in general operation, but instead exists for completeness.
 */
int vgic_v5_vmt_free(void)
{
	if (!vmt_info)
		return 0;

	if (!vmt_info->two_level) {
		kfree(vmt_info->linear.vmt_base);
	} else {
		/* Free the L2 tables; kfree(NULL) is safe */
		for (int i = 0; i < vmt_info->l2.num_l1_ents; ++i)
			kfree(vmt_info->l2.l2ptrs[i]);
		kfree(vmt_info->l2.l2ptrs);

		/* And now free the L1 table */
		kfree(vmt_info->l2.vmt_base);
	}

	ida_destroy(&vmt_info->vm_id_ida);
	kfree(vmt_info);
	vmt_info = NULL;

	return 0;
}

/*
 * Look up a VMT Entry by VM ID.
 */
static struct vmtl2_entry *vgic_v5_get_l2_vmte(u32 vm_id)
{
	unsigned int l1_index, l2_index;
	struct vmtl2_entry *l2_table;

	if (vm_id == VGIC_V5_VM_ID_INVAL)
		return ERR_PTR(-EINVAL);

	if (!vmt_info->two_level)
		return &vmt_info->linear.vmt_base[vm_id];

	l1_index = vm_id / GICV5_VMT_L2_TABLE_ENTRIES;
	l2_index = vm_id % GICV5_VMT_L2_TABLE_ENTRIES;

	if (l1_index >= vmt_info->l2.num_l1_ents)
		return ERR_PTR(-E2BIG);

	if (!vmt_info->l2.l2ptrs[l1_index])
		return ERR_PTR(-EINVAL);

	l2_table = vmt_info->l2.l2ptrs[l1_index];
	return &l2_table[l2_index];
}

/*
 * Zero a VMT Entry, and flush & invalidate to the PoC, if required.
 */
static int vgic_v5_reset_vmte(struct kvm *kvm)
{
	u32 vm_id = vgic_v5_vm_id(kvm);
	struct vmtl2_entry *vmte;

	vmte = vgic_v5_get_l2_vmte(vm_id);
	if (IS_ERR(vmte))
		return PTR_ERR(vmte);

	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
		/*
		 * The VMT is normal memory shared with the IRS. Invalidate before
		 * rewriting the entry so that cacheline-granular maintenance cannot
		 * later push stale data for neighbouring IRS-visible state back to
		 * memory.
		 */
		vgic_v5_clean_inval(vmte, sizeof(*vmte));

		/*
		 * Prevent the compiler from eliding the individual VMTE
		 * stores. Ordering and visibility to the IRS are provided by the
		 * surrounding cache maintenance and command protocol, not by
		 * WRITE_ONCE().
		 *
		 * The same compiler-access constraint applies to READ_ONCE() users in
		 * this file: when inspecting IRS-visible table entries, read the field
		 * exactly once and prevent the compiler from reusing, merging or
		 * tearing the access. Coherency and freshness for non-coherent IRSes
		 * still come from the surrounding cache maintenance.
		 */
		WRITE_ONCE(vmte->val[0], cpu_to_le64(0ULL));
		WRITE_ONCE(vmte->val[1], cpu_to_le64(0ULL));
		WRITE_ONCE(vmte->val[2], cpu_to_le64(0ULL));
		WRITE_ONCE(vmte->val[3], cpu_to_le64(0ULL));

		/* And make our write visible to the IRS (if non-coherent) */
		vgic_v5_clean_inval(vmte, sizeof(*vmte));
	}

	return 0;
}

/*
 * Use the IDA to allocate a new VM ID, and track it in the gicv5_vm data
 * structure. If we're out of VM IDs, the IDA catches that, and we return the
 * error (-ENOSPC). If we've previously allocated a VM ID, we catch that too and
 * return -EBUSY.
 */
int vgic_v5_allocate_vm_id(struct kvm *kvm)
{
	int id;

	if (kvm->arch.vgic.gicv5_vm.vm_id != VGIC_V5_VM_ID_INVAL)
		return -EBUSY;

	id = ida_alloc_max(&vmt_info->vm_id_ida, vmt_info->num_entries - 1u,
			   GFP_KERNEL);
	if (id < 0)
		return id;

	kvm->arch.vgic.gicv5_vm.vm_id = id;

	return 0;
}

/*
 * Release the VM ID to allow it to be reallocated in the future.
 */
void vgic_v5_release_vm_id(struct kvm *kvm)
{
	if (kvm->arch.vgic.gicv5_vm.vm_id == VGIC_V5_VM_ID_INVAL)
		return;

	ida_free(&vmt_info->vm_id_ida, kvm->arch.vgic.gicv5_vm.vm_id);
	kvm->arch.vgic.gicv5_vm.vm_id = VGIC_V5_VM_ID_INVAL;
}

/*
 * Initialise an entry in the VMT based on the index of the VM.
 *
 * Note: We don't mark the VMTE as valid as this needs to be done by
 * the hardware.
 */
int vgic_v5_vmte_init(struct kvm *kvm)
{
	size_t vmd_alloc_size, vpet_alloc_size, vped_alloc_size;
	void *vped_base = NULL, *vmd = NULL;
	struct vgic_v5_vm_info *vmi = NULL;
	u64 tmp, vmte_val0 = 0, vmte_val1;
	u32 vm_id = vgic_v5_vm_id(kvm);
	int ret, nr_cpus, nr_vcpus;
	bool vmi_inserted = false;
	struct vmtl2_entry *vmte;
	vpe_entry *vpet = NULL;
	struct kvm_vcpu *vcpu;
	u16 max_vpe_id = 0;
	unsigned long i;

	nr_vcpus = atomic_read(&kvm->online_vcpus);
	if (nr_vcpus > vmt_info->max_vpes)
		return -E2BIG;

	/*
	 * If we're using two-level VMTs, L2 is allocated on demand. For linear
	 * VMTs, this is a NOP.
	 */
	ret = vgic_v5_alloc_l2_vmt(kvm);
	if (ret)
		return ret;

	vmte = vgic_v5_get_l2_vmte(vm_id);
	if (IS_ERR(vmte))
		return PTR_ERR(vmte);

	/* If the entry is already valid, something went wrong */
	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
		vgic_v5_clean_inval(vmte, sizeof(*vmte));
		if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID)
			return -EINVAL;
	}

	ret = vgic_v5_reset_vmte(kvm);
	if (ret)
		return ret;

	vmi = kzalloc_obj(*vmi);
	if (!vmi) {
		ret = -ENOMEM;
		goto out_fail;
	}

	ret = xa_insert(&vm_info, vm_id, vmi, GFP_KERNEL);
	if (ret)
		goto out_fail;
	vmi_inserted = true;

	/*
	 * If we are restoring the state of a guest, we need to re-inject any
	 * IRQs that were pending when the state of the guest was originally
	 * saved. We use the pending_irqs list for this.
	 */
	INIT_LIST_HEAD(&vmi->pending_irqs);

	/* Allocate and assign the VM Descriptor, if required. */
	if (vmt_info->vmd_size != 0) {
		vmd_alloc_size = round_up(vmt_info->vmd_size,
					  dma_get_cache_alignment());
		vmd = kzalloc(vmd_alloc_size, GFP_KERNEL);
		if (!vmd) {
			ret = -ENOMEM;
			goto out_fail;
		}

		/* Stash the VA so we can free it later */
		vmi->vmd_base = vmd;

		tmp = FIELD_PREP(GICV5_VMTEL2E_VMD_ADDR,
				 virt_to_phys(vmd) >> GICV5_VMTEL2E_VMD_ADDR_SHIFT);
		vmte_val0 = tmp;
	}

	/*
	 * Allocate and assign the VPE Table.
	 *
	 * First of all, iterate over all vcpus to find the highest VPE ID we
	 * require - we need to ensure that we have enough storage for all
	 * vcpu_id values that userspace has picked and not just the total
	 * number of vcpus. This gives us the number of VPEs required for the
	 * VM.
	 *
	 * Round up the number of VPEs to a whole power of two as we cannot
	 * describe non-powers-of-two in the VMTE field as it conveys the number
	 * of ID bits used and not the number of vPEs. IRS_IDR1.IAFFID_BITS is
	 * encoded as N - 1, so expose at least one VPE ID bit even for a
	 * single-vCPU VM to keep the views consistent.
	 */
	kvm_for_each_vcpu(i, vcpu, kvm) {
		u16 vpe_id = vgic_v5_vpe_id(vcpu);

		if (vpe_id > max_vpe_id)
			max_vpe_id = vpe_id;
	}

	nr_cpus = max(2UL, roundup_pow_of_two(max_vpe_id + 1));
	vmi->vpe_id_bits = fls(nr_cpus) - 1;

	vpet_alloc_size = round_up((size_t)nr_cpus * sizeof(*vpet),
				   dma_get_cache_alignment());
	vpet = kzalloc(vpet_alloc_size, GFP_KERNEL);
	if (!vpet) {
		ret = -ENOMEM;
		goto out_fail;
	}

	/* Stash the VA so we can free it later */
	vmi->vpet_base = vpet;

	tmp = FIELD_PREP(GICV5_VMTEL2E_VPET_ADDR,
			 virt_to_phys(vpet) >> GICV5_VMTEL2E_VPET_ADDR_SHIFT);
	tmp |= FIELD_PREP(GICV5_VMTEL2E_VPE_ID_BITS, vmi->vpe_id_bits);
	vmte_val1 = tmp;

	/*
	 * Allocate a dense VPED array indexed by vcpu_idx. The VPET is indexed
	 * by the potentially sparse vcpu_id, but using that ID here would waste
	 * memory. Given that this is not userspace visible, we can cheat a bit
	 * and use the dense index instead. This is the ONLY place that we do
	 * this.
	 *
	 * Round the requested size up to a whole cacheline. kzalloc() gurantees
	 * natural alignment, so we ensure that the cachelines cannot be shared
	 * with unrelated slab objects. VPED and cacheline sizes are powers of
	 * two, so this also preserves the required VPED alignment when a VPED
	 * is larger than a cacheline.
	 */
	vped_alloc_size = round_up((size_t)nr_vcpus * vmt_info->vped_size,
				   dma_get_cache_alignment());
	vped_base = kzalloc(vped_alloc_size, GFP_KERNEL);
	if (!vped_base) {
		ret = -ENOMEM;
		goto out_fail;
	}
	vmi->vped_base = vped_base;

	if (vmd)
		vgic_v5_clean_inval(vmd, vmd_alloc_size);
	vgic_v5_clean_inval(vpet, vpet_alloc_size);
	vgic_v5_clean_inval(vped_base, vped_alloc_size);

	/* Publish the VMTE while serialising access to the shared VMT. */
	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
		WRITE_ONCE(vmte->val[0], cpu_to_le64(vmte_val0));
		WRITE_ONCE(vmte->val[1], cpu_to_le64(vmte_val1));
		vgic_v5_clean_inval(vmte, sizeof(*vmte));
	}

	kvm->arch.vgic.gicv5_vm.vmte_allocated = true;

	return 0;

out_fail:
	/* kfree(NULL) is safe so we can just kfree() at leisure */
	kfree(vmd);
	kfree(vpet);
	kfree(vped_base);
	if (vmi_inserted)
		xa_erase(&vm_info, vm_id);
	kfree(vmi);

	vgic_v5_reset_vmte(kvm);

	return ret;
}

/*
 * Release the VMT Entry, freeing up any allocated data structures before
 * zeroing the VMTE.
 *
 * The VMTE must be marked as invalid before it is released.
 */
int vgic_v5_vmte_release(struct kvm *kvm)
{
	u32 vm_id = vgic_v5_vm_id(kvm);
	struct vgic_v5_vm_info *vmi;
	struct vmtl2_entry *vmte;
	int ret;

	vmte = vgic_v5_get_l2_vmte(vm_id);
	if (IS_ERR(vmte))
		return PTR_ERR(vmte);

	/* Reject if the VMTE has not been marked as invalid! */
	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
		vgic_v5_clean_inval(vmte, sizeof(*vmte));
		if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID)
			return -EINVAL;
	}

	vmi = xa_load(&vm_info, vm_id);
	if (!vmi)
		goto no_vmi;

	/* Unlikely, but possible. Avoid leaking the memory. */
	vgic_v5_drain_pending_irqs(kvm, vmi, false);

	/* If we have an LPI IST, free it */
	if (vmi->h_lpi_ist) {
		ret = vgic_v5_lpi_ist_free(kvm);
		if (ret)
			return ret;
	}
	vmi->h_lpi_ist = NULL;

	/* If we have an SPI IST, free it */
	if (vmi->h_spi_ist) {
		ret = vgic_v5_spi_ist_free(kvm);
		if (ret)
			return ret;
	}
	vmi->h_spi_ist = NULL;

	kfree(vmi->vped_base);
	kfree(vmi->vpet_base);
	kfree(vmi->vmd_base);

	xa_erase(&vm_info, vm_id);
	kfree(vmi);

no_vmi:
	/*
	 * If we didn't get far enough into allocating a VMTE to create the VM
	 * info structure, then we just zero the VMTE and move on. There's
	 * nothing else we can realistically do here.
	 */
	ret = vgic_v5_reset_vmte(kvm);
	if (ret)
		return ret;

	kvm->arch.vgic.gicv5_vm.vmte_allocated = false;

	return 0;
}

/*
 * Provide a way for the IRS MMIO emulation to correctly populate the number of
 * IAFFID bits (which correspond to our vpe_id_bits.
 */
u8 vgic_v5_vmte_vpe_id_bits(struct kvm_vcpu *vcpu)
{
	u32 vm_id = vgic_v5_vm_id(vcpu->kvm);
	struct vgic_v5_vm_info *vmi;

	vmi = xa_load(&vm_info, vm_id);
	if (!vmi)
		return 0;

	return vmi->vpe_id_bits;
}

/* Provide the preallocated VPE descriptor to the hardware via the VPE Table. */
int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu)
{
	u32 vm_id = vgic_v5_vm_id(vcpu->kvm);
	u16 vpe_id = vgic_v5_vpe_id(vcpu);
	struct vgic_v5_vm_info *vmi;
	vpe_entry *vpet_base;
	void *vped;

	/* Make sure we're not over what the hardware supports */
	if (vpe_id >= vmt_info->max_vpes)
		return -E2BIG;

	vmi = xa_load(&vm_info, vm_id);
	if (!vmi)
		return -EINVAL;

	if (vpe_id >= 1 << vmi->vpe_id_bits)
		return -E2BIG;

	vpet_base = vmi->vpet_base;

	/* If the VPETE for this CPU is already valid we've gone wrong */
	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
		vgic_v5_clean_inval(&vpet_base[vpe_id], sizeof(*vpet_base));
		if (le64_to_cpu(READ_ONCE(vpet_base[vpe_id])) & GICV5_VPE_VALID)
			return -EBUSY;
	}

	vped = (u8 *)vmi->vped_base +
		(size_t)vcpu->vcpu_idx * vmt_info->vped_size;

	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
		phys_addr_t paddr = virt_to_phys(vped) & GICV5_VPED_ADDR;
		WRITE_ONCE(vpet_base[vpe_id], cpu_to_le64(paddr));
		vgic_v5_clean_inval(vpet_base + vpe_id, sizeof(vpe_entry));
	}

	return 0;
}

/* Clear the VPE's table entry after the VMTE has been made invalid. */
int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu)
{
	u32 vm_id = vgic_v5_vm_id(vcpu->kvm);
	u16 vpe_id = vgic_v5_vpe_id(vcpu);
	struct vgic_v5_vm_info *vmi;
	struct vmtl2_entry *vmte;
	vpe_entry *vpet_base;

	vmte = vgic_v5_get_l2_vmte(vm_id);
	if (IS_ERR(vmte))
		return PTR_ERR(vmte);

	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
		vgic_v5_clean_inval(vmte, sizeof(*vmte));
		if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID)
			return -EBUSY;
	}

	vmi = xa_load(&vm_info, vm_id);
	if (!vmi)
		return -EINVAL;

	if (vpe_id >= 1 << vmi->vpe_id_bits)
		return -E2BIG;

	vpet_base = vmi->vpet_base;
	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
		WRITE_ONCE(vpet_base[vpe_id], 0ULL);
		vgic_v5_clean_inval(vpet_base + vpe_id, sizeof(vpe_entry));
	}

	return 0;
}

phys_addr_t vgic_v5_get_vmt_base(void)
{
	phys_addr_t vmt_base;

	if (!vmt_info->two_level)
		vmt_base = virt_to_phys(vmt_info->linear.vmt_base);
	else
		vmt_base = virt_to_phys(vmt_info->l2.vmt_base);

	return vmt_base;
}

/*
 * Assign an already allocated IST to the VM by populating the fields in the
 * corresponding VMTE. We re-use this code for both an SPI IST and LPI IST, even
 * if the paths to reach it might be vastly different.
 */
static int vgic_v5_vmte_assign_ist(struct kvm *kvm, phys_addr_t ist_base,
				   bool two_level, unsigned int id_bits,
				   unsigned int l2sz, unsigned int istsz,
				   bool spi_ist)
{
	struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
	u32 vm_id = vgic_v5_vm_id(kvm);
	enum gicv5_vcpu_cmd cmd;
	struct vmtl2_entry *vmte;
	unsigned int section;
	u64 tmp;
	int ret;

	/*
	 * The L2 VMTE comprises four 64-bit "sections", where sections 2 & 3
	 * describe the LPI and SPI ISTs, respectively. Both the LPI and SPI
	 * sections have the same layout, and as we are either operating on SPIs
	 * or LPIs we pick a section of the VMTE to modify up-front.
	 *
	 * See the GICv5 EAC0 Specification 11.2.2 for more details about the
	 * VMTE layout.
	 */
	section = spi_ist ? GICV5_VMTEL2_SPI_SECTION : GICV5_VMTEL2_LPI_SECTION;

	if (ist_base & ~GICV5_VMTEL2E_IST_ADDR) {
		pr_err_ratelimited("kvm [%i]: IST misaligned: address 0x%llx, mask 0x%llx\n",
				   task_pid_nr(current), ist_base,
				   GICV5_VMTEL2E_IST_ADDR);
		return -EINVAL;
	}

	vmte = vgic_v5_get_l2_vmte(vm_id);
	if (IS_ERR(vmte))
		return PTR_ERR(vmte);

	tmp = FIELD_PREP(GICV5_VMTEL2E_IST_L2SZ, l2sz);
	tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ADDR,
			ist_base >> GICV5_VMTEL2E_IST_ADDR_SHIFT);
	tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ISTSZ, istsz);
	tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ID_BITS, id_bits);
	if (two_level)
		tmp |= GICV5_VMTEL2E_IST_STRUCTURE;

	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
		/* Bail if already allocated */
		vgic_v5_clean_inval(vmte, sizeof(*vmte));
		if (le64_to_cpu(READ_ONCE(vmte->val[section])) &
		    GICV5_VMTEL2E_IST_VALID)
			return -EINVAL;

		WRITE_ONCE(vmte->val[section], cpu_to_le64(tmp));
		vgic_v5_clean_inval(vmte, sizeof(*vmte));
	}

	/* Finally, mark the entry as valid */
	cmd = spi_ist ? SPI_VIST_MAKE_VALID : LPI_VIST_MAKE_VALID;
	ret = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu0), &cmd);
	if (ret) {
		/*
		 * A timeout does not tell us whether the IRS consumed the
		 * command, so we're in some indeterminate and potentially
		 * transient state.  It may still make the VIST valid and access
		 * the backing memory, so leave both the VMTE and allocation
		 * intact. Killing the VM ensures that teardown is the only path
		 * that can reclaim them, after it has successfully made the
		 * complete VMTE invalid.
		 */
		if (ret == -ETIMEDOUT) {
			kvm_vm_dead(kvm);
			return ret;
		}

		scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
			WRITE_ONCE(vmte->val[section], 0ULL);
			vgic_v5_clean_inval(vmte, sizeof(*vmte));
		}

		return ret;
	}

	return 0;
}

/*
 * Allocate a Linear IST - always used for SPIs and potentially LPIs.
 *
 * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0
 * spec.
 *
 * NOTE: istsz is the FIELD used by GICv5, not the actual size (or log2() of the
 * size).
 */
static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist,
				    unsigned int id_bits, unsigned int istsz)
{
	const size_t n = max(5, id_bits + 1 + istsz);
	u32 vm_id = vgic_v5_vm_id(kvm);
	struct vgic_v5_vm_info *vmi;
	__le64 *ist;
	u32 l1sz;

	vmi = xa_load(&vm_info, vm_id);
	if (!vmi)
		return -EINVAL;

	/*
	 * Allocate the IST. We only have one level, so we just use the L2 ISTE.
	 */
	l1sz = BIT(n + 1);
	ist = kzalloc(l1sz, GFP_KERNEL_ACCOUNT);
	if (!ist)
		return -ENOMEM;

	if (spi_ist) {
		vmi->h_spi_ist = ist;
	} else {
		vmi->h_lpi_ist_structure = false;
		vmi->h_lpi_ist = ist;
	}

	vgic_v5_clean_inval(ist, l1sz);

	return 0;
}

/*
 * Allocate the first level of a two-level IST - LPI, only.
 *
 * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0
 * spec.
 *
 * NOTE: istsz and l2sz are the FIELDS used by GICv5, not the actual sizes (or
 * log2() of the sizes).
 */
static int vgic_v5_alloc_l1_ist(struct kvm *kvm, unsigned int id_bits,
				unsigned int istsz, unsigned int l2sz)
{
	const size_t n =  max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1);
	u32 vm_id = vgic_v5_vm_id(kvm);
	const u32 l1_size = BIT(n + 1);
	struct vgic_v5_vm_info *vmi;
	__le64 *ist;

	vmi = xa_load(&vm_info, vm_id);
	if (!vmi)
		return -EINVAL;

	ist = kzalloc(l1_size, GFP_KERNEL_ACCOUNT);
	if (!ist)
		return -ENOMEM;

	vmi->h_lpi_ist_structure = true;
	vmi->h_lpi_ist = ist;

	vgic_v5_clean_inval(ist, l1_size);

	return 0;
}

/*
 * Allocate ALL of the second level ISTs for a two-level IST - LPI, only.
 *
 * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0
 * spec. The l2_size calculation is from section 11.2.3 of the same document.
 *
 * NOTE: istsz and l2sz are the FIELDS used by GICv5, not the actual sizes (or
 * log2() of the sizes).
 */
static int vgic_v5_alloc_l2_ists(struct kvm *kvm, unsigned int id_bits,
				 unsigned int istsz, unsigned int l2sz)
{
	const size_t n =  max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1);
	const int l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE;
	const size_t l2_size = BIT(11 + (2 * l2sz) + 1);
	u32 vm_id = vgic_v5_vm_id(kvm);
	struct vgic_v5_vm_info *vmi;
	__le64 *l2ist;
	__le64 *l1ist;
	int index;
	u64 val;

	vmi = xa_load(&vm_info, vm_id);
	if (!vmi)
		return -EINVAL;

	l1ist = vmi->h_lpi_ist;

	/*
	 * Allocate the storage for the pointers to the L2 ISTs (used when
	 * freeing later).
	 */
	vmi->h_lpi_l2_ists = kzalloc_objs(*vmi->h_lpi_l2_ists, l1_entries,
					  GFP_KERNEL_ACCOUNT);
	if (!vmi->h_lpi_l2_ists)
		return -ENOMEM;

	/* Allocate the L2 IST for each L1 IST entry */
	for (index = 0; index < l1_entries; ++index) {
		l2ist = kzalloc(l2_size, GFP_KERNEL_ACCOUNT);
		if (!l2ist) {
			while (--index >= 0)
				kfree(vmi->h_lpi_l2_ists[index]);

			kfree(vmi->h_lpi_l2_ists);
			vmi->h_lpi_l2_ists = NULL;

			return -ENOMEM;
		}

		/*
		 * We are not doing on-demand allocation of the L2 ISTs, and are
		 * instead provisioning the whole IST up front. This means that
		 * we are able to mark the L2 ISTs as valid in the L1 ISTEs as
		 * the overall IST is not yet valid.
		 */
		val = (virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK) |
		      GICV5_ISTL1E_VALID;
		l1ist[index] = cpu_to_le64(val);

		vmi->h_lpi_l2_ists[index] = l2ist;

		vgic_v5_clean_inval(l2ist, l2_size);
	}

	/* Handle CMOs for the whole L1 IST in one go */
	vgic_v5_clean_inval(l1ist, l1_entries * sizeof(*l1ist));

	return 0;
}

/* Allocate a two-level IST - LPIs, only */
static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm, unsigned int id_bits,
					   unsigned int istsz, unsigned int l2sz)
{
	u32 vm_id = vgic_v5_vm_id(kvm);
	struct vgic_v5_vm_info *vmi;
	int ret;

	/*
	 * Allocate the L1 IST first, then all of the L2s. Everything
	 * is preallocated and we do no on-demand IST allocation. This
	 * is to avoid needing to track if and when the guest is doing
	 * on-demand IST allocation.
	 */
	ret = vgic_v5_alloc_l1_ist(kvm, id_bits, istsz, l2sz);
	if (ret)
		return ret;

	ret = vgic_v5_alloc_l2_ists(kvm, id_bits, istsz, l2sz);
	if (ret) {
		/* Free the L1 IST again */
		vmi = xa_load(&vm_info, vm_id);
		kfree(vmi->h_lpi_ist);
		vmi->h_lpi_ist = NULL;

		return ret;
	}

	return 0;
}

static void vgic_v5_free_allocated_lpi_ist(struct vgic_v5_vm_info *vmi,
					   unsigned int id_bits,
					   unsigned int istsz,
					   unsigned int l2sz)
{
	if (!vmi->h_lpi_ist_structure) {
		kfree(vmi->h_lpi_ist);
		vmi->h_lpi_ist = NULL;
		return;
	}

	if (vmi->h_lpi_l2_ists) {
		const size_t n = max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1);
		const int l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE;
		int index;

		for (index = 0; index < l1_entries; ++index)
			kfree(vmi->h_lpi_l2_ists[index]);

		kfree(vmi->h_lpi_l2_ists);
		vmi->h_lpi_l2_ists = NULL;
	}

	kfree(vmi->h_lpi_ist);
	vmi->h_lpi_ist = NULL;
}

static void vgic_v5_free_allocated_spi_ist(struct kvm *kvm)
{
	u32 vm_id = vgic_v5_vm_id(kvm);
	struct vgic_v5_vm_info *vmi;

	vmi = xa_load(&vm_info, vm_id);
	if (!vmi)
		return;

	kfree(vmi->h_spi_ist);
	vmi->h_spi_ist = NULL;
}

/*
 * Free a Linear IST. Can only happen once the VM is dead.
 */
static int vgic_v5_linear_ist_free(struct kvm *kvm, bool spi)
{
	u32 vm_id = vgic_v5_vm_id(kvm);
	struct vmtl2_entry *vmte;
	struct vgic_v5_vm_info *vmi;
	int section;

	vmi = xa_load(&vm_info, vm_id);
	if (!vmi)
		return -EINVAL;

	vmte = vgic_v5_get_l2_vmte(vm_id);
	if (IS_ERR(vmte))
		return PTR_ERR(vmte);

	if (spi) {
		section = GICV5_VMTEL2_SPI_SECTION;
		vgic_v5_free_allocated_spi_ist(kvm);
	} else {
		section = GICV5_VMTEL2_LPI_SECTION;
		vgic_v5_free_allocated_lpi_ist(vmi, 0, 0, 0);
	}

	/* The VM should be dead here, so we can just zero the VMT section */
	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
		WRITE_ONCE(vmte->val[section], cpu_to_le64(0));
		vgic_v5_clean_inval(vmte, sizeof(*vmte));
	}

	return 0;
}

/*
 * Free a Two-Level IST. Can only happen once the VM is dead.
 */
static int vgic_v5_two_level_ist_free(struct kvm *kvm, bool spi)
{
	unsigned int id_bits, istsz, l2sz;
	u32 vm_id = vgic_v5_vm_id(kvm);
	struct vgic_v5_vm_info *vmi;
	struct vmtl2_entry *vmte;
	u64 tmp;
	int section;

	/* We don't create two-level SPI ISTs, so freeing is a bad idea! */
	if (spi)
		return -EINVAL;

	vmi = xa_load(&vm_info, vm_id);
	if (!vmi)
		return -EINVAL;

	section = GICV5_VMTEL2_LPI_SECTION;

	if (!vmi->h_lpi_ist_structure)
		return -EINVAL;

	vmte = vgic_v5_get_l2_vmte(vm_id);
	if (IS_ERR(vmte))
		return PTR_ERR(vmte);

	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
		vgic_v5_clean_inval(vmte, sizeof(*vmte));
		tmp = le64_to_cpu(READ_ONCE(vmte->val[section]));
	}

	id_bits = FIELD_GET(GICV5_VMTEL2E_IST_ID_BITS, tmp);
	istsz = FIELD_GET(GICV5_VMTEL2E_IST_ISTSZ, tmp);
	l2sz = FIELD_GET(GICV5_VMTEL2E_IST_L2SZ, tmp);

	vgic_v5_free_allocated_lpi_ist(vmi, id_bits, istsz, l2sz);

	/* The VM must be dead, so we can just zero the VMT section */
	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
		WRITE_ONCE(vmte->val[section], cpu_to_le64(0));
		vgic_v5_clean_inval(vmte, sizeof(*vmte));
	}

	return 0;
}

/* Helper to determine ISTE size based on metadata requirements */
static unsigned int vgic_v5_ist_istsz(unsigned int id_bits)
{
	if (!vgic_v5_irs_istmd(&irs_caps))
		return GICV5_IRS_IST_CFGR_ISTSZ_4;

	if (id_bits >= vgic_v5_irs_istmd_sz(&irs_caps))
		return GICV5_IRS_IST_CFGR_ISTSZ_16;

	return GICV5_IRS_IST_CFGR_ISTSZ_8;
}

/*
 * Allocate an IST for SPIs.
 *
 * We don't anticipate a large number of SPIs being allocated. Therefore, we
 * always allocate a Linear IST for SPIs. This will need to be revisited should
 * that assumption no longer hold.
 */
int vgic_v5_spi_ist_alloc(struct kvm *kvm, unsigned int id_bits)
{
	u32 vm_id = vgic_v5_vm_id(kvm);
	struct vgic_v5_vm_info *vmi;
	phys_addr_t base_addr;
	unsigned int istsz;
	int ret;

	lockdep_assert_held(&kvm->arch.config_lock);

	istsz = vgic_v5_ist_istsz(id_bits);

	vmi = xa_load(&vm_info, vm_id);
	if (!vmi)
		return -EINVAL;

	if (vmi->h_spi_ist)
		return -EBUSY;

	ret = vgic_v5_alloc_linear_ist(kvm, true, id_bits, istsz);
	if (ret)
		return ret;
	base_addr = virt_to_phys(vmi->h_spi_ist);

	ret = vgic_v5_vmte_assign_ist(kvm, base_addr, false, id_bits, 0, istsz,
				      true);
	if (ret) {
		if (ret != -ETIMEDOUT)
			vgic_v5_free_allocated_spi_ist(kvm);
		return ret;
	}

	return 0;
}

/*
 * Free the IST for SPIs. Should only happen once the VM is dead.
 */
static int vgic_v5_spi_ist_free(struct kvm *kvm)
{
	lockdep_assert_held(&kvm->arch.config_lock);

	return vgic_v5_linear_ist_free(kvm, true);
}

int vgic_v5_lpi_ist_exists(struct kvm *kvm)
{
	u32 vm_id = vgic_v5_vm_id(kvm);
	struct vgic_v5_vm_info *vmi;

	vmi = xa_load(&vm_info, vm_id);
	if (!vmi)
		return -ENXIO;

	return !!vmi->h_lpi_ist;
}

/*
 * Allocate an IST for LPIs.
 *
 * Unlike with SPIs, we anticipate that the guest will allocate a relatively
 * large number of LPIs. Therefore, while we support doing a linear LPI IST, it
 * is expected that LPI ISTs will be two-level.
 */
int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits)
{
	u32 vm_id = vgic_v5_vm_id(kvm);
	struct vgic_v5_vm_info *vmi;
	unsigned int istsz, l2sz;
	phys_addr_t phys_addr;
	bool two_level;
	int ret;

	lockdep_assert_held(&kvm->arch.config_lock);

	vmi = xa_load(&vm_info, vm_id);
	if (!vmi)
		return -EINVAL;

	if (vmi->h_lpi_ist)
		return -EBUSY;

	istsz = vgic_v5_ist_istsz(id_bits);
	l2sz = gicv5_irs_l2_sz(vgic_v5_irs_ist_l2sz(&irs_caps));

	/*
	 * Determine if we want to create a Linear or a Two-Level IST.
	 *
	 * A two-level IST is only required when a single L2 IST cannot cover
	 * the requested ID space. This depends on the L2 IST size selected for
	 * the IRS, not PAGE_SIZE. Using PAGE_SIZE here would switch to
	 * two-level too early when the selected L2 IST is larger than a page,
	 * and the allocation sizing arithmetic would underflow.
	 */
	two_level = vgic_v5_irs_ist_levels(&irs_caps) &&
		id_bits > ((10 - istsz) + (2 * l2sz));

	if (!two_level)
		ret = vgic_v5_alloc_linear_ist(kvm, false /* LPIs, not SPIs */,
					       id_bits, istsz);
	else
		ret = vgic_v5_alloc_two_level_lpi_ist(kvm, id_bits, istsz,
						      l2sz);

	if (ret)
		return ret;

	phys_addr = virt_to_phys(vmi->h_lpi_ist);
	ret = vgic_v5_vmte_assign_ist(kvm, phys_addr, two_level, id_bits, l2sz,
				      istsz, false);
	if (ret && ret != -ETIMEDOUT)
		vgic_v5_free_allocated_lpi_ist(vmi, id_bits, istsz, l2sz);

	return ret;
}

/* Free the LPI IST again */
int vgic_v5_lpi_ist_free(struct kvm *kvm)
{
	u32 vm_id = vgic_v5_vm_id(kvm);
	struct vgic_v5_vm_info *vmi;

	lockdep_assert_held(&kvm->arch.config_lock);

	vmi = xa_load(&vm_info, vm_id);
	if (!vmi)
		return -ENXIO;

	if (!vmi->h_lpi_ist_structure)
		return vgic_v5_linear_ist_free(kvm, false);
	else
		return vgic_v5_two_level_ist_free(kvm, false);
}

static struct vgic_v5_two_level_ist_shape
vgic_v5_two_level_ist_shape(const struct vgic_v5_ist_desc *ist)
{
	struct vgic_v5_two_level_ist_shape shape;
	size_t l2bits, n;

	l2bits = (10 - ist->istsz) + (2 * ist->l2sz);
	n = max(2, ist->id_bits - l2bits + 3 - 1);

	shape.l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE;
	shape.l2_entries = BIT(l2bits);

	return shape;
}

static int vgic_v5_read_vm_ist_desc(struct kvm *kvm, unsigned int section,
				    struct vgic_v5_ist_desc *ist)
{
	u32 vm_id = vgic_v5_vm_id(kvm);
	struct vmtl2_entry *vmte;
	u64 vmte_ist_section;

	vmte = vgic_v5_get_l2_vmte(vm_id);
	if (IS_ERR(vmte))
		return PTR_ERR(vmte);

	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
		vgic_v5_clean_inval(vmte, sizeof(*vmte));
		vmte_ist_section = le64_to_cpu(READ_ONCE(vmte->val[section]));
	}

	ist->id_bits = FIELD_GET(GICV5_VMTEL2E_IST_ID_BITS, vmte_ist_section);
	ist->istsz = FIELD_GET(GICV5_VMTEL2E_IST_ISTSZ, vmte_ist_section);
	ist->l2sz = FIELD_GET(GICV5_VMTEL2E_IST_L2SZ, vmte_ist_section);
	ist->iste_size = GICV5_ISTE_SIZE(ist->istsz);

	return !!(vmte_ist_section & GICV5_VMTEL2E_IST_VALID);
}

static int vgic_v5_get_spi_ist_desc(struct kvm *kvm,
				    struct vgic_v5_ist_desc *ist)
{
	u32 vm_id = vgic_v5_vm_id(kvm);
	int ret;

	memset(ist, 0, sizeof(*ist));

	ist->vmi = xa_load(&vm_info, vm_id);
	if (!ist->vmi)
		return -ENXIO;

	ret = vgic_v5_read_vm_ist_desc(kvm, GICV5_VMTEL2_SPI_SECTION, ist);
	if (ret < 0)
		return ret;

	ist->base = ist->vmi->h_spi_ist;
	if (!ret || !ist->base)
		return -ENXIO;

	ist->present = true;
	return 0;
}

static int vgic_v5_get_lpi_ist_desc(struct kvm *kvm,
				    struct vgic_v5_ist_desc *ist)
{
	u32 vm_id = vgic_v5_vm_id(kvm);
	bool guest_valid, host_valid;
	int ret;

	memset(ist, 0, sizeof(*ist));

	ist->vmi = xa_load(&vm_info, vm_id);
	if (WARN_ON_ONCE(!ist->vmi))
		return -ENXIO;

	ret = vgic_v5_read_vm_ist_desc(kvm, GICV5_VMTEL2_LPI_SECTION, ist);
	if (ret < 0)
		return ret;

	host_valid = ret;
	guest_valid = kvm->arch.vgic.vgic_v5_irs_data->ist_baser.valid;
	ist->base = ist->vmi->h_lpi_ist;

	/* If there is no IST to save/restore, return without error. */
	if (!guest_valid && !host_valid && !ist->base)
		return 0;

	/* Mismatched combination of valid state */
	if (!guest_valid || !host_valid || !ist->base)
		return -ENXIO;

	if (ist->vmi->h_lpi_ist_structure && !ist->vmi->h_lpi_l2_ists)
		return -ENXIO;

	ist->present = true;
	return 0;
}

/*
 * Save a linear host IST to userspace memory.
 *
 * Only the architected 32-bit ISTE state is stored. Metadata is skipped when
 * striding through the host IST.
 */
static int vgic_v5_save_linear_ist(const struct vgic_v5_ist_desc *ist,
				   u32 __user *uaddr, size_t nr_entries)
{
	size_t index;
	int ret;

	vgic_v5_clean_inval(ist->base,
			    GICV5_LINEAR_IST_SIZE(ist->id_bits, ist->istsz));

	for (index = 0; index < nr_entries; index++) {
		__le32 *h_iste_addr = ist->base + index * ist->iste_size;
		u32 h_iste;

		h_iste = le32_to_cpu(READ_ONCE(*h_iste_addr));
		ret = put_user(h_iste, uaddr);
		if (ret)
			return ret;

		uaddr++;
	}

	return 0;
}

/*
 * Save a two-level host IST to userspace memory.
 *
 * Only the architected 32-bit ISTE state is stored. Metadata is skipped when
 * striding through the host IST.
 */
static int vgic_v5_save_two_level_ist(const struct vgic_v5_ist_desc *ist,
				      u32 __user *uaddr)
{
	struct vgic_v5_two_level_ist_shape shape;
	size_t h_l1_index, h_l2_index;
	void *h_l2_ist_base;
	int ret;

	shape = vgic_v5_two_level_ist_shape(ist);

	vgic_v5_clean_inval(ist->base,
			    shape.l1_entries * sizeof(*ist->vmi->h_lpi_ist));

	for (h_l1_index = 0; h_l1_index < shape.l1_entries; h_l1_index++) {
		u64 l1_iste;

		/*
		 * Host L2 ISTs are preallocated. Any invalid L1 entry means the
		 * host IST state is inconsistent.
		 */
		l1_iste = le64_to_cpu(READ_ONCE(ist->vmi->h_lpi_ist[h_l1_index]));
		if (!FIELD_GET(GICV5_ISTL1E_VALID, l1_iste))
			return -ENXIO;

		h_l2_ist_base = ist->vmi->h_lpi_l2_ists[h_l1_index];
		if (!h_l2_ist_base)
			return -ENXIO;

		vgic_v5_clean_inval(h_l2_ist_base,
				    shape.l2_entries * ist->iste_size);

		for (h_l2_index = 0; h_l2_index < shape.l2_entries; h_l2_index++) {
			void *h_iste_addr = h_l2_ist_base + h_l2_index * ist->iste_size;
			u32 h_iste = le32_to_cpu(*(__le32 *)h_iste_addr);

			ret = put_user(h_iste, uaddr);
			if (ret)
				return ret;

			uaddr++;
		}
	}

	return 0;
}

/*
 * Save the SPI IST to userspace-provided memory.
 */
int vgic_v5_save_spi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr)
{
	struct vgic_v5_ist_desc ist;
	u32 __user *uaddr;
	int ret;

	ret = vgic_v5_get_spi_ist_desc(kvm, &ist);
	if (ret)
		return ret;

	uaddr = (u32 __user *)(unsigned long)ist_attr->spi_ist_addr;

	/* The host SPI IST is always linear. */
	return vgic_v5_save_linear_ist(&ist, uaddr,
				       kvm->arch.vgic.nr_spis);
}

/*
 * Save the LPI IST to userspace memory.
 *
 * The LPI IST may be linear or two-level, so host iteration depends on the
 * allocated host shape.
 */
int vgic_v5_save_lpi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr)
{
	struct vgic_v5_ist_desc ist;
	u32 __user *uaddr;
	int ret;

	ret = vgic_v5_get_lpi_ist_desc(kvm, &ist);
	if (ret)
		return ret;

	if (!ist.present)
		return 0;

	/*
	 * Userspace sized the buffer from the guest-visible configuration.
	 * Refuse to walk a host IST with a different number of entries.
	 */
	if (ist.id_bits !=
	    kvm->arch.vgic.vgic_v5_irs_data->ist_cfgr.lpi_id_bits)
		return -EINVAL;

	uaddr = (u32 __user *)(unsigned long)ist_attr->lpi_ist_addr;

	if (!ist.vmi->h_lpi_ist_structure)
		return vgic_v5_save_linear_ist(&ist, uaddr,
					       BIT(ist.id_bits));

	return vgic_v5_save_two_level_ist(&ist, uaddr);
}

/*
 * Track any SPIs and LPIs that were marked as pending at the point where the
 * IST was restored.
 *
 * Restored pending state is cleared from the host ISTE and replayed with VDPEND
 * before the VM first runs.
 */
static int vgic_v5_track_pending_irq(struct list_head *pending_irqs, u32 intid,
				     u32 type)
{
	struct vgic_v5_pending_irq *pirq;

	pirq = kzalloc_obj(*pirq);
	if (!pirq)
		return -ENOMEM;

	/* Encode the interrupt as a GICv5 IntID. */
	pirq->irq = FIELD_PREP(GICV5_HWIRQ_TYPE, type) |
		    FIELD_PREP(GICV5_HWIRQ_ID, intid);

	INIT_LIST_HEAD(&pirq->next);
	list_add_tail(&pirq->next, pending_irqs);

	return 0;
}

/*
 * Process and sanitise each restored ISTE.
 *
 * HWU is for hardware use and must not survive migration. Pending state is
 * tracked, cleared from the ISTE, and replayed before the VM first runs.
 */
static int vgic_v5_process_iste(__le32 *iste, struct list_head *pending_irqs,
				u32 intid, u32 type)
{
	u32 iste_data = le32_to_cpu(READ_ONCE(*iste));
	int ret;

	/* Pending state is replayed later with VDPEND. */
	if (iste_data & GICV5_ISTL2E_PENDING) {
		ret = vgic_v5_track_pending_irq(pending_irqs, intid, type);
		if (ret)
			return ret;
	}

	iste_data &= ~GICV5_ISTL2E_PENDING;
	iste_data &= ~GICV5_ISTL2E_HWU;

	WRITE_ONCE(*iste, cpu_to_le32(iste_data));

	return 0;
}

static void vgic_v5_restore_spi_config(struct kvm *kvm, __le32 iste, u32 spi)
{
	u32 iste_data = le32_to_cpu(iste);
	bool pending = iste_data & GICV5_ISTL2E_PENDING;
	struct vgic_irq *irq;
	unsigned long flags;

	irq = vgic_get_irq(kvm, vgic_v5_make_spi(spi));
	if (WARN_ON_ONCE(!irq))
		return;

	raw_spin_lock_irqsave(&irq->irq_lock, flags);

	if (iste_data & GICV5_ISTL2E_HM)
		irq->config = VGIC_CONFIG_LEVEL;
	else
		irq->config = VGIC_CONFIG_EDGE;

	if (irq->config == VGIC_CONFIG_EDGE)
		irq->pending_latch = pending;
	else if (pending)
		irq->pending_latch = true;
	else if (!irq->active)
		irq->pending_latch = false;

	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
	vgic_put_irq(kvm, irq);
}

static int vgic_v5_restore_ist_entry(struct kvm *kvm,
				     const struct vgic_v5_ist_desc *ist,
				     void *h_iste_addr, __le32 h_iste,
				     u32 intid, u32 intid_type)
{
	__le32 raw_iste = h_iste;
	int ret;

	/*
	 * Sanitise the IST, clearing HWU & pending fields. Pending state is
	 * later replayed via GIC VDPEND.
	 */
	ret = vgic_v5_process_iste(&h_iste, &ist->vmi->pending_irqs,
				   intid, intid_type);
	if (ret)
		return ret;

	if (intid_type == GICV5_HWIRQ_TYPE_SPI)
		vgic_v5_restore_spi_config(kvm, raw_iste, intid);

	/*
	 * Zero the full ISTE (incl metadata), and write back the non-metadata
	 * region, only.
	 */
	memset(h_iste_addr, 0, ist->iste_size);
	WRITE_ONCE(*(__le32 *)h_iste_addr, h_iste);
	vgic_v5_clean_inval(h_iste_addr, ist->iste_size);

	return 0;
}

/*
 * Restore a userspace IST image to a linear host IST.
 *
 * The userspace IST image is a linear array of 32-bit ISTEs.
 */
static int vgic_v5_restore_linear_ist(struct kvm *kvm,
				      const struct vgic_v5_ist_desc *ist,
				      u32 __user *uaddr, size_t nr_entries,
				      u32 intid_type)
{
	size_t index;
	int ret;

	for (index = 0; index < nr_entries; index++) {
		void *h_iste_addr = ist->base + index * ist->iste_size;
		u32 h_iste;

		ret = get_user(h_iste, uaddr);
		if (ret)
			return ret;

		ret = vgic_v5_restore_ist_entry(kvm, ist, h_iste_addr,
						cpu_to_le32(h_iste), index, intid_type);
		if (ret)
			return ret;

		uaddr++;
	}

	return 0;
}

/*
 * Restore a userspace IST image to a two-level host IST.
 *
 * The userspace IST image is a linear array of 32-bit ISTEs.
 */
static int vgic_v5_restore_two_level_ist(struct kvm *kvm,
					 const struct vgic_v5_ist_desc *ist,
					 u32 __user *uaddr, u32 intid_type)
{
	struct vgic_v5_two_level_ist_shape shape;
	size_t h_l1_index, h_l2_index;
	void *h_l2_ist_base;
	int ret;

	shape = vgic_v5_two_level_ist_shape(ist);

	vgic_v5_clean_inval(ist->vmi->h_lpi_ist,
			    shape.l1_entries * sizeof(*ist->vmi->h_lpi_ist));

	for (h_l1_index = 0; h_l1_index < shape.l1_entries; ++h_l1_index) {
		u64 l1_iste;

		/*
		 * Host L2 ISTs are preallocated. Any invalid L1 entry means the
		 * host IST state is inconsistent.
		 */
		l1_iste = le64_to_cpu(READ_ONCE(ist->vmi->h_lpi_ist[h_l1_index]));
		if (!FIELD_GET(GICV5_ISTL1E_VALID, l1_iste))
			return -ENXIO;

		h_l2_ist_base = ist->vmi->h_lpi_l2_ists[h_l1_index];
		if (!h_l2_ist_base)
			return -ENXIO;

		for (h_l2_index = 0; h_l2_index < shape.l2_entries; h_l2_index++) {
			void *h_iste_addr = h_l2_ist_base +
					    h_l2_index * ist->iste_size;
			u32 intid = h_l1_index * shape.l2_entries + h_l2_index;
			u32 h_iste;

			ret = get_user(h_iste, uaddr);
			if (ret)
				return ret;

			ret = vgic_v5_restore_ist_entry(kvm, ist, h_iste_addr,
							cpu_to_le32(h_iste), intid,
							intid_type);
			if (ret)
				return ret;

			uaddr++;
		}
	}

	return 0;
}

/*
 * Restore the SPI IST from userspace-provided buffer to the host-allocated IST.
 */
int vgic_v5_restore_spi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr)
{
	u32 __user *uaddr;
	struct vgic_v5_ist_desc ist;
	int ret;

	ret = vgic_v5_get_spi_ist_desc(kvm, &ist);
	if (ret)
		return ret;

	uaddr = (u32 __user *)(unsigned long)ist_attr->spi_ist_addr;

	/* The host SPI IST is always linear. */
	return vgic_v5_restore_linear_ist(kvm, &ist, uaddr,
					  kvm->arch.vgic.nr_spis,
					  GICV5_HWIRQ_TYPE_SPI);
}

/*
 * Restore the LPI IST from userspace memory to the host-allocated LPI IST.
 *
 * The host LPI IST may be linear or two-level, so host iteration depends on the
 * host IST's shape.
 */
int vgic_v5_restore_lpi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr)
{
	u32 __user *uaddr;
	struct vgic_v5_ist_desc ist;
	int ret;

	ret = vgic_v5_get_lpi_ist_desc(kvm, &ist);
	if (ret)
		return ret;

	if (!ist.present)
		return 0;

	uaddr = (u32 __user *)(unsigned long)ist_attr->lpi_ist_addr;

	if (!ist.vmi->h_lpi_ist_structure)
		return vgic_v5_restore_linear_ist(kvm, &ist, uaddr,
						  BIT(ist.id_bits),
						  GICV5_HWIRQ_TYPE_LPI);

	return vgic_v5_restore_two_level_ist(kvm, &ist, uaddr,
					     GICV5_HWIRQ_TYPE_LPI);
}

/*
 * Process the pending IRQs removing them from the list and optionally injecting
 * them.
 */
static int vgic_v5_process_pending_irqs(struct kvm *kvm, bool inject)
{
	u32 vm_id = vgic_v5_vm_id(kvm);
	struct vgic_v5_vm_info *vmi;

	lockdep_assert_held(&kvm->arch.config_lock);

	vmi = xa_load(&vm_info, vm_id);
	if (!vmi)
		return -ENXIO;

	vgic_v5_drain_pending_irqs(kvm, vmi, inject);

	return 0;
}

/* Replay pending state that was cleared while restoring guest IST state. */
int vgic_v5_restore_pending_irqs(struct kvm *kvm)
{
	return vgic_v5_process_pending_irqs(kvm, true);
}

/* Drop pending state collected by a failed IST restore. */
void vgic_v5_discard_pending_irqs(struct kvm *kvm)
{
	vgic_v5_process_pending_irqs(kvm, false);
}