#ifndef _ASM_X86_KVM_HOST_H
#define _ASM_X86_KVM_HOST_H
#include <linux/types.h>
#include <linux/mm.h>
#include <linux/mmu_notifier.h>
#include <linux/tracepoint.h>
#include <linux/cpumask.h>
#include <linux/irq_work.h>
#include <linux/irq.h>
#include <linux/workqueue.h>
#include <linux/xarray.h>
#include <linux/kvm.h>
#include <linux/kvm_para.h>
#include <linux/kvm_types.h>
#include <linux/perf_event.h>
#include <linux/pvclock_gtod.h>
#include <linux/clocksource.h>
#include <linux/irqbypass.h>
#include <linux/kfifo.h>
#include <linux/sched/vhost_task.h>
#include <linux/call_once.h>
#include <linux/atomic.h>
#include <asm/apic.h>
#include <asm/pvclock-abi.h>
#include <asm/debugreg.h>
#include <asm/desc.h>
#include <asm/mtrr.h>
#include <asm/msr-index.h>
#include <asm/msr.h>
#include <asm/asm.h>
#include <asm/irq_remapping.h>
#include <asm/kvm_page_track.h>
#include <asm/kvm_vcpu_regs.h>
#include <asm/virt.h>
#include <hyperv/hvhdk.h>
#define kvm_file_to_kvm_arch x86
#define __KVM_HAVE_ARCH_VCPU_DEBUGFS
#define __KVM_HAVE_MSI
#define __KVM_HAVE_NO_POLL
#define __KVM_HAVE_READONLY_MEM
#ifdef CONFIG_KVM_MAX_NR_VCPUS
#define KVM_MAX_VCPUS CONFIG_KVM_MAX_NR_VCPUS
#else
#define KVM_MAX_VCPUS 1024
#endif
#define KVM_VCPU_ID_RATIO 4
#define KVM_MAX_VCPU_IDS (KVM_MAX_VCPUS * KVM_VCPU_ID_RATIO)
#define KVM_INTERNAL_MEM_SLOTS 3
#define KVM_HALT_POLL_NS_DEFAULT 200000
#define KVM_IRQCHIP_NUM_PINS KVM_IOAPIC_NUM_PINS
#define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \
KVM_DIRTY_LOG_INITIALLY_SET)
#define KVM_REQ_MIGRATE_TIMER KVM_ARCH_REQ(0)
#define KVM_REQ_REPORT_TPR_ACCESS KVM_ARCH_REQ(1)
#define KVM_REQ_TRIPLE_FAULT KVM_ARCH_REQ(2)
#define KVM_REQ_MMU_SYNC KVM_ARCH_REQ(3)
#define KVM_REQ_CLOCK_UPDATE KVM_ARCH_REQ(4)
#define KVM_REQ_LOAD_MMU_PGD KVM_ARCH_REQ(5)
#define KVM_REQ_EVENT KVM_ARCH_REQ(6)
#define KVM_REQ_APF_HALT KVM_ARCH_REQ(7)
#define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(8)
#define KVM_REQ_NMI KVM_ARCH_REQ(9)
#define KVM_REQ_PMU KVM_ARCH_REQ(10)
#define KVM_REQ_PMI KVM_ARCH_REQ(11)
#ifdef CONFIG_KVM_SMM
#define KVM_REQ_SMI KVM_ARCH_REQ(12)
#endif
#define KVM_REQ_MASTERCLOCK_UPDATE KVM_ARCH_REQ(13)
#define KVM_REQ_MCLOCK_INPROGRESS \
KVM_ARCH_REQ_FLAGS(14, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_SCAN_IOAPIC \
KVM_ARCH_REQ_FLAGS(15, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_GLOBAL_CLOCK_UPDATE KVM_ARCH_REQ(16)
#define KVM_REQ_APIC_PAGE_RELOAD \
KVM_ARCH_REQ_FLAGS(17, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_HV_CRASH KVM_ARCH_REQ(18)
#define KVM_REQ_IOAPIC_EOI_EXIT KVM_ARCH_REQ(19)
#define KVM_REQ_HV_RESET KVM_ARCH_REQ(20)
#define KVM_REQ_HV_EXIT KVM_ARCH_REQ(21)
#define KVM_REQ_HV_STIMER KVM_ARCH_REQ(22)
#define KVM_REQ_LOAD_EOI_EXITMAP KVM_ARCH_REQ(23)
#define KVM_REQ_GET_NESTED_STATE_PAGES KVM_ARCH_REQ(24)
#define KVM_REQ_APICV_UPDATE \
KVM_ARCH_REQ_FLAGS(25, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_TLB_FLUSH_CURRENT KVM_ARCH_REQ(26)
#define KVM_REQ_TLB_FLUSH_GUEST \
KVM_ARCH_REQ_FLAGS(27, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_APF_READY KVM_ARCH_REQ(28)
#define KVM_REQ_RECALC_INTERCEPTS KVM_ARCH_REQ(29)
#define KVM_REQ_UPDATE_CPU_DIRTY_LOGGING \
KVM_ARCH_REQ_FLAGS(30, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_MMU_FREE_OBSOLETE_ROOTS \
KVM_ARCH_REQ_FLAGS(31, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_HV_TLB_FLUSH \
KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_VMSA_PAGE_RELOAD \
KVM_ARCH_REQ_FLAGS(33, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_UPDATE_PROTECTED_GUEST_STATE \
KVM_ARCH_REQ_FLAGS(34, KVM_REQUEST_WAIT)
#define INVALID_PAGE (~(hpa_t)0)
#define VALID_PAGE(x) ((x) != INVALID_PAGE)
#define KVM_MAX_HUGEPAGE_LEVEL PG_LEVEL_1G
#define KVM_NR_PAGE_SIZES (KVM_MAX_HUGEPAGE_LEVEL - PG_LEVEL_4K + 1)
#define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9)
#define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + KVM_HPAGE_GFN_SHIFT(x))
#define KVM_HPAGE_SIZE(x) (1UL << KVM_HPAGE_SHIFT(x))
#define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1))
#define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE)
#define KVM_MAX_CPUID_ENTRIES 256
#define KVM_NR_VAR_MTRR 8
#define ASYNC_PF_PER_VCPU 64
enum kvm_reg {
VCPU_REGS_RAX = __VCPU_REGS_RAX,
VCPU_REGS_RCX = __VCPU_REGS_RCX,
VCPU_REGS_RDX = __VCPU_REGS_RDX,
VCPU_REGS_RBX = __VCPU_REGS_RBX,
VCPU_REGS_RSP = __VCPU_REGS_RSP,
VCPU_REGS_RBP = __VCPU_REGS_RBP,
VCPU_REGS_RSI = __VCPU_REGS_RSI,
VCPU_REGS_RDI = __VCPU_REGS_RDI,
#ifdef CONFIG_X86_64
VCPU_REGS_R8 = 8,
VCPU_REGS_R9,
VCPU_REGS_R10,
VCPU_REGS_R11,
VCPU_REGS_R12,
VCPU_REGS_R13,
VCPU_REGS_R14,
VCPU_REGS_R15,
#endif
NR_VCPU_GENERAL_PURPOSE_REGS,
VCPU_REG_RIP = NR_VCPU_GENERAL_PURPOSE_REGS,
VCPU_REG_PDPTR,
VCPU_REG_CR0,
VCPU_REG_CR3,
VCPU_REG_ERAPS = VCPU_REG_CR3,
VCPU_REG_CR4,
VCPU_REG_RFLAGS,
VCPU_REG_SEGMENTS,
VCPU_REG_EXIT_INFO_1,
VCPU_REG_EXIT_INFO_2,
NR_VCPU_TOTAL_REGS,
};
enum {
VCPU_SREG_ES,
VCPU_SREG_CS,
VCPU_SREG_SS,
VCPU_SREG_DS,
VCPU_SREG_FS,
VCPU_SREG_GS,
VCPU_SREG_TR,
VCPU_SREG_LDTR,
};
enum exit_fastpath_completion {
EXIT_FASTPATH_NONE,
EXIT_FASTPATH_REENTER_GUEST,
EXIT_FASTPATH_EXIT_HANDLED,
EXIT_FASTPATH_EXIT_USERSPACE,
};
typedef enum exit_fastpath_completion fastpath_t;
struct x86_emulate_ctxt;
struct x86_exception;
union kvm_smram;
enum x86_intercept;
enum x86_intercept_stage;
#define KVM_NR_DB_REGS 4
#define PFERR_PRESENT_MASK BIT(0)
#define PFERR_WRITE_MASK BIT(1)
#define PFERR_USER_MASK BIT(2)
#define PFERR_RSVD_MASK BIT(3)
#define PFERR_FETCH_MASK BIT(4)
#define PFERR_PK_MASK BIT(5)
#define PFERR_SS_MASK BIT(6)
#define PFERR_SGX_MASK BIT(15)
#define PFERR_GUEST_RMP_MASK BIT_ULL(31)
#define PFERR_GUEST_FINAL_MASK BIT_ULL(32)
#define PFERR_GUEST_PAGE_MASK BIT_ULL(33)
#define PFERR_GUEST_FAULT_STAGE_MASK \
(PFERR_GUEST_FINAL_MASK | PFERR_GUEST_PAGE_MASK)
#define PFERR_GUEST_ENC_MASK BIT_ULL(34)
#define PFERR_GUEST_SIZEM_MASK BIT_ULL(35)
#define PFERR_GUEST_VMPL_MASK BIT_ULL(36)
#define PFERR_IMPLICIT_ACCESS BIT_ULL(48)
#define PFERR_PRIVATE_ACCESS BIT_ULL(49)
#define PFERR_SYNTHETIC_MASK (PFERR_IMPLICIT_ACCESS | PFERR_PRIVATE_ACCESS)
struct kvm_kernel_irqfd;
struct kvm_kernel_irq_routing_entry;
struct kvm_apic_map;
struct kvm_x86_msr_filter;
struct kvm_x86_pmu_event_filter;
struct kvm_caps {
bool has_tsc_control;
u32 max_guest_tsc_khz;
u8 tsc_scaling_ratio_frac_bits;
u64 max_tsc_scaling_ratio;
u64 default_tsc_scaling_ratio;
bool has_bus_lock_exit;
bool has_notify_vmexit;
u32 supported_vm_types;
u64 supported_mce_cap;
u64 supported_xcr0;
u64 supported_xss;
u64 supported_perf_cap;
u64 supported_efer_bits;
u64 supported_quirks;
u64 inapplicable_quirks;
};
extern struct kvm_caps kvm_caps;
struct kvm_host_values {
u8 maxphyaddr;
u64 efer;
u64 xcr0;
u64 xss;
u64 s_cet;
u64 arch_capabilities;
};
extern struct kvm_host_values kvm_host;
union kvm_mmu_page_role {
u32 word;
struct {
unsigned level:4;
unsigned has_4_byte_gpte:1;
unsigned quadrant:2;
unsigned direct:1;
unsigned access:4;
unsigned invalid:1;
unsigned efer_nx:1;
unsigned cr0_wp:1;
unsigned smap_andnot_wp:1;
unsigned ad_disabled:1;
unsigned guest_mode:1;
unsigned passthrough:1;
unsigned is_mirror:1;
unsigned cr4_smep:1;
unsigned:3;
unsigned smm:8;
};
};
union kvm_mmu_extended_role {
u32 word;
struct {
unsigned int valid:1;
unsigned int execonly:1;
unsigned int cr4_pse:1;
unsigned int cr4_pke:1;
unsigned int cr4_smap:1;
unsigned int cr4_la57:1;
unsigned int efer_lma:1;
unsigned int has_pferr_fetch:1;
};
};
union kvm_cpu_role {
u64 as_u64;
struct {
union kvm_mmu_page_role base;
union kvm_mmu_extended_role ext;
};
};
struct kvm_rmap_head {
atomic_long_t val;
};
struct kvm_pio_request {
unsigned long count;
int in;
int port;
int size;
};
#define PT64_ROOT_MAX_LEVEL 5
struct kvm_page_format {
u64 rsvd_bits_mask[2][PT64_ROOT_MAX_LEVEL];
u64 bad_mt_xwr;
u32 pkru_mask;
u16 permissions[16];
};
struct kvm_mmu_root_info {
gpa_t pgd;
hpa_t hpa;
};
#define KVM_MMU_ROOT_INFO_INVALID \
((struct kvm_mmu_root_info) { .pgd = INVALID_PAGE, .hpa = INVALID_PAGE })
#define KVM_MMU_NUM_PREV_ROOTS 3
#define KVM_MMU_ROOT_CURRENT BIT(0)
#define KVM_MMU_ROOT_PREVIOUS(i) BIT(1+i)
#define KVM_MMU_ROOTS_ALL (BIT(1 + KVM_MMU_NUM_PREV_ROOTS) - 1)
#define KVM_HAVE_MMU_RWLOCK
struct kvm_mmu_page;
struct kvm_page_fault;
struct kvm_pagewalk {
unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu);
u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
void (*inject_page_fault)(struct kvm_vcpu *vcpu,
struct x86_exception *fault,
bool from_hardware);
gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
gpa_t gva_or_gpa, u64 access,
struct x86_exception *exception);
union kvm_cpu_role cpu_role;
struct kvm_page_format fmt;
};
struct kvm_mmu {
int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
int (*sync_spte)(struct kvm_vcpu *vcpu,
struct kvm_mmu_page *sp, int i);
struct kvm_pagewalk *w;
struct kvm_mmu_root_info root;
hpa_t mirror_root_hpa;
union kvm_mmu_page_role root_role;
struct kvm_mmu_root_info prev_roots[KVM_MMU_NUM_PREV_ROOTS];
u64 *pae_root;
u64 *pml4_root;
u64 *pml5_root;
struct kvm_page_format fmt;
};
enum pmc_type {
KVM_PMC_GP = 0,
KVM_PMC_FIXED,
};
struct kvm_pmc {
enum pmc_type type;
u8 idx;
bool is_paused;
bool intr;
u64 counter;
u64 emulated_counter;
u64 eventsel;
u64 eventsel_hw;
struct perf_event *perf_event;
struct kvm_vcpu *vcpu;
u64 current_config;
};
#define KVM_MAX(a, b) ((a) >= (b) ? (a) : (b))
#define KVM_MAX_NR_INTEL_GP_COUNTERS 8
#define KVM_MAX_NR_AMD_GP_COUNTERS 6
#define KVM_MAX_NR_GP_COUNTERS KVM_MAX(KVM_MAX_NR_INTEL_GP_COUNTERS, \
KVM_MAX_NR_AMD_GP_COUNTERS)
#define KVM_MAX_NR_INTEL_FIXED_COUNTERS 3
#define KVM_MAX_NR_AMD_FIXED_COUNTERS 0
#define KVM_MAX_NR_FIXED_COUNTERS KVM_MAX(KVM_MAX_NR_INTEL_FIXED_COUNTERS, \
KVM_MAX_NR_AMD_FIXED_COUNTERS)
struct kvm_pmu {
u8 version;
unsigned nr_arch_gp_counters;
unsigned nr_arch_fixed_counters;
unsigned available_event_types;
u64 fixed_ctr_ctrl;
u64 fixed_ctr_ctrl_hw;
u64 fixed_ctr_ctrl_rsvd;
u64 global_ctrl;
u64 global_status;
u64 counter_bitmask[2];
u64 global_ctrl_rsvd;
u64 global_status_rsvd;
u64 reserved_bits;
u64 raw_event_mask;
struct kvm_pmc gp_counters[KVM_MAX_NR_GP_COUNTERS];
struct kvm_pmc fixed_counters[KVM_MAX_NR_FIXED_COUNTERS];
union {
DECLARE_BITMAP(reprogram_pmi, X86_PMC_IDX_MAX);
atomic64_t __reprogram_pmi;
};
DECLARE_BITMAP(all_valid_pmc_idx, X86_PMC_IDX_MAX);
DECLARE_BITMAP(pmc_in_use, X86_PMC_IDX_MAX);
DECLARE_BITMAP(pmc_counting_instructions, X86_PMC_IDX_MAX);
DECLARE_BITMAP(pmc_counting_branches, X86_PMC_IDX_MAX);
DECLARE_BITMAP(pmc_has_mode_specific_enables, X86_PMC_IDX_MAX);
u64 ds_area;
u64 pebs_enable;
u64 pebs_enable_rsvd;
u64 pebs_data_cfg;
u64 pebs_data_cfg_rsvd;
bool need_cleanup;
u8 event_count;
};
struct kvm_pmu_ops;
enum {
KVM_DEBUGREG_BP_ENABLED = BIT(0),
KVM_DEBUGREG_WONT_EXIT = BIT(1),
KVM_DEBUGREG_AUTO_SWITCH = BIT(2),
};
struct kvm_mtrr {
u64 var[KVM_NR_VAR_MTRR * 2];
u64 fixed_64k;
u64 fixed_16k[2];
u64 fixed_4k[8];
u64 deftype;
};
struct kvm_hypervisor_cpuid {
u32 base;
u32 limit;
};
#ifdef CONFIG_KVM_XEN
struct kvm_vcpu_xen {
u64 hypercall_rip;
u32 current_runstate;
u8 upcall_vector;
struct gfn_to_pfn_cache vcpu_info_cache;
struct gfn_to_pfn_cache vcpu_time_info_cache;
struct gfn_to_pfn_cache runstate_cache;
struct gfn_to_pfn_cache runstate2_cache;
u64 last_steal;
u64 runstate_entry_time;
u64 runstate_times[4];
unsigned long evtchn_pending_sel;
u32 vcpu_id;
u32 timer_virq;
u64 timer_expires;
atomic_t timer_pending;
struct hrtimer timer;
int poll_evtchn;
struct timer_list poll_timer;
struct kvm_hypervisor_cpuid cpuid;
};
#endif
struct kvm_vcpu_hv;
struct kvm_queued_exception {
bool pending;
bool injected;
bool has_error_code;
u8 vector;
u32 error_code;
unsigned long payload;
bool has_payload;
};
enum kvm_only_cpuid_leafs {
CPUID_12_EAX = NCAPINTS,
CPUID_7_1_EDX,
CPUID_8000_0007_EDX,
CPUID_8000_0022_EAX,
CPUID_7_2_EDX,
CPUID_24_0_EBX,
CPUID_8000_0021_ECX,
CPUID_7_1_ECX,
CPUID_1E_1_EAX,
CPUID_24_1_ECX,
NR_KVM_CPU_CAPS,
NKVMCAPINTS = NR_KVM_CPU_CAPS - NCAPINTS,
};
struct kvm_vcpu_arch {
unsigned long regs[NR_VCPU_GENERAL_PURPOSE_REGS];
unsigned long rip;
DECLARE_BITMAP(regs_avail, NR_VCPU_TOTAL_REGS);
DECLARE_BITMAP(regs_dirty, NR_VCPU_TOTAL_REGS);
unsigned long cr0;
unsigned long cr0_guest_owned_bits;
unsigned long cr2;
unsigned long cr3;
unsigned long cr4;
unsigned long cr4_guest_owned_bits;
unsigned long cr4_guest_rsvd_bits;
unsigned long cr8;
u32 host_pkru;
u32 pkru;
u32 hflags;
u64 efer;
u64 host_debugctl;
u64 apic_base;
struct kvm_lapic *apic;
bool load_eoi_exitmap_pending;
DECLARE_BITMAP(ioapic_handled_vectors, 256);
unsigned long apic_attention;
int32_t apic_arb_prio;
int mp_state;
u64 ia32_misc_enable_msr;
u64 smbase;
u64 smi_count;
bool at_instruction_boundary;
bool tpr_access_reporting;
bool xfd_no_write_intercept;
u64 microcode_version;
u64 arch_capabilities;
u64 perf_capabilities;
struct kvm_mmu *mmu;
struct kvm_mmu root_mmu;
struct kvm_mmu guest_mmu;
struct kvm_pagewalk ngpa_walk;
struct kvm_pagewalk gva_walk;
u64 pdptrs[4];
struct kvm_mmu_memory_cache mmu_pte_list_desc_cache;
struct kvm_mmu_memory_cache mmu_shadow_page_cache;
struct kvm_mmu_memory_cache mmu_shadowed_info_cache;
struct kvm_mmu_memory_cache mmu_page_header_cache;
struct kvm_mmu_memory_cache mmu_external_spt_cache;
struct page *pml_page;
struct fpu_guest guest_fpu;
u64 xcr0;
u64 guest_supported_xcr0;
u64 ia32_xss;
u64 guest_supported_xss;
struct kvm_pio_request pio;
void *pio_data;
void *sev_pio_data;
unsigned sev_pio_count;
u8 event_exit_inst_len;
bool exception_from_userspace;
struct kvm_queued_exception exception;
struct kvm_queued_exception exception_vmexit;
struct kvm_queued_interrupt {
bool injected;
bool soft;
u8 nr;
} interrupt;
int halt_request;
int cpuid_nent;
struct kvm_cpuid_entry2 *cpuid_entries;
bool cpuid_dynamic_bits_dirty;
bool is_amd_compatible;
u32 cpu_caps[NR_KVM_CPU_CAPS];
u64 reserved_gpa_bits;
int maxphyaddr;
struct x86_emulate_ctxt *emulate_ctxt;
bool emulate_regs_need_sync_to_vcpu;
bool emulate_regs_need_sync_from_vcpu;
int (*complete_userspace_io)(struct kvm_vcpu *vcpu);
unsigned long cui_linear_rip;
int cui_rdmsr_imm_reg;
gpa_t time;
s8 pvclock_tsc_shift;
u32 pvclock_tsc_mul;
u64 hw_tsc_hz;
struct gfn_to_pfn_cache pv_time;
bool pvclock_set_guest_stopped_request;
struct {
u8 preempted;
u64 msr_val;
u64 last_steal;
struct gfn_to_hva_cache cache;
} st;
u64 l1_tsc_offset;
u64 tsc_offset;
u64 last_guest_tsc;
u64 last_host_tsc;
u64 tsc_offset_adjustment;
u64 this_tsc_nsec;
u64 this_tsc_write;
u64 this_tsc_generation;
bool tsc_catchup;
bool tsc_always_catchup;
s8 virtual_tsc_shift;
u32 virtual_tsc_mult;
u32 virtual_tsc_khz;
s64 ia32_tsc_adjust_msr;
u64 msr_ia32_power_ctl;
u64 l1_tsc_scaling_ratio;
u64 tsc_scaling_ratio;
atomic_t nmi_queued;
unsigned int nmi_pending;
bool nmi_injected;
bool smi_pending;
u8 handling_intr_from_guest;
struct kvm_mtrr mtrr_state;
u64 pat;
unsigned switch_db_regs;
unsigned long db[KVM_NR_DB_REGS];
unsigned long dr6;
unsigned long dr7;
unsigned long eff_db[KVM_NR_DB_REGS];
unsigned long guest_debug_dr7;
u64 msr_platform_info;
u64 msr_misc_features_enables;
u64 mcg_cap;
u64 mcg_status;
u64 mcg_ctl;
u64 mcg_ext_ctl;
u64 *mce_banks;
u64 *mci_ctl2_banks;
u64 mmio_gva;
unsigned mmio_access;
gfn_t mmio_gfn;
u64 mmio_gen;
struct kvm_pmu pmu;
unsigned long singlestep_rip;
#ifdef CONFIG_KVM_HYPERV
bool hyperv_enabled;
struct kvm_vcpu_hv *hyperv;
#endif
#ifdef CONFIG_KVM_XEN
struct kvm_vcpu_xen xen;
#endif
cpumask_var_t wbinvd_dirty_mask;
unsigned long last_retry_eip;
unsigned long last_retry_addr;
struct {
bool halted;
gfn_t gfns[ASYNC_PF_PER_VCPU];
struct gfn_to_hva_cache data;
u64 msr_en_val;
u64 msr_int_val;
u16 vec;
u32 id;
u32 host_apf_flags;
bool pageready_pending;
} apf;
struct {
u64 length;
u64 status;
} osvw;
struct {
u64 msr_val;
struct gfn_to_hva_cache data;
} pv_eoi;
u64 msr_kvm_poll_control;
struct {
bool pv_unhalted;
} pv;
u8 pending_ioapic_eoi;
int pending_external_vector;
int highest_stale_pending_ioapic_eoi;
bool preempted_in_kernel;
int last_vmentry_cpu;
u64 msr_hwcr;
struct {
u32 features;
bool enforce;
} pv_cpuid;
bool guest_state_protected;
bool guest_tsc_protected;
bool pdptrs_from_userspace;
#define KVM_NESTED_RUN_PENDING 1
#define KVM_NESTED_RUN_PENDING_UNTRUSTED 2
u8 nested_run_pending;
#if IS_ENABLED(CONFIG_HYPERV)
hpa_t hv_root_tdp;
#endif
};
struct kvm_lpage_info {
int disallow_lpage;
};
struct kvm_arch_memory_slot {
struct kvm_rmap_head *rmap[KVM_NR_PAGE_SIZES];
struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1];
unsigned short *gfn_write_track;
};
struct kvm_hv_syndbg {
struct {
u64 control;
u64 status;
u64 send_page;
u64 recv_page;
u64 pending_page;
} control;
u64 options;
};
enum hv_tsc_page_status {
HV_TSC_PAGE_UNSET = 0,
HV_TSC_PAGE_GUEST_CHANGED,
HV_TSC_PAGE_HOST_CHANGED,
HV_TSC_PAGE_SET,
HV_TSC_PAGE_BROKEN,
};
#ifdef CONFIG_KVM_HYPERV
struct kvm_hv {
struct mutex hv_lock;
u64 hv_guest_os_id;
u64 hv_hypercall;
u64 hv_tsc_page;
enum hv_tsc_page_status hv_tsc_page_status;
u64 hv_crash_param[HV_X64_MSR_CRASH_PARAMS];
u64 hv_crash_ctl;
struct ms_hyperv_tsc_page tsc_ref;
struct idr conn_to_evt;
u64 hv_reenlightenment_control;
u64 hv_tsc_emulation_control;
u64 hv_tsc_emulation_status;
u64 hv_invtsc_control;
atomic_t num_mismatched_vp_indexes;
unsigned int synic_auto_eoi_used;
struct kvm_hv_syndbg hv_syndbg;
bool xsaves_xsavec_checked;
};
#endif
#ifdef CONFIG_KVM_XEN
struct kvm_xen {
struct mutex xen_lock;
u32 xen_version;
bool long_mode;
bool runstate_update_flag;
u8 upcall_vector;
struct gfn_to_pfn_cache shinfo_cache;
struct xarray evtchn_ports;
unsigned long poll_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)];
struct kvm_xen_hvm_config hvm_config;
};
#endif
enum kvm_irqchip_mode {
KVM_IRQCHIP_NONE,
#ifdef CONFIG_KVM_IOAPIC
KVM_IRQCHIP_KERNEL,
#endif
KVM_IRQCHIP_SPLIT,
};
enum kvm_suppress_eoi_broadcast_mode {
KVM_SUPPRESS_EOI_BROADCAST_QUIRKED,
KVM_SUPPRESS_EOI_BROADCAST_ENABLED,
KVM_SUPPRESS_EOI_BROADCAST_DISABLED
};
struct kvm_possible_nx_huge_pages {
struct list_head pages;
u64 nr_pages;
};
enum kvm_mmu_type {
KVM_SHADOW_MMU,
#ifdef CONFIG_X86_64
KVM_TDP_MMU,
#endif
KVM_NR_MMU_TYPES,
};
struct kvm_arch {
unsigned long n_requested_mmu_pages;
unsigned long n_max_mmu_pages;
unsigned int indirect_shadow_pages;
u8 mmu_valid_gen;
u8 vm_type;
bool has_private_mem;
bool has_protected_page_tables;
bool has_protected_state;
bool has_protected_eoi;
bool has_protected_pmu;
bool pre_fault_allowed;
struct hlist_head *mmu_page_hash;
struct list_head active_mmu_pages;
struct kvm_possible_nx_huge_pages possible_nx_huge_pages[KVM_NR_MMU_TYPES];
#ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING
struct kvm_page_track_notifier_head track_notifier_head;
#endif
spinlock_t mmu_unsync_pages_lock;
u64 shadow_mmio_value;
#define __KVM_HAVE_ARCH_NONCOHERENT_DMA
atomic_t noncoherent_dma_count;
unsigned long nr_possible_bypass_irqs;
#ifdef CONFIG_KVM_IOAPIC
struct kvm_pic *vpic;
struct kvm_ioapic *vioapic;
struct kvm_pit *vpit;
#endif
atomic_t vapics_in_nmi_mode;
struct mutex apic_map_lock;
struct kvm_apic_map __rcu *apic_map;
atomic_t apic_map_dirty;
bool apic_access_memslot_enabled;
bool apic_access_memslot_inhibited;
____cacheline_aligned
struct rw_semaphore apicv_update_lock;
atomic_t apicv_nr_irq_window_req;
____cacheline_aligned
unsigned long apicv_inhibit_reasons;
gpa_t wall_clock;
u64 disabled_exits;
s64 kvmclock_offset;
raw_spinlock_t tsc_write_lock;
u64 last_tsc_nsec;
u64 last_tsc_write;
u32 last_tsc_khz;
u64 last_tsc_offset;
u64 last_tsc_scaling_ratio;
u64 cur_tsc_nsec;
u64 cur_tsc_write;
u64 cur_tsc_offset;
u64 cur_tsc_generation;
int nr_vcpus_matched_tsc;
u32 default_tsc_khz;
bool user_set_tsc;
u64 apic_bus_cycle_ns;
seqcount_raw_spinlock_t pvclock_sc;
bool use_master_clock;
u64 master_kernel_ns;
u64 master_cycle_now;
struct ratelimit_state kvmclock_update_rs;
u64 master_tsc_scaling_ratio;
s8 master_tsc_shift;
u32 master_tsc_mul;
#ifdef CONFIG_KVM_HYPERV
struct kvm_hv hyperv;
#endif
#ifdef CONFIG_KVM_XEN
struct kvm_xen xen;
#endif
bool backwards_tsc_observed;
bool boot_vcpu_runs_old_kvmclock;
u32 bsp_vcpu_id;
u64 disabled_quirks;
enum kvm_irqchip_mode irqchip_mode;
u8 nr_reserved_ioapic_pins;
bool disabled_lapic_found;
bool x2apic_format;
bool x2apic_broadcast_quirk_disabled;
enum kvm_suppress_eoi_broadcast_mode suppress_eoi_broadcast_mode;
bool has_mapped_host_mmio;
bool guest_can_read_msr_platform_info;
bool exception_payload_enabled;
bool triple_fault_event;
bool bus_lock_detection_enabled;
bool enable_pmu;
bool created_mediated_pmu;
u32 notify_window;
u32 notify_vmexit_flags;
bool exit_on_emulation_error;
u32 user_space_msr_mask;
struct kvm_x86_msr_filter __rcu *msr_filter;
u32 hypercall_exit_enabled;
bool sgx_provisioning_allowed;
struct kvm_x86_pmu_event_filter __rcu *pmu_event_filter;
struct vhost_task *nx_huge_page_recovery_thread;
u64 nx_huge_page_last;
struct once nx_once;
#ifdef CONFIG_X86_64
#ifdef CONFIG_KVM_PROVE_MMU
atomic64_t tdp_mmu_pages;
#endif
struct list_head tdp_mmu_roots;
spinlock_t tdp_mmu_pages_lock;
#endif /* CONFIG_X86_64 */
bool shadow_root_allocated;
#ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING
bool external_write_tracking_enabled;
#endif
#if IS_ENABLED(CONFIG_HYPERV)
hpa_t hv_root_tdp;
spinlock_t hv_root_tdp_lock;
struct hv_partition_assist_pg *hv_pa_pg;
#endif
u32 max_vcpu_ids;
bool disable_nx_huge_pages;
struct kvm_mmu_memory_cache split_shadow_page_cache;
struct kvm_mmu_memory_cache split_page_header_cache;
#define SPLIT_DESC_CACHE_MIN_NR_OBJECTS (SPTE_ENT_PER_PAGE + 1)
struct kvm_mmu_memory_cache split_desc_cache;
gfn_t gfn_direct_bits;
int mirror_root_level;
int cpu_dirty_log_size;
};
struct kvm_vm_stat {
struct kvm_vm_stat_generic generic;
u64 mmu_shadow_zapped;
u64 mmu_pte_write;
u64 mmu_pde_zapped;
u64 mmu_flooded;
u64 mmu_recycled;
u64 mmu_cache_miss;
u64 mmu_unsync;
u64 mmu_shadow_pages;
union {
struct {
atomic64_t pages_4k;
atomic64_t pages_2m;
atomic64_t pages_1g;
};
atomic64_t pages[KVM_NR_PAGE_SIZES];
};
u64 nx_lpage_splits;
u64 max_mmu_page_hash_collisions;
u64 max_mmu_rmap_size;
};
struct kvm_vcpu_stat {
struct kvm_vcpu_stat_generic generic;
u64 pf_taken;
u64 pf_fixed;
u64 pf_emulate;
u64 pf_spurious;
u64 pf_fast;
u64 pf_mmio_spte_created;
u64 pf_guest;
u64 tlb_flush;
u64 invlpg;
u64 exits;
u64 io_exits;
u64 mmio_exits;
u64 signal_exits;
u64 irq_window_exits;
u64 nmi_window_exits;
u64 l1d_flush;
u64 halt_exits;
u64 request_irq_exits;
u64 irq_exits;
u64 host_state_reload;
u64 fpu_reload;
u64 insn_emulation;
u64 insn_emulation_fail;
u64 hypercalls;
u64 irq_injections;
u64 nmi_injections;
u64 req_event;
u64 nested_run;
u64 directed_yield_attempted;
u64 directed_yield_successful;
u64 preemption_reported;
u64 preemption_other;
u64 guest_mode;
u64 notify_window_exits;
};
struct x86_instruction_info;
struct msr_data {
bool host_initiated;
u32 index;
u64 data;
};
struct kvm_lapic_irq {
u32 vector;
u16 delivery_mode;
u16 dest_mode;
bool level;
u16 trig_mode;
u32 shorthand;
u32 dest_id;
bool msi_redir_hint;
};
enum kvm_x86_run_flags {
KVM_RUN_FORCE_IMMEDIATE_EXIT = BIT(0),
KVM_RUN_LOAD_GUEST_DR6 = BIT(1),
KVM_RUN_LOAD_DEBUGCTL = BIT(2),
};
struct kvm_x86_ops {
const char *name;
int (*check_processor_compatibility)(void);
int (*enable_virtualization_cpu)(void);
void (*disable_virtualization_cpu)(void);
cpu_emergency_virt_cb *emergency_disable_virtualization_cpu;
void (*hardware_unsetup)(void);
bool (*has_emulated_msr)(struct kvm *kvm, u32 index);
void (*vcpu_after_set_cpuid)(struct kvm_vcpu *vcpu);
unsigned int vm_size;
int (*vm_init)(struct kvm *kvm);
void (*vm_destroy)(struct kvm *kvm);
void (*vm_pre_destroy)(struct kvm *kvm);
int (*vcpu_precreate)(struct kvm *kvm);
int (*vcpu_create)(struct kvm_vcpu *vcpu);
void (*vcpu_free)(struct kvm_vcpu *vcpu);
void (*vcpu_reset)(struct kvm_vcpu *vcpu, bool init_event);
void (*prepare_switch_to_guest)(struct kvm_vcpu *vcpu);
void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu);
void (*vcpu_put)(struct kvm_vcpu *vcpu);
const u64 HOST_OWNED_DEBUGCTL;
void (*update_exception_bitmap)(struct kvm_vcpu *vcpu);
int (*get_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr);
int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr);
u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg);
void (*get_segment)(struct kvm_vcpu *vcpu,
struct kvm_segment *var, int seg);
int (*get_cpl)(struct kvm_vcpu *vcpu);
int (*get_cpl_no_cache)(struct kvm_vcpu *vcpu);
void (*set_segment)(struct kvm_vcpu *vcpu,
struct kvm_segment *var, int seg);
void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l);
bool (*is_valid_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0);
void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0);
void (*post_set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
bool (*is_valid_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4);
void (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4);
int (*set_efer)(struct kvm_vcpu *vcpu, u64 efer);
void (*get_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
void (*set_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
void (*get_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
void (*set_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
void (*sync_dirty_debug_regs)(struct kvm_vcpu *vcpu);
void (*set_dr7)(struct kvm_vcpu *vcpu, unsigned long value);
void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg);
unsigned long (*get_rflags)(struct kvm_vcpu *vcpu);
void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags);
bool (*get_if_flag)(struct kvm_vcpu *vcpu);
void (*flush_tlb_all)(struct kvm_vcpu *vcpu);
void (*flush_tlb_current)(struct kvm_vcpu *vcpu);
#if IS_ENABLED(CONFIG_HYPERV)
int (*flush_remote_tlbs)(struct kvm *kvm);
int (*flush_remote_tlbs_range)(struct kvm *kvm, gfn_t gfn,
gfn_t nr_pages);
#endif
void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr);
void (*flush_tlb_guest)(struct kvm_vcpu *vcpu);
bool (*vcpu_needs_initialization)(struct kvm_vcpu *vcpu);
enum exit_fastpath_completion (*vcpu_run)(struct kvm_vcpu *vcpu,
u64 run_flags);
int (*handle_exit)(struct kvm_vcpu *vcpu,
enum exit_fastpath_completion exit_fastpath);
int (*skip_emulated_instruction)(struct kvm_vcpu *vcpu);
void (*update_emulated_instruction)(struct kvm_vcpu *vcpu);
bool (*unhandleable_emulation_required)(struct kvm_vcpu *vcpu);
void (*set_interrupt_shadow)(struct kvm_vcpu *vcpu, int mask);
u32 (*get_interrupt_shadow)(struct kvm_vcpu *vcpu);
void (*patch_hypercall)(struct kvm_vcpu *vcpu,
unsigned char *hypercall_addr);
void (*inject_irq)(struct kvm_vcpu *vcpu, bool reinjected);
void (*inject_nmi)(struct kvm_vcpu *vcpu);
void (*inject_exception)(struct kvm_vcpu *vcpu);
void (*cancel_injection)(struct kvm_vcpu *vcpu);
int (*interrupt_allowed)(struct kvm_vcpu *vcpu, bool for_injection);
int (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection);
bool (*get_nmi_mask)(struct kvm_vcpu *vcpu);
void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked);
void (*clear_hlt)(struct kvm_vcpu *vcpu);
bool (*is_vnmi_pending)(struct kvm_vcpu *vcpu);
bool (*set_vnmi_pending)(struct kvm_vcpu *vcpu);
void (*enable_nmi_window)(struct kvm_vcpu *vcpu);
void (*enable_irq_window)(struct kvm_vcpu *vcpu);
void (*update_cr8_intercept)(struct kvm_vcpu *vcpu, int tpr, int irr);
const bool x2apic_icr_is_split;
const unsigned long required_apicv_inhibits;
bool allow_apicv_in_x2apic_without_x2apic_virtualization;
void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu);
void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr);
void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap);
void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu);
void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu);
void (*deliver_interrupt)(struct kvm_lapic *apic, int delivery_mode,
int trig_mode, int vector);
int (*sync_pir_to_irr)(struct kvm_vcpu *vcpu);
int (*set_tss_addr)(struct kvm *kvm, unsigned int addr);
int (*set_identity_map_addr)(struct kvm *kvm, u64 ident_addr);
u8 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio);
bool (*tdp_has_smep)(struct kvm *kvm);
void (*load_mmu_pgd)(struct kvm_vcpu *vcpu, hpa_t root_hpa,
int root_level);
int (*set_external_spte)(struct kvm *kvm, gfn_t gfn, u64 old_spte,
u64 new_spte, enum pg_level level);
void (*free_external_spt)(struct kvm *kvm, struct kvm_mmu_page *sp);
int (*topup_external_cache)(struct kvm_vcpu *vcpu, int min_nr_spts);
bool (*has_wbinvd_exit)(void);
u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu);
u64 (*get_l2_tsc_multiplier)(struct kvm_vcpu *vcpu);
void (*write_tsc_offset)(struct kvm_vcpu *vcpu);
void (*write_tsc_multiplier)(struct kvm_vcpu *vcpu);
void (*get_exit_info)(struct kvm_vcpu *vcpu, u32 *reason,
u64 *info1, u64 *info2,
u32 *intr_info, u32 *error_code);
void (*get_entry_info)(struct kvm_vcpu *vcpu,
u32 *intr_info, u32 *error_code);
int (*check_intercept)(struct kvm_vcpu *vcpu,
struct x86_instruction_info *info,
enum x86_intercept_stage stage,
struct x86_exception *exception);
void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu);
void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu, bool enable);
void (*vcpu_blocking)(struct kvm_vcpu *vcpu);
void (*vcpu_unblocking)(struct kvm_vcpu *vcpu);
int (*pi_update_irte)(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm,
unsigned int host_irq, uint32_t guest_irq,
struct kvm_vcpu *vcpu, u32 vector);
void (*pi_start_bypass)(struct kvm *kvm);
void (*apicv_pre_state_restore)(struct kvm_vcpu *vcpu);
void (*apicv_post_state_restore)(struct kvm_vcpu *vcpu);
bool (*dy_apicv_has_pending_interrupt)(struct kvm_vcpu *vcpu);
bool (*protected_apic_has_interrupt)(struct kvm_vcpu *vcpu);
int (*set_hv_timer)(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc,
bool *expired);
void (*cancel_hv_timer)(struct kvm_vcpu *vcpu);
void (*setup_mce)(struct kvm_vcpu *vcpu);
#ifdef CONFIG_KVM_SMM
int (*smi_allowed)(struct kvm_vcpu *vcpu, bool for_injection);
int (*enter_smm)(struct kvm_vcpu *vcpu, union kvm_smram *smram);
int (*leave_smm)(struct kvm_vcpu *vcpu, const union kvm_smram *smram);
void (*enable_smi_window)(struct kvm_vcpu *vcpu);
#endif
int (*dev_get_attr)(u32 group, u64 attr, u64 *val);
int (*mem_enc_ioctl)(struct kvm *kvm, void __user *argp);
int (*vcpu_mem_enc_ioctl)(struct kvm_vcpu *vcpu, void __user *argp);
int (*vcpu_mem_enc_unlocked_ioctl)(struct kvm_vcpu *vcpu, void __user *argp);
int (*mem_enc_register_region)(struct kvm *kvm, struct kvm_enc_region *argp);
int (*mem_enc_unregister_region)(struct kvm *kvm, struct kvm_enc_region *argp);
int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd);
int (*vm_move_enc_context_from)(struct kvm *kvm, unsigned int source_fd);
void (*guest_memory_reclaimed)(struct kvm *kvm);
void (*reload_vmsa)(struct kvm_vcpu *vcpu);
int (*get_feature_msr)(u32 msr, u64 *data);
int (*check_emulate_instruction)(struct kvm_vcpu *vcpu, int emul_type,
void *insn, int insn_len);
bool (*apic_init_signal_blocked)(struct kvm_vcpu *vcpu);
int (*enable_l2_tlb_flush)(struct kvm_vcpu *vcpu);
void (*migrate_timers)(struct kvm_vcpu *vcpu);
void (*recalc_intercepts)(struct kvm_vcpu *vcpu);
int (*complete_emulated_msr)(struct kvm_vcpu *vcpu, int err);
void (*vcpu_deliver_sipi_vector)(struct kvm_vcpu *vcpu, u8 vector);
unsigned long (*vcpu_get_apicv_inhibit_reasons)(struct kvm_vcpu *vcpu);
gva_t (*get_untagged_addr)(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags);
void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu);
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
int (*gmem_make_private)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
kvm_pfn_t nr_pages);
#endif
#if defined(CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT) || defined(CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM)
void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages);
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range);
#endif
int (*gmem_max_mapping_level)(struct kvm *kvm, kvm_pfn_t pfn, bool is_private);
};
struct kvm_x86_nested_ops {
bool enabled;
void (*leave_nested)(struct kvm_vcpu *vcpu);
bool (*is_exception_vmexit)(struct kvm_vcpu *vcpu, u8 vector,
u32 error_code);
int (*check_events)(struct kvm_vcpu *vcpu);
bool (*has_events)(struct kvm_vcpu *vcpu, bool for_injection);
void (*triple_fault)(struct kvm_vcpu *vcpu);
int (*get_state)(struct kvm_vcpu *vcpu,
struct kvm_nested_state __user *user_kvm_nested_state,
unsigned user_data_size);
int (*set_state)(struct kvm_vcpu *vcpu,
struct kvm_nested_state __user *user_kvm_nested_state,
struct kvm_nested_state *kvm_state);
bool (*get_nested_state_pages)(struct kvm_vcpu *vcpu);
int (*write_log_dirty)(struct kvm_vcpu *vcpu, gpa_t l2_gpa);
gpa_t (*translate_nested_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa,
u64 access,
struct x86_exception *exception,
u64 pte_access);
int (*enable_evmcs)(struct kvm_vcpu *vcpu,
uint16_t *vmcs_version);
uint16_t (*get_evmcs_version)(struct kvm_vcpu *vcpu);
void (*hv_inject_synthetic_vmexit_post_tlb_flush)(struct kvm_vcpu *vcpu);
};
struct kvm_x86_init_ops {
int (*hardware_setup)(void);
unsigned int (*handle_intel_pt_intr)(void);
struct kvm_x86_ops *runtime_ops;
struct kvm_pmu_ops *pmu_ops;
struct kvm_x86_nested_ops *nested_ops;
};
struct kvm_arch_async_pf {
u32 token;
gfn_t gfn;
unsigned long cr3;
bool direct_map;
u64 error_code;
};
extern bool __read_mostly allow_smaller_maxphyaddr;
extern bool __read_mostly enable_apicv;
extern bool __read_mostly enable_ipiv;
extern bool __read_mostly enable_device_posted_irqs;
extern struct kvm_x86_ops kvm_x86_ops;
extern struct kvm_x86_nested_ops kvm_nested_ops __read_mostly;
#define kvm_x86_call(func) static_call(kvm_x86_##func)
#define KVM_X86_OP(func) \
DECLARE_STATIC_CALL(kvm_x86_##func, *(((struct kvm_x86_ops *)0)->func));
#define KVM_X86_OP_OPTIONAL KVM_X86_OP
#define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP
#include <asm/kvm-x86-ops.h>
#define kvm_nested_call(func) static_call(kvm_x86_nested_##func)
#define KVM_X86_NESTED_OP(func) \
DECLARE_STATIC_CALL(kvm_x86_nested_##func, *(((struct kvm_x86_nested_ops *)0)->func));
#define KVM_X86_NESTED_OP_OPTIONAL KVM_X86_NESTED_OP
#define KVM_X86_NESTED_OP_OPTIONAL_RET0 KVM_X86_NESTED_OP
#include <asm/kvm-x86-nested-ops.h>
#define __KVM_HAVE_ARCH_VM_ALLOC
static inline struct kvm *kvm_arch_alloc_vm(void)
{
return kvzalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT);
}
#define __KVM_HAVE_ARCH_VM_FREE
void kvm_arch_free_vm(struct kvm *kvm);
#if IS_ENABLED(CONFIG_HYPERV)
#define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS
static inline int kvm_arch_flush_remote_tlbs(struct kvm *kvm)
{
if (kvm_x86_ops.flush_remote_tlbs &&
!kvm_x86_call(flush_remote_tlbs)(kvm))
return 0;
else
return -ENOTSUPP;
}
#define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE
static inline int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn,
u64 nr_pages)
{
if (!kvm_x86_ops.flush_remote_tlbs_range)
return -EOPNOTSUPP;
return kvm_x86_call(flush_remote_tlbs_range)(kvm, gfn, nr_pages);
}
#endif /* CONFIG_HYPERV */
enum kvm_intr_type {
KVM_HANDLING_IRQ = 1,
KVM_HANDLING_NMI,
};
#define kvm_arch_pmi_in_guest(vcpu) \
((vcpu) && (vcpu)->arch.handling_intr_from_guest && \
(!!in_nmi() == ((vcpu)->arch.handling_intr_from_guest == KVM_HANDLING_NMI)))
#if defined(CONFIG_KVM_SW_PROTECTED_VM) || \
defined(CONFIG_KVM_INTEL_TDX) || \
defined(CONFIG_KVM_AMD_SEV)
#define kvm_arch_has_private_mem(kvm) ((kvm)->arch.has_private_mem)
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
#define kvm_arch_has_gmem_convert() (!!kvm_x86_ops.gmem_make_private)
#endif
#define kvm_arch_has_readonly_mem(kvm) (!(kvm)->arch.has_protected_state)
#ifdef CONFIG_X86_64
static inline unsigned long read_msr(unsigned long msr)
{
u64 value;
rdmsrq(msr, value);
return value;
}
#endif
#define HF_GUEST_MASK (1 << 0) /* VCPU is in guest-mode */
#ifdef CONFIG_KVM_SMM
#define HF_SMM_MASK (1 << 1)
#define HF_SMM_INSIDE_NMI_MASK (1 << 2)
# define KVM_MAX_NR_ADDRESS_SPACES 2
# define kvm_arch_nr_memslot_as_ids(kvm) (kvm_arch_has_private_mem(kvm) ? 1 : 2)
# define kvm_arch_vcpu_memslots_id(vcpu) ((vcpu)->arch.hflags & HF_SMM_MASK ? 1 : 0)
# define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm)
#else
# define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, 0)
#endif
bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
struct kvm_async_pf *work);
void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
struct kvm_async_pf *work);
void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
struct kvm_async_pf *work);
void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu);
bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu);
static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
{
kvm_x86_call(vcpu_blocking)(vcpu);
}
static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
{
kvm_x86_call(vcpu_unblocking)(vcpu);
}
static inline bool kvm_arch_has_irq_bypass(void)
{
return enable_device_posted_irqs;
}
#endif /* _ASM_X86_KVM_HOST_H */