#include <linux/linkage.h>
#include <asm/asm.h>
#include <asm/asm-offsets.h>
#include <asm/bitsperlong.h>
#include <asm/frame.h>
#include <asm/nospec-branch.h>
#include "kvm-asm-offsets.h"
#include "vmenter.h"
#define SVM_vmcb01_pa (SVM_vmcb01 + KVM_VMCB_pa)
.section .noinstr.text, "ax"
.macro RESTORE_GUEST_SPEC_CTRL
ALTERNATIVE_2 "", \
"jmp 800f", X86_FEATURE_MSR_SPEC_CTRL, \
"", X86_FEATURE_V_SPEC_CTRL
.endm
.macro RESTORE_HOST_SPEC_CTRL
ALTERNATIVE_2 "", \
"jmp 900f", X86_FEATURE_MSR_SPEC_CTRL, \
"", X86_FEATURE_V_SPEC_CTRL
.endm
#define SVM_CLEAR_CPU_BUFFERS \
ALTERNATIVE "", __CLEAR_CPU_BUFFERS, X86_FEATURE_CLEAR_CPU_BUF_VM
SYM_FUNC_START(__svm_vcpu_run)
push %_ASM_BP
mov %_ASM_SP, %_ASM_BP
#ifdef CONFIG_X86_64
push %r15
push %r14
push %r13
push %r12
#else
push %edi
push %esi
#endif
push %_ASM_BX
push %_ASM_ARG2
__ASM_SIZE(push) PER_CPU_VAR(svm_data + SD_save_area_pa)
push %_ASM_ARG1
.ifnc _ASM_ARG1, _ASM_DI
mov %_ASM_ARG1, %_ASM_DI
.endif
RESTORE_GUEST_SPEC_CTRL
801:
mov SVM_vmcb01_pa(%_ASM_DI), %_ASM_AX
1: vmload %_ASM_AX
2:
mov SVM_current_vmcb(%_ASM_DI), %_ASM_AX
mov KVM_VMCB_pa(%_ASM_AX), %_ASM_AX
LOAD_REGS %_ASM_DI, SVM_vcpu_arch_regs, \
%_ASM_CX, %_ASM_DX, %_ASM_BX, %_ASM_BP, %_ASM_SI
#ifdef CONFIG_X86_64
LOAD_REGS %_ASM_DI, SVM_vcpu_arch_regs, \
%r8, %r9, %r10, %r11, %r12, %r13, %r14, %r15
#endif
LOAD_REGS %_ASM_DI, SVM_vcpu_arch_regs, %_ASM_DI
SVM_CLEAR_CPU_BUFFERS
3: vmrun %_ASM_AX
4:
pop %_ASM_AX
STORE_REGS %_ASM_AX, SVM_vcpu_arch_regs, \
%_ASM_CX, %_ASM_DX, %_ASM_BX, %_ASM_BP, %_ASM_SI, %_ASM_DI
#ifdef CONFIG_X86_64
STORE_REGS %_ASM_AX, SVM_vcpu_arch_regs, \
%r8, %r9, %r10, %r11, %r12, %r13, %r14, %r15
#endif
mov %_ASM_AX, %_ASM_DI
mov SVM_vmcb01_pa(%_ASM_DI), %_ASM_AX
5: vmsave %_ASM_AX
6:
pop %_ASM_AX
7: vmload %_ASM_AX
8:
FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_VMEXIT
RESTORE_HOST_SPEC_CTRL
901:
UNTRAIN_RET_VM
CLEAR_REGS %ecx, %edx, %ebx, %ebp, %esi, %edi
#ifdef CONFIG_X86_64
CLEAR_REGS %r8d, %r9d, %r10d, %r11d, %r12d, %r13d, %r14d, %r15d
#endif
pop %_ASM_BX
pop %_ASM_BX
#ifdef CONFIG_X86_64
pop %r12
pop %r13
pop %r14
pop %r15
#else
pop %esi
pop %edi
#endif
pop %_ASM_BP
RET
800:
RESTORE_GUEST_SPEC_CTRL_BODY SVM_spec_ctrl(%_ASM_DI), 801b
jmp 801b
900:
RESTORE_HOST_SPEC_CTRL_BODY SVM_spec_ctrl(%_ASM_DI), (%_ASM_SP), 901b
jmp 901b
10: cmpb $0, _ASM_RIP(virt_rebooting)
jne 2b
ud2
30: cmpb $0, _ASM_RIP(virt_rebooting)
jne 4b
ud2
50: cmpb $0, _ASM_RIP(virt_rebooting)
jne 6b
ud2
70: cmpb $0, _ASM_RIP(virt_rebooting)
jne 8b
ud2
_ASM_EXTABLE(1b, 10b)
_ASM_EXTABLE(3b, 30b)
_ASM_EXTABLE(5b, 50b)
_ASM_EXTABLE(7b, 70b)
SYM_FUNC_END(__svm_vcpu_run)
#ifdef CONFIG_KVM_AMD_SEV
#define SEV_ES_GPRS_BASE 0x300
SYM_FUNC_START(__svm_sev_es_vcpu_run)
FRAME_BEGIN
STORE_REGS %rdx, SEV_ES_GPRS_BASE, %rbp, %r15, %r14, %r13, %r12, %rbx
STORE_REGS %rdx, SEV_ES_GPRS_BASE, %rdi, %rsi
RESTORE_GUEST_SPEC_CTRL
801:
mov SVM_current_vmcb(%rdi), %rax
mov KVM_VMCB_pa(%rax), %rax
SVM_CLEAR_CPU_BUFFERS
1: vmrun %rax
2:
FILL_RETURN_BUFFER %rax, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_VMEXIT
RESTORE_HOST_SPEC_CTRL
901:
UNTRAIN_RET_VM
FRAME_END
RET
800:
RESTORE_GUEST_SPEC_CTRL_BODY SVM_spec_ctrl(%_ASM_DI), 801b
jmp 801b
900:
RESTORE_HOST_SPEC_CTRL_BODY SVM_spec_ctrl(%_ASM_DI), %esi, 901b
jmp 901b
3: cmpb $0, virt_rebooting(%rip)
jne 2b
ud2
_ASM_EXTABLE(1b, 3b)
SYM_FUNC_END(__svm_sev_es_vcpu_run)
#endif /* CONFIG_KVM_AMD_SEV */