github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-06 21:58:01 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

KVM VMX changes for 7.4

Browse Source 
- Enable Notify VM-Exits (now called Instruction Timeout in the SDM) and Bus
    Lock VM-Exits for TDX guests when they're enabled by userspace (KVM forgot
    to wire up the TDX Module knobs).
 
  - Move all common (between VMX and TDX) APIs/helpers to common.h/main.c, and
    change all namespace prefixes from "vmx" to "vt".
 
  - Fix a thankfully benign bug (due to other areas of KVM being hardened) where
    usersepace can bypass INIT_MEM_REGION's address wrap check by providing a
    nr_pages that will overflow a u64 when converted to bytes, and fix an also-
    benign off-by-one bug in the check (inclusive vs. exclusive).
 
  - Set the vmcs.GUEST_ACTIVITY_STATE back to ACTIVE when userspace sets KVM's
    mp_state to anything other than HALTED, e.g. so that a vCPU doesn't get
    stuck in HLT if userspace emulates RESET.
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEKTobbabEP7vbhhN9OlYIJqCjN/0FAmrRoB0ACgkQOlYIJqCj
 N/1/JA//Qq4oU67xamqNE92mFuqsO6/5hc0iwhm22qm55VZ2BQGeGtYh6u96ewVS
 QDNQzeEPCawzXzqsOASsPz/auXnZVqjfkIlNVC5WF3+ZW9Xb81Px4f7POLjN4ey6
 a3eY/4v4hS09yn/FubYRnwLYAq8Nl+p758T6SfmXf9sMy+DLfm5R3oWueZL/hmoH
 HLnxN8qLhAmMoVQU/lFsd86AqVJfsltFosdqOZ+pP6H7A8yW0dMW503Zn4cO2+9I
 CYeFcQ21Swx6niIkMHJOMgrWuEGApwe+Il1+VviMSMBxT31oOP7b+g13PmWRKCn0
 LyfOs20Ubkj4IHA/7YVJJ3I7Og0gVFgILytosKGYYiddgz5mNyiHKUsoaz0+/h0b
 4401JMAZwgmcurWt/rY8jyhxuuO0SZ0UUiceU6lvxkTSKKDL2bOtal2AtysIaRAf
 dQ5194V71bFj1HpWgcvrrGd9tFot/zFPliz8QIucD9ROQ27RwjZywShwt/qIoWxp
 nK02c69UkOnxCFnSx/moPw2DD0v0ybQ+RhXL88yrjsm+bw+GignYAcn/4XhJp9iT
 /YBmPQR+p+Ns96gKoX8cMW3QKL91JD34S5j+Vk4mr0MiubrXJTky7i5oyoql8soZ
 3wnhpxszyDIXGabHs2xO1aiy3A0dJVAhcBotIOt/oD7/qc9sAkY=
 =DpB7
 -----END PGP SIGNATURE-----

Merge tag 'kvm-x86-vmx-7.4' of https://github.com/kvm-x86/linux into HEAD

KVM VMX changes for 7.4

 - Enable Notify VM-Exits (now called Instruction Timeout in the SDM) and Bus
   Lock VM-Exits for TDX guests when they're enabled by userspace (KVM forgot
   to wire up the TDX Module knobs).

 - Move all common (between VMX and TDX) APIs/helpers to common.h/main.c, and
   change all namespace prefixes from "vmx" to "vt".

 - Fix a thankfully benign bug (due to other areas of KVM being hardened) where
   usersepace can bypass INIT_MEM_REGION's address wrap check by providing a
   nr_pages that will overflow a u64 when converted to bytes, and fix an also-
   benign off-by-one bug in the check (inclusive vs. exclusive).

 - Set the vmcs.GUEST_ACTIVITY_STATE back to ACTIVE when userspace sets KVM's
   mp_state to anything other than HALTED, e.g. so that a vCPU doesn't get
   stuck in HLT if userspace emulates RESET.
This commit is contained in:
Paolo Bonzini 2026-10-16 11:02:29 +02:00
commit c7c2f51339
13 changed files with 397 additions and 291 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
arch/x86/include/asm/kvm-x86-ops.h
View File

@ -82,6 +82,7 @@ KVM_X86_OP(interrupt_allowed)
KVM_X86_OP(nmi_allowed)
KVM_X86_OP(get_nmi_mask)
KVM_X86_OP(set_nmi_mask)
KVM_X86_OP_OPTIONAL(clear_hlt)
KVM_X86_OP(enable_nmi_window)
KVM_X86_OP(enable_irq_window)
KVM_X86_OP_OPTIONAL(update_cr8_intercept)

7
arch/x86/include/asm/kvm_host.h
View File

@ -1621,6 +1621,13 @@ struct kvm_x86_ops {
int (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection);
bool (*get_nmi_mask)(struct kvm_vcpu *vcpu);
void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked);
/*
* Force the vCPU out of any hardware-tracked halted/inactive state so
* that it will fetch and execute instructions on the next VM-Enter.
* Only needed by VMX, where VMCS.GUEST_ACTIVITY_STATE persists across
* VM-Exit/VM-Enter; SVM has no equivalent VMCB field.
*/
void (*clear_hlt)(struct kvm_vcpu *vcpu);
/* Whether or not a virtual NMI is pending in hardware. */
bool (*is_vnmi_pending)(struct kvm_vcpu *vcpu);
/*

8
arch/x86/include/asm/vmx.h
View File

@ -20,6 +20,14 @@
#include <asm/trapnr.h>
#include <asm/vmxfeatures.h>
/*
* UNDEFINED is a KVM-defined value used to indicate that the basic exit reason
* is undefined/invalid, e.g. on VM-Fail or if TDX never attempted VM-Enter.
* The value itself is arbitrary, it just needs to not collide with a real exit
* reason.
*/
#define EXIT_REASON_UNDEFINED 0xdead
struct vmcs_hdr {
u32 revision_id:31;
u32 shadow_vmcs:1;

90
arch/x86/kvm/vmx/common.h
View File

@ -2,10 +2,13 @@
#ifndef __KVM_X86_VMX_COMMON_H
#define __KVM_X86_VMX_COMMON_H
#include <linux/container_of.h>
#include <linux/kvm_host.h>
#include <asm/posted_intr.h>
#include <asm/vmx.h>
#include "mmu.h"
#include "vmx_ops.h"
union vmx_exit_reason {
struct {
@ -55,6 +58,8 @@ struct vcpu_vt {
#endif
};
noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu);
#ifdef CONFIG_KVM_INTEL_TDX
static __always_inline bool is_td(struct kvm *kvm)
@ -74,14 +79,62 @@ static __always_inline bool is_td_vcpu(struct kvm_vcpu *vcpu) { return false; }
#endif
struct vcpu_vmx_tdx {
struct kvm_vcpu vcpu;
struct vcpu_vt vt;
};
static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu)
{
return &(container_of(vcpu, struct vcpu_vmx_tdx, vcpu)->vt);
}
static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt)
{
return &(container_of(vt, struct vcpu_vmx_tdx, vt)->vcpu);
}
static __always_inline union vmx_exit_reason vt_get_exit_reason(struct kvm_vcpu *vcpu)
{
return to_vt(vcpu)->exit_reason;
}
static __always_inline unsigned long vt_get_exit_qual(struct kvm_vcpu *vcpu)
{
struct vcpu_vt *vt = to_vt(vcpu);
if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_1) &&
!WARN_ON_ONCE(is_td_vcpu(vcpu)))
vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
return vt->exit_qualification;
}
static __always_inline u32 vt_get_intr_info(struct kvm_vcpu *vcpu)
{
struct vcpu_vt *vt = to_vt(vcpu);
if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_2) &&
!WARN_ON_ONCE(is_td_vcpu(vcpu)))
vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
return vt->exit_intr_info;
}
static inline bool is_xfd_nm_fault(struct kvm_vcpu *vcpu)
{
return vcpu->arch.guest_fpu.fpstate->xfd &&
!kvm_is_cr0_bit_set(vcpu, X86_CR0_TS);
}
static inline bool vt_is_tdx_private_gpa(struct kvm *kvm, gpa_t gpa)
{
/* For TDX the direct mask is the shared mask. */
return !kvm_is_addr_direct(kvm, gpa);
}
static inline int __vmx_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa,
unsigned long exit_qualification)
static inline int __vt_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa,
unsigned long exit_qualification)
{
u64 error_code;
@ -164,8 +217,8 @@ static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu,
* Post an interrupt to a vCPU's PIR and trigger the vCPU to process the
* interrupt if necessary.
*/
static inline void __vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu,
struct pi_desc *pi_desc, int vector)
static inline void __vt_deliver_posted_interrupt(struct kvm_vcpu *vcpu,
struct pi_desc *pi_desc, int vector)
{
if (pi_test_and_set_pir(vector, pi_desc))
return;
@ -183,6 +236,33 @@ static inline void __vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu,
kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_VECTOR);
}
noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu);
static inline int __vt_handle_notify(struct kvm_vcpu *vcpu,
unsigned long exit_qual)
{
bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID;
++vcpu->stat.notify_window_exits;
if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER ||
context_invalid) {
vcpu->run->exit_reason = KVM_EXIT_NOTIFY;
vcpu->run->notify.flags = context_invalid ?
KVM_NOTIFY_CONTEXT_INVALID : 0;
return 0;
}
return 1;
}
static inline int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu)
{
/*
* Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK
* VM-Exits. Unconditionally set the flag here and leave the handling
* to .handle_exit() callback.
*/
to_vt(vcpu)->exit_reason.bus_lock_detected = true;
return 1;
}
#endif /* __KVM_X86_VMX_COMMON_H */

145
arch/x86/kvm/vmx/main.c
View File

@ -1,4 +1,5 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/entry-common.h>
#include <linux/moduleparam.h>
#include "x86_ops.h"
@ -10,8 +11,12 @@
#include "tdx.h"
#include "tdx_arch.h"
#pragma GCC poison to_vmx
static_assert(offsetof(struct vcpu_vmx_tdx, vt) == offsetof(struct vcpu_vmx, vt));
#ifdef CONFIG_KVM_INTEL_TDX
static_assert(offsetof(struct vcpu_vmx, vt) == offsetof(struct vcpu_tdx, vt));
static_assert(offsetof(struct vcpu_vmx_tdx, vt) == offsetof(struct vcpu_tdx, vt));
static void vt_disable_virtualization_cpu(void)
{
@ -154,15 +159,6 @@ static fastpath_t vt_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
return vmx_vcpu_run(vcpu, run_flags);
}
static int vt_handle_exit(struct kvm_vcpu *vcpu,
enum exit_fastpath_completion fastpath)
{
if (is_td_vcpu(vcpu))
return tdx_handle_exit(vcpu, fastpath);
return vmx_handle_exit(vcpu, fastpath);
}
static bool vt_unhandleable_emulation_required(struct kvm_vcpu *vcpu)
{
if (is_td_vcpu(vcpu)) {
@ -601,6 +597,18 @@ static void vt_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
vmx_set_nmi_mask(vcpu, masked);
}
static void vt_clear_hlt(struct kvm_vcpu *vcpu)
{
/*
* TDX doesn't support disabling HLT-exiting, and KVM can't access a
* TD's VMCS, so there is never any hardware halted state to clear.
*/
if (is_td_vcpu(vcpu))
return;
vmx_clear_hlt(vcpu);
}
static void vt_enable_nmi_window(struct kvm_vcpu *vcpu)
{
/* Refer to the comments in tdx_inject_nmi(). */
@ -807,7 +815,6 @@ static void vt_write_tsc_multiplier(struct kvm_vcpu *vcpu)
vmx_write_tsc_multiplier(vcpu);
}
#ifdef CONFIG_X86_64
static int vt_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc,
bool *expired)
{
@ -826,7 +833,6 @@ static void vt_cancel_hv_timer(struct kvm_vcpu *vcpu)
vmx_cancel_hv_timer(vcpu);
}
#endif
static void vt_setup_mce(struct kvm_vcpu *vcpu)
{
@ -876,6 +882,112 @@ static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn,
#define vt_op_tdx_only(name) NULL
#endif /* CONFIG_KVM_INTEL_TDX */
noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu)
{
if ((u16)vt_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI ||
!is_nmi(vt_get_intr_info(vcpu)))
return;
kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR);
kvm_after_interrupt(vcpu);
}
static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu)
{
/*
* Save xfd_err to guest_fpu before interrupt is enabled, so the
* MSR value is not clobbered by the host activity before the guest
* has chance to consume it.
*
* Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM
* interception may have been caused by L1 interception. Per the SDM,
* XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1.
*
* Note, XFD_ERR is updated _before_ the #NM interception check, i.e.
* unlike CR2 and DR6, the value is not a payload that is attached to
* the #NM exception.
*/
if (is_xfd_nm_fault(vcpu))
rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);
}
static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info)
{
/* if exit due to PF check for async PF */
if (is_page_fault(intr_info))
vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags();
/* if exit due to NM, handle before interrupts are enabled */
else if (is_nm_fault(intr_info))
handle_nm_fault_irqoff(vcpu);
/* Handle machine checks before interrupts are enabled */
else if (is_machine_check(intr_info))
kvm_machine_check();
}
static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu,
u32 intr_info)
{
unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK;
if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm,
"unexpected VM-Exit interrupt info: 0x%x", intr_info))
return;
kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector);
kvm_after_interrupt(vcpu);
vcpu->arch.at_instruction_boundary = true;
}
static void vt_handle_exit_irqoff(struct kvm_vcpu *vcpu)
{
if (to_vt(vcpu)->emulation_required)
return;
switch (vt_get_exit_reason(vcpu).basic) {
case EXIT_REASON_EXTERNAL_INTERRUPT:
handle_external_interrupt_irqoff(vcpu, vt_get_intr_info(vcpu));
break;
case EXIT_REASON_EXCEPTION_NMI:
handle_exception_irqoff(vcpu, vt_get_intr_info(vcpu));
break;
case EXIT_REASON_MCE_DURING_VMENTRY:
kvm_machine_check();
break;
default:
break;
}
}
static int vt_handle_exit(struct kvm_vcpu *vcpu,
enum exit_fastpath_completion fastpath)
{
int ret;
#ifdef CONFIG_KVM_INTEL_TDX
if (is_td_vcpu(vcpu))
ret = tdx_handle_exit(vcpu, fastpath);
else
#endif
ret = vmx_handle_exit(vcpu, fastpath);
/*
* Exit to user space when bus lock detected to inform that there is
* a bus lock in guest.
*/
if (vt_get_exit_reason(vcpu).bus_lock_detected) {
if (ret > 0) {
vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK;
ret = 0;
}
vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK;
}
return ret;
}
#define VMX_REQUIRED_APICV_INHIBITS \
(BIT(APICV_INHIBIT_REASON_DISABLED) | \
BIT(APICV_INHIBIT_REASON_ABSENT) | \
@ -949,7 +1061,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = {
.vcpu_needs_initialization = vt_op_tdx_only(vcpu_needs_initialization),
.vcpu_run = vt_op(vcpu_run),
.handle_exit = vt_op(handle_exit),
.handle_exit = vt_handle_exit,
.skip_emulated_instruction = vmx_skip_emulated_instruction,
.update_emulated_instruction = vmx_update_emulated_instruction,
.unhandleable_emulation_required = vt_op(unhandleable_emulation_required),
@ -964,6 +1076,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = {
.nmi_allowed = vt_op(nmi_allowed),
.get_nmi_mask = vt_op(get_nmi_mask),
.set_nmi_mask = vt_op(set_nmi_mask),
.clear_hlt = vt_op(clear_hlt),
.enable_nmi_window = vt_op(enable_nmi_window),
.enable_irq_window = vt_op(enable_irq_window),
.update_cr8_intercept = vt_op(update_cr8_intercept),
@ -1000,12 +1113,12 @@ struct kvm_x86_ops vt_x86_ops __initdata = {
.load_mmu_pgd = vt_op(load_mmu_pgd),
.check_intercept = vmx_check_intercept,
.handle_exit_irqoff = vmx_handle_exit_irqoff,
.handle_exit_irqoff = vt_handle_exit_irqoff,
.update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging),
.pi_update_irte = vmx_pi_update_irte,
.pi_start_bypass = vmx_pi_start_bypass,
.pi_update_irte = vt_pi_update_irte,
.pi_start_bypass = vt_pi_start_bypass,
#ifdef CONFIG_X86_64
.set_hv_timer = vt_op(set_hv_timer),

36
arch/x86/kvm/vmx/nested.c
View File

@ -446,7 +446,7 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
* "NMI unblocking due to IRET", i.e. the bit can be propagated
* as-is from the original EXIT_QUALIFICATION.
*/
exit_qualification = vmx_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI;
exit_qualification = vt_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI;
} else {
if (fault->error_code & PFERR_RSVD_MASK) {
vm_exit_reason = EXIT_REASON_EPT_MISCONFIG;
@ -472,7 +472,7 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
* and walks the faulting GPA.
*/
if (from_hardware)
exit_qualification |= vmx_get_exit_qual(vcpu) & mask;
exit_qualification |= vt_get_exit_qual(vcpu) & mask;
else
exit_qualification |= fault->exit_qualification & mask;
@ -4748,7 +4748,7 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
{
/* update exit information fields: */
vmcs12->vm_exit_reason = vm_exit_reason;
if (vmx_get_exit_reason(vcpu).enclave_mode)
if (vt_get_exit_reason(vcpu).enclave_mode)
vmcs12->vm_exit_reason |= VMX_EXIT_REASONS_SGX_ENCLAVE_MODE;
vmcs12->exit_qualification = exit_qualification;
@ -5374,7 +5374,7 @@ static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer,
struct x86_exception e;
int r;
if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu),
vmcs_read32(VMX_INSTRUCTION_INFO), false,
sizeof(*vmpointer), &gva)) {
*ret = 1;
@ -5666,7 +5666,7 @@ static int handle_vmread(struct kvm_vcpu *vcpu)
{
struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu)
: get_vmcs12(vcpu);
unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
unsigned long exit_qualification = vt_get_exit_qual(vcpu);
u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
struct vcpu_vmx *vmx = to_vmx(vcpu);
struct x86_exception e;
@ -5772,7 +5772,7 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu)
{
struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu)
: get_vmcs12(vcpu);
unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
unsigned long exit_qualification = vt_get_exit_qual(vcpu);
u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
struct vcpu_vmx *vmx = to_vmx(vcpu);
struct x86_exception e;
@ -5963,7 +5963,7 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu)
/* Emulate the VMPTRST instruction */
static int handle_vmptrst(struct kvm_vcpu *vcpu)
{
unsigned long exit_qual = vmx_get_exit_qual(vcpu);
unsigned long exit_qual = vt_get_exit_qual(vcpu);
u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
gpa_t current_vmptr;
struct x86_exception e;
@ -6032,7 +6032,7 @@ static int handle_invept(struct kvm_vcpu *vcpu)
/* According to the Intel VMX instruction reference, the memory
* operand is read even if it isn't needed (e.g., for type==global)
*/
if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu),
vmx_instruction_info, false, sizeof(operand), &gva))
return 1;
r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);
@ -6115,7 +6115,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
/* according to the intel vmx instruction reference, the memory
* operand is read even if it isn't needed (e.g., for type==global)
*/
if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu),
vmx_instruction_info, false, sizeof(operand), &gva))
return 1;
r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);
@ -6247,8 +6247,8 @@ static int handle_vmfunc(struct kvm_vcpu *vcpu)
* EXIT_REASON_VMFUNC as the exit reason.
*/
nested_vmx_vmexit(vcpu, vmx->vt.exit_reason.full,
vmx_get_intr_info(vcpu),
vmx_get_exit_qual(vcpu));
vt_get_intr_info(vcpu),
vt_get_exit_qual(vcpu));
return 1;
}
@ -6299,7 +6299,7 @@ static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu,
if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING);
exit_qualification = vmx_get_exit_qual(vcpu);
exit_qualification = vt_get_exit_qual(vcpu);
port = exit_qualification >> 16;
size = (exit_qualification & 7) + 1;
@ -6325,7 +6325,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
if (exit_reason.basic == EXIT_REASON_MSR_READ_IMM ||
exit_reason.basic == EXIT_REASON_MSR_WRITE_IMM)
msr_index = vmx_get_exit_qual(vcpu);
msr_index = vt_get_exit_qual(vcpu);
else
msr_index = kvm_ecx_read(vcpu);
@ -6361,7 +6361,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
struct vmcs12 *vmcs12)
{
unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
unsigned long exit_qualification = vt_get_exit_qual(vcpu);
int cr = exit_qualification & 15;
int reg;
unsigned long val;
@ -6495,7 +6495,7 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu,
switch ((u16)exit_reason.basic) {
case EXIT_REASON_EXCEPTION_NMI:
intr_info = vmx_get_intr_info(vcpu);
intr_info = vt_get_intr_info(vcpu);
if (is_nmi(intr_info))
return true;
else if (is_page_fault(intr_info))
@ -6578,7 +6578,7 @@ static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu,
switch ((u16)exit_reason.basic) {
case EXIT_REASON_EXCEPTION_NMI:
intr_info = vmx_get_intr_info(vcpu);
intr_info = vt_get_intr_info(vcpu);
if (is_nmi(intr_info))
return true;
else if (is_page_fault(intr_info))
@ -6748,14 +6748,14 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu)
* need to be synthesized by querying the in-kernel LAPIC, but external
* interrupts are never reflected to L1 so it's a non-issue.
*/
exit_intr_info = vmx_get_intr_info(vcpu);
exit_intr_info = vt_get_intr_info(vcpu);
if (is_exception_with_error_code(exit_intr_info)) {
struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
vmcs12->vm_exit_intr_error_code =
vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
}
exit_qual = vmx_get_exit_qual(vcpu);
exit_qual = vt_get_exit_qual(vcpu);
reflect_vmexit:
nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual);

24
arch/x86/kvm/vmx/posted_intr.c
View File

@ -14,6 +14,8 @@
#include "vmx.h"
#include "tdx.h"
#pragma GCC poison to_vmx
/*
* Maintain a per-CPU list of vCPUs that need to be awakened by wakeup_handler()
* when a WAKEUP_VECTOR interrupted is posted. vCPUs are added to the list when
@ -54,7 +56,7 @@ static int pi_try_set_control(struct pi_desc *pi_desc, u64 *pold, u64 new)
return 0;
}
void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
void vt_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
{
struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
struct vcpu_vt *vt = to_vt(vcpu);
@ -144,11 +146,11 @@ void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
pi_set_on(pi_desc);
}
static bool vmx_can_use_vtd_pi(struct kvm *kvm)
static bool vt_can_use_vtd_pi(struct kvm *kvm)
{
/*
* Note, reading the number of possible bypass IRQs can race with a
* bypass IRQ being attached to the VM. vmx_pi_start_bypass() ensures
* bypass IRQ being attached to the VM. vt_pi_start_bypass() ensures
* blockng vCPUs will see an elevated count or get KVM_REQ_UNBLOCK.
*/
return irqchip_in_kernel(kvm) && kvm_arch_has_irq_bypass() &&
@ -206,7 +208,7 @@ static void pi_enable_wakeup_handler(struct kvm_vcpu *vcpu)
__apic_send_IPI_self(POSTED_INTR_WAKEUP_VECTOR);
}
static bool vmx_needs_pi_wakeup(struct kvm_vcpu *vcpu)
static bool vt_needs_pi_wakeup(struct kvm_vcpu *vcpu)
{
/*
* The default posted interrupt vector does nothing when
@ -217,14 +219,14 @@ static bool vmx_needs_pi_wakeup(struct kvm_vcpu *vcpu)
* back to the pi_wakeup_handler() function.
*/
return (vmx_can_use_ipiv(vcpu) && !is_td_vcpu(vcpu)) ||
vmx_can_use_vtd_pi(vcpu->kvm);
vt_can_use_vtd_pi(vcpu->kvm);
}
void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
void vt_vcpu_pi_put(struct kvm_vcpu *vcpu)
{
struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
if (!vmx_needs_pi_wakeup(vcpu))
if (!vt_needs_pi_wakeup(vcpu))
return;
/*
@ -292,17 +294,17 @@ bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu)
/*
* Kick all vCPUs when the first possible bypass IRQ is attached to a VM, as
* blocking vCPUs may scheduled out without reconfiguring PID.NV to the wakeup
* vector, i.e. if the bypass IRQ came along after vmx_vcpu_pi_put().
* vector, i.e. if the bypass IRQ came along after vt_vcpu_pi_put().
*/
void vmx_pi_start_bypass(struct kvm *kvm)
void vt_pi_start_bypass(struct kvm *kvm)
{
if (WARN_ON_ONCE(!vmx_can_use_vtd_pi(kvm)))
if (WARN_ON_ONCE(!vt_can_use_vtd_pi(kvm)))
return;
kvm_make_all_cpus_request(kvm, KVM_REQ_UNBLOCK);
}
int vmx_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm,
int vt_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm,
unsigned int host_irq, uint32_t guest_irq,
struct kvm_vcpu *vcpu, u32 vector)
{

8
arch/x86/kvm/vmx/posted_intr.h
View File

@ -8,16 +8,16 @@
#include <asm/posted_intr.h>
void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu);
void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu);
void vt_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu);
void vt_vcpu_pi_put(struct kvm_vcpu *vcpu);
void pi_wakeup_handler(void);
void __init pi_init_cpu(int cpu);
void pi_apicv_pre_state_restore(struct kvm_vcpu *vcpu);
bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu);
int vmx_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm,
int vt_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm,
unsigned int host_irq, uint32_t guest_irq,
struct kvm_vcpu *vcpu, u32 vector);
void vmx_pi_start_bypass(struct kvm *kvm);
void vt_pi_start_bypass(struct kvm *kvm);
static inline int pi_find_highest_vector(struct pi_desc *pi_desc)
{

112
arch/x86/kvm/vmx/tdx.c
View File

@ -714,7 +714,7 @@ void tdx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
struct vcpu_tdx *tdx = to_tdx(vcpu);
vmx_vcpu_pi_load(vcpu, cpu);
vt_vcpu_pi_load(vcpu, cpu);
if (vcpu->cpu == cpu || !is_hkid_assigned(to_kvm_tdx(vcpu->kvm)))
return;
@ -739,7 +739,7 @@ bool tdx_interrupt_allowed(struct kvm_vcpu *vcpu)
* interrupt is always allowed unless TDX guest calls TDVMCALL with HLT,
* which passes the interrupt blocked flag.
*/
return vmx_get_exit_reason(vcpu).basic != EXIT_REASON_HLT ||
return vt_get_exit_reason(vcpu).basic != EXIT_REASON_HLT ||
!to_tdx(vcpu)->vp_enter_args.r12;
}
@ -757,7 +757,7 @@ static bool tdx_protected_apic_has_interrupt(struct kvm_vcpu *vcpu)
* otherwise the interrupt would have been serviced at the instruction
* boundary.
*/
if (vmx_get_exit_reason(vcpu).basic != EXIT_REASON_HLT ||
if (vt_get_exit_reason(vcpu).basic != EXIT_REASON_HLT ||
to_tdx(vcpu)->vp_enter_args.r12)
return false;
@ -824,7 +824,7 @@ static void tdx_prepare_switch_to_host(struct kvm_vcpu *vcpu)
void tdx_vcpu_put(struct kvm_vcpu *vcpu)
{
vmx_vcpu_pi_put(vcpu);
vt_vcpu_pi_put(vcpu);
tdx_prepare_switch_to_host(vcpu);
}
@ -919,36 +919,51 @@ static __always_inline u32 tdcall_to_vmx_exit_reason(struct kvm_vcpu *vcpu)
return EXIT_REASON_TDCALL;
}
static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu)
static __always_inline bool tdx_is_exit_reason_valid(u64 vp_enter_ret)
{
struct vcpu_tdx *tdx = to_tdx(vcpu);
u32 exit_reason;
switch (tdx->vp_enter_ret & TDX_SEAMCALL_STATUS_MASK) {
switch (vp_enter_ret & TDX_SEAMCALL_STATUS_MASK) {
case TDX_SUCCESS:
case TDX_NON_RECOVERABLE_VCPU:
case TDX_NON_RECOVERABLE_TD:
case TDX_NON_RECOVERABLE_TD_NON_ACCESSIBLE:
case TDX_NON_RECOVERABLE_TD_WRONG_APIC_MODE:
break;
return true;
default:
return -1u;
return false;
}
}
exit_reason = tdx->vp_enter_ret;
static __always_inline union vmx_exit_reason tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu)
{
struct vcpu_tdx *tdx = to_tdx(vcpu);
union vmx_exit_reason exit_reason;
switch (exit_reason) {
/*
* Return the synthesized invalid Exit Reason, as the TDX module
* never attempted to run the vCPU, i.e. the Exit Reason is undefined,
* but this is NOT a failed VM-Enter.
*/
if (!tdx_is_exit_reason_valid(tdx->vp_enter_ret))
return (union vmx_exit_reason) {
.basic = EXIT_REASON_UNDEFINED,
};
exit_reason.full = (u32)tdx->vp_enter_ret;
switch (exit_reason.basic) {
case EXIT_REASON_TDCALL:
if (tdvmcall_exit_type(vcpu))
return EXIT_REASON_VMCALL;
return tdcall_to_vmx_exit_reason(vcpu);
exit_reason.basic = EXIT_REASON_VMCALL;
else
exit_reason.basic = tdcall_to_vmx_exit_reason(vcpu);
break;
case EXIT_REASON_EPT_MISCONFIG:
/*
* Defer KVM_BUG_ON() until tdx_handle_exit() because this is in
* non-instrumentable code with interrupts disabled.
*/
return -1u;
exit_reason.basic = EXIT_REASON_UNDEFINED;
break;
default:
break;
}
@ -965,22 +980,21 @@ static noinstr void tdx_vcpu_enter_exit(struct kvm_vcpu *vcpu)
tdx->vp_enter_ret = tdh_vp_enter(&tdx->vp, &tdx->vp_enter_args);
vt->exit_reason.full = tdx_to_vmx_exit_reason(vcpu);
vt->exit_reason = tdx_to_vmx_exit_reason(vcpu);
vt->exit_qualification = tdx->vp_enter_args.rcx;
tdx->ext_exit_qualification = tdx->vp_enter_args.rdx;
tdx->exit_gpa = tdx->vp_enter_args.r8;
vt->exit_intr_info = tdx->vp_enter_args.r9;
vmx_handle_nmi(vcpu);
vt_handle_nmi(vcpu);
guest_state_exit_irqoff();
}
static bool tdx_failed_vmentry(struct kvm_vcpu *vcpu)
{
return vmx_get_exit_reason(vcpu).failed_vmentry &&
vmx_get_exit_reason(vcpu).full != -1u;
return vt_get_exit_reason(vcpu).failed_vmentry;
}
static fastpath_t tdx_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
@ -1069,8 +1083,16 @@ fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
* allowing vCPU entry to avoid contention with tdh_vp_enter() and
* TDCALLs.
*/
if (unlikely(READ_ONCE(to_kvm_tdx(vcpu->kvm)->wait_for_sept_zap)))
if (unlikely(READ_ONCE(to_kvm_tdx(vcpu->kvm)->wait_for_sept_zap))) {
/*
* The vCPU never entered the guest, but this looks like a
* handled exit to the caller. Synthesize an invalid exit
* reason so the previous exit's stale value isn't consumed
* a second time.
*/
vt->exit_reason.full = EXIT_REASON_UNDEFINED;
return EXIT_FASTPATH_EXIT_HANDLED;
}
trace_kvm_entry(vcpu, run_flags & KVM_RUN_FORCE_IMMEDIATE_EXIT);
@ -1129,7 +1151,7 @@ void tdx_inject_nmi(struct kvm_vcpu *vcpu)
static int tdx_handle_exception_nmi(struct kvm_vcpu *vcpu)
{
u32 intr_info = vmx_get_intr_info(vcpu);
u32 intr_info = vt_get_intr_info(vcpu);
/*
* Machine checks are handled by handle_exception_irqoff(), or by
@ -1890,7 +1912,7 @@ void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,
struct vcpu_tdx *tdx = to_tdx(vcpu);
/* TDX supports only posted interrupt. No lapic emulation. */
__vmx_deliver_posted_interrupt(vcpu, &tdx->vt.pi_desc, vector);
__vt_deliver_posted_interrupt(vcpu, &tdx->vt.pi_desc, vector);
trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector);
}
@ -1898,7 +1920,7 @@ void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,
static inline bool tdx_is_sept_violation_unexpected_pending(struct kvm_vcpu *vcpu)
{
u64 eeq_type = to_tdx(vcpu)->ext_exit_qualification & TDX_EXT_EXIT_QUAL_TYPE_MASK;
u64 eq = vmx_get_exit_qual(vcpu);
u64 eq = vt_get_exit_qual(vcpu);
if (eeq_type != TDX_EXT_EXIT_QUAL_TYPE_PENDING_EPT_VIOLATION)
return false;
@ -1934,7 +1956,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu)
/* Only private GPA triggers zero-step mitigation */
local_retry = true;
} else {
exit_qual = vmx_get_exit_qual(vcpu);
exit_qual = vt_get_exit_qual(vcpu);
/*
* EPT violation due to instruction fetch should never be
* triggered from shared memory in TDX guest. If such EPT
@ -1982,7 +2004,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu)
while (1) {
struct kvm_memory_slot *slot;
ret = __vmx_handle_ept_violation(vcpu, gpa, exit_qual);
ret = __vt_handle_ept_violation(vcpu, gpa, exit_qual);
if (ret != RET_PF_RETRY || !local_retry)
break;
@ -2016,7 +2038,7 @@ int tdx_complete_emulated_msr(struct kvm_vcpu *vcpu, int err)
return 1;
}
if (vmx_get_exit_reason(vcpu).basic == EXIT_REASON_MSR_READ)
if (vt_get_exit_reason(vcpu).basic == EXIT_REASON_MSR_READ)
tdvmcall_set_return_val(vcpu, kvm_read_edx_eax(vcpu));
return 1;
@ -2027,7 +2049,7 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath)
{
struct vcpu_tdx *tdx = to_tdx(vcpu);
u64 vp_enter_ret = tdx->vp_enter_ret;
union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu);
union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu);
if (fastpath != EXIT_FASTPATH_NONE)
return 1;
@ -2121,6 +2143,11 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath)
* - If it's not an MSMI, no need to do anything here.
*/
return 1;
case EXIT_REASON_NOTIFY:
/* NMI blocking state is handled by TDX module */
return __vt_handle_notify(vcpu, vt_get_exit_qual(vcpu));
case EXIT_REASON_BUS_LOCK:
return vt_handle_bus_lock_vmexit(vcpu);
default:
break;
}
@ -2136,10 +2163,10 @@ void tdx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
struct vcpu_tdx *tdx = to_tdx(vcpu);
*reason = tdx->vt.exit_reason.full;
if (*reason != -1u) {
*info1 = vmx_get_exit_qual(vcpu);
if (tdx_is_exit_reason_valid(tdx->vp_enter_ret)) {
*info1 = vt_get_exit_qual(vcpu);
*info2 = tdx->ext_exit_qualification;
*intr_info = vmx_get_intr_info(vcpu);
*intr_info = vt_get_intr_info(vcpu);
} else {
*info1 = 0;
*info2 = 0;
@ -3152,6 +3179,17 @@ static int tdx_vcpu_init(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *cmd)
td_vmcs_write64(tdx, POSTED_INTR_DESC_ADDR, __pa(&tdx->vt.pi_desc));
td_vmcs_setbit32(tdx, PIN_BASED_VM_EXEC_CONTROL, PIN_BASED_POSTED_INTR);
if (kvm_notify_vmexit_enabled(vcpu->kvm)) {
td_vmcs_setbit32(tdx, SECONDARY_VM_EXEC_CONTROL,
SECONDARY_EXEC_NOTIFY_VM_EXITING);
td_vmcs_write32(tdx, NOTIFY_WINDOW,
vcpu->kvm->arch.notify_window);
}
if (vcpu->kvm->arch.bus_lock_detection_enabled)
td_vmcs_setbit32(tdx, SECONDARY_VM_EXEC_CONTROL,
SECONDARY_EXEC_BUS_LOCK_DETECTION);
tdx->state = VCPU_TD_STATE_INITIALIZED;
return 0;
@ -3217,6 +3255,7 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
struct kvm_tdx_init_mem_region region;
struct tdx_gmem_post_populate_arg arg;
gpa_t nr_bytes, end_gpa;
long gmem_ret;
int ret;
@ -3235,10 +3274,13 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c
if (!PAGE_ALIGNED(region.source_addr) ||
(!gmem_in_place_conversion && !region.source_addr) ||
!PAGE_ALIGNED(region.gpa) || !region.nr_pages ||
region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa ||
!PAGE_ALIGNED(region.gpa) || !region.nr_pages)
return -EINVAL;
if (check_shl_overflow(region.nr_pages, PAGE_SHIFT, &nr_bytes) ||
check_add_overflow(region.gpa, nr_bytes - 1, &end_gpa) ||
!vt_is_tdx_private_gpa(kvm, region.gpa) ||
!vt_is_tdx_private_gpa(kvm, region.gpa + (region.nr_pages << PAGE_SHIFT) - 1))
!vt_is_tdx_private_gpa(kvm, end_gpa))
return -EINVAL;
ret = 0;

205
arch/x86/kvm/vmx/vmx.c
View File

@ -1577,12 +1577,12 @@ void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
vmx_vcpu_load_vmcs(vcpu, cpu);
vmx_vcpu_pi_load(vcpu, cpu);
vt_vcpu_pi_load(vcpu, cpu);
}
void vmx_vcpu_put(struct kvm_vcpu *vcpu)
{
vmx_vcpu_pi_put(vcpu);
vt_vcpu_pi_put(vcpu);
vmx_prepare_switch_to_host(to_vmx(vcpu));
}
@ -1791,7 +1791,7 @@ int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
* so that guest userspace can't DoS the guest simply by triggering
* emulation (enclaves are CPL3 only).
*/
if (vmx_get_exit_reason(vcpu).enclave_mode) {
if (vt_get_exit_reason(vcpu).enclave_mode) {
kvm_queue_exception(vcpu, UD_VECTOR);
return X86EMUL_PROPAGATE_FAULT;
}
@ -1806,7 +1806,7 @@ int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu);
union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu);
unsigned long rip, orig_rip;
u32 instr_len;
@ -1909,7 +1909,7 @@ int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu)
return skip_emulated_instruction(vcpu);
}
static void vmx_clear_hlt(struct kvm_vcpu *vcpu)
void vmx_clear_hlt(struct kvm_vcpu *vcpu)
{
/*
* Ensure that we clear the HLT state in the VMCS. We don't need to
@ -4436,7 +4436,7 @@ static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
if (!vcpu->arch.apic->apicv_active)
return -1;
__vmx_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector);
__vt_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector);
return 0;
}
@ -5392,15 +5392,9 @@ bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu)
(kvm_get_rflags(vcpu) & X86_EFLAGS_AC);
}
static bool is_xfd_nm_fault(struct kvm_vcpu *vcpu)
{
return vcpu->arch.guest_fpu.fpstate->xfd &&
!kvm_is_cr0_bit_set(vcpu, X86_CR0_TS);
}
static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code)
{
unsigned long cr2 = vmx_get_exit_qual(vcpu);
unsigned long cr2 = vt_get_exit_qual(vcpu);
if (vcpu->arch.apf.host_apf_flags)
goto handle_pf;
@ -5448,7 +5442,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu)
u32 vect_info;
vect_info = vmx->idt_vectoring_info;
intr_info = vmx_get_intr_info(vcpu);
intr_info = vt_get_intr_info(vcpu);
/*
* Machine checks are handled by handle_exception_irqoff(), or by
@ -5529,7 +5523,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu)
switch (ex_no) {
case DB_VECTOR:
dr6 = vmx_get_exit_qual(vcpu);
dr6 = vt_get_exit_qual(vcpu);
if (!(vcpu->guest_debug &
(KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
/*
@ -5605,7 +5599,7 @@ static int handle_io(struct kvm_vcpu *vcpu)
int size, in, string;
unsigned port;
exit_qualification = vmx_get_exit_qual(vcpu);
exit_qualification = vt_get_exit_qual(vcpu);
string = (exit_qualification & 16) != 0;
++vcpu->stat.io_exits;
@ -5694,7 +5688,7 @@ static int handle_cr(struct kvm_vcpu *vcpu)
int err;
int ret;
exit_qualification = vmx_get_exit_qual(vcpu);
exit_qualification = vt_get_exit_qual(vcpu);
cr = exit_qualification & 15;
reg = (exit_qualification >> 8) & 15;
switch ((exit_qualification >> 4) & 3) {
@ -5772,7 +5766,7 @@ static int handle_dr(struct kvm_vcpu *vcpu)
int dr, dr7, reg;
int err = 1;
exit_qualification = vmx_get_exit_qual(vcpu);
exit_qualification = vt_get_exit_qual(vcpu);
dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
/* First, if DR does not exist, trigger UD */
@ -5868,7 +5862,7 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu)
static int handle_invlpg(struct kvm_vcpu *vcpu)
{
unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
unsigned long exit_qualification = vt_get_exit_qual(vcpu);
kvm_mmu_invlpg(vcpu, exit_qualification);
return kvm_skip_emulated_instruction(vcpu);
@ -5877,7 +5871,7 @@ static int handle_invlpg(struct kvm_vcpu *vcpu)
static int handle_apic_access(struct kvm_vcpu *vcpu)
{
if (likely(fasteoi)) {
unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
unsigned long exit_qualification = vt_get_exit_qual(vcpu);
int access_type, offset;
access_type = exit_qualification & APIC_ACCESS_TYPE;
@ -5898,7 +5892,7 @@ static int handle_apic_access(struct kvm_vcpu *vcpu)
static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu)
{
unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
unsigned long exit_qualification = vt_get_exit_qual(vcpu);
/* EOI-induced VM exit is trap-like and thus no need to adjust IP */
kvm_apic_set_eoi_accelerated(vcpu, exit_qualification & 0xff);
@ -5907,7 +5901,7 @@ static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu)
static int handle_apic_write(struct kvm_vcpu *vcpu)
{
unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
unsigned long exit_qualification = vt_get_exit_qual(vcpu);
/*
* APIC-write VM-Exit is trap-like, KVM doesn't need to advance RIP and
@ -5935,7 +5929,7 @@ static int handle_task_switch(struct kvm_vcpu *vcpu)
idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK);
type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
exit_qualification = vmx_get_exit_qual(vcpu);
exit_qualification = vt_get_exit_qual(vcpu);
reason = (u32)exit_qualification >> 30;
if (reason == TASK_SWITCH_GATE && idt_v) {
@ -5981,7 +5975,7 @@ static int handle_task_switch(struct kvm_vcpu *vcpu)
static int handle_ept_violation(struct kvm_vcpu *vcpu)
{
unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
unsigned long exit_qualification = vt_get_exit_qual(vcpu);
gpa_t gpa;
/*
@ -6009,7 +6003,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu)
if (unlikely(allow_smaller_maxphyaddr && !kvm_vcpu_is_legal_gpa(vcpu, gpa)))
return kvm_emulate_instruction(vcpu, 0);
return __vmx_handle_ept_violation(vcpu, gpa, exit_qualification);
return __vt_handle_ept_violation(vcpu, gpa, exit_qualification);
}
static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
@ -6172,7 +6166,7 @@ static int handle_invpcid(struct kvm_vcpu *vcpu)
/* According to the Intel instruction reference, the memory operand
* is read even if it isn't needed (e.g., for type==all)
*/
if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu),
vmx_instruction_info, false,
sizeof(operand), &gva))
return 1;
@ -6186,7 +6180,7 @@ static int handle_pml_full(struct kvm_vcpu *vcpu)
trace_kvm_pml_full(vcpu->vcpu_id);
exit_qualification = vmx_get_exit_qual(vcpu);
exit_qualification = vt_get_exit_qual(vcpu);
/*
* PML buffer FULL happened while executing iret from NMI,
@ -6277,23 +6271,9 @@ static int handle_encls(struct kvm_vcpu *vcpu)
}
#endif /* CONFIG_X86_SGX_KVM */
static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu)
{
/*
* Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK
* VM-Exits. Unconditionally set the flag here and leave the handling to
* vmx_handle_exit().
*/
to_vt(vcpu)->exit_reason.bus_lock_detected = true;
return 1;
}
static int handle_notify(struct kvm_vcpu *vcpu)
{
unsigned long exit_qual = vmx_get_exit_qual(vcpu);
bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID;
++vcpu->stat.notify_window_exits;
unsigned long exit_qual = vt_get_exit_qual(vcpu);
/*
* Notify VM exit happened while executing iret from NMI,
@ -6303,15 +6283,7 @@ static int handle_notify(struct kvm_vcpu *vcpu)
vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
GUEST_INTR_STATE_NMI);
if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER ||
context_invalid) {
vcpu->run->exit_reason = KVM_EXIT_NOTIFY;
vcpu->run->notify.flags = context_invalid ?
KVM_NOTIFY_CONTEXT_INVALID : 0;
return 0;
}
return 1;
return __vt_handle_notify(vcpu, exit_qual);
}
static int vmx_get_msr_imm_reg(struct kvm_vcpu *vcpu)
@ -6321,13 +6293,13 @@ static int vmx_get_msr_imm_reg(struct kvm_vcpu *vcpu)
static int handle_rdmsr_imm(struct kvm_vcpu *vcpu)
{
return kvm_emulate_rdmsr_imm(vcpu, vmx_get_exit_qual(vcpu),
return kvm_emulate_rdmsr_imm(vcpu, vt_get_exit_qual(vcpu),
vmx_get_msr_imm_reg(vcpu));
}
static int handle_wrmsr_imm(struct kvm_vcpu *vcpu)
{
return kvm_emulate_wrmsr_imm(vcpu, vmx_get_exit_qual(vcpu),
return kvm_emulate_wrmsr_imm(vcpu, vt_get_exit_qual(vcpu),
vmx_get_msr_imm_reg(vcpu));
}
@ -6387,7 +6359,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
[EXIT_REASON_VMFUNC] = handle_vmx_instruction,
[EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer,
[EXIT_REASON_ENCLS] = handle_encls,
[EXIT_REASON_BUS_LOCK] = handle_bus_lock_vmexit,
[EXIT_REASON_BUS_LOCK] = vt_handle_bus_lock_vmexit,
[EXIT_REASON_NOTIFY] = handle_notify,
[EXIT_REASON_SEAMCALL] = handle_tdx_instruction,
[EXIT_REASON_TDCALL] = handle_tdx_instruction,
@ -6398,16 +6370,18 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
static const int kvm_vmx_max_exit_handlers =
ARRAY_SIZE(kvm_vmx_exit_handlers);
static_assert(EXIT_REASON_UNDEFINED >= ARRAY_SIZE(kvm_vmx_exit_handlers));
void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code)
{
struct vcpu_vmx *vmx = to_vmx(vcpu);
*reason = vmx->vt.exit_reason.full;
*info1 = vmx_get_exit_qual(vcpu);
*info1 = vt_get_exit_qual(vcpu);
if (!(vmx->vt.exit_reason.failed_vmentry)) {
*info2 = vmx->idt_vectoring_info;
*intr_info = vmx_get_intr_info(vcpu);
*intr_info = vt_get_intr_info(vcpu);
if (is_exception_with_error_code(*intr_info))
*error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
else
@ -6692,10 +6666,10 @@ void dump_vmcs(struct kvm_vcpu *vcpu)
* The guest has exited. See if we can fix it or if we need userspace
* assistance.
*/
static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
{
struct vcpu_vmx *vmx = to_vmx(vcpu);
union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu);
union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu);
u32 vectoring_info = vmx->idt_vectoring_info;
u16 exit_handler_index;
@ -6851,24 +6825,6 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
return 0;
}
int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
{
int ret = __vmx_handle_exit(vcpu, exit_fastpath);
/*
* Exit to user space when bus lock detected to inform that there is
* a bus lock in guest.
*/
if (vmx_get_exit_reason(vcpu).bus_lock_detected) {
if (ret > 0)
vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK;
vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK;
return 0;
}
return ret;
}
void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
{
int tpr_threshold;
@ -7132,74 +7088,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
}
static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu)
{
/*
* Save xfd_err to guest_fpu before interrupt is enabled, so the
* MSR value is not clobbered by the host activity before the guest
* has chance to consume it.
*
* Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM
* interception may have been caused by L1 interception. Per the SDM,
* XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1.
*
* Note, XFD_ERR is updated _before_ the #NM interception check, i.e.
* unlike CR2 and DR6, the value is not a payload that is attached to
* the #NM exception.
*/
if (is_xfd_nm_fault(vcpu))
rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);
}
static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info)
{
/* if exit due to PF check for async PF */
if (is_page_fault(intr_info))
vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags();
/* if exit due to NM, handle before interrupts are enabled */
else if (is_nm_fault(intr_info))
handle_nm_fault_irqoff(vcpu);
/* Handle machine checks before interrupts are enabled */
else if (is_machine_check(intr_info))
kvm_machine_check();
}
static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu,
u32 intr_info)
{
unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK;
if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm,
"unexpected VM-Exit interrupt info: 0x%x", intr_info))
return;
kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector);
kvm_after_interrupt(vcpu);
vcpu->arch.at_instruction_boundary = true;
}
void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu)
{
if (to_vt(vcpu)->emulation_required)
return;
switch (vmx_get_exit_reason(vcpu).basic) {
case EXIT_REASON_EXTERNAL_INTERRUPT:
handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu));
break;
case EXIT_REASON_EXCEPTION_NMI:
handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu));
break;
case EXIT_REASON_MCE_DURING_VMENTRY:
kvm_machine_check();
break;
default:
break;
}
}
/*
* The kvm parameter can be NULL (module initialization, or invocation before
* VM creation). Be sure to check the kvm parameter before using it.
@ -7239,7 +7127,7 @@ static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
if (vmx->loaded_vmcs->nmi_known_unmasked)
return;
exit_intr_info = vmx_get_intr_info(&vmx->vcpu);
exit_intr_info = vt_get_intr_info(&vmx->vcpu);
unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
/*
@ -7406,14 +7294,14 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu,
* the fastpath even, all other exits must use the slow path.
*/
if (is_guest_mode(vcpu) &&
vmx_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER)
vt_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER)
return EXIT_FASTPATH_NONE;
switch (vmx_get_exit_reason(vcpu).basic) {
switch (vt_get_exit_reason(vcpu).basic) {
case EXIT_REASON_MSR_WRITE:
return handle_fastpath_wrmsr(vcpu);
case EXIT_REASON_MSR_WRITE_IMM:
return handle_fastpath_wrmsr_imm(vcpu, vmx_get_exit_qual(vcpu),
return handle_fastpath_wrmsr_imm(vcpu, vt_get_exit_qual(vcpu),
vmx_get_msr_imm_reg(vcpu));
case EXIT_REASON_PREEMPTION_TIMER:
return handle_fastpath_preemption_timer(vcpu, force_immediate_exit);
@ -7426,17 +7314,6 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu,
}
}
noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu)
{
if ((u16)vmx_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI ||
!is_nmi(vmx_get_intr_info(vcpu)))
return;
kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR);
kvm_after_interrupt(vcpu);
}
static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,
unsigned int flags)
{
@ -7461,15 +7338,15 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,
vmx_enable_fb_clear(vmx);
if (unlikely(vmx->fail)) {
vmx->vt.exit_reason.full = 0xdead;
vmx->vt.exit_reason.full = EXIT_REASON_UNDEFINED;
goto out;
}
vmx->vt.exit_reason.full = vmcs_read32(VM_EXIT_REASON);
if (likely(!vmx_get_exit_reason(vcpu).failed_vmentry))
if (likely(!vt_get_exit_reason(vcpu).failed_vmentry))
vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
vmx_handle_nmi(vcpu);
vt_handle_nmi(vcpu);
out:
guest_state_exit_irqoff();
@ -7606,7 +7483,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
* checking.
*/
if (vcpu->arch.nested_run_pending &&
!vmx_get_exit_reason(vcpu).failed_vmentry)
!vt_get_exit_reason(vcpu).failed_vmentry)
++vcpu->stat.nested_run;
vcpu->arch.nested_run_pending = 0;
@ -7617,7 +7494,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
trace_kvm_exit(vcpu, KVM_ISA_VMX);
if (unlikely(vmx_get_exit_reason(vcpu).failed_vmentry))
if (unlikely(vt_get_exit_reason(vcpu).failed_vmentry))
return EXIT_FASTPATH_NONE;
vmx->loaded_vmcs->launched = 1;

37
arch/x86/kvm/vmx/vmx.h
View File

@ -297,43 +297,6 @@ struct kvm_vmx {
u64 *pid_table;
};
static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu)
{
return &(container_of(vcpu, struct vcpu_vmx, vcpu)->vt);
}
static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt)
{
return &(container_of(vt, struct vcpu_vmx, vt)->vcpu);
}
static __always_inline union vmx_exit_reason vmx_get_exit_reason(struct kvm_vcpu *vcpu)
{
return to_vt(vcpu)->exit_reason;
}
static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu)
{
struct vcpu_vt *vt = to_vt(vcpu);
if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_1) &&
!WARN_ON_ONCE(is_td_vcpu(vcpu)))
vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
return vt->exit_qualification;
}
static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu)
{
struct vcpu_vt *vt = to_vt(vcpu);
if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_2) &&
!WARN_ON_ONCE(is_td_vcpu(vcpu)))
vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
return vt->exit_intr_info;
}
void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu);
int allocate_vpid(void);
void free_vpid(int vpid);

2
arch/x86/kvm/vmx/x86_ops.h
View File

@ -27,7 +27,6 @@ void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event);
void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
void vmx_vcpu_put(struct kvm_vcpu *vcpu);
int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath);
void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu);
int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu);
void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu);
bool vmx_unhandleable_emulation_required(struct kvm_vcpu *vcpu);
@ -95,6 +94,7 @@ int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection);
int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection);
bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu);
void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked);
void vmx_clear_hlt(struct kvm_vcpu *vcpu);
void vmx_enable_nmi_window(struct kvm_vcpu *vcpu);
void vmx_enable_irq_window(struct kvm_vcpu *vcpu);
void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr);

13
arch/x86/kvm/x86.c
View File

@ -9136,6 +9136,19 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
}
kvm_set_mp_state(vcpu, mp_state->mp_state);
/*
* Force the vCPU out of any hardware-tracked halted state, e.g. VMX's
* GUEST_ACTIVITY_STATE=HLT, when userspace puts the vCPU into a state
* other than HALTED. The hardware state is sticky across VM-Exit and
* VM-Enter and is not touched by any other ioctl, so a vCPU that halted
* with HLT-exiting disabled stays wedged even after userspace rewrites
* its registers, e.g. when a VMM emulates a machine reset.
*/
if (kvm_hlt_in_guest(vcpu->kvm) &&
mp_state->mp_state != KVM_MP_STATE_HALTED)
kvm_x86_call(clear_hlt)(vcpu);
kvm_make_request(KVM_REQ_EVENT, vcpu);
ret = 0;