github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 17:47:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

KVM misc x86 changes for 7.2

Browse Source 
- Handle EXIT_FASTPATH_EXIT_USERSPACE in vendor code to ensure vendor code
    gets a chance to handle things like reaping the PML buffer.
 
  - Ensure KVM's copy of CR0 and CR3 are up-to-date on SVM prior to invoking
    fastpath handlers.
 
  - Update KVM's view of PV async enabling if and only if the MSR write fully
    succeeds.
 
  - Fix a variety of issues where the emulator doesn't honor guest-debug state,
    and clean up related code along the way.
 
  - Synthesize EPT Violation and #NPF "error code" bits when injecting faults
    into L1 that didn't originate in hardware (in which case the VMCS/VMCB
    doesn't hold relevant information).
 
  - Add support for virtualizing (well, emulating) AMD's flavor of CPL>0 CPUID
    faulting.
 
  - Clean up the GPR APIs so that KVM's use of "raw" is consistent, and fix a
    variety of minor bugs along the way.
 
  - Fix an OOB memory access due to not checking the VP ID when handling a
    Hyper-V PV TLB flush for L2.
 
  - Fix a bug in the mediated PMU's handling of fixed counters that allowed the
    guest to bypass the PMU event filter.
 
  - Allow userspace to return EAGAIN when handling SNP and TDX hypercalls, so
    the KVM can forward a "retry" status code to the guest, and reserve all
    unused error codes for future usage.
 
  - Misc fixes and cleanups.
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEKTobbabEP7vbhhN9OlYIJqCjN/0FAmorPacACgkQOlYIJqCj
 N/2f4BAAiWMEfbazgtGLYQyiSd2QtIw41Dpcz9cnSBxvuFztulewTpd5z2B4/8kX
 dSeZ3vyofgOVUVIj4AlD6Ap8ZO7XL3FizLcjWv9c8l4j/sTREL1ix0oejPJpBASg
 5EmSueBjmj+8VbBNxHGCYMuU4okR0UBmU86sC0wBuho+CvqkNegSrmCpntHUh4Kn
 34/HZ6QTik7iZNXV7KVT7/2De/bZV/9RJ7T8EAOkOGRfWft7cVUmkCXBP184pqee
 Z9X1gcymXb17Z73GbOFbM3SEewcYLiy1LdDyk1Tid3Wk6nJlMj0Tc0djsI6Vz7+8
 UvaKNoAXyWydPzJmYuCuTP+OnrZrha3RuEZvpP9UBJqGQgNjzs1jis92jiPL3yf4
 38zcNesukLU8RWKMHw4PYMQtDIVwqPloSPN9tPZ6U+9IWZx88H/IRMd080Jxsbem
 UNCuB88PawtlYV7gNk2XvQ0gcLib3LTjVjrWyTloUDWTjsMEXYxQ7Ehn6Jk+dfya
 BMjivWY246SqsMBmkj4Lj0S6uD25z9Yv3uPcwshZ3nuXCjdmBKlWNSs7cFYopI1t
 h4urP9xX/eURs63f++nB+6ZsqMR2ESQXmbybvsevR/+GFb0Iuy3obJk4ab2kKvP/
 OhFpo8AmJ3jOK2RrIbZ/wUb5rfZbvnCxjAqSQ1YZHIMkrD9vlTY=
 =IHTg
 -----END PGP SIGNATURE-----

Merge tag 'kvm-x86-misc-7.2' of https://github.com/kvm-x86/linux into HEAD

KVM misc x86 changes for 7.2

 - Handle EXIT_FASTPATH_EXIT_USERSPACE in vendor code to ensure vendor code
   gets a chance to handle things like reaping the PML buffer.

 - Ensure KVM's copy of CR0 and CR3 are up-to-date on SVM prior to invoking
   fastpath handlers.

 - Update KVM's view of PV async enabling if and only if the MSR write fully
   succeeds.

 - Fix a variety of issues where the emulator doesn't honor guest-debug state,
   and clean up related code along the way.

 - Synthesize EPT Violation and #NPF "error code" bits when injecting faults
   into L1 that didn't originate in hardware (in which case the VMCS/VMCB
   doesn't hold relevant information).

 - Add support for virtualizing (well, emulating) AMD's flavor of CPL>0 CPUID
   faulting.

 - Clean up the GPR APIs so that KVM's use of "raw" is consistent, and fix a
   variety of minor bugs along the way.

 - Fix an OOB memory access due to not checking the VP ID when handling a
   Hyper-V PV TLB flush for L2.

 - Fix a bug in the mediated PMU's handling of fixed counters that allowed the
   guest to bypass the PMU event filter.

 - Allow userspace to return EAGAIN when handling SNP and TDX hypercalls, so
   the KVM can forward a "retry" status code to the guest, and reserve all
   unused error codes for future usage.

 - Misc fixes and cleanups.
This commit is contained in:
Paolo Bonzini 2026-06-12 10:11:09 +02:00
commit f6d6be78b2
42 changed files with 1116 additions and 547 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Documentation/virt/kvm/api.rst
View File

@ -8818,6 +8818,9 @@ block sizes is exposed in KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES as a
This capability, if enabled, will cause KVM to exit to userspace
with KVM_EXIT_HYPERCALL exit reason to process some hypercalls.
Userspace may fail the hypercall by setting hypercall.ret to EINVAL
or may request the hypercall to be retried the next time the guest run
by setting hypercall.ret to EAGAIN.
Calling KVM_CHECK_EXTENSION for this capability will return a bitmask
of hypercalls that can be configured to exit to userspace.
@ -9363,6 +9366,8 @@ means the VM type with value @n is supported. Possible values of @n are::
#define KVM_X86_SW_PROTECTED_VM 1
#define KVM_X86_SEV_VM 2
#define KVM_X86_SEV_ES_VM 3
#define KVM_X86_SNP_VM 4
#define KVM_X86_TDX_VM 5
Note, KVM_X86_SW_PROTECTED_VM is currently only for development and testing.
Do not use KVM_X86_SW_PROTECTED_VM for "real" VMs, and especially not in

1
MAINTAINERS
View File

@ -14162,6 +14162,7 @@ L: kvm@vger.kernel.org
S: Supported
P: Documentation/process/maintainer-kvm-x86.rst
T: git git://git.kernel.org/pub/scm/virt/kvm/kvm.git
F: Documentation/process/maintainer-kvm-x86.rst
F: arch/x86/include/asm/kvm*
F: arch/x86/include/asm/svm.h
F: arch/x86/include/asm/vmx*.h

27
arch/x86/include/asm/kvm_host.h
View File

@ -284,6 +284,8 @@ enum x86_intercept_stage;
#define PFERR_GUEST_RMP_MASK BIT_ULL(31)
#define PFERR_GUEST_FINAL_MASK BIT_ULL(32)
#define PFERR_GUEST_PAGE_MASK BIT_ULL(33)
#define PFERR_GUEST_FAULT_STAGE_MASK \
(PFERR_GUEST_FINAL_MASK | PFERR_GUEST_PAGE_MASK)
#define PFERR_GUEST_ENC_MASK BIT_ULL(34)
#define PFERR_GUEST_SIZEM_MASK BIT_ULL(35)
#define PFERR_GUEST_VMPL_MASK BIT_ULL(36)
@ -484,7 +486,8 @@ struct kvm_mmu {
u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
void (*inject_page_fault)(struct kvm_vcpu *vcpu,
struct x86_exception *fault);
struct x86_exception *fault,
bool from_hardware);
gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
gpa_t gva_or_gpa, u64 access,
struct x86_exception *exception);
@ -1057,8 +1060,6 @@ struct kvm_vcpu_arch {
u16 vec;
u32 id;
u32 host_apf_flags;
bool send_always;
bool delivery_as_pf_vmexit;
bool pageready_pending;
} apf;
@ -1441,6 +1442,7 @@ struct kvm_arch {
bool has_private_mem;
bool has_protected_state;
bool has_protected_eoi;
bool has_protected_pmu;
bool pre_fault_allowed;
struct hlist_head *mmu_page_hash;
struct list_head active_mmu_pages;
@ -2155,8 +2157,6 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3);
extern bool tdp_enabled;
u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu);
/*
* EMULTYPE_NO_DECODE - Set when re-emulating an instruction (after completing
* userspace I/O) to indicate that the emulation context
@ -2276,7 +2276,6 @@ int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu);
void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
void kvm_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, int seg);
void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector);
int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
@ -2307,10 +2306,18 @@ void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, unsigned long payload);
void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned int nr,
bool has_error_code, u32 error_code);
void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault);
void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
struct x86_exception *fault);
bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl);
void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault,
bool from_hardware);
void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
struct x86_exception *fault,
bool from_hardware);
static inline void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
struct x86_exception *fault)
{
__kvm_inject_emulated_page_fault(vcpu, fault, false);
}
bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr);
static inline int __kvm_irq_line_state(unsigned long *irq_state,

1
arch/x86/include/asm/msr-index.h
View File

@ -899,6 +899,7 @@
#define MSR_K7_HWCR_IRPERF_EN_BIT 30
#define MSR_K7_HWCR_IRPERF_EN BIT_ULL(MSR_K7_HWCR_IRPERF_EN_BIT)
#define MSR_K7_HWCR_CPUID_USER_DIS_BIT 35
#define MSR_K7_HWCR_CPUID_USER_DIS BIT_ULL(MSR_K7_HWCR_CPUID_USER_DIS_BIT)
#define MSR_K7_FID_VID_CTL 0xc0010041
#define MSR_K7_FID_VID_STATUS 0xc0010042
#define MSR_K7_HWCR_CPB_DIS_BIT 25

21
arch/x86/kvm/cpuid.c
View File

@ -11,7 +11,7 @@
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/kvm_host.h>
#include "linux/lockdep.h"
#include <linux/lockdep.h>
#include <linux/export.h>
#include <linux/vmalloc.h>
#include <linux/uaccess.h>
@ -1248,7 +1248,7 @@ void kvm_initialize_cpu_caps(void)
F(AUTOIBRS),
EMULATED_F(NO_SMM_CTL_MSR),
/* PrefetchCtlMsr */
/* GpOnUserCpuid */
EMULATED_F(GP_ON_USER_CPUID),
/* EPSF */
F(PREFETCHI),
F(AVX512_BMM),
@ -2161,17 +2161,18 @@ int kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
{
u32 eax, ebx, ecx, edx;
if (!is_smm(vcpu) && cpuid_fault_enabled(vcpu) &&
!kvm_require_cpl(vcpu, 0))
if (!kvm_is_cpuid_allowed(vcpu)) {
kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
return 1;
}
eax = kvm_rax_read(vcpu);
ecx = kvm_rcx_read(vcpu);
eax = kvm_eax_read(vcpu);
ecx = kvm_ecx_read(vcpu);
kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx, false);
kvm_rax_write(vcpu, eax);
kvm_rbx_write(vcpu, ebx);
kvm_rcx_write(vcpu, ecx);
kvm_rdx_write(vcpu, edx);
kvm_eax_write(vcpu, eax);
kvm_ebx_write(vcpu, ebx);
kvm_ecx_write(vcpu, ecx);
kvm_edx_write(vcpu, edx);
return kvm_skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_cpuid);

20
arch/x86/kvm/cpuid.h
View File

@ -7,6 +7,8 @@
#include <asm/processor.h>
#include <uapi/asm/kvm_para.h>
#include "smm.h"
extern u32 kvm_cpu_caps[NR_KVM_CPU_CAPS] __read_mostly;
extern bool kvm_is_configuring_cpu_caps __read_mostly;
@ -124,7 +126,7 @@ static __always_inline bool guest_cpuid_has(struct kvm_vcpu *vcpu,
entry = kvm_find_cpuid_entry_index(vcpu, cpuid.function, cpuid.index);
if (!entry)
return NULL;
return false;
reg = __cpuid_entry_get_reg(entry, cpuid.reg);
if (!reg)
@ -181,15 +183,17 @@ static inline int guest_cpuid_stepping(struct kvm_vcpu *vcpu)
return x86_stepping(best->eax);
}
static inline bool supports_cpuid_fault(struct kvm_vcpu *vcpu)
{
return vcpu->arch.msr_platform_info & MSR_PLATFORM_INFO_CPUID_FAULT;
}
static inline bool cpuid_fault_enabled(struct kvm_vcpu *vcpu)
{
return vcpu->arch.msr_misc_features_enables &
MSR_MISC_FEATURES_ENABLES_CPUID_FAULT;
return (vcpu->arch.msr_misc_features_enables &
MSR_MISC_FEATURES_ENABLES_CPUID_FAULT) ||
(vcpu->arch.msr_hwcr & MSR_K7_HWCR_CPUID_USER_DIS);
}
static inline bool kvm_is_cpuid_allowed(struct kvm_vcpu *vcpu)
{
return !cpuid_fault_enabled(vcpu) || is_smm(vcpu) ||
!kvm_x86_call(get_cpl)(vcpu);
}
static __always_inline void kvm_cpu_cap_clear(unsigned int x86_feature)

2
arch/x86/kvm/debugfs.c
View File

@ -121,7 +121,7 @@ static int kvm_mmu_rmaps_stat_show(struct seq_file *m, void *v)
lpage_size = kvm_mmu_slot_lpages(slot, k + 1);
cur = log[k];
for (l = 0; l < lpage_size; l++) {
index = ffs(pte_list_count(&rmap[l]));
index = fls(pte_list_count(&rmap[l]));
if (WARN_ON_ONCE(index >= RMAP_LOG_SIZE))
index = RMAP_LOG_SIZE - 1;
cur[index]++;

22
arch/x86/kvm/emulate.c
View File

@ -20,7 +20,7 @@
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/kvm_host.h>
#include "kvm_cache_regs.h"
#include "regs.h"
#include "kvm_emulate.h"
#include <linux/stringify.h>
#include <asm/debugreg.h>
@ -540,8 +540,9 @@ static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
return X86EMUL_PROPAGATE_FAULT;
}
static int emulate_db(struct x86_emulate_ctxt *ctxt)
static int emulate_db(struct x86_emulate_ctxt *ctxt, unsigned long dr6)
{
ctxt->exception.dr6 = dr6;
return emulate_exception(ctxt, DB_VECTOR, 0, false);
}
@ -3593,12 +3594,8 @@ static int em_sti(struct x86_emulate_ctxt *ctxt)
static int em_cpuid(struct x86_emulate_ctxt *ctxt)
{
u32 eax, ebx, ecx, edx;
u64 msr = 0;
ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
if (!ctxt->ops->is_smm(ctxt) &&
(msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT) &&
ctxt->ops->cpl(ctxt))
if (!ctxt->ops->is_cpuid_allowed(ctxt))
return emulate_gp(ctxt, 0);
eax = reg_read(ctxt, VCPU_REGS_RAX);
@ -3847,15 +3844,8 @@ static int check_dr_read(struct x86_emulate_ctxt *ctxt)
if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
return emulate_ud(ctxt);
if (ctxt->ops->get_dr(ctxt, 7) & DR7_GD) {
ulong dr6;
dr6 = ctxt->ops->get_dr(ctxt, 6);
dr6 &= ~DR_TRAP_BITS;
dr6 |= DR6_BD | DR6_ACTIVE_LOW;
ctxt->ops->set_dr(ctxt, 6, dr6);
return emulate_db(ctxt);
}
if (ctxt->ops->get_effective_dr7(ctxt) & DR7_GD)
return emulate_db(ctxt, DR6_BD);
return X86EMUL_CONTINUE;
}

26
arch/x86/kvm/hyperv.c
View File

@ -1839,6 +1839,11 @@ static bool hv_is_vp_in_sparse_set(u32 vp_id, u64 valid_bank_mask, u64 sparse_ba
int valid_bit_nr = vp_id / HV_VCPUS_PER_SPARSE_BANK;
unsigned long sbank;
BUILD_BUG_ON(BITS_PER_TYPE(valid_bank_mask) != HV_MAX_SPARSE_VCPU_BANKS);
if (valid_bit_nr >= HV_MAX_SPARSE_VCPU_BANKS)
return false;
if (!test_bit(valid_bit_nr, (unsigned long *)&valid_bank_mask))
return false;
@ -2377,10 +2382,10 @@ static void kvm_hv_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result)
longmode = is_64_bit_hypercall(vcpu);
if (longmode)
kvm_rax_write(vcpu, result);
kvm_rax_write_raw(vcpu, result);
else {
kvm_rdx_write(vcpu, result >> 32);
kvm_rax_write(vcpu, result & 0xffffffff);
kvm_edx_write(vcpu, result >> 32);
kvm_eax_write(vcpu, result);
}
}
@ -2544,18 +2549,15 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
#ifdef CONFIG_X86_64
if (is_64_bit_hypercall(vcpu)) {
hc.param = kvm_rcx_read(vcpu);
hc.ingpa = kvm_rdx_read(vcpu);
hc.outgpa = kvm_r8_read(vcpu);
hc.param = kvm_rcx_read_raw(vcpu);
hc.ingpa = kvm_rdx_read_raw(vcpu);
hc.outgpa = kvm_r8_read_raw(vcpu);
} else
#endif
{
hc.param = ((u64)kvm_rdx_read(vcpu) << 32) |
(kvm_rax_read(vcpu) & 0xffffffff);
hc.ingpa = ((u64)kvm_rbx_read(vcpu) << 32) |
(kvm_rcx_read(vcpu) & 0xffffffff);
hc.outgpa = ((u64)kvm_rdi_read(vcpu) << 32) |
(kvm_rsi_read(vcpu) & 0xffffffff);
hc.param = ((u64)kvm_edx_read(vcpu) << 32) | kvm_eax_read(vcpu);
hc.ingpa = ((u64)kvm_ebx_read(vcpu) << 32) | kvm_ecx_read(vcpu);
hc.outgpa = ((u64)kvm_edi_read(vcpu) << 32) | kvm_esi_read(vcpu);
}
hc.code = hc.param & 0xffff;

4
arch/x86/kvm/hyperv.h
View File

@ -232,8 +232,8 @@ static inline bool kvm_hv_is_tlb_flush_hcall(struct kvm_vcpu *vcpu)
if (!hv_vcpu)
return false;
code = is_64_bit_hypercall(vcpu) ? kvm_rcx_read(vcpu) :
kvm_rax_read(vcpu);
code = is_64_bit_hypercall(vcpu) ? kvm_rcx_read_raw(vcpu) :
kvm_eax_read(vcpu);
return (code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE ||
code == HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST ||

2
arch/x86/kvm/ioapic.c
View File

@ -441,7 +441,7 @@ static void ioapic_write_indirect(struct kvm_ioapic *ioapic, u32 val)
irq.dest_id = old_dest_id;
irq.dest_mode =
kvm_lapic_irq_dest_mode(
!!e->fields.dest_mode);
!!old_dest_mode);
kvm_bitmap_or_dest_vcpus(ioapic->kvm, &irq,
vcpu_bitmap);
}

4
arch/x86/kvm/irq.c
View File

@ -585,12 +585,16 @@ int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
r = 0;
switch (chip->chip_id) {
case KVM_IRQCHIP_PIC_MASTER:
spin_lock(&pic->lock);
memcpy(&chip->chip.pic, &pic->pics[0],
sizeof(struct kvm_pic_state));
spin_unlock(&pic->lock);
break;
case KVM_IRQCHIP_PIC_SLAVE:
spin_lock(&pic->lock);
memcpy(&chip->chip.pic, &pic->pics[1],
sizeof(struct kvm_pic_state));
spin_unlock(&pic->lock);
break;
case KVM_IRQCHIP_IOAPIC:
kvm_get_ioapic(kvm, &chip->chip.ioapic);

17
arch/x86/kvm/kvm_emulate.h
View File

@ -22,9 +22,13 @@ enum x86_intercept_stage;
struct x86_exception {
u8 vector;
bool error_code_valid;
u16 error_code;
u64 error_code;
bool nested_page_fault;
u64 address; /* cr2 or nested page fault gpa */
union {
u64 address; /* cr2 or nested page fault gpa */
unsigned long dr6;
u64 payload;
};
u8 async_page_fault;
unsigned long exit_qualification;
};
@ -211,6 +215,7 @@ struct x86_emulate_ops {
ulong (*get_cr)(struct x86_emulate_ctxt *ctxt, int cr);
int (*set_cr)(struct x86_emulate_ctxt *ctxt, int cr, ulong val);
int (*cpl)(struct x86_emulate_ctxt *ctxt);
ulong (*get_effective_dr7)(struct x86_emulate_ctxt *ctxt);
ulong (*get_dr)(struct x86_emulate_ctxt *ctxt, int dr);
int (*set_dr)(struct x86_emulate_ctxt *ctxt, int dr, ulong value);
int (*set_msr_with_filter)(struct x86_emulate_ctxt *ctxt, u32 msr_index, u64 data);
@ -225,6 +230,7 @@ struct x86_emulate_ops {
struct x86_instruction_info *info,
enum x86_intercept_stage stage);
bool (*is_cpuid_allowed)(struct x86_emulate_ctxt *ctxt);
bool (*get_cpuid)(struct x86_emulate_ctxt *ctxt, u32 *eax, u32 *ebx,
u32 *ecx, u32 *edx, bool exact_only);
bool (*guest_has_movbe)(struct x86_emulate_ctxt *ctxt);
@ -520,13 +526,6 @@ enum x86_intercept {
nr_x86_intercepts
};
/* Host execution mode. */
#if defined(CONFIG_X86_32)
#define X86EMUL_MODE_HOST X86EMUL_MODE_PROT32
#elif defined(CONFIG_X86_64)
#define X86EMUL_MODE_HOST X86EMUL_MODE_PROT64
#endif
int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len, int emulation_type);
bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt);
#define EMULATION_FAILED -1

28
arch/x86/kvm/lapic.c
View File

@ -37,7 +37,7 @@
#include <asm/delay.h>
#include <linux/atomic.h>
#include <linux/jump_label.h>
#include "kvm_cache_regs.h"
#include "regs.h"
#include "irq.h"
#include "ioapic.h"
#include "trace.h"
@ -2744,6 +2744,32 @@ u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
return (tpr & 0xf0) >> 4;
}
void kvm_lapic_update_cr8_intercept(struct kvm_vcpu *vcpu)
{
int max_irr, tpr;
if (!kvm_x86_ops.update_cr8_intercept)
return;
if (!lapic_in_kernel(vcpu))
return;
if (vcpu->arch.apic->apicv_active)
return;
if (!vcpu->arch.apic->vapic_addr)
max_irr = kvm_lapic_find_highest_irr(vcpu);
else
max_irr = -1;
if (max_irr != -1)
max_irr >>= 4;
tpr = kvm_lapic_get_cr8(vcpu);
kvm_x86_call(update_cr8_intercept)(vcpu, tpr, max_irr);
}
static void __kvm_apic_set_base(struct kvm_vcpu *vcpu, u64 value)
{
u64 old_value = vcpu->arch.apic_base;

1
arch/x86/kvm/lapic.h
View File

@ -100,6 +100,7 @@ int kvm_apic_accept_events(struct kvm_vcpu *vcpu);
void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event);
u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu);
void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8);
void kvm_lapic_update_cr8_intercept(struct kvm_vcpu *vcpu);
void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu);
void kvm_apic_set_version(struct kvm_vcpu *vcpu);
void kvm_apic_after_set_mcg_cap(struct kvm_vcpu *vcpu);

2
arch/x86/kvm/mmu.h
View File

@ -3,7 +3,7 @@
#define __KVM_X86_MMU_H
#include <linux/kvm_host.h>
#include "kvm_cache_regs.h"
#include "regs.h"
#include "x86.h"
#include "cpuid.h"

2
arch/x86/kvm/mmu/mmu.c
View File

@ -22,7 +22,7 @@
#include "mmu_internal.h"
#include "tdp_mmu.h"
#include "x86.h"
#include "kvm_cache_regs.h"
#include "regs.h"
#include "smm.h"
#include "kvm_emulate.h"
#include "page_track.h"

36
arch/x86/kvm/mmu/paging_tmpl.h
View File

@ -328,6 +328,12 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
const int write_fault = access & PFERR_WRITE_MASK;
const int user_fault = access & PFERR_USER_MASK;
const int fetch_fault = access & PFERR_FETCH_MASK;
/*
* Note! Track the error_code that's common to legacy shadow paging
* and NPT shadow paging as a u16 to guard against unintentionally
* setting any of bits 63:16. Architecturally, the #PF error code is
* 32 bits, and Intel CPUs don't support settings bits 31:16.
*/
u16 errcode = 0;
gpa_t real_gpa;
gfn_t gfn;
@ -391,16 +397,6 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
nested_access | PFERR_GUEST_PAGE_MASK,
&walker->fault, 0);
/*
* FIXME: This can happen if emulation (for of an INS/OUTS
* instruction) triggers a nested page fault. The exit
* qualification / exit info field will incorrectly have
* "guest page access" as the nested page fault's cause,
* instead of "guest page structure access". To fix this,
* the x86_exception struct should be augmented with enough
* information to fix the exit_qualification or exit_info_1
* fields.
*/
if (unlikely(real_gpa == INVALID_GPA))
return 0;
@ -506,7 +502,8 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
* [2:0] - Derive from the access bits. The exit_qualification might be
* out of date if it is serving an EPT misconfiguration.
* [5:3] - Calculated by the page walk of the guest EPT page tables
* [7:11] - Derived from [7:11] of real exit_qualification
* [7:8] - Derived from "fault stage" access bits
* [9:11] - Derived from [9:11] of real exit_qualification
*
* The other bits are set to 0.
*/
@ -520,6 +517,14 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
else
walker->fault.exit_qualification |= EPT_VIOLATION_ACC_READ;
/*
* KVM doesn't emulate features that access GPAs directly, e.g.
* Intel Processor Trace. Assume the GVA is always valid; when
* propagating faults from hardware, KVM will discard this info
* and use the EXIT_QUALIFICATION bits from the VMCS.
*/
walker->fault.exit_qualification |= EPT_VIOLATION_GVA_IS_VALID;
/*
* Accesses to guest paging structures are either "reads" or
* "read+write" accesses, so consider them the latter if write_fault
@ -527,6 +532,8 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
*/
if (access & PFERR_GUEST_PAGE_MASK)
walker->fault.exit_qualification |= EPT_VIOLATION_ACC_READ;
else
walker->fault.exit_qualification |= EPT_VIOLATION_GVA_TRANSLATED;
/*
* Note, pte_access holds the raw RWX bits from the EPTE, not
@ -542,6 +549,11 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
walker->fault.nested_page_fault = mmu != vcpu->arch.walk_mmu;
walker->fault.async_page_fault = false;
#if PTTYPE != PTTYPE_EPT
if (walker->fault.nested_page_fault)
walker->fault.error_code |= access & PFERR_GUEST_FAULT_STAGE_MASK;
#endif
trace_kvm_mmu_walker_error(walker->fault.error_code);
return 0;
}
@ -807,7 +819,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
*/
if (!r) {
if (!fault->prefetch)
kvm_inject_emulated_page_fault(vcpu, &walker.fault);
__kvm_inject_emulated_page_fault(vcpu, &walker.fault, true);
return RET_PF_RETRY;
}

200
arch/x86/kvm/kvm_cache_regs.h → arch/x86/kvm/regs.h
View File

@ -1,6 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef ASM_KVM_CACHE_REGS_H
#define ASM_KVM_CACHE_REGS_H
#ifndef ARCH_X86_KVM_REGS_H
#define ARCH_X86_KVM_REGS_H
#include <linux/kvm_host.h>
@ -16,32 +16,91 @@
static_assert(!(KVM_POSSIBLE_CR0_GUEST_BITS & X86_CR0_PDPTR_BITS));
#define BUILD_KVM_GPR_ACCESSORS(lname, uname) \
static __always_inline unsigned long kvm_##lname##_read(struct kvm_vcpu *vcpu)\
{ \
return vcpu->arch.regs[VCPU_REGS_##uname]; \
} \
static __always_inline void kvm_##lname##_write(struct kvm_vcpu *vcpu, \
unsigned long val) \
{ \
vcpu->arch.regs[VCPU_REGS_##uname] = val; \
}
BUILD_KVM_GPR_ACCESSORS(rax, RAX)
BUILD_KVM_GPR_ACCESSORS(rbx, RBX)
BUILD_KVM_GPR_ACCESSORS(rcx, RCX)
BUILD_KVM_GPR_ACCESSORS(rdx, RDX)
BUILD_KVM_GPR_ACCESSORS(rbp, RBP)
BUILD_KVM_GPR_ACCESSORS(rsi, RSI)
BUILD_KVM_GPR_ACCESSORS(rdi, RDI)
static inline bool is_long_mode(struct kvm_vcpu *vcpu)
{
#ifdef CONFIG_X86_64
BUILD_KVM_GPR_ACCESSORS(r8, R8)
BUILD_KVM_GPR_ACCESSORS(r9, R9)
BUILD_KVM_GPR_ACCESSORS(r10, R10)
BUILD_KVM_GPR_ACCESSORS(r11, R11)
BUILD_KVM_GPR_ACCESSORS(r12, R12)
BUILD_KVM_GPR_ACCESSORS(r13, R13)
BUILD_KVM_GPR_ACCESSORS(r14, R14)
BUILD_KVM_GPR_ACCESSORS(r15, R15)
return !!(vcpu->arch.efer & EFER_LMA);
#else
return false;
#endif
}
static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu)
{
int cs_db, cs_l;
WARN_ON_ONCE(vcpu->arch.guest_state_protected);
if (!is_long_mode(vcpu))
return false;
kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
return cs_l;
}
static inline bool is_64_bit_hypercall(struct kvm_vcpu *vcpu)
{
#ifdef CONFIG_X86_64
/*
* If running with protected guest state, the CS register is not
* accessible. The hypercall register values will have had to been
* provided in 64-bit mode, so assume the guest is in 64-bit.
*/
return vcpu->arch.guest_state_protected || is_64_bit_mode(vcpu);
#else
return false;
#endif
}
static __always_inline unsigned long kvm_reg_mode_mask(struct kvm_vcpu *vcpu)
{
#ifdef CONFIG_X86_64
return is_64_bit_mode(vcpu) ? GENMASK(63, 0) : GENMASK(31, 0);
#else
return GENMASK(31, 0);
#endif
}
#define __BUILD_KVM_GPR_ACCESSORS(lname, uname) \
static __always_inline unsigned long kvm_##lname##_read(struct kvm_vcpu *vcpu) \
{ \
return vcpu->arch.regs[VCPU_REGS_##uname] & kvm_reg_mode_mask(vcpu); \
} \
static __always_inline unsigned long kvm_##lname##_read_raw(struct kvm_vcpu *vcpu) \
{ \
return vcpu->arch.regs[VCPU_REGS_##uname]; \
} \
static __always_inline void kvm_##lname##_write_raw(struct kvm_vcpu *vcpu, \
unsigned long val) \
{ \
vcpu->arch.regs[VCPU_REGS_##uname] = val; \
}
#define BUILD_KVM_GPR_ACCESSORS(lname, uname) \
static __always_inline u32 kvm_e##lname##_read(struct kvm_vcpu *vcpu) \
{ \
return vcpu->arch.regs[VCPU_REGS_##uname]; \
} \
static __always_inline void kvm_e##lname##_write(struct kvm_vcpu *vcpu, u32 val) \
{ \
vcpu->arch.regs[VCPU_REGS_##uname] = val; \
} \
__BUILD_KVM_GPR_ACCESSORS(r##lname, uname)
BUILD_KVM_GPR_ACCESSORS(ax, RAX)
BUILD_KVM_GPR_ACCESSORS(bx, RBX)
BUILD_KVM_GPR_ACCESSORS(cx, RCX)
BUILD_KVM_GPR_ACCESSORS(dx, RDX)
BUILD_KVM_GPR_ACCESSORS(bp, RBP)
BUILD_KVM_GPR_ACCESSORS(si, RSI)
BUILD_KVM_GPR_ACCESSORS(di, RDI)
#ifdef CONFIG_X86_64
__BUILD_KVM_GPR_ACCESSORS(r8, R8)
__BUILD_KVM_GPR_ACCESSORS(r9, R9)
__BUILD_KVM_GPR_ACCESSORS(r10, R10)
__BUILD_KVM_GPR_ACCESSORS(r11, R11)
__BUILD_KVM_GPR_ACCESSORS(r12, R12)
__BUILD_KVM_GPR_ACCESSORS(r13, R13)
__BUILD_KVM_GPR_ACCESSORS(r14, R14)
__BUILD_KVM_GPR_ACCESSORS(r15, R15)
#endif
/*
@ -143,6 +202,11 @@ static inline unsigned long kvm_register_read_raw(struct kvm_vcpu *vcpu, int reg
return vcpu->arch.regs[reg];
}
static inline unsigned long kvm_register_read(struct kvm_vcpu *vcpu, int reg)
{
return kvm_register_read_raw(vcpu, reg) & kvm_reg_mode_mask(vcpu);
}
static inline void kvm_register_write_raw(struct kvm_vcpu *vcpu, int reg,
unsigned long val)
{
@ -153,6 +217,12 @@ static inline void kvm_register_write_raw(struct kvm_vcpu *vcpu, int reg,
kvm_register_mark_dirty(vcpu, reg);
}
static inline void kvm_register_write(struct kvm_vcpu *vcpu,
int reg, unsigned long val)
{
return kvm_register_write_raw(vcpu, reg, val & kvm_reg_mode_mask(vcpu));
}
static inline unsigned long kvm_rip_read(struct kvm_vcpu *vcpu)
{
if (!kvm_register_is_available(vcpu, VCPU_REG_RIP))
@ -177,6 +247,11 @@ static inline void kvm_rsp_write(struct kvm_vcpu *vcpu, unsigned long val)
kvm_register_write_raw(vcpu, VCPU_REGS_RSP, val);
}
static inline u64 kvm_read_edx_eax(struct kvm_vcpu *vcpu)
{
return kvm_eax_read(vcpu) | (u64)(kvm_edx_read(vcpu)) << 32;
}
static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index)
{
might_sleep(); /* on svm */
@ -243,10 +318,75 @@ static inline ulong kvm_read_cr4(struct kvm_vcpu *vcpu)
return kvm_read_cr4_bits(vcpu, ~0UL);
}
static inline u64 kvm_read_edx_eax(struct kvm_vcpu *vcpu)
static inline bool __kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
return (kvm_rax_read(vcpu) & -1u)
| ((u64)(kvm_rdx_read(vcpu) & -1u) << 32);
return !(cr4 & vcpu->arch.cr4_guest_rsvd_bits);
}
#define __cr4_reserved_bits(__cpu_has, __c) \
({ \
u64 __reserved_bits = CR4_RESERVED_BITS; \
\
if (!__cpu_has(__c, X86_FEATURE_XSAVE)) \
__reserved_bits |= X86_CR4_OSXSAVE; \
if (!__cpu_has(__c, X86_FEATURE_SMEP)) \
__reserved_bits |= X86_CR4_SMEP; \
if (!__cpu_has(__c, X86_FEATURE_SMAP)) \
__reserved_bits |= X86_CR4_SMAP; \
if (!__cpu_has(__c, X86_FEATURE_FSGSBASE)) \
__reserved_bits |= X86_CR4_FSGSBASE; \
if (!__cpu_has(__c, X86_FEATURE_PKU)) \
__reserved_bits |= X86_CR4_PKE; \
if (!__cpu_has(__c, X86_FEATURE_LA57)) \
__reserved_bits |= X86_CR4_LA57; \
if (!__cpu_has(__c, X86_FEATURE_UMIP)) \
__reserved_bits |= X86_CR4_UMIP; \
if (!__cpu_has(__c, X86_FEATURE_VMX)) \
__reserved_bits |= X86_CR4_VMXE; \
if (!__cpu_has(__c, X86_FEATURE_PCID)) \
__reserved_bits |= X86_CR4_PCIDE; \
if (!__cpu_has(__c, X86_FEATURE_LAM)) \
__reserved_bits |= X86_CR4_LAM_SUP; \
if (!__cpu_has(__c, X86_FEATURE_SHSTK) && \
!__cpu_has(__c, X86_FEATURE_IBT)) \
__reserved_bits |= X86_CR4_CET; \
__reserved_bits; \
})
static inline bool is_protmode(struct kvm_vcpu *vcpu)
{
return kvm_is_cr0_bit_set(vcpu, X86_CR0_PE);
}
static inline bool is_pae(struct kvm_vcpu *vcpu)
{
return kvm_is_cr4_bit_set(vcpu, X86_CR4_PAE);
}
static inline bool is_pse(struct kvm_vcpu *vcpu)
{
return kvm_is_cr4_bit_set(vcpu, X86_CR4_PSE);
}
static inline bool is_paging(struct kvm_vcpu *vcpu)
{
return likely(kvm_is_cr0_bit_set(vcpu, X86_CR0_PG));
}
static inline bool is_pae_paging(struct kvm_vcpu *vcpu)
{
return !is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu);
}
static inline bool kvm_dr7_valid(u64 data)
{
/* Bits [63:32] are reserved */
return !(data >> 32);
}
static inline bool kvm_dr6_valid(u64 data)
{
/* Bits [63:32] are reserved */
return !(data >> 32);
}
static inline void enter_guest_mode(struct kvm_vcpu *vcpu)

2
arch/x86/kvm/smm.c
View File

@ -3,7 +3,7 @@
#include <linux/kvm_host.h>
#include "x86.h"
#include "kvm_cache_regs.h"
#include "regs.h"
#include "kvm_emulate.h"
#include "smm.h"
#include "cpuid.h"

46
arch/x86/kvm/svm/nested.c
View File

@ -34,23 +34,37 @@
#define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK
static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
struct x86_exception *fault)
struct x86_exception *fault,
bool from_hardware)
{
struct vcpu_svm *svm = to_svm(vcpu);
struct vmcb *vmcb = svm->vmcb;
u64 fault_stage;
if (vmcb->control.exit_code != SVM_EXIT_NPF) {
/*
* TODO: track the cause of the nested page fault, and
* correctly fill in the high bits of exit_info_1.
*/
vmcb->control.exit_code = SVM_EXIT_NPF;
vmcb->control.exit_info_1 = (1ULL << 32);
vmcb->control.exit_info_2 = fault->address;
}
/*
* For hardware NPF exits, the GUEST_FAULT_STAGE bits are only
* available in the hardware exit_info_1, since the guest_mmu
* walker doesn't know whether the faulting GPA was a page table
* page or final page from L2's perspective.
*/
if (from_hardware)
fault_stage = vmcb->control.exit_info_1 &
PFERR_GUEST_FAULT_STAGE_MASK;
else
fault_stage = fault->error_code & PFERR_GUEST_FAULT_STAGE_MASK;
vmcb->control.exit_info_1 &= ~0xffffffffULL;
vmcb->control.exit_info_1 |= fault->error_code;
/*
* All nested page faults should be annotated as occurring on the
* final translation *or* the page walk. Arbitrarily choose "final"
* if KVM is buggy and enumerated both or neither.
*/
if (WARN_ON_ONCE(hweight64(fault_stage) != 1))
fault_stage = PFERR_GUEST_FINAL_MASK;
vmcb->control.exit_code = SVM_EXIT_NPF;
vmcb->control.exit_info_1 = fault_stage |
(fault->error_code & ~PFERR_GUEST_FAULT_STAGE_MASK);
vmcb->control.exit_info_2 = fault->address;
nested_svm_vmexit(svm);
}
@ -771,7 +785,7 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm)
svm->vcpu.arch.cr2 = save->cr2;
kvm_rax_write(vcpu, save->rax);
kvm_rax_write_raw(vcpu, save->rax);
kvm_rsp_write(vcpu, save->rsp);
kvm_rip_write(vcpu, save->rip);
@ -1112,7 +1126,7 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu)
if (WARN_ON_ONCE(!svm->nested.initialized))
return -EINVAL;
vmcb12_gpa = kvm_register_read(vcpu, VCPU_REGS_RAX);
vmcb12_gpa = kvm_rax_read(vcpu);
if (!page_address_valid(vcpu, vmcb12_gpa)) {
kvm_inject_gp(vcpu, 0);
return 1;
@ -1237,7 +1251,7 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu)
vmcb12->save.rflags = kvm_get_rflags(vcpu);
vmcb12->save.rip = kvm_rip_read(vcpu);
vmcb12->save.rsp = kvm_rsp_read(vcpu);
vmcb12->save.rax = kvm_rax_read(vcpu);
vmcb12->save.rax = kvm_rax_read_raw(vcpu);
vmcb12->save.dr7 = vmcb02->save.dr7;
vmcb12->save.dr6 = svm->vcpu.arch.dr6;
vmcb12->save.cpl = vmcb02->save.cpl;
@ -1384,7 +1398,7 @@ void nested_svm_vmexit(struct vcpu_svm *svm)
svm_set_efer(vcpu, vmcb01->save.efer);
svm_set_cr0(vcpu, vmcb01->save.cr0 | X86_CR0_PE);
svm_set_cr4(vcpu, vmcb01->save.cr4);
kvm_rax_write(vcpu, vmcb01->save.rax);
kvm_rax_write_raw(vcpu, vmcb01->save.rax);
kvm_rsp_write(vcpu, vmcb01->save.rsp);
kvm_rip_write(vcpu, vmcb01->save.rip);

12
arch/x86/kvm/svm/sev.c
View File

@ -3711,9 +3711,13 @@ static int snp_rmptable_psmash(kvm_pfn_t pfn)
static int snp_complete_psc_msr(struct kvm_vcpu *vcpu)
{
u64 hypercall_ret = READ_ONCE(vcpu->run->hypercall.ret);
struct vcpu_svm *svm = to_svm(vcpu);
if (vcpu->run->hypercall.ret)
if (!kvm_is_valid_map_gpa_range_ret(hypercall_ret))
return -EINVAL;
if (hypercall_ret)
set_ghcb_msr(svm, GHCB_MSR_PSC_RESP_ERROR);
else
set_ghcb_msr(svm, GHCB_MSR_PSC_RESP);
@ -3802,9 +3806,13 @@ static void __snp_complete_one_psc(struct vcpu_svm *svm)
static int snp_complete_one_psc(struct kvm_vcpu *vcpu)
{
u64 hypercall_ret = READ_ONCE(vcpu->run->hypercall.ret);
struct vcpu_svm *svm = to_svm(vcpu);
if (vcpu->run->hypercall.ret) {
if (!kvm_is_valid_map_gpa_range_ret(hypercall_ret))
return -EINVAL;
if (hypercall_ret) {
snp_complete_psc(svm, VMGEXIT_PSC_ERROR_GENERIC);
return 1; /* resume guest */
}

28
arch/x86/kvm/svm/svm.c
View File

@ -4,7 +4,7 @@
#include "irq.h"
#include "mmu.h"
#include "kvm_cache_regs.h"
#include "regs.h"
#include "x86.h"
#include "smm.h"
#include "cpuid.h"
@ -2217,7 +2217,7 @@ static int intr_interception(struct kvm_vcpu *vcpu)
static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload)
{
u64 vmcb12_gpa = kvm_register_read(vcpu, VCPU_REGS_RAX);
u64 vmcb12_gpa = kvm_rax_read(vcpu);
struct vcpu_svm *svm = to_svm(vcpu);
struct vmcb *vmcb12;
struct kvm_host_map map;
@ -2325,7 +2325,7 @@ static int gp_interception(struct kvm_vcpu *vcpu)
if (nested_svm_check_permissions(vcpu))
return 1;
if (!page_address_valid(vcpu, kvm_register_read(vcpu, VCPU_REGS_RAX)))
if (!page_address_valid(vcpu, kvm_rax_read(vcpu)))
goto reinject;
/*
@ -2408,16 +2408,13 @@ static int clgi_interception(struct kvm_vcpu *vcpu)
static int invlpga_interception(struct kvm_vcpu *vcpu)
{
/* FIXME: Handle an address size prefix. */
gva_t gva = kvm_rax_read(vcpu);
u32 asid = kvm_rcx_read(vcpu);
u32 asid = kvm_ecx_read(vcpu);
if (nested_svm_check_permissions(vcpu))
return 1;
/* FIXME: Handle an address size prefix. */
if (!is_long_mode(vcpu))
gva = (u32)gva;
trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva);
/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
@ -3674,13 +3671,8 @@ static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
struct vcpu_svm *svm = to_svm(vcpu);
struct kvm_run *kvm_run = vcpu->run;
/* SEV-ES guests must use the CR write traps to track CR registers. */
if (!is_sev_es_guest(vcpu)) {
if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE))
vcpu->arch.cr0 = svm->vmcb->save.cr0;
if (npt_enabled)
vcpu->arch.cr3 = svm->vmcb->save.cr3;
}
if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE))
return 0;
if (is_guest_mode(vcpu)) {
int vmexit;
@ -4535,11 +4527,17 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
x86_spec_ctrl_restore_host(svm->virt_spec_ctrl);
/* SEV-ES guests must use the CR write traps to track CR registers. */
if (!is_sev_es_guest(vcpu)) {
vcpu->arch.cr2 = svm->vmcb->save.cr2;
vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
vcpu->arch.rip = svm->vmcb->save.rip;
if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE))
vcpu->arch.cr0 = svm->vmcb->save.cr0;
if (npt_enabled)
vcpu->arch.cr3 = svm->vmcb->save.cr3;
}
kvm_reset_dirty_registers(vcpu);

2
arch/x86/kvm/svm/svm.h
View File

@ -23,7 +23,7 @@
#include <asm/sev-common.h>
#include "cpuid.h"
#include "kvm_cache_regs.h"
#include "regs.h"
/*
* Helpers to convert to/from physical addresses for pages whose address is

39
arch/x86/kvm/vmx/nested.c
View File

@ -411,7 +411,8 @@ static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp,
}
static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
struct x86_exception *fault)
struct x86_exception *fault,
bool from_hardware)
{
struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
struct vcpu_vmx *vmx = to_vmx(vcpu);
@ -444,13 +445,29 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
exit_qualification = 0;
} else {
u64 mask = EPT_VIOLATION_GVA_IS_VALID |
EPT_VIOLATION_GVA_TRANSLATED;
EPT_VIOLATION_GVA_TRANSLATED;
if (vmx->nested.msrs.ept_caps & VMX_EPT_ADVANCED_VMEXIT_INFO_BIT)
mask |= EPT_VIOLATION_GVA_USER |
EPT_VIOLATION_GVA_WRITABLE |
EPT_VIOLATION_GVA_NX;
exit_qualification = fault->exit_qualification;
exit_qualification |= vmx_get_exit_qual(vcpu) & mask;
EPT_VIOLATION_GVA_WRITABLE |
EPT_VIOLATION_GVA_NX;
exit_qualification = fault->exit_qualification & ~mask;
/*
* Use the EXIT_QUALIFICATION from the VMCS if and only
* if the hardware VM-Exit from L2 was an EPT Violation.
* If the fault is synthesized, then EXIT_QUALIFICATION
* is stale and/or holds entirely different data. And
* conversely, KVM _must_ rely on EXIT_QUALIFICATION if
* the fault came from hardware, because KVM only sees
* and walks the faulting GPA.
*/
if (from_hardware)
exit_qualification |= vmx_get_exit_qual(vcpu) & mask;
else
exit_qualification |= fault->exit_qualification & mask;
vm_exit_reason = EXIT_REASON_EPT_VIOLATION;
}
@ -6148,7 +6165,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu,
struct vmcs12 *vmcs12)
{
u32 index = kvm_rcx_read(vcpu);
u32 index = kvm_ecx_read(vcpu);
u64 new_eptp;
if (WARN_ON_ONCE(!nested_cpu_has_ept(vmcs12)))
@ -6182,7 +6199,7 @@ static int handle_vmfunc(struct kvm_vcpu *vcpu)
{
struct vcpu_vmx *vmx = to_vmx(vcpu);
struct vmcs12 *vmcs12;
u32 function = kvm_rax_read(vcpu);
u32 function = kvm_eax_read(vcpu);
/*
* VMFUNC should never execute cleanly while L1 is active; KVM supports
@ -6304,7 +6321,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
exit_reason.basic == EXIT_REASON_MSR_WRITE_IMM)
msr_index = vmx_get_exit_qual(vcpu);
else
msr_index = kvm_rcx_read(vcpu);
msr_index = kvm_ecx_read(vcpu);
/*
* The MSR_BITMAP page is divided into four 1024-byte bitmaps,
@ -6414,7 +6431,7 @@ static bool nested_vmx_exit_handled_encls(struct kvm_vcpu *vcpu,
!nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENCLS_EXITING))
return false;
encls_leaf = kvm_rax_read(vcpu);
encls_leaf = kvm_eax_read(vcpu);
if (encls_leaf > 62)
encls_leaf = 63;
return vmcs12->encls_exiting_bitmap & BIT_ULL(encls_leaf);
@ -6535,6 +6552,8 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu,
nested_evmcs_l2_tlb_flush_enabled(vcpu) &&
kvm_hv_is_tlb_flush_hcall(vcpu);
#endif
case EXIT_REASON_CPUID:
return !kvm_is_cpuid_allowed(vcpu);
default:
break;
}

2
arch/x86/kvm/vmx/nested.h
View File

@ -2,7 +2,7 @@
#ifndef __KVM_X86_VMX_NESTED_H
#define __KVM_X86_VMX_NESTED_H
#include "kvm_cache_regs.h"
#include "regs.h"
#include "hyperv.h"
#include "vmcs12.h"
#include "vmx.h"

10
arch/x86/kvm/vmx/pmu_intel.c
View File

@ -56,8 +56,16 @@ static struct x86_pmu_lbr *vcpu_to_lbr_records(struct kvm_vcpu *vcpu)
static void reprogram_fixed_counters(struct kvm_pmu *pmu, u64 data)
{
/*
* Compare against the value the mediated PMU shoves into hardware, not
* the guest's desired value. For the emulated PMU (proxied via perf),
* they are one and the same (fixed_ctr_ctrl_hw isn't used other than
* here). For the mediated PMU, KVM needs to reprogram the actual MSR,
* and so needs to react to potential changes in the value shoved into
* hardware, e.g. to ensure the event filter is enforced.
*/
u64 old_fixed_ctr_ctrl = pmu->fixed_ctr_ctrl_hw;
struct kvm_pmc *pmc;
u64 old_fixed_ctr_ctrl = pmu->fixed_ctr_ctrl;
int i;
pmu->fixed_ctr_ctrl = data;

6
arch/x86/kvm/vmx/sgx.c
View File

@ -6,7 +6,7 @@
#include <asm/sgx.h>
#include "x86.h"
#include "kvm_cache_regs.h"
#include "regs.h"
#include "nested.h"
#include "sgx.h"
#include "vmx.h"
@ -352,7 +352,7 @@ static int handle_encls_einit(struct kvm_vcpu *vcpu)
rflags &= ~X86_EFLAGS_ZF;
vmx_set_rflags(vcpu, rflags);
kvm_rax_write(vcpu, ret);
kvm_eax_write(vcpu, ret);
return kvm_skip_emulated_instruction(vcpu);
}
@ -380,7 +380,7 @@ static inline bool sgx_enabled_in_guest_bios(struct kvm_vcpu *vcpu)
int handle_encls(struct kvm_vcpu *vcpu)
{
u32 leaf = (u32)kvm_rax_read(vcpu);
u32 leaf = kvm_eax_read(vcpu);
if (!enable_sgx || !guest_cpu_cap_has(vcpu, X86_FEATURE_SGX) ||
!guest_cpu_cap_has(vcpu, X86_FEATURE_SGX1)) {

52
arch/x86/kvm/vmx/tdx.c
View File

@ -638,6 +638,12 @@ int tdx_vm_init(struct kvm *kvm)
kvm->arch.has_private_mem = true;
kvm->arch.disabled_quirks |= KVM_X86_QUIRK_IGNORE_GUEST_PAT;
/*
* PMU support is provided by the TDX-Module (if enabled for the VM).
* From KVM's perspective, the VM doesn't have a virtual PMU.
*/
kvm->arch.has_protected_pmu = true;
/*
* Because guest TD is protected, VMM can't parse the instruction in TD.
* Instead, guest uses MMIO hypercall. For unmodified device driver,
@ -1163,11 +1169,11 @@ static int complete_hypercall_exit(struct kvm_vcpu *vcpu)
static int tdx_emulate_vmcall(struct kvm_vcpu *vcpu)
{
kvm_rax_write(vcpu, to_tdx(vcpu)->vp_enter_args.r10);
kvm_rbx_write(vcpu, to_tdx(vcpu)->vp_enter_args.r11);
kvm_rcx_write(vcpu, to_tdx(vcpu)->vp_enter_args.r12);
kvm_rdx_write(vcpu, to_tdx(vcpu)->vp_enter_args.r13);
kvm_rsi_write(vcpu, to_tdx(vcpu)->vp_enter_args.r14);
kvm_rax_write_raw(vcpu, to_tdx(vcpu)->vp_enter_args.r10);
kvm_rbx_write_raw(vcpu, to_tdx(vcpu)->vp_enter_args.r11);
kvm_rcx_write_raw(vcpu, to_tdx(vcpu)->vp_enter_args.r12);
kvm_rdx_write_raw(vcpu, to_tdx(vcpu)->vp_enter_args.r13);
kvm_rsi_write_raw(vcpu, to_tdx(vcpu)->vp_enter_args.r14);
return __kvm_emulate_hypercall(vcpu, 0, complete_hypercall_exit);
}
@ -1182,12 +1188,22 @@ static void __tdx_map_gpa(struct vcpu_tdx *tdx);
static int tdx_complete_vmcall_map_gpa(struct kvm_vcpu *vcpu)
{
u64 hypercall_ret = READ_ONCE(vcpu->run->hypercall.ret);
struct vcpu_tdx *tdx = to_tdx(vcpu);
long rc;
if (vcpu->run->hypercall.ret) {
tdvmcall_set_return_code(vcpu, TDVMCALL_STATUS_INVALID_OPERAND);
tdx->vp_enter_args.r11 = tdx->map_gpa_next;
return 1;
switch (hypercall_ret) {
case 0:
break;
case EAGAIN:
rc = TDVMCALL_STATUS_RETRY;
goto propagate_error;
case EINVAL:
rc = TDVMCALL_STATUS_INVALID_OPERAND;
goto propagate_error;
default:
WARN_ON_ONCE(kvm_is_valid_map_gpa_range_ret(hypercall_ret));
return -EINVAL;
}
tdx->map_gpa_next += TDX_MAP_GPA_MAX_LEN;
@ -1200,13 +1216,17 @@ static int tdx_complete_vmcall_map_gpa(struct kvm_vcpu *vcpu)
* TDVMCALL_MAP_GPA, see comments in tdx_protected_apic_has_interrupt().
*/
if (kvm_vcpu_has_events(vcpu)) {
tdvmcall_set_return_code(vcpu, TDVMCALL_STATUS_RETRY);
tdx->vp_enter_args.r11 = tdx->map_gpa_next;
return 1;
rc = TDVMCALL_STATUS_RETRY;
goto propagate_error;
}
__tdx_map_gpa(tdx);
return 0;
propagate_error:
tdvmcall_set_return_code(vcpu, rc);
tdx->vp_enter_args.r11 = tdx->map_gpa_next;
return 1;
}
static void __tdx_map_gpa(struct vcpu_tdx *tdx)
@ -2031,12 +2051,12 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath)
case EXIT_REASON_IO_INSTRUCTION:
return tdx_emulate_io(vcpu);
case EXIT_REASON_MSR_READ:
kvm_rcx_write(vcpu, tdx->vp_enter_args.r12);
kvm_ecx_write(vcpu, tdx->vp_enter_args.r12);
return kvm_emulate_rdmsr(vcpu);
case EXIT_REASON_MSR_WRITE:
kvm_rcx_write(vcpu, tdx->vp_enter_args.r12);
kvm_rax_write(vcpu, tdx->vp_enter_args.r13 & -1u);
kvm_rdx_write(vcpu, tdx->vp_enter_args.r13 >> 32);
kvm_ecx_write(vcpu, tdx->vp_enter_args.r12);
kvm_eax_write(vcpu, tdx->vp_enter_args.r13);
kvm_edx_write(vcpu, tdx->vp_enter_args.r13 >> 32);
return kvm_emulate_wrmsr(vcpu);
case EXIT_REASON_EPT_MISCONFIG:
return tdx_emulate_mmio(vcpu);

40
arch/x86/kvm/vmx/vmx.c
View File

@ -59,7 +59,7 @@
#include "hyperv.h"
#include "kvm_onhyperv.h"
#include "irq.h"
#include "kvm_cache_regs.h"
#include "regs.h"
#include "lapic.h"
#include "mmu.h"
#include "nested.h"
@ -1912,6 +1912,24 @@ void vmx_inject_exception(struct kvm_vcpu *vcpu)
u32 intr_info = ex->vector | INTR_INFO_VALID_MASK;
struct vcpu_vmx *vmx = to_vmx(vcpu);
/*
* When injecting a #DB, single-stepping is enabled in RFLAGS, and STI
* or MOV-SS blocking is active, set vmcs.PENDING_DBG_EXCEPTIONS.BS to
* prevent a false positive from VM-Entry consistency check. VM-Entry
* asserts that a single-step #DB _must_ be pending in this scenario,
* as the previous instruction cannot have toggled RFLAGS.TF 0=>1
* (because STI and POP/MOV don't modify RFLAGS), therefore the one
* instruction delay when activating single-step breakpoints must have
* already expired. However, the CPU isn't smart enough to peek at
* vmcs.VM_ENTRY_INTR_INFO_FIELD and so doesn't realize that yes, there
* is indeed a #DB pending/imminent.
*/
if (ex->vector == DB_VECTOR &&
(vmx_get_rflags(vcpu) & X86_EFLAGS_TF) &&
vmx_get_interrupt_shadow(vcpu))
vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS) | DR6_BS);
kvm_deliver_exception_payload(vcpu, ex);
if (ex->has_error_code) {
@ -5495,26 +5513,9 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu)
* avoid single-step #DB and MTF updates, as ICEBP is
* higher priority. Note, skipping ICEBP still clears
* STI and MOVSS blocking.
*
* For all other #DBs, set vmcs.PENDING_DBG_EXCEPTIONS.BS
* if single-step is enabled in RFLAGS and STI or MOVSS
* blocking is active, as the CPU doesn't set the bit
* on VM-Exit due to #DB interception. VM-Entry has a
* consistency check that a single-step #DB is pending
* in this scenario as the previous instruction cannot
* have toggled RFLAGS.TF 0=>1 (because STI and POP/MOV
* don't modify RFLAGS), therefore the one instruction
* delay when activating single-step breakpoints must
* have already expired. Note, the CPU sets/clears BS
* as appropriate for all other VM-Exits types.
*/
if (is_icebp(intr_info))
WARN_ON(!skip_emulated_instruction(vcpu));
else if ((vmx_get_rflags(vcpu) & X86_EFLAGS_TF) &&
(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)))
vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS) | DR6_BS);
kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
return 1;
@ -6715,6 +6716,9 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
if (enable_pml && !is_guest_mode(vcpu))
vmx_flush_pml_buffer(vcpu);
if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE))
return 0;
/*
* KVM should never reach this point with a pending nested VM-Enter.
* More specifically, short-circuiting VM-Entry to emulate L2 due to

2
arch/x86/kvm/vmx/vmx.h
View File

@ -10,7 +10,7 @@
#include <asm/posted_intr.h>
#include "capabilities.h"
#include "../kvm_cache_regs.h"
#include "../regs.h"
#include "pmu_intel.h"
#include "vmcs.h"
#include "vmx_ops.h"

378
arch/x86/kvm/x86.c
View File

@ -23,7 +23,7 @@
#include "mmu.h"
#include "i8254.h"
#include "tss.h"
#include "kvm_cache_regs.h"
#include "regs.h"
#include "kvm_emulate.h"
#include "mmu/page_track.h"
#include "x86.h"
@ -128,12 +128,10 @@ static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST | \
KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST)
static void update_cr8_intercept(struct kvm_vcpu *vcpu);
static void process_nmi(struct kvm_vcpu *vcpu);
static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
static void store_regs(struct kvm_vcpu *vcpu);
static int sync_regs(struct kvm_vcpu *vcpu);
static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu);
static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
@ -152,6 +150,7 @@ struct kvm_x86_ops kvm_x86_ops __read_mostly;
#include <asm/kvm-x86-ops.h>
EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits);
EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg);
EXPORT_STATIC_CALL_GPL(kvm_x86_get_cpl);
static bool __read_mostly ignore_msrs = 0;
module_param(ignore_msrs, bool, 0644);
@ -576,13 +575,6 @@ static struct kmem_cache *kvm_alloc_emulator_cache(void)
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
{
int i;
for (i = 0; i < ASYNC_PF_PER_VCPU; i++)
vcpu->arch.apf.gfns[i] = ~0;
}
static void kvm_destroy_user_return_msrs(void)
{
int cpu;
@ -970,7 +962,8 @@ static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err)
EMULTYPE_COMPLETE_USER_EXIT);
}
void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault,
bool from_hardware)
{
++vcpu->stat.pf_guest;
@ -987,8 +980,9 @@ void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
fault->address);
}
void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
struct x86_exception *fault)
void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
struct x86_exception *fault,
bool from_hardware)
{
struct kvm_mmu *fault_mmu;
WARN_ON_ONCE(fault->vector != PF_VECTOR);
@ -1005,9 +999,9 @@ void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
kvm_mmu_invalidate_addr(vcpu, fault_mmu, fault->address,
KVM_MMU_ROOT_CURRENT);
fault_mmu->inject_page_fault(vcpu, fault);
fault_mmu->inject_page_fault(vcpu, fault, from_hardware);
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_inject_emulated_page_fault);
EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_inject_emulated_page_fault);
void kvm_inject_nmi(struct kvm_vcpu *vcpu)
{
@ -1021,18 +1015,6 @@ void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_queue_exception_e);
/*
* Checks if cpl <= required_cpl; if true, return true. Otherwise queue
* a #GP and return false.
*/
bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
{
if (kvm_x86_call(get_cpl)(vcpu) <= required_cpl)
return true;
kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
return false;
}
bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
{
if ((dr != 4 && dr != 5) || !kvm_is_cr4_bit_set(vcpu, X86_CR4_DE))
@ -1043,13 +1025,6 @@ bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_require_dr);
static bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)
{
u64 mask = KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT;
return (vcpu->arch.apf.msr_en_val & mask) == mask;
}
static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
{
return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2);
@ -1314,7 +1289,7 @@ int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu)
{
/* Note, #UD due to CR4.OSXSAVE=0 has priority over the intercept. */
if (kvm_x86_call(get_cpl)(vcpu) != 0 ||
__kvm_set_xcr(vcpu, kvm_rcx_read(vcpu), kvm_read_edx_eax(vcpu))) {
__kvm_set_xcr(vcpu, kvm_ecx_read(vcpu), kvm_read_edx_eax(vcpu))) {
kvm_inject_gp(vcpu, 0);
return 1;
}
@ -1601,9 +1576,17 @@ unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr)
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dr);
static unsigned long kvm_get_effective_dr7(struct kvm_vcpu *vcpu)
{
if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
return vcpu->arch.guest_debug_dr7;
return vcpu->arch.dr7;
}
int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu)
{
u32 pmc = kvm_rcx_read(vcpu);
u32 pmc = kvm_ecx_read(vcpu);
u64 data;
if (kvm_pmu_rdpmc(vcpu, pmc, &data)) {
@ -1611,8 +1594,8 @@ int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu)
return 1;
}
kvm_rax_write(vcpu, (u32)data);
kvm_rdx_write(vcpu, data >> 32);
kvm_eax_write(vcpu, data);
kvm_edx_write(vcpu, data >> 32);
return kvm_skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdpmc);
@ -2059,8 +2042,8 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_msr_write);
static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu)
{
if (!vcpu->run->msr.error) {
kvm_rax_write(vcpu, (u32)vcpu->run->msr.data);
kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);
kvm_eax_write(vcpu, vcpu->run->msr.data);
kvm_edx_write(vcpu, vcpu->run->msr.data >> 32);
}
}
@ -2141,8 +2124,8 @@ static int __kvm_emulate_rdmsr(struct kvm_vcpu *vcpu, u32 msr, int reg,
trace_kvm_msr_read(msr, data);
if (reg < 0) {
kvm_rax_write(vcpu, data & -1u);
kvm_rdx_write(vcpu, (data >> 32) & -1u);
kvm_eax_write(vcpu, data);
kvm_edx_write(vcpu, data >> 32);
} else {
kvm_register_write(vcpu, reg, data);
}
@ -2159,7 +2142,7 @@ static int __kvm_emulate_rdmsr(struct kvm_vcpu *vcpu, u32 msr, int reg,
int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
{
return __kvm_emulate_rdmsr(vcpu, kvm_rcx_read(vcpu), -1,
return __kvm_emulate_rdmsr(vcpu, kvm_ecx_read(vcpu), -1,
complete_fast_rdmsr);
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdmsr);
@ -2195,7 +2178,7 @@ static int __kvm_emulate_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
{
return __kvm_emulate_wrmsr(vcpu, kvm_rcx_read(vcpu),
return __kvm_emulate_wrmsr(vcpu, kvm_ecx_read(vcpu),
kvm_read_edx_eax(vcpu));
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_wrmsr);
@ -2305,7 +2288,7 @@ static fastpath_t __handle_fastpath_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 da
fastpath_t handle_fastpath_wrmsr(struct kvm_vcpu *vcpu)
{
return __handle_fastpath_wrmsr(vcpu, kvm_rcx_read(vcpu),
return __handle_fastpath_wrmsr(vcpu, kvm_ecx_read(vcpu),
kvm_read_edx_eax(vcpu));
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_wrmsr);
@ -3648,23 +3631,19 @@ static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
if (!lapic_in_kernel(vcpu))
return data ? 1 : 0;
vcpu->arch.apf.msr_en_val = data;
if (!kvm_pv_async_pf_enabled(vcpu)) {
kvm_clear_async_pf_completion_queue(vcpu);
kvm_async_pf_hash_reset(vcpu);
return 0;
}
if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
sizeof(u64)))
if (__kvm_pv_async_pf_enabled(data) &&
kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
sizeof(u64)))
return 1;
vcpu->arch.apf.send_always = (data & KVM_ASYNC_PF_SEND_ALWAYS);
vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
kvm_async_pf_wakeup_all(vcpu);
vcpu->arch.apf.msr_en_val = data;
if (__kvm_pv_async_pf_enabled(data)) {
kvm_async_pf_wakeup_all(vcpu);
} else {
kvm_clear_async_pf_completion_queue(vcpu);
kvm_async_pf_hash_reset(vcpu);
}
return 0;
}
@ -4003,22 +3982,28 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
break;
case MSR_EFER:
return set_efer(vcpu, msr_info);
case MSR_K7_HWCR:
data &= ~(u64)0x40; /* ignore flush filter disable */
data &= ~(u64)0x100; /* ignore ignne emulation enable */
data &= ~(u64)0x8; /* ignore TLB cache disable */
case MSR_K7_HWCR: {
/*
* Allow McStatusWrEn and TscFreqSel. (Linux guests from v3.2
* through at least v6.6 whine if TscFreqSel is clear,
* depending on F/M/S.
*/
if (data & ~(BIT_ULL(18) | BIT_ULL(24))) {
u64 valid = BIT_ULL(18) | BIT_ULL(24);
data &= ~(u64)0x40; /* ignore flush filter disable */
data &= ~(u64)0x100; /* ignore ignne emulation enable */
data &= ~(u64)0x8; /* ignore TLB cache disable */
if (guest_cpu_cap_has(vcpu, X86_FEATURE_GP_ON_USER_CPUID))
valid |= MSR_K7_HWCR_CPUID_USER_DIS;
if (data & ~valid) {
kvm_pr_unimpl_wrmsr(vcpu, msr, data);
return 1;
}
vcpu->arch.msr_hwcr = data;
break;
}
case MSR_FAM10H_MMIO_CONF_BASE:
if (data != 0) {
kvm_pr_unimpl_wrmsr(vcpu, msr, data);
@ -4265,7 +4250,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
case MSR_MISC_FEATURES_ENABLES:
if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT ||
(data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
!supports_cpuid_fault(vcpu)))
!(vcpu->arch.msr_platform_info & MSR_PLATFORM_INFO_CPUID_FAULT)))
return 1;
vcpu->arch.msr_misc_features_enables = data;
break;
@ -5341,7 +5326,7 @@ static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
r = kvm_apic_set_state(vcpu, s);
if (r)
return r;
update_cr8_intercept(vcpu);
kvm_lapic_update_cr8_intercept(vcpu);
return 0;
}
@ -6915,6 +6900,10 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
if (!enable_pmu || (cap->args[0] & ~KVM_CAP_PMU_VALID_MASK))
break;
if (kvm->arch.has_protected_pmu &&
cap->args[0] != KVM_PMU_CAP_DISABLE)
break;
mutex_lock(&kvm->lock);
if (!kvm->created_vcpus && !kvm->arch.created_mediated_pmu) {
kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CAP_DISABLE);
@ -8553,6 +8542,11 @@ static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));
}
static unsigned long emulator_get_effective_dr7(struct x86_emulate_ctxt *ctxt)
{
return kvm_get_effective_dr7(emul_to_vcpu(ctxt));
}
static unsigned long emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr)
{
return kvm_get_dr(emul_to_vcpu(ctxt), dr);
@ -8810,6 +8804,11 @@ static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
&ctxt->exception);
}
static bool emulator_is_cpuid_allowed(struct x86_emulate_ctxt *ctxt)
{
return kvm_is_cpuid_allowed(emul_to_vcpu(ctxt));
}
static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
u32 *eax, u32 *ebx, u32 *ecx, u32 *edx,
bool exact_only)
@ -8935,6 +8934,7 @@ static const struct x86_emulate_ops emulate_ops = {
.get_cr = emulator_get_cr,
.set_cr = emulator_set_cr,
.cpl = emulator_get_cpl,
.get_effective_dr7 = emulator_get_effective_dr7,
.get_dr = emulator_get_dr,
.set_dr = emulator_set_dr,
.set_msr_with_filter = emulator_set_msr_with_filter,
@ -8946,6 +8946,7 @@ static const struct x86_emulate_ops emulate_ops = {
.wbinvd = emulator_wbinvd,
.fix_hypercall = emulator_fix_hypercall,
.intercept = emulator_intercept,
.is_cpuid_allowed = emulator_is_cpuid_allowed,
.get_cpuid = emulator_get_cpuid,
.guest_has_movbe = emulator_guest_has_movbe,
.guest_has_fxsr = emulator_guest_has_fxsr,
@ -8981,17 +8982,36 @@ static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
}
}
static void inject_emulated_exception(struct kvm_vcpu *vcpu)
static int kvm_inject_emulated_db(struct kvm_vcpu *vcpu, unsigned long dr6)
{
struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
struct kvm_run *kvm_run = vcpu->run;
if (ctxt->exception.vector == PF_VECTOR)
kvm_inject_emulated_page_fault(vcpu, &ctxt->exception);
else if (ctxt->exception.error_code_valid)
kvm_queue_exception_e(vcpu, ctxt->exception.vector,
ctxt->exception.error_code);
if (vcpu->guest_debug & (KVM_GUESTDBG_USE_HW_BP | KVM_GUESTDBG_SINGLESTEP)) {
kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW;
kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);
kvm_run->debug.arch.exception = DB_VECTOR;
kvm_run->exit_reason = KVM_EXIT_DEBUG;
return 0;
}
kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
return 1;
}
static int inject_emulated_exception(struct kvm_vcpu *vcpu)
{
struct x86_exception *ex = &vcpu->arch.emulate_ctxt->exception;
if (ex->vector == DB_VECTOR)
return kvm_inject_emulated_db(vcpu, ex->dr6);
if (ex->vector == PF_VECTOR)
kvm_inject_emulated_page_fault(vcpu, ex);
else if (ex->error_code_valid)
kvm_queue_exception_e(vcpu, ex->vector, ex->error_code);
else
kvm_queue_exception(vcpu, ctxt->exception.vector);
kvm_queue_exception(vcpu, ex->vector);
return 1;
}
static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu)
@ -9031,6 +9051,7 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
ctxt->interruptibility = 0;
ctxt->have_exception = false;
ctxt->exception.vector = -1;
ctxt->exception.payload = 0;
ctxt->perm_ok = false;
init_decode_cache(ctxt);
@ -9248,21 +9269,6 @@ static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7,
return dr6;
}
static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)
{
struct kvm_run *kvm_run = vcpu->run;
if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
kvm_run->debug.arch.dr6 = DR6_BS | DR6_ACTIVE_LOW;
kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);
kvm_run->debug.arch.exception = DB_VECTOR;
kvm_run->exit_reason = KVM_EXIT_DEBUG;
return 0;
}
kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);
return 1;
}
int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
unsigned long rflags = kvm_x86_call(get_rflags)(vcpu);
@ -9283,13 +9289,16 @@ int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
* that sets the TF flag".
*/
if (unlikely(rflags & X86_EFLAGS_TF))
r = kvm_vcpu_do_singlestep(vcpu);
r = kvm_inject_emulated_db(vcpu, DR6_BS);
return r;
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_skip_emulated_instruction);
static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu)
{
if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
return false;
if (kvm_get_rflags(vcpu) & X86_EFLAGS_RF)
return true;
@ -9306,6 +9315,8 @@ static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu)
static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu,
int emulation_type, int *r)
{
unsigned long dr7 = kvm_get_effective_dr7(vcpu);
WARN_ON_ONCE(emulation_type & EMULTYPE_NO_DECODE);
/*
@ -9326,34 +9337,14 @@ static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu,
EMULTYPE_TRAP_UD | EMULTYPE_VMWARE_GP | EMULTYPE_PF))
return false;
if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
(vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
struct kvm_run *kvm_run = vcpu->run;
unsigned long eip = kvm_get_linear_rip(vcpu);
u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
vcpu->arch.guest_debug_dr7,
vcpu->arch.eff_db);
if (dr6 != 0) {
kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW;
kvm_run->debug.arch.pc = eip;
kvm_run->debug.arch.exception = DB_VECTOR;
kvm_run->exit_reason = KVM_EXIT_DEBUG;
*r = 0;
return true;
}
}
if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&
if (unlikely(dr7 & DR7_BP_EN_MASK) &&
!kvm_is_code_breakpoint_inhibited(vcpu)) {
unsigned long eip = kvm_get_linear_rip(vcpu);
u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
vcpu->arch.dr7,
vcpu->arch.db);
u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0, dr7,
vcpu->arch.eff_db);
if (dr6 != 0) {
kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
*r = 1;
if (dr6) {
*r = kvm_inject_emulated_db(vcpu, dr6);
return true;
}
}
@ -9499,8 +9490,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
*/
WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR ||
exception_type(ctxt->exception.vector) == EXCPT_TRAP);
inject_emulated_exception(vcpu);
return 1;
return inject_emulated_exception(vcpu);
}
return handle_emulation_failure(vcpu, emulation_type);
}
@ -9595,8 +9585,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
if (ctxt->have_exception) {
WARN_ON_ONCE(vcpu->mmio_needed && !vcpu->mmio_is_write);
vcpu->mmio_needed = false;
r = 1;
inject_emulated_exception(vcpu);
r = inject_emulated_exception(vcpu);
} else if (vcpu->arch.pio.count) {
if (!vcpu->arch.pio.in) {
/* FIXME: return into emulator if single-stepping. */
@ -9639,7 +9628,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
kvm_pmu_branch_retired(vcpu);
kvm_rip_write(vcpu, ctxt->eip);
if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))
r = kvm_vcpu_do_singlestep(vcpu);
r = kvm_inject_emulated_db(vcpu, DR6_BS);
kvm_x86_call(update_emulated_instruction)(vcpu);
__kvm_set_rflags(vcpu, ctxt->eflags);
}
@ -9690,7 +9679,7 @@ static int complete_fast_pio_out(struct kvm_vcpu *vcpu)
static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,
unsigned short port)
{
unsigned long val = kvm_rax_read(vcpu);
unsigned long val = kvm_rax_read_raw(vcpu);
int ret = emulator_pio_out(vcpu, size, port, &val, 1);
if (ret)
@ -9726,10 +9715,10 @@ static int complete_fast_pio_in(struct kvm_vcpu *vcpu)
}
/* For size less than 4 we merge, else we zero extend */
val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;
val = (vcpu->arch.pio.size < 4) ? kvm_rax_read_raw(vcpu) : 0;
complete_emulator_pio_in(vcpu, &val);
kvm_rax_write(vcpu, val);
kvm_rax_write_raw(vcpu, val);
return kvm_skip_emulated_instruction(vcpu);
}
@ -9741,11 +9730,11 @@ static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,
int ret;
/* For size less than 4 we merge, else we zero extend */
val = (size < 4) ? kvm_rax_read(vcpu) : 0;
val = (size < 4) ? kvm_rax_read_raw(vcpu) : 0;
ret = emulator_pio_in(vcpu, size, port, &val, 1);
if (ret) {
kvm_rax_write(vcpu, val);
kvm_rax_write_raw(vcpu, val);
return ret;
}
@ -10412,33 +10401,34 @@ static int complete_hypercall_exit(struct kvm_vcpu *vcpu)
if (!is_64_bit_hypercall(vcpu))
ret = (u32)ret;
kvm_rax_write(vcpu, ret);
kvm_rax_write_raw(vcpu, ret);
return kvm_skip_emulated_instruction(vcpu);
}
int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, int cpl,
int (*complete_hypercall)(struct kvm_vcpu *))
{
unsigned long ret;
unsigned long nr = kvm_rax_read(vcpu);
unsigned long a0 = kvm_rbx_read(vcpu);
unsigned long a1 = kvm_rcx_read(vcpu);
unsigned long a2 = kvm_rdx_read(vcpu);
unsigned long a3 = kvm_rsi_read(vcpu);
int op_64_bit = is_64_bit_hypercall(vcpu);
unsigned long ret, nr, a0, a1, a2, a3;
++vcpu->stat.hypercalls;
trace_kvm_hypercall(nr, a0, a1, a2, a3);
if (!op_64_bit) {
nr &= 0xFFFFFFFF;
a0 &= 0xFFFFFFFF;
a1 &= 0xFFFFFFFF;
a2 &= 0xFFFFFFFF;
a3 &= 0xFFFFFFFF;
if (op_64_bit) {
nr = kvm_rax_read_raw(vcpu);
a0 = kvm_rbx_read_raw(vcpu);
a1 = kvm_rcx_read_raw(vcpu);
a2 = kvm_rdx_read_raw(vcpu);
a3 = kvm_rsi_read_raw(vcpu);
} else {
nr = kvm_eax_read(vcpu);
a0 = kvm_ebx_read(vcpu);
a1 = kvm_ecx_read(vcpu);
a2 = kvm_edx_read(vcpu);
a3 = kvm_esi_read(vcpu);
}
trace_kvm_hypercall(nr, a0, a1, a2, a3);
if (cpl) {
ret = -KVM_EPERM;
goto out;
@ -10581,33 +10571,6 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu)
kvm_run->flags |= KVM_RUN_X86_GUEST_MODE;
}
static void update_cr8_intercept(struct kvm_vcpu *vcpu)
{
int max_irr, tpr;
if (!kvm_x86_ops.update_cr8_intercept)
return;
if (!lapic_in_kernel(vcpu))
return;
if (vcpu->arch.apic->apicv_active)
return;
if (!vcpu->arch.apic->vapic_addr)
max_irr = kvm_lapic_find_highest_irr(vcpu);
else
max_irr = -1;
if (max_irr != -1)
max_irr >>= 4;
tpr = kvm_lapic_get_cr8(vcpu);
kvm_x86_call(update_cr8_intercept)(vcpu, tpr, max_irr);
}
int kvm_check_nested_events(struct kvm_vcpu *vcpu)
{
if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
@ -11348,7 +11311,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
kvm_x86_call(enable_irq_window)(vcpu);
if (kvm_lapic_enabled(vcpu)) {
update_cr8_intercept(vcpu);
kvm_lapic_update_cr8_intercept(vcpu);
kvm_lapic_sync_to_vapic(vcpu);
}
}
@ -11593,9 +11556,6 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
if (vcpu->arch.apic_attention)
kvm_lapic_sync_from_vapic(vcpu);
if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE))
return 0;
r = kvm_x86_call(handle_exit)(vcpu, exit_fastpath);
return r;
@ -12135,23 +12095,23 @@ static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);
vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
}
regs->rax = kvm_rax_read(vcpu);
regs->rbx = kvm_rbx_read(vcpu);
regs->rcx = kvm_rcx_read(vcpu);
regs->rdx = kvm_rdx_read(vcpu);
regs->rsi = kvm_rsi_read(vcpu);
regs->rdi = kvm_rdi_read(vcpu);
regs->rax = kvm_rax_read_raw(vcpu);
regs->rbx = kvm_rbx_read_raw(vcpu);
regs->rcx = kvm_rcx_read_raw(vcpu);
regs->rdx = kvm_rdx_read_raw(vcpu);
regs->rsi = kvm_rsi_read_raw(vcpu);
regs->rdi = kvm_rdi_read_raw(vcpu);
regs->rsp = kvm_rsp_read(vcpu);
regs->rbp = kvm_rbp_read(vcpu);
regs->rbp = kvm_rbp_read_raw(vcpu);
#ifdef CONFIG_X86_64
regs->r8 = kvm_r8_read(vcpu);
regs->r9 = kvm_r9_read(vcpu);
regs->r10 = kvm_r10_read(vcpu);
regs->r11 = kvm_r11_read(vcpu);
regs->r12 = kvm_r12_read(vcpu);
regs->r13 = kvm_r13_read(vcpu);
regs->r14 = kvm_r14_read(vcpu);
regs->r15 = kvm_r15_read(vcpu);
regs->r8 = kvm_r8_read_raw(vcpu);
regs->r9 = kvm_r9_read_raw(vcpu);
regs->r10 = kvm_r10_read_raw(vcpu);
regs->r11 = kvm_r11_read_raw(vcpu);
regs->r12 = kvm_r12_read_raw(vcpu);
regs->r13 = kvm_r13_read_raw(vcpu);
regs->r14 = kvm_r14_read_raw(vcpu);
regs->r15 = kvm_r15_read_raw(vcpu);
#endif
regs->rip = kvm_rip_read(vcpu);
@ -12175,23 +12135,23 @@ static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
kvm_rax_write(vcpu, regs->rax);
kvm_rbx_write(vcpu, regs->rbx);
kvm_rcx_write(vcpu, regs->rcx);
kvm_rdx_write(vcpu, regs->rdx);
kvm_rsi_write(vcpu, regs->rsi);
kvm_rdi_write(vcpu, regs->rdi);
kvm_rax_write_raw(vcpu, regs->rax);
kvm_rbx_write_raw(vcpu, regs->rbx);
kvm_rcx_write_raw(vcpu, regs->rcx);
kvm_rdx_write_raw(vcpu, regs->rdx);
kvm_rsi_write_raw(vcpu, regs->rsi);
kvm_rdi_write_raw(vcpu, regs->rdi);
kvm_rsp_write(vcpu, regs->rsp);
kvm_rbp_write(vcpu, regs->rbp);
kvm_rbp_write_raw(vcpu, regs->rbp);
#ifdef CONFIG_X86_64
kvm_r8_write(vcpu, regs->r8);
kvm_r9_write(vcpu, regs->r9);
kvm_r10_write(vcpu, regs->r10);
kvm_r11_write(vcpu, regs->r11);
kvm_r12_write(vcpu, regs->r12);
kvm_r13_write(vcpu, regs->r13);
kvm_r14_write(vcpu, regs->r14);
kvm_r15_write(vcpu, regs->r15);
kvm_r8_write_raw(vcpu, regs->r8);
kvm_r9_write_raw(vcpu, regs->r9);
kvm_r10_write_raw(vcpu, regs->r10);
kvm_r11_write_raw(vcpu, regs->r11);
kvm_r12_write_raw(vcpu, regs->r12);
kvm_r13_write_raw(vcpu, regs->r13);
kvm_r14_write_raw(vcpu, regs->r14);
kvm_r15_write_raw(vcpu, regs->r15);
#endif
kvm_rip_write(vcpu, regs->rip);
@ -12497,7 +12457,7 @@ static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs,
kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
update_cr8_intercept(vcpu);
kvm_lapic_update_cr8_intercept(vcpu);
/* Older userspace won't unhalt the vcpu on reset. */
if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
@ -13094,7 +13054,7 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
* on RESET. But, go through the motions in case that's ever remedied.
*/
cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1);
kvm_rdx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600);
kvm_edx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600);
kvm_x86_call(vcpu_reset)(vcpu, init_event);
@ -13368,7 +13328,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
kvm->arch.default_tsc_khz = max_tsc_khz ? : tsc_khz;
kvm->arch.apic_bus_cycle_ns = APIC_BUS_CYCLE_NS_DEFAULT;
kvm->arch.guest_can_read_msr_platform_info = true;
kvm->arch.enable_pmu = enable_pmu;
kvm->arch.enable_pmu = enable_pmu && !kvm->arch.has_protected_pmu;
#if IS_ENABLED(CONFIG_HYPERV)
spin_lock_init(&kvm->arch.hv_root_tdp_lock);
@ -14013,7 +13973,7 @@ static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
if (!kvm_pv_async_pf_enabled(vcpu))
return false;
if (!vcpu->arch.apf.send_always &&
if (!(vcpu->arch.apf.msr_en_val & KVM_ASYNC_PF_SEND_ALWAYS) &&
(vcpu->arch.guest_state_protected || !kvm_x86_call(get_cpl)(vcpu)))
return false;
@ -14022,7 +13982,7 @@ static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
* L1 needs to opt into the special #PF vmexits that are
* used to deliver async page faults.
*/
return vcpu->arch.apf.delivery_as_pf_vmexit;
return vcpu->arch.apf.msr_en_val & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
} else {
/*
* Play it safe in case the guest temporarily disables paging.
@ -14066,7 +14026,7 @@ bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
fault.nested_page_fault = false;
fault.address = work->arch.token;
fault.async_page_fault = true;
kvm_inject_page_fault(vcpu, &fault);
kvm_inject_page_fault(vcpu, &fault, false);
return true;
} else {
/*
@ -14237,7 +14197,7 @@ void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_c
fault.address = gva;
fault.async_page_fault = false;
}
vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault);
vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault, true);
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_fixup_and_inject_pf_error);

134
arch/x86/kvm/x86.h
View File

@ -6,7 +6,7 @@
#include <asm/fpu/xstate.h>
#include <asm/mce.h>
#include <asm/pvclock.h>
#include "kvm_cache_regs.h"
#include "regs.h"
#include "kvm_emulate.h"
#include "cpuid.h"
@ -243,42 +243,6 @@ static inline bool kvm_exception_is_soft(unsigned int nr)
return (nr == BP_VECTOR) || (nr == OF_VECTOR);
}
static inline bool is_protmode(struct kvm_vcpu *vcpu)
{
return kvm_is_cr0_bit_set(vcpu, X86_CR0_PE);
}
static inline bool is_long_mode(struct kvm_vcpu *vcpu)
{
#ifdef CONFIG_X86_64
return !!(vcpu->arch.efer & EFER_LMA);
#else
return false;
#endif
}
static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu)
{
int cs_db, cs_l;
WARN_ON_ONCE(vcpu->arch.guest_state_protected);
if (!is_long_mode(vcpu))
return false;
kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
return cs_l;
}
static inline bool is_64_bit_hypercall(struct kvm_vcpu *vcpu)
{
/*
* If running with protected guest state, the CS register is not
* accessible. The hypercall register values will have had to been
* provided in 64-bit mode, so assume the guest is in 64-bit.
*/
return vcpu->arch.guest_state_protected || is_64_bit_mode(vcpu);
}
static inline bool x86_exception_has_error_code(unsigned int vector)
{
static u32 exception_has_error_code = BIT(DF_VECTOR) | BIT(TS_VECTOR) |
@ -293,26 +257,6 @@ static inline bool mmu_is_nested(struct kvm_vcpu *vcpu)
return vcpu->arch.walk_mmu == &vcpu->arch.nested_mmu;
}
static inline bool is_pae(struct kvm_vcpu *vcpu)
{
return kvm_is_cr4_bit_set(vcpu, X86_CR4_PAE);
}
static inline bool is_pse(struct kvm_vcpu *vcpu)
{
return kvm_is_cr4_bit_set(vcpu, X86_CR4_PSE);
}
static inline bool is_paging(struct kvm_vcpu *vcpu)
{
return likely(kvm_is_cr0_bit_set(vcpu, X86_CR0_PG));
}
static inline bool is_pae_paging(struct kvm_vcpu *vcpu)
{
return !is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu);
}
static inline u8 vcpu_virt_addr_bits(struct kvm_vcpu *vcpu)
{
return kvm_is_cr4_bit_set(vcpu, X86_CR4_LA57) ? 57 : 48;
@ -421,21 +365,6 @@ static inline bool vcpu_match_mmio_gpa(struct kvm_vcpu *vcpu, gpa_t gpa)
return false;
}
static inline unsigned long kvm_register_read(struct kvm_vcpu *vcpu, int reg)
{
unsigned long val = kvm_register_read_raw(vcpu, reg);
return is_64_bit_mode(vcpu) ? val : (u32)val;
}
static inline void kvm_register_write(struct kvm_vcpu *vcpu,
int reg, unsigned long val)
{
if (!is_64_bit_mode(vcpu))
val = (u32)val;
return kvm_register_write_raw(vcpu, reg, val);
}
static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk)
{
return !(kvm->arch.disabled_quirks & quirk);
@ -630,15 +559,23 @@ static inline bool kvm_pat_valid(u64 data)
return (data | ((data & 0x0202020202020202ull) << 1)) == data;
}
static inline bool kvm_dr7_valid(u64 data)
static inline bool __kvm_pv_async_pf_enabled(u64 data)
{
/* Bits [63:32] are reserved */
return !(data >> 32);
u64 mask = KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT;
return (data & mask) == mask;
}
static inline bool kvm_dr6_valid(u64 data)
static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)
{
/* Bits [63:32] are reserved */
return !(data >> 32);
return __kvm_pv_async_pf_enabled(vcpu->arch.apf.msr_en_val);
}
static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
{
int i;
for (i = 0; i < ASYNC_PF_PER_VCPU; i++)
vcpu->arch.apf.gfns[i] = ~0;
}
/*
@ -687,41 +624,6 @@ enum kvm_msr_access {
#define KVM_MSR_RET_UNSUPPORTED 2
#define KVM_MSR_RET_FILTERED 3
static inline bool __kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
return !(cr4 & vcpu->arch.cr4_guest_rsvd_bits);
}
#define __cr4_reserved_bits(__cpu_has, __c) \
({ \
u64 __reserved_bits = CR4_RESERVED_BITS; \
\
if (!__cpu_has(__c, X86_FEATURE_XSAVE)) \
__reserved_bits |= X86_CR4_OSXSAVE; \
if (!__cpu_has(__c, X86_FEATURE_SMEP)) \
__reserved_bits |= X86_CR4_SMEP; \
if (!__cpu_has(__c, X86_FEATURE_SMAP)) \
__reserved_bits |= X86_CR4_SMAP; \
if (!__cpu_has(__c, X86_FEATURE_FSGSBASE)) \
__reserved_bits |= X86_CR4_FSGSBASE; \
if (!__cpu_has(__c, X86_FEATURE_PKU)) \
__reserved_bits |= X86_CR4_PKE; \
if (!__cpu_has(__c, X86_FEATURE_LA57)) \
__reserved_bits |= X86_CR4_LA57; \
if (!__cpu_has(__c, X86_FEATURE_UMIP)) \
__reserved_bits |= X86_CR4_UMIP; \
if (!__cpu_has(__c, X86_FEATURE_VMX)) \
__reserved_bits |= X86_CR4_VMXE; \
if (!__cpu_has(__c, X86_FEATURE_PCID)) \
__reserved_bits |= X86_CR4_PCIDE; \
if (!__cpu_has(__c, X86_FEATURE_LAM)) \
__reserved_bits |= X86_CR4_LAM_SUP; \
if (!__cpu_has(__c, X86_FEATURE_SHSTK) && \
!__cpu_has(__c, X86_FEATURE_IBT)) \
__reserved_bits |= X86_CR4_CET; \
__reserved_bits; \
})
int kvm_sev_es_mmio(struct kvm_vcpu *vcpu, bool is_write, gpa_t gpa,
unsigned int bytes, void *data);
int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size,
@ -754,6 +656,12 @@ static inline void kvm_prepare_emulated_mmio_exit(struct kvm_vcpu *vcpu,
frag->data, vcpu->mmio_is_write);
}
static inline bool kvm_is_valid_map_gpa_range_ret(u64 hypercall_ret)
{
return !hypercall_ret || hypercall_ret == EINVAL ||
hypercall_ret == EAGAIN;
}
static inline bool user_exit_on_hypercall(struct kvm *kvm, unsigned long hc_nr)
{
return kvm->arch.hypercall_exit_enabled & BIT(hc_nr);

39
arch/x86/kvm/xen.c
View File

@ -1408,7 +1408,7 @@ int kvm_xen_hvm_config(struct kvm *kvm, struct kvm_xen_hvm_config *xhc)
static int kvm_xen_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result)
{
kvm_rax_write(vcpu, result);
kvm_rax_write_raw(vcpu, result);
return kvm_skip_emulated_instruction(vcpu);
}
@ -1678,32 +1678,35 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu)
bool handled = false;
u8 cpl;
input = (u64)kvm_register_read(vcpu, VCPU_REGS_RAX);
/* Hyper-V hypercalls get bit 31 set in EAX */
if ((input & 0x80000000) &&
if ((kvm_rax_read_raw(vcpu) & 0x80000000) &&
kvm_hv_hypercall_enabled(vcpu))
return kvm_hv_hypercall(vcpu);
longmode = is_64_bit_hypercall(vcpu);
if (!longmode) {
params[0] = (u32)kvm_rbx_read(vcpu);
params[1] = (u32)kvm_rcx_read(vcpu);
params[2] = (u32)kvm_rdx_read(vcpu);
params[3] = (u32)kvm_rsi_read(vcpu);
params[4] = (u32)kvm_rdi_read(vcpu);
params[5] = (u32)kvm_rbp_read(vcpu);
input = kvm_eax_read(vcpu);
params[0] = kvm_ebx_read(vcpu);
params[1] = kvm_ecx_read(vcpu);
params[2] = kvm_edx_read(vcpu);
params[3] = kvm_esi_read(vcpu);
params[4] = kvm_edi_read(vcpu);
params[5] = kvm_ebp_read(vcpu);
}
#ifdef CONFIG_X86_64
else {
params[0] = (u64)kvm_rdi_read(vcpu);
params[1] = (u64)kvm_rsi_read(vcpu);
params[2] = (u64)kvm_rdx_read(vcpu);
params[3] = (u64)kvm_r10_read(vcpu);
params[4] = (u64)kvm_r8_read(vcpu);
params[5] = (u64)kvm_r9_read(vcpu);
}
#ifdef CONFIG_X86_64
input = (u64)kvm_rax_read_raw(vcpu);
params[0] = (u64)kvm_rdi_read_raw(vcpu);
params[1] = (u64)kvm_rsi_read_raw(vcpu);
params[2] = (u64)kvm_rdx_read_raw(vcpu);
params[3] = (u64)kvm_r10_read_raw(vcpu);
params[4] = (u64)kvm_r8_read_raw(vcpu);
params[5] = (u64)kvm_r9_read_raw(vcpu);
#else
KVM_BUG_ON(1, vcpu->kvm);
return -EIO;
#endif
}
cpl = kvm_x86_call(get_cpl)(vcpu);
trace_kvm_xen_hypercall(cpl, input, params[0], params[1], params[2],
params[3], params[4], params[5]);

1
tools/testing/selftests/kvm/Makefile.kvm
View File

@ -97,6 +97,7 @@ TEST_GEN_PROGS_x86 += x86/nested_emulation_test
TEST_GEN_PROGS_x86 += x86/nested_exceptions_test
TEST_GEN_PROGS_x86 += x86/nested_invalid_cr3_test
TEST_GEN_PROGS_x86 += x86/nested_set_state_test
TEST_GEN_PROGS_x86 += x86/nested_tdp_fault_test
TEST_GEN_PROGS_x86 += x86/nested_tsc_adjust_test
TEST_GEN_PROGS_x86 += x86/nested_tsc_scaling_test
TEST_GEN_PROGS_x86 += x86/nested_vmsave_vmload_test

29
tools/testing/selftests/kvm/include/x86/processor.h
View File

@ -38,7 +38,24 @@ extern u64 guest_tsc_khz;
const char *ex_str(int vector);
#define X86_EFLAGS_FIXED (1u << 1)
#define X86_EFLAGS_CF BIT(0) /* Carry Flag */
#define X86_EFLAGS_FIXED BIT(1) /* Bit 1 - always on */
#define X86_EFLAGS_PF BIT(2) /* Parity Flag */
#define X86_EFLAGS_AF BIT(4) /* Auxiliary carry Flag */
#define X86_EFLAGS_ZF BIT(6) /* Zero Flag */
#define X86_EFLAGS_SF BIT(7) /* Sign Flag */
#define X86_EFLAGS_TF BIT(8) /* Trap Flag */
#define X86_EFLAGS_IF BIT(9) /* Interrupt Flag */
#define X86_EFLAGS_DF BIT(10) /* Direction Flag */
#define X86_EFLAGS_OF BIT(11) /* Overflow Flag */
#define X86_EFLAGS_IOPL BIT(12) /* I/O Privilege Level (2 bits) */
#define X86_EFLAGS_NT BIT(14) /* Nested Task */
#define X86_EFLAGS_RF BIT(16) /* Resume Flag */
#define X86_EFLAGS_VM BIT(17) /* Virtual Mode */
#define X86_EFLAGS_AC BIT(18) /* Alignment Check/Access Control */
#define X86_EFLAGS_VIF BIT(19) /* Virtual Interrupt Flag */
#define X86_EFLAGS_VIP BIT(20) /* Virtual Interrupt Pending */
#define X86_EFLAGS_ID BIT(21) /* CPUID detection */
#define X86_CR4_VME (1ul << 0)
#define X86_CR4_PVI (1ul << 1)
@ -209,6 +226,7 @@ struct kvm_x86_cpu_feature {
#define X86_FEATURE_SEV KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 1)
#define X86_FEATURE_SEV_ES KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 3)
#define X86_FEATURE_SEV_SNP KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 4)
#define X86_FEATURE_GP_ON_USER_CPUID KVM_X86_CPU_FEATURE(0x80000021, 0, EAX, 17)
#define X86_FEATURE_PERFMON_V2 KVM_X86_CPU_FEATURE(0x80000022, 0, EAX, 0)
#define X86_FEATURE_LBR_PMC_FREEZE KVM_X86_CPU_FEATURE(0x80000022, 0, EAX, 2)
@ -1556,6 +1574,15 @@ u64 *tdp_get_pte(struct kvm_vm *vm, u64 l2_gpa);
#define PFERR_GUEST_PAGE_MASK BIT_ULL(PFERR_GUEST_PAGE_BIT)
#define PFERR_IMPLICIT_ACCESS BIT_ULL(PFERR_IMPLICIT_ACCESS_BIT)
#define EPT_VIOLATION_ACC_READ BIT(0)
#define EPT_VIOLATION_ACC_WRITE BIT(1)
#define EPT_VIOLATION_ACC_INSTR BIT(2)
#define EPT_VIOLATION_PROT_READ BIT(3)
#define EPT_VIOLATION_PROT_WRITE BIT(4)
#define EPT_VIOLATION_PROT_EXEC BIT(5)
#define EPT_VIOLATION_GVA_IS_VALID BIT(7)
#define EPT_VIOLATION_GVA_TRANSLATED BIT(8)
bool sys_clocksource_is_based_on_tsc(void);
#endif /* SELFTEST_KVM_PROCESSOR_H */

2
tools/testing/selftests/kvm/lib/x86/processor.c
View File

@ -848,7 +848,7 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id)
/* Setup guest general purpose registers */
vcpu_regs_get(vcpu, &regs);
regs.rflags = regs.rflags | 0x2;
regs.rflags = regs.rflags | X86_EFLAGS_FIXED;
regs.rsp = stack_gva;
vcpu_regs_set(vcpu, &regs);

2
tools/testing/selftests/kvm/lib/x86/vmx.c
View File

@ -360,7 +360,7 @@ static inline void init_vmcs_guest_state(void *rip, void *rsp)
vmwrite(GUEST_DR7, 0x400);
vmwrite(GUEST_RSP, (u64)rsp);
vmwrite(GUEST_RIP, (u64)rip);
vmwrite(GUEST_RFLAGS, 2);
vmwrite(GUEST_RFLAGS, X86_EFLAGS_FIXED);
vmwrite(GUEST_PENDING_DBG_EXCEPTIONS, 0);
vmwrite(GUEST_SYSENTER_ESP, vmreadz(HOST_IA32_SYSENTER_ESP));
vmwrite(GUEST_SYSENTER_EIP, vmreadz(HOST_IA32_SYSENTER_EIP));

88
tools/testing/selftests/kvm/x86/debug_regs.c
View File

@ -15,10 +15,51 @@
#define IRQ_VECTOR 0xAA
#define CAST_TO_RIP(v) ((unsigned long long)&(v))
/* For testing data access debug BP */
u32 guest_value;
extern unsigned char sw_bp, hw_bp, write_data, ss_start, bd_start;
extern unsigned char fep_bd_start, fep_sti_start, fep_sti_end;
static int irqs_received;
static void guest_db_handler(struct ex_regs *regs)
{
static int count;
unsigned long target_rips[2] = {
CAST_TO_RIP(fep_sti_start),
CAST_TO_RIP(fep_sti_end),
};
__GUEST_ASSERT(regs->rip == target_rips[count],
"STI[%u]: unexpected rip 0x%lx (should be 0x%lx)",
count, regs->rip, target_rips[count]);
regs->rflags &= ~X86_EFLAGS_TF;
count++;
}
static void guest_irq_handler(struct ex_regs *regs)
{
/*
* The pending IRQ should finally be take when KVM_GUESTDBG_BLOCKIRQ is
* cleared and IRQs are enabled. Note, the IRQ is expected to arrive
* on the instruction immediately after STI, even though its in an STI
* shadow. Because the next instruction has a coincident #DB, and #DBs
* are not subject to STI-blocking, the #DB will push RFLAGS.IF=1 on
* the stack, and the eventual IRET will unmask IRQs and obliterate the
* STI shadow in the process.
*/
unsigned long target_rip = CAST_TO_RIP(fep_sti_start);
__GUEST_ASSERT(regs->rip == target_rip,
"IRQ: unexpected rip 0x%lx (should be 0x%lx)",
regs->rip, target_rip);
irqs_received++;
x2apic_write_reg(APIC_EOI, 0);
}
static void guest_code(void)
{
@ -64,11 +105,33 @@ static void guest_code(void)
/* DR6.BD test */
asm volatile("bd_start: mov %%dr0, %%rax" : : : "rax");
/*
* Note, the IRET from the #DB that occurs in the below STI-shadow will
* unmask IRQs, i.e. the pending interrupt will be delivered after #DB
* handling, on the CLI!
*/
if (is_forced_emulation_enabled) {
asm volatile(KVM_FEP "fep_bd_start: mov %%dr0, %%rax" : : : "rax");
/* pending debug exceptions for emulation */
asm volatile("pushf\n\t"
"orq $" __stringify(X86_EFLAGS_TF) ", (%rsp)\n\t"
"popf\n\t"
"sti\n\t"
"fep_sti_start:"
"cli\n\t"
"pushf\n\t"
"orq $" __stringify(X86_EFLAGS_TF) ", (%rsp)\n\t"
"popf\n\t"
KVM_FEP "sti\n\t"
"fep_sti_end:"
"cli\n\t");
GUEST_ASSERT(irqs_received == 1);
}
GUEST_DONE();
}
#define CAST_TO_RIP(v) ((unsigned long long)&(v))
static void vcpu_skip_insn(struct kvm_vcpu *vcpu, int insn_len)
{
struct kvm_regs regs;
@ -185,7 +248,7 @@ int main(void)
target_dr6);
}
/* Finally test global disable */
/* test global disable */
memset(&debug, 0, sizeof(debug));
debug.control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP;
debug.arch.debugreg[7] = 0x400 | DR7_GD;
@ -202,10 +265,29 @@ int main(void)
run->debug.arch.pc, target_rip, run->debug.arch.dr6,
target_dr6);
/* test global disable in emulation */
if (is_forced_emulation_enabled) {
/* Skip the 3-bytes "mov dr0" */
vcpu_skip_insn(vcpu, 3);
vcpu_run(vcpu);
TEST_ASSERT(run->exit_reason == KVM_EXIT_DEBUG &&
run->debug.arch.exception == DB_VECTOR &&
run->debug.arch.pc == CAST_TO_RIP(fep_bd_start) &&
run->debug.arch.dr6 == target_dr6,
"DR7.GD: exit %d exception %d rip 0x%llx "
"(should be 0x%llx) dr6 0x%llx (should be 0x%llx)",
run->exit_reason, run->debug.arch.exception,
run->debug.arch.pc, CAST_TO_RIP(fep_bd_start),
run->debug.arch.dr6, target_dr6);
}
/* Disable all debug controls, run to the end */
memset(&debug, 0, sizeof(debug));
vcpu_guest_debug_set(vcpu, &debug);
vm_install_exception_handler(vm, DB_VECTOR, guest_db_handler);
vm_install_exception_handler(vm, IRQ_VECTOR, guest_irq_handler);
vcpu_run(vcpu);
TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
cmd = get_ucall(vcpu, &uc);

9
tools/testing/selftests/kvm/x86/hwcr_msr_test.c
View File

@ -11,12 +11,17 @@
void test_hwcr_bit(struct kvm_vcpu *vcpu, unsigned int bit)
{
const u64 ignored = BIT_ULL(3) | BIT_ULL(6) | BIT_ULL(8);
const u64 valid = BIT_ULL(18) | BIT_ULL(24);
const u64 legal = ignored | valid;
u64 valid = BIT_ULL(18) | BIT_ULL(24);
u64 val = BIT_ULL(bit);
u64 actual;
u64 legal;
int r;
if (kvm_cpu_has(X86_FEATURE_GP_ON_USER_CPUID))
valid |= BIT_ULL(35);
legal = ignored | valid;
r = _vcpu_set_msr(vcpu, MSR_K7_HWCR, val);
TEST_ASSERT(val & ~legal ? !r : r == 1,
"Expected KVM_SET_MSRS(MSR_K7_HWCR) = 0x%lx to %s",

313
tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c Normal file
View File

@ -0,0 +1,313 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2025, Google, Inc.
*/
#include "test_util.h"
#include "kvm_util.h"
#include "processor.h"
#include "svm_util.h"
#include "vmx.h"
#define L2_GUEST_STACK_SIZE 64
enum test_type {
TEST_FINAL_PAGE_UNMAPPED, /* Final data page not present */
TEST_PT_PAGE_UNMAPPED, /* Page table page not present */
TEST_FINAL_PAGE_WRITE_PROTECTED, /* Final data page read-only */
TEST_PT_PAGE_WRITE_PROTECTED, /* Page table page read-only */
};
static gva_t l2_test_page;
static void (*l2_entry)(void);
#define TEST_IO_PORT 0x80
#define TEST1_VADDR 0x8000000ULL
#define TEST2_VADDR 0x10000000ULL
#define TEST3_VADDR 0x18000000ULL
#define TEST4_VADDR 0x20000000ULL
/*
* L2 executes OUTS reading from l2_test_page, triggering a nested page
* fault on the read access.
*/
static void l2_guest_code_outs(void)
{
asm volatile("outsb" ::"S"(l2_test_page), "d"(TEST_IO_PORT) : "memory");
GUEST_FAIL("L2 should not reach here");
}
/*
* L2 executes INS writing to l2_test_page, triggering a nested page
* fault on the write access.
*/
static void l2_guest_code_ins(void)
{
asm volatile("insb" ::"D"(l2_test_page), "d"(TEST_IO_PORT) : "memory");
GUEST_FAIL("L2 should not reach here");
}
#define GUEST_ASSERT_EXIT_QUAL(ac_eq, ex_eq) \
__GUEST_ASSERT((ac_eq) == (ex_eq), \
"Wanted EXIT_QUAL '0x%lx', got '0x%lx'", ex_eq, ac_eq)
static void l1_vmx_code(struct vmx_pages *vmx, u64 expected_fault_gpa,
u64 test_type)
{
unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
u64 exit_qual;
GUEST_ASSERT(vmx->vmcs_gpa);
GUEST_ASSERT(prepare_for_vmx_operation(vmx));
GUEST_ASSERT(load_vmcs(vmx));
prepare_vmcs(vmx, l2_entry, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
GUEST_ASSERT(!vmlaunch());
/* Verify we got an EPT violation exit */
__GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_EPT_VIOLATION,
"Expected EPT violation (0x%x), got 0x%lx",
EXIT_REASON_EPT_VIOLATION,
vmreadz(VM_EXIT_REASON));
__GUEST_ASSERT(vmreadz(GUEST_PHYSICAL_ADDRESS) == expected_fault_gpa,
"Expected guest_physical_address = 0x%lx, got 0x%lx",
expected_fault_gpa,
vmreadz(GUEST_PHYSICAL_ADDRESS));
exit_qual = vmreadz(EXIT_QUALIFICATION);
/*
* Note, EPT page table accesses are always read+write, e.g. so that
* the CPU can do A/D updates at-will.
*/
switch (test_type) {
case TEST_FINAL_PAGE_UNMAPPED:
GUEST_ASSERT_EXIT_QUAL(exit_qual, EPT_VIOLATION_ACC_READ |
EPT_VIOLATION_GVA_IS_VALID |
EPT_VIOLATION_GVA_TRANSLATED);
break;
case TEST_PT_PAGE_UNMAPPED:
GUEST_ASSERT_EXIT_QUAL(exit_qual, EPT_VIOLATION_ACC_READ |
EPT_VIOLATION_ACC_WRITE |
EPT_VIOLATION_GVA_IS_VALID);
break;
case TEST_FINAL_PAGE_WRITE_PROTECTED:
GUEST_ASSERT_EXIT_QUAL(exit_qual, EPT_VIOLATION_ACC_WRITE |
EPT_VIOLATION_PROT_READ |
EPT_VIOLATION_PROT_EXEC |
EPT_VIOLATION_GVA_IS_VALID |
EPT_VIOLATION_GVA_TRANSLATED);
break;
case TEST_PT_PAGE_WRITE_PROTECTED:
GUEST_ASSERT_EXIT_QUAL(exit_qual, EPT_VIOLATION_ACC_READ |
EPT_VIOLATION_ACC_WRITE |
EPT_VIOLATION_PROT_READ |
EPT_VIOLATION_PROT_EXEC |
EPT_VIOLATION_GVA_IS_VALID);
break;
}
GUEST_DONE();
}
#define GUEST_ASSERT_NPF_EC(ac_ec, ex_ec) \
__GUEST_ASSERT((ac_ec) == (ex_ec), \
"Wanted NPF error code '0x%lx', got '0x%lx'", (u64)(ex_ec), ac_ec)
static void l1_svm_code(struct svm_test_data *svm, u64 expected_fault_gpa,
u64 test_type)
{
unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
struct vmcb *vmcb = svm->vmcb;
u64 exit_info_1;
generic_svm_setup(svm, l2_entry,
&l2_guest_stack[L2_GUEST_STACK_SIZE]);
run_guest(vmcb, svm->vmcb_gpa);
/* Verify we got an NPF exit */
__GUEST_ASSERT(vmcb->control.exit_code == SVM_EXIT_NPF,
"Expected NPF exit (0x%x), got 0x%lx", SVM_EXIT_NPF,
vmcb->control.exit_code);
__GUEST_ASSERT(vmcb->control.exit_info_2 == expected_fault_gpa,
"Expected exit_info_2 = 0x%lx, got 0x%lx",
expected_fault_gpa,
vmcb->control.exit_info_2);
exit_info_1 = vmcb->control.exit_info_1;
/*
* Note, without GMET enabled, NPT walks are always user accesses. And
* like EPT, page table accesses are always read+write.
*/
switch (test_type) {
case TEST_FINAL_PAGE_UNMAPPED:
GUEST_ASSERT_NPF_EC(exit_info_1, PFERR_USER_MASK |
PFERR_GUEST_FINAL_MASK);
break;
case TEST_PT_PAGE_UNMAPPED:
GUEST_ASSERT_NPF_EC(exit_info_1, PFERR_WRITE_MASK |
PFERR_USER_MASK |
PFERR_GUEST_PAGE_MASK);
break;
case TEST_FINAL_PAGE_WRITE_PROTECTED:
GUEST_ASSERT_NPF_EC(exit_info_1, PFERR_PRESENT_MASK |
PFERR_WRITE_MASK |
PFERR_USER_MASK |
PFERR_GUEST_FINAL_MASK);
break;
case TEST_PT_PAGE_WRITE_PROTECTED:
GUEST_ASSERT_NPF_EC(exit_info_1, PFERR_PRESENT_MASK |
PFERR_WRITE_MASK |
PFERR_USER_MASK |
PFERR_GUEST_PAGE_MASK);
break;
}
GUEST_DONE();
}
static void l1_guest_code(void *data, u64 expected_fault_gpa,
u64 test_type)
{
if (this_cpu_has(X86_FEATURE_VMX))
l1_vmx_code(data, expected_fault_gpa, test_type);
else
l1_svm_code(data, expected_fault_gpa, test_type);
}
/* Returns the GPA of the PT page that maps @vaddr. */
static u64 get_pt_gpa_for_vaddr(struct kvm_vm *vm, u64 vaddr)
{
u64 *pte;
pte = vm_get_pte(vm, vaddr);
TEST_ASSERT(pte && (*pte & 0x1), "PTE not present for vaddr 0x%lx",
(unsigned long)vaddr);
return addr_hva2gpa(vm, (void *)((u64)pte & ~0xFFFULL));
}
static void run_test(enum test_type type)
{
gpa_t expected_fault_gpa;
gva_t nested_gva;
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
struct ucall uc;
vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code);
vm_enable_tdp(vm);
if (kvm_cpu_has(X86_FEATURE_VMX))
vcpu_alloc_vmx(vm, &nested_gva);
else
vcpu_alloc_svm(vm, &nested_gva);
switch (type) {
case TEST_FINAL_PAGE_UNMAPPED:
/*
* Unmap the final data page from NPT/EPT. The guest page
* table walk succeeds, but the final GPA->HPA translation
* fails. L2 reads from the page via OUTS.
*/
l2_entry = l2_guest_code_outs;
l2_test_page = vm_alloc(vm, vm->page_size, TEST1_VADDR);
expected_fault_gpa = addr_gva2gpa(vm, l2_test_page);
break;
case TEST_PT_PAGE_UNMAPPED:
/*
* Unmap a page table page from NPT/EPT. The hardware page
* table walk fails when translating the PT page's GPA
* through NPT/EPT. L2 reads from the page via OUTS.
*/
l2_entry = l2_guest_code_outs;
l2_test_page = vm_alloc(vm, vm->page_size, TEST2_VADDR);
expected_fault_gpa = get_pt_gpa_for_vaddr(vm, l2_test_page);
break;
case TEST_FINAL_PAGE_WRITE_PROTECTED:
/*
* Write-protect the final data page in NPT/EPT. The page
* is present and readable, but not writable. L2 writes to
* the page via INS, triggering a protection violation.
*/
l2_entry = l2_guest_code_ins;
l2_test_page = vm_alloc(vm, vm->page_size, TEST3_VADDR);
expected_fault_gpa = addr_gva2gpa(vm, l2_test_page);
break;
case TEST_PT_PAGE_WRITE_PROTECTED:
/*
* Write-protect a page table page in NPT/EPT. The page is
* present and readable, but not writable. The guest page
* table walk needs write access to set A/D bits, so it
* triggers a protection violation on the PT page.
* L2 reads from the page via OUTS.
*/
l2_entry = l2_guest_code_outs;
l2_test_page = vm_alloc(vm, vm->page_size, TEST4_VADDR);
expected_fault_gpa = get_pt_gpa_for_vaddr(vm, l2_test_page);
break;
}
tdp_identity_map_default_memslots(vm);
if (type == TEST_FINAL_PAGE_WRITE_PROTECTED ||
type == TEST_PT_PAGE_WRITE_PROTECTED)
*tdp_get_pte(vm, expected_fault_gpa) &= ~PTE_WRITABLE_MASK(&vm->stage2_mmu);
else
*tdp_get_pte(vm, expected_fault_gpa) &= ~(PTE_PRESENT_MASK(&vm->stage2_mmu) |
PTE_READABLE_MASK(&vm->stage2_mmu) |
PTE_WRITABLE_MASK(&vm->stage2_mmu) |
PTE_EXECUTABLE_MASK(&vm->stage2_mmu));
sync_global_to_guest(vm, l2_entry);
sync_global_to_guest(vm, l2_test_page);
vcpu_args_set(vcpu, 3, nested_gva, expected_fault_gpa, (u64)type);
/*
* For the INS-based write test, KVM emulates the instruction and
* first reads from the I/O port, which exits to userspace.
* Re-enter the guest so emulation can proceed to the memory
* write, where the nested page fault is triggered.
*/
for (;;) {
vcpu_run(vcpu);
if (vcpu->run->exit_reason == KVM_EXIT_IO &&
vcpu->run->io.port == TEST_IO_PORT &&
vcpu->run->io.direction == KVM_EXIT_IO_IN) {
continue;
}
break;
}
switch (get_ucall(vcpu, &uc)) {
case UCALL_DONE:
break;
case UCALL_ABORT:
REPORT_GUEST_ASSERT(uc);
default:
TEST_FAIL("Unexpected exit reason: %d", vcpu->run->exit_reason);
}
kvm_vm_free(vm);
}
int main(int argc, char *argv[])
{
TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) || kvm_cpu_has(X86_FEATURE_SVM));
TEST_REQUIRE(kvm_cpu_has_tdp());
run_test(TEST_FINAL_PAGE_UNMAPPED);
run_test(TEST_PT_PAGE_UNMAPPED);
run_test(TEST_FINAL_PAGE_WRITE_PROTECTED);
run_test(TEST_PT_PAGE_WRITE_PROTECTED);
return 0;
}

6
tools/testing/selftests/kvm/x86/pmu_event_filter_test.c
View File

@ -731,6 +731,8 @@ static void test_filter_ioctl(struct kvm_vcpu *vcpu)
static void intel_run_fixed_counter_guest_code(u8 idx)
{
u8 nr_fixed_counters = this_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
for (;;) {
wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
wrmsr(MSR_CORE_PERF_FIXED_CTR0 + idx, 0);
@ -738,6 +740,10 @@ static void intel_run_fixed_counter_guest_code(u8 idx)
/* Only OS_EN bit is enabled for fixed counter[idx]. */
wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, FIXED_PMC_CTRL(idx, FIXED_PMC_KERNEL));
wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, FIXED_PMC_GLOBAL_CTRL_ENABLE(idx));
if (nr_fixed_counters > 1)
wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL,
FIXED_PMC_CTRL(idx, FIXED_PMC_KERNEL) |
FIXED_PMC_CTRL((idx + 1) % nr_fixed_counters, FIXED_PMC_KERNEL));
__asm__ __volatile__("loop ." : "+c"((int){NUM_BRANCHES}));
wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);