diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 4f27ccbc5ba1..e95c9cda085d 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -9760,6 +9760,31 @@ On older versions of Linux, CPU[EAX=1]:ECX[24] (TSC_DEADLINE) is not reported by is present and the kernel has enabled in-kernel emulation of the local APIC. On newer versions, ``KVM_GET_SUPPORTED_CPUID`` does report the bit as available. +Long mode segment limits +~~~~~~~~~~~~~~~~~~~~~~~~ + +CPU[EAX=0x80000008]:EBX[20] (EFER_LMSLE_MBZ) is a "defeature" bit: it is set +when the CPU does *not* support long mode segment limits, and so requires +EFER.LMSLE to be zero. KVM reports the bit via ``KVM_GET_SUPPORTED_CPUID`` if +and only if KVM refuses to set EFER.LMSLE, i.e. if the CPU doesn't support long +mode segment limits, or if nested SVM is unsupported. KVM therefore reports +the bit on all Intel hosts, as KVM allows EFER.LMSLE only when nested SVM is +enabled. + +KVM never reports the bit via ``KVM_GET_EMULATED_CPUID``, but userspace may set +it via ``KVM_SET_CPUID2`` even on a host where KVM doesn't report it. KVM +honors the guest's enumeration and rejects EFER.LMSLE=1 accordingly. That lets +userspace defeature a vCPU on a host that *does* support long mode segment +limits, so that the vCPU can later be migrated to a host that doesn't, e.g. so +that a vCPU created on AMD Rome can be migrated to Milan and later, which +dropped support for long mode segment limits. The opposite direction needs no +emulation, as a host that lacks long mode segment limits already enumerates the +defeature. + +Note, ``KVM_SET_MSRS`` is exempt from the check, as host-initiated MSR writes +skip guest CPUID checks so that userspace can set MSRs before it sets guest +CPUID. ``KVM_SET_SREGS`` and nested VMRUN are not exempt. + CPU topology ~~~~~~~~~~~~ diff --git a/Documentation/virt/kvm/x86/amd-memory-encryption.rst b/Documentation/virt/kvm/x86/amd-memory-encryption.rst index 5977fbe33b98..eeb0812cb10a 100644 --- a/Documentation/virt/kvm/x86/amd-memory-encryption.rst +++ b/Documentation/virt/kvm/x86/amd-memory-encryption.rst @@ -170,7 +170,7 @@ Returns: 0 on success, -negative on error :: - struct kvm_sev_launch_update { + struct kvm_sev_launch_update_data { __u64 uaddr; /* userspace address to be encrypted (must be 16-byte aligned) */ __u32 len; /* length of the data to be encrypted (must be 16-byte aligned) */ }; @@ -316,7 +316,7 @@ used by the guest owner with the KVM_SEV_LAUNCH_MEASURE. If len is zero on entry, the measurement blob length is written to len and uaddr is unused. -Parameters (in): struct kvm_sev_attestation +Parameters (in): struct kvm_sev_attestation_report Returns: 0 on success, -negative on error @@ -377,7 +377,7 @@ Returns: 0 on success, -negative on error :: - struct kvm_sev_launch_send_update_data { + struct kvm_sev_send_update_data { __u64 hdr_uaddr; /* userspace address containing the packet header */ __u32 hdr_len; @@ -447,7 +447,7 @@ Returns: 0 on success, -negative on error :: - struct kvm_sev_launch_receive_update_data { + struct kvm_sev_receive_update_data { __u64 hdr_uaddr; /* userspace address containing the packet header */ __u32 hdr_len; diff --git a/Documentation/virt/kvm/x86/hypercalls.rst b/Documentation/virt/kvm/x86/hypercalls.rst index 521ecf9a8a36..e0b6c11f19ce 100644 --- a/Documentation/virt/kvm/x86/hypercalls.rst +++ b/Documentation/virt/kvm/x86/hypercalls.rst @@ -109,7 +109,7 @@ The template for each hypercall is: Usage: a0: guest physical address where host copies -"struct kvm_clock_offset" structure. +"struct kvm_clock_pairing" structure. a1: clock_type, ATM only KVM_CLOCK_PAIRING_WALLCLOCK (0) is supported (corresponding to the host's CLOCK_REALTIME clock). diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index fe84cf4e97d3..e2c21e58190d 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -238,6 +238,7 @@ // free: was #define X86_FEATURE_PVUNLOCK ( 8*32+20) /* PV unlock function */ #define X86_FEATURE_VCPUPREEMPT ( 8*32+21) /* PV vcpu_is_preempted function */ #define X86_FEATURE_TDX_GUEST ( 8*32+22) /* "tdx_guest" Intel Trust Domain Extensions Guest */ +#define X86_FEATURE_PML ( 8*32+23) /* AMD Page Modification logging */ /* Intel-defined CPU features, CPUID level 0x00000007:0 (EBX), word 9 */ #define X86_FEATURE_FSGSBASE ( 9*32+ 0) /* "fsgsbase" RDFSBASE, WRFSBASE, RDGSBASE, WRGSBASE instructions*/ diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 4b070981016a..2a0245fa7e26 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -794,6 +794,8 @@ struct kvm_vcpu_arch { */ struct kvm_mmu_memory_cache mmu_external_spt_cache; + struct page *pml_page; + /* * QEMU userspace and the guest each have their own FPU state. * In vcpu_run, we switch between the user and guest FPU contexts. @@ -979,7 +981,7 @@ struct kvm_vcpu_arch { bool pv_unhalted; } pv; - int pending_ioapic_eoi; + u8 pending_ioapic_eoi; int pending_external_vector; int highest_stale_pending_ioapic_eoi; @@ -1171,6 +1173,7 @@ struct kvm_arch { u8 mmu_valid_gen; u8 vm_type; bool has_private_mem; + bool has_protected_page_tables; bool has_protected_state; bool has_protected_eoi; bool has_protected_pmu; @@ -1416,6 +1419,7 @@ struct kvm_arch { struct kvm_mmu_memory_cache split_desc_cache; gfn_t gfn_direct_bits; + int mirror_root_level; /* * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer. A Zero @@ -1588,7 +1592,7 @@ struct kvm_x86_ops { * Can potentially get non-canonical addresses through INVLPGs, which * the implementation may choose to ignore if appropriate. */ - void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr, bool *full); + void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr); /* * Flush any TLB entries created by the guest. Like tlb_flush_gva(), @@ -1679,7 +1683,7 @@ struct kvm_x86_ops { struct x86_exception *exception); void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu); - void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu); + void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu, bool enable); void (*vcpu_blocking)(struct kvm_vcpu *vcpu); void (*vcpu_unblocking)(struct kvm_vcpu *vcpu); diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index aa63431ba92c..e0a7549a7c72 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -165,7 +165,10 @@ struct __attribute__ ((__packed__)) vmcb_control_area { u8 reserved_9[22]; u64 allowed_sev_features; /* Offset 0x138 */ u64 guest_sev_features; /* Offset 0x140 */ - u8 reserved_10[664]; + u8 reserved_10[128]; + u64 pml_addr; /* Offset 0x1c8 */ + u16 pml_index; /* Offset 0x1d0 */ + u8 reserved_11[526]; /* * Offset 0x3e0, 32 bytes reserved * for use by hypervisor/software. @@ -240,10 +243,11 @@ struct __attribute__ ((__packed__)) vmcb_control_area { #define SVM_IOIO_SIZE_MASK (7 << SVM_IOIO_SIZE_SHIFT) #define SVM_IOIO_ASIZE_MASK (7 << SVM_IOIO_ASIZE_SHIFT) -#define SVM_MISC_ENABLE_NP BIT(0) -#define SVM_MISC_ENABLE_SEV BIT(1) -#define SVM_MISC_ENABLE_SEV_ES BIT(2) -#define SVM_MISC_ENABLE_GMET BIT(3) +#define SVM_MISC_ENABLE_NP BIT_ULL(0) +#define SVM_MISC_ENABLE_SEV BIT_ULL(1) +#define SVM_MISC_ENABLE_SEV_ES BIT_ULL(2) +#define SVM_MISC_ENABLE_GMET BIT_ULL(3) +#define SVM_MISC_ENABLE_PML BIT_ULL(11) #define SVM_MISC2_ENABLE_V_LBR BIT_ULL(0) #define SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE BIT_ULL(1) diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index 010a45c9f614..e80676185092 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h @@ -101,6 +101,7 @@ #define SVM_EXIT_AVIC_INCOMPLETE_IPI 0x401 #define SVM_EXIT_AVIC_UNACCELERATED_ACCESS 0x402 #define SVM_EXIT_VMGEXIT 0x403 +#define SVM_EXIT_PML_FULL 0x407 /* SEV-ES software-defined VMGEXIT events */ #define SVM_VMGEXIT_MMIO_READ 0x80000001ull @@ -236,6 +237,7 @@ { SVM_EXIT_AVIC_INCOMPLETE_IPI, "avic_incomplete_ipi" }, \ { SVM_EXIT_AVIC_UNACCELERATED_ACCESS, "avic_unaccelerated_access" }, \ { SVM_EXIT_VMGEXIT, "vmgexit" }, \ + { SVM_EXIT_PML_FULL, "pml_full" }, \ { SVM_VMGEXIT_MMIO_READ, "vmgexit_mmio_read" }, \ { SVM_VMGEXIT_MMIO_WRITE, "vmgexit_mmio_write" }, \ { SVM_VMGEXIT_NMI_COMPLETE, "vmgexit_nmi_complete" }, \ diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index 8665a6474806..b18cc9c88dd5 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -55,6 +55,7 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_AMD_FAST_CPPC, CPUID_EDX, 15, 0x80000007, 0 }, { X86_FEATURE_CPPC_PERF_PRIO, CPUID_EDX, 16, 0x80000007, 0 }, { X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 }, + { X86_FEATURE_PML, CPUID_ECX, 4, 0x8000000a, 0 }, { X86_FEATURE_X2AVIC_EXT, CPUID_ECX, 6, 0x8000000a, 0 }, { X86_FEATURE_COHERENCY_SFW_NO, CPUID_EBX, 31, 0x8000001f, 0 }, { X86_FEATURE_SMBA, CPUID_EBX, 2, 0x80000020, 0 }, diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 38450e9fca9a..adafd6fa2c54 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -484,20 +484,6 @@ int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu) return 36; } -int cpuid_query_maxguestphyaddr(struct kvm_vcpu *vcpu) -{ - struct kvm_cpuid_entry2 *best; - - best = kvm_find_cpuid_entry(vcpu, 0x80000000); - if (!best || best->eax < 0x80000008) - goto not_found; - best = kvm_find_cpuid_entry(vcpu, 0x80000008); - if (best) - return (best->eax >> 16) & 0xff; -not_found: - return 0; -} - /* * This "raw" version returns the reserved GPA bits without any adjustments for * encryption technologies that usurp bits. The raw mask should be used if and @@ -1096,7 +1082,7 @@ void kvm_initialize_cpu_caps(void) ); kvm_cpu_cap_init(CPUID_24_1_ECX, - F(AVX10_VNNI_INT), + F(AVX10_V1_AUX), ); kvm_cpu_cap_init(CPUID_8000_0001_ECX, @@ -1172,6 +1158,10 @@ void kvm_initialize_cpu_caps(void) F(AMD_STIBP), F(AMD_STIBP_ALWAYS_ON), F(AMD_IBRS_SAME_MODE), + /* + * Vendor code also sets EFER_LMSLE_MBZ if KVM itself + * can't support EFER.LMSLE, e.g. if nested SVM is disabled. + */ PASSTHROUGH_F(EFER_LMSLE_MBZ), F(AMD_PSFD), F(AMD_IBPB_RET), @@ -1416,6 +1406,22 @@ static int cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, u32 func, u32 ind if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) entry->ecx = feature_bit(RDPID); return 1; + case 0x80000008: + /* + * Honor the guest's EFER_LMSLE_MBZ even if the underlying CPU + * allows setting EFER.LMSLE, e.g. to allow migrating a vCPU + * between hosts with and without EFER.LMSLE support. To avoid + * breaking existing setups that reflect KVM's supported CPUID + * into the guest, KVM doesn't advertise EFER_LMSLE_MBZ unless + * KVM *can't* support EFER.LMSLE=1. + */ + if (include_partially_emulated && + !kvm_cpu_cap_has(X86_FEATURE_EFER_LMSLE_MBZ)) { + entry->ebx |= feature_bit(EFER_LMSLE_MBZ); + return 1; + } + /* Nothing in 0x80000008 is fully emulated, don't emit an entry. */ + return 0; default: return 0; } diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h index 8d863f45585d..46bfe8699e67 100644 --- a/arch/x86/kvm/cpuid.h +++ b/arch/x86/kvm/cpuid.h @@ -68,7 +68,6 @@ void __init kvm_init_xstate_sizes(void); u32 xstate_required_size(u64 xstate_bv, bool compacted); int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu); -int cpuid_query_maxguestphyaddr(struct kvm_vcpu *vcpu); u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu); static inline int cpuid_maxphyaddr(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 604651cb2739..5d2e3617c952 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -802,7 +802,9 @@ static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint, if (no_retry) return 0; + hv_hdr.message_flags.asu8 = 0; hv_hdr.message_flags.msg_pending = 1; + r = kvm_vcpu_write_guest_page(vcpu, msg_page_gfn, &hv_hdr.message_flags, msg_off + @@ -1995,7 +1997,6 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) u64 entries[KVM_HV_TLB_FLUSH_FIFO_SIZE]; int i, j, count; gva_t gva; - bool full = false; if (!tdp_enabled) return -EINVAL; @@ -2006,7 +2007,7 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) count = kfifo_out(&tlb_flush_fifo->entries, entries, KVM_HV_TLB_FLUSH_FIFO_SIZE); - for (i = 0; i < count && !full; i++) { + for (i = 0; i < count; i++) { if (entries[i] == KVM_HV_TLB_FLUSHALL_ENTRY) goto out_flush_all; @@ -2015,11 +2016,11 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) * pages to flush. */ gva = entries[i] & PAGE_MASK; - for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1 && !full; j++) { + for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1; j++) { if (is_noncanonical_invlpg_address(gva + j * PAGE_SIZE, vcpu)) continue; - kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE, &full); + kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE); } ++vcpu->stat.tlb_flush; diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h index 3e375af15c03..10886bea82c9 100644 --- a/arch/x86/kvm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h @@ -354,8 +354,8 @@ struct x86_emulate_ctxt { bool have_exception; struct x86_exception exception; - /* GPA available */ - bool gpa_available; + /* Saved GPA and permitted emulated accesses. */ + u64 gpa_access; gpa_t gpa_val; /* diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index e1f3cea14765..ecf611845fb1 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1552,7 +1552,7 @@ static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector) return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors); } -static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) +static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, u8 vector) { int __maybe_unused trigger_mode; @@ -1622,7 +1622,7 @@ static int apic_set_eoi(struct kvm_lapic *apic) * this interface assumes a trap-like exit, which has already finished * desired side effect including vISR and vPPR update. */ -void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) +void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, u8 vector) { struct kvm_lapic *apic = vcpu->arch.apic; diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index bd1098c89d99..db071d0b172d 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -177,7 +177,7 @@ u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu); void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data); void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset); -void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector); +void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, u8 vector); int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr); void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 5a83f09b4d68..205dfe00111b 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5921,6 +5921,9 @@ static union kvm_cpu_role kvm_calc_cpu_role(struct kvm_vcpu *vcpu, return role; } + if (KVM_BUG_ON(____is_efer_lma(regs) && !____is_cr4_pae(regs), vcpu->kvm)) + *(u64 *)®s->efer &= ~EFER_LMA; + role.base.efer_nx = ____is_efer_nx(regs); role.base.cr0_wp = ____is_cr0_wp(regs); role.base.cr4_smep = ____is_cr4_smep(regs); @@ -5964,19 +5967,22 @@ void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu, static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu) { - int maxpa; - - if (vcpu->kvm->arch.vm_type == KVM_X86_TDX_VM) - maxpa = cpuid_query_maxguestphyaddr(vcpu); - else - maxpa = cpuid_maxphyaddr(vcpu); - /* tdp_root_level is architecture forced level, use it if nonzero */ if (tdp_root_level) return tdp_root_level; + /* + * If the VM has mirror roots, then the root level is predefined as the + * mirror root (and by extension the normal root) needs to match the + * root level that was configured for the external page tables that are + * being mirrored by KVM. + */ + if (kvm_has_mirrored_tdp(vcpu->kvm) && + !WARN_ON_ONCE(!vcpu->kvm->arch.mirror_root_level)) + return vcpu->kvm->arch.mirror_root_level; + /* Use 5-level TDP if and only if it's useful/necessary. */ - if (max_tdp_level == 5 && maxpa <= 48) + if (max_tdp_level == 5 && cpuid_maxphyaddr(vcpu) <= 48) return 4; return max_tdp_level; @@ -6643,6 +6649,9 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err return r; emulate: + if (error_code & PFERR_WRITE_MASK) + emulation_type |= EMULTYPE_PF_WRITE; + return x86_emulate_instruction(vcpu, cr2_or_gpa, emulation_type, insn, insn_len); } @@ -6715,7 +6724,7 @@ void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w, if (is_noncanonical_invlpg_address(addr, vcpu)) return; - kvm_x86_call(flush_tlb_gva)(vcpu, addr, NULL); + kvm_x86_call(flush_tlb_gva)(vcpu, addr); if (tdp_enabled) return; @@ -6845,7 +6854,7 @@ static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, struct k * other exception is for shadowing L1's 32-bit or PAE NPT on 64-bit * KVM; that horror is handled on-demand by mmu_alloc_special_roots(). */ - if (tdp_enabled && kvm_mmu_get_tdp_level(vcpu) > PT32E_ROOT_LEVEL) + if (tdp_enabled && kvm_mmu_get_max_tdp_level() > PT32E_ROOT_LEVEL) return 0; page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32); diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 27427e7f22fa..c8ec47b09264 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -368,13 +368,14 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, pte_access = ~0; /* - * Queue a page fault for injection if this assertion fails, as callers - * assume that walker.fault contains sane info on a walk failure. I.e. - * avoid making the situation worse by inducing even worse badness - * between when the assertion fails and when KVM kicks the vCPU out to - * userspace (because the VM is bugged). + * Queue a page fault for injection if any of the below assertions fail, + * as callers assume that walker.fault contains sane info on a walk + * failure. I.e. avoid making the situation worse by inducing even + * worse badness between when the assertion fails and when KVM kicks + * the vCPU out to userspace (because the VM is bugged). */ - if (KVM_BUG_ON(is_long_mode(vcpu) && !is_pae(vcpu), vcpu->kvm)) + if (KVM_BUG_ON(is_long_mode(vcpu) && !is_pae(vcpu), vcpu->kvm) || + KVM_BUG_ON(walker->max_level > PT_MAX_FULL_LEVELS, vcpu->kvm)) goto error; ++walker->level; @@ -391,7 +392,9 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, offset = index * sizeof(pt_element_t); pte_gpa = gfn_to_gpa(table_gfn) + offset; - BUG_ON(walker->level < 1); + if (KVM_BUG_ON(walker->level < 1, vcpu->kvm)) + goto error; + walker->table_gfn[walker->level - 1] = table_gfn; walker->pte_gpa[walker->level - 1] = pte_gpa; diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c index 37d5bed3529f..e91650af44c2 100644 --- a/arch/x86/kvm/msrs.c +++ b/arch/x86/kvm/msrs.c @@ -598,9 +598,19 @@ static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) if (efer & EFER_NX && !guest_cpu_cap_has(vcpu, X86_FEATURE_NX)) return false; - return true; + /* + * EFER_LMSLE_MBZ is a "defeature" bit, i.e. is set when the CPU does + * *not* support long mode segment limits, and so is the only EFER + * check whose polarity is inverted: EFER.LMSLE is legal if and only if + * the guest does *not* have the defeature. + */ + if (efer & EFER_LMSLE && + guest_cpu_cap_has(vcpu, X86_FEATURE_EFER_LMSLE_MBZ)) + return false; + return true; } + bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) { if (efer & ~kvm_caps.supported_efer_bits) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index d2fd47ee5ec8..eef2f792dc9e 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -662,7 +662,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) DECLARE_BITMAP(bitmap, X86_PMC_IDX_MAX); struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); struct kvm_pmc *pmc; - int bit; + int bit, r; bitmap_copy(bitmap, pmu->reprogram_pmi, X86_PMC_IDX_MAX); @@ -676,12 +676,14 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) kvm_for_each_pmc(pmu, pmc, bit, bitmap) { /* - * If reprogramming fails, e.g. due to contention, re-set the - * reprogram bit, i.e. opportunistically try again on the next - * PMU refresh. Don't make a new request as doing so can stall - * the guest if reprogramming repeatedly fails. + * If reprogramming fails on a transient condition, e.g. due to + * contention, re-set the reprogram bit, i.e. opportunistically + * try again on the next PMU refresh. Don't make a new request + * as doing so can stall the guest if reprogramming repeatedly + * fails. */ - if (reprogram_counter(pmc)) + r = reprogram_counter(pmc); + if (r == -EBUSY || r == -ENOMEM) set_bit(pmc->idx, pmu->reprogram_pmi); } diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c index 8f66438989e4..3621f1a47d37 100644 --- a/arch/x86/kvm/regs.c +++ b/arch/x86/kvm/regs.c @@ -440,9 +440,14 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr4); +static bool kvm_is_valid_cr8(unsigned long cr8) +{ + return !(cr8 & CR8_RESERVED_BITS); +} + int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) { - if (cr8 & CR8_RESERVED_BITS) + if (!kvm_is_valid_cr8(cr8)) return 1; if (lapic_in_kernel(vcpu)) kvm_lapic_set_tpr(vcpu, cr8); @@ -565,6 +570,7 @@ static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) return kvm_is_valid_cr4(vcpu, sregs->cr4) && kvm_is_valid_cr0(vcpu, sregs->cr0) && + kvm_is_valid_cr8(sregs->cr8) && kvm_valid_efer(vcpu, sregs->efer); } @@ -766,17 +772,19 @@ void kvm_update_dr7(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); -static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) +unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) { - u64 fixed = DR6_FIXED_1; + unsigned long fixed = DR6_ACTIVE_LOW & ~DR6_VOLATILE; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM)) fixed |= DR6_RTM; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) fixed |= DR6_BUS_LOCK; + return fixed; } +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dr6_fixed_1); int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) { @@ -792,7 +800,7 @@ int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) case 6: if (!kvm_dr6_valid(val)) return 1; /* #GP */ - vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu); + vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_get_dr6_fixed_1(vcpu); break; case 5: default: /* 7 */ diff --git a/arch/x86/kvm/regs.h b/arch/x86/kvm/regs.h index 447f0ec3e63e..97c17b120a57 100644 --- a/arch/x86/kvm/regs.h +++ b/arch/x86/kvm/regs.h @@ -38,16 +38,14 @@ static_assert(!(KVM_POSSIBLE_CR0_GUEST_BITS & X86_CR0_PDPTR_BITS)); #define DR6_BT (1 << 15) #define DR6_RTM (1 << 16) /* - * DR6_ACTIVE_LOW combines fixed-1 and active-low bits. - * We can regard all the bits in DR6_FIXED_1 as active_low bits; - * they will never be 0 for now, but when they are defined - * in the future it will require no code change. + * DR6_ACTIVE_LOW combines fixed-1 and active-low bits (bits that are currently + * fixed-1 are guaranteed to have active-low semantics if they are ever used to + * support a new feature). * - * DR6_ACTIVE_LOW is also used as the init/reset value for DR6. + * Note, DR6_ACTIVE_LOW is also the INIT/RESET value for DR6. */ #define DR6_ACTIVE_LOW 0xffff0ff0 #define DR6_VOLATILE 0x0001e80f -#define DR6_FIXED_1 (DR6_ACTIVE_LOW & ~DR6_VOLATILE) #define DR7_BP_EN_MASK 0x000000ff #define DR7_GE (1 << 9) @@ -62,6 +60,7 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8); int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val); unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr); +unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu); unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu); void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw); int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); diff --git a/arch/x86/kvm/reverse_cpuid.h b/arch/x86/kvm/reverse_cpuid.h index 2ad25781cefb..89345af0f7ad 100644 --- a/arch/x86/kvm/reverse_cpuid.h +++ b/arch/x86/kvm/reverse_cpuid.h @@ -66,7 +66,7 @@ #define X86_FEATURE_AVX10_512 KVM_X86_FEATURE(CPUID_24_0_EBX, 18) /* Intel-defined sub-features, CPUID level 0x00000024:1 (ECX) */ -#define X86_FEATURE_AVX10_VNNI_INT KVM_X86_FEATURE(CPUID_24_1_ECX, 2) +#define X86_FEATURE_AVX10_V1_AUX KVM_X86_FEATURE(CPUID_24_1_ECX, 2) /* CPUID level 0x80000007 (EDX). */ #define KVM_X86_FEATURE_CONSTANT_TSC KVM_X86_FEATURE(CPUID_8000_0007_EDX, 8) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index f9090b601efa..8845f469adf3 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -527,12 +527,16 @@ void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, /* Always clear misc_ctl bits that the guest cannot use */ to->misc_ctl = from->misc_ctl; + to->misc_ctl2 = from->misc_ctl2; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_NPT)) to->misc_ctl &= ~SVM_MISC_ENABLE_NP; if (!gmet_enabled || !guest_cpu_cap_has(vcpu, X86_FEATURE_GMET)) to->misc_ctl &= ~SVM_MISC_ENABLE_GMET; + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV)) + to->misc_ctl2 &= ~SVM_MISC2_ENABLE_V_LBR; + to->iopm_base_pa = from->iopm_base_pa & PAGE_MASK; to->msrpm_base_pa = from->msrpm_base_pa & PAGE_MASK; to->tsc_offset = from->tsc_offset; @@ -550,7 +554,6 @@ void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, to->event_inj_err = from->event_inj_err; to->next_rip = from->next_rip; to->nested_cr3 = from->nested_cr3; - to->misc_ctl2 = from->misc_ctl2; to->pause_filter_count = from->pause_filter_count; to->pause_filter_thresh = from->pause_filter_thresh; @@ -735,12 +738,6 @@ static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, return 0; } -static bool nested_vmcb12_has_lbrv(struct kvm_vcpu *vcpu) -{ - return guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && - (to_svm(vcpu)->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR); -} - static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) { struct vmcb_ctrl_area_cached *control = &svm->nested.ctl; @@ -789,6 +786,10 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) kvm_set_rflags(vcpu, save->rflags | X86_EFLAGS_FIXED); + /* SVM ignores EFER.LMA if EFER.LME=0 (instead of failing VMRUN). */ + if (!(svm->nested.save.efer & EFER_LME)) + svm->nested.save.efer &= ~EFER_LMA; + svm_set_efer(vcpu, svm->nested.save.efer); svm_set_cr0(vcpu, svm->nested.save.cr0); @@ -807,17 +808,17 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_DR))) { vmcb02->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1; - svm->vcpu.arch.dr6 = svm->nested.save.dr6 | DR6_ACTIVE_LOW; + svm->vcpu.arch.dr6 = svm->nested.save.dr6 | kvm_get_dr6_fixed_1(vcpu); vmcb_mark_dirty(vmcb02, VMCB_DR); } - if (nested_vmcb12_has_lbrv(vcpu)) { + if (control->misc_ctl2 & SVM_MISC2_ENABLE_V_LBR) { /* * Reserved bits of DEBUGCTL are ignored. Be consistent with * svm_set_msr's definition of reserved bits. */ svm_copy_lbrs(&vmcb02->save, save); - vmcb02->save.dbgctl &= ~DEBUGCTL_RESERVED_BITS; + vmcb02->save.dbgctl &= svm_get_supported_debugctl(vcpu); } else { svm_copy_lbrs(&vmcb02->save, &vmcb01->save); } @@ -901,6 +902,13 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa; vmcb_mark_dirty(vmcb02, VMCB_PERM_MAP); + /* + * PML is never enabled in hardware for L2. Make sure that an + * unexpected PML write would trigger a PML_FULL VM-Exit. + */ + if (pml) + vmcb02->control.pml_index = -1; + /* * Stash vmcb02's counter if the guest hasn't moved past the guilty * instruction; otherwise, reset the counter to '0'. @@ -1199,7 +1207,7 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu) } /* Copy state save area fields which are handled by VMRUN */ -void svm_copy_vmrun_state(struct vmcb_save_area *to_save, +void svm_copy_vmrun_state(struct kvm_vcpu *vcpu, struct vmcb_save_area *to_save, struct vmcb_save_area *from_save) { to_save->es = from_save->es; @@ -1226,7 +1234,7 @@ void svm_copy_vmrun_state(struct vmcb_save_area *to_save, if (kvm_cpu_cap_has(X86_FEATURE_LBRV)) { svm_copy_lbrs(to_save, from_save); - to_save->dbgctl &= ~DEBUGCTL_RESERVED_BITS; + to_save->dbgctl &= svm_get_supported_debugctl(vcpu); } } @@ -1297,7 +1305,7 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu) if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) vmcb12->control.next_rip = vmcb02->control.next_rip; - if (nested_vmcb12_has_lbrv(vcpu)) + if (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR) svm_copy_lbrs(&vmcb12->save, &vmcb02->save); vmcb12->control.event_inj = 0; @@ -1374,7 +1382,7 @@ void nested_svm_vmexit(struct vcpu_svm *svm) if (!nested_exit_on_intr(svm)) kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); - if (!nested_vmcb12_has_lbrv(vcpu)) { + if (!(svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR)) { svm_copy_lbrs(&vmcb01->save, &vmcb02->save); vmcb_mark_dirty(vmcb01, VMCB_LBR); } @@ -1820,6 +1828,13 @@ int nested_svm_exit_special(struct vcpu_svm *svm) if (nested_svm_is_l2_tlb_flush_hcall(vcpu)) return NESTED_EXIT_HOST; break; + case SVM_EXIT_PML_FULL: + /* + * All PML full exits are handled by KVM. KVM emulates PML in + * software for L1, but never enables PML in hardware on behalf + * of L1. + */ + return NESTED_EXIT_HOST; default: break; } @@ -2028,6 +2043,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, if (!(save->cr0 & X86_CR0_PG) || !(save->cr0 & X86_CR0_PE) || (save->rflags & X86_EFLAGS_VM) || + ((save->efer & EFER_LMA) && !(save->efer & EFER_LME)) || !nested_vmcb_check_save(vcpu, &save_cached, false)) goto out_free; @@ -2064,7 +2080,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa; - svm_copy_vmrun_state(&svm->vmcb01.ptr->save, save); + svm_copy_vmrun_state(vcpu, &svm->vmcb01.ptr->save, save); nested_copy_vmcb_control_to_cache(svm, ctl); svm_switch_vmcb(svm, &svm->nested.vmcb02); diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 5c421eaa5c07..93fbfb7d1f65 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -573,6 +573,7 @@ static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp, INIT_LIST_HEAD(&sev->regions_list); INIT_LIST_HEAD(&sev->mirror_vms); sev->need_init = false; + kvm->arch.has_protected_page_tables = true; kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_SEV); @@ -2058,12 +2059,14 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) dst->pages_locked = src->pages_locked; dst->es_active = src->es_active; dst->vmsa_features = src->vmsa_features; + dst_kvm->arch.has_protected_page_tables = true; src->asid = 0; src->active = false; src->handle = 0; src->pages_locked = 0; src->es_active = false; + src_kvm->arch.has_protected_page_tables = false; list_cut_before(&dst->regions_list, &src->regions_list, &src->regions_list); @@ -2920,6 +2923,7 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) mutex_unlock(&sev_mirror_lock); mirror_sev->active = true; + kvm->arch.has_protected_page_tables = true; mirror_sev->asid = source_sev->asid; mirror_sev->fd = source_sev->fd; mirror_sev->es_active = source_sev->es_active; @@ -2978,6 +2982,7 @@ void sev_vm_init(struct kvm *kvm) kvm->arch.has_protected_state = true; fallthrough; case KVM_X86_SEV_VM: + kvm->arch.has_protected_page_tables = true; kvm->arch.pre_fault_allowed = !kvm->arch.has_private_mem; to_kvm_sev_info(kvm)->need_init = true; break; @@ -5007,7 +5012,7 @@ struct page *snp_safe_alloc_page_node(int node, gfp_t gfp) * Allocate an SNP-safe page to workaround the SNP erratum where * the CPU will incorrectly signal an RMP violation #PF if a * hugepage (2MB or 1GB) collides with the RMP entry of a - * 2MB-aligned VMCB, VMSA, or AVIC backing page. + * 2MB-aligned VMCB, VMSA, PML or AVIC backing page. * * Allocate one extra page, choose a page which is not * 2MB-aligned, and free the other. diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 103197238d24..3b96a4f63637 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -179,6 +179,9 @@ module_param(vnmi, bool, 0444); module_param(enable_mediated_pmu, bool, 0444); +bool __ro_after_init pml = true; +module_param(pml, bool, 0444); + static bool __ro_after_init svm_gp_erratum_intercept = true; static u8 rsm_ins_bytes[] = "\x0f\xaa"; @@ -872,13 +875,22 @@ static void __svm_disable_lbrv(struct kvm_vcpu *vcpu) to_svm(vcpu)->vmcb->control.misc_ctl2 &= ~SVM_MISC2_ENABLE_V_LBR; } +static bool svm_need_lbr_virtualization(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + if (svm->vmcb->save.dbgctl & (DEBUGCTLMSR_LBR | DEBUGCTLMSR_BUS_LOCK_DETECT)) + return true; + + return is_guest_mode(vcpu) && + (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR); +} + void svm_update_lbrv(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); bool current_enable_lbrv = svm->vmcb->control.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR; - bool enable_lbrv = (svm->vmcb->save.dbgctl & DEBUGCTLMSR_LBR) || - (is_guest_mode(vcpu) && guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && - (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR)); + bool enable_lbrv = svm_need_lbr_virtualization(vcpu); if (enable_lbrv && !current_enable_lbrv) __svm_enable_lbrv(vcpu); @@ -1258,6 +1270,23 @@ static void init_vmcb(struct kvm_vcpu *vcpu, bool init_event) if (vcpu->kvm->arch.bus_lock_detection_enabled) svm_set_intercept(svm, INTERCEPT_BUSLOCK); + if (pml) { + /* + * Populate the page address and index here, PML is enabled + * when dirty logging is enabled on the memslot through + * svm_update_cpu_dirty_logging() + */ + control->pml_addr = (u64)__sme_set(page_to_phys(vcpu->arch.pml_page)); + + /* + * The VMCB control area remains valid after an intercepted SHUTDOWN + * (AMD APM Rev 3.45+), so only initialize PML index on reset to avoid + * discarding already-logged entries that haven't been flushed. + */ + if (!init_event) + control->pml_index = PML_HEAD_INDEX; + } + if (is_sev_guest(vcpu)) sev_init_vmcb(svm, init_event); @@ -1317,9 +1346,15 @@ static int svm_vcpu_create(struct kvm_vcpu *vcpu) if (!vmcb01_page) goto out; + if (pml) { + vcpu->arch.pml_page = snp_safe_alloc_page(); + if (!vcpu->arch.pml_page) + goto error_free_vmcb_page; + } + err = sev_vcpu_create(vcpu); if (err) - goto error_free_vmcb_page; + goto error_free_pml_page; err = avic_init_vcpu(svm); if (err) @@ -1344,6 +1379,9 @@ static int svm_vcpu_create(struct kvm_vcpu *vcpu) error_free_sev: sev_free_vcpu(vcpu); +error_free_pml_page: + if (vcpu->arch.pml_page) + __free_page(vcpu->arch.pml_page); error_free_vmcb_page: __free_page(vmcb01_page); out: @@ -1361,6 +1399,9 @@ static void svm_vcpu_free(struct kvm_vcpu *vcpu) sev_free_vcpu(vcpu); + if (pml && vcpu->arch.pml_page) + __free_page(vcpu->arch.pml_page); + __free_page(__sme_pa_to_page(svm->vmcb01.pa)); svm_vcpu_free_msrpm(svm->msrpm); } @@ -2749,10 +2790,19 @@ static int efer_trap(struct kvm_vcpu *vcpu) * bit in svm_set_efer(), but __kvm_valid_efer() checks it against * whether the guest has X86_FEATURE_SVM - this avoids a failure if * the guest doesn't have X86_FEATURE_SVM. + * + * Clear EFER_LMSLE for a related reason: EFER writes are *trapped*, + * not intercepted, i.e. hardware has already committed the write by + * the time KVM gains control, and the trap is enabled if and only if + * the guest is SEV-ES, whose EFER lives in the encrypted VMSA and so + * can't be fixed up by KVM. Rejecting EFER.LMSLE=1 would inject a #GP + * *and* leave EFER.LMSLE set in the guest, which is strictly worse + * than honoring a write that hardware itself allowed. */ msr_info.host_initiated = false; msr_info.index = MSR_EFER; - msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME; + msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & + ~(EFER_SVME | EFER_LMSLE); ret = kvm_set_msr_common(vcpu, &msr_info); return kvm_complete_insn_gp(vcpu, ret); @@ -3165,7 +3215,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) data &= ~DEBUGCTLMSR_BTF; } - if (data & DEBUGCTL_RESERVED_BITS) + if (data & ~svm_get_supported_debugctl(vcpu)) return 1; if (svm->vmcb->save.dbgctl == data) @@ -3334,6 +3384,45 @@ static int vmmcall_interception(struct kvm_vcpu *vcpu) return kvm_emulate_hypercall(vcpu); } +static void svm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb *vmcb01 = svm->vmcb01.ptr; + + if (enable) + vmcb01->control.misc_ctl |= SVM_MISC_ENABLE_PML; + else + vmcb01->control.misc_ctl &= ~SVM_MISC_ENABLE_PML; + + vmcb_mark_dirty(vmcb01, VMCB_NPT); +} + +static void svm_flush_pml_buffer(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb_control_area *control = &svm->vmcb->control; + + /* Do nothing if PML buffer is empty */ + if (control->pml_index == PML_HEAD_INDEX) + return; + + kvm_flush_pml_buffer(vcpu, control->pml_index); + + /* Reset the PML index */ + control->pml_index = PML_HEAD_INDEX; +} + +static int pml_full_interception(struct kvm_vcpu *vcpu) +{ + trace_kvm_pml_full(vcpu->vcpu_id); + + /* + * PML buffer is already flushed at the beginning of svm_handle_exit(). + * Nothing to do here. + */ + return 1; +} + static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { [SVM_EXIT_READ_CR0] = cr_interception, [SVM_EXIT_READ_CR3] = cr_interception, @@ -3411,6 +3500,7 @@ static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { #ifdef CONFIG_KVM_AMD_SEV [SVM_EXIT_VMGEXIT] = sev_handle_vmgexit, #endif + [SVM_EXIT_PML_FULL] = pml_full_interception, }; static void dump_vmcb(struct kvm_vcpu *vcpu) @@ -3460,8 +3550,14 @@ static void dump_vmcb(struct kvm_vcpu *vcpu) pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2); pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info); pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err); - pr_err("%-20s%lld\n", "misc_ctl:", control->misc_ctl); + pr_err("%-20s%llx\n", "misc_ctl:", control->misc_ctl); pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3); + + if (pml) { + pr_err("%-20s%016llx\n", "pml_addr:", control->pml_addr); + pr_err("%-20s%04x\n", "pml_index:", control->pml_index); + } + pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar); pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa); pr_err("%-20s%08x\n", "event_inj:", control->event_inj); @@ -3639,6 +3735,13 @@ int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 __exit_code) (u64)exit_code != __exit_code) goto unexpected_vmexit; + /* + * PML is never enabled when running L2, bail immediately if a PML full + * exit occurs as something is horribly wrong. + */ + if (unlikely(is_guest_mode(vcpu) && exit_code == SVM_EXIT_PML_FULL)) + goto unexpected_vmexit; + #ifdef CONFIG_MITIGATION_RETPOLINE if (exit_code == SVM_EXIT_MSR) return msr_interception(vcpu); @@ -3707,6 +3810,14 @@ static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) struct vcpu_svm *svm = to_svm(vcpu); struct kvm_run *kvm_run = vcpu->run; + /* + * Opportunistically flush the PML buffer on VM exit. This keeps the + * dirty bitmap current by processing logged GPAs rather than waiting for + * PML_FULL exit. + */ + if (vcpu->kvm->arch.cpu_dirty_log_size && !is_guest_mode(vcpu)) + svm_flush_pml_buffer(vcpu); + if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE)) return 0; @@ -4235,7 +4346,7 @@ static void svm_flush_tlb_guest(struct kvm_vcpu *vcpu) svm_flush_tlb_asid(vcpu); } -static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva, bool *full) +static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) { struct vcpu_svm *svm = to_svm(vcpu); @@ -4251,8 +4362,6 @@ static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva, bool *full) } svm_flush_tlb_guest(vcpu); - if (full) - *full = true; } static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu) @@ -5049,7 +5158,7 @@ static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400); - svm_copy_vmrun_state(m_save.map.hva + 0x400, &svm->vmcb01.ptr->save); + svm_copy_vmrun_state(vcpu, m_save.map.hva + 0x400, &svm->vmcb01.ptr->save); return 0; } @@ -5089,7 +5198,7 @@ static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) * used during SMM (see svm_enter_smm()) */ - svm_copy_vmrun_state(&svm->vmcb01.ptr->save, m_save.map.hva + 0x400); + svm_copy_vmrun_state(vcpu, &svm->vmcb01.ptr->save, m_save.map.hva + 0x400); /* * Enter the nested guest now @@ -5308,6 +5417,9 @@ static int svm_vm_init(struct kvm *kvm) if (!pause_filter_count || !pause_filter_thresh) kvm_disable_exits(kvm, KVM_X86_DISABLE_EXITS_PAUSE); + if (pml) + kvm->arch.cpu_dirty_log_size = PML_LOG_NR_ENTRIES; + svm_srso_vm_init(); return 0; } @@ -5426,6 +5538,8 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .check_intercept = svm_check_intercept, .handle_exit_irqoff = svm_handle_exit_irqoff, + .update_cpu_dirty_logging = svm_update_cpu_dirty_logging, + .deliver_interrupt = svm_deliver_interrupt, .pi_update_irte = avic_pi_update_irte, .setup_mce = svm_setup_mce, @@ -5568,6 +5682,16 @@ static __init void svm_set_cpu_caps(void) boot_cpu_has(X86_FEATURE_AMD_SSBD)) kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); + /* + * Tell userspace that EFER.LMSLE must be zero if nested SVM is + * disabled, as KVM allows EFER.LMSLE if and only if nested SVM is + * supported (a historical artifact of commit eec4b140c924 ("KVM: SVM: + * Allow EFER.LMSLE to be set with nested svm"), not an architectural + * requirement). + */ + if (!nested) + kvm_cpu_cap_set(X86_FEATURE_EFER_LMSLE_MBZ); + if (enable_pmu) { /* * Enumerate support for PERFCTR_CORE if and only if KVM has @@ -5592,9 +5716,17 @@ static __init void svm_set_cpu_caps(void) * Clear capabilities that are automatically configured by common code, * but that require explicit SVM support (that isn't yet implemented). */ - kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT); kvm_cpu_cap_clear(X86_FEATURE_MSR_IMM); + /* + * LBR Virtualization must be enabled to support BusLockTrap inside the + * guest, since BusLockTrap is enabled through MSR_IA32_DEBUGCTLMSR and + * MSR_IA32_DEBUGCTLMSR is virtualized only if LBR Virtualization is + * enabled. + */ + if (!lbrv) + kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT); + kvm_setup_xss_caps(); kvm_finalize_cpu_caps(); } @@ -5687,6 +5819,10 @@ static __init int svm_hardware_setup(void) nrips = nrips && boot_cpu_has(X86_FEATURE_NRIPS); + pml = pml && npt_enabled && cpu_feature_enabled(X86_FEATURE_PML); + if (pml) + pr_info("Page modification logging supported\n"); + if (lbrv) { if (!boot_cpu_has(X86_FEATURE_LBRV)) lbrv = false; diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 84f19026d3e8..4b0b2ee13a3a 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -53,6 +53,7 @@ extern int vgif; extern bool intercept_smi; extern bool vnmi; extern int lbrv; +extern bool pml; extern int tsc_aux_uret_slot __ro_after_init; @@ -786,7 +787,15 @@ BUILD_SVM_MSR_BITMAP_HELPERS(bool, test, test) BUILD_SVM_MSR_BITMAP_HELPERS(void, clear, __clear) BUILD_SVM_MSR_BITMAP_HELPERS(void, set, __set) -#define DEBUGCTL_RESERVED_BITS (~DEBUGCTLMSR_LBR) +static inline u64 svm_get_supported_debugctl(struct kvm_vcpu *vcpu) +{ + u64 debugctl = DEBUGCTLMSR_LBR; + + if (guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) + debugctl |= DEBUGCTLMSR_BUS_LOCK_DETECT; + + return debugctl; +} /* svm.c */ extern bool dump_invalid_vmcb; @@ -876,7 +885,7 @@ void svm_leave_nested(struct kvm_vcpu *vcpu); void svm_free_nested(struct vcpu_svm *svm); int svm_allocate_nested(struct vcpu_svm *svm); int nested_svm_vmrun(struct kvm_vcpu *vcpu); -void svm_copy_vmrun_state(struct vmcb_save_area *to_save, +void svm_copy_vmrun_state(struct kvm_vcpu *vcpu, struct vmcb_save_area *to_save, struct vmcb_save_area *from_save); void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb); void nested_svm_vmexit(struct vcpu_svm *svm); diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 4c52ab8d0786..797206912ae9 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -108,7 +108,7 @@ static void vt_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vmx_vcpu_load(vcpu, cpu); } -static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) +static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable) { /* * Basic TDX does not support feature PML. KVM does not enable PML in @@ -117,7 +117,7 @@ static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) if (WARN_ON_ONCE(is_td_vcpu(vcpu))) return; - vmx_update_cpu_dirty_logging(vcpu); + vmx_update_cpu_dirty_logging(vcpu, enable); } static void vt_prepare_switch_to_guest(struct kvm_vcpu *vcpu) @@ -543,12 +543,12 @@ static void vt_flush_tlb_current(struct kvm_vcpu *vcpu) vmx_flush_tlb_current(vcpu); } -static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full) +static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) { if (is_td_vcpu(vcpu)) return; - vmx_flush_tlb_gva(vcpu, addr, full); + vmx_flush_tlb_gva(vcpu, addr); } static void vt_flush_tlb_guest(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index e7950b98b4bb..3fbe08907afe 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -56,9 +56,6 @@ bool enable_tdx __ro_after_init; module_param_named(tdx, enable_tdx, bool, 0444); -#define TDX_SHARED_BIT_PWL_5 gpa_to_gfn(BIT_ULL(51)) -#define TDX_SHARED_BIT_PWL_4 gpa_to_gfn(BIT_ULL(47)) - static const struct tdx_sys_info *tdx_sysinfo; void tdh_vp_rd_failed(struct vcpu_tdx *tdx, char *uclass, u32 field, u64 err) @@ -619,6 +616,7 @@ int tdx_vm_init(struct kvm *kvm) { struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); + kvm->arch.has_protected_page_tables = true; kvm->arch.has_protected_state = true; /* * TDX Module doesn't allow the hypervisor to modify the EOI-bitmap, @@ -1609,10 +1607,7 @@ static int handle_tdvmcall(struct kvm_vcpu *vcpu) void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int pgd_level) { - u64 shared_bit = (pgd_level == 5) ? TDX_SHARED_BIT_PWL_5 : - TDX_SHARED_BIT_PWL_4; - - if (KVM_BUG_ON(shared_bit != kvm_gfn_direct_bits(vcpu->kvm), vcpu->kvm)) + if (KVM_BUG_ON(pgd_level != vcpu->kvm->arch.mirror_root_level, vcpu->kvm)) return; td_vmcs_write64(to_tdx(vcpu), SHARED_EPT_POINTER, root_hpa); @@ -2755,6 +2750,14 @@ DEFINE_CLASS(tdx_vm_state_guard, tdx_vm_state_guard_t, if (!IS_ERR(_T)) tdx_release_vm_state_locks(_T), tdx_acquire_vm_state_locks(kvm), struct kvm *kvm); +static __always_inline void tdx_set_mirror_root_level(struct kvm *kvm, int level) +{ + BUILD_BUG_ON(level != 4 && level != 5); + + kvm->arch.mirror_root_level = level; + kvm->arch.gfn_direct_bits = gpa_to_gfn(BIT_ULL(level == 4 ? 47 : 51)); +} + static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) { struct kvm_tdx_init_vm __user *user_data = u64_to_user_ptr(cmd->data); @@ -2818,9 +2821,9 @@ static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) kvm_tdx->xfam = td_params->xfam; if (td_params->config_flags & TDX_CONFIG_FLAGS_MAX_GPAW) - kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_5; + tdx_set_mirror_root_level(kvm, 5); else - kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_4; + tdx_set_mirror_root_level(kvm, 4); kvm_tdx->state = TD_STATE_INITIALIZED; out: diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 54d93167e342..686e586f3290 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -3389,7 +3389,7 @@ void vmx_flush_tlb_current(struct kvm_vcpu *vcpu) vpid_sync_context(vmx_get_current_vpid(vcpu)); } -void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full) +void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) { /* * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in @@ -4893,10 +4893,11 @@ int vmx_vcpu_precreate(struct kvm *kvm) #define VMX_XSS_EXIT_BITMAP 0 -static void init_vmcs(struct vcpu_vmx *vmx) +static void init_vmcs(struct kvm_vcpu *vcpu) { - struct kvm *kvm = vmx->vcpu.kvm; + struct kvm *kvm = vcpu->kvm; struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); + struct vcpu_vmx *vmx = to_vmx(vcpu); if (nested) nested_vmx_set_vmcs_shadowing_bitmap(); @@ -4921,7 +4922,7 @@ static void init_vmcs(struct vcpu_vmx *vmx) if (cpu_has_tertiary_exec_ctrls()) tertiary_exec_controls_set(vmx, vmx_tertiary_exec_control(vmx)); - if (enable_apicv && lapic_in_kernel(&vmx->vcpu)) { + if (enable_apicv && lapic_in_kernel(vcpu)) { vmcs_write64(EOI_EXIT_BITMAP0, 0); vmcs_write64(EOI_EXIT_BITMAP1, 0); vmcs_write64(EOI_EXIT_BITMAP2, 0); @@ -4933,7 +4934,7 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->vt.pi_desc))); } - if (vmx_can_use_ipiv(&vmx->vcpu)) { + if (vmx_can_use_ipiv(vcpu)) { vmcs_write64(PID_POINTER_TABLE, __pa(kvm_vmx->pid_table)); vmcs_write16(LAST_PID_POINTER_INDEX, kvm->arch.max_vcpu_ids - 1); } @@ -4968,15 +4969,15 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) - vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); + vmcs_write64(GUEST_IA32_PAT, vcpu->arch.pat); vm_exit_controls_set(vmx, vmx_get_initial_vmexit_ctrl()); /* 22.2.1, 20.8.1 */ vm_entry_controls_set(vmx, vmx_get_initial_vmentry_ctrl()); - vmx->vcpu.arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); - vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits); + vcpu->arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); + vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); set_cr4_guest_host_mask(vmx); @@ -4987,11 +4988,11 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP); if (enable_pml) { - vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); + vmcs_write64(PML_ADDRESS, page_to_phys(vcpu->arch.pml_page)); vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); } - vmx_write_encls_bitmap(&vmx->vcpu, NULL); + vmx_write_encls_bitmap(vcpu, NULL); if (vmx_pt_mode_is_host_guest()) { memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc)); @@ -5004,13 +5005,13 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_writel(GUEST_SYSENTER_ESP, 0); vmcs_writel(GUEST_SYSENTER_EIP, 0); - vmx_guest_debugctl_write(&vmx->vcpu, 0); + vmx_guest_debugctl_write(vcpu, 0); if (cpu_has_vmx_tpr_shadow()) { vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); - if (cpu_need_tpr_shadow(&vmx->vcpu)) + if (cpu_need_tpr_shadow(vcpu)) vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, - __pa(vmx->vcpu.arch.apic->regs)); + __pa(vcpu->arch.apic->regs)); vmcs_write32(TPR_THRESHOLD, 0); } @@ -5021,7 +5022,7 @@ static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - init_vmcs(vmx); + init_vmcs(vcpu); if (nested && kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) @@ -5898,10 +5899,9 @@ static int handle_apic_access(struct kvm_vcpu *vcpu) static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) { unsigned long exit_qualification = vmx_get_exit_qual(vcpu); - int vector = exit_qualification & 0xff; /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ - kvm_apic_set_eoi_accelerated(vcpu, vector); + kvm_apic_set_eoi_accelerated(vcpu, exit_qualification & 0xff); return 1; } @@ -6428,48 +6428,25 @@ void vmx_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code) *error_code = 0; } -static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx) +static void vmx_destroy_pml_buffer(struct kvm_vcpu *vcpu) { - if (vmx->pml_pg) { - __free_page(vmx->pml_pg); - vmx->pml_pg = NULL; + if (vcpu->arch.pml_page) { + __free_page(vcpu->arch.pml_page); + vcpu->arch.pml_page = NULL; } } static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) { - struct vcpu_vmx *vmx = to_vmx(vcpu); - u16 pml_idx, pml_tail_index; - u64 *pml_buf; - int i; + u16 pml_idx; pml_idx = vmcs_read16(GUEST_PML_INDEX); /* Do nothing if PML buffer is empty */ if (pml_idx == PML_HEAD_INDEX) return; - /* - * PML index always points to the next available PML buffer entity - * unless PML log has just overflowed. - */ - pml_tail_index = (pml_idx >= PML_LOG_NR_ENTRIES) ? 0 : pml_idx + 1; - /* - * PML log is written backwards: the CPU first writes the entry 511 - * then the entry 510, and so on. - * - * Read the entries in the same order they were written, to ensure that - * the dirty ring is filled in the same order the CPU wrote them. - */ - pml_buf = page_address(vmx->pml_pg); - - for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) { - u64 gpa; - - gpa = pml_buf[i]; - WARN_ON(gpa & (PAGE_SIZE - 1)); - kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); - } + kvm_flush_pml_buffer(vcpu, pml_idx); /* reset PML index */ vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); @@ -7658,7 +7635,7 @@ void vmx_vcpu_free(struct kvm_vcpu *vcpu) struct vcpu_vmx *vmx = to_vmx(vcpu); if (enable_pml) - vmx_destroy_pml_buffer(vmx); + vmx_destroy_pml_buffer(vcpu); free_vpid(vmx->vpid); nested_vmx_free_vcpu(vcpu); free_loaded_vmcs(vmx->loaded_vmcs); @@ -7690,8 +7667,8 @@ int vmx_vcpu_create(struct kvm_vcpu *vcpu) * for the guest), etc. */ if (enable_pml) { - vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); - if (!vmx->pml_pg) + vcpu->arch.pml_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!vcpu->arch.pml_page) goto free_vpid; } @@ -7762,7 +7739,7 @@ int vmx_vcpu_create(struct kvm_vcpu *vcpu) free_vmcs: free_loaded_vmcs(vmx->loaded_vmcs); free_pml: - vmx_destroy_pml_buffer(vmx); + vmx_destroy_pml_buffer(vcpu); free_vpid: free_vpid(vmx->vpid); return err; @@ -8147,6 +8124,13 @@ static __init void vmx_set_cpu_caps(void) kvm_cpu_cap_clear(X86_FEATURE_IBT); } + /* + * CPUID 0x80000008. Tell userspace that EFER.LMSLE must be zero; KVM + * never allows EFER.LMSLE to be set on Intel CPUs, as KVM supports long + * mode segment limits only in conjunction with nested SVM. + */ + kvm_cpu_cap_set(X86_FEATURE_EFER_LMSLE_MBZ); + kvm_setup_xss_caps(); kvm_finalize_cpu_caps(); } @@ -8450,21 +8434,13 @@ static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit } #endif -void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (WARN_ON_ONCE(!enable_pml)) - return; - guard(vmx_vmcs01)(vcpu); - /* - * Note, nr_memslots_dirty_logging can be changed concurrent with this - * code, but in that case another update request will be made and so - * the guest will never run with a stale PML value. - */ - if (atomic_read(&vcpu->kvm->nr_memslots_dirty_logging)) + if (enable) secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_ENABLE_PML); else secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML); diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index dc8517f15bc4..5f505ad14dde 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -265,13 +265,6 @@ struct vcpu_vmx { unsigned int ple_window; bool ple_window_dirty; - /* Support for PML */ -#define PML_LOG_NR_ENTRIES 512 - /* PML is written backwards: this is the first entry written by the CPU */ -#define PML_HEAD_INDEX (PML_LOG_NR_ENTRIES-1) - - struct page *pml_pg; - /* apic deadline value in host tsc */ u64 hv_deadline_tsc; @@ -397,7 +390,7 @@ u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu); gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); -void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu); +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable); u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated); bool vmx_is_valid_debugctl(struct kvm_vcpu *vcpu, u64 data, bool host_initiated); diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h index 054fd14bb2e1..3351b91e924e 100644 --- a/arch/x86/kvm/vmx/x86_ops.h +++ b/arch/x86/kvm/vmx/x86_ops.h @@ -82,7 +82,7 @@ void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); bool vmx_get_if_flag(struct kvm_vcpu *vcpu); void vmx_flush_tlb_all(struct kvm_vcpu *vcpu); void vmx_flush_tlb_current(struct kvm_vcpu *vcpu); -void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full); +void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr); void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu); void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask); u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu); @@ -118,7 +118,7 @@ u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu); u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu); void vmx_write_tsc_offset(struct kvm_vcpu *vcpu); void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu); -void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu); +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable); #ifdef CONFIG_X86_64 int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, bool *expired); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index dc3ed1a05fbd..269d9886248e 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -401,30 +401,37 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, switch (ex->vector) { case DB_VECTOR: /* - * "Certain debug exceptions may clear bit 0-3. The - * remaining contents of the DR6 register are never - * cleared by the processor". + * DR6 is a mess. Reserved/unused bits are fixed-to-1, and so + * to maintain backwards compatibility with existing software, + * features that use previously-reserved bits have active-low + * semantics, i.e. clear the bit when the feature is present in + * the payload. + * + * Further complicating matters, some DR6 bits are preserved by + * hardware, while others are explicitly modified on every #DB. + * The trap bits are always set based on the payload, as is the + * RTM flag (but it's active low). All other bits are modified + * if and only if a relevant debug exception occurs, e.g. BD, + * BS, and BT are never cleared by hardware, and BLD is never + * set by hardware (when supported, excepting RESET). + * + * Lastly, the payload does NOT have active-low semantics, e.g. + * so that it's compatible VMX's pending debug exceptions and + * qualification fields, and to avoid bleeding the DR6 madness + * into other KVM code. + * + * To compute DR6: + * + * 1. "Reset" the bits that are modified on all #DBs + * 2. Clear active-low bits that are present in the payload. + * 3. Set active-high bits that are present in the payload. + * 4. Clear fixed-0 bits. + * 5. Set fixed-1 bits. */ vcpu->arch.dr6 &= ~DR_TRAP_BITS; - /* - * In order to reflect the #DB exception payload in guest - * dr6, three components need to be considered: active low - * bit, FIXED_1 bits and active high bits (e.g. DR6_BD, - * DR6_BS and DR6_BT) - * DR6_ACTIVE_LOW contains the FIXED_1 and active low bits. - * In the target guest dr6: - * FIXED_1 bits should always be set. - * Active low bits should be cleared if 1-setting in payload. - * Active high bits should be set if 1-setting in payload. - * - * Note, the payload is compatible with the pending debug - * exceptions/exit qualification under VMX, that active_low bits - * are active high in payload. - * So they need to be flipped for DR6. - */ - vcpu->arch.dr6 |= DR6_ACTIVE_LOW; - vcpu->arch.dr6 |= ex->payload; - vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW; + vcpu->arch.dr6 |= DR6_RTM; + vcpu->arch.dr6 &= ~(ex->payload & DR6_ACTIVE_LOW); + vcpu->arch.dr6 |= (ex->payload & ~DR6_ACTIVE_LOW); /* * The #DB payload is defined as compatible with the 'pending @@ -433,6 +440,7 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, * breakpoint), it is reserved and must be zero in DR6. */ vcpu->arch.dr6 &= ~BIT(12); + vcpu->arch.dr6 |= kvm_get_dr6_fixed_1(vcpu); break; case PF_VECTOR: vcpu->arch.cr2 = ex->payload; @@ -450,6 +458,8 @@ static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vecto { struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; + kvm_make_request(KVM_REQ_EVENT, vcpu); + ex->vector = vector; ex->injected = false; ex->pending = true; @@ -3895,6 +3905,37 @@ void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) kvm_vcpu_kick(vcpu); } +void kvm_flush_pml_buffer(struct kvm_vcpu *vcpu, u16 pml_idx) +{ + u16 pml_tail_index; + u64 *pml_buf; + int i; + + /* + * PML index always points to the next available PML buffer entity + * unless PML log has just overflowed. + */ + pml_tail_index = (pml_idx >= PML_LOG_NR_ENTRIES) ? 0 : pml_idx + 1; + + /* + * PML log is written backwards: the CPU first writes the entry 511 + * then the entry 510, and so on. + * + * Read the entries in the same order they were written, to ensure that + * the dirty ring is filled in the same order the CPU wrote them. + */ + pml_buf = page_address(vcpu->arch.pml_page); + + for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) { + u64 gpa; + + gpa = pml_buf[i]; + WARN_ON_ONCE(gpa & (PAGE_SIZE - 1)); + kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); + } +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_flush_pml_buffer); + int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) { @@ -5043,7 +5084,8 @@ static int emulator_read_write_onepage(unsigned long addr, void *val, * operation using rep will only have the initial GPA from the NPF * occurred. */ - if (ctxt->gpa_available && emulator_can_use_gpa(ctxt) && + if ((ctxt->gpa_access & (write ? ACC_WRITE_MASK : ACC_READ_MASK)) && + emulator_can_use_gpa(ctxt) && (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) { gpa = ctxt->gpa_val; ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write); @@ -5892,7 +5934,7 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu) kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); - ctxt->gpa_available = false; + ctxt->gpa_access = 0; ctxt->eflags = kvm_get_rflags(vcpu); ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0; @@ -6408,7 +6450,17 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, /* With shadow page tables, cr2 contains a GVA or nGPA. */ if (vcpu->arch.mmu->root_role.direct) { - ctxt->gpa_available = true; + ctxt->gpa_access = ACC_READ_MASK; + /* + * Always allow writes for guests with protected page + * tables, as KVM can't walk the guest's page tables, + * i.e. KVM can't get the RMW protections for a given + * GVA to see if the write side of a RMW operation + * should be allowed. + */ + if ((emulation_type & EMULTYPE_PF_WRITE) || + vcpu->kvm->arch.has_protected_page_tables) + ctxt->gpa_access |= ACC_WRITE_MASK; ctxt->gpa_val = cr2_or_gpa; } } else { @@ -6884,7 +6936,14 @@ static void kvm_setup_efer_caps(void) if (kvm_cpu_cap_has(X86_FEATURE_SVM)) { kvm_caps.supported_efer_bits |= EFER_SVME; - if (!boot_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ)) + + /* + * Enumerating EFER_LMSLE_MBZ and allowing EFER.LMSLE=1 + * would be nonsensical. Note, vendor code sets the defeature + * if KVM can't support EFER.LMSLE for any reason, i.e. this + * needs to consult KVM's capabilities, not just raw CPUID. + */ + if (!kvm_cpu_cap_has(X86_FEATURE_EFER_LMSLE_MBZ)) kvm_caps.supported_efer_bits |= EFER_LMSLE; } } @@ -7996,6 +8055,21 @@ static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) kvm_x86_call(set_apic_access_page_addr)(vcpu); } +static void kvm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) +{ + /* + * Note, nr_memslots_dirty_logging can be changed concurrent with this + * code, but in that case another update request will be made and so + * the guest will never run with a stale PML value. + */ + bool enable = atomic_read(&vcpu->kvm->nr_memslots_dirty_logging); + + if (WARN_ON_ONCE(!vcpu->kvm->arch.cpu_dirty_log_size)) + return; + + kvm_x86_call(update_cpu_dirty_logging)(vcpu, enable); +} + /* * Called within kvm->srcu read side. * Returns 1 to let vcpu_run() continue the guest execution loop without @@ -8109,7 +8183,6 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_NMI, vcpu)) process_nmi(vcpu); if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) { - BUG_ON(vcpu->arch.pending_ioapic_eoi > 255); if (test_bit(vcpu->arch.pending_ioapic_eoi, vcpu->arch.ioapic_handled_vectors)) { vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI; @@ -8166,7 +8239,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) kvm_x86_call(recalc_intercepts)(vcpu); if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu)) - kvm_x86_call(update_cpu_dirty_logging)(vcpu); + kvm_update_cpu_dirty_logging(vcpu); if (kvm_check_request(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, vcpu)) { kvm_vcpu_reset(vcpu, true); @@ -8355,6 +8428,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) WARN_ON(vcpu->arch.switch_db_regs & KVM_DEBUGREG_AUTO_SWITCH); kvm_x86_call(sync_dirty_debug_regs)(vcpu); kvm_update_dr0123(vcpu); + vcpu->arch.dr6 |= kvm_get_dr6_fixed_1(vcpu); kvm_update_dr7(vcpu); } diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 1d9a66048b01..cfe96962a4e4 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -408,6 +408,8 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, * EMULTYPE_PF - Set when an intercepted #PF triggers the emulation, in which case * the CR2/GPA value pass on the stack is valid. * + * EMULTYPE_PF_WRITE - Set with EMULTYPE_PF when hardware reports a write access. + * * EMULTYPE_COMPLETE_USER_EXIT - Set when the emulator should update interruptibility * state and inject single-step #DBs after skipping * an instruction (after completing userspace I/O). @@ -446,6 +448,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, #define EMULTYPE_COMPLETE_USER_EXIT (1 << 7) #define EMULTYPE_WRITE_PF_TO_SP (1 << 8) #define EMULTYPE_SKIP_SOFT_INT (1 << 9) +#define EMULTYPE_PF_WRITE (1 << 10) #define EMULTYPE_SET_SOFT_INT_VECTOR(v) ((u32)((v) & 0xff) << 16) #define EMULTYPE_GET_SOFT_INT_VECTOR(e) (((e) >> 16) & 0xff) @@ -901,4 +904,11 @@ int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, int cpl, int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); +/* Support for PML */ +#define PML_LOG_NR_ENTRIES 512 +/* PML is written backwards: this is the first entry written by the CPU */ +#define PML_HEAD_INDEX (PML_LOG_NR_ENTRIES-1) + +void kvm_flush_pml_buffer(struct kvm_vcpu *vcpu, u16 pml_idx); + #endif diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 42b50f7badad..b96f1bb23e29 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -118,7 +118,7 @@ TEST_GEN_PROGS_x86 += x86/state_test TEST_GEN_PROGS_x86 += x86/vmx_preemption_timer_test TEST_GEN_PROGS_x86 += x86/svm_vmcall_test TEST_GEN_PROGS_x86 += x86/svm_int_ctl_test -TEST_GEN_PROGS_x86 += x86/svm_nested_clear_efer_svme +TEST_GEN_PROGS_x86 += x86/svm_nested_efer_test TEST_GEN_PROGS_x86 += x86/svm_nested_shutdown_test TEST_GEN_PROGS_x86 += x86/svm_nested_soft_inject_test TEST_GEN_PROGS_x86 += x86/svm_nested_vmcb12_gpa diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index 6e6f70035508..6f110987c51a 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -216,6 +216,7 @@ struct kvm_x86_cpu_feature { #define X86_FEATURE_INVTSC KVM_X86_CPU_FEATURE(0x80000007, 0, EDX, 8) #define X86_FEATURE_RDPRU KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 4) #define X86_FEATURE_AMD_IBPB KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 12) +#define X86_FEATURE_EFER_LMSLE_MBZ KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 20) #define X86_FEATURE_NPT KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 0) #define X86_FEATURE_LBRV KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 1) #define X86_FEATURE_NRIPS KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 3) @@ -1472,6 +1473,19 @@ static inline bool kvm_is_lbrv_enabled(void) return !!get_kvm_amd_param_integer("lbrv"); } +/* + * Do NOT use this to check for nVMX or nSVM support. Querying kvm_cpu_has() + * for either of X86_FEATURE_{VMX,SVM} is the idiomatic way to check for nested + * virtualization support. Use this *only* to validate KVM's own enumeration. + */ +static inline bool kvm_is_nested_virtualization_enabled(void) +{ + if (host_cpu_is_intel) + return get_kvm_intel_param_bool("nested"); + + return get_kvm_amd_param_integer("nested"); +} + u64 *vm_get_pte(struct kvm_vm *vm, gva_t gva); u64 kvm_hypercall(u64 nr, u64 a0, u64 a1, u64 a2, u64 a3); diff --git a/tools/testing/selftests/kvm/x86/set_sregs_test.c b/tools/testing/selftests/kvm/x86/set_sregs_test.c index 603226ffe437..5afc4187f5eb 100644 --- a/tools/testing/selftests/kvm/x86/set_sregs_test.c +++ b/tools/testing/selftests/kvm/x86/set_sregs_test.c @@ -135,6 +135,17 @@ static void test_cr_bits(struct kvm_vcpu *vcpu, u64 cr4) /* NW without CD is illegal, as is PG without PE. */ TEST_INVALID_SREG_BIT(vcpu, cr0, sregs, X86_CR0_NW); TEST_INVALID_SREG_BIT(vcpu, cr0, sregs, X86_CR0_PG); + + /* CR8 bits 3:0 are writable; bits 63:4 are reserved. */ + vcpu_sregs_get(vcpu, &sregs); + sregs.cr8 = 0xf; + vcpu_sregs_set(vcpu, &sregs); + + vcpu_sregs_get(vcpu, &sregs); + TEST_ASSERT_EQ(sregs.cr8, 0xf); + + for (i = 4; i < 64; i++) + TEST_INVALID_SREG_BIT(vcpu, cr8, sregs, BIT_ULL(i)); } static void test_efer_bits(struct kvm_vcpu *vcpu, u64 efer) diff --git a/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c b/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c deleted file mode 100644 index 6bc301207cbc..000000000000 --- a/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2026, Google LLC. - */ -#include "kvm_util.h" -#include "vmx.h" -#include "svm_util.h" -#include "kselftest.h" - - -static void l2_guest_code(void) -{ - unsigned long efer = rdmsr(MSR_EFER); - - /* generic_svm_setup() initializes EFER_SVME set for L2 */ - GUEST_ASSERT(efer & EFER_SVME); - wrmsr(MSR_EFER, efer & ~EFER_SVME); - - /* Unreachable, L1 should be shutdown */ - GUEST_ASSERT(0); -} - -static void l1_guest_code(struct svm_test_data *svm) -{ - generic_svm_setup(svm, l2_guest_code); - run_guest(svm->vmcb, svm->vmcb_gpa); - - /* Unreachable, L1 should be shutdown */ - GUEST_ASSERT(0); -} - -int main(int argc, char *argv[]) -{ - struct kvm_vcpu *vcpu; - struct kvm_vm *vm; - gva_t nested_gva = 0; - - TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM)); - - vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); - - vcpu_alloc_svm(vm, &nested_gva); - vcpu_args_set(vcpu, 1, nested_gva); - - vcpu_run(vcpu); - TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_SHUTDOWN); - - kvm_vm_free(vm); - return 0; -} diff --git a/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c b/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c new file mode 100644 index 000000000000..fa0d76a9c390 --- /dev/null +++ b/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c @@ -0,0 +1,266 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Tests for KVM's handling of EFER bits whose behavior is tied to nested SVM. + * + * Copyright (C) 2026, Google LLC. + */ +#include "test_util.h" +#include "kvm_util.h" +#include "processor.h" +#include "svm_util.h" +#include "kselftest.h" + +static bool l2_ran; + +static void l2_clear_efer_svme(void) +{ + u64 efer = rdmsr(MSR_EFER); + + /* generic_svm_setup() initializes EFER_SVME set for L2 */ + GUEST_ASSERT(efer & EFER_SVME); + wrmsr(MSR_EFER, efer & ~EFER_SVME); + + /* Unreachable, L1 should be shutdown */ + GUEST_ASSERT(0); +} + +static void l1_clear_efer_svme(struct svm_test_data *svm) +{ + generic_svm_setup(svm, l2_clear_efer_svme); + run_guest(svm->vmcb, svm->vmcb_gpa); + + /* Unreachable, L1 should be shutdown */ + GUEST_ASSERT(0); +} + +static void l2_lmsle(void) +{ + GUEST_ASSERT(rdmsr(MSR_EFER) & EFER_LMSLE); + l2_ran = true; + vmmcall(); +} + +static void l1_lmsle(struct svm_test_data *svm) +{ + bool lmsle_mbz = this_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ); + struct vmcb *vmcb = svm->vmcb; + u64 efer = rdmsr(MSR_EFER); + + /* + * Selftests' vCPUs are created with EFER.LMSLE clear; the sub-tests + * below need to start from a clean slate. + */ + GUEST_ASSERT(!(efer & EFER_LMSLE)); + GUEST_ASSERT(!l2_ran); + + /* + * Per the APM, if EFER_LMSLE_MBZ is enumerated in CPUID, "64-bit mode + * segment limit checking is not supported and attempting to set + * EFER.LMSLE = 1 causes a #GP exception". + */ + if (lmsle_mbz) { + GUEST_ASSERT_EQ(wrmsr_safe(MSR_EFER, efer | EFER_LMSLE), GP_VECTOR); + GUEST_ASSERT(!(rdmsr(MSR_EFER) & EFER_LMSLE)); + } else { + GUEST_ASSERT_EQ(wrmsr_safe(MSR_EFER, efer | EFER_LMSLE), 0); + GUEST_ASSERT(rdmsr(MSR_EFER) & EFER_LMSLE); + + /* + * Restore EFER so that generic_svm_setup() doesn't propagate + * EFER.LMSLE into vmcb12 on its own, i.e. so that the VMRUN + * sub-test actually tests what it thinks it's testing. + */ + wrmsr(MSR_EFER, efer); + } + + /* + * VMRUN's consistency checks reject "any MBZ bit of EFER", i.e. a + * vmcb12 with EFER.LMSLE set must generate VMEXIT_INVALID when the + * defeature is enumerated. + */ + generic_svm_setup(svm, l2_lmsle); + vmcb->save.efer |= EFER_LMSLE; + run_guest(vmcb, svm->vmcb_gpa); + + if (lmsle_mbz) { + GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_ERR); + GUEST_ASSERT(!l2_ran); + } else { + GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_VMMCALL); + GUEST_ASSERT(l2_ran); + GUEST_ASSERT(vmcb->save.efer & EFER_LMSLE); + } + + GUEST_DONE(); +} + +static struct kvm_vcpu *create_l1_vcpu(struct kvm_vm **vm, void *l1_guest_code) +{ + struct kvm_vcpu *vcpu; + gva_t svm_gva; + + *vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); + + vcpu_alloc_svm(*vm, &svm_gva); + vcpu_args_set(vcpu, 1, svm_gva); + + return vcpu; +} + +static void test_enumeration(void) +{ + bool lmsle_mbz; + + /* + * EFER_LMSLE_MBZ, CPUID.80000008H:EBX[bit 20], is a "defeature" bit, + * i.e. is set when the CPU does *not* support long mode segment + * limits. KVM enumerates the defeature if and only if KVM refuses to + * set EFER.LMSLE, i.e. if the CPU doesn't support LMSLE, or if KVM + * doesn't support nested SVM. Derive the expectation from raw CPUID + * and kvm_amd's "nested" module param rather than from + * kvm_cpu_has(X86_FEATURE_SVM), so that the assertion doesn't simply + * compare KVM's enumeration to itself. + */ + lmsle_mbz = this_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ) || + !this_cpu_has(X86_FEATURE_SVM) || + !kvm_is_nested_virtualization_enabled(); + + TEST_ASSERT_EQ(kvm_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ), lmsle_mbz); + + ksft_test_result_pass("KVM enumerates EFER_LMSLE_MBZ=%d\n", lmsle_mbz); +} + +static void test_clear_efer_svme(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + + vcpu = create_l1_vcpu(&vm, l1_clear_efer_svme); + + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_SHUTDOWN); + + kvm_vm_free(vm); + ksft_test_result_pass("L2 clearing EFER.SVME shuts down L1\n"); +} + +static void test_lmsle(bool lmsle_mbz) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + + vcpu = create_l1_vcpu(&vm, l1_lmsle); + + vcpu_set_or_clear_cpuid_feature(vcpu, X86_FEATURE_EFER_LMSLE_MBZ, + lmsle_mbz); + + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + case UCALL_DONE: + break; + default: + TEST_FAIL("Unexpected ucall: %lu", uc.cmd); + } + + kvm_vm_free(vm); + ksft_test_result_pass("Guest EFER_LMSLE_MBZ=%d\n", lmsle_mbz); +} + +static void test_host_initiated_lmsle(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + u64 efer; + + vm = vm_create_with_one_vcpu(&vcpu, NULL); + vcpu_set_cpuid_feature(vcpu, X86_FEATURE_EFER_LMSLE_MBZ); + + /* + * EFER_LMSLE_MBZ is a guest CPUID consistency check, not a host + * capability, i.e. must not be enforced against host-initiated writes, + * so that userspace can set MSRs before it sets guest CPUID. + */ + efer = vcpu_get_msr(vcpu, MSR_EFER); + TEST_ASSERT(!(efer & EFER_LMSLE), "EFER.LMSLE unexpectedly set"); + + vcpu_set_msr(vcpu, MSR_EFER, efer | EFER_LMSLE); + TEST_ASSERT_EQ(vcpu_get_msr(vcpu, MSR_EFER), efer | EFER_LMSLE); + + kvm_vm_free(vm); + ksft_test_result_pass("Host-initiated EFER.LMSLE=1 is allowed\n"); +} + +static void test_sregs_lmsle(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_sregs sregs; + struct kvm_vm *vm; + int rc; + + vm = vm_create_with_one_vcpu(&vcpu, NULL); + vcpu_set_cpuid_feature(vcpu, X86_FEATURE_EFER_LMSLE_MBZ); + + /* + * Unlike KVM_SET_MSRS, KVM_SET_SREGS runs the full set of guest CPUID + * checks, i.e. rejects EFER.LMSLE even though it's host-initiated. + */ + vcpu_sregs_get(vcpu, &sregs); + TEST_ASSERT(!(sregs.efer & EFER_LMSLE), "EFER.LMSLE unexpectedly set"); + + sregs.efer |= EFER_LMSLE; + rc = _vcpu_sregs_set(vcpu, &sregs); + TEST_ASSERT(rc, "KVM allowed EFER.LMSLE with EFER_LMSLE_MBZ set"); + + kvm_vm_free(vm); + ksft_test_result_pass("KVM_SET_SREGS rejects EFER.LMSLE=1\n"); +} + +int main(int argc, char *argv[]) +{ + bool has_nested_svm, has_lmsle; + + ksft_print_header(); + ksft_set_plan(6); + + test_enumeration(); + + /* + * The sub-tests below need to actually run a nested guest, and the + * EFER.LMSLE sub-tests additionally need KVM to allow EFER.LMSLE. + * It's KVM's view of the world, not raw CPUID, that dictates whether + * EFER.LMSLE is allowed, i.e. whether the defeature is emulated. + */ + has_nested_svm = kvm_cpu_has(X86_FEATURE_SVM); + has_lmsle = has_nested_svm && + !kvm_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ); + + if (!has_nested_svm) + ksft_print_msg("Nested SVM unsupported\n"); + else if (!has_lmsle) + ksft_print_msg("KVM doesn't support EFER.LMSLE\n"); + + if (has_nested_svm) { + test_clear_efer_svme(); + test_lmsle(true); + } else { + ksft_test_result_skip("L2 clearing EFER.SVME shuts down L1\n"); + ksft_test_result_skip("Guest EFER_LMSLE_MBZ=1\n"); + } + + if (has_lmsle) { + test_lmsle(false); + test_host_initiated_lmsle(); + test_sregs_lmsle(); + } else { + ksft_test_result_skip("Guest EFER_LMSLE_MBZ=0\n"); + ksft_test_result_skip("Host-initiated EFER.LMSLE=1 is allowed\n"); + ksft_test_result_skip("KVM_SET_SREGS rejects EFER.LMSLE=1\n"); + } + + ksft_finished(); +} diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 0d8cc3cc0641..a97aa5b92616 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2175,7 +2175,7 @@ static int kvm_set_memory_region(struct kvm *kvm, int kvm_set_internal_memslot(struct kvm *kvm, const struct kvm_userspace_memory_region2 *mem) { - if (WARN_ON_ONCE(mem->slot < KVM_USER_MEM_SLOTS)) + if (WARN_ON_ONCE((u16)mem->slot < KVM_USER_MEM_SLOTS)) return -EINVAL; if (WARN_ON_ONCE(mem->flags))