From 7d2a727f44d833c5f5771e61fc721359f01a0b6f Mon Sep 17 00:00:00 2001 From: Zeng Chi Date: Thu, 27 Aug 2026 19:22:00 +0800 Subject: [PATCH 01/45] KVM: Mask off the address space ID in kvm_set_internal_memslot() check kvm_set_internal_memslot() is supposed to reject memslot IDs that belong to userspace, i.e. IDs below KVM_USER_MEM_SLOTS, but it compares the raw "slot" field, which packs both the address space ID and the memslot ID (as_id << 16 | id). For any address space other than 0, the packed value is always >= 65536, which is larger than KVM_USER_MEM_SLOTS, and so the sanity check is a nop regardless of the actual memslot ID. E.g. x86 sets internal memslots for every address space, including SMM, so the check would silently accept a userspace memslot ID for the SMM address space while rejecting the same ID for the normal address space. Extract the memslot ID the same way kvm_vm_ioctl_set_memory_region() and kvm_set_memory_region() do, so that the check covers all address spaces. No in-tree caller trips the check; this only makes the sanity check effective for all address spaces. Fixes: 156bffdb2b49 ("KVM: Add a dedicated API for setting KVM-internal memslots") Signed-off-by: Zeng Chi Link: https://patch.msgid.link/20260827112200.866476-1-zeng_chi911@163.com Signed-off-by: Sean Christopherson --- virt/kvm/kvm_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 108d42c5c1d6..d9da8b51614a 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2138,7 +2138,7 @@ static int kvm_set_memory_region(struct kvm *kvm, int kvm_set_internal_memslot(struct kvm *kvm, const struct kvm_userspace_memory_region2 *mem) { - if (WARN_ON_ONCE(mem->slot < KVM_USER_MEM_SLOTS)) + if (WARN_ON_ONCE((u16)mem->slot < KVM_USER_MEM_SLOTS)) return -EINVAL; if (WARN_ON_ONCE(mem->flags)) From 2616f497506a772db53e7bb1024cde50b4c165e0 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 11:44:49 -0700 Subject: [PATCH 02/45] KVM: x86: Zero reserved synic flags when delivering message to avoid stack leak Ensure all reserved flags are zeroed when setting the msg_pending flag as part of SynIC message delivery, as setting only msg_pending can leak seven bits of kernel stack data to the guest. E.g. gcc typically uses a bitwise-OR to set the msg_pending flag, without initializing the on-stack variable. Precisely zero the flags, e.g. as opposed to zeroing the entire structure, to make it somewhat more obvious that the flags *need* to be zeroed. E.g. it would be quite easy to misread the initial read of guest memory and think that zeroing the entire structure is completely superfluous. Fixes: 3a0e7731724f ("x86: kvm: hyperv: simplify SynIC message delivery") Cc: stable@vger.kernel.org Reported-by: Stefan Teodorescu Reviewed-by: Vitaly Kuznetsov Link: https://patch.msgid.link/20260826184449.814130-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/hyperv.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 604651cb2739..8d2669d8ef34 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -802,7 +802,9 @@ static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint, if (no_retry) return 0; + hv_hdr.message_flags.asu8 = 0; hv_hdr.message_flags.msg_pending = 1; + r = kvm_vcpu_write_guest_page(vcpu, msg_page_gfn, &hv_hdr.message_flags, msg_off + From 926a9ea9bdf0b09c1cd8e896b251152d892f2566 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 6 Aug 2026 10:14:00 -0700 Subject: [PATCH 03/45] KVM: x86: Track kvm_vcpu_arch.pending_ioapic_eoi as a u8 to drop dead BUG_ON() Track the vector associated with KVM_EXIT_IOAPIC_EOI as a u8, as the vector is zero-initialized and is only ever set to a legal vector. I.e. unlike many of the other vector variables in KVM, it can't be -1 (and can't be greater than 255, though that should hold true for all vectors in KVM). Drop the now fully dead BUG_ON(), which aside from being completely unnecessary, was also useless, as it wouldn't guard against a negative value, i.e. wouldn't detect KVM's magic -1 value. For all intents and purposes, no functional change intended. Suggested-by: Paolo Bonzini Reviewed-by: Kai Huang Reviewed-by: Paolo Bonzini Link: https://patch.msgid.link/20260806171400.4119301-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/lapic.c | 4 ++-- arch/x86/kvm/lapic.h | 2 +- arch/x86/kvm/vmx/vmx.c | 3 +-- arch/x86/kvm/x86.c | 1 - 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 3af7395c2430..14865d36fa49 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -974,7 +974,7 @@ struct kvm_vcpu_arch { bool pv_unhalted; } pv; - int pending_ioapic_eoi; + u8 pending_ioapic_eoi; int pending_external_vector; int highest_stale_pending_ioapic_eoi; diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index e1f3cea14765..ecf611845fb1 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1552,7 +1552,7 @@ static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector) return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors); } -static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) +static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, u8 vector) { int __maybe_unused trigger_mode; @@ -1622,7 +1622,7 @@ static int apic_set_eoi(struct kvm_lapic *apic) * this interface assumes a trap-like exit, which has already finished * desired side effect including vISR and vPPR update. */ -void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) +void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, u8 vector) { struct kvm_lapic *apic = vcpu->arch.apic; diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index bd1098c89d99..db071d0b172d 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -177,7 +177,7 @@ u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu); void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data); void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset); -void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector); +void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, u8 vector); int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr); void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 612ab07d4100..58078e50c4de 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5888,10 +5888,9 @@ static int handle_apic_access(struct kvm_vcpu *vcpu) static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) { unsigned long exit_qualification = vmx_get_exit_qual(vcpu); - int vector = exit_qualification & 0xff; /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ - kvm_apic_set_eoi_accelerated(vcpu, vector); + kvm_apic_set_eoi_accelerated(vcpu, exit_qualification & 0xff); return 1; } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 79468ddfe473..28172424c502 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8151,7 +8151,6 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_NMI, vcpu)) process_nmi(vcpu); if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) { - BUG_ON(vcpu->arch.pending_ioapic_eoi > 255); if (test_bit(vcpu->arch.pending_ioapic_eoi, vcpu->arch.ioapic_handled_vectors)) { vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI; From 7ed619909773d1cc2d026b63b003010693ef0c5f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 16:09:30 -0700 Subject: [PATCH 04/45] KVM: x86/mmu: Use KVM's max TDP level to determine need for 32-bit TDP root When checking to see if KVM needs to allocate a TDP PAE root that's 32-bit addressable, query KVM's overall max TDP level, not the vCPU-specific TDP level, as the logic is specific to using NPT on 32-bit hosts. Querying the vCPU's alleged TDP level is flawed and confusing, as KVM selects between 4-level vs. 5-level based on the guest's MAXPHYADDR, and MAXPHYADDR isn't yet configured (via CPUID) when the vCPU is being created. I.e. as is, it would *appear* that KVM is violating its own rules with respect to changing guest CPUID (see kvm_mmu_after_set_cpuid()). In practice, the flaw is benign as the goal is purely to see if KVM needs to use PAE-paging; whether KVM will use 4-level vs. 5-level is irrelevant. More importantly, avoiding kvm_mmu_get_tdp_level() during vCPU creation will allow hardening KVM's handling of S-EPT mirror root level. For all intents and purposes, no functional change intended. Reviewed-by: Xiaoyao Li Link: https://patch.msgid.link/20260902230932.2760127-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 064ecc33b926..c9a684151420 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -6834,7 +6834,7 @@ static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, struct k * other exception is for shadowing L1's 32-bit or PAE NPT on 64-bit * KVM; that horror is handled on-demand by mmu_alloc_special_roots(). */ - if (tdp_enabled && kvm_mmu_get_tdp_level(vcpu) > PT32E_ROOT_LEVEL) + if (tdp_enabled && kvm_mmu_get_max_tdp_level() > PT32E_ROOT_LEVEL) return 0; page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32); From cc8a8f2a90b45cd4c8509db7bc61c2ac9dbd2cf2 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 16:09:31 -0700 Subject: [PATCH 05/45] KVM: VMX: Explicitly track TDX VMs' root level instead of guessing it from CPUID Explicitly track the root level for TDX VMs instead of trying to infer the depth of the paging tree based on an individual vCPU's CPUID information. Applying KVM's existing logic to select the root level to TDX is flawed as nothing *requires* userspace to fill in the correct guest.MAXPHYADDR for a vCPU's CPUID. Guessing at the correct root level is also ridiculous given that userspace has already told KVM the root level during TD initialization. Relying on userspace to set the expected/correct CPUID lets a misbehaving userspace trip the KVM_BUG_ON() in tdx_load_mmu_pgd() by configuring guest CPUID to use an "incorrect" guest.MAXPHYADDR. Don't use kvm_gfn_direct_bits() to infer the mirror root level, as the connection between TDX's one and only "direct" bit and the predetermined root level is a TDX implementation detail. I.e. avoid baking in the assumption that there is exactly one "direct bits", that the one bit is a pivot between normal and mirror roots, and that the pivot bit is the most significant bit of the effective GPA space. For the same reason, set the root level and direct bits in TDX code, i.e. don't provide a helper in the MMU, because from the MMU's perspective, they are two separate concepts. Keep gfn_direct_bits even though it can be trivially derived from mirror_root_level as saving a whole eight bytes per VM is meaningless, keeping the TDX details buried in TDX would require a kvm_x86_ops hook, and the value is queried fairly often and in hot paths. And for the moment, keep the S-bit sanity check in tdx_load_mmu_pgd(), even though it really only needs to ensure the incoming level matches the preconfigured mirror root level. Because KVM manually configures the S-bit location, there's technically a risk that the S-bit location and mirror root level could get out of sync. That can be addressed by more programmatically computing the S-bit, but that doesn't need to be done now. Cc: Rick Edgecombe Cc: Xiaoyao Li Cc: Kai Huang Cc: Yan Zhao Fixes: 20d913729c11 ("KVM: x86/mmu: Taking guest pa into consideration when calculate tdp level") Reviewed-by: Rick Edgecombe Tested-by: Yan Zhao Tested-by: Binbin Wu Reviewed-by: Binbin Wu Reviewed-by: Xiaoyao Li Link: https://patch.msgid.link/20260902230932.2760127-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/cpuid.c | 14 -------------- arch/x86/kvm/cpuid.h | 1 - arch/x86/kvm/mmu/mmu.c | 19 +++++++++++-------- arch/x86/kvm/vmx/tdx.c | 14 ++++++++++++-- 5 files changed, 24 insertions(+), 25 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 14865d36fa49..e369faf3c285 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1407,6 +1407,7 @@ struct kvm_arch { struct kvm_mmu_memory_cache split_desc_cache; gfn_t gfn_direct_bits; + int mirror_root_level; /* * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer. A Zero diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 851f151efb35..bb68a021cfaf 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -483,20 +483,6 @@ int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu) return 36; } -int cpuid_query_maxguestphyaddr(struct kvm_vcpu *vcpu) -{ - struct kvm_cpuid_entry2 *best; - - best = kvm_find_cpuid_entry(vcpu, 0x80000000); - if (!best || best->eax < 0x80000008) - goto not_found; - best = kvm_find_cpuid_entry(vcpu, 0x80000008); - if (best) - return (best->eax >> 16) & 0xff; -not_found: - return 0; -} - /* * This "raw" version returns the reserved GPA bits without any adjustments for * encryption technologies that usurp bits. The raw mask should be used if and diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h index 8d863f45585d..46bfe8699e67 100644 --- a/arch/x86/kvm/cpuid.h +++ b/arch/x86/kvm/cpuid.h @@ -68,7 +68,6 @@ void __init kvm_init_xstate_sizes(void); u32 xstate_required_size(u64 xstate_bv, bool compacted); int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu); -int cpuid_query_maxguestphyaddr(struct kvm_vcpu *vcpu); u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu); static inline int cpuid_maxphyaddr(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index c9a684151420..d43b8b6c3b17 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5953,19 +5953,22 @@ void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu, static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu) { - int maxpa; - - if (vcpu->kvm->arch.vm_type == KVM_X86_TDX_VM) - maxpa = cpuid_query_maxguestphyaddr(vcpu); - else - maxpa = cpuid_maxphyaddr(vcpu); - /* tdp_root_level is architecture forced level, use it if nonzero */ if (tdp_root_level) return tdp_root_level; + /* + * If the VM has mirror roots, then the root level is predefined as the + * mirror root (and by extension the normal root) needs to match the + * root level that was configured for the external page tables that are + * being mirrored by KVM. + */ + if (kvm_has_mirrored_tdp(vcpu->kvm) && + !WARN_ON_ONCE(!vcpu->kvm->arch.mirror_root_level)) + return vcpu->kvm->arch.mirror_root_level; + /* Use 5-level TDP if and only if it's useful/necessary. */ - if (max_tdp_level == 5 && maxpa <= 48) + if (max_tdp_level == 5 && cpuid_maxphyaddr(vcpu) <= 48) return 4; return max_tdp_level; diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index b272c20586a7..e6b7da616817 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2760,6 +2760,13 @@ DEFINE_CLASS(tdx_vm_state_guard, tdx_vm_state_guard_t, if (!IS_ERR(_T)) tdx_release_vm_state_locks(_T), tdx_acquire_vm_state_locks(kvm), struct kvm *kvm); +static __always_inline void tdx_set_mirror_root_level(struct kvm *kvm, int level) +{ + BUILD_BUG_ON(level != 4 && level != 5); + + kvm->arch.mirror_root_level = level; +} + static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) { struct kvm_tdx_init_vm __user *user_data = u64_to_user_ptr(cmd->data); @@ -2822,10 +2829,13 @@ static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) kvm_tdx->attributes = td_params->attributes; kvm_tdx->xfam = td_params->xfam; - if (td_params->config_flags & TDX_CONFIG_FLAGS_MAX_GPAW) + if (td_params->config_flags & TDX_CONFIG_FLAGS_MAX_GPAW) { kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_5; - else + tdx_set_mirror_root_level(kvm, 5); + } else { kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_4; + tdx_set_mirror_root_level(kvm, 4); + } kvm_tdx->state = TD_STATE_INITIALIZED; out: From 37e3197c50c824720c3af3118942bda3678f832a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 16:09:32 -0700 Subject: [PATCH 06/45] KVM: VMX: Drop TDX_SHARED_BIT_PWL_{4,5} and dedup related code Fold the GPA => GFN conversion and bitshift logic for identifying the S-bit given the EPT root level into tdx_set_mirror_root_level() to dedup the math and drop TDX_SHARED_BIT_PWL_{4,5} in the process. In addition to deduping a small amount of code, using the level to compute the S-bit position more or less eliminates the risk of the mirror_root_level and gfn_direct_bits getting out of sync. Opportunistically switch the sanity check in tdx_load_mmu_pgd() to check the mirror root level, not the S-Bit location, now that it's all but impossible for the two things to get out of sync. For all intents and purposes, no functional change intended. Reviewed-by: Xiaoyao Li Link: https://patch.msgid.link/20260902230932.2760127-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index e6b7da616817..a0bc9f818f43 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -56,9 +56,6 @@ bool enable_tdx __ro_after_init; module_param_named(tdx, enable_tdx, bool, 0444); -#define TDX_SHARED_BIT_PWL_5 gpa_to_gfn(BIT_ULL(51)) -#define TDX_SHARED_BIT_PWL_4 gpa_to_gfn(BIT_ULL(47)) - static const struct tdx_sys_info *tdx_sysinfo; void tdh_vp_rd_failed(struct vcpu_tdx *tdx, char *uclass, u32 field, u64 err) @@ -1609,10 +1606,7 @@ static int handle_tdvmcall(struct kvm_vcpu *vcpu) void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int pgd_level) { - u64 shared_bit = (pgd_level == 5) ? TDX_SHARED_BIT_PWL_5 : - TDX_SHARED_BIT_PWL_4; - - if (KVM_BUG_ON(shared_bit != kvm_gfn_direct_bits(vcpu->kvm), vcpu->kvm)) + if (KVM_BUG_ON(pgd_level != vcpu->kvm->arch.mirror_root_level, vcpu->kvm)) return; td_vmcs_write64(to_tdx(vcpu), SHARED_EPT_POINTER, root_hpa); @@ -2765,6 +2759,7 @@ static __always_inline void tdx_set_mirror_root_level(struct kvm *kvm, int level BUILD_BUG_ON(level != 4 && level != 5); kvm->arch.mirror_root_level = level; + kvm->arch.gfn_direct_bits = gpa_to_gfn(BIT_ULL(level == 4 ? 47 : 51)); } static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) @@ -2829,13 +2824,10 @@ static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) kvm_tdx->attributes = td_params->attributes; kvm_tdx->xfam = td_params->xfam; - if (td_params->config_flags & TDX_CONFIG_FLAGS_MAX_GPAW) { - kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_5; + if (td_params->config_flags & TDX_CONFIG_FLAGS_MAX_GPAW) tdx_set_mirror_root_level(kvm, 5); - } else { - kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_4; + else tdx_set_mirror_root_level(kvm, 4); - } kvm_tdx->state = TD_STATE_INITIALIZED; out: From 5dc500a354cc8e109df678076ae3488094649468 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 16:20:24 -0700 Subject: [PATCH 07/45] KVM: nSVM: Reject KVM_SET_NESTED_STATE if L1 has EFER.LMA=1 && EFER.LME=0 Reject KVM_SET_NESTED_STATE if the incoming L1 host state has what is effectively an impossible EFER combination of LMA=1 but LME=0, i.e. if the state says long mode is active but not enabled. Unlike VMX, SVM doesn't have an explicit consistent check for the illegal combination; presumably hardware simply ignores EFER.LMA if EFER.LME=0. Unfortunately, KVM doesn't ignore EFER.LMA in this case and consumes the illegal state when constructing the shadow MMU for L2. E.g. if userspace also clears CR4.PAE, then kvm_calc_cpu_role() will compute a role with 4 or 5 levels of paging, but shadow_mmu_init_context() will wire up the MMU to use the paging32 template, which maxes out its levels at 2. Note, the "real badness" is effectively the same as what happened with the nVMX bug fixed by commit 112e66017bff ("KVM: nVMX: add missing consistency checks for CR0 and CR4"). Unfortunately, the sanity check added by commit 72e2fb24a0b0 ("KVM: x86/mmu: Bug the VM if a vCPU ends up in long mode without PAE enabled") doesn't work for this case, since L2 state is active at the time of the page fault, but it's L1 that has the bad state. Fixes: cc440cdad5b7 ("KVM: nSVM: implement KVM_GET_NESTED_STATE and KVM_SET_NESTED_STATE") Cc: stable@vger.kernel.org Cc: Yosry Ahmed Reported-by: Stefan Teodorescu Link: https://patch.msgid.link/20260902232028.2767071-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 73f37b050d0a..49fb10ad1f9f 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -2028,6 +2028,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, if (!(save->cr0 & X86_CR0_PG) || !(save->cr0 & X86_CR0_PE) || (save->rflags & X86_EFLAGS_VM) || + ((save->efer & EFER_LMA) && !(save->efer & EFER_LME)) || !nested_vmcb_check_save(vcpu, &save_cached, false)) goto out_free; From 1c4c84a643b704e473cb171f7b61359ff29b12a4 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 16:20:25 -0700 Subject: [PATCH 08/45] KVM: nSVM: Ignore EFER.LMA if EFER.LME=0 when preparing L2 state Force EFER.LMA=0 if EFER.LME=0 when preparing L2 state for VMRUN, i.e. mimic real hardware's behavior of ignoring EFER.LMA if EFER.LME=0. VMRUN unfortunately allows the nonsensical combination, i.e. doesn't fail, but KVM itself has an invariant EFER.LMA can be set et if and only if EFER.LME is set. Breaking that invariant can lead to a variety of issue, particularly in MMU code that keys off EFER.LMA when determining whether to emulate/virtualization 4/5-level paging versus PAE paging. Cc: stable@vger.kernel.org Cc: Yosry Ahmed Link: https://patch.msgid.link/20260902232028.2767071-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 49fb10ad1f9f..23d29597d6bf 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -789,6 +789,10 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) kvm_set_rflags(vcpu, save->rflags | X86_EFLAGS_FIXED); + /* SVM ignores EFER.LMA if EFER.LME=0 (instead of failing VMRUN). */ + if (!(svm->nested.save.efer & EFER_LME)) + svm->nested.save.efer &= ~EFER_LMA; + svm_set_efer(vcpu, svm->nested.save.efer); svm_set_cr0(vcpu, svm->nested.save.cr0); From 115f9405a2ed6d78ee7ad8436bdd638743f6d1d4 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 16:20:26 -0700 Subject: [PATCH 09/45] KVM: x86/mmu: Bug the VM if KVM attempts to walk more levels than the MMU has Extend the "EFER.LMA && !CR4.PAE" check, which exists largely to guard against KVM configuring a paging32 MMU with more than 2 levels of paging, with a very explicit check for exactly that: that KVM isn't trying to walk more levels of paging than the MMU template provides. I.e. harden KVM against all bugs that would cause KVM to generates accesses beyond the bounds of guest_walker's arrays, regardless of how KVM ended up with the misconfigured MMU. Note, don't use w->cpu_role.base.level directly as the paging64 template only provides two levels of page tables for PAE paging on 32-bit hosts, and handles the third level by manually emulating the PDPTR access. Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260902232028.2767071-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/paging_tmpl.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 27427e7f22fa..f925b11d76dd 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -368,13 +368,14 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, pte_access = ~0; /* - * Queue a page fault for injection if this assertion fails, as callers - * assume that walker.fault contains sane info on a walk failure. I.e. - * avoid making the situation worse by inducing even worse badness - * between when the assertion fails and when KVM kicks the vCPU out to - * userspace (because the VM is bugged). + * Queue a page fault for injection if any of the below assertions fail, + * as callers assume that walker.fault contains sane info on a walk + * failure. I.e. avoid making the situation worse by inducing even + * worse badness between when the assertion fails and when KVM kicks + * the vCPU out to userspace (because the VM is bugged). */ - if (KVM_BUG_ON(is_long_mode(vcpu) && !is_pae(vcpu), vcpu->kvm)) + if (KVM_BUG_ON(is_long_mode(vcpu) && !is_pae(vcpu), vcpu->kvm) || + KVM_BUG_ON(walker->max_level > PT_MAX_FULL_LEVELS, vcpu->kvm)) goto error; ++walker->level; From 34dacbe7ae93f2d3c26cd0bf3e08200b1fa2c180 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 16:20:27 -0700 Subject: [PATCH 10/45] KVM: x86/mmu: Bug the VM if KVM calcs a CPU role with EFER.LMA=1 && CR4.PAE=0 Bug the VM if KVM attempts to construct a CPU role with the should-be- impossible combination of long mode being active without PAE paging being enabled. KVM's MMU construction assumes that EFER.LMA can be set if and only CR4.PAE is set, and will create a completely invalid MMU if that assumption fails. FNAME(walk_addr_generic) already has sanity checks to try and mitigate the fallout, but attempt to catch such bugs earlier, as this is (at least) the second time KVM has had bugs that escaped into FNAME(walk_addr_generic), and it's entirely possible the bad state could cause problems elsewhere. Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260902232028.2767071-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index d43b8b6c3b17..0601964934a1 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5910,6 +5910,9 @@ static union kvm_cpu_role kvm_calc_cpu_role(struct kvm_vcpu *vcpu, return role; } + if (KVM_BUG_ON(____is_efer_lma(regs) && !____is_cr4_pae(regs), vcpu->kvm)) + *(u64 *)®s->efer &= ~EFER_LMA; + role.base.efer_nx = ____is_efer_nx(regs); role.base.cr0_wp = ____is_cr0_wp(regs); role.base.cr4_smep = ____is_cr4_smep(regs); From 47b437b2d903137d547365391c7a5153b01dee6a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 16:20:28 -0700 Subject: [PATCH 11/45] KVM: x86/mmu: Convert MMU walker's bounds check from BUG_ON() to KVM_BUG_ON() Bug the VM, not the host, if KVM's sanity check that walking guest PTEs doesn't underflow the walker's level fires. Bugging the host while holding mmu_lock is all but guaranteed to panic the host, KVM hasn't _yet_ consumed the out-of-bounds level (i.e. hasn't corrupted memory), and KVM is already committed to bugging the VM and synthesizing a guest page fault if a fatal MMU error occurs while walking guest PTEs. I.e. there's no reason to keep the BUG_ON() at this point. Link: https://patch.msgid.link/20260902232028.2767071-6-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/paging_tmpl.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index f925b11d76dd..c8ec47b09264 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -392,7 +392,9 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, offset = index * sizeof(pt_element_t); pte_gpa = gfn_to_gpa(table_gfn) + offset; - BUG_ON(walker->level < 1); + if (KVM_BUG_ON(walker->level < 1, vcpu->kvm)) + goto error; + walker->table_gfn[walker->level - 1] = table_gfn; walker->pte_gpa[walker->level - 1] = pte_gpa; From 674717509d9ecfd3362c3fbdde1fe477fe117fe0 Mon Sep 17 00:00:00 2001 From: Hao Xiang Date: Tue, 8 Sep 2026 19:54:51 +0800 Subject: [PATCH 12/45] KVM: x86: Rename CPUID feature bit AVX10_VNNI_INT to AVX10_V1_AUX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename AVX10_VNNI_INT is to AVX10_V1_AUX to match the most recent version of the AVX10.2 spec (revision 7.0) [*]. Intel renamed the CPUID flag to apply to a variety of "convert" instructions as well as VNNI instructions. [*] Intel® Advanced Vector Extensions 10.2 Architecture Specification (rev 7.0) Signed-off-by: Hao Xiang Reviewed-by: Zhao Liu Link: https://patch.msgid.link/047d5e3a-b64d-434c-9a40-140468441de2.hao.xiang@linux.alibaba.com [sean: massage changelog] Signed-off-by: Sean Christopherson --- arch/x86/kvm/cpuid.c | 2 +- arch/x86/kvm/reverse_cpuid.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index bb68a021cfaf..cbf1fe4ae01f 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1081,7 +1081,7 @@ void kvm_initialize_cpu_caps(void) ); kvm_cpu_cap_init(CPUID_24_1_ECX, - F(AVX10_VNNI_INT), + F(AVX10_V1_AUX), ); kvm_cpu_cap_init(CPUID_8000_0001_ECX, diff --git a/arch/x86/kvm/reverse_cpuid.h b/arch/x86/kvm/reverse_cpuid.h index 2ad25781cefb..89345af0f7ad 100644 --- a/arch/x86/kvm/reverse_cpuid.h +++ b/arch/x86/kvm/reverse_cpuid.h @@ -66,7 +66,7 @@ #define X86_FEATURE_AVX10_512 KVM_X86_FEATURE(CPUID_24_0_EBX, 18) /* Intel-defined sub-features, CPUID level 0x00000024:1 (ECX) */ -#define X86_FEATURE_AVX10_VNNI_INT KVM_X86_FEATURE(CPUID_24_1_ECX, 2) +#define X86_FEATURE_AVX10_V1_AUX KVM_X86_FEATURE(CPUID_24_1_ECX, 2) /* CPUID level 0x80000007 (EDX). */ #define KVM_X86_FEATURE_CONSTANT_TSC KVM_X86_FEATURE(CPUID_8000_0007_EDX, 8) From 3e7a842fe53301e4d0c2717108b9cd41df009a65 Mon Sep 17 00:00:00 2001 From: Tharit Tangkijwanichakul Date: Tue, 8 Sep 2026 23:04:25 +0700 Subject: [PATCH 13/45] KVM: x86: Reject reserved CR8 bits in KVM_SET_SREGS kvm_is_valid_sregs() validates the incoming CR0, CR4, and efer values but never checks CR8. When userspace passes a CR8 value with any of the reserved bits [63:4] set, __set_sregs_common() forwards it to kvm_set_cr8(), which rejects the reserved bits and returns early. That return value is not checked, so the ioctl reports success while the requested value is silently dropped. A subsequent KVM_GET_SREGS then returns a CR8 different from the one userspace believed it had written. Factor the reserved-bit check out into kvm_is_valid_cr8() and use it both in kvm_set_cr8() and in kvm_is_valid_sregs(). Note, prior to commit 7017fc3d1a12 ("KVM: Define and use cr8 access functions"), which unintentionally introduced the silent dropping behavior, KVM didn't check the incoming CR8 at all, i.e. allowed illegal values and likely generated failed VM-Entries. Fixes: 6aa8b732ca01 ("[PATCH] kvm: userspace interface") Fixes: 7017fc3d1a12 ("KVM: Define and use cr8 access functions") Signed-off-by: Tharit Tangkijwanichakul Link: https://patch.msgid.link/20260908160426.6547-2-tharitt97@gmail.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/regs.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c index 8f66438989e4..fad31b59c622 100644 --- a/arch/x86/kvm/regs.c +++ b/arch/x86/kvm/regs.c @@ -440,9 +440,14 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr4); +static bool kvm_is_valid_cr8(unsigned long cr8) +{ + return !(cr8 & CR8_RESERVED_BITS); +} + int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) { - if (cr8 & CR8_RESERVED_BITS) + if (!kvm_is_valid_cr8(cr8)) return 1; if (lapic_in_kernel(vcpu)) kvm_lapic_set_tpr(vcpu, cr8); @@ -565,6 +570,7 @@ static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) return kvm_is_valid_cr4(vcpu, sregs->cr4) && kvm_is_valid_cr0(vcpu, sregs->cr0) && + kvm_is_valid_cr8(sregs->cr8) && kvm_valid_efer(vcpu, sregs->efer); } From 2b0e0aa54fa27af7c4afd502ef805e61f80c4498 Mon Sep 17 00:00:00 2001 From: Tharit Tangkijwanichakul Date: Tue, 8 Sep 2026 23:04:26 +0700 Subject: [PATCH 14/45] KVM: selftests: Add CR8 reserved-bit checks to set_sregs_test Extend set_sregs_test to cover CR8. Verify that the architecturally allowed bits [3:0] can be set and read back, that every reserved bit [63:4] is rejected by KVM_SET_SREGS, and that a successful set is observable via KVM_GET_SREGS. Without the accompanying fix to kvm_is_valid_sregs(), setting a reserved CR8 bit is silently accepted and the read-back value diverges from the value written, so this test fails on an unpatched kernel and passes once the fix is applied. Signed-off-by: Tharit Tangkijwanichakul Link: https://patch.msgid.link/20260908160426.6547-3-tharitt97@gmail.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/set_sregs_test.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tools/testing/selftests/kvm/x86/set_sregs_test.c b/tools/testing/selftests/kvm/x86/set_sregs_test.c index 603226ffe437..5afc4187f5eb 100644 --- a/tools/testing/selftests/kvm/x86/set_sregs_test.c +++ b/tools/testing/selftests/kvm/x86/set_sregs_test.c @@ -135,6 +135,17 @@ static void test_cr_bits(struct kvm_vcpu *vcpu, u64 cr4) /* NW without CD is illegal, as is PG without PE. */ TEST_INVALID_SREG_BIT(vcpu, cr0, sregs, X86_CR0_NW); TEST_INVALID_SREG_BIT(vcpu, cr0, sregs, X86_CR0_PG); + + /* CR8 bits 3:0 are writable; bits 63:4 are reserved. */ + vcpu_sregs_get(vcpu, &sregs); + sregs.cr8 = 0xf; + vcpu_sregs_set(vcpu, &sregs); + + vcpu_sregs_get(vcpu, &sregs); + TEST_ASSERT_EQ(sregs.cr8, 0xf); + + for (i = 4; i < 64; i++) + TEST_INVALID_SREG_BIT(vcpu, cr8, sregs, BIT_ULL(i)); } static void test_efer_bits(struct kvm_vcpu *vcpu, u64 efer) From fe02f37155c80ccfe8cbe10fd2bec1b0689028be Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 5 Sep 2026 11:09:12 +0200 Subject: [PATCH 15/45] Documentation: KVM: SEV: Use the uapi structure names The document names struct kvm_sev_launch_update, struct kvm_sev_attestation, struct kvm_sev_launch_send_update_data and struct kvm_sev_launch_receive_update_data. None of them exists in arch/x86/include/uapi/asm/kvm.h; the structures are kvm_sev_launch_update_data, kvm_sev_attestation_report, kvm_sev_send_update_data and kvm_sev_receive_update_data. Use the uapi names. Fixes: dc48bae01e5a ("KVM: Define SEV key management command id") Fixes: 2c07ded06427 ("KVM/SVM: add support for SEV attestation command") Fixes: d3d1af85e2c7 ("KVM: SVM: Add KVM_SEND_UPDATE_DATA command") Fixes: 15fb7de1a7f5 ("KVM: SVM: Add KVM_SEV_RECEIVE_UPDATE_DATA command") Assisted-by: LLM Signed-off-by: Karl Mehltretter Link: https://patch.msgid.link/20260905090912.40740-1-kmehltretter@gmail.com Signed-off-by: Sean Christopherson --- Documentation/virt/kvm/x86/amd-memory-encryption.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/virt/kvm/x86/amd-memory-encryption.rst b/Documentation/virt/kvm/x86/amd-memory-encryption.rst index bd04a908a8db..9ebd99aa1d0b 100644 --- a/Documentation/virt/kvm/x86/amd-memory-encryption.rst +++ b/Documentation/virt/kvm/x86/amd-memory-encryption.rst @@ -170,7 +170,7 @@ Returns: 0 on success, -negative on error :: - struct kvm_sev_launch_update { + struct kvm_sev_launch_update_data { __u64 uaddr; /* userspace address to be encrypted (must be 16-byte aligned) */ __u32 len; /* length of the data to be encrypted (must be 16-byte aligned) */ }; @@ -316,7 +316,7 @@ used by the guest owner with the KVM_SEV_LAUNCH_MEASURE. If len is zero on entry, the measurement blob length is written to len and uaddr is unused. -Parameters (in): struct kvm_sev_attestation +Parameters (in): struct kvm_sev_attestation_report Returns: 0 on success, -negative on error @@ -377,7 +377,7 @@ Returns: 0 on success, -negative on error :: - struct kvm_sev_launch_send_update_data { + struct kvm_sev_send_update_data { __u64 hdr_uaddr; /* userspace address containing the packet header */ __u32 hdr_len; @@ -447,7 +447,7 @@ Returns: 0 on success, -negative on error :: - struct kvm_sev_launch_receive_update_data { + struct kvm_sev_receive_update_data { __u64 hdr_uaddr; /* userspace address containing the packet header */ __u32 hdr_len; From 07de87723b6f7a7d4a699eb94e75ebe85b6b8c98 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 5 Sep 2026 11:10:08 +0200 Subject: [PATCH 16/45] Documentation: KVM: x86: Use struct kvm_clock_pairing in hypercalls.rst struct kvm_clock_offset does not exist; the name is struct kvm_clock_pairing. Use struct kvm_clock_pairing. Fixes: 55dd00a73a51 ("KVM: x86: add KVM_HC_CLOCK_PAIRING hypercall") Assisted-by: LLM Signed-off-by: Karl Mehltretter Link: https://patch.msgid.link/20260905091008.40802-1-kmehltretter@gmail.com Signed-off-by: Sean Christopherson --- Documentation/virt/kvm/x86/hypercalls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/virt/kvm/x86/hypercalls.rst b/Documentation/virt/kvm/x86/hypercalls.rst index 521ecf9a8a36..e0b6c11f19ce 100644 --- a/Documentation/virt/kvm/x86/hypercalls.rst +++ b/Documentation/virt/kvm/x86/hypercalls.rst @@ -109,7 +109,7 @@ The template for each hypercall is: Usage: a0: guest physical address where host copies -"struct kvm_clock_offset" structure. +"struct kvm_clock_pairing" structure. a1: clock_type, ATM only KVM_CLOCK_PAIRING_WALLCLOCK (0) is supported (corresponding to the host's CLOCK_REALTIME clock). From ba70065fec49766cd9e5a53ca6540703c9f53b85 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 17 Sep 2026 06:03:20 -0400 Subject: [PATCH 17/45] KVM: x86: remove "full" out-argument from flush_tlb_gva The "full" out-argument in flush_tlb_gva is only ever set on AMD processors, where the TLB flush is little more than a memory write and thus extremely cheap anyway. Simplify the code by letting the implementation of Hyper-V TLB flush hypercalls always go through all the addresses queued in tlb_flush_fifo; there are no performance concerns since that is already the case on VMX, and also was on SVM prior to commit 26505e1b5b54 ("KVM: SVM: make svm_flush_tlb_gva do a full asid flush if NPT enabled", 2026-08-06). Suggested-by: Sean Christopherson Signed-off-by: Paolo Bonzini Link: https://patch.msgid.link/20260917100320.21899-1-pbonzini@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/hyperv.c | 7 +++---- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/svm/svm.c | 4 +--- arch/x86/kvm/vmx/main.c | 4 ++-- arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/kvm/vmx/x86_ops.h | 2 +- 7 files changed, 10 insertions(+), 13 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index e369faf3c285..1bfe56fbb8a6 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1580,7 +1580,7 @@ struct kvm_x86_ops { * Can potentially get non-canonical addresses through INVLPGs, which * the implementation may choose to ignore if appropriate. */ - void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr, bool *full); + void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr); /* * Flush any TLB entries created by the guest. Like tlb_flush_gva(), diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 8d2669d8ef34..5d2e3617c952 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -1997,7 +1997,6 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) u64 entries[KVM_HV_TLB_FLUSH_FIFO_SIZE]; int i, j, count; gva_t gva; - bool full = false; if (!tdp_enabled) return -EINVAL; @@ -2008,7 +2007,7 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) count = kfifo_out(&tlb_flush_fifo->entries, entries, KVM_HV_TLB_FLUSH_FIFO_SIZE); - for (i = 0; i < count && !full; i++) { + for (i = 0; i < count; i++) { if (entries[i] == KVM_HV_TLB_FLUSHALL_ENTRY) goto out_flush_all; @@ -2017,11 +2016,11 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) * pages to flush. */ gva = entries[i] & PAGE_MASK; - for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1 && !full; j++) { + for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1; j++) { if (is_noncanonical_invlpg_address(gva + j * PAGE_SIZE, vcpu)) continue; - kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE, &full); + kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE); } ++vcpu->stat.tlb_flush; diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 0601964934a1..f037fa665414 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -6710,7 +6710,7 @@ void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w, if (is_noncanonical_invlpg_address(addr, vcpu)) return; - kvm_x86_call(flush_tlb_gva)(vcpu, addr, NULL); + kvm_x86_call(flush_tlb_gva)(vcpu, addr); if (tdp_enabled) return; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 7d59d301e1e5..351b9c7233f0 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -4245,7 +4245,7 @@ static void svm_flush_tlb_guest(struct kvm_vcpu *vcpu) svm_flush_tlb_asid(vcpu); } -static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva, bool *full) +static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) { struct vcpu_svm *svm = to_svm(vcpu); @@ -4261,8 +4261,6 @@ static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva, bool *full) } svm_flush_tlb_guest(vcpu); - if (full) - *full = true; } static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 4c52ab8d0786..0ff3230fd95e 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -543,12 +543,12 @@ static void vt_flush_tlb_current(struct kvm_vcpu *vcpu) vmx_flush_tlb_current(vcpu); } -static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full) +static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) { if (is_td_vcpu(vcpu)) return; - vmx_flush_tlb_gva(vcpu, addr, full); + vmx_flush_tlb_gva(vcpu, addr); } static void vt_flush_tlb_guest(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 58078e50c4de..4395d0809833 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -3379,7 +3379,7 @@ void vmx_flush_tlb_current(struct kvm_vcpu *vcpu) vpid_sync_context(vmx_get_current_vpid(vcpu)); } -void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full) +void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) { /* * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h index 054fd14bb2e1..cdb38d940cfb 100644 --- a/arch/x86/kvm/vmx/x86_ops.h +++ b/arch/x86/kvm/vmx/x86_ops.h @@ -82,7 +82,7 @@ void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); bool vmx_get_if_flag(struct kvm_vcpu *vcpu); void vmx_flush_tlb_all(struct kvm_vcpu *vcpu); void vmx_flush_tlb_current(struct kvm_vcpu *vcpu); -void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full); +void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr); void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu); void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask); u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu); From 1065274e5cbbf43683c125c654b66a7867ead140 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 7 Sep 2026 06:38:59 +0000 Subject: [PATCH 18/45] KVM: VMX: Pass @vcpu, not @vmx to init_vmcs() Pass @vcpu instead of @vmx to init_vmcs(), and switch all of the vmx->vcpu usage to a simple vcpu. No functional change intended. Signed-off-by: Nikunj A Dadhania Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260907063906.1964557-2-nikunj@amd.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/vmx.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 4395d0809833..928627a2ac8c 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4883,10 +4883,11 @@ int vmx_vcpu_precreate(struct kvm *kvm) #define VMX_XSS_EXIT_BITMAP 0 -static void init_vmcs(struct vcpu_vmx *vmx) +static void init_vmcs(struct kvm_vcpu *vcpu) { - struct kvm *kvm = vmx->vcpu.kvm; + struct kvm *kvm = vcpu->kvm; struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); + struct vcpu_vmx *vmx = to_vmx(vcpu); if (nested) nested_vmx_set_vmcs_shadowing_bitmap(); @@ -4911,7 +4912,7 @@ static void init_vmcs(struct vcpu_vmx *vmx) if (cpu_has_tertiary_exec_ctrls()) tertiary_exec_controls_set(vmx, vmx_tertiary_exec_control(vmx)); - if (enable_apicv && lapic_in_kernel(&vmx->vcpu)) { + if (enable_apicv && lapic_in_kernel(vcpu)) { vmcs_write64(EOI_EXIT_BITMAP0, 0); vmcs_write64(EOI_EXIT_BITMAP1, 0); vmcs_write64(EOI_EXIT_BITMAP2, 0); @@ -4923,7 +4924,7 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->vt.pi_desc))); } - if (vmx_can_use_ipiv(&vmx->vcpu)) { + if (vmx_can_use_ipiv(vcpu)) { vmcs_write64(PID_POINTER_TABLE, __pa(kvm_vmx->pid_table)); vmcs_write16(LAST_PID_POINTER_INDEX, kvm->arch.max_vcpu_ids - 1); } @@ -4958,15 +4959,15 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) - vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); + vmcs_write64(GUEST_IA32_PAT, vcpu->arch.pat); vm_exit_controls_set(vmx, vmx_get_initial_vmexit_ctrl()); /* 22.2.1, 20.8.1 */ vm_entry_controls_set(vmx, vmx_get_initial_vmentry_ctrl()); - vmx->vcpu.arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); - vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits); + vcpu->arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); + vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); set_cr4_guest_host_mask(vmx); @@ -4981,7 +4982,7 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); } - vmx_write_encls_bitmap(&vmx->vcpu, NULL); + vmx_write_encls_bitmap(vcpu, NULL); if (vmx_pt_mode_is_host_guest()) { memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc)); @@ -4994,13 +4995,13 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_writel(GUEST_SYSENTER_ESP, 0); vmcs_writel(GUEST_SYSENTER_EIP, 0); - vmx_guest_debugctl_write(&vmx->vcpu, 0); + vmx_guest_debugctl_write(vcpu, 0); if (cpu_has_vmx_tpr_shadow()) { vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); - if (cpu_need_tpr_shadow(&vmx->vcpu)) + if (cpu_need_tpr_shadow(vcpu)) vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, - __pa(vmx->vcpu.arch.apic->regs)); + __pa(vcpu->arch.apic->regs)); vmcs_write32(TPR_THRESHOLD, 0); } @@ -5011,7 +5012,7 @@ static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - init_vmcs(vmx); + init_vmcs(vcpu); if (nested && kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) From d75bb23606ff5d142d115f12adac339e3266eb5c Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Mon, 7 Sep 2026 06:39:00 +0000 Subject: [PATCH 19/45] KVM: x86: Move PML page to common vcpu arch structure Move the PML page pointer from VMX-specific vcpu_vmx structure to the common kvm_vcpu_arch structure to enable sharing between VMX and SVM implementations. Only the page pointer is moved to x86 common code while keeping allocation logic vendor-specific, since AMD requires snp_safe_alloc_page() for PML buffer allocation. Update all VMX references accordingly. No functional change, restructuring to prepare for SVM PML support. Suggested-by: Kai Huang Reviewed-by: Kai Huang Signed-off-by: Nikunj A Dadhania Link: https://patch.msgid.link/20260907063906.1964557-3-nikunj@amd.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/vmx/vmx.c | 21 ++++++++++----------- arch/x86/kvm/vmx/vmx.h | 7 ------- arch/x86/kvm/x86.h | 5 +++++ 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 1bfe56fbb8a6..6fce5a0061c2 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -789,6 +789,8 @@ struct kvm_vcpu_arch { */ struct kvm_mmu_memory_cache mmu_external_spt_cache; + struct page *pml_page; + /* * QEMU userspace and the guest each have their own FPU state. * In vcpu_run, we switch between the user and guest FPU contexts. diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 928627a2ac8c..c214633453a8 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4978,7 +4978,7 @@ static void init_vmcs(struct kvm_vcpu *vcpu) vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP); if (enable_pml) { - vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); + vmcs_write64(PML_ADDRESS, page_to_phys(vcpu->arch.pml_page)); vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); } @@ -6418,17 +6418,16 @@ void vmx_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code) *error_code = 0; } -static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx) +static void vmx_destroy_pml_buffer(struct kvm_vcpu *vcpu) { - if (vmx->pml_pg) { - __free_page(vmx->pml_pg); - vmx->pml_pg = NULL; + if (vcpu->arch.pml_page) { + __free_page(vcpu->arch.pml_page); + vcpu->arch.pml_page = NULL; } } static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) { - struct vcpu_vmx *vmx = to_vmx(vcpu); u16 pml_idx, pml_tail_index; u64 *pml_buf; int i; @@ -6451,7 +6450,7 @@ static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) * Read the entries in the same order they were written, to ensure that * the dirty ring is filled in the same order the CPU wrote them. */ - pml_buf = page_address(vmx->pml_pg); + pml_buf = page_address(vcpu->arch.pml_page); for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) { u64 gpa; @@ -7648,7 +7647,7 @@ void vmx_vcpu_free(struct kvm_vcpu *vcpu) struct vcpu_vmx *vmx = to_vmx(vcpu); if (enable_pml) - vmx_destroy_pml_buffer(vmx); + vmx_destroy_pml_buffer(vcpu); free_vpid(vmx->vpid); nested_vmx_free_vcpu(vcpu); free_loaded_vmcs(vmx->loaded_vmcs); @@ -7680,8 +7679,8 @@ int vmx_vcpu_create(struct kvm_vcpu *vcpu) * for the guest), etc. */ if (enable_pml) { - vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); - if (!vmx->pml_pg) + vcpu->arch.pml_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!vcpu->arch.pml_page) goto free_vpid; } @@ -7752,7 +7751,7 @@ int vmx_vcpu_create(struct kvm_vcpu *vcpu) free_vmcs: free_loaded_vmcs(vmx->loaded_vmcs); free_pml: - vmx_destroy_pml_buffer(vmx); + vmx_destroy_pml_buffer(vcpu); free_vpid: free_vpid(vmx->vpid); return err; diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index dc8517f15bc4..6bcca3487c59 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -265,13 +265,6 @@ struct vcpu_vmx { unsigned int ple_window; bool ple_window_dirty; - /* Support for PML */ -#define PML_LOG_NR_ENTRIES 512 - /* PML is written backwards: this is the first entry written by the CPU */ -#define PML_HEAD_INDEX (PML_LOG_NR_ENTRIES-1) - - struct page *pml_pg; - /* apic deadline value in host tsc */ u64 hv_deadline_tsc; diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 0f5919b092e4..05c31ee984c9 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -900,4 +900,9 @@ int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, int cpl, int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); +/* Support for PML */ +#define PML_LOG_NR_ENTRIES 512 +/* PML is written backwards: this is the first entry written by the CPU */ +#define PML_HEAD_INDEX (PML_LOG_NR_ENTRIES-1) + #endif From e1e1934fea48bf4193dcd3c2778421ce5610f5ee Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Mon, 7 Sep 2026 06:39:01 +0000 Subject: [PATCH 20/45] KVM: x86: Carve out PML flush routine Move the PML (Page Modification Logging) buffer flushing logic from VMX-specific code to common x86 KVM code to enable reuse by SVM and avoid code duplication. The AMD SVM PML implementations share the same behavior as VMX PML: 1) The PML buffer is a 4K page with 512 entries 2) Hardware records dirty GPAs in reverse order (from index 511 to 0) 3) Hardware clears bits 11:0 when recording GPAs No functional change intended for VMX, except tone down the WARN_ON() to WARN_ON_ONCE() for the page alignment check. If hardware exhibits this behavior once, it's likely to occur repeatedly, so use WARN_ON_ONCE() to avoid log flooding while still capturing the unexpected condition. The refactoring prepares for SVM to leverage the same PML flushing implementation. Reviewed-by: Kai Huang Signed-off-by: Nikunj A Dadhania Link: https://patch.msgid.link/20260907063906.1964557-4-nikunj@amd.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/vmx.c | 26 ++------------------------ arch/x86/kvm/x86.c | 31 +++++++++++++++++++++++++++++++ arch/x86/kvm/x86.h | 2 ++ 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index c214633453a8..2650b6e299af 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6428,37 +6428,15 @@ static void vmx_destroy_pml_buffer(struct kvm_vcpu *vcpu) static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) { - u16 pml_idx, pml_tail_index; - u64 *pml_buf; - int i; + u16 pml_idx; pml_idx = vmcs_read16(GUEST_PML_INDEX); /* Do nothing if PML buffer is empty */ if (pml_idx == PML_HEAD_INDEX) return; - /* - * PML index always points to the next available PML buffer entity - * unless PML log has just overflowed. - */ - pml_tail_index = (pml_idx >= PML_LOG_NR_ENTRIES) ? 0 : pml_idx + 1; - /* - * PML log is written backwards: the CPU first writes the entry 511 - * then the entry 510, and so on. - * - * Read the entries in the same order they were written, to ensure that - * the dirty ring is filled in the same order the CPU wrote them. - */ - pml_buf = page_address(vcpu->arch.pml_page); - - for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) { - u64 gpa; - - gpa = pml_buf[i]; - WARN_ON(gpa & (PAGE_SIZE - 1)); - kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); - } + kvm_flush_pml_buffer(vcpu, pml_idx); /* reset PML index */ vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 28172424c502..85ce54fa268b 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3941,6 +3941,37 @@ void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) kvm_vcpu_kick(vcpu); } +void kvm_flush_pml_buffer(struct kvm_vcpu *vcpu, u16 pml_idx) +{ + u16 pml_tail_index; + u64 *pml_buf; + int i; + + /* + * PML index always points to the next available PML buffer entity + * unless PML log has just overflowed. + */ + pml_tail_index = (pml_idx >= PML_LOG_NR_ENTRIES) ? 0 : pml_idx + 1; + + /* + * PML log is written backwards: the CPU first writes the entry 511 + * then the entry 510, and so on. + * + * Read the entries in the same order they were written, to ensure that + * the dirty ring is filled in the same order the CPU wrote them. + */ + pml_buf = page_address(vcpu->arch.pml_page); + + for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) { + u64 gpa; + + gpa = pml_buf[i]; + WARN_ON_ONCE(gpa & (PAGE_SIZE - 1)); + kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); + } +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_flush_pml_buffer); + int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) { diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 05c31ee984c9..2d747c20965c 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -905,4 +905,6 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); /* PML is written backwards: this is the first entry written by the CPU */ #define PML_HEAD_INDEX (PML_LOG_NR_ENTRIES-1) +void kvm_flush_pml_buffer(struct kvm_vcpu *vcpu, u16 pml_idx); + #endif From 8ac111b6dc4a523c406f5d827afb5379fbd4e025 Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Mon, 7 Sep 2026 06:39:02 +0000 Subject: [PATCH 21/45] KVM: VMX: Use cpu_dirty_log_size instead of enable_pml for PML checks Replace the enable_pml check with cpu_dirty_log_size in VMX PML code to determine whether PML is enabled on a per-VM basis. The enable_pml module parameter is a global setting that doesn't reflect per-VM capabilities, whereas cpu_dirty_log_size accurately indicates whether a specific VM has PML enabled. For example, TDX VMs don't yet support PML. Using cpu_dirty_log_size ensures the check correctly reflects this, while enable_pml would incorrectly indicate PML is available. This also improves consistency with kvm_mmu_update_cpu_dirty_logging(), which already uses cpu_dirty_log_size to determine PML enablement. Suggested-by: Kai Huang Reviewed-by: Kai Huang Reviewed-by: Pankaj Gupta Signed-off-by: Nikunj A Dadhania Link: https://patch.msgid.link/20260907063906.1964557-5-nikunj@amd.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/vmx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 2650b6e299af..d11642e4da95 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -8421,7 +8421,7 @@ void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (WARN_ON_ONCE(!enable_pml)) + if (WARN_ON_ONCE(!vcpu->kvm->arch.cpu_dirty_log_size)) return; guard(vmx_vmcs01)(vcpu); From 8301c447b24031b73d51ba04de04636c59317828 Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Mon, 7 Sep 2026 06:39:03 +0000 Subject: [PATCH 22/45] KVM: x86: Carve out common dirty logging update Move the PML enable/disable decision logic from the VMX into a new common wrapper kvm_update_cpu_dirty_logging() in x86.c to avoid code duplication. Update the update_cpu_dirty_logging() signature to accept a bool enable parameter accordingly. Note, this subtly changes the ordering for VMX: currently guard(vmx_vmcs01) is taken before the atomic_read of nr_memslots_dirty_logging; after this refactoring the atomic_read happens in common code before guard(vmx_vmcs01) is taken. The ordering flip is harmless as a concurrent change to nr_memslots_dirty_logging will queue another update request. No functional change intended for VMX, restructuring to prepare for SVM PML support. Suggested-by: Sean Christopherson Signed-off-by: Nikunj A Dadhania Link: https://patch.msgid.link/20260907063906.1964557-6-nikunj@amd.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/vmx/main.c | 4 ++-- arch/x86/kvm/vmx/vmx.c | 12 ++---------- arch/x86/kvm/vmx/vmx.h | 2 +- arch/x86/kvm/vmx/x86_ops.h | 2 +- arch/x86/kvm/x86.c | 17 ++++++++++++++++- 6 files changed, 23 insertions(+), 16 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 6fce5a0061c2..3335c4b1afad 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1673,7 +1673,7 @@ struct kvm_x86_ops { struct x86_exception *exception); void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu); - void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu); + void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu, bool enable); void (*vcpu_blocking)(struct kvm_vcpu *vcpu); void (*vcpu_unblocking)(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 0ff3230fd95e..797206912ae9 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -108,7 +108,7 @@ static void vt_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vmx_vcpu_load(vcpu, cpu); } -static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) +static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable) { /* * Basic TDX does not support feature PML. KVM does not enable PML in @@ -117,7 +117,7 @@ static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) if (WARN_ON_ONCE(is_td_vcpu(vcpu))) return; - vmx_update_cpu_dirty_logging(vcpu); + vmx_update_cpu_dirty_logging(vcpu, enable); } static void vt_prepare_switch_to_guest(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index d11642e4da95..c9f20144a70a 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -8417,21 +8417,13 @@ static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit } #endif -void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (WARN_ON_ONCE(!vcpu->kvm->arch.cpu_dirty_log_size)) - return; - guard(vmx_vmcs01)(vcpu); - /* - * Note, nr_memslots_dirty_logging can be changed concurrent with this - * code, but in that case another update request will be made and so - * the guest will never run with a stale PML value. - */ - if (atomic_read(&vcpu->kvm->nr_memslots_dirty_logging)) + if (enable) secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_ENABLE_PML); else secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML); diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 6bcca3487c59..5f505ad14dde 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -390,7 +390,7 @@ u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu); gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); -void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu); +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable); u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated); bool vmx_is_valid_debugctl(struct kvm_vcpu *vcpu, u64 data, bool host_initiated); diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h index cdb38d940cfb..3351b91e924e 100644 --- a/arch/x86/kvm/vmx/x86_ops.h +++ b/arch/x86/kvm/vmx/x86_ops.h @@ -118,7 +118,7 @@ u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu); u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu); void vmx_write_tsc_offset(struct kvm_vcpu *vcpu); void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu); -void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu); +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable); #ifdef CONFIG_X86_64 int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, bool *expired); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 85ce54fa268b..be117e09cdd2 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8073,6 +8073,21 @@ static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) kvm_x86_call(set_apic_access_page_addr)(vcpu); } +static void kvm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) +{ + /* + * Note, nr_memslots_dirty_logging can be changed concurrent with this + * code, but in that case another update request will be made and so + * the guest will never run with a stale PML value. + */ + bool enable = atomic_read(&vcpu->kvm->nr_memslots_dirty_logging); + + if (WARN_ON_ONCE(!vcpu->kvm->arch.cpu_dirty_log_size)) + return; + + kvm_x86_call(update_cpu_dirty_logging)(vcpu, enable); +} + /* * Called within kvm->srcu read side. * Returns 1 to let vcpu_run() continue the guest execution loop without @@ -8238,7 +8253,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) kvm_x86_call(recalc_intercepts)(vcpu); if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu)) - kvm_x86_call(update_cpu_dirty_logging)(vcpu); + kvm_update_cpu_dirty_logging(vcpu); if (kvm_check_request(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, vcpu)) { kvm_vcpu_reset(vcpu, true); From 732cf7ec3224f05f29258c77648d11edc489130c Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Mon, 7 Sep 2026 06:39:04 +0000 Subject: [PATCH 23/45] x86/cpufeatures: Add Page modification logging Page modification logging(PML) is a hardware feature designed to track guest modified memory pages. PML enables the hypervisor to identify which pages in a guest's memory have been changed since the last checkpoint or during live migration. The PML feature is advertised via CPUID leaf 0x8000000A ECX[4] bit. Acked-by: Borislav Petkov (AMD) Signed-off-by: Nikunj A Dadhania Link: https://patch.msgid.link/20260907063906.1964557-7-nikunj@amd.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/kernel/cpu/scattered.c | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index f70ee74b5f92..cbfcc66a18e9 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -238,6 +238,7 @@ // free: was #define X86_FEATURE_PVUNLOCK ( 8*32+20) /* PV unlock function */ #define X86_FEATURE_VCPUPREEMPT ( 8*32+21) /* PV vcpu_is_preempted function */ #define X86_FEATURE_TDX_GUEST ( 8*32+22) /* "tdx_guest" Intel Trust Domain Extensions Guest */ +#define X86_FEATURE_PML ( 8*32+23) /* AMD Page Modification logging */ /* Intel-defined CPU features, CPUID level 0x00000007:0 (EBX), word 9 */ #define X86_FEATURE_FSGSBASE ( 9*32+ 0) /* "fsgsbase" RDFSBASE, WRFSBASE, RDGSBASE, WRGSBASE instructions*/ diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index 8665a6474806..b18cc9c88dd5 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -55,6 +55,7 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_AMD_FAST_CPPC, CPUID_EDX, 15, 0x80000007, 0 }, { X86_FEATURE_CPPC_PERF_PRIO, CPUID_EDX, 16, 0x80000007, 0 }, { X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 }, + { X86_FEATURE_PML, CPUID_ECX, 4, 0x8000000a, 0 }, { X86_FEATURE_X2AVIC_EXT, CPUID_ECX, 6, 0x8000000a, 0 }, { X86_FEATURE_COHERENCY_SFW_NO, CPUID_EBX, 31, 0x8000001f, 0 }, { X86_FEATURE_SMBA, CPUID_EBX, 2, 0x80000020, 0 }, From 5ebaa85a746e886ca23f03b8d2990a3c16d7f01f Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Mon, 7 Sep 2026 06:39:05 +0000 Subject: [PATCH 24/45] KVM: SVM: Use BIT_ULL for 64-bit misc_ctl bit definitions Replace BIT() with BIT_ULL() for SVM misc control bit definitions since misc_ctl is a 64-bit field in the VMCB control area structure. No functional change intended. Signed-off-by: Nikunj A Dadhania Link: https://patch.msgid.link/20260907063906.1964557-8-nikunj@amd.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/svm.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index aa63431ba92c..f199c52709df 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -240,10 +240,10 @@ struct __attribute__ ((__packed__)) vmcb_control_area { #define SVM_IOIO_SIZE_MASK (7 << SVM_IOIO_SIZE_SHIFT) #define SVM_IOIO_ASIZE_MASK (7 << SVM_IOIO_ASIZE_SHIFT) -#define SVM_MISC_ENABLE_NP BIT(0) -#define SVM_MISC_ENABLE_SEV BIT(1) -#define SVM_MISC_ENABLE_SEV_ES BIT(2) -#define SVM_MISC_ENABLE_GMET BIT(3) +#define SVM_MISC_ENABLE_NP BIT_ULL(0) +#define SVM_MISC_ENABLE_SEV BIT_ULL(1) +#define SVM_MISC_ENABLE_SEV_ES BIT_ULL(2) +#define SVM_MISC_ENABLE_GMET BIT_ULL(3) #define SVM_MISC2_ENABLE_V_LBR BIT_ULL(0) #define SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE BIT_ULL(1) From 0d7fbc059f33527caadf1f0387d3eea61962c402 Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Mon, 7 Sep 2026 06:39:06 +0000 Subject: [PATCH 25/45] KVM: SVM: Add Page modification logging support Currently, dirty logging relies on write protecting guest memory and marking dirty GFNs during subsequent write faults. This method works but incurs overhead due to additional write faults for each dirty GFN. Implement support for the Page Modification Logging (PML) feature, a hardware-assisted method for efficient dirty logging. PML automatically logs dirty GPA[51:12] to a 4K buffer when the CPU sets NPT D-bits. Two new VMCB fields are utilized: PML_ADDR and PML_INDEX. The PML_INDEX is initialized to 511 (8 bytes per GPA entry), and the CPU decreases the PML_INDEX after logging each GPA. When the PML buffer is full, a VMEXIT(PML_FULL) with exit code 0x407 is generated. Since PML_INDEX in the VMCB control area remains valid after an intercepted SHUTDOWN, only initialize it on reset and leave it unchanged on INIT to avoid discarding already-logged entries that haven't been flushed. PML operates on guest physical addresses at the NPT level, tracking D-bit updates in page tables rather than memory content. This allows it to work identically for normal and confidential computing guests (SEV/SEV-ES/SEV-SNP), enabling cpu_dirty_log_size to be set uniformly for all AMD VMs without special-casing encrypted guests. Use vmcb01 directly when updating PML controls to ensure L1's state remains correct, as svm->vmcb points to vmcb02 when L2 is active. PML is not enabled in hardware for nested guests; treat PML_FULL as unexpected exits. Add a new module parameter to enable/disable PML, and enable it by default when supported. Acked-by: Kai Huang Signed-off-by: Nikunj A Dadhania Link: https://patch.msgid.link/20260907063906.1964557-9-nikunj@amd.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/svm.h | 6 +- arch/x86/include/uapi/asm/svm.h | 2 + arch/x86/kvm/svm/nested.c | 14 +++++ arch/x86/kvm/svm/sev.c | 2 +- arch/x86/kvm/svm/svm.c | 106 +++++++++++++++++++++++++++++++- arch/x86/kvm/svm/svm.h | 1 + 6 files changed, 127 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index f199c52709df..e0a7549a7c72 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -165,7 +165,10 @@ struct __attribute__ ((__packed__)) vmcb_control_area { u8 reserved_9[22]; u64 allowed_sev_features; /* Offset 0x138 */ u64 guest_sev_features; /* Offset 0x140 */ - u8 reserved_10[664]; + u8 reserved_10[128]; + u64 pml_addr; /* Offset 0x1c8 */ + u16 pml_index; /* Offset 0x1d0 */ + u8 reserved_11[526]; /* * Offset 0x3e0, 32 bytes reserved * for use by hypervisor/software. @@ -244,6 +247,7 @@ struct __attribute__ ((__packed__)) vmcb_control_area { #define SVM_MISC_ENABLE_SEV BIT_ULL(1) #define SVM_MISC_ENABLE_SEV_ES BIT_ULL(2) #define SVM_MISC_ENABLE_GMET BIT_ULL(3) +#define SVM_MISC_ENABLE_PML BIT_ULL(11) #define SVM_MISC2_ENABLE_V_LBR BIT_ULL(0) #define SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE BIT_ULL(1) diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index 010a45c9f614..e80676185092 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h @@ -101,6 +101,7 @@ #define SVM_EXIT_AVIC_INCOMPLETE_IPI 0x401 #define SVM_EXIT_AVIC_UNACCELERATED_ACCESS 0x402 #define SVM_EXIT_VMGEXIT 0x403 +#define SVM_EXIT_PML_FULL 0x407 /* SEV-ES software-defined VMGEXIT events */ #define SVM_VMGEXIT_MMIO_READ 0x80000001ull @@ -236,6 +237,7 @@ { SVM_EXIT_AVIC_INCOMPLETE_IPI, "avic_incomplete_ipi" }, \ { SVM_EXIT_AVIC_UNACCELERATED_ACCESS, "avic_unaccelerated_access" }, \ { SVM_EXIT_VMGEXIT, "vmgexit" }, \ + { SVM_EXIT_PML_FULL, "pml_full" }, \ { SVM_VMGEXIT_MMIO_READ, "vmgexit_mmio_read" }, \ { SVM_VMGEXIT_MMIO_WRITE, "vmgexit_mmio_write" }, \ { SVM_VMGEXIT_NMI_COMPLETE, "vmgexit_nmi_complete" }, \ diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 23d29597d6bf..ddbded0dee87 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -905,6 +905,13 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa; vmcb_mark_dirty(vmcb02, VMCB_PERM_MAP); + /* + * PML is never enabled in hardware for L2. Make sure that an + * unexpected PML write would trigger a PML_FULL VM-Exit. + */ + if (pml) + vmcb02->control.pml_index = -1; + /* * Stash vmcb02's counter if the guest hasn't moved past the guilty * instruction; otherwise, reset the counter to '0'. @@ -1824,6 +1831,13 @@ int nested_svm_exit_special(struct vcpu_svm *svm) if (nested_svm_is_l2_tlb_flush_hcall(vcpu)) return NESTED_EXIT_HOST; break; + case SVM_EXIT_PML_FULL: + /* + * All PML full exits are handled by KVM. KVM emulates PML in + * software for L1, but never enables PML in hardware on behalf + * of L1. + */ + return NESTED_EXIT_HOST; default: break; } diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 5705723f1f41..cfe990f80504 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -4994,7 +4994,7 @@ struct page *snp_safe_alloc_page_node(int node, gfp_t gfp) * Allocate an SNP-safe page to workaround the SNP erratum where * the CPU will incorrectly signal an RMP violation #PF if a * hugepage (2MB or 1GB) collides with the RMP entry of a - * 2MB-aligned VMCB, VMSA, or AVIC backing page. + * 2MB-aligned VMCB, VMSA, PML or AVIC backing page. * * Allocate one extra page, choose a page which is not * 2MB-aligned, and free the other. diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 351b9c7233f0..842c16483f6c 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -179,6 +179,9 @@ module_param(vnmi, bool, 0444); module_param(enable_mediated_pmu, bool, 0444); +bool __ro_after_init pml = true; +module_param(pml, bool, 0444); + static bool __ro_after_init svm_gp_erratum_intercept = true; static u8 rsm_ins_bytes[] = "\x0f\xaa"; @@ -1267,6 +1270,23 @@ static void init_vmcb(struct kvm_vcpu *vcpu, bool init_event) if (vcpu->kvm->arch.bus_lock_detection_enabled) svm_set_intercept(svm, INTERCEPT_BUSLOCK); + if (pml) { + /* + * Populate the page address and index here, PML is enabled + * when dirty logging is enabled on the memslot through + * svm_update_cpu_dirty_logging() + */ + control->pml_addr = (u64)__sme_set(page_to_phys(vcpu->arch.pml_page)); + + /* + * The VMCB control area remains valid after an intercepted SHUTDOWN + * (AMD APM Rev 3.45+), so only initialize PML index on reset to avoid + * discarding already-logged entries that haven't been flushed. + */ + if (!init_event) + control->pml_index = PML_HEAD_INDEX; + } + if (is_sev_guest(vcpu)) sev_init_vmcb(svm, init_event); @@ -1326,9 +1346,15 @@ static int svm_vcpu_create(struct kvm_vcpu *vcpu) if (!vmcb01_page) goto out; + if (pml) { + vcpu->arch.pml_page = snp_safe_alloc_page(); + if (!vcpu->arch.pml_page) + goto error_free_vmcb_page; + } + err = sev_vcpu_create(vcpu); if (err) - goto error_free_vmcb_page; + goto error_free_pml_page; err = avic_init_vcpu(svm); if (err) @@ -1353,6 +1379,9 @@ static int svm_vcpu_create(struct kvm_vcpu *vcpu) error_free_sev: sev_free_vcpu(vcpu); +error_free_pml_page: + if (vcpu->arch.pml_page) + __free_page(vcpu->arch.pml_page); error_free_vmcb_page: __free_page(vmcb01_page); out: @@ -1370,6 +1399,9 @@ static void svm_vcpu_free(struct kvm_vcpu *vcpu) sev_free_vcpu(vcpu); + if (pml && vcpu->arch.pml_page) + __free_page(vcpu->arch.pml_page); + __free_page(__sme_pa_to_page(svm->vmcb01.pa)); svm_vcpu_free_msrpm(svm->msrpm); } @@ -3344,6 +3376,45 @@ static int vmmcall_interception(struct kvm_vcpu *vcpu) return kvm_emulate_hypercall(vcpu); } +static void svm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb *vmcb01 = svm->vmcb01.ptr; + + if (enable) + vmcb01->control.misc_ctl |= SVM_MISC_ENABLE_PML; + else + vmcb01->control.misc_ctl &= ~SVM_MISC_ENABLE_PML; + + vmcb_mark_dirty(vmcb01, VMCB_NPT); +} + +static void svm_flush_pml_buffer(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb_control_area *control = &svm->vmcb->control; + + /* Do nothing if PML buffer is empty */ + if (control->pml_index == PML_HEAD_INDEX) + return; + + kvm_flush_pml_buffer(vcpu, control->pml_index); + + /* Reset the PML index */ + control->pml_index = PML_HEAD_INDEX; +} + +static int pml_full_interception(struct kvm_vcpu *vcpu) +{ + trace_kvm_pml_full(vcpu->vcpu_id); + + /* + * PML buffer is already flushed at the beginning of svm_handle_exit(). + * Nothing to do here. + */ + return 1; +} + static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { [SVM_EXIT_READ_CR0] = cr_interception, [SVM_EXIT_READ_CR3] = cr_interception, @@ -3421,6 +3492,7 @@ static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { #ifdef CONFIG_KVM_AMD_SEV [SVM_EXIT_VMGEXIT] = sev_handle_vmgexit, #endif + [SVM_EXIT_PML_FULL] = pml_full_interception, }; static void dump_vmcb(struct kvm_vcpu *vcpu) @@ -3470,8 +3542,14 @@ static void dump_vmcb(struct kvm_vcpu *vcpu) pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2); pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info); pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err); - pr_err("%-20s%lld\n", "misc_ctl:", control->misc_ctl); + pr_err("%-20s%llx\n", "misc_ctl:", control->misc_ctl); pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3); + + if (pml) { + pr_err("%-20s%016llx\n", "pml_addr:", control->pml_addr); + pr_err("%-20s%04x\n", "pml_index:", control->pml_index); + } + pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar); pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa); pr_err("%-20s%08x\n", "event_inj:", control->event_inj); @@ -3649,6 +3727,13 @@ int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 __exit_code) (u64)exit_code != __exit_code) goto unexpected_vmexit; + /* + * PML is never enabled when running L2, bail immediately if a PML full + * exit occurs as something is horribly wrong. + */ + if (unlikely(is_guest_mode(vcpu) && exit_code == SVM_EXIT_PML_FULL)) + goto unexpected_vmexit; + #ifdef CONFIG_MITIGATION_RETPOLINE if (exit_code == SVM_EXIT_MSR) return msr_interception(vcpu); @@ -3717,6 +3802,14 @@ static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) struct vcpu_svm *svm = to_svm(vcpu); struct kvm_run *kvm_run = vcpu->run; + /* + * Opportunistically flush the PML buffer on VM exit. This keeps the + * dirty bitmap current by processing logged GPAs rather than waiting for + * PML_FULL exit. + */ + if (vcpu->kvm->arch.cpu_dirty_log_size && !is_guest_mode(vcpu)) + svm_flush_pml_buffer(vcpu); + if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE)) return 0; @@ -5309,6 +5402,9 @@ static int svm_vm_init(struct kvm *kvm) if (!pause_filter_count || !pause_filter_thresh) kvm_disable_exits(kvm, KVM_X86_DISABLE_EXITS_PAUSE); + if (pml) + kvm->arch.cpu_dirty_log_size = PML_LOG_NR_ENTRIES; + svm_srso_vm_init(); return 0; } @@ -5427,6 +5523,8 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .check_intercept = svm_check_intercept, .handle_exit_irqoff = svm_handle_exit_irqoff, + .update_cpu_dirty_logging = svm_update_cpu_dirty_logging, + .deliver_interrupt = svm_deliver_interrupt, .pi_update_irte = avic_pi_update_irte, .setup_mce = svm_setup_mce, @@ -5687,6 +5785,10 @@ static __init int svm_hardware_setup(void) nrips = nrips && boot_cpu_has(X86_FEATURE_NRIPS); + pml = pml && npt_enabled && cpu_feature_enabled(X86_FEATURE_PML); + if (pml) + pr_info("Page modification logging supported\n"); + if (lbrv) { if (!boot_cpu_has(X86_FEATURE_LBRV)) lbrv = false; diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index e958943b8162..f566dae44bb0 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -53,6 +53,7 @@ extern int vgif; extern bool intercept_smi; extern bool vnmi; extern int lbrv; +extern bool pml; extern int tsc_aux_uret_slot __ro_after_init; From 1c24893fdb4df1e8713d921a849ef14626e50954 Mon Sep 17 00:00:00 2001 From: Luka Absandze Date: Sun, 20 Sep 2026 15:54:53 +0000 Subject: [PATCH 26/45] KVM: x86/pmu: Don't retry a counter whose config was rejected kvm_pmu_handle_event() re-arms the reprogram bit for every failed reprogram, on the assumption that the failure is transient and a later refresh will succeed. That is true for contention, e.g. the -EBUSY from x86_reserve_hardware(), but not for a configuration the host PMU driver rejects outright. A rejected config can never succeed on retry, so the counter is reprogrammed on every PMU refresh for as long as the guest leaves it enabled, and every attempt fails the same way. Re-arm only for the errnos that indicate a transient condition, i.e. -EBUSY and -ENOMEM. Suggested-by: Sean Christopherson Link: https://lore.kernel.org/all/20260916123315.89042-1-absandze@amazon.de/ Signed-off-by: Luka Absandze Reviewed-by: Sandipan Das Link: https://patch.msgid.link/20260920155453.10836-1-absandze@amazon.de Signed-off-by: Sean Christopherson --- arch/x86/kvm/pmu.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index a7d60c8785cd..03a470c49a74 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -662,7 +662,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) DECLARE_BITMAP(bitmap, X86_PMC_IDX_MAX); struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); struct kvm_pmc *pmc; - int bit; + int bit, r; bitmap_copy(bitmap, pmu->reprogram_pmi, X86_PMC_IDX_MAX); @@ -676,12 +676,14 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) kvm_for_each_pmc(pmu, pmc, bit, bitmap) { /* - * If reprogramming fails, e.g. due to contention, re-set the - * reprogram bit, i.e. opportunistically try again on the next - * PMU refresh. Don't make a new request as doing so can stall - * the guest if reprogramming repeatedly fails. + * If reprogramming fails on a transient condition, e.g. due to + * contention, re-set the reprogram bit, i.e. opportunistically + * try again on the next PMU refresh. Don't make a new request + * as doing so can stall the guest if reprogramming repeatedly + * fails. */ - if (reprogram_counter(pmc)) + r = reprogram_counter(pmc); + if (r == -EBUSY || r == -ENOMEM) set_bit(pmc->idx, pmu->reprogram_pmi); } From e70f812ccdd6f45b002ff054b0108bbf8db1c0ed Mon Sep 17 00:00:00 2001 From: Kyle Zeng Date: Thu, 17 Sep 2026 20:36:58 -0700 Subject: [PATCH 27/45] KVM: x86: Restrict saved GPA writes to hardware write faults A GPA supplied by a hardware page fault describes the access that faulted, not an arbitrary instruction decoded afterwards. A guest can change an MMIO read into a store before KVM fetches the instruction. The saved-GPA shortcut then skips the write-aware guest page-table walk and can issue a write through a guest-read-only mapping. Carry hardware write-fault information into the emulator and retain it alongside the saved GPA. For guests without protected page tables, only reuse that GPA for a write when hardware reported a write access. Reads and faults with unknown access direction do not authorize an emulated write; fall back to the existing permission-aware translation in those cases. Keep the authorization across MMIO completion without decoding a new instruction, and reset it when initializing a fresh emulator context. This also covers writes reached through the cmpxchg fallback. Preserve the shortcut for hardware-reported writes and for the read side of read-modify-write emulation after such a fault. For guests with protected page tables, preserve the existing behavior and always allow writes through the saved GPA. An RMW instruction can fault on its initial read, and KVM cannot walk the protected guest page tables to check the subsequent write. Track this property in common KVM state for SEV and TDX. Also account for legacy SEV initialization and encryption-context copies and moves. Fixes: 0f89b207b04a ("kvm: svm: Use the hardware provided GPA instead of page walk") Reported-by: Oleg Boiko Assisted-by: Codex:gpt-6-astra Signed-off-by: Kyle Zeng Link: https://patch.msgid.link/20260918033658.21842-1-kylebot@openai.com [sean: clear has_protected_page_tables on the source after migration] Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/kvm_emulate.h | 4 ++-- arch/x86/kvm/mmu/mmu.c | 3 +++ arch/x86/kvm/svm/sev.c | 5 +++++ arch/x86/kvm/vmx/tdx.c | 1 + arch/x86/kvm/x86.c | 17 ++++++++++++++--- arch/x86/kvm/x86.h | 3 +++ 7 files changed, 29 insertions(+), 5 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 3335c4b1afad..2159f1ba4f33 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1168,6 +1168,7 @@ struct kvm_arch { u8 mmu_valid_gen; u8 vm_type; bool has_private_mem; + bool has_protected_page_tables; bool has_protected_state; bool has_protected_eoi; bool has_protected_pmu; diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h index 3e375af15c03..10886bea82c9 100644 --- a/arch/x86/kvm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h @@ -354,8 +354,8 @@ struct x86_emulate_ctxt { bool have_exception; struct x86_exception exception; - /* GPA available */ - bool gpa_available; + /* Saved GPA and permitted emulated accesses. */ + u64 gpa_access; gpa_t gpa_val; /* diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index f037fa665414..21a9a11b9ce6 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -6638,6 +6638,9 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err return r; emulate: + if (error_code & PFERR_WRITE_MASK) + emulation_type |= EMULTYPE_PF_WRITE; + return x86_emulate_instruction(vcpu, cr2_or_gpa, emulation_type, insn, insn_len); } diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index cfe990f80504..dd0943cd0133 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -560,6 +560,7 @@ static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp, INIT_LIST_HEAD(&sev->regions_list); INIT_LIST_HEAD(&sev->mirror_vms); sev->need_init = false; + kvm->arch.has_protected_page_tables = true; kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_SEV); @@ -2041,12 +2042,14 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) dst->pages_locked = src->pages_locked; dst->es_active = src->es_active; dst->vmsa_features = src->vmsa_features; + dst_kvm->arch.has_protected_page_tables = true; src->asid = 0; src->active = false; src->handle = 0; src->pages_locked = 0; src->es_active = false; + src_kvm->arch.has_protected_page_tables = false; list_cut_before(&dst->regions_list, &src->regions_list, &src->regions_list); @@ -2907,6 +2910,7 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) mutex_unlock(&sev_mirror_lock); mirror_sev->active = true; + kvm->arch.has_protected_page_tables = true; mirror_sev->asid = source_sev->asid; mirror_sev->fd = source_sev->fd; mirror_sev->es_active = source_sev->es_active; @@ -2965,6 +2969,7 @@ void sev_vm_init(struct kvm *kvm) kvm->arch.has_protected_state = true; fallthrough; case KVM_X86_SEV_VM: + kvm->arch.has_protected_page_tables = true; kvm->arch.pre_fault_allowed = !kvm->arch.has_private_mem; to_kvm_sev_info(kvm)->need_init = true; break; diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index a0bc9f818f43..2d2febd4a5ef 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -616,6 +616,7 @@ int tdx_vm_init(struct kvm *kvm) { struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); + kvm->arch.has_protected_page_tables = true; kvm->arch.has_protected_state = true; /* * TDX Module doesn't allow the hypervisor to modify the EOI-bitmap, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index be117e09cdd2..de77b222870e 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -5120,7 +5120,8 @@ static int emulator_read_write_onepage(unsigned long addr, void *val, * operation using rep will only have the initial GPA from the NPF * occurred. */ - if (ctxt->gpa_available && emulator_can_use_gpa(ctxt) && + if ((ctxt->gpa_access & (write ? ACC_WRITE_MASK : ACC_READ_MASK)) && + emulator_can_use_gpa(ctxt) && (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) { gpa = ctxt->gpa_val; ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write); @@ -5969,7 +5970,7 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu) kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); - ctxt->gpa_available = false; + ctxt->gpa_access = 0; ctxt->eflags = kvm_get_rflags(vcpu); ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0; @@ -6485,7 +6486,17 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, /* With shadow page tables, cr2 contains a GVA or nGPA. */ if (vcpu->arch.mmu->root_role.direct) { - ctxt->gpa_available = true; + ctxt->gpa_access = ACC_READ_MASK; + /* + * Always allow writes for guests with protected page + * tables, as KVM can't walk the guest's page tables, + * i.e. KVM can't get the RMW protections for a given + * GVA to see if the write side of a RMW operation + * should be allowed. + */ + if ((emulation_type & EMULTYPE_PF_WRITE) || + vcpu->kvm->arch.has_protected_page_tables) + ctxt->gpa_access |= ACC_WRITE_MASK; ctxt->gpa_val = cr2_or_gpa; } } else { diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 2d747c20965c..cbcf8e3ff31a 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -407,6 +407,8 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, * EMULTYPE_PF - Set when an intercepted #PF triggers the emulation, in which case * the CR2/GPA value pass on the stack is valid. * + * EMULTYPE_PF_WRITE - Set with EMULTYPE_PF when hardware reports a write access. + * * EMULTYPE_COMPLETE_USER_EXIT - Set when the emulator should update interruptibility * state and inject single-step #DBs after skipping * an instruction (after completing userspace I/O). @@ -445,6 +447,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, #define EMULTYPE_COMPLETE_USER_EXIT (1 << 7) #define EMULTYPE_WRITE_PF_TO_SP (1 << 8) #define EMULTYPE_SKIP_SOFT_INT (1 << 9) +#define EMULTYPE_PF_WRITE (1 << 10) #define EMULTYPE_SET_SOFT_INT_VECTOR(v) ((u32)((v) & 0xff) << 16) #define EMULTYPE_GET_SOFT_INT_VECTOR(e) (((e) >> 16) & 0xff) From 719a1d58bf43edd03b271df90b67c58c1e9a7da4 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 12:36:23 -0700 Subject: [PATCH 28/45] KVM: x86: Preserve DR6.BLD (Bus Lock Detect) when delivering #DB payload Preserve Bus Lock Detect if it isn't present in the payload. Unlike RTM, which is explicitly modified on every #DB, DR6.BLD is modified if and only if the #DB is due to a (coincident) Bus Lock Detect. Per Intel's SDM: Other debug exceptions do not modify this bit. And AMD's APM All other #DB exceptions leave DR6[BLD] unmodified, Restore the explicit RTM handling that was clobbered by commit 9a3ecd5e2aa1 ("KVM: X86: Rename DR6_INIT to DR6_ACTIVE_LOW"), as the semantics of any given bit are feature specific, i.e. are obviously not consistent for active-high versus active-low. In other words, while the bug was technically introduced when support for Bus Lock Detect was added, it's really that attempt to genericize handling of active-low bits that set KVM up to fail. Fixes: e8ea85fb280e ("KVM: X86: Add support for the emulation of DR6_BUS_LOCK bit") Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260930193634.3415089-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 52 ++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index de77b222870e..997a6b79c03c 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -399,30 +399,37 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, switch (ex->vector) { case DB_VECTOR: /* - * "Certain debug exceptions may clear bit 0-3. The - * remaining contents of the DR6 register are never - * cleared by the processor". + * DR6 is a mess. Reserved/unused bits are fixed-to-1, and so + * to maintain backwards compatibility with existing software, + * features that use previously-reserved bits have active-low + * semantics, i.e. clear the bit when the feature is present in + * the payload. + * + * Further complicating matters, some DR6 bits are preserved by + * hardware, while others are explicitly modified on every #DB. + * The trap bits are always set based on the payload, as is the + * RTM flag (but it's active low). All other bits are modified + * if and only if a relevant debug exception occurs, e.g. BD, + * BS, and BT are never cleared by hardware, and BLD is never + * set by hardware (when supported, excepting RESET). + * + * Lastly, the payload does NOT have active-low semantics, e.g. + * so that it's compatible VMX's pending debug exceptions and + * qualification fields, and to avoid bleeding the DR6 madness + * into other KVM code. + * + * To compute DR6: + * + * 1. "Reset" the bits that are modified on all #DBs + * 2. Clear active-low bits that are present in the payload. + * 3. Set active-high bits that are present in the payload. + * 4. Clear fixed-0 bits. + * 5. Set fixed-1 bits. */ vcpu->arch.dr6 &= ~DR_TRAP_BITS; - /* - * In order to reflect the #DB exception payload in guest - * dr6, three components need to be considered: active low - * bit, FIXED_1 bits and active high bits (e.g. DR6_BD, - * DR6_BS and DR6_BT) - * DR6_ACTIVE_LOW contains the FIXED_1 and active low bits. - * In the target guest dr6: - * FIXED_1 bits should always be set. - * Active low bits should be cleared if 1-setting in payload. - * Active high bits should be set if 1-setting in payload. - * - * Note, the payload is compatible with the pending debug - * exceptions/exit qualification under VMX, that active_low bits - * are active high in payload. - * So they need to be flipped for DR6. - */ - vcpu->arch.dr6 |= DR6_ACTIVE_LOW; - vcpu->arch.dr6 |= ex->payload; - vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW; + vcpu->arch.dr6 |= DR6_RTM; + vcpu->arch.dr6 &= ~(ex->payload & DR6_ACTIVE_LOW); + vcpu->arch.dr6 |= (ex->payload & ~DR6_ACTIVE_LOW); /* * The #DB payload is defined as compatible with the 'pending @@ -431,6 +438,7 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, * breakpoint), it is reserved and must be zero in DR6. */ vcpu->arch.dr6 &= ~BIT(12); + vcpu->arch.dr6 |= DR6_FIXED_1; break; case PF_VECTOR: vcpu->arch.cr2 = ex->payload; From cc551e8d4f77c278fd0bb0ed45d1103cb5a90889 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 12:36:24 -0700 Subject: [PATCH 29/45] KVM: x86: Rename kvm_dr6_fixed() => kvm_get_dr6_fixed_1() Rename kvm_dr6_fixed() to kvm_get_dr6_fixed_1() to make it more clear that the helper retrieves the fixed-1 DR6 bits for a given vCPU. No functional change intended. Link: https://patch.msgid.link/20260930193634.3415089-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/regs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c index fad31b59c622..62075443f6b9 100644 --- a/arch/x86/kvm/regs.c +++ b/arch/x86/kvm/regs.c @@ -772,7 +772,7 @@ void kvm_update_dr7(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); -static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) +static u64 kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) { u64 fixed = DR6_FIXED_1; @@ -798,7 +798,7 @@ int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) case 6: if (!kvm_dr6_valid(val)) return 1; /* #GP */ - vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu); + vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_get_dr6_fixed_1(vcpu); break; case 5: default: /* 7 */ From 28bb033664eef3c18c6acb2c075fc01d7ada9296 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 12:36:25 -0700 Subject: [PATCH 30/45] KVM: Return an "unsigned long", not "u64" for the fixed-1 DR6 bits Return an "unsigned long" instead of a "u64" from the helper to get the fixed-1 DR6 bits, as debug registers are natural width registers, not 64-bit registers. No functional change intended (the fixed-1 bits only set bits in 31:0, and the sole caller truncates the value to an "unsigned long"). Link: https://patch.msgid.link/20260930193634.3415089-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/regs.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c index 62075443f6b9..f74a29621661 100644 --- a/arch/x86/kvm/regs.c +++ b/arch/x86/kvm/regs.c @@ -772,15 +772,16 @@ void kvm_update_dr7(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); -static u64 kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) +static unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) { - u64 fixed = DR6_FIXED_1; + unsigned long fixed = DR6_FIXED_1; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM)) fixed |= DR6_RTM; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) fixed |= DR6_BUS_LOCK; + return fixed; } From 7b88577a0983af221a78fc66c56532eb97d8fd2f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 12:36:26 -0700 Subject: [PATCH 31/45] KVM: x86: Force fixed-1 bits in DR6 after synchronizing with hardware Set all fixed-1 bits in KVM's version of DR6 after synchronizing with hardware to paper over as much of the virtualization hole as possible. Because KVM dynamically disables DR interception, a guest can write any DR6 bit that isn't fixed in bare metal, even if the bit is supposed to be fixed from the guest's perspective. In addition to providing some amount of consistency in KVM, this will allow adding sanity checks that KVM never ends up with fixed-1 bits clear in its version of DR6. Link: https://patch.msgid.link/20260930193634.3415089-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/regs.c | 2 +- arch/x86/kvm/regs.h | 1 + arch/x86/kvm/x86.c | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c index f74a29621661..db43ade8ceb1 100644 --- a/arch/x86/kvm/regs.c +++ b/arch/x86/kvm/regs.c @@ -772,7 +772,7 @@ void kvm_update_dr7(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); -static unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) +unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) { unsigned long fixed = DR6_FIXED_1; diff --git a/arch/x86/kvm/regs.h b/arch/x86/kvm/regs.h index 447f0ec3e63e..3ca5cd8a8d95 100644 --- a/arch/x86/kvm/regs.h +++ b/arch/x86/kvm/regs.h @@ -62,6 +62,7 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8); int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val); unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr); +unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu); unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu); void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw); int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 997a6b79c03c..c2e24f4d2a1e 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8461,6 +8461,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) WARN_ON(vcpu->arch.switch_db_regs & KVM_DEBUGREG_AUTO_SWITCH); kvm_x86_call(sync_dirty_debug_regs)(vcpu); kvm_update_dr0123(vcpu); + vcpu->arch.dr6 |= kvm_get_dr6_fixed_1(vcpu); kvm_update_dr7(vcpu); } From c62ffee78db8837a6251af83798907af37b53ed4 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 12:36:27 -0700 Subject: [PATCH 32/45] KVM: x86: Set guest's fixed-1 DR6 bits when delivering #DB payload When delivering a #DB payload, set all DR6 bits that are fixed-1 from the guest's perspective, not just the host's perspective. Don't WARN on a "bad" payload (or existing DR6) even though KVM itself shouldn't set DR6 or synthesize a payload without the fixed-1 bits set, as userspace can provide a completely arbitrary payload via KVM_SET_VCPU_EVENTS (and "hardening" that flow is not worth the effort as there's no benefit in practice). Link: https://patch.msgid.link/20260930193634.3415089-6-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index c2e24f4d2a1e..1e24ae7b8879 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -438,7 +438,7 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, * breakpoint), it is reserved and must be zero in DR6. */ vcpu->arch.dr6 &= ~BIT(12); - vcpu->arch.dr6 |= DR6_FIXED_1; + vcpu->arch.dr6 |= kvm_get_dr6_fixed_1(vcpu); break; case PF_VECTOR: vcpu->arch.cr2 = ex->payload; From 0e2342b023801746ece4320d1ad0714f48ea6b1e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 12:36:28 -0700 Subject: [PATCH 33/45] KVM: x86: Kill off DR6_FIXED_1 to prevent future misuse Now that the only user of DR6_FIXED_1 is kvm_get_dr6_fixed_1(), drop the hardcoded macro to force all references to fixed-1 bits to use the vCPU- aware helper. Opportunistically reword the comment about DR6_ACTIVE_LOW to explain why treating fixed-1 bits as active-low is correct (barring a *very* egregious architectural goof). No functional change intended. Link: https://patch.msgid.link/20260930193634.3415089-7-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/regs.c | 2 +- arch/x86/kvm/regs.h | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c index db43ade8ceb1..0c76cf17e884 100644 --- a/arch/x86/kvm/regs.c +++ b/arch/x86/kvm/regs.c @@ -774,7 +774,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) { - unsigned long fixed = DR6_FIXED_1; + unsigned long fixed = DR6_ACTIVE_LOW & ~DR6_VOLATILE; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM)) fixed |= DR6_RTM; diff --git a/arch/x86/kvm/regs.h b/arch/x86/kvm/regs.h index 3ca5cd8a8d95..97c17b120a57 100644 --- a/arch/x86/kvm/regs.h +++ b/arch/x86/kvm/regs.h @@ -38,16 +38,14 @@ static_assert(!(KVM_POSSIBLE_CR0_GUEST_BITS & X86_CR0_PDPTR_BITS)); #define DR6_BT (1 << 15) #define DR6_RTM (1 << 16) /* - * DR6_ACTIVE_LOW combines fixed-1 and active-low bits. - * We can regard all the bits in DR6_FIXED_1 as active_low bits; - * they will never be 0 for now, but when they are defined - * in the future it will require no code change. + * DR6_ACTIVE_LOW combines fixed-1 and active-low bits (bits that are currently + * fixed-1 are guaranteed to have active-low semantics if they are ever used to + * support a new feature). * - * DR6_ACTIVE_LOW is also used as the init/reset value for DR6. + * Note, DR6_ACTIVE_LOW is also the INIT/RESET value for DR6. */ #define DR6_ACTIVE_LOW 0xffff0ff0 #define DR6_VOLATILE 0x0001e80f -#define DR6_FIXED_1 (DR6_ACTIVE_LOW & ~DR6_VOLATILE) #define DR7_BP_EN_MASK 0x000000ff #define DR7_GE (1 << 9) From 7faa73133e40c59d15abd7201145b75834b067c6 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 12:36:29 -0700 Subject: [PATCH 34/45] KVM: SVM: Add helper to query if LBR virtualization needs to be enabled Extract the logic for determining whether or not LBR virtualization needs to be enabled into a helper in anticipation of growing the list of features that depend on LBR virtualization. No functional change intended. Link: https://patch.msgid.link/20260930193634.3415089-8-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/svm.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 842c16483f6c..2bb1c6576ded 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -884,13 +884,22 @@ static void __svm_disable_lbrv(struct kvm_vcpu *vcpu) to_svm(vcpu)->vmcb->control.misc_ctl2 &= ~SVM_MISC2_ENABLE_V_LBR; } +static bool svm_need_lbr_virtualization(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + if (svm->vmcb->save.dbgctl & DEBUGCTLMSR_LBR) + return true; + + return is_guest_mode(vcpu) && guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && + (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR); +} + void svm_update_lbrv(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); bool current_enable_lbrv = svm->vmcb->control.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR; - bool enable_lbrv = (svm->vmcb->save.dbgctl & DEBUGCTLMSR_LBR) || - (is_guest_mode(vcpu) && guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && - (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR)); + bool enable_lbrv = svm_need_lbr_virtualization(vcpu); if (enable_lbrv && !current_enable_lbrv) __svm_enable_lbrv(vcpu); From 5cb0577b74971d578ab105f3b6a8f43fce1f3968 Mon Sep 17 00:00:00 2001 From: Shivansh Dhiman Date: Wed, 30 Sep 2026 12:36:30 -0700 Subject: [PATCH 35/45] KVM: nSVM: Disable LBRV in nested control cache when unsupported Clear SVM_MISC2_ENABLE_V_LBR in __nested_copy_vmcb_control_to_cache() when the vCPU does not support LBR Virtualization. This lets the cached value be consumed directly instead of re-checking X86_FEATURE_LBRV on every access. Suggested-by: Yosry Ahmed Signed-off-by: Shivansh Dhiman Link: https://patch.msgid.link/20260930193634.3415089-9-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 8 +++++--- arch/x86/kvm/svm/svm.c | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index ddbded0dee87..64464bdbe644 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -527,12 +527,16 @@ void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, /* Always clear misc_ctl bits that the guest cannot use */ to->misc_ctl = from->misc_ctl; + to->misc_ctl2 = from->misc_ctl2; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_NPT)) to->misc_ctl &= ~SVM_MISC_ENABLE_NP; if (!gmet_enabled || !guest_cpu_cap_has(vcpu, X86_FEATURE_GMET)) to->misc_ctl &= ~SVM_MISC_ENABLE_GMET; + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV)) + to->misc_ctl2 &= ~SVM_MISC2_ENABLE_V_LBR; + to->iopm_base_pa = from->iopm_base_pa & PAGE_MASK; to->msrpm_base_pa = from->msrpm_base_pa & PAGE_MASK; to->tsc_offset = from->tsc_offset; @@ -550,7 +554,6 @@ void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, to->event_inj_err = from->event_inj_err; to->next_rip = from->next_rip; to->nested_cr3 = from->nested_cr3; - to->misc_ctl2 = from->misc_ctl2; to->pause_filter_count = from->pause_filter_count; to->pause_filter_thresh = from->pause_filter_thresh; @@ -737,8 +740,7 @@ static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, static bool nested_vmcb12_has_lbrv(struct kvm_vcpu *vcpu) { - return guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && - (to_svm(vcpu)->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR); + return to_svm(vcpu)->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR; } static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 2bb1c6576ded..cbd4b0806d23 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -891,7 +891,7 @@ static bool svm_need_lbr_virtualization(struct kvm_vcpu *vcpu) if (svm->vmcb->save.dbgctl & DEBUGCTLMSR_LBR) return true; - return is_guest_mode(vcpu) && guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && + return is_guest_mode(vcpu) && (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR); } From cab4449bddbdcd8bf121ebe67ddb44fc61ca1b8e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 12:36:31 -0700 Subject: [PATCH 36/45] KVM: nSVM: Open code check on LBR virtualization being enabled in vmcb12 Now that KVM clears ENABLE_V_LBR if LBR virtualization isn't supported when caching vmcb12 control fields, i.e. now that the logic is a simple one-liner, open the check and drop the now-superfluous wrapper. No functional change intended. Link: https://patch.msgid.link/20260930193634.3415089-10-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 64464bdbe644..3c1c3e9698bb 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -738,11 +738,6 @@ static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, return 0; } -static bool nested_vmcb12_has_lbrv(struct kvm_vcpu *vcpu) -{ - return to_svm(vcpu)->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR; -} - static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) { struct vmcb_ctrl_area_cached *control = &svm->nested.ctl; @@ -817,7 +812,7 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) vmcb_mark_dirty(vmcb02, VMCB_DR); } - if (nested_vmcb12_has_lbrv(vcpu)) { + if (control->misc_ctl2 & SVM_MISC2_ENABLE_V_LBR) { /* * Reserved bits of DEBUGCTL are ignored. Be consistent with * svm_set_msr's definition of reserved bits. @@ -1310,7 +1305,7 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu) if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) vmcb12->control.next_rip = vmcb02->control.next_rip; - if (nested_vmcb12_has_lbrv(vcpu)) + if (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR) svm_copy_lbrs(&vmcb12->save, &vmcb02->save); vmcb12->control.event_inj = 0; @@ -1387,7 +1382,7 @@ void nested_svm_vmexit(struct vcpu_svm *svm) if (!nested_exit_on_intr(svm)) kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); - if (!nested_vmcb12_has_lbrv(vcpu)) { + if (!(svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR)) { svm_copy_lbrs(&vmcb01->save, &vmcb02->save); vmcb_mark_dirty(vmcb01, VMCB_LBR); } From 39080c3ca8b1edefdca7a4a189696427bbd8f6f7 Mon Sep 17 00:00:00 2001 From: Shivansh Dhiman Date: Wed, 30 Sep 2026 12:36:32 -0700 Subject: [PATCH 37/45] KVM: nSVM: Don't assume all active-low bits DR6 are fixed-1 When preparing vmcb02 for nested VMRUN, force only the actual fixed-1 bits instead of setting all active-low bits. The flaw is currently benign, as the only active-low bits supported by KVM are RTM (Restricted Transactional Memory) and BLD (Bus Lock Detect), neither of which is currently supported on SVM, but that's about to change. Signed-off-by: Shivansh Dhiman [sean: massage changelog] Link: https://patch.msgid.link/20260930193634.3415089-11-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/regs.c | 1 + arch/x86/kvm/svm/nested.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c index 0c76cf17e884..3621f1a47d37 100644 --- a/arch/x86/kvm/regs.c +++ b/arch/x86/kvm/regs.c @@ -784,6 +784,7 @@ unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) return fixed; } +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dr6_fixed_1); int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) { diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 3c1c3e9698bb..eb1ea172f2bb 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -808,7 +808,7 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_DR))) { vmcb02->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1; - svm->vcpu.arch.dr6 = svm->nested.save.dr6 | DR6_ACTIVE_LOW; + svm->vcpu.arch.dr6 = svm->nested.save.dr6 | kvm_get_dr6_fixed_1(vcpu); vmcb_mark_dirty(vmcb02, VMCB_DR); } From cd0e6f4d7cec11be97fe6476309719b23ab5accb Mon Sep 17 00:00:00 2001 From: Shivansh Dhiman Date: Wed, 30 Sep 2026 12:36:33 -0700 Subject: [PATCH 38/45] KVM: SVM: Add a vCPU-aware helper to get supported DEBUGCTL bits Replace DEBUGCTL_RESERVED_BITS with a vCPU-aware helper that provides the set of supported bits so that KVM can handle bits that are conditionally supported based on the vCPU model, e.g. for features like Bus Lock Detect. No functional change intended. Signed-off-by: Shivansh Dhiman Reviewed-by: Nikunj A Dadhania [sean: invert from reserved => supported, massage changelog] Link: https://patch.msgid.link/20260930193634.3415089-12-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 8 ++++---- arch/x86/kvm/svm/svm.c | 6 +++--- arch/x86/kvm/svm/svm.h | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index eb1ea172f2bb..3f547c01bd20 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -818,7 +818,7 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) * svm_set_msr's definition of reserved bits. */ svm_copy_lbrs(&vmcb02->save, save); - vmcb02->save.dbgctl &= ~DEBUGCTL_RESERVED_BITS; + vmcb02->save.dbgctl &= svm_get_supported_debugctl(vcpu); } else { svm_copy_lbrs(&vmcb02->save, &vmcb01->save); } @@ -1207,7 +1207,7 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu) } /* Copy state save area fields which are handled by VMRUN */ -void svm_copy_vmrun_state(struct vmcb_save_area *to_save, +void svm_copy_vmrun_state(struct kvm_vcpu *vcpu, struct vmcb_save_area *to_save, struct vmcb_save_area *from_save) { to_save->es = from_save->es; @@ -1234,7 +1234,7 @@ void svm_copy_vmrun_state(struct vmcb_save_area *to_save, if (kvm_cpu_cap_has(X86_FEATURE_LBRV)) { svm_copy_lbrs(to_save, from_save); - to_save->dbgctl &= ~DEBUGCTL_RESERVED_BITS; + to_save->dbgctl &= svm_get_supported_debugctl(vcpu); } } @@ -2080,7 +2080,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa; - svm_copy_vmrun_state(&svm->vmcb01.ptr->save, save); + svm_copy_vmrun_state(vcpu, &svm->vmcb01.ptr->save, save); nested_copy_vmcb_control_to_cache(svm, ctl); svm_switch_vmcb(svm, &svm->nested.vmcb02); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index cbd4b0806d23..88349a9862c2 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3216,7 +3216,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) data &= ~DEBUGCTLMSR_BTF; } - if (data & DEBUGCTL_RESERVED_BITS) + if (data & ~svm_get_supported_debugctl(vcpu)) return 1; if (svm->vmcb->save.dbgctl == data) @@ -5152,7 +5152,7 @@ static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400); - svm_copy_vmrun_state(m_save.map.hva + 0x400, &svm->vmcb01.ptr->save); + svm_copy_vmrun_state(vcpu, m_save.map.hva + 0x400, &svm->vmcb01.ptr->save); return 0; } @@ -5192,7 +5192,7 @@ static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) * used during SMM (see svm_enter_smm()) */ - svm_copy_vmrun_state(&svm->vmcb01.ptr->save, m_save.map.hva + 0x400); + svm_copy_vmrun_state(vcpu, &svm->vmcb01.ptr->save, m_save.map.hva + 0x400); /* * Enter the nested guest now diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index f566dae44bb0..61f9c5e4cee2 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -786,7 +786,10 @@ BUILD_SVM_MSR_BITMAP_HELPERS(bool, test, test) BUILD_SVM_MSR_BITMAP_HELPERS(void, clear, __clear) BUILD_SVM_MSR_BITMAP_HELPERS(void, set, __set) -#define DEBUGCTL_RESERVED_BITS (~DEBUGCTLMSR_LBR) +static inline u64 svm_get_supported_debugctl(struct kvm_vcpu *vcpu) +{ + return DEBUGCTLMSR_LBR; +} /* svm.c */ extern bool dump_invalid_vmcb; @@ -876,7 +879,7 @@ void svm_leave_nested(struct kvm_vcpu *vcpu); void svm_free_nested(struct vcpu_svm *svm); int svm_allocate_nested(struct vcpu_svm *svm); int nested_svm_vmrun(struct kvm_vcpu *vcpu); -void svm_copy_vmrun_state(struct vmcb_save_area *to_save, +void svm_copy_vmrun_state(struct kvm_vcpu *vcpu, struct vmcb_save_area *to_save, struct vmcb_save_area *from_save); void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb); void nested_svm_vmexit(struct vcpu_svm *svm); From 62ad461079e72c7bc16fcc535aa72a49ad89a54f Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Wed, 30 Sep 2026 12:36:34 -0700 Subject: [PATCH 39/45] KVM: SVM: Add support for virtualizating Bus Lock Detect Advertise support for virtualizing Bus Lock Detect and allow the guest to enable BUS_LOCK_DETECT if LBR Virtualization is enabled (Bus Lock Detect is enabled through MSR_IA32_DEBUGCTLMSR, which context-switched if and only if LBR Virtualization is enabled). Signed-off-by: Ravi Bangoria Reviewed-by: Tom Lendacky Co-developed-by: Shivansh Dhiman Signed-off-by: Shivansh Dhiman [sean: massage changelog] Link: https://patch.msgid.link/20260930193634.3415089-13-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/svm.c | 12 ++++++++++-- arch/x86/kvm/svm/svm.h | 7 ++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 88349a9862c2..3331fb4d9cd4 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -888,7 +888,7 @@ static bool svm_need_lbr_virtualization(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - if (svm->vmcb->save.dbgctl & DEBUGCTLMSR_LBR) + if (svm->vmcb->save.dbgctl & (DEBUGCTLMSR_LBR | DEBUGCTLMSR_BUS_LOCK_DETECT)) return true; return is_guest_mode(vcpu) && @@ -5700,9 +5700,17 @@ static __init void svm_set_cpu_caps(void) * Clear capabilities that are automatically configured by common code, * but that require explicit SVM support (that isn't yet implemented). */ - kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT); kvm_cpu_cap_clear(X86_FEATURE_MSR_IMM); + /* + * LBR Virtualization must be enabled to support BusLockTrap inside the + * guest, since BusLockTrap is enabled through MSR_IA32_DEBUGCTLMSR and + * MSR_IA32_DEBUGCTLMSR is virtualized only if LBR Virtualization is + * enabled. + */ + if (!lbrv) + kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT); + kvm_setup_xss_caps(); kvm_finalize_cpu_caps(); } diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 61f9c5e4cee2..14094a94f6aa 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -788,7 +788,12 @@ BUILD_SVM_MSR_BITMAP_HELPERS(void, set, __set) static inline u64 svm_get_supported_debugctl(struct kvm_vcpu *vcpu) { - return DEBUGCTLMSR_LBR; + u64 debugctl = DEBUGCTLMSR_LBR; + + if (guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) + debugctl |= DEBUGCTLMSR_BUS_LOCK_DETECT; + + return debugctl; } /* svm.c */ From 192dbad1bc0fb23460beef0c72b3590bfe613f35 Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Wed, 30 Sep 2026 14:21:10 -0700 Subject: [PATCH 40/45] KVM: x86: Advertise EFER_LMSLE_MBZ when KVM disallows EFER.LMSLE Set EFER_LMSLE_MBZ, CPUID.80000008H:EBX[bit 20], in KVM's supported CPUID whenever KVM refuses to set EFER.LMSLE, not just when hardware enumerates the defeature. KVM allows EFER.LMSLE if and only if nested SVM is supported, but passes through hardware's EFER_LMSLE_MBZ as-is, i.e. KVM tells userspace that long mode segment limits are available on an Intel CPU, and on an AMD CPU with kvm_amd.nested=0, and then rejects WRMSR(EFER) with EFER.LMSLE=1. The bit means precisely "EFER.LMSLE must be zero." Note, this is purely an enumeration change. EFER.LMSLE is already gated on nested SVM: kvm_setup_efer_caps() adds EFER_LMSLE to supported_efer_bits only if X86_FEATURE_SVM is supported, and svm_set_cpu_caps() sets X86_FEATURE_SVM if and only if "nested" is true. KVM's handling of EFER is unchanged; only what KVM tells userspace changes. Note #2, KVM now sets the defeature on Intel CPUs as well. That is no different than KVM setting NullSelectorClearsBase, another AMD-defined bit, on CPUs that don't have the errata. And LMSLE's dependency on nested SVM is a historical artifact of commit eec4b140c924 ("KVM: SVM: Allow EFER.LMSLE to be set with nested svm"), not an architectural requirement. Be aware that this changes KVM_GET_SUPPORTED_CPUID on all Intel hosts and on AMD hosts with kvm_amd.nested=0, i.e. userspace that reflects KVM's supported CPUID into the guest will start enumerating EFER_LMSLE_MBZ=1 to newly created guests. That's the correct enumeration, and it takes away only an EFER bit that KVM has never allowed to be set on such hosts, but it *is* guest visible. Deliberately omit a Fixes: tag: the misenumeration is benign in practice, as KVM's WRMSR(EFER) behavior is and always has been correct, and backporting a guest-visible CPUID change isn't worth the risk. Opportunistically key the EFER_LMSLE decision off kvm_cpu_cap_has() instead of boot_cpu_has(), as PASSTHROUGH_F() sets EFER_LMSLE_MBZ based on *raw* CPUID, i.e. "clearcpuid=13:20" would otherwise get KVM to enumerate the defeature and allow EFER.LMSLE=1, which is guaranteed to fail on VMRUN. Document KVM's enumeration of the defeature in api.rst. Assisted-by: LLM Signed-off-by: Jim Mattson Link: https://patch.msgid.link/20260930212114.3500045-2-seanjc@google.com Signed-off-by: Sean Christopherson --- Documentation/virt/kvm/api.rst | 11 +++++++++++ arch/x86/kvm/cpuid.c | 4 ++++ arch/x86/kvm/svm/svm.c | 10 ++++++++++ arch/x86/kvm/vmx/vmx.c | 7 +++++++ arch/x86/kvm/x86.c | 9 ++++++++- 5 files changed, 40 insertions(+), 1 deletion(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index e0430cc750c9..3c5bcdb0923f 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -9635,6 +9635,17 @@ On older versions of Linux, CPU[EAX=1]:ECX[24] (TSC_DEADLINE) is not reported by is present and the kernel has enabled in-kernel emulation of the local APIC. On newer versions, ``KVM_GET_SUPPORTED_CPUID`` does report the bit as available. +Long mode segment limits +~~~~~~~~~~~~~~~~~~~~~~~~ + +CPU[EAX=0x80000008]:EBX[20] (EFER_LMSLE_MBZ) is a "defeature" bit: it is set +when the CPU does *not* support long mode segment limits, and so requires +EFER.LMSLE to be zero. KVM reports the bit via ``KVM_GET_SUPPORTED_CPUID`` if +and only if KVM refuses to set EFER.LMSLE, i.e. if the CPU doesn't support long +mode segment limits, or if nested SVM is unsupported. KVM therefore reports +the bit on all Intel hosts, as KVM allows EFER.LMSLE only when nested SVM is +enabled. + CPU topology ~~~~~~~~~~~~ diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index cbf1fe4ae01f..65029e16d5d8 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1157,6 +1157,10 @@ void kvm_initialize_cpu_caps(void) F(AMD_STIBP), F(AMD_STIBP_ALWAYS_ON), F(AMD_IBRS_SAME_MODE), + /* + * Vendor code also sets EFER_LMSLE_MBZ if KVM itself + * can't support EFER.LMSLE, e.g. if nested SVM is disabled. + */ PASSTHROUGH_F(EFER_LMSLE_MBZ), F(AMD_PSFD), F(AMD_IBPB_RET), diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 3331fb4d9cd4..b09640629a14 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5676,6 +5676,16 @@ static __init void svm_set_cpu_caps(void) boot_cpu_has(X86_FEATURE_AMD_SSBD)) kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); + /* + * Tell userspace that EFER.LMSLE must be zero if nested SVM is + * disabled, as KVM allows EFER.LMSLE if and only if nested SVM is + * supported (a historical artifact of commit eec4b140c924 ("KVM: SVM: + * Allow EFER.LMSLE to be set with nested svm"), not an architectural + * requirement). + */ + if (!nested) + kvm_cpu_cap_set(X86_FEATURE_EFER_LMSLE_MBZ); + if (enable_pmu) { /* * Enumerate support for PERFCTR_CORE if and only if KVM has diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index c9f20144a70a..50f98343fc80 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -8114,6 +8114,13 @@ static __init void vmx_set_cpu_caps(void) kvm_cpu_cap_clear(X86_FEATURE_IBT); } + /* + * CPUID 0x80000008. Tell userspace that EFER.LMSLE must be zero; KVM + * never allows EFER.LMSLE to be set on Intel CPUs, as KVM supports long + * mode segment limits only in conjunction with nested SVM. + */ + kvm_cpu_cap_set(X86_FEATURE_EFER_LMSLE_MBZ); + kvm_setup_xss_caps(); kvm_finalize_cpu_caps(); } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 1e24ae7b8879..5efd17b360de 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -6980,7 +6980,14 @@ static void kvm_setup_efer_caps(void) if (kvm_cpu_cap_has(X86_FEATURE_SVM)) { kvm_caps.supported_efer_bits |= EFER_SVME; - if (!boot_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ)) + + /* + * Enumerating EFER_LMSLE_MBZ and allowing EFER.LMSLE=1 + * would be nonsensical. Note, vendor code sets the defeature + * if KVM can't support EFER.LMSLE for any reason, i.e. this + * needs to consult KVM's capabilities, not just raw CPUID. + */ + if (!kvm_cpu_cap_has(X86_FEATURE_EFER_LMSLE_MBZ)) kvm_caps.supported_efer_bits |= EFER_LMSLE; } } From 7358d13e1f1975bc8ddd2b02df70d2cdf638273a Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Wed, 30 Sep 2026 14:21:11 -0700 Subject: [PATCH 41/45] KVM: x86: Honor the guest's EFER_LMSLE_MBZ Reject guest attempts to set EFER.LMSLE if userspace enumerates EFER_LMSLE_MBZ, CPUID.80000008H:EBX[bit 20], in guest CPUID, so that userspace can hide long mode segment limits from the guest even on CPUs that support LMSLE, e.g. for migration compatibility between Rome and Milan. Enumerate the defeature as *partially* emulated, i.e. as supported by KVM but never advertised, so that kvm_vcpu_after_set_cpuid() propagates userspace's bit into vcpu->arch.cpu_caps. Checking guest_cpu_cap_has() in __kvm_valid_efer() doesn't suffice on its own, because KVM clears the defeature in kvm_cpu_caps precisely on the CPUs where it needs to be emulated, i.e. cpu_caps would never hold the bit no matter what userspace puts in guest CPUID. Don't advertise the defeature in KVM_GET_SUPPORTED_CPUID, as that would take EFER.LMSLE away from userspace that blindly stuffs KVM's supported CPUID into KVM_SET_CPUID2. MONITOR/MWAIT gets the same treatment. Enforce the defeature only for guest-initiated writes, as it's a guest CPUID consistency check, not a host capability. See commit 11988499e62b ("KVM: x86: Skip EFER vs. guest CPUID checks for host-initiated writes"). Note, KVM_SET_SREGS does reject EFER.LMSLE, as it runs the full set of guest CPUID checks, as does nested VMRUN, i.e. L1 can't sneak EFER.LMSLE into L2 via vmcb12. Mask EFER_LMSLE out of the value consumed by efer_trap(), which handles SVM_EXIT_EFER_WRITE_TRAP. EFER writes are *trapped*, not intercepted, i.e. hardware has already committed the write by the time KVM gains control, and the trap is enabled if and only if the guest is SEV-ES, whose EFER lives in the encrypted VMSA and so can't be fixed up by KVM. Rejecting the write would inject a #GP *and* leave EFER.LMSLE set in the guest, which is strictly worse than honoring a write that hardware itself allowed. EFER_SVME is already masked out for a similar "KVM can't enforce this here" reason. Opportunistically fix the whitespace damage around __kvm_valid_efer(). Document the resulting two-level contract in api.rst, i.e. that userspace may set the defeature even when KVM doesn't enumerate it. Suggested-by: Sean Christopherson Assisted-by: LLM Signed-off-by: Jim Mattson Link: https://patch.msgid.link/20260930212114.3500045-3-seanjc@google.com Signed-off-by: Sean Christopherson --- Documentation/virt/kvm/api.rst | 14 ++++++++++++++ arch/x86/kvm/cpuid.c | 16 ++++++++++++++++ arch/x86/kvm/msrs.c | 12 +++++++++++- arch/x86/kvm/svm/svm.c | 11 ++++++++++- 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 3c5bcdb0923f..2852bf94828b 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -9646,6 +9646,20 @@ mode segment limits, or if nested SVM is unsupported. KVM therefore reports the bit on all Intel hosts, as KVM allows EFER.LMSLE only when nested SVM is enabled. +KVM never reports the bit via ``KVM_GET_EMULATED_CPUID``, but userspace may set +it via ``KVM_SET_CPUID2`` even on a host where KVM doesn't report it. KVM +honors the guest's enumeration and rejects EFER.LMSLE=1 accordingly. That lets +userspace defeature a vCPU on a host that *does* support long mode segment +limits, so that the vCPU can later be migrated to a host that doesn't, e.g. so +that a vCPU created on AMD Rome can be migrated to Milan and later, which +dropped support for long mode segment limits. The opposite direction needs no +emulation, as a host that lacks long mode segment limits already enumerates the +defeature. + +Note, ``KVM_SET_MSRS`` is exempt from the check, as host-initiated MSR writes +skip guest CPUID checks so that userspace can set MSRs before it sets guest +CPUID. ``KVM_SET_SREGS`` and nested VMRUN are not exempt. + CPU topology ~~~~~~~~~~~~ diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 65029e16d5d8..697d3009816e 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1405,6 +1405,22 @@ static int cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, u32 func, u32 ind if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) entry->ecx = feature_bit(RDPID); return 1; + case 0x80000008: + /* + * Honor the guest's EFER_LMSLE_MBZ even if the underlying CPU + * allows setting EFER.LMSLE, e.g. to allow migrating a vCPU + * between hosts with and without EFER.LMSLE support. To avoid + * breaking existing setups that reflect KVM's supported CPUID + * into the guest, KVM doesn't advertise EFER_LMSLE_MBZ unless + * KVM *can't* support EFER.LMSLE=1. + */ + if (include_partially_emulated && + !kvm_cpu_cap_has(X86_FEATURE_EFER_LMSLE_MBZ)) { + entry->ebx |= feature_bit(EFER_LMSLE_MBZ); + return 1; + } + /* Nothing in 0x80000008 is fully emulated, don't emit an entry. */ + return 0; default: return 0; } diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c index dd3bb04878ca..b519fb90776e 100644 --- a/arch/x86/kvm/msrs.c +++ b/arch/x86/kvm/msrs.c @@ -598,9 +598,19 @@ static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) if (efer & EFER_NX && !guest_cpu_cap_has(vcpu, X86_FEATURE_NX)) return false; - return true; + /* + * EFER_LMSLE_MBZ is a "defeature" bit, i.e. is set when the CPU does + * *not* support long mode segment limits, and so is the only EFER + * check whose polarity is inverted: EFER.LMSLE is legal if and only if + * the guest does *not* have the defeature. + */ + if (efer & EFER_LMSLE && + guest_cpu_cap_has(vcpu, X86_FEATURE_EFER_LMSLE_MBZ)) + return false; + return true; } + bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) { if (efer & ~kvm_caps.supported_efer_bits) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index b09640629a14..aacc493e1b90 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2800,10 +2800,19 @@ static int efer_trap(struct kvm_vcpu *vcpu) * bit in svm_set_efer(), but __kvm_valid_efer() checks it against * whether the guest has X86_FEATURE_SVM - this avoids a failure if * the guest doesn't have X86_FEATURE_SVM. + * + * Clear EFER_LMSLE for a related reason: EFER writes are *trapped*, + * not intercepted, i.e. hardware has already committed the write by + * the time KVM gains control, and the trap is enabled if and only if + * the guest is SEV-ES, whose EFER lives in the encrypted VMSA and so + * can't be fixed up by KVM. Rejecting EFER.LMSLE=1 would inject a #GP + * *and* leave EFER.LMSLE set in the guest, which is strictly worse + * than honoring a write that hardware itself allowed. */ msr_info.host_initiated = false; msr_info.index = MSR_EFER; - msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME; + msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & + ~(EFER_SVME | EFER_LMSLE); ret = kvm_set_msr_common(vcpu, &msr_info); return kvm_complete_insn_gp(vcpu, ret); From cc2b9cc15ab702a3d4ada479f129fd287e40bd7e Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Wed, 30 Sep 2026 14:21:12 -0700 Subject: [PATCH 42/45] KVM: selftests: Rename svm_nested_clear_efer_svme to svm_nested_efer_test Rename svm_nested_clear_efer_svme.c to svm_nested_efer_test.c so that the test can host coverage for other EFER bits whose behavior is tied to nested SVM, and so that the name matches the svm_nested__test convention used by svm_nested_pat_test, svm_nested_shutdown_test, etc. No functional change intended. Assisted-by: LLM Signed-off-by: Jim Mattson Link: https://patch.msgid.link/20260930212114.3500045-4-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/Makefile.kvm | 2 +- .../{svm_nested_clear_efer_svme.c => svm_nested_efer_test.c} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tools/testing/selftests/kvm/x86/{svm_nested_clear_efer_svme.c => svm_nested_efer_test.c} (100%) diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 96bab7002d39..2554464d8b96 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -117,7 +117,7 @@ TEST_GEN_PROGS_x86 += x86/state_test TEST_GEN_PROGS_x86 += x86/vmx_preemption_timer_test TEST_GEN_PROGS_x86 += x86/svm_vmcall_test TEST_GEN_PROGS_x86 += x86/svm_int_ctl_test -TEST_GEN_PROGS_x86 += x86/svm_nested_clear_efer_svme +TEST_GEN_PROGS_x86 += x86/svm_nested_efer_test TEST_GEN_PROGS_x86 += x86/svm_nested_shutdown_test TEST_GEN_PROGS_x86 += x86/svm_nested_soft_inject_test TEST_GEN_PROGS_x86 += x86/svm_nested_vmcb12_gpa diff --git a/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c b/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c similarity index 100% rename from tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c rename to tools/testing/selftests/kvm/x86/svm_nested_efer_test.c From 58f4dc83da6e5baf8b9a9752e433360c6b17df38 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 14:21:13 -0700 Subject: [PATCH 43/45] KVM: selftests: Add module param API to check if nested virtualization is enabled Add an API to check if the nested virtualization module param is enabled, e.g. so that tests can verify KVM's own enumeration of the corresponding CPU feature. Provide a big warning above the API to make it as obvious as possible clear that tests should use "kvm_cpu_has(X86_FEATURE_VMX) || kvm_cpu_has(X86_FEATURE_SVM)" when checking for nested virtualization. It doesn't really matter which method tests use, as KVM is supposed to keep "nested" synchronized with VMX/SVM enumeration, but make the comment super scary to try and prevent KVM selftests from having two different ways of doing the same thing. Note, the bool vs. int mess is intentional, because KVM's params are terrible. kvm_amd.ko provides an "int", while kvm_intel.ko provides a "bool". Link: https://patch.msgid.link/20260930212114.3500045-5-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/processor.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index 6e6f70035508..3d3a7d9f2a95 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -1472,6 +1472,19 @@ static inline bool kvm_is_lbrv_enabled(void) return !!get_kvm_amd_param_integer("lbrv"); } +/* + * Do NOT use this to check for nVMX or nSVM support. Querying kvm_cpu_has() + * for either of X86_FEATURE_{VMX,SVM} is the idiomatic way to check for nested + * virtualization support. Use this *only* to validate KVM's own enumeration. + */ +static inline bool kvm_is_nested_virtualization_enabled(void) +{ + if (host_cpu_is_intel) + return get_kvm_intel_param_bool("nested"); + + return get_kvm_amd_param_integer("nested"); +} + u64 *vm_get_pte(struct kvm_vm *vm, gva_t gva); u64 kvm_hypercall(u64 nr, u64 a0, u64 a1, u64 a2, u64 a3); From 34f749161dabac48a6aeae36ffd5f55f84023d61 Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Wed, 30 Sep 2026 14:21:14 -0700 Subject: [PATCH 44/45] KVM: selftests: Add coverage for the EFER_LMSLE_MBZ defeature Add coverage for KVM's virtualization of EFER_LMSLE_MBZ, CPUID.80000008H:EBX[bit 20], which is set when the CPU does *not* support long mode segment limits. Verify that KVM's enumeration of the defeature matches the expectation derived from hardware plus kvm_amd's "nested" module param, so that the assertion doesn't compare KVM's enumeration against itself, and that WRMSR(EFER), nested VMRUN, and KVM_SET_SREGS reject EFER.LMSLE=1 if and only if guest CPUID enumerates the defeature. Also verify that host-initiated KVM_SET_MSRS is exempt from the check, as host-initiated writes skip guest CPUID checks so that userspace can set MSRs before it sets guest CPUID. Extend svm_nested_efer_test rather than add yet another test binary for a single CPUID bit; the test already has the L1/L2 harness needed to exercise EFER.LMSLE. Note, only a CPU that supports LMSLE, i.e. Rome and earlier, exercises the emulation path; elsewhere KVM enumerates the defeature straight from hardware. Opportunistically drop the unnecessary #include of vmx.h. Assisted-by: LLM Signed-off-by: Jim Mattson [sean: drop specific APM volume references, use common "nested" getter] Link: https://patch.msgid.link/20260930212114.3500045-6-seanjc@google.com Signed-off-by: Sean Christopherson --- .../selftests/kvm/include/x86/processor.h | 1 + .../selftests/kvm/x86/svm_nested_efer_test.c | 244 +++++++++++++++++- 2 files changed, 231 insertions(+), 14 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index 3d3a7d9f2a95..6f110987c51a 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -216,6 +216,7 @@ struct kvm_x86_cpu_feature { #define X86_FEATURE_INVTSC KVM_X86_CPU_FEATURE(0x80000007, 0, EDX, 8) #define X86_FEATURE_RDPRU KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 4) #define X86_FEATURE_AMD_IBPB KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 12) +#define X86_FEATURE_EFER_LMSLE_MBZ KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 20) #define X86_FEATURE_NPT KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 0) #define X86_FEATURE_LBRV KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 1) #define X86_FEATURE_NRIPS KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 3) diff --git a/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c b/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c index 6bc301207cbc..fa0d76a9c390 100644 --- a/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c +++ b/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c @@ -1,16 +1,20 @@ // SPDX-License-Identifier: GPL-2.0-only /* + * Tests for KVM's handling of EFER bits whose behavior is tied to nested SVM. + * * Copyright (C) 2026, Google LLC. */ +#include "test_util.h" #include "kvm_util.h" -#include "vmx.h" +#include "processor.h" #include "svm_util.h" #include "kselftest.h" +static bool l2_ran; -static void l2_guest_code(void) +static void l2_clear_efer_svme(void) { - unsigned long efer = rdmsr(MSR_EFER); + u64 efer = rdmsr(MSR_EFER); /* generic_svm_setup() initializes EFER_SVME set for L2 */ GUEST_ASSERT(efer & EFER_SVME); @@ -20,31 +24,243 @@ static void l2_guest_code(void) GUEST_ASSERT(0); } -static void l1_guest_code(struct svm_test_data *svm) +static void l1_clear_efer_svme(struct svm_test_data *svm) { - generic_svm_setup(svm, l2_guest_code); + generic_svm_setup(svm, l2_clear_efer_svme); run_guest(svm->vmcb, svm->vmcb_gpa); /* Unreachable, L1 should be shutdown */ GUEST_ASSERT(0); } -int main(int argc, char *argv[]) +static void l2_lmsle(void) +{ + GUEST_ASSERT(rdmsr(MSR_EFER) & EFER_LMSLE); + l2_ran = true; + vmmcall(); +} + +static void l1_lmsle(struct svm_test_data *svm) +{ + bool lmsle_mbz = this_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ); + struct vmcb *vmcb = svm->vmcb; + u64 efer = rdmsr(MSR_EFER); + + /* + * Selftests' vCPUs are created with EFER.LMSLE clear; the sub-tests + * below need to start from a clean slate. + */ + GUEST_ASSERT(!(efer & EFER_LMSLE)); + GUEST_ASSERT(!l2_ran); + + /* + * Per the APM, if EFER_LMSLE_MBZ is enumerated in CPUID, "64-bit mode + * segment limit checking is not supported and attempting to set + * EFER.LMSLE = 1 causes a #GP exception". + */ + if (lmsle_mbz) { + GUEST_ASSERT_EQ(wrmsr_safe(MSR_EFER, efer | EFER_LMSLE), GP_VECTOR); + GUEST_ASSERT(!(rdmsr(MSR_EFER) & EFER_LMSLE)); + } else { + GUEST_ASSERT_EQ(wrmsr_safe(MSR_EFER, efer | EFER_LMSLE), 0); + GUEST_ASSERT(rdmsr(MSR_EFER) & EFER_LMSLE); + + /* + * Restore EFER so that generic_svm_setup() doesn't propagate + * EFER.LMSLE into vmcb12 on its own, i.e. so that the VMRUN + * sub-test actually tests what it thinks it's testing. + */ + wrmsr(MSR_EFER, efer); + } + + /* + * VMRUN's consistency checks reject "any MBZ bit of EFER", i.e. a + * vmcb12 with EFER.LMSLE set must generate VMEXIT_INVALID when the + * defeature is enumerated. + */ + generic_svm_setup(svm, l2_lmsle); + vmcb->save.efer |= EFER_LMSLE; + run_guest(vmcb, svm->vmcb_gpa); + + if (lmsle_mbz) { + GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_ERR); + GUEST_ASSERT(!l2_ran); + } else { + GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_VMMCALL); + GUEST_ASSERT(l2_ran); + GUEST_ASSERT(vmcb->save.efer & EFER_LMSLE); + } + + GUEST_DONE(); +} + +static struct kvm_vcpu *create_l1_vcpu(struct kvm_vm **vm, void *l1_guest_code) +{ + struct kvm_vcpu *vcpu; + gva_t svm_gva; + + *vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); + + vcpu_alloc_svm(*vm, &svm_gva); + vcpu_args_set(vcpu, 1, svm_gva); + + return vcpu; +} + +static void test_enumeration(void) +{ + bool lmsle_mbz; + + /* + * EFER_LMSLE_MBZ, CPUID.80000008H:EBX[bit 20], is a "defeature" bit, + * i.e. is set when the CPU does *not* support long mode segment + * limits. KVM enumerates the defeature if and only if KVM refuses to + * set EFER.LMSLE, i.e. if the CPU doesn't support LMSLE, or if KVM + * doesn't support nested SVM. Derive the expectation from raw CPUID + * and kvm_amd's "nested" module param rather than from + * kvm_cpu_has(X86_FEATURE_SVM), so that the assertion doesn't simply + * compare KVM's enumeration to itself. + */ + lmsle_mbz = this_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ) || + !this_cpu_has(X86_FEATURE_SVM) || + !kvm_is_nested_virtualization_enabled(); + + TEST_ASSERT_EQ(kvm_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ), lmsle_mbz); + + ksft_test_result_pass("KVM enumerates EFER_LMSLE_MBZ=%d\n", lmsle_mbz); +} + +static void test_clear_efer_svme(void) { struct kvm_vcpu *vcpu; struct kvm_vm *vm; - gva_t nested_gva = 0; - TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM)); - - vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); - - vcpu_alloc_svm(vm, &nested_gva); - vcpu_args_set(vcpu, 1, nested_gva); + vcpu = create_l1_vcpu(&vm, l1_clear_efer_svme); vcpu_run(vcpu); TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_SHUTDOWN); kvm_vm_free(vm); - return 0; + ksft_test_result_pass("L2 clearing EFER.SVME shuts down L1\n"); +} + +static void test_lmsle(bool lmsle_mbz) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + + vcpu = create_l1_vcpu(&vm, l1_lmsle); + + vcpu_set_or_clear_cpuid_feature(vcpu, X86_FEATURE_EFER_LMSLE_MBZ, + lmsle_mbz); + + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + case UCALL_DONE: + break; + default: + TEST_FAIL("Unexpected ucall: %lu", uc.cmd); + } + + kvm_vm_free(vm); + ksft_test_result_pass("Guest EFER_LMSLE_MBZ=%d\n", lmsle_mbz); +} + +static void test_host_initiated_lmsle(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + u64 efer; + + vm = vm_create_with_one_vcpu(&vcpu, NULL); + vcpu_set_cpuid_feature(vcpu, X86_FEATURE_EFER_LMSLE_MBZ); + + /* + * EFER_LMSLE_MBZ is a guest CPUID consistency check, not a host + * capability, i.e. must not be enforced against host-initiated writes, + * so that userspace can set MSRs before it sets guest CPUID. + */ + efer = vcpu_get_msr(vcpu, MSR_EFER); + TEST_ASSERT(!(efer & EFER_LMSLE), "EFER.LMSLE unexpectedly set"); + + vcpu_set_msr(vcpu, MSR_EFER, efer | EFER_LMSLE); + TEST_ASSERT_EQ(vcpu_get_msr(vcpu, MSR_EFER), efer | EFER_LMSLE); + + kvm_vm_free(vm); + ksft_test_result_pass("Host-initiated EFER.LMSLE=1 is allowed\n"); +} + +static void test_sregs_lmsle(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_sregs sregs; + struct kvm_vm *vm; + int rc; + + vm = vm_create_with_one_vcpu(&vcpu, NULL); + vcpu_set_cpuid_feature(vcpu, X86_FEATURE_EFER_LMSLE_MBZ); + + /* + * Unlike KVM_SET_MSRS, KVM_SET_SREGS runs the full set of guest CPUID + * checks, i.e. rejects EFER.LMSLE even though it's host-initiated. + */ + vcpu_sregs_get(vcpu, &sregs); + TEST_ASSERT(!(sregs.efer & EFER_LMSLE), "EFER.LMSLE unexpectedly set"); + + sregs.efer |= EFER_LMSLE; + rc = _vcpu_sregs_set(vcpu, &sregs); + TEST_ASSERT(rc, "KVM allowed EFER.LMSLE with EFER_LMSLE_MBZ set"); + + kvm_vm_free(vm); + ksft_test_result_pass("KVM_SET_SREGS rejects EFER.LMSLE=1\n"); +} + +int main(int argc, char *argv[]) +{ + bool has_nested_svm, has_lmsle; + + ksft_print_header(); + ksft_set_plan(6); + + test_enumeration(); + + /* + * The sub-tests below need to actually run a nested guest, and the + * EFER.LMSLE sub-tests additionally need KVM to allow EFER.LMSLE. + * It's KVM's view of the world, not raw CPUID, that dictates whether + * EFER.LMSLE is allowed, i.e. whether the defeature is emulated. + */ + has_nested_svm = kvm_cpu_has(X86_FEATURE_SVM); + has_lmsle = has_nested_svm && + !kvm_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ); + + if (!has_nested_svm) + ksft_print_msg("Nested SVM unsupported\n"); + else if (!has_lmsle) + ksft_print_msg("KVM doesn't support EFER.LMSLE\n"); + + if (has_nested_svm) { + test_clear_efer_svme(); + test_lmsle(true); + } else { + ksft_test_result_skip("L2 clearing EFER.SVME shuts down L1\n"); + ksft_test_result_skip("Guest EFER_LMSLE_MBZ=1\n"); + } + + if (has_lmsle) { + test_lmsle(false); + test_host_initiated_lmsle(); + test_sregs_lmsle(); + } else { + ksft_test_result_skip("Guest EFER_LMSLE_MBZ=0\n"); + ksft_test_result_skip("Host-initiated EFER.LMSLE=1 is allowed\n"); + ksft_test_result_skip("KVM_SET_SREGS rejects EFER.LMSLE=1\n"); + } + + ksft_finished(); } From 88b23a2e8286b7016c3b58dc10093cf28c22aa3d Mon Sep 17 00:00:00 2001 From: Loc Nguyen Date: Fri, 2 Oct 2026 05:42:54 +0000 Subject: [PATCH 45/45] KVM: x86: Request event processing for exception VM-Exits Request event processing whenever kvm_queue_exception_vmexit() queues an exception VM-Exit. Most callers reach the helper through kvm_multiple_exception(), which already sets KVM_REQ_EVENT, but nested asynchronous page faults call it directly. Without another event request, vcpu_enter_guest() skips kvm_check_and_inject_events() and re-enters L2. A subsequent VM-Exit can clear the pending exception while reconstructing vectoring state, losing the synthetic #PF VM-Exit. The APF reason remains PAGE_NOT_PRESENT and a later regular #PF can consume the stale value. Setting KVM_REQ_EVENT in the common helper ensures that L1 receives the queued VM-Exit before KVM re-enters L2. Fixes: 7709aba8f716 ("KVM: x86: Morph pending exceptions to pending VM-Exits at queue time") Cc: stable@vger.kernel.org Signed-off-by: Loc Nguyen Link: https://patch.msgid.link/20261002053723.3275202-2-loc@ryzome.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 5efd17b360de..727d68c1331d 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -456,6 +456,8 @@ static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vecto { struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; + kvm_make_request(KVM_REQ_EVENT, vcpu); + ex->vector = vector; ex->injected = false; ex->pending = true;