From b59a87c1ba3c347e8aec5f932eb7abf13064fd62 Mon Sep 17 00:00:00 2001 From: Chengfeng Ye Date: Wed, 26 Aug 2026 02:39:24 +0800 Subject: [PATCH 01/10] KVM: x86: Take SRCU in kvm_zap_gfn_range() kvm_zap_gfn_range() walks memslots and rmaps and may drop mmu_lock to reschedule. Callers that do not already hold kvm->srcu (or slots_lock) can race with memslot deletion: synchronize_srcu_expedited() does not wait, kvm_free_memslot() frees the old slot and its rmap, and the zap resumes on freed memory. The VFIO noncoherent-DMA path hits this by zapping the entire GPA space without SRCU. KASAN reported: BUG: KASAN: vmalloc-out-of-bounds in slot_rmap_walk_next+0x82/0x1c0 Read of size 8 at addr ffffc900005c1008 Call Trace: slot_rmap_walk_next+0x82/0x1c0 __kvm_rmap_zap_gfn_range+0x17a/0x280 kvm_zap_gfn_range+0x2a6/0x6a0 kvm_vfio_set_attr+0x576/0x770 kvm_device_ioctl+0x1ff/0x3b0 Take SRCU inside kvm_zap_gfn_range() so every caller is covered. Nesting with an existing kvm->srcu critical section is fine; the helper uses a local index. Drop the now-redundant SRCU pair from __kvm_set_or_clear_apicv_inhibit(). Fixes: 362ff6dca541 ("KVM: x86/mmu: Zap KVM TDP when noncoherent DMA assignment starts/stops") Cc: stable@vger.kernel.org Signed-off-by: Chengfeng Ye Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260825183924.237727-1-nicoyip.dev@gmail.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 2 ++ arch/x86/kvm/x86.c | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 064ecc33b926..87518f238e5e 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -7117,6 +7117,8 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) if (WARN_ON_ONCE(gfn_end <= gfn_start)) return; + guard(srcu)(&kvm->srcu); + write_lock(&kvm->mmu_lock); kvm_mmu_invalidate_start(kvm); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 79468ddfe473..15721ce659bb 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -7925,10 +7925,8 @@ void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, kvm->arch.apicv_inhibit_reasons = new; if (new) { unsigned long gfn = gpa_to_gfn(APIC_DEFAULT_PHYS_BASE); - int idx = srcu_read_lock(&kvm->srcu); kvm_zap_gfn_range(kvm, gfn, gfn+1); - srcu_read_unlock(&kvm->srcu, idx); } } else { kvm->arch.apicv_inhibit_reasons = new; From 355323a4dc945d0176f54c324aa44943820f932f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 09:42:11 -0700 Subject: [PATCH 02/10] KVM: x86/mmu: Reload MMU on *every* page pre-fault attempt/iteration Reload the MMU (which is a nop if the MMU doesn't need to be reloaded) on every attempt to pre-fault a guest page, i.e. when the page fault path signals that the caller should retry. If the synchronize_srcu_expedited() in kvm_invalidate_memslot() completes before kvm_vcpu_pre_fault_memory() grabs SRCU, but kvm_mmu_reload() in the pre-fault path completes before kvm_invalidate_memslot() triggers x86's "fast zap all", then the pre-fault task will reach kvm_tdp_page_prefault() with an invalid root. Attempting to fault-in memory with an invalid root ultimately puts kvm_tdp_page_prefault() into an infinite (breakable) retry loop, which manifests most obviously as a hang in the pre_fault_memory_test selftest, but also eventually causes RCU (SRCU?) to complain. INFO: rcu_tasks detected stalls on tasks: 000000000cda47bd: .. nvcsw: 6/6 holdout: 1 idle_cpu: -1/25 task:pre_fault_memor state:R running task stack:12696 pid:95588 tgid:95588 ppid:95584 task_flags:0x400000 flags:0x00080801 Call Trace: lock_release+0x4e/0x320 __get_user_pages+0x546/0xcd0 up_read+0x1b/0x30 get_user_pages_unlocked+0xee/0x350 hva_to_pfn+0xd3/0x3d0 [kvm] lock_release+0x4e/0x320 xa_load+0x5c/0x170 xa_load+0x14c/0x170 __kvm_faultin_pfn+0xd9/0x130 [kvm] lock_acquire+0x65/0x2b0 lock_release+0x4e/0x320 kvm_mmu_faultin_pfn+0x1e1/0x690 [kvm] gup_fast_fallback+0x63e/0xdf0 kvm_tdp_page_fault+0xeb/0x140 [kvm] kvm_mmu_do_page_fault+0x12e/0x200 [kvm] kvm_arch_vcpu_pre_fault_memory+0x16e/0x200 [kvm] kvm_vcpu_pre_fault_memory+0xc1/0x1f0 [kvm] kvm_vcpu_pre_fault_memory+0x116/0x1f0 [kvm] kvm_vcpu_ioctl+0x3a4/0x6b0 [kvm] clockevents_program_event+0x5d/0x170 __se_sys_ioctl+0x6d/0xb0 entry_SYSCALL_64_after_hwframe+0x4b/0x53 do_syscall_64+0x10a/0x480 __irq_exit_rcu+0x8e/0x140 entry_SYSCALL_64_after_hwframe+0x4b/0x53 Fixes: 6e01b7601dfe ("KVM: x86: Implement kvm_arch_vcpu_pre_fault_memory()") Reviewed-by: Rick Edgecombe Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260826164214.756512-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 87518f238e5e..786ddbc30dd5 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5061,6 +5061,10 @@ static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa, if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) return -EIO; + r = kvm_mmu_reload(vcpu); + if (r) + return r; + cond_resched(); r = kvm_mmu_do_page_fault(vcpu, gpa, error_code, true, NULL, level); } while (r == RET_PF_RETRY); @@ -5101,14 +5105,6 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, if (kvm_is_gfn_alias(vcpu->kvm, gpa_to_gfn(range->gpa))) return -EINVAL; - /* - * reload is efficient when called repeatedly, so we can do it on - * every iteration. - */ - r = kvm_mmu_reload(vcpu); - if (r) - return r; - direct_bits = 0; if (kvm_arch_has_private_mem(vcpu->kvm) && kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(range->gpa))) From 698b8c88e15914a88f686b552c14c0c5caff3cc7 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 09:42:12 -0700 Subject: [PATCH 03/10] KVM: x86/mmu: Harden "map private PFN" against unexpected root invalidation Move kvm_tdp_mmu_map_private_pfn()'s reload of the MMU into its tight loop so that an unexpected root invalidation has a better chance of being handled gracefully, even though it should be impossible for the vCPU's root to be invalidated after the initial reload. As is, encountering an invalid root is *guaranteed* to put the task into an infinite loop (albeit a breakable loop that honors NEED_RESCHED). Add a WARN to try and detect bugs that break KVM's expectations, along with a comment to explain why it should be impossible for the root to be invalidated. Note, the loop in question doesn't actually check for a stale page fault, i.e. likely won't detect an invalid loop in the first place. That bug will be addressed shortly. Cc: Kai Huang Cc: Yan Zhao Cc: Rick Edgecombe Link: https://patch.msgid.link/20260826164214.756512-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 786ddbc30dd5..f8d29fa2e538 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5209,10 +5209,6 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) if (kvm_gfn_is_write_tracked(kvm, fault.slot, fault.gfn)) return -EPERM; - r = kvm_mmu_reload(vcpu); - if (r) - return r; - r = mmu_topup_memory_caches(vcpu, false); if (r) return r; @@ -5224,10 +5220,22 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) return -EIO; + r = kvm_mmu_reload(vcpu); + if (r) + return r; + cond_resched(); guard(read_lock)(&kvm->mmu_lock); + /* + * Because slots_lock is held, it should be impossible for *any* + * roots to be invalidated after the initial MMU reload. WARN, + * but continue on; the above MMU reload will do the right thing + * if the current root is actually invalid. + */ + WARN_ON_ONCE(kvm_test_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu)); + r = kvm_tdp_mmu_map(vcpu, &fault); } while (r == RET_PF_RETRY); From 6415d9c4ee19be3816c89974562d67e2c00c8638 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 09:42:13 -0700 Subject: [PATCH 04/10] KVM: x86/mmu: Top-up memory caches when retrying "map private PFN" When mapping a private PFN in TDX's post-populate callback, top-up the memory caches on every attempt to map the PFN to harden against bugs in the map flow that could consume cache entries even if mapping ultimately fails. E.g. as pointed out by Sashiko, the in-progress Dynamic PAMT support could consume PAMT cache entries on TDX-Module lock contention. Harden KVM even though consuming an entry on failure is considered a KVM bug. Retry should only be encountered if KVM is buggy (the locks held by the sole call path will prevent retries from being needed due to TDX-specific details, and memory can be faulted in only once the VM is TD_STATE_RUNNABLE, and KVM_TDX_INIT_MEM_REGION is only usable if the VM is *not* TD_STATE_RUNNABLE), top-up is "free" if there's no work to be done, and populating a TDX guest's memory is a slow path, i.e. there's no meaningful downside to the hardening. Reported-by: Sashiko Bot Closes: https://lore.kernel.org/all/20260718061050.E17B01F000E9@smtp.kernel.org Reviewed-by: Rick Edgecombe Link: https://patch.msgid.link/20260826164214.756512-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index f8d29fa2e538..45522105a472 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5209,10 +5209,6 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) if (kvm_gfn_is_write_tracked(kvm, fault.slot, fault.gfn)) return -EPERM; - r = mmu_topup_memory_caches(vcpu, false); - if (r) - return r; - do { if (signal_pending(current)) return -EINTR; @@ -5224,6 +5220,10 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) if (r) return r; + r = mmu_topup_memory_caches(vcpu, false); + if (r) + return r; + cond_resched(); guard(read_lock)(&kvm->mmu_lock); From f7a3be22761953f4a1f754786b7a35b4c5ea6824 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 09:42:14 -0700 Subject: [PATCH 05/10] KVM: x86/mmu: Add sanity check to detect stale page faults in "map private PFN" Harden the "map private PFN" flow against potentially-fatal bugs or future KVM changes by checking for a stale "fault" prior to actually mapping the PFN into the guest. While it should be impossible for the "page fault" to become stale, the sanity check is cheap, whereas a broken assumption would have a high probability of leading to a guest-expoitable use-after-free. Snapshot the invalidation sequence after acquiring mmu_lock to avoid false positives, even though doing so completely voids anys and all protection against unexpected invalidations. Pretty much the entire point of kvm_tdp_mmu_map_private_pfn() is that it allows mapping a PFN that was gifted by the caller, i.e. the caller would have to mess up its one and only responsibility. Link: https://patch.msgid.link/20260826164214.756512-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 45522105a472..319125faf9fa 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5236,6 +5236,18 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) */ WARN_ON_ONCE(kvm_test_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu)); + /* + * Snapshot the invalidation sequence counter after acquiring + * mmu_lock, as guest_memfd guarantees the validity of the pfn, + * i.e. any concurrent invalidations are guaranteed to be + * irrelevant. + */ + fault.mmu_seq = vcpu->kvm->mmu_invalidate_seq; + if (is_page_fault_stale(vcpu, &fault)) { + r = RET_PF_RETRY; + continue; + } + r = kvm_tdp_mmu_map(vcpu, &fault); } while (r == RET_PF_RETRY); From df269c8e097f306b09d9f371a60a936e2530c10d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 14:17:59 -0700 Subject: [PATCH 06/10] KVM: x86/mmu: Always guard rmaps with mmu_lock on PREEMPT_RT=y kernels For all intents and purposes, revert KVM's ability to walk rmaps outside of mmu_lock when running on a realtime (PREEMPT_RT=y) kernel. I.e. don't use a non-sleepable bit-spinlock to protect rmap entries, as realtime kernels are highly unlikely to benefit from increased aging throughput and reduced jitter for memory-overcommitted nested VMs, whereas using a non-sleepable lock is currently buggy and goes against the spirit of realtime kernels. Because KVM's rmap locks are hand-crafted bit-spinlocks, preemption must be disabled before acquiring the lock, otherwise a preempted lock holder will result in all other walkers of the locked rmap to spin and wait, with no tracked owner for PI to boost. For non-RT kernels, acquiring mmu_lock suffices, as mmu_lock is a non-sleepable rwlock. But on RT, where mmu_lock becomes sleepable, preemption is left enabled for rmap writers: WARNING: arch/x86/kvm/mmu/mmu.c:920 at __kvm_rmap_lock+0x1a7/0x1e0 [kvm], CPU#16: vmx_apic_update/3708 CPU: 16 UID: 0 PID: 3708 Comm: vmx_apic_update Not tainted 7.2.0-rc7 #52 PREEMPT_{RT,LAZY} RIP: 0010:__kvm_rmap_lock+0x1a7/0x1e0 [kvm] Call Trace: pte_list_add+0x67/0x4d0 [kvm] __link_shadow_page+0x249/0x480 [kvm] ept_fetch+0x4d5/0x1220 [kvm] ept_page_fault+0x60b/0x850 [kvm] kvm_mmu_do_page_fault+0x252/0x690 [kvm] Alternatively, KVM could manually disable preemption when grabbing an rmap lock, but as above, that isn't what RT kernels generally want, and it's actually more complex to implement (cleanly). To not completely lose the scaling advantage of per-rmap locks, take mmu_lock for read in the aging path, i.e. allow multiple concurrent aging tasks, as the aging code needs to use atomic SPTE accesses no matter what, i.e. no extra code/work is required to guard against concurrent aging of SPTEs. Reported-by: David Woodhouse Closes: https://lore.kernel.org/all/8d47b43e1829ac92703723e6a1a4afc7a2eaacb5.camel@infradead.org Fixes: 4834eaded91e ("KVM: x86/mmu: Add infrastructure to allow walking rmaps outside of mmu_lock") Reviewed-by: David Woodhouse Tested-by: David Woodhouse Reviewed-by: James Houghton Link: https://patch.msgid.link/20260902211759.2700289-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 48 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 319125faf9fa..855c4f030f4c 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -895,6 +895,7 @@ static struct kvm_memory_slot *gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu */ #define KVM_RMAP_MANY BIT(0) +#ifndef CONFIG_PREEMPT_RT /* * rmaps and PTE lists are mostly protected by mmu_lock (the shadow MMU always * operates with mmu_lock held for write), but rmaps can be walked without @@ -1008,7 +1009,8 @@ static unsigned long kvm_rmap_get(struct kvm_rmap_head *rmap_head) * actual locking is the same, but the caller is disallowed from modifying the * rmap, and so the unlock flow is a nop if the rmap is/was empty. */ -static unsigned long kvm_rmap_lock_readonly(struct kvm_rmap_head *rmap_head) +static unsigned long kvm_rmap_lock_readonly(struct kvm *kvm, + struct kvm_rmap_head *rmap_head) { unsigned long rmap_val; @@ -1032,6 +1034,35 @@ static void kvm_rmap_unlock_readonly(struct kvm_rmap_head *rmap_head, __kvm_rmap_unlock(rmap_head, old_val); preempt_enable(); } +#else +static unsigned long kvm_rmap_get(struct kvm_rmap_head *rmap_head) +{ + return atomic_long_read(&rmap_head->val); +} +static unsigned long kvm_rmap_lock(struct kvm *kvm, + struct kvm_rmap_head *rmap_head) +{ + lockdep_assert_held_write(&kvm->mmu_lock); + return kvm_rmap_get(rmap_head); +} + +static void kvm_rmap_unlock(struct kvm *kvm, + struct kvm_rmap_head *rmap_head, + unsigned long new_val) +{ + atomic_long_set_release(&rmap_head->val, new_val); +} + +static unsigned long kvm_rmap_lock_readonly(struct kvm *kvm, + struct kvm_rmap_head *rmap_head) +{ + lockdep_assert_held_read(&kvm->mmu_lock); + return kvm_rmap_get(rmap_head); +} + +static void kvm_rmap_unlock_readonly(struct kvm_rmap_head *rmap_head, + unsigned long old_val) { } +#endif /* * Returns the number of pointers in the rmap chain, not counting the new one. @@ -1745,11 +1776,24 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm, gfn_t gfn; int level; + BUILD_BUG_ON(!IS_ENABLED(CONFIG_KVM_MMU_LOCKLESS_AGING)); + + /* + * For realtime kernels, do aging under mmu_lock (per-rmap locking is + * compiled out), as realtime deployments are unlikely to benefit from + * increased aging throughput and reduced jitter for memory-overcommitted + * nested VMs, whereas keeping preemption enabled is extremely valuable + * (mmu_lock becomes a sleepable lock on realtime kernels). + */ +#ifdef CONFIG_PREEMPT_RT + guard(read_lock)(&kvm->mmu_lock); +#endif + for (level = PG_LEVEL_4K; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) { for (gfn = range->start; gfn < range->end; gfn += KVM_PAGES_PER_HPAGE(level)) { rmap_head = gfn_to_rmap(gfn, level, range->slot); - rmap_val = kvm_rmap_lock_readonly(rmap_head); + rmap_val = kvm_rmap_lock_readonly(kvm, rmap_head); for_each_rmap_spte_lockless(rmap_val, &iter, sptep, old_spte) { if (!is_accessed_spte(old_spte)) From 27f02d851410ece239f4024aae87143ed00f4354 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 6 Aug 2026 12:19:45 -0700 Subject: [PATCH 07/10] KVM: x86/mmu: Bug the VM if KVM attempts to unsync an upper-level shadow page Bug the VM and skip marking a shadow page unsync if KVM attempts to unsync an upper-level shadow page, as marking an upper-level SP unsync is all but guaranteed to be far worse than terminating the VM. E.g. sync_spte() (rightly) assumes SPTEs in the SP are leaf SPTEs and so calls drop_spte() instead of drop_parent_pte(), which can effectively crash the host via when running with CONFIG_BUG_ON_DATA_CORRUPTION=y due to there not being an rmap entry for the SPTE. Simply not marking the SP unsync, i.e. letting the VM continue on, is equally dangerous for the guest as it means KVM's shadow MMU is using stale information, e.g. could unintentionally corrupt the guest. Cc: Jinu Kim Link: https://patch.msgid.link/20260806191945.4192732-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 855c4f030f4c..2bf6ed878589 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3065,6 +3065,9 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot, if (prefetch) return -EEXIST; + if (KVM_BUG_ON(sp->role.level != PG_LEVEL_4K, kvm)) + continue; + /* * TDP MMU page faults require an additional spinlock as they * run with mmu_lock held for read, not write, and the unsync @@ -3088,7 +3091,6 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot, continue; } - WARN_ON_ONCE(sp->role.level != PG_LEVEL_4K); kvm_unsync_page(kvm, sp); } if (locked) From 98a843669196ace8789a859796e45caffb970e40 Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Sun, 20 Sep 2026 07:24:59 +0000 Subject: [PATCH 08/10] KVM: x86/mmu: Convert can't-happen fault path EFAULTs to KVM_BUG_ON() + -EIO Four -EFAULT returns in x86's page fault path are guarded by WARN_ON_ONCE() because they describe KVM bugs, not conditions the guest or userspace can reach: - direct_map() and FNAME(fetch)() completing the walk at a level other than the goal level, i.e. KVM's shadow page table walk disagreeing with the mapping level KVM itself just computed. - a CR2 with bits 63:32 set on 32-bit KVM, which the hardware cannot produce. - PFERR_PRIVATE_ACCESS set on a reserved-bit fault. KVM sets that synthetic bit itself, and only when PFERR_RSVD_MASK is clear. Returning -EFAULT for a KVM bug conflicts with KVM_CAP_MEMORY_FAULT_INFO, which promises that an -EFAULT out of KVM_RUN on a guest page fault VM-Exit is accompanied by kvm_run.memory_fault. Describing a broken KVM invariant as a memory fault would be actively misleading, as there is no guest access for userspace to resolve and retry. Convert the four sites to KVM_BUG_ON() + -EIO, the established way to report that KVM is hosed, so that -EFAULT out of the fault path always means "guest memory access KVM could not resolve". Suggested-by: Sean Christopherson Link: https://lore.kernel.org/all/Zr-8M9rYplgN6IS3@google.com/ Signed-off-by: Mikhail Malyshev Link: https://patch.msgid.link/20260920072459.3485710-3-mike.malyshev@gmail.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 12 ++++++------ arch/x86/kvm/mmu/paging_tmpl.h | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 2bf6ed878589..f14440a2b547 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3577,8 +3577,8 @@ static int direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) fault->req_level >= it.level); } - if (WARN_ON_ONCE(it.level != fault->goal_level)) - return -EFAULT; + if (KVM_BUG_ON(it.level != fault->goal_level, vcpu->kvm)) + return -EIO; ret = mmu_set_spte(vcpu, fault->slot, it.sptep, access, base_gfn, fault->pfn, fault); @@ -4941,8 +4941,8 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, #ifndef CONFIG_X86_64 /* A 64-bit CR2 should be impossible on 32-bit KVM. */ - if (WARN_ON_ONCE(fault_address >> 32)) - return -EFAULT; + if (KVM_BUG_ON(fault_address >> 32, vcpu->kvm)) + return -EIO; #endif /* * Legacy #PF exception only have a 32-bit error code. Simply drop the @@ -6651,8 +6651,8 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err r = RET_PF_INVALID; if (unlikely(error_code & PFERR_RSVD_MASK)) { - if (WARN_ON_ONCE(error_code & PFERR_PRIVATE_ACCESS)) - return -EFAULT; + if (KVM_BUG_ON(error_code & PFERR_PRIVATE_ACCESS, vcpu->kvm)) + return -EIO; r = handle_mmio_page_fault(vcpu, cr2_or_gpa, direct); if (r == RET_PF_EMULATE) diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 27427e7f22fa..4f5369cf3753 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -775,8 +775,8 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, fault->req_level >= it.level); } - if (WARN_ON_ONCE(it.level != fault->goal_level)) - return -EFAULT; + if (KVM_BUG_ON(it.level != fault->goal_level, vcpu->kvm)) + return -EIO; ret = mmu_set_spte(vcpu, fault->slot, it.sptep, gw->pte_access, base_gfn, fault->pfn, fault); From 64f1cf2884403895feed3c442c65e1cb785945b9 Mon Sep 17 00:00:00 2001 From: Anish Moorthy Date: Sun, 20 Sep 2026 07:24:58 +0000 Subject: [PATCH 09/10] KVM: x86/mmu: Report a memory fault exit when the fault handler EFAULTs KVM_CAP_MEMORY_FAULT_INFO documents that KVM_RUN will fill kvm_run.memory_fault when KVM cannot resolve a guest page fault VM-Exit, "e.g. if there is a valid memslot but no backing VMA for the corresponding host virtual address". kvm_handle_error_pfn() does not honor that guarantee. It returns a bare -EFAULT, leaving userspace with no indication of which guest physical address faulted, or even that the exit was a memory fault at all. Userspace cannot distinguish a transient, resolvable condition from a fatal one, so in practice the VMM terminates the guest. Fill kvm_run.memory_fault before returning -EFAULT. A concrete user is an Intel integrated GPU assigned to a guest via vfio-pci. The guest driver clears PCI_COMMAND.MEM on one vCPU while another vCPU is mid-MMIO to a BAR of the same device. Clearing PCI_COMMAND.MEM makes vfio-pci zap the BAR's mmap, so the second vCPU's fault finds a valid memslot whose VMA can no longer supply a PFN, and KVM_RUN fails with a bare -EFAULT. The VM dies, even though the guest did nothing architecturally invalid and the condition clears as soon as the driver re-enables memory decoding. Reproduce by pairing a vCPU that spins on accesses to the assigned device's BAR0 with a vCPU that toggles PCI_COMMAND.MEM; the race is hit within minutes. The same crash has been observed in the field on production edge hardware. Reporting the fault does not by itself define the access semantics. Userspace still has to decide what a read or write to a BAR with memory decoding disabled returns. But it is the information userspace needs in order to make that decision instead of killing the guest. Suggested-by: Sean Christopherson Fixes: 16f95f3b95ca ("KVM: Add KVM_EXIT_MEMORY_FAULT exit to report faults to userspace") Link: https://lore.kernel.org/all/20240809205158.1340255-1-amoorthy@google.com/ Link: https://lore.kernel.org/all/Zr-8M9rYplgN6IS3@google.com/ Signed-off-by: Anish Moorthy Co-developed-by: Mikhail Malyshev Signed-off-by: Mikhail Malyshev Link: https://patch.msgid.link/20260920072459.3485710-2-mike.malyshev@gmail.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index f14440a2b547..4d22e0d3f8a1 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3616,6 +3616,7 @@ static int kvm_handle_error_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fa return RET_PF_RETRY; } + kvm_mmu_prepare_memory_fault_exit(vcpu, fault); return -EFAULT; } From 31adfe155db60a58b61e5b3d472cf85336ff3daf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20L=C3=B3pez?= Date: Wed, 16 Sep 2026 16:33:11 +0200 Subject: [PATCH 10/10] KVM: x86/mmu: Fix NX page unaccounting for TDP MMU pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit unaccount_nx_huge_page() unconditionally passes KVM_SHADOW_MMU to untrack_possible_nx_huge_page(), which is incorrect, particularly on the path from kvm_recover_nx_huge_pages(), which handles pages from both MMUs. Fix this by passing down the MMU type from the callers. Note that __kvm_mmu_prepare_zap_page() is only used for shadow MMU pages. Fixes: 6777885605e1 ("KVM: x86/mmu: Track possible NX huge pages separately for TDP vs. Shadow MMU") Signed-off-by: Carlos López Link: https://patch.msgid.link/20260916143310.1909993-2-clopez@suse.de Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 4d22e0d3f8a1..021fbc33a2b8 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -864,11 +864,12 @@ void untrack_possible_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp, list_del_init(&sp->possible_nx_huge_page_link); } -static void unaccount_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp) +static void unaccount_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp, + enum kvm_mmu_type mmu_type) { sp->nx_huge_page_disallowed = false; - untrack_possible_nx_huge_page(kvm, sp, KVM_SHADOW_MMU); + untrack_possible_nx_huge_page(kvm, sp, mmu_type); } static struct kvm_memory_slot *gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, @@ -2829,7 +2830,7 @@ static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm, } if (sp->nx_huge_page_disallowed) - unaccount_nx_huge_page(kvm, sp); + unaccount_nx_huge_page(kvm, sp, KVM_SHADOW_MMU); sp->role.invalid = 1; @@ -8050,7 +8051,7 @@ static void kvm_recover_nx_huge_pages(struct kvm *kvm, WARN_ON_ONCE(!sp->nx_huge_page_disallowed); WARN_ON_ONCE(!sp->role.direct); - unaccount_nx_huge_page(kvm, sp); + unaccount_nx_huge_page(kvm, sp, mmu_type); if (is_tdp_mmu) spin_unlock(tdp_mmu_pages_lock);