From e428f9779a43737d830111238816f1928b07aefb Mon Sep 17 00:00:00 2001 From: Phil Rosenthal Date: Mon, 20 Jul 2026 13:45:49 -0400 Subject: [PATCH 1/3] KVM: x86/mmu: Consume the locked rmap value in the lockless rmap walk __kvm_rmap_lock() deliberately elides the rmap lock when it observes an empty rmap. In that case kvm_rmap_lock_readonly() also re-enables preemption and returns zero, so the caller holds neither the rmap lock nor a preemption reference. The elision documents the invariant it relies on: * Elide the lock if the rmap is empty, as lockless walkers (read-only * mode) don't need to (and can't) walk an empty rmap, nor can they add * entries to the rmap. I.e. the only paths that process empty rmaps * do so while holding mmu_lock for write, and are mutually exclusive. kvm_rmap_age_gfn_range() ignores the returned value and unconditionally enters for_each_rmap_spte_lockless(). The iterator started with rmap_get_first(), which re-reads rmap_head->val rather than using the value returned by the lock. If a writer populates the rmap between the lock's read and the iterator's re-read, the aging path walks the newly installed rmap without holding its lock. For a KVM_RMAP_MANY rmap this leaves the walker following a pte_list_desc chain that it never locked. A writer holding mmu_lock for write may free that chain (e.g. kvm_zap_all_rmap_sptes() on the recycle path, or any rmap zap) via kmem_cache_free() while the walk is in progress, giving a slab use-after-free. Nothing serialises the two: the aging path runs without mmu_lock when CONFIG_KVM_MMU_LOCKLESS_AGING=y, and the rmap lock that would otherwise exclude the writer was elided. Because the empty path re-enables preemption, the interval between the two reads can span an arbitrary scheduling delay. Fix the class of bug by having the lockless walk consume the value returned by the lock instead of re-reading the rmap. Split rmap_get_first() into __rmap_get_first(), which starts an iterator from an already-read rmap value, and make for_each_rmap_spte_lockless() take that value and call __rmap_get_first() directly. kvm_rmap_age_gfn_range() passes the value returned by kvm_rmap_lock_readonly(): when the lock was elided the value is zero, __rmap_get_first() returns NULL, and the walk is skipped. No lockless walker re-reads the rmap, so the lock-elision invariant cannot be violated, and no lock()-without-paired-unlock() path is added to the aging code. Fixes: af3b6a9eba48 ("KVM: x86/mmu: Walk rmaps (shadow MMU) without holding mmu_lock when aging gfns") Suggested-by: Sean Christopherson Cc: stable@vger.kernel.org Signed-off-by: Phil Rosenthal Link: https://patch.msgid.link/20260720-rmap-age-elided-submit-v2-1-668973030d47@phil.gs Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 6c13da942bfc..440db8891f21 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1227,18 +1227,9 @@ struct rmap_iterator { int pos; /* index of the sptep */ }; -/* - * Iteration must be started by this function. This should also be used after - * removing/dropping sptes from the rmap link because in such cases the - * information in the iterator may not be valid. - * - * Returns sptep if found, NULL otherwise. - */ -static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head, - struct rmap_iterator *iter) +static u64 *__rmap_get_first(unsigned long rmap_val, + struct rmap_iterator *iter) { - unsigned long rmap_val = kvm_rmap_get(rmap_head); - if (!rmap_val) return NULL; @@ -1252,6 +1243,19 @@ static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head, return iter->desc->sptes[iter->pos]; } +/* + * Iteration must be started by this function. This should also be used after + * removing/dropping sptes from the rmap link because in such cases the + * information in the iterator may not be valid. + * + * Returns sptep if found, NULL otherwise. + */ +static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head, + struct rmap_iterator *iter) +{ + return __rmap_get_first(kvm_rmap_get(rmap_head), iter); +} + /* * Must be used with a valid iterator: e.g. after rmap_get_first(). * @@ -1286,8 +1290,9 @@ static u64 *rmap_get_next(struct rmap_iterator *iter) __for_each_rmap_spte(_rmap_head_, _iter_, _sptep_) \ if (!WARN_ON_ONCE(!is_shadow_present_pte(*(_sptep_)))) \ -#define for_each_rmap_spte_lockless(_rmap_head_, _iter_, _sptep_, _spte_) \ - __for_each_rmap_spte(_rmap_head_, _iter_, _sptep_) \ +#define for_each_rmap_spte_lockless(_rmap_val_, _iter_, _sptep_, _spte_) \ + for (_sptep_ = __rmap_get_first(_rmap_val_, _iter_); \ + _sptep_; _sptep_ = rmap_get_next(_iter_)) \ if (is_shadow_present_pte(_spte_ = mmu_spte_get_lockless(sptep))) static void drop_spte(struct kvm *kvm, u64 *sptep) @@ -1725,7 +1730,7 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm, rmap_head = gfn_to_rmap(gfn, level, range->slot); rmap_val = kvm_rmap_lock_readonly(rmap_head); - for_each_rmap_spte_lockless(rmap_head, &iter, sptep, spte) { + for_each_rmap_spte_lockless(rmap_val, &iter, sptep, spte) { if (!is_accessed_spte(spte)) continue; From 3d679b7cb31f74bf2303123ce4ee3266eeee999a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 27 Jul 2026 17:22:35 -0700 Subject: [PATCH 2/3] KVM: x86/mmu: Use CMPXCHG when clearing Accessed bit in TDP MMU Use LOCK CMPXCHG instead of LOCK AND to clear the Accessed bit when aging SPTEs in the TDP MMU, as doing a LOCK AND can corrupt a FROZEN SPTE and allow a third CPU to effectively overwrite the FROZEN SPTE. As pointed out by AI of some kind, because the magic FROZEN_SPTE value is a "full" SPTE, not a single bit, and includes the Accessed bit, clearing the Accessed bit in a FROZEN SPTE will result in is_frozen_spte() getting a false negative. E.g. if CPU0 freezes an SPTE, and CPU1 clears the Accessed bit in the frozen SPTE, then CPU2 could come along and overwrite the frozen SPTE with a shadow-present SPTE. Thankfully, the false negative is largely benign, because outside of TDX, which doesn't support aging, KVM only freezes leaf SPTEs when removing an upper level shadow page. So while KVM could clobber a frozen SPTE back to a shadow-present SPTE, and could even use the new SPTE, the subsequent TLB flush will make the orphaned, shadow-present SPTE unreachable. Failure to ever zap the orphaned leaf SPTE would show up in KVM's stats, but otherwise is benign (because KVM no longer keeps an elevated refcount for leaf SPTEs). Opportunistically add a comment to warn future developers away from using kvm_tdp_mmu_write_spte_atomic() and tdp_mmu_clear_spte_bits_atomic(), as they are generally unsafe. Keep the helpers, e.g. instead of open-coding the atomic64_fetch_and() in tdp_mmu_clear_spte_bits(), as scary warnings usually are more effective deterrent against recidivism than removal of the dangerous code. Alternatively, KVM could use different bits for the magic FROZEN_SPTE value, e.g. setting the Dirty bits (with effective IPAT and Global aliases) would likely be "ok", as IPAT/Global are extremely unlikely to be cleared without doing a full SPTE write, and KVM's clearing of Dirty bits shares logic with Write-Protection, which must do a full SPTE write (via cmpxchg64() in the TDP MMU) to ensure KVM isn't clobbering state. But there is zero reason to carry that risk (beyond stubbornness in wanting to preserve a "cute" idea), as the cost of LOCK CMPXCHG and LOCK AND are within 1-2 uops of each other on modern hardware. Fixes: b146a9b34aed ("KVM: x86/mmu: Age TDP MMU SPTEs without holding mmu_lock") Cc: stable@vger.kernel.org Reviewed-by: Kai Huang Reviewed-by: James Houghton Link: https://patch.msgid.link/20260728002236.869865-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/tdp_iter.h | 7 +++++++ arch/x86/kvm/mmu/tdp_mmu.c | 20 +++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/mmu/tdp_iter.h b/arch/x86/kvm/mmu/tdp_iter.h index 364c5da6c499..f898d8d0d93c 100644 --- a/arch/x86/kvm/mmu/tdp_iter.h +++ b/arch/x86/kvm/mmu/tdp_iter.h @@ -19,6 +19,13 @@ static inline u64 kvm_tdp_mmu_read_spte(tdp_ptep_t sptep) return READ_ONCE(*rcu_dereference(sptep)); } +/* + * WARNING! mmu_lock must be held for write when using the "write atomic" or + * "clear bits atomic" APIs, otherwise KVM could overwrite the "wrong" old SPTE + * value, i.e. clobber an update from a different CPU. The only exception is + * when KVM is freezing a leaf SPTE for removal, in which case KVM doesn't care + * about the exact old SPTE value (KVM will react to the actual old value). + */ static inline u64 kvm_tdp_mmu_write_spte_atomic(tdp_ptep_t sptep, u64 new_spte) { KVM_MMU_WARN_ON(is_ept_ve_possible(new_spte)); diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index ce3f2efadb05..44dad106fad1 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -1335,19 +1335,17 @@ static void kvm_tdp_mmu_age_spte(struct kvm *kvm, struct tdp_iter *iter) if (WARN_ON_ONCE(is_mirror_sptep(iter->sptep))) return; - if (spte_ad_enabled(iter->old_spte)) { - iter->old_spte = tdp_mmu_clear_spte_bits_atomic(iter->sptep, - shadow_accessed_mask); + if (spte_ad_enabled(iter->old_spte)) new_spte = iter->old_spte & ~shadow_accessed_mask; - } else { + else new_spte = mark_spte_for_access_track(iter->old_spte); - /* - * It is safe for the following cmpxchg to fail. Leave the - * Accessed bit set, as the spte is most likely young anyway. - */ - if (__tdp_mmu_set_spte_atomic(kvm, iter, new_spte)) - return; - } + + /* + * Don't bother retrying if another CPU modified the SPTE, the SPTE is + * either being zapped or is likely still in-use, i.e. is still young. + */ + if (__tdp_mmu_set_spte_atomic(kvm, iter, new_spte)) + return; trace_kvm_tdp_mmu_spte_changed(iter->as_id, iter->gfn, iter->level, iter->old_spte, new_spte); From fb25ee778aae357da2c1a8dc1d240b6061482454 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 27 Jul 2026 17:22:36 -0700 Subject: [PATCH 3/3] KVM: x86/mmu: Use CMPXCHG when clearing Accessed bit in the shadow MMU Use CMPXCHG instead of clear_bit(), which currently emits a LOCK BTR since the to-be-cleared bit isn't a compile-time constant, when aging SPTEs in the shadow MMU to align with the approach taken by the TDP MMU, and because using CMPXCHG is far more robust against bugs in KVM. E.g. if the SPTE is somehow no longer an SPTE due to a KVM bug, CMPXCHG will fail gracefully, whereas clear_bit() would potentially corrupt/clobber memory. Clearing the Accessed bit without atomically ensuring the SPTE is still the old SPTE is "fine", as holding the rmap's lock ensures zapping the old SPTE can't fully complete, which in turn ensures a new, different SPTE can't be installed. But that chain of logic isn't exactly obvious, and there's zero reason to avoid CMPXCHG as its cost on modern hardware is within ~1-2 uops of LOCK BTR (and may even be cheaper on some microarchitectures). Doing a 64-bit CMPXCHG on 32-bit kernels does require a more expensive CMPXCHG8B, but 32-bit KVM is all but dead at this point. Cc: James Houghton Reviewed-by: Kai Huang Reviewed-by: James Houghton Link: https://patch.msgid.link/20260728002236.869865-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 440db8891f21..71beb169d78d 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1718,11 +1718,11 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm, struct kvm_rmap_head *rmap_head; struct rmap_iterator iter; unsigned long rmap_val; + u64 old_spte, new_spte; bool young = false; u64 *sptep; gfn_t gfn; int level; - u64 spte; for (level = PG_LEVEL_4K; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) { for (gfn = range->start; gfn < range->end; @@ -1730,8 +1730,8 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm, rmap_head = gfn_to_rmap(gfn, level, range->slot); rmap_val = kvm_rmap_lock_readonly(rmap_head); - for_each_rmap_spte_lockless(rmap_val, &iter, sptep, spte) { - if (!is_accessed_spte(spte)) + for_each_rmap_spte_lockless(rmap_val, &iter, sptep, old_spte) { + if (!is_accessed_spte(old_spte)) continue; if (test_only) { @@ -1739,17 +1739,18 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm, return true; } - if (spte_ad_enabled(spte)) - clear_bit((ffs(shadow_accessed_mask) - 1), - (unsigned long *)sptep); + if (spte_ad_enabled(old_spte)) + new_spte = old_spte & ~shadow_accessed_mask; else - /* - * If the following cmpxchg fails, the - * spte is being concurrently modified - * and should most likely stay young. - */ - cmpxchg64(sptep, spte, - mark_spte_for_access_track(spte)); + new_spte = mark_spte_for_access_track(old_spte); + + /* + * Don't bother retrying if the CMPXCHG fails, + * i.e. if another CPU modified the SPTE. The + * SPTE is either being zapped or is likely + * still in-use, i.e. is still young. + */ + cmpxchg64(sptep, old_spte, new_spte); young = true; }