github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-26 10:02:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

KVM x86 MMU changes for 7.3

Browse Source 
- Fix a bug where KVM would walk a newly created rmap without holding the rmap
    lock (or mmu_lock) during aging.
 
  - Fix a bug where aging TDP MMU SPTEs could clobber FROZEN SPTEs.
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEKTobbabEP7vbhhN9OlYIJqCjN/0FAmp8wS8ACgkQOlYIJqCj
 N/2SmQ//ecBzKzw5uQVwigrVGwWNZS1I00d4DZsYQpD/cAfSuKCYG4YGfG5IBtUy
 tvMmoPswmTbullysJxB2bW0JOPjmGGEE6lsNhCH7CmgpKXdeBgqUo2x+KHVV8hFv
 YlXXBA2rkbtObMjQgfaByIjv2M3d0trSkOfe39WvPEcfqjOBp3GK22km9eHw4qN2
 PqidjSHtAzwcsd+id1cmw5uoel9RKOpux7FPM5mfVIWHJByDrY5sUU+u9Ug0p4F1
 2wWbG6ghdnO0LsFFrGD2K2Ummux1KaBuFny/q4cqv3dg3IsMrLcX5xpmC9Z+N1Ey
 mmq0D87VXzxEaYXDiLB7hWp6ZM+5dFXunSyhoX/Qu8ALUrUKyXY7MLOdLPXvsYpx
 YEy5OTsZCdlebvziu2w8+aNOTxsctYF9cj8yraz8PtmKqgJ2Wg1seNLl7rfGwCWP
 DGtvaPrS20cm+OIVD26or4zfR+6DGhilfz1BItoQejzA4ryVo3XEcYn9U11ph6mn
 9Mu0W/XVl9AX44lliENPbcNmLDB5X65twv+1eIZkbevFRpSFsJYFg4Fdg6/0JK8g
 B2zPyKmsTi+PgAx6dnEZeVNDhYY6P7E1OaGbXqIvNSdzGM6Ps3gbfIYiIOPD515B
 kGi8y9kDKnNZ7PQrACTWU+u6stB8f4BoXBE7Au1+GNPbSRlqu9M=
 =axEq
 -----END PGP SIGNATURE-----

Merge tag 'kvm-x86-mmu-7.3' of https://github.com/kvm-x86/linux into HEAD

KVM x86 MMU changes for 7.3

 - Fix a bug where KVM would walk a newly created rmap without holding the rmap
   lock (or mmu_lock) during aging.

 - Fix a bug where aging TDP MMU SPTEs could clobber FROZEN SPTEs.
This commit is contained in:
Paolo Bonzini 2026-08-18 13:26:37 +02:00
commit abad8c4cb9
3 changed files with 48 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
arch/x86/kvm/mmu/mmu.c
View File

@ -1248,18 +1248,9 @@ struct rmap_iterator {
int pos; /* index of the sptep */
};
/*
* Iteration must be started by this function. This should also be used after
* removing/dropping sptes from the rmap link because in such cases the
* information in the iterator may not be valid.
*
* Returns sptep if found, NULL otherwise.
*/
static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
struct rmap_iterator *iter)
static u64 *__rmap_get_first(unsigned long rmap_val,
struct rmap_iterator *iter)
{
unsigned long rmap_val = kvm_rmap_get(rmap_head);
if (!rmap_val)
return NULL;
@ -1273,6 +1264,19 @@ static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
return iter->desc->sptes[iter->pos];
}
/*
* Iteration must be started by this function. This should also be used after
* removing/dropping sptes from the rmap link because in such cases the
* information in the iterator may not be valid.
*
* Returns sptep if found, NULL otherwise.
*/
static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
struct rmap_iterator *iter)
{
return __rmap_get_first(kvm_rmap_get(rmap_head), iter);
}
/*
* Must be used with a valid iterator: e.g. after rmap_get_first().
*
@ -1307,8 +1311,9 @@ static u64 *rmap_get_next(struct rmap_iterator *iter)
__for_each_rmap_spte(_rmap_head_, _iter_, _sptep_) \
if (!WARN_ON_ONCE(!is_shadow_present_pte(*(_sptep_)))) \
#define for_each_rmap_spte_lockless(_rmap_head_, _iter_, _sptep_, _spte_) \
__for_each_rmap_spte(_rmap_head_, _iter_, _sptep_) \
#define for_each_rmap_spte_lockless(_rmap_val_, _iter_, _sptep_, _spte_) \
for (_sptep_ = __rmap_get_first(_rmap_val_, _iter_); \
_sptep_; _sptep_ = rmap_get_next(_iter_)) \
if (is_shadow_present_pte(_spte_ = mmu_spte_get_lockless(sptep)))
static void drop_spte(struct kvm *kvm, u64 *sptep)
@ -1734,11 +1739,11 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm,
struct kvm_rmap_head *rmap_head;
struct rmap_iterator iter;
unsigned long rmap_val;
u64 old_spte, new_spte;
bool young = false;
u64 *sptep;
gfn_t gfn;
int level;
u64 spte;
for (level = PG_LEVEL_4K; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) {
for (gfn = range->start; gfn < range->end;
@ -1746,8 +1751,8 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm,
rmap_head = gfn_to_rmap(gfn, level, range->slot);
rmap_val = kvm_rmap_lock_readonly(rmap_head);
for_each_rmap_spte_lockless(rmap_head, &iter, sptep, spte) {
if (!is_accessed_spte(spte))
for_each_rmap_spte_lockless(rmap_val, &iter, sptep, old_spte) {
if (!is_accessed_spte(old_spte))
continue;
if (test_only) {
@ -1755,17 +1760,18 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm,
return true;
}
if (spte_ad_enabled(spte))
clear_bit((ffs(shadow_accessed_mask) - 1),
(unsigned long *)sptep);
if (spte_ad_enabled(old_spte))
new_spte = old_spte & ~shadow_accessed_mask;
else
/*
* If the following cmpxchg fails, the
* spte is being concurrently modified
* and should most likely stay young.
*/
cmpxchg64(sptep, spte,
mark_spte_for_access_track(spte));
new_spte = mark_spte_for_access_track(old_spte);
/*
* Don't bother retrying if the CMPXCHG fails,
* i.e. if another CPU modified the SPTE. The
* SPTE is either being zapped or is likely
* still in-use, i.e. is still young.
*/
cmpxchg64(sptep, old_spte, new_spte);
young = true;
}

7
arch/x86/kvm/mmu/tdp_iter.h
View File

@ -19,6 +19,13 @@ static inline u64 kvm_tdp_mmu_read_spte(tdp_ptep_t sptep)
return READ_ONCE(*rcu_dereference(sptep));
}
/*
* WARNING! mmu_lock must be held for write when using the "write atomic" or
* "clear bits atomic" APIs, otherwise KVM could overwrite the "wrong" old SPTE
* value, i.e. clobber an update from a different CPU. The only exception is
* when KVM is freezing a leaf SPTE for removal, in which case KVM doesn't care
* about the exact old SPTE value (KVM will react to the actual old value).
*/
static inline u64 kvm_tdp_mmu_write_spte_atomic(tdp_ptep_t sptep, u64 new_spte)
{
KVM_MMU_WARN_ON(is_ept_ve_possible(new_spte));

20
arch/x86/kvm/mmu/tdp_mmu.c
View File

@ -1335,19 +1335,17 @@ static void kvm_tdp_mmu_age_spte(struct kvm *kvm, struct tdp_iter *iter)
if (WARN_ON_ONCE(is_mirror_sptep(iter->sptep)))
return;
if (spte_ad_enabled(iter->old_spte)) {
iter->old_spte = tdp_mmu_clear_spte_bits_atomic(iter->sptep,
shadow_accessed_mask);
if (spte_ad_enabled(iter->old_spte))
new_spte = iter->old_spte & ~shadow_accessed_mask;
} else {
else
new_spte = mark_spte_for_access_track(iter->old_spte);
/*
* It is safe for the following cmpxchg to fail. Leave the
* Accessed bit set, as the spte is most likely young anyway.
*/
if (__tdp_mmu_set_spte_atomic(kvm, iter, new_spte))
return;
}
/*
* Don't bother retrying if another CPU modified the SPTE, the SPTE is
* either being zapped or is likely still in-use, i.e. is still young.
*/
if (__tdp_mmu_set_spte_atomic(kvm, iter, new_spte))
return;
trace_kvm_tdp_mmu_spte_changed(iter->as_id, iter->gfn, iter->level,
iter->old_spte, new_spte);