github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

KVM x86 MMU changes for 7.4

Browse Source 
- Grab KVM's SRCU "lock" in kvm_zap_gfn_range() to fix a recurring bug where
    callers fail to do so.
 
  - Fix a variety of issues in the pre-fault code that are mostly benign in the
    current code base, but will show up in big ways if/when the pre-faulting
    logic is used to implementing prefetch-on-fault for the TDP MMU.
 
  - Guard rmaps with mmu_lock for PREEMPT_RT=y kernels to fix a longstanding
    issue introducing by "lockless aging" where KVM would take its effective
    raw bit-spinlock with preemption enabled (because mmu_lock gets turned into
    a sleepable lock).
 
  - Prepare a memory_fault exit (paired with -EFAULT) when KVM hits an error
    PFN, e.g. so that userspace can resolve transient issues instead of killing
    the guest.  Convert a variety of WARN()+EFAULT paths into KVM_BUG_ON()+EIO
    so that KVM can hard commit to filling memory_fault on -EFAULTs that occur
    in the page fault path.
 
  - Fix a NX hugepage un-accounting bug where KVM would always uncharge the
    shadow MMU, even when handling TDP MMU pages.
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEKTobbabEP7vbhhN9OlYIJqCjN/0FAmrRm3oACgkQOlYIJqCj
 N/2stRAAlmTOGuS0zvfPEouScJRKIk1JMkG90bO2/yaDv2wf+7oK6CgvZpu3/kbQ
 yINxFg7jAqc7z2Aj9FZ8U/AgNjzjPWgM6rEKpuChqiXicBSw8Q58L+DajxSBKcCf
 US5IXcjaaQeJWxz2j8RDcGLB00CLIR2ZmfUFvFcdPUlw6MHROj2YCpMut6+lRz37
 RO6NRGL5OQWkabEBgAcn7pJPBD4psFd98J90s3qUYinWRxbXWTZ3DKjfYyOayOGh
 N2KihDYisZ7fmm2D6qGOhM0bzJj+Q+Pa3bTK6cow4jzlva3T/qMLX0F6lHRYSJyD
 H5h8fIiwIeoviA538aEjmK3skkdOFy7F+OstECgW2ZrviKASKDJxG4YTXN3+UAas
 AYTlIR9Jvn/IACF06N7czMs5dlsmRs6tVVIrpiviTRoJzVa4WFMeVg43tIsNd6uy
 EpAJf47/w/e+tvt2F+BQF/EO3vuPv53BDc8khWgZdd3J/APQKLegJg67aDO2ArV1
 JqCRrLLn/4X/OdcZMNSfZjx9SFU1cDOzoBOp8vMiQ67X0t/ZQWI28JLf/Sripc7O
 kSWr6RaxB1TwOlu2yUu+dZZUq2v++MYSguz0ucmHLjqGxUdyvVLY9Z/sjCeA8ZDE
 hDO7OSbROmAqxUyqkpOeVfwEh/x4KRakLMgunbIlPMlmx0IXT3c=
 =potp
 -----END PGP SIGNATURE-----

Merge tag 'kvm-x86-mmu-7.4' of https://github.com/kvm-x86/linux into HEAD

KVM x86 MMU changes for 7.4

 - Grab KVM's SRCU "lock" in kvm_zap_gfn_range() to fix a recurring bug where
   callers fail to do so.

 - Fix a variety of issues in the pre-fault code that are mostly benign in the
   current code base, but will show up in big ways if/when the pre-faulting
   logic is used to implementing prefetch-on-fault for the TDP MMU.

 - Guard rmaps with mmu_lock for PREEMPT_RT=y kernels to fix a longstanding
   issue introducing by "lockless aging" where KVM would take its effective
   raw bit-spinlock with preemption enabled (because mmu_lock gets turned into
   a sleepable lock).

 - Prepare a memory_fault exit (paired with -EFAULT) when KVM hits an error
   PFN, e.g. so that userspace can resolve transient issues instead of killing
   the guest.  Convert a variety of WARN()+EFAULT paths into KVM_BUG_ON()+EIO
   so that KVM can hard commit to filling memory_fault on -EFAULTs that occur
   in the page fault path.

 - Fix a NX hugepage un-accounting bug where KVM would always uncharge the
   shadow MMU, even when handling TDP MMU pages.
This commit is contained in:
Paolo Bonzini 2026-10-16 09:51:00 +02:00
commit 7930afcf14
3 changed files with 97 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

124
arch/x86/kvm/mmu/mmu.c
View File

@ -864,11 +864,12 @@ void untrack_possible_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp,
list_del_init(&sp->possible_nx_huge_page_link);
}
static void unaccount_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp)
static void unaccount_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp,
enum kvm_mmu_type mmu_type)
{
sp->nx_huge_page_disallowed = false;
untrack_possible_nx_huge_page(kvm, sp, KVM_SHADOW_MMU);
untrack_possible_nx_huge_page(kvm, sp, mmu_type);
}
static struct kvm_memory_slot *gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu,
@ -895,6 +896,7 @@ static struct kvm_memory_slot *gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu
*/
#define KVM_RMAP_MANY BIT(0)
#ifndef CONFIG_PREEMPT_RT
/*
* rmaps and PTE lists are mostly protected by mmu_lock (the shadow MMU always
* operates with mmu_lock held for write), but rmaps can be walked without
@ -1008,7 +1010,8 @@ static unsigned long kvm_rmap_get(struct kvm_rmap_head *rmap_head)
* actual locking is the same, but the caller is disallowed from modifying the
* rmap, and so the unlock flow is a nop if the rmap is/was empty.
*/
static unsigned long kvm_rmap_lock_readonly(struct kvm_rmap_head *rmap_head)
static unsigned long kvm_rmap_lock_readonly(struct kvm *kvm,
struct kvm_rmap_head *rmap_head)
{
unsigned long rmap_val;
@ -1032,6 +1035,35 @@ static void kvm_rmap_unlock_readonly(struct kvm_rmap_head *rmap_head,
__kvm_rmap_unlock(rmap_head, old_val);
preempt_enable();
}
#else
static unsigned long kvm_rmap_get(struct kvm_rmap_head *rmap_head)
{
return atomic_long_read(&rmap_head->val);
}
static unsigned long kvm_rmap_lock(struct kvm *kvm,
struct kvm_rmap_head *rmap_head)
{
lockdep_assert_held_write(&kvm->mmu_lock);
return kvm_rmap_get(rmap_head);
}
static void kvm_rmap_unlock(struct kvm *kvm,
struct kvm_rmap_head *rmap_head,
unsigned long new_val)
{
atomic_long_set_release(&rmap_head->val, new_val);
}
static unsigned long kvm_rmap_lock_readonly(struct kvm *kvm,
struct kvm_rmap_head *rmap_head)
{
lockdep_assert_held_read(&kvm->mmu_lock);
return kvm_rmap_get(rmap_head);
}
static void kvm_rmap_unlock_readonly(struct kvm_rmap_head *rmap_head,
unsigned long old_val) { }
#endif
/*
* Returns the number of pointers in the rmap chain, not counting the new one.
@ -1745,11 +1777,24 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm,
gfn_t gfn;
int level;
BUILD_BUG_ON(!IS_ENABLED(CONFIG_KVM_MMU_LOCKLESS_AGING));
/*
* For realtime kernels, do aging under mmu_lock (per-rmap locking is
* compiled out), as realtime deployments are unlikely to benefit from
* increased aging throughput and reduced jitter for memory-overcommitted
* nested VMs, whereas keeping preemption enabled is extremely valuable
* (mmu_lock becomes a sleepable lock on realtime kernels).
*/
#ifdef CONFIG_PREEMPT_RT
guard(read_lock)(&kvm->mmu_lock);
#endif
for (level = PG_LEVEL_4K; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) {
for (gfn = range->start; gfn < range->end;
gfn += KVM_PAGES_PER_HPAGE(level)) {
rmap_head = gfn_to_rmap(gfn, level, range->slot);
rmap_val = kvm_rmap_lock_readonly(rmap_head);
rmap_val = kvm_rmap_lock_readonly(kvm, rmap_head);
for_each_rmap_spte_lockless(rmap_val, &iter, sptep, old_spte) {
if (!is_accessed_spte(old_spte))
@ -2791,7 +2836,7 @@ static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm,
}
if (sp->nx_huge_page_disallowed)
unaccount_nx_huge_page(kvm, sp);
unaccount_nx_huge_page(kvm, sp, KVM_SHADOW_MMU);
sp->role.invalid = 1;
@ -3027,6 +3072,9 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot,
if (prefetch)
return -EEXIST;
if (KVM_BUG_ON(sp->role.level != PG_LEVEL_4K, kvm))
continue;
/*
* TDP MMU page faults require an additional spinlock as they
* run with mmu_lock held for read, not write, and the unsync
@ -3050,7 +3098,6 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot,
continue;
}
WARN_ON_ONCE(sp->role.level != PG_LEVEL_4K);
kvm_unsync_page(kvm, sp);
}
if (locked)
@ -3537,8 +3584,8 @@ static int direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
fault->req_level >= it.level);
}
if (WARN_ON_ONCE(it.level != fault->goal_level))
return -EFAULT;
if (KVM_BUG_ON(it.level != fault->goal_level, vcpu->kvm))
return -EIO;
ret = mmu_set_spte(vcpu, fault->slot, it.sptep, access,
base_gfn, fault->pfn, fault);
@ -3576,6 +3623,7 @@ static int kvm_handle_error_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fa
return RET_PF_RETRY;
}
kvm_mmu_prepare_memory_fault_exit(vcpu, fault);
return -EFAULT;
}
@ -4901,8 +4949,8 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
#ifndef CONFIG_X86_64
/* A 64-bit CR2 should be impossible on 32-bit KVM. */
if (WARN_ON_ONCE(fault_address >> 32))
return -EFAULT;
if (KVM_BUG_ON(fault_address >> 32, vcpu->kvm))
return -EIO;
#endif
/*
* Legacy #PF exception only have a 32-bit error code. Simply drop the
@ -5067,6 +5115,10 @@ static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa,
if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu))
return -EIO;
r = kvm_mmu_reload(vcpu);
if (r)
return r;
cond_resched();
r = kvm_mmu_do_page_fault(vcpu, gpa, error_code, true, NULL, level);
} while (r == RET_PF_RETRY);
@ -5112,14 +5164,6 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
if (kvm_is_gfn_alias(vcpu->kvm, gpa_to_gfn(range->gpa)))
return -EINVAL;
/*
* reload is efficient when called repeatedly, so we can do it on
* every iteration.
*/
r = kvm_mmu_reload(vcpu);
if (r)
return r;
direct_bits = 0;
if (kvm_arch_has_private_mem(vcpu->kvm) &&
kvm_is_private_gfn(vcpu->kvm, gpa_to_gfn(range->gpa)))
@ -5224,14 +5268,6 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
if (kvm_gfn_is_write_tracked(kvm, fault.slot, fault.gfn))
return -EPERM;
r = kvm_mmu_reload(vcpu);
if (r)
return r;
r = mmu_topup_memory_caches(vcpu, false);
if (r)
return r;
do {
if (signal_pending(current))
return -EINTR;
@ -5239,10 +5275,38 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu))
return -EIO;
r = kvm_mmu_reload(vcpu);
if (r)
return r;
r = mmu_topup_memory_caches(vcpu, false);
if (r)
return r;
cond_resched();
guard(read_lock)(&kvm->mmu_lock);
/*
* Because slots_lock is held, it should be impossible for *any*
* roots to be invalidated after the initial MMU reload. WARN,
* but continue on; the above MMU reload will do the right thing
* if the current root is actually invalid.
*/
WARN_ON_ONCE(kvm_test_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu));
/*
* Snapshot the invalidation sequence counter after acquiring
* mmu_lock, as guest_memfd guarantees the validity of the pfn,
* i.e. any concurrent invalidations are guaranteed to be
* irrelevant.
*/
fault.mmu_seq = vcpu->kvm->mmu_invalidate_seq;
if (is_page_fault_stale(vcpu, &fault)) {
r = RET_PF_RETRY;
continue;
}
r = kvm_tdp_mmu_map(vcpu, &fault);
} while (r == RET_PF_RETRY);
@ -6606,8 +6670,8 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err
r = RET_PF_INVALID;
if (unlikely(error_code & PFERR_RSVD_MASK)) {
if (WARN_ON_ONCE(error_code & PFERR_PRIVATE_ACCESS))
return -EFAULT;
if (KVM_BUG_ON(error_code & PFERR_PRIVATE_ACCESS, vcpu->kvm))
return -EIO;
r = handle_mmio_page_fault(vcpu, cr2_or_gpa, direct);
if (r == RET_PF_EMULATE)
@ -7137,6 +7201,8 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
if (WARN_ON_ONCE(gfn_end <= gfn_start))
return;
guard(srcu)(&kvm->srcu);
write_lock(&kvm->mmu_lock);
kvm_mmu_invalidate_start(kvm);
@ -8005,7 +8071,7 @@ static void kvm_recover_nx_huge_pages(struct kvm *kvm,
WARN_ON_ONCE(!sp->nx_huge_page_disallowed);
WARN_ON_ONCE(!sp->role.direct);
unaccount_nx_huge_page(kvm, sp);
unaccount_nx_huge_page(kvm, sp, mmu_type);
if (is_tdp_mmu)
spin_unlock(tdp_mmu_pages_lock);

4
arch/x86/kvm/mmu/paging_tmpl.h
View File

@ -778,8 +778,8 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
fault->req_level >= it.level);
}
if (WARN_ON_ONCE(it.level != fault->goal_level))
return -EFAULT;
if (KVM_BUG_ON(it.level != fault->goal_level, vcpu->kvm))
return -EIO;
ret = mmu_set_spte(vcpu, fault->slot, it.sptep, gw->pte_access,
base_gfn, fault->pfn, fault);

2
arch/x86/kvm/x86.c
View File

@ -7938,10 +7938,8 @@ void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
kvm->arch.apicv_inhibit_reasons = new;
if (new) {
unsigned long gfn = gpa_to_gfn(APIC_DEFAULT_PHYS_BASE);
int idx = srcu_read_lock(&kvm->srcu);
kvm_zap_gfn_range(kvm, gfn, gfn+1);
srcu_read_unlock(&kvm->srcu, idx);
}
} else {
kvm->arch.apicv_inhibit_reasons = new;