github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-23 22:14:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

KVM: x86/mmu: unify root_gva_walk and ngva_walk

Browse Source 
At this point, vcpu->arch.ngva_walk and vcpu->arch.root_gva_walk contain
the same information; compare init_kvm_page_walk() on one side with
init_kvm_softmmu() + shadow_mmu_init_context() on the other.  They only
differ in when each is active, and root_gva_walk is also used by shadow
paging, via FNAME(walk_addr) and its callers.

Always use the same instance of kvm_pagewalk to do GVA->GPA translations,
for both guest emulation and shadow paging, instead of flipping the
gva_walk pointer back and forth.  After all the page walking does behave
the same no matter if you are in guest mode or not; the difference lies
in the behavior of kvm_translate_gpa and thus in vcpu->arch.mmu, not in
the page walker itself.

This completes the transition from walk_mmu/nested_mmu as the page
walking entry points to gva_walk/ngpa_walk, and removes duplicated
code between the initialization of root_mmu.w and ngva_walk (The
Struct Formerly Known As nested_mmu).

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
This commit is contained in:
Paolo Bonzini 2026-05-11 06:19:00 -04:00
parent 0a59a8b8a2
commit 148fe965df
9 changed files with 60 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
arch/x86/include/asm/kvm_host.h
View File

@ -905,26 +905,15 @@ struct kvm_vcpu_arch {
/* Non-nested MMU for L1 */
struct kvm_mmu root_mmu;
struct kvm_pagewalk root_gva_walk;
/* L1 TDP when running nested */
struct kvm_mmu guest_mmu;
struct kvm_pagewalk ngpa_walk;
/*
* Paging state of an L2 guest (used for nested npt)
*
* This context will save all necessary information to walk page tables
* of an L2 guest. This context is only initialized for page table
* walking and not for faulting since we never handle l2 page faults on
* the host.
*/
struct kvm_pagewalk ngva_walk;
/*
* Pagewalk context used for gva_to_gpa translations.
*/
struct kvm_pagewalk *gva_walk;
struct kvm_pagewalk gva_walk;
u64 pdptrs[4]; /* pae */

2
arch/x86/kvm/hyperv.c
View File

@ -2046,7 +2046,7 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
* read with kvm_read_guest().
*/
if (!hc->fast) {
hc->ingpa = kvm_translate_gpa(vcpu, vcpu->arch.gva_walk, hc->ingpa,
hc->ingpa = kvm_translate_gpa(vcpu, &vcpu->arch.gva_walk, hc->ingpa,
PFERR_GUEST_FINAL_MASK, NULL, 0);
if (unlikely(hc->ingpa == INVALID_GPA))
return HV_STATUS_INVALID_HYPERCALL_INPUT;

8
arch/x86/kvm/mmu.h
View File

@ -253,9 +253,9 @@ static inline void kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
* @w's snapshot of CR0.WP and thus all related paging metadata may
* be stale. Refresh CR0.WP and the metadata on-demand when checking
* for permission faults. Exempt nested MMUs, i.e. MMUs for shadowing
* nEPT and nNPT, as CR0.WP is ignored in both cases. Note, KVM does
* need to refresh ngva_walk, a.k.a. the walker used to translate L2
* GVAs to GPAs, so as to honor L2's CR0.WP.
* nEPT and nNPT, as CR0.WP is ignored in both cases. Note, KVM will
* still refresh gva_walk, so as to honor L2's CR0.WP when translating
* L2 GVAs to GPAs.
*/
if (!tdp_enabled || w == &vcpu->arch.ngpa_walk)
return;
@ -382,7 +382,7 @@ static inline gpa_t kvm_translate_gpa(struct kvm_vcpu *vcpu,
struct x86_exception *exception,
u64 pte_access)
{
if (w != &vcpu->arch.ngva_walk)
if (!mmu_is_nested(vcpu) || w == &vcpu->arch.ngpa_walk)
return gpa;
return kvm_x86_ops.nested_ops->translate_nested_gpa(vcpu, gpa, access,
exception,

120
arch/x86/kvm/mmu/mmu.c
View File

@ -5212,7 +5212,6 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_tdp_mmu_map_private_pfn);
static void nonpaging_init_context(struct kvm_mmu *context)
{
context->page_fault = nonpaging_page_fault;
context->w->gva_to_gpa = nonpaging_gva_to_gpa;
context->sync_spte = NULL;
}
@ -5843,14 +5842,12 @@ static void reset_guest_paging_metadata(struct kvm_vcpu *vcpu,
static void paging64_init_context(struct kvm_mmu *context)
{
context->page_fault = paging64_page_fault;
context->w->gva_to_gpa = paging64_gva_to_gpa;
context->sync_spte = paging64_sync_spte;
}
static void paging32_init_context(struct kvm_mmu *context)
{
context->page_fault = paging32_page_fault;
context->w->gva_to_gpa = paging32_gva_to_gpa;
context->sync_spte = paging32_sync_spte;
}
@ -5965,39 +5962,22 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
struct kvm_mmu *context = &vcpu->arch.root_mmu;
union kvm_mmu_page_role root_role = kvm_calc_tdp_mmu_root_page_role(vcpu, cpu_role);
if (cpu_role.as_u64 == context->w->cpu_role.as_u64 &&
root_role.word == context->root_role.word)
if (root_role.word == context->root_role.word)
return;
context->w->cpu_role.as_u64 = cpu_role.as_u64;
context->root_role.word = root_role.word;
context->page_fault = kvm_tdp_page_fault;
context->sync_spte = NULL;
context->w->inject_page_fault = kvm_inject_page_fault;
context->w->get_pdptr = kvm_pdptr_read;
context->w->get_guest_pgd = get_guest_cr3;
if (!is_cr0_pg(context->w))
context->w->gva_to_gpa = nonpaging_gva_to_gpa;
else if (is_cr4_pae(context->w))
context->w->gva_to_gpa = paging64_gva_to_gpa;
else
context->w->gva_to_gpa = paging32_gva_to_gpa;
reset_guest_paging_metadata(vcpu, context->w);
reset_tdp_shadow_zero_bits_mask(context);
}
static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *context,
union kvm_cpu_role cpu_role,
union kvm_mmu_page_role root_role)
{
if (cpu_role.as_u64 == context->w->cpu_role.as_u64 &&
root_role.word == context->root_role.word)
if (root_role.word == context->root_role.word)
return;
context->w->cpu_role.as_u64 = cpu_role.as_u64;
context->root_role.word = root_role.word;
if (!is_cr0_pg(context->w))
@ -6007,7 +5987,6 @@ static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *conte
else
paging32_init_context(context);
reset_guest_paging_metadata(vcpu, context->w);
reset_shadow_zero_bits_mask(vcpu, context);
}
@ -6033,7 +6012,28 @@ static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu,
*/
root_role.efer_nx = true;
shadow_mmu_init_context(vcpu, context, cpu_role, root_role);
shadow_mmu_init_context(vcpu, context, root_role);
}
static void init_kvm_page_walk(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
union kvm_cpu_role cpu_role)
{
if (cpu_role.as_u64 == w->cpu_role.as_u64)
return;
w->cpu_role.as_u64 = cpu_role.as_u64;
w->inject_page_fault = kvm_inject_page_fault;
w->get_pdptr = kvm_pdptr_read;
w->get_guest_pgd = get_guest_cr3;
if (!is_cr0_pg(w))
w->gva_to_gpa = nonpaging_gva_to_gpa;
else if (is_cr4_pae(w))
w->gva_to_gpa = paging64_gva_to_gpa;
else
w->gva_to_gpa = paging32_gva_to_gpa;
reset_guest_paging_metadata(vcpu, w);
}
void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr4,
@ -6052,13 +6052,15 @@ void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr4,
WARN_ON_ONCE(cpu_role.base.direct || !cpu_role.base.guest_mode);
cpu_role.base.cr4_smep = (misc_ctl & SVM_MISC_ENABLE_GMET) != 0;
init_kvm_page_walk(vcpu, &vcpu->arch.ngpa_walk, cpu_role);
root_role = cpu_role.base;
root_role.level = kvm_mmu_get_tdp_level(vcpu);
if (root_role.level == PT64_ROOT_5LEVEL &&
cpu_role.base.level == PT64_ROOT_4LEVEL)
root_role.passthrough = 1;
shadow_mmu_init_context(vcpu, context, cpu_role, root_role);
shadow_mmu_init_context(vcpu, context, root_role);
kvm_mmu_new_pgd(vcpu, nested_cr3);
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_init_shadow_npt_mmu);
@ -6123,46 +6125,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_init_shadow_ept_mmu);
static void init_kvm_softmmu(struct kvm_vcpu *vcpu,
union kvm_cpu_role cpu_role)
{
struct kvm_mmu *context = &vcpu->arch.root_mmu;
kvm_init_shadow_mmu(vcpu, cpu_role);
context->w->inject_page_fault = kvm_inject_page_fault;
context->w->get_pdptr = kvm_pdptr_read;
context->w->get_guest_pgd = get_guest_cr3;
}
static void init_kvm_ngva_walk(struct kvm_vcpu *vcpu,
union kvm_cpu_role new_mode)
{
struct kvm_pagewalk *g_context = &vcpu->arch.ngva_walk;
if (new_mode.as_u64 == g_context->cpu_role.as_u64)
return;
g_context->cpu_role.as_u64 = new_mode.as_u64;
g_context->inject_page_fault = kvm_inject_page_fault;
g_context->get_pdptr = kvm_pdptr_read;
g_context->get_guest_pgd = get_guest_cr3;
/*
* Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using
* L1's nested page tables (e.g. EPT12). The nested translation
* of l2_gva to l1_gpa is done by arch.ngva_walk.gva_to_gpa using
* L2's page tables as the first level of translation and L1's
* nested page tables as the second level of translation. Basically
* the gva_to_gpa functions between mmu and ngva_walk are swapped.
*/
if (!is_paging(vcpu))
g_context->gva_to_gpa = nonpaging_gva_to_gpa;
else if (is_long_mode(vcpu))
g_context->gva_to_gpa = paging64_gva_to_gpa;
else if (is_pae(vcpu))
g_context->gva_to_gpa = paging64_gva_to_gpa;
else
g_context->gva_to_gpa = paging32_gva_to_gpa;
reset_guest_paging_metadata(vcpu, g_context);
}
void kvm_init_mmu(struct kvm_vcpu *vcpu)
@ -6170,12 +6133,14 @@ void kvm_init_mmu(struct kvm_vcpu *vcpu)
struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu);
union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, &regs);
if (mmu_is_nested(vcpu))
init_kvm_ngva_walk(vcpu, cpu_role);
else if (tdp_enabled)
init_kvm_tdp_mmu(vcpu, cpu_role);
else
init_kvm_softmmu(vcpu, cpu_role);
init_kvm_page_walk(vcpu, &vcpu->arch.gva_walk, cpu_role);
if (!mmu_is_nested(vcpu)) {
if (tdp_enabled)
init_kvm_tdp_mmu(vcpu, cpu_role);
else
init_kvm_softmmu(vcpu, cpu_role);
}
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_init_mmu);
@ -6195,9 +6160,8 @@ void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu)
*/
vcpu->arch.root_mmu.root_role.invalid = 1;
vcpu->arch.guest_mmu.root_role.invalid = 1;
vcpu->arch.root_gva_walk.cpu_role.ext.valid = 0;
vcpu->arch.ngpa_walk.cpu_role.ext.valid = 0;
vcpu->arch.ngva_walk.cpu_role.ext.valid = 0;
vcpu->arch.gva_walk.cpu_role.ext.valid = 0;
kvm_mmu_reset_context(vcpu);
KVM_BUG_ON(!kvm_can_set_cpuid_and_feature_msrs(vcpu), vcpu->kvm);
@ -6693,13 +6657,14 @@ void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
WARN_ON_ONCE(roots & ~KVM_MMU_ROOTS_ALL);
/* It's actually a GPA for vcpu->arch.guest_mmu. */
if (w == vcpu->arch.gva_walk) {
if (w == &vcpu->arch.gva_walk) {
/* INVLPG on a non-canonical address is a NOP according to the SDM. */
if (is_noncanonical_invlpg_address(addr, vcpu))
return;
kvm_x86_call(flush_tlb_gva)(vcpu, addr);
if (w == &vcpu->arch.ngva_walk)
if (tdp_enabled)
return;
mmu = &vcpu->arch.root_mmu;
@ -6733,7 +6698,7 @@ void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
* be synced when switching to that new cr3, so nothing needs to be
* done here for them.
*/
kvm_mmu_invalidate_addr(vcpu, vcpu->arch.gva_walk, gva, KVM_MMU_ROOTS_ALL);
kvm_mmu_invalidate_addr(vcpu, &vcpu->arch.gva_walk, gva, KVM_MMU_ROOTS_ALL);
++vcpu->stat.invlpg;
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_invlpg);
@ -6755,7 +6720,7 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
}
if (roots)
kvm_mmu_invalidate_addr(vcpu, mmu->w, gva, roots);
kvm_mmu_invalidate_addr(vcpu, &vcpu->arch.gva_walk, gva, roots);
++vcpu->stat.invlpg;
/*
@ -6871,13 +6836,12 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu)
vcpu->arch.mmu_shadow_page_cache.gfp_zero = __GFP_ZERO;
vcpu->arch.mmu = &vcpu->arch.root_mmu;
vcpu->arch.gva_walk = &vcpu->arch.root_gva_walk;
ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu, &vcpu->arch.ngpa_walk);
if (ret)
return ret;
ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu, &vcpu->arch.root_gva_walk);
ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu, &vcpu->arch.gva_walk);
if (ret)
goto fail_allocate_root;

4
arch/x86/kvm/mmu/paging_tmpl.h
View File

@ -548,7 +548,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
}
#endif
walker->fault.address = addr;
walker->fault.nested_page_fault = w != vcpu->arch.gva_walk;
walker->fault.nested_page_fault = w != &vcpu->arch.gva_walk;
walker->fault.async_page_fault = false;
#if PTTYPE != PTTYPE_EPT
@ -906,7 +906,7 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
#ifndef CONFIG_X86_64
/* A 64-bit GVA should be impossible on 32-bit KVM. */
WARN_ON_ONCE((addr >> 32) && w == vcpu->arch.gva_walk);
WARN_ON_ONCE((addr >> 32) && w == &vcpu->arch.gva_walk);
#endif
r = FNAME(walk_addr_generic)(&walker, vcpu, w, addr, access);

2
arch/x86/kvm/regs.c
View File

@ -154,7 +154,7 @@ static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
*/
int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
{
struct kvm_pagewalk *w = vcpu->arch.gva_walk;
struct kvm_pagewalk *w = &vcpu->arch.gva_walk;
gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
gpa_t real_gpa;
int i;

2
arch/x86/kvm/svm/nested.c
View File

@ -117,13 +117,11 @@ static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
vcpu->arch.ngpa_walk.get_guest_pgd = nested_svm_get_tdp_cr3;
vcpu->arch.ngpa_walk.get_pdptr = nested_svm_get_tdp_pdptr;
vcpu->arch.ngpa_walk.inject_page_fault = nested_svm_inject_npf_exit;
vcpu->arch.gva_walk = &vcpu->arch.ngva_walk;
}
static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
{
vcpu->arch.mmu = &vcpu->arch.root_mmu;
vcpu->arch.gva_walk = vcpu->arch.root_mmu.w;
}
static bool nested_vmcb_needs_vls_intercept(struct vcpu_svm *svm)

3
arch/x86/kvm/vmx/nested.c
View File

@ -516,14 +516,11 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
vcpu->arch.ngpa_walk.get_pdptr = kvm_pdptr_read;
vcpu->arch.ngpa_walk.inject_page_fault = nested_ept_inject_page_fault;
vcpu->arch.gva_walk = &vcpu->arch.ngva_walk;
}
static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
{
vcpu->arch.mmu = &vcpu->arch.root_mmu;
vcpu->arch.gva_walk = vcpu->arch.root_mmu.w;
}
static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,

18
arch/x86/kvm/x86.c
View File

@ -587,7 +587,7 @@ void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
WARN_ON_ONCE(fault->vector != PF_VECTOR);
fault_walk = fault->nested_page_fault ? &vcpu->arch.ngpa_walk :
vcpu->arch.gva_walk;
&vcpu->arch.gva_walk;
/*
* Invalidate the TLB entry for the faulting address, if it exists,
@ -4769,7 +4769,7 @@ static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
struct x86_exception *exception)
{
struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, exception);
@ -4779,7 +4779,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_gva_to_gpa_read);
gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
struct x86_exception *exception)
{
struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
access |= PFERR_WRITE_MASK;
@ -4791,7 +4791,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_gva_to_gpa_write);
gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
struct x86_exception *exception)
{
struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, 0, exception);
}
@ -4800,7 +4800,7 @@ static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
struct kvm_vcpu *vcpu, u64 access,
struct x86_exception *exception)
{
struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
void *data = val;
int r = X86EMUL_CONTINUE;
@ -4833,7 +4833,7 @@ static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
struct x86_exception *exception)
{
struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
unsigned offset;
int ret;
@ -4892,7 +4892,7 @@ static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes
struct kvm_vcpu *vcpu, u64 access,
struct x86_exception *exception)
{
struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
void *data = val;
int r = X86EMUL_CONTINUE;
@ -4998,7 +4998,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
gpa_t *gpa, struct x86_exception *exception,
bool write)
{
struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
u64 access = ((kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0)
| (write ? PFERR_WRITE_MASK : 0);
@ -10601,7 +10601,7 @@ void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end)
void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code)
{
struct kvm_pagewalk *gva_walk = vcpu->arch.gva_walk;
struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
struct x86_exception fault;
u64 access = error_code &
(PFERR_WRITE_MASK | PFERR_FETCH_MASK | PFERR_USER_MASK);