From 4487492b92a4f3df4166ab27a277b3d34867e5f2 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 28 Jan 2026 17:14:33 -0800 Subject: [PATCH 01/24] x86/tdx: Use pg_level in TDX APIs, not the TDX-Module's 0-based level Rework the TDX APIs to take the kernel's 1-based pg_level enum, not the TDX-Module's 0-based level. The APIs are _kernel_ APIs, not TDX-Module APIs, and the kernel (and KVM) uses "enum pg_level" literally everywhere. Using "enum pg_level" eliminates ambiguity when looking at the APIs (it's NOT clear that "int level" refers to the TDX-Module's level), and will allow for using existing helpers like page_level_size() when support for hugepages is added to the S-EPT APIs. No functional change intended. Cc: Kai Huang Cc: Dave Hansen Cc: Rick Edgecombe Cc: Yan Zhao Cc: Vishal Annapurve Cc: Ackerley Tng Acked-by: Kiryl Shutsemau Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Rick Edgecombe Tested-by: Rick Edgecombe Acked-by: Dave Hansen Link: https://patch.msgid.link/20260129011517.3545883-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/tdx.h | 14 ++++---------- arch/x86/kvm/vmx/tdx.c | 11 ++++------- arch/x86/virt/vmx/tdx/tdx.c | 26 ++++++++++++++++++-------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index a149740b24e8..c140ddde59ff 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -187,19 +187,13 @@ static inline u64 mk_keyed_paddr(u16 hkid, struct page *page) return ret; } -static inline int pg_level_to_tdx_sept_level(enum pg_level level) -{ - WARN_ON_ONCE(level == PG_LEVEL_NONE); - return level - 1; -} - u64 tdh_vp_enter(struct tdx_vp *vp, struct tdx_module_args *args); u64 tdh_mng_addcx(struct tdx_td *td, struct page *tdcs_page); u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, struct page *page, struct page *source, u64 *ext_err1, u64 *ext_err2); -u64 tdh_mem_sept_add(struct tdx_td *td, u64 gpa, int level, struct page *page, u64 *ext_err1, u64 *ext_err2); +u64 tdh_mem_sept_add(struct tdx_td *td, u64 gpa, enum pg_level level, struct page *page, u64 *ext_err1, u64 *ext_err2); u64 tdh_vp_addcx(struct tdx_vp *vp, struct page *tdcx_page); -u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, int level, struct page *page, u64 *ext_err1, u64 *ext_err2); -u64 tdh_mem_range_block(struct tdx_td *td, u64 gpa, int level, u64 *ext_err1, u64 *ext_err2); +u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, enum pg_level level, struct page *page, u64 *ext_err1, u64 *ext_err2); +u64 tdh_mem_range_block(struct tdx_td *td, u64 gpa, enum pg_level level, u64 *ext_err1, u64 *ext_err2); u64 tdh_mng_key_config(struct tdx_td *td); u64 tdh_mng_create(struct tdx_td *td, u16 hkid); u64 tdh_vp_create(struct tdx_td *td, struct tdx_vp *vp); @@ -215,7 +209,7 @@ u64 tdh_vp_rd(struct tdx_vp *vp, u64 field, u64 *data); u64 tdh_vp_wr(struct tdx_vp *vp, u64 field, u64 data, u64 mask); u64 tdh_phymem_page_reclaim(struct page *page, u64 *tdx_pt, u64 *tdx_owner, u64 *tdx_size); u64 tdh_mem_track(struct tdx_td *tdr); -u64 tdh_mem_page_remove(struct tdx_td *td, u64 gpa, u64 level, u64 *ext_err1, u64 *ext_err2); +u64 tdh_mem_page_remove(struct tdx_td *td, u64 gpa, enum pg_level level, u64 *ext_err1, u64 *ext_err2); u64 tdh_phymem_cache_wb(bool resume); u64 tdh_phymem_page_wbinvd_tdr(struct tdx_td *td); u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, struct page *page); diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index b8c3d3d8bbfe..04aecdd3a6c0 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1638,14 +1638,13 @@ static int tdx_mem_page_add(struct kvm *kvm, gfn_t gfn, enum pg_level level, static int tdx_mem_page_aug(struct kvm *kvm, gfn_t gfn, enum pg_level level, kvm_pfn_t pfn) { - int tdx_level = pg_level_to_tdx_sept_level(level); struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); struct page *page = pfn_to_page(pfn); gpa_t gpa = gfn_to_gpa(gfn); u64 entry, level_state; u64 err; - err = tdh_mem_page_aug(&kvm_tdx->td, gpa, tdx_level, page, &entry, &level_state); + err = tdh_mem_page_aug(&kvm_tdx->td, gpa, level, page, &entry, &level_state); if (unlikely(tdx_operand_busy(err))) return -EBUSY; @@ -1689,12 +1688,11 @@ static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, static int tdx_sept_link_private_spt(struct kvm *kvm, gfn_t gfn, enum pg_level level, void *private_spt) { - int tdx_level = pg_level_to_tdx_sept_level(level); gpa_t gpa = gfn_to_gpa(gfn); struct page *page = virt_to_page(private_spt); u64 err, entry, level_state; - err = tdh_mem_sept_add(&to_kvm_tdx(kvm)->td, gpa, tdx_level, page, &entry, + err = tdh_mem_sept_add(&to_kvm_tdx(kvm)->td, gpa, level, page, &entry, &level_state); if (unlikely(tdx_operand_busy(err))) return -EBUSY; @@ -1778,7 +1776,6 @@ static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, enum pg_level level, u64 mirror_spte) { struct page *page = pfn_to_page(spte_to_pfn(mirror_spte)); - int tdx_level = pg_level_to_tdx_sept_level(level); struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); gpa_t gpa = gfn_to_gpa(gfn); u64 err, entry, level_state; @@ -1798,7 +1795,7 @@ static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, return; err = tdh_do_no_vcpus(tdh_mem_range_block, kvm, &kvm_tdx->td, gpa, - tdx_level, &entry, &level_state); + level, &entry, &level_state); if (TDX_BUG_ON_2(err, TDH_MEM_RANGE_BLOCK, entry, level_state, kvm)) return; @@ -1814,7 +1811,7 @@ static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, * Race with TDH.VP.ENTER due to (0-step mitigation) and Guest TDCALLs. */ err = tdh_do_no_vcpus(tdh_mem_page_remove, kvm, &kvm_tdx->td, gpa, - tdx_level, &entry, &level_state); + level, &entry, &level_state); if (TDX_BUG_ON_2(err, TDH_MEM_PAGE_REMOVE, entry, level_state, kvm)) return; diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index cb9b3210ab71..a6e77afafa79 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -1568,6 +1568,12 @@ static void tdx_clflush_page(struct page *page) clflush_cache_range(page_to_virt(page), PAGE_SIZE); } +static int pg_level_to_tdx_sept_level(enum pg_level level) +{ + WARN_ON_ONCE(level == PG_LEVEL_NONE); + return level - 1; +} + noinstr u64 tdh_vp_enter(struct tdx_vp *td, struct tdx_module_args *args) { args->rcx = td->tdvpr_pa; @@ -1608,10 +1614,11 @@ u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, struct page *page, struct page } EXPORT_SYMBOL_FOR_KVM(tdh_mem_page_add); -u64 tdh_mem_sept_add(struct tdx_td *td, u64 gpa, int level, struct page *page, u64 *ext_err1, u64 *ext_err2) +u64 tdh_mem_sept_add(struct tdx_td *td, u64 gpa, enum pg_level level, + struct page *page, u64 *ext_err1, u64 *ext_err2) { struct tdx_module_args args = { - .rcx = gpa | level, + .rcx = gpa | pg_level_to_tdx_sept_level(level), .rdx = tdx_tdr_pa(td), .r8 = page_to_phys(page), }; @@ -1639,10 +1646,11 @@ u64 tdh_vp_addcx(struct tdx_vp *vp, struct page *tdcx_page) } EXPORT_SYMBOL_FOR_KVM(tdh_vp_addcx); -u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, int level, struct page *page, u64 *ext_err1, u64 *ext_err2) +u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, enum pg_level level, + struct page *page, u64 *ext_err1, u64 *ext_err2) { struct tdx_module_args args = { - .rcx = gpa | level, + .rcx = gpa | pg_level_to_tdx_sept_level(level), .rdx = tdx_tdr_pa(td), .r8 = page_to_phys(page), }; @@ -1658,10 +1666,11 @@ u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, int level, struct page *page, u } EXPORT_SYMBOL_FOR_KVM(tdh_mem_page_aug); -u64 tdh_mem_range_block(struct tdx_td *td, u64 gpa, int level, u64 *ext_err1, u64 *ext_err2) +u64 tdh_mem_range_block(struct tdx_td *td, u64 gpa, enum pg_level level, + u64 *ext_err1, u64 *ext_err2) { struct tdx_module_args args = { - .rcx = gpa | level, + .rcx = gpa | pg_level_to_tdx_sept_level(level), .rdx = tdx_tdr_pa(td), }; u64 ret; @@ -1874,10 +1883,11 @@ u64 tdh_mem_track(struct tdx_td *td) } EXPORT_SYMBOL_FOR_KVM(tdh_mem_track); -u64 tdh_mem_page_remove(struct tdx_td *td, u64 gpa, u64 level, u64 *ext_err1, u64 *ext_err2) +u64 tdh_mem_page_remove(struct tdx_td *td, u64 gpa, enum pg_level level, + u64 *ext_err1, u64 *ext_err2) { struct tdx_module_args args = { - .rcx = gpa | level, + .rcx = gpa | pg_level_to_tdx_sept_level(level), .rdx = tdx_tdr_pa(td), }; u64 ret; From 02eaaffdd8656084f130e000a7c1a4ef27ac87a8 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 28 Jan 2026 17:14:34 -0800 Subject: [PATCH 02/24] KVM: x86/mmu: Update iter->old_spte if cmpxchg64 on mirror SPTE "fails" Pass a pointer to iter->old_spte, not simply its value, when setting an external SPTE in __tdp_mmu_set_spte_atomic(), so that the iterator's value will be updated if the cmpxchg64 to freeze the mirror SPTE fails. The bug is currently benign as TDX is mutualy exclusive with all paths that do "local" retry", e.g. clear_dirty_gfn_range() and wrprot_gfn_range(). Fixes: 77ac7079e66d ("KVM: x86/tdp_mmu: Propagate building mirror page tables") Reviewed-by: Kai Huang Reviewed-by: Rick Edgecombe Link: https://patch.msgid.link/20260129011517.3545883-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/tdp_mmu.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 5a2f8ce9a32b..f98afc3422ce 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -509,10 +509,10 @@ static void *get_external_spt(gfn_t gfn, u64 new_spte, int level) } static int __must_check set_external_spte_present(struct kvm *kvm, tdp_ptep_t sptep, - gfn_t gfn, u64 old_spte, + gfn_t gfn, u64 *old_spte, u64 new_spte, int level) { - bool was_present = is_shadow_present_pte(old_spte); + bool was_present = is_shadow_present_pte(*old_spte); bool is_present = is_shadow_present_pte(new_spte); bool is_leaf = is_present && is_last_spte(new_spte, level); int ret = 0; @@ -525,7 +525,7 @@ static int __must_check set_external_spte_present(struct kvm *kvm, tdp_ptep_t sp * page table has been modified. Use FROZEN_SPTE similar to * the zapping case. */ - if (!try_cmpxchg64(rcu_dereference(sptep), &old_spte, FROZEN_SPTE)) + if (!try_cmpxchg64(rcu_dereference(sptep), old_spte, FROZEN_SPTE)) return -EBUSY; /* @@ -541,7 +541,7 @@ static int __must_check set_external_spte_present(struct kvm *kvm, tdp_ptep_t sp ret = kvm_x86_call(link_external_spt)(kvm, gfn, level, external_spt); } if (ret) - __kvm_tdp_mmu_write_spte(sptep, old_spte); + __kvm_tdp_mmu_write_spte(sptep, *old_spte); else __kvm_tdp_mmu_write_spte(sptep, new_spte); return ret; @@ -670,7 +670,7 @@ static inline int __must_check __tdp_mmu_set_spte_atomic(struct kvm *kvm, return -EBUSY; ret = set_external_spte_present(kvm, iter->sptep, iter->gfn, - iter->old_spte, new_spte, iter->level); + &iter->old_spte, new_spte, iter->level); if (ret) return ret; } else { From a8b2924676ec42d76597f849baba31acc3ba1bfc Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 28 Jan 2026 17:14:35 -0800 Subject: [PATCH 03/24] KVM: TDX: Account all non-transient page allocations for per-TD structures Account all non-transient allocations associated with a single TD (or its vCPUs), as KVM's ABI is that allocations that are active for the lifetime of a VM are accounted. Leave temporary allocations, i.e. allocations that are freed within a single function/ioctl, unaccounted, to again align with KVM's existing behavior, e.g. see commit dd103407ca31 ("KVM: X86: Remove unnecessary GFP_KERNEL_ACCOUNT for temporary variables"). Fixes: 8d032b683c29 ("KVM: TDX: create/destroy VM structure") Fixes: a50f673f25e0 ("KVM: TDX: Do TDX specific vcpu initialization") Cc: stable@vger.kernel.org Reviewed-by: Kai Huang Reviewed-by: Rick Edgecombe Link: https://patch.msgid.link/20260129011517.3545883-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 04aecdd3a6c0..72d916c957bc 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2384,20 +2384,20 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params, ret = -ENOMEM; - tdr_page = alloc_page(GFP_KERNEL); + tdr_page = alloc_page(GFP_KERNEL_ACCOUNT); if (!tdr_page) goto free_hkid; kvm_tdx->td.tdcs_nr_pages = tdx_sysinfo->td_ctrl.tdcs_base_size / PAGE_SIZE; /* TDVPS = TDVPR(4K page) + TDCX(multiple 4K pages), -1 for TDVPR. */ kvm_tdx->td.tdcx_nr_pages = tdx_sysinfo->td_ctrl.tdvps_base_size / PAGE_SIZE - 1; - tdcs_pages = kzalloc_objs(*kvm_tdx->td.tdcs_pages, - kvm_tdx->td.tdcs_nr_pages); + tdcs_pages = kzalloc_objs(*kvm_tdx->td.tdcs_pages, kvm_tdx->td.tdcs_nr_pages, + GFP_KERNEL_ACCOUNT); if (!tdcs_pages) goto free_tdr; for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) { - tdcs_pages[i] = alloc_page(GFP_KERNEL); + tdcs_pages[i] = alloc_page(GFP_KERNEL_ACCOUNT); if (!tdcs_pages[i]) goto free_tdcs; } @@ -2872,7 +2872,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx) int ret, i; u64 err; - page = alloc_page(GFP_KERNEL); + page = alloc_page(GFP_KERNEL_ACCOUNT); if (!page) return -ENOMEM; tdx->vp.tdvpr_page = page; @@ -2885,14 +2885,14 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx) tdx->vp.tdvpr_pa = page_to_phys(tdx->vp.tdvpr_page); tdx->vp.tdcx_pages = kcalloc(kvm_tdx->td.tdcx_nr_pages, sizeof(*tdx->vp.tdcx_pages), - GFP_KERNEL); + GFP_KERNEL_ACCOUNT); if (!tdx->vp.tdcx_pages) { ret = -ENOMEM; goto free_tdvpr; } for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) { - page = alloc_page(GFP_KERNEL); + page = alloc_page(GFP_KERNEL_ACCOUNT); if (!page) { ret = -ENOMEM; goto free_tdcx; From e1a31ca28c9d1c36dd4e40468436b9ebce14e4d2 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 28 Jan 2026 17:14:36 -0800 Subject: [PATCH 04/24] KVM: x86: Make "external SPTE" ops that can fail RET0 static calls Define kvm_x86_ops .link_external_spt(), .set_external_spte(), and .free_external_spt() as RET0 static calls so that an unexpected call to a a default operation doesn't consume garbage. Fixes: 77ac7079e66d ("KVM: x86/tdp_mmu: Propagate building mirror page tables") Fixes: 94faba8999b9 ("KVM: x86/tdp_mmu: Propagate tearing down mirror page tables") Reviewed-by: Kai Huang Reviewed-by: Rick Edgecombe Link: https://patch.msgid.link/20260129011517.3545883-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index e4fca997ec79..b0269325646c 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -96,9 +96,9 @@ KVM_X86_OP_OPTIONAL_RET0(set_identity_map_addr) KVM_X86_OP_OPTIONAL_RET0(get_mt_mask) KVM_X86_OP_OPTIONAL_RET0(tdp_has_smep) KVM_X86_OP(load_mmu_pgd) -KVM_X86_OP_OPTIONAL(link_external_spt) -KVM_X86_OP_OPTIONAL(set_external_spte) -KVM_X86_OP_OPTIONAL(free_external_spt) +KVM_X86_OP_OPTIONAL_RET0(link_external_spt) +KVM_X86_OP_OPTIONAL_RET0(set_external_spte) +KVM_X86_OP_OPTIONAL_RET0(free_external_spt) KVM_X86_OP_OPTIONAL(remove_external_spte) KVM_X86_OP(has_wbinvd_exit) KVM_X86_OP(get_l2_tsc_offset) From 6ad0badd765ce6c7ddf2c70ac3b26882069a40c9 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 30 Apr 2026 09:49:29 +0800 Subject: [PATCH 05/24] x86/tdx: Use PFN directly for mapping guest private memory Remove struct page assumptions/constraints in the SEAMCALL wrapper APIs for mapping guest private memory and have them take PFN directly. Having core TDX make assumptions that guest private memory must be backed by struct page (and/or folio) will create subtle dependencies on how KVM/guest_memfd allocates/manages memory (e.g., whether it uses memory allocated from core MM, if the memory is refcounted, or if the folio is split) that are easily avoided. [1]. KVM's MMUs work with PFNs. This is very much an intentional design choice. It ensures that the KVM MMUs remain flexible and are not too tied to the regular CPU MMUs and the kernel code around them. Using 'struct page' for TDX guest memory is not a good fit anywhere near the KVM MMU code [2]. Use "kvm_pfn_t pfn" for type safety. Using this KVM type is appropriate since APIs tdh_mem_page_add() and tdh_mem_page_aug() are exported to KVM only. [ Yan: Replace "u64 pfn" with "kvm_pfn_t pfn" ] Signed-off-by: Yan Zhao Link: https://lore.kernel.org/all/aWgyhmTJphGQqO0Y@google.com [1] Link: https://lore.kernel.org/all/ac7V0g2q2hN3dU5u@google.com [2] Acked-by: Kiryl Shutsemau Reviewed-by: Xiaoyao Li Reviewed-by: Ackerley Tng Acked-by: Dave Hansen Link: https://patch.msgid.link/20260430014929.24210-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/tdx.h | 7 +++++-- arch/x86/kvm/vmx/tdx.c | 7 +++---- arch/x86/virt/vmx/tdx/tdx.c | 19 ++++++++++++------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index c140ddde59ff..648d350616e4 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -189,10 +190,12 @@ static inline u64 mk_keyed_paddr(u16 hkid, struct page *page) u64 tdh_vp_enter(struct tdx_vp *vp, struct tdx_module_args *args); u64 tdh_mng_addcx(struct tdx_td *td, struct page *tdcs_page); -u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, struct page *page, struct page *source, u64 *ext_err1, u64 *ext_err2); +u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, kvm_pfn_t pfn, struct page *source, + u64 *ext_err1, u64 *ext_err2); u64 tdh_mem_sept_add(struct tdx_td *td, u64 gpa, enum pg_level level, struct page *page, u64 *ext_err1, u64 *ext_err2); u64 tdh_vp_addcx(struct tdx_vp *vp, struct page *tdcx_page); -u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, enum pg_level level, struct page *page, u64 *ext_err1, u64 *ext_err2); +u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, enum pg_level level, kvm_pfn_t pfn, + u64 *ext_err1, u64 *ext_err2); u64 tdh_mem_range_block(struct tdx_td *td, u64 gpa, enum pg_level level, u64 *ext_err1, u64 *ext_err2); u64 tdh_mng_key_config(struct tdx_td *td); u64 tdh_mng_create(struct tdx_td *td, u16 hkid); diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 72d916c957bc..43bb9c492a4e 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1624,8 +1624,8 @@ static int tdx_mem_page_add(struct kvm *kvm, gfn_t gfn, enum pg_level level, KVM_BUG_ON(!kvm_tdx->page_add_src, kvm)) return -EIO; - err = tdh_mem_page_add(&kvm_tdx->td, gpa, pfn_to_page(pfn), - kvm_tdx->page_add_src, &entry, &level_state); + err = tdh_mem_page_add(&kvm_tdx->td, gpa, pfn, kvm_tdx->page_add_src, + &entry, &level_state); if (unlikely(tdx_operand_busy(err))) return -EBUSY; @@ -1639,12 +1639,11 @@ static int tdx_mem_page_aug(struct kvm *kvm, gfn_t gfn, enum pg_level level, kvm_pfn_t pfn) { struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); - struct page *page = pfn_to_page(pfn); gpa_t gpa = gfn_to_gpa(gfn); u64 entry, level_state; u64 err; - err = tdh_mem_page_aug(&kvm_tdx->td, gpa, level, page, &entry, &level_state); + err = tdh_mem_page_aug(&kvm_tdx->td, gpa, level, pfn, &entry, &level_state); if (unlikely(tdx_operand_busy(err))) return -EBUSY; diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index a6e77afafa79..b24b81cea5ea 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -30,7 +30,6 @@ #include #include #include -#include #include #include #include @@ -1568,6 +1567,11 @@ static void tdx_clflush_page(struct page *page) clflush_cache_range(page_to_virt(page), PAGE_SIZE); } +static void tdx_clflush_pfn(kvm_pfn_t pfn) +{ + clflush_cache_range(__va(PFN_PHYS(pfn)), PAGE_SIZE); +} + static int pg_level_to_tdx_sept_level(enum pg_level level) { WARN_ON_ONCE(level == PG_LEVEL_NONE); @@ -1594,17 +1598,18 @@ u64 tdh_mng_addcx(struct tdx_td *td, struct page *tdcs_page) } EXPORT_SYMBOL_FOR_KVM(tdh_mng_addcx); -u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, struct page *page, struct page *source, u64 *ext_err1, u64 *ext_err2) +u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, kvm_pfn_t pfn, struct page *source, + u64 *ext_err1, u64 *ext_err2) { struct tdx_module_args args = { .rcx = gpa, .rdx = tdx_tdr_pa(td), - .r8 = page_to_phys(page), + .r8 = PFN_PHYS(pfn), .r9 = page_to_phys(source), }; u64 ret; - tdx_clflush_page(page); + tdx_clflush_pfn(pfn); ret = seamcall_ret(TDH_MEM_PAGE_ADD, &args); *ext_err1 = args.rcx; @@ -1647,16 +1652,16 @@ u64 tdh_vp_addcx(struct tdx_vp *vp, struct page *tdcx_page) EXPORT_SYMBOL_FOR_KVM(tdh_vp_addcx); u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, enum pg_level level, - struct page *page, u64 *ext_err1, u64 *ext_err2) + kvm_pfn_t pfn, u64 *ext_err1, u64 *ext_err2) { struct tdx_module_args args = { .rcx = gpa | pg_level_to_tdx_sept_level(level), .rdx = tdx_tdr_pa(td), - .r8 = page_to_phys(page), + .r8 = PFN_PHYS(pfn), }; u64 ret; - tdx_clflush_page(page); + tdx_clflush_pfn(pfn); ret = seamcall_ret(TDH_MEM_PAGE_AUG, &args); *ext_err1 = args.rcx; From 4c7a1247646c46f6ab906167a5f6d5577ea63472 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 30 Apr 2026 09:49:48 +0800 Subject: [PATCH 06/24] x86/tdx: Use PFN directly for unmapping guest private memory Remove struct page assumptions/constraints in APIs for unmapping guest private memory and have them take physical address directly. Having core TDX make assumptions that guest private memory must be backed by struct page (and/or folio) will create subtle dependencies on how KVM/guest_memfd allocates/manages memory (e.g., whether it uses memory allocated from core MM, if the memory is refcounted, or if the folio is split) that are easily avoided. [1]. KVM's MMUs work with PFNs. This is very much an intentional design choice. It ensures that the KVM MMUs remain flexible and are not too tightly tied to the regular CPU MMUs and the kernel code around them. Using "struct page" for TDX guest memory is not a good fit anywhere near the KVM MMU code [2]. Therefore, for unmapping guest private memory: export tdx_quirk_reset_paddr() for direct KVM invocation, and convert the SEAMCALL wrapper API tdh_phymem_page_wbinvd_hkid() to take PFN as input (thus updating mk_keyed_paddr() and tdh_phymem_page_wbinvd_tdr()). Intentionally have KVM pass PAGE_SIZE (rather than KVM_HPAGE_SIZE(level)) to tdx_quirk_reset_paddr() in tdx_sept_remove_private_spte() to avoid mixing in huge page changes. The KVM_BUG_ON() check for !PG_LEVEL_4K in tdx_sept_remove_private_spte() justifies using PAGE_SIZE. Do not convert tdx_reclaim_page() to use PFN as input since it currently does not remove guest private memory. Use "kvm_pfn_t pfn" for type safety. Using this KVM type is appropriate since APIs tdh_phymem_page_wbinvd_hkid() and tdx_quirk_reset_paddr() are exported to KVM only. [Yan: Use kvm_pfn_t,exclude tdx_reclaim_page(),use tdx_quirk_reset_paddr()] Signed-off-by: Yan Zhao Link: https://lore.kernel.org/all/aWgyhmTJphGQqO0Y@google.com [1] Link: https://lore.kernel.org/all/ac7V0g2q2hN3dU5u@google.com [2] Acked-by: Kiryl Shutsemau Reviewed-by: Xiaoyao Li Reviewed-by: Ackerley Tng Acked-by: Dave Hansen Link: https://patch.msgid.link/20260430014948.24226-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/tdx.h | 14 +++++--------- arch/x86/kvm/vmx/tdx.c | 6 +++--- arch/x86/virt/vmx/tdx/tdx.c | 9 +++++---- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index 648d350616e4..046060ef44c7 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -154,6 +154,7 @@ u32 tdx_get_nr_guest_keyids(void); void tdx_guest_keyid_free(unsigned int keyid); void tdx_quirk_reset_page(struct page *page); +void tdx_quirk_reset_paddr(unsigned long base, unsigned long size); struct tdx_td { /* TD root structure: */ @@ -177,15 +178,10 @@ struct tdx_vp { struct page **tdcx_pages; }; -static inline u64 mk_keyed_paddr(u16 hkid, struct page *page) +static inline u64 mk_keyed_paddr(u16 hkid, kvm_pfn_t pfn) { - u64 ret; - - ret = page_to_phys(page); - /* KeyID bits are just above the physical address bits: */ - ret |= (u64)hkid << boot_cpu_data.x86_phys_bits; - - return ret; + /* KeyID bits are just above the physical address bits. */ + return PFN_PHYS(pfn) | ((u64)hkid << boot_cpu_data.x86_phys_bits); } u64 tdh_vp_enter(struct tdx_vp *vp, struct tdx_module_args *args); @@ -215,7 +211,7 @@ u64 tdh_mem_track(struct tdx_td *tdr); u64 tdh_mem_page_remove(struct tdx_td *td, u64 gpa, enum pg_level level, u64 *ext_err1, u64 *ext_err2); u64 tdh_phymem_cache_wb(bool resume); u64 tdh_phymem_page_wbinvd_tdr(struct tdx_td *td); -u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, struct page *page); +u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, kvm_pfn_t pfn); #else static inline void tdx_init(void) { } static inline u32 tdx_get_nr_guest_keyids(void) { return 0; } diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 43bb9c492a4e..d8ed1d3b7252 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1774,8 +1774,8 @@ static int tdx_sept_free_private_spt(struct kvm *kvm, gfn_t gfn, static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, enum pg_level level, u64 mirror_spte) { - struct page *page = pfn_to_page(spte_to_pfn(mirror_spte)); struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); + kvm_pfn_t pfn = spte_to_pfn(mirror_spte); gpa_t gpa = gfn_to_gpa(gfn); u64 err, entry, level_state; @@ -1814,11 +1814,11 @@ static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, if (TDX_BUG_ON_2(err, TDH_MEM_PAGE_REMOVE, entry, level_state, kvm)) return; - err = tdh_phymem_page_wbinvd_hkid((u16)kvm_tdx->hkid, page); + err = tdh_phymem_page_wbinvd_hkid((u16)kvm_tdx->hkid, pfn); if (TDX_BUG_ON(err, TDH_PHYMEM_PAGE_WBINVD, kvm)) return; - tdx_quirk_reset_page(page); + tdx_quirk_reset_paddr(PFN_PHYS(pfn), PAGE_SIZE); } void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index b24b81cea5ea..e5a37ea2d4a0 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -710,7 +710,7 @@ static __init int tdmrs_set_up_pamt_all(struct tdmr_info_list *tdmr_list, * to normal kernel memory. Systems with the X86_BUG_TDX_PW_MCE erratum need to * do the conversion explicitly via MOVDIR64B. */ -static void tdx_quirk_reset_paddr(unsigned long base, unsigned long size) +void tdx_quirk_reset_paddr(unsigned long base, unsigned long size) { const void *zero_page = (const void *)page_address(ZERO_PAGE(0)); unsigned long phys, end; @@ -729,6 +729,7 @@ static void tdx_quirk_reset_paddr(unsigned long base, unsigned long size) */ mb(); } +EXPORT_SYMBOL_FOR_KVM(tdx_quirk_reset_paddr); void tdx_quirk_reset_page(struct page *page) { @@ -1920,17 +1921,17 @@ u64 tdh_phymem_page_wbinvd_tdr(struct tdx_td *td) { struct tdx_module_args args = {}; - args.rcx = mk_keyed_paddr(tdx_global_keyid, td->tdr_page); + args.rcx = mk_keyed_paddr(tdx_global_keyid, page_to_pfn(td->tdr_page)); return seamcall(TDH_PHYMEM_PAGE_WBINVD, &args); } EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_wbinvd_tdr); -u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, struct page *page) +u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, kvm_pfn_t pfn) { struct tdx_module_args args = {}; - args.rcx = mk_keyed_paddr(hkid, page); + args.rcx = mk_keyed_paddr(hkid, pfn); return seamcall(TDH_PHYMEM_PAGE_WBINVD, &args); } From 4a72a6dc447d697441f578bb144d1add3ea8f326 Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Thu, 30 Apr 2026 09:50:01 +0800 Subject: [PATCH 07/24] x86/tdx: Drop exported function tdx_quirk_reset_page() KVM invokes tdx_quirk_reset_page() to reset TDX control pages (including S-EPT pages, TDR page, etc.), as all those pages are allocated by KVM TDX and thus always have struct page. However, it's also reasonable for KVM to reset those TDX control pages via tdx_quirk_reset_paddr() directly, eliminating the need to export two parallel APIs. Keeping tdx_quirk_reset_page() as a one-line helper in the header file is also unnecessary. No functional change intended. Suggested-by: Paolo Bonzini Suggested-by: Xiaoyao Li Signed-off-by: Yan Zhao Acked-by: Kiryl Shutsemau Reviewed-by: Xiaoyao Li Reviewed-by: Ackerley Tng Acked-by: Dave Hansen Link: https://patch.msgid.link/20260430015001.24242-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/tdx.h | 1 - arch/x86/kvm/vmx/tdx.c | 4 ++-- arch/x86/virt/vmx/tdx/tdx.c | 6 ------ 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index 046060ef44c7..1a83ef88af17 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -153,7 +153,6 @@ int tdx_guest_keyid_alloc(void); u32 tdx_get_nr_guest_keyids(void); void tdx_guest_keyid_free(unsigned int keyid); -void tdx_quirk_reset_page(struct page *page); void tdx_quirk_reset_paddr(unsigned long base, unsigned long size); struct tdx_td { diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index d8ed1d3b7252..3f956dde4a51 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -343,7 +343,7 @@ static int tdx_reclaim_page(struct page *page) r = __tdx_reclaim_page(page); if (!r) - tdx_quirk_reset_page(page); + tdx_quirk_reset_paddr(page_to_phys(page), PAGE_SIZE); return r; } @@ -597,7 +597,7 @@ static void tdx_reclaim_td_control_pages(struct kvm *kvm) if (TDX_BUG_ON(err, TDH_PHYMEM_PAGE_WBINVD, kvm)) return; - tdx_quirk_reset_page(kvm_tdx->td.tdr_page); + tdx_quirk_reset_paddr(page_to_phys(kvm_tdx->td.tdr_page), PAGE_SIZE); __free_page(kvm_tdx->td.tdr_page); kvm_tdx->td.tdr_page = NULL; diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index e5a37ea2d4a0..deb67e68f85f 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -731,12 +731,6 @@ void tdx_quirk_reset_paddr(unsigned long base, unsigned long size) } EXPORT_SYMBOL_FOR_KVM(tdx_quirk_reset_paddr); -void tdx_quirk_reset_page(struct page *page) -{ - tdx_quirk_reset_paddr(page_to_phys(page), PAGE_SIZE); -} -EXPORT_SYMBOL_FOR_KVM(tdx_quirk_reset_page); - static __init void tdmr_quirk_reset_pamt(struct tdmr_info *tdmr) { From 3f330fbb918fbbfb5b50b190fff8a13d7810b6f0 Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Thu, 30 Apr 2026 09:50:14 +0800 Subject: [PATCH 08/24] x86/virt/tdx: Move mk_keyed_paddr() to tdx.c due to no external users Move mk_keyed_paddr() from tdx.h to tdx.c to avoid unnecessary header inclusion and improve encapsulation since there are no users outside of tdx.c. No functional change intended. Signed-off-by: Yan Zhao Acked-by: Kiryl Shutsemau Reviewed-by: Xiaoyao Li Acked-by: Dave Hansen Link: https://patch.msgid.link/20260430015014.24261-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/tdx.h | 6 ------ arch/x86/virt/vmx/tdx/tdx.c | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index 1a83ef88af17..32fbdf8f55ae 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -177,12 +177,6 @@ struct tdx_vp { struct page **tdcx_pages; }; -static inline u64 mk_keyed_paddr(u16 hkid, kvm_pfn_t pfn) -{ - /* KeyID bits are just above the physical address bits. */ - return PFN_PHYS(pfn) | ((u64)hkid << boot_cpu_data.x86_phys_bits); -} - u64 tdh_vp_enter(struct tdx_vp *vp, struct tdx_module_args *args); u64 tdh_mng_addcx(struct tdx_td *td, struct page *tdcs_page); u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, kvm_pfn_t pfn, struct page *source, diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index deb67e68f85f..967482ae3c80 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -1911,6 +1911,12 @@ u64 tdh_phymem_cache_wb(bool resume) } EXPORT_SYMBOL_FOR_KVM(tdh_phymem_cache_wb); +static inline u64 mk_keyed_paddr(u16 hkid, kvm_pfn_t pfn) +{ + /* KeyID bits are just above the physical address bits. */ + return PFN_PHYS(pfn) | ((u64)hkid << boot_cpu_data.x86_phys_bits); +} + u64 tdh_phymem_page_wbinvd_tdr(struct tdx_td *td) { struct tdx_module_args args = {}; From 12d595795de758dbaab2da6dac14ebd354b0f57d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 9 May 2026 15:53:57 +0800 Subject: [PATCH 09/24] KVM: TDX: Drop kvm_x86_ops.link_external_spt() Drop the dedicated .link_external_spt() for linking S-EPT pages, and instead funnel everything through .set_external_spte() for mapping S-EPT entries. Using separate hooks doesn't help prevent TDP MMU details from bleeding into TDX, and vice versa; to the contrary, dedicated callbacks will result in _more_ pollution when hugepage support is added, e.g. will require the TDP MMU to know details about the splitting rules for TDX that aren't all that relevant to the TDP MMU. Ideally, KVM would provide a single pair of hooks to set S-EPT entries, one hook for setting SPTEs under write-lock and another for setting SPTEs under read-lock (e.g. to ensure the entire operation is "atomic", to allow for failure, etc.). Sadly, TDX's requirement that all child S-EPT entries are removed before the parent makes that impractical: the TDP MMU deliberately prunes non-leaf SPTEs and _then_ processes its children, thus making it quite important for the TDP MMU to differentiate between zapping leaf and non-leaf S-EPT entries. However, that's the _only_ case that's truly special, and even that case could be shoehorned into a single hook; it just wouldn't be a net positive. Signed-off-by: Rick Edgecombe Signed-off-by: Yan Zhao Link: https://patch.msgid.link/20260509075357.4113-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 1 - arch/x86/include/asm/kvm_host.h | 3 -- arch/x86/kvm/mmu/tdp_mmu.c | 29 +------------- arch/x86/kvm/vmx/tdx.c | 63 ++++++++++++++++++++---------- 4 files changed, 44 insertions(+), 52 deletions(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index b0269325646c..2cb393000ee9 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -96,7 +96,6 @@ KVM_X86_OP_OPTIONAL_RET0(set_identity_map_addr) KVM_X86_OP_OPTIONAL_RET0(get_mt_mask) KVM_X86_OP_OPTIONAL_RET0(tdp_has_smep) KVM_X86_OP(load_mmu_pgd) -KVM_X86_OP_OPTIONAL_RET0(link_external_spt) KVM_X86_OP_OPTIONAL_RET0(set_external_spte) KVM_X86_OP_OPTIONAL_RET0(free_external_spt) KVM_X86_OP_OPTIONAL(remove_external_spte) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 8a53ca619570..85339d43a9ff 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1911,9 +1911,6 @@ struct kvm_x86_ops { void (*load_mmu_pgd)(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level); - /* Update external mapping with page table link. */ - int (*link_external_spt)(struct kvm *kvm, gfn_t gfn, enum pg_level level, - void *external_spt); /* Update the external page table from spte getting set. */ int (*set_external_spte)(struct kvm *kvm, gfn_t gfn, enum pg_level level, u64 mirror_spte); diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index f98afc3422ce..0dc1b0597f8a 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -495,27 +495,12 @@ static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared) call_rcu(&sp->rcu_head, tdp_mmu_free_sp_rcu_callback); } -static void *get_external_spt(gfn_t gfn, u64 new_spte, int level) -{ - if (is_shadow_present_pte(new_spte) && !is_last_spte(new_spte, level)) { - struct kvm_mmu_page *sp = spte_to_child_sp(new_spte); - - WARN_ON_ONCE(sp->role.level + 1 != level); - WARN_ON_ONCE(sp->gfn != gfn); - return sp->external_spt; - } - - return NULL; -} - static int __must_check set_external_spte_present(struct kvm *kvm, tdp_ptep_t sptep, gfn_t gfn, u64 *old_spte, u64 new_spte, int level) { bool was_present = is_shadow_present_pte(*old_spte); - bool is_present = is_shadow_present_pte(new_spte); - bool is_leaf = is_present && is_last_spte(new_spte, level); - int ret = 0; + int ret; KVM_BUG_ON(was_present, kvm); @@ -528,18 +513,8 @@ static int __must_check set_external_spte_present(struct kvm *kvm, tdp_ptep_t sp if (!try_cmpxchg64(rcu_dereference(sptep), old_spte, FROZEN_SPTE)) return -EBUSY; - /* - * Use different call to either set up middle level - * external page table, or leaf. - */ - if (is_leaf) { - ret = kvm_x86_call(set_external_spte)(kvm, gfn, level, new_spte); - } else { - void *external_spt = get_external_spt(gfn, new_spte, level); + ret = kvm_x86_call(set_external_spte)(kvm, gfn, level, new_spte); - KVM_BUG_ON(!external_spt, kvm); - ret = kvm_x86_call(link_external_spt)(kvm, gfn, level, external_spt); - } if (ret) __kvm_tdp_mmu_write_spte(sptep, *old_spte); else diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 3f956dde4a51..2dfc90c449a7 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1653,18 +1653,58 @@ static int tdx_mem_page_aug(struct kvm *kvm, gfn_t gfn, return 0; } +static struct page *tdx_spte_to_sept_pt(struct kvm *kvm, gfn_t gfn, + u64 new_spte, enum pg_level level) +{ + struct kvm_mmu_page *sp = spte_to_child_sp(new_spte); + + if (KVM_BUG_ON(!sp->external_spt, kvm) || + KVM_BUG_ON(sp->role.level + 1 != level, kvm) || + KVM_BUG_ON(sp->gfn != gfn, kvm)) + return NULL; + + return virt_to_page(sp->external_spt); +} + +static int tdx_sept_link_private_spt(struct kvm *kvm, gfn_t gfn, + enum pg_level level, u64 mirror_spte) +{ + gpa_t gpa = gfn_to_gpa(gfn); + u64 err, entry, level_state; + struct page *sept_pt; + + sept_pt = tdx_spte_to_sept_pt(kvm, gfn, mirror_spte, level); + if (!sept_pt) + return -EIO; + + err = tdh_mem_sept_add(&to_kvm_tdx(kvm)->td, gpa, level, sept_pt, + &entry, &level_state); + if (unlikely(tdx_operand_busy(err))) + return -EBUSY; + + if (TDX_BUG_ON_2(err, TDH_MEM_SEPT_ADD, entry, level_state, kvm)) + return -EIO; + + return 0; +} + static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, enum pg_level level, u64 mirror_spte) { struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); kvm_pfn_t pfn = spte_to_pfn(mirror_spte); + if (KVM_BUG_ON(!is_shadow_present_pte(mirror_spte), kvm)) + return -EIO; + + if (!is_last_spte(mirror_spte, level)) + return tdx_sept_link_private_spt(kvm, gfn, level, mirror_spte); + /* TODO: handle large pages. */ if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm)) return -EIO; - WARN_ON_ONCE(!is_shadow_present_pte(mirror_spte) || - (mirror_spte & VMX_EPT_RWX_MASK) != VMX_EPT_RWX_MASK); + WARN_ON_ONCE((mirror_spte & VMX_EPT_RWX_MASK) != VMX_EPT_RWX_MASK); /* * Ensure pre_fault_allowed is read by kvm_arch_vcpu_pre_fault_memory() @@ -1684,24 +1724,6 @@ static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, return tdx_mem_page_aug(kvm, gfn, level, pfn); } -static int tdx_sept_link_private_spt(struct kvm *kvm, gfn_t gfn, - enum pg_level level, void *private_spt) -{ - gpa_t gpa = gfn_to_gpa(gfn); - struct page *page = virt_to_page(private_spt); - u64 err, entry, level_state; - - err = tdh_mem_sept_add(&to_kvm_tdx(kvm)->td, gpa, level, page, &entry, - &level_state); - if (unlikely(tdx_operand_busy(err))) - return -EBUSY; - - if (TDX_BUG_ON_2(err, TDH_MEM_SEPT_ADD, entry, level_state, kvm)) - return -EIO; - - return 0; -} - /* * Ensure shared and private EPTs to be flushed on all vCPUs. * tdh_mem_track() is the only caller that increases TD epoch. An increase in @@ -3411,7 +3433,6 @@ int __init tdx_hardware_setup(void) vt_x86_ops.vm_size = max_t(unsigned int, vt_x86_ops.vm_size, sizeof(struct kvm_tdx)); - vt_x86_ops.link_external_spt = tdx_sept_link_private_spt; vt_x86_ops.set_external_spte = tdx_sept_set_private_spte; vt_x86_ops.free_external_spt = tdx_sept_free_private_spt; vt_x86_ops.remove_external_spte = tdx_sept_remove_private_spte; From 176dcd88f4b46ecf8df579f54b9168c105c87c60 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 9 May 2026 15:55:00 +0800 Subject: [PATCH 10/24] KVM: TDX: Wrap mapping of leaf and non-leaf S-EPT entries into helpers Add a helper, tdx_sept_map_leaf_spte(), to wrap and isolate PAGE.ADD and PAGE.AUG operations. Rename tdx_sept_link_private_spt() to tdx_sept_map_nonleaf_spte() to wrap SEPT.ADD for symmetry. Thus, transition tdx_sept_set_private_spte() into a "dispatch" routine for setting/writing S-EPT entries. No functional change intended. Signed-off-by: Rick Edgecombe Signed-off-by: Yan Zhao Link: https://patch.msgid.link/20260509075500.4157-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 2dfc90c449a7..965c1244e733 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1666,7 +1666,7 @@ static struct page *tdx_spte_to_sept_pt(struct kvm *kvm, gfn_t gfn, return virt_to_page(sp->external_spt); } -static int tdx_sept_link_private_spt(struct kvm *kvm, gfn_t gfn, +static int tdx_sept_map_nonleaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level level, u64 mirror_spte) { gpa_t gpa = gfn_to_gpa(gfn); @@ -1688,18 +1688,12 @@ static int tdx_sept_link_private_spt(struct kvm *kvm, gfn_t gfn, return 0; } -static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, - enum pg_level level, u64 mirror_spte) +static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level level, + u64 mirror_spte) { struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); kvm_pfn_t pfn = spte_to_pfn(mirror_spte); - if (KVM_BUG_ON(!is_shadow_present_pte(mirror_spte), kvm)) - return -EIO; - - if (!is_last_spte(mirror_spte, level)) - return tdx_sept_link_private_spt(kvm, gfn, level, mirror_spte); - /* TODO: handle large pages. */ if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm)) return -EIO; @@ -1724,6 +1718,18 @@ static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, return tdx_mem_page_aug(kvm, gfn, level, pfn); } +static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, + enum pg_level level, u64 mirror_spte) +{ + if (KVM_BUG_ON(!is_shadow_present_pte(mirror_spte), kvm)) + return -EIO; + + if (!is_last_spte(mirror_spte, level)) + return tdx_sept_map_nonleaf_spte(kvm, gfn, level, mirror_spte); + + return tdx_sept_map_leaf_spte(kvm, gfn, level, mirror_spte); +} + /* * Ensure shared and private EPTs to be flushed on all vCPUs. * tdh_mem_track() is the only caller that increases TD epoch. An increase in From 78d23e299bba68d12c212f729fc7cf47a66a5251 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 9 May 2026 15:55:19 +0800 Subject: [PATCH 11/24] KVM: x86/mmu: Fold set_external_spte_present() into its sole caller Fold set_external_spte_present() into __tdp_mmu_set_spte_atomic() in anticipation of propagating *all* changes (like atomic zap) triggered by tdp_mmu_set_spte_atomic() to the external PTEs. No functional change intended. Signed-off-by: Yan Zhao Link: https://patch.msgid.link/20260509075520.4177-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/tdp_mmu.c | 72 ++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 41 deletions(-) diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 0dc1b0597f8a..128089571e34 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -495,33 +495,6 @@ static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared) call_rcu(&sp->rcu_head, tdp_mmu_free_sp_rcu_callback); } -static int __must_check set_external_spte_present(struct kvm *kvm, tdp_ptep_t sptep, - gfn_t gfn, u64 *old_spte, - u64 new_spte, int level) -{ - bool was_present = is_shadow_present_pte(*old_spte); - int ret; - - KVM_BUG_ON(was_present, kvm); - - lockdep_assert_held(&kvm->mmu_lock); - /* - * We need to lock out other updates to the SPTE until the external - * page table has been modified. Use FROZEN_SPTE similar to - * the zapping case. - */ - if (!try_cmpxchg64(rcu_dereference(sptep), old_spte, FROZEN_SPTE)) - return -EBUSY; - - ret = kvm_x86_call(set_external_spte)(kvm, gfn, level, new_spte); - - if (ret) - __kvm_tdp_mmu_write_spte(sptep, *old_spte); - else - __kvm_tdp_mmu_write_spte(sptep, new_spte); - return ret; -} - /** * handle_changed_spte - handle bookkeeping associated with an SPTE change * @kvm: kvm instance @@ -626,6 +599,8 @@ static inline int __must_check __tdp_mmu_set_spte_atomic(struct kvm *kvm, struct tdp_iter *iter, u64 new_spte) { + u64 *raw_sptep = rcu_dereference(iter->sptep); + /* * The caller is responsible for ensuring the old SPTE is not a FROZEN * SPTE. KVM should never attempt to zap or manipulate a FROZEN SPTE, @@ -635,8 +610,13 @@ static inline int __must_check __tdp_mmu_set_spte_atomic(struct kvm *kvm, WARN_ON_ONCE(iter->yielded || is_frozen_spte(iter->old_spte)); if (is_mirror_sptep(iter->sptep) && !is_frozen_spte(new_spte)) { + bool was_present = is_shadow_present_pte(iter->old_spte); int ret; + KVM_BUG_ON(was_present, kvm); + + lockdep_assert_held(&kvm->mmu_lock); + /* * Users of atomic zapping don't operate on mirror roots, * so don't handle it and bug the VM if it's seen. @@ -644,25 +624,35 @@ static inline int __must_check __tdp_mmu_set_spte_atomic(struct kvm *kvm, if (KVM_BUG_ON(!is_shadow_present_pte(new_spte), kvm)) return -EBUSY; - ret = set_external_spte_present(kvm, iter->sptep, iter->gfn, - &iter->old_spte, new_spte, iter->level); - if (ret) - return ret; - } else { - u64 *sptep = rcu_dereference(iter->sptep); - /* - * Note, fast_pf_fix_direct_spte() can also modify TDP MMU SPTEs - * and does not hold the mmu_lock. On failure, i.e. if a - * different logical CPU modified the SPTE, try_cmpxchg64() - * updates iter->old_spte with the current value, so the caller - * operates on fresh data, e.g. if it retries - * tdp_mmu_set_spte_atomic() + * We need to lock out other updates to the SPTE until the external + * page table has been modified. Use FROZEN_SPTE similar to + * the zapping case. */ - if (!try_cmpxchg64(sptep, &iter->old_spte, new_spte)) + if (!try_cmpxchg64(raw_sptep, &iter->old_spte, FROZEN_SPTE)) return -EBUSY; + + ret = kvm_x86_call(set_external_spte)(kvm, iter->gfn, iter->level, + new_spte); + + if (ret) + __kvm_tdp_mmu_write_spte(iter->sptep, iter->old_spte); + else + __kvm_tdp_mmu_write_spte(iter->sptep, new_spte); + + return ret; } + /* + * Note, fast_pf_fix_direct_spte() can also modify TDP MMU SPTEs and + * does not hold the mmu_lock. On failure, i.e. if a different logical + * CPU modified the SPTE, try_cmpxchg64() updates iter->old_spte with + * the current value, so the caller operates on fresh data, e.g. if it + * retries tdp_mmu_set_spte_atomic(). + */ + if (!try_cmpxchg64(raw_sptep, &iter->old_spte, new_spte)) + return -EBUSY; + return 0; } From 291bcb3a75835233f6904b20c64b941108330f87 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 9 May 2026 15:55:33 +0800 Subject: [PATCH 12/24] KVM: x86/mmu: Plumb param "old_spte" into kvm_x86_ops.set_external_spte() If tdp_mmu_set_spte_atomic() triggers an atomic zap on a mirror SPTE (though currently no paths trigger it), the change is propagated via the set_external_spte() op. Plumb the old SPTE into the set_external_spte() op, so TDX code rather than TDP MMU code can warn if the atomic zap isn't allowed, i.e. to let TDX enforce TDX's rules (inasmuch as possible). Rename mirror_spte to new_spte to follow the TDP MMU's naming, and to make it more obvious what value the parameter holds. Opportunistically tweak the ordering of parameters to match the pattern of most TDP MMU functions, which do "old, new, level". Signed-off-by: Rick Edgecombe Signed-off-by: Yan Zhao Link: https://patch.msgid.link/20260509075533.4193-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 4 ++-- arch/x86/kvm/mmu/tdp_mmu.c | 4 ++-- arch/x86/kvm/vmx/tdx.c | 22 +++++++++++----------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 85339d43a9ff..b8ca43241b11 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1912,8 +1912,8 @@ struct kvm_x86_ops { int root_level); /* Update the external page table from spte getting set. */ - int (*set_external_spte)(struct kvm *kvm, gfn_t gfn, enum pg_level level, - u64 mirror_spte); + int (*set_external_spte)(struct kvm *kvm, gfn_t gfn, u64 old_spte, + u64 new_spte, enum pg_level level); /* Update external page tables for page table about to be freed. */ int (*free_external_spt)(struct kvm *kvm, gfn_t gfn, enum pg_level level, diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 128089571e34..4e961b331780 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -632,8 +632,8 @@ static inline int __must_check __tdp_mmu_set_spte_atomic(struct kvm *kvm, if (!try_cmpxchg64(raw_sptep, &iter->old_spte, FROZEN_SPTE)) return -EBUSY; - ret = kvm_x86_call(set_external_spte)(kvm, iter->gfn, iter->level, - new_spte); + ret = kvm_x86_call(set_external_spte)(kvm, iter->gfn, iter->old_spte, + new_spte, iter->level); if (ret) __kvm_tdp_mmu_write_spte(iter->sptep, iter->old_spte); diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 965c1244e733..c7ca5c79ada2 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1667,13 +1667,13 @@ static struct page *tdx_spte_to_sept_pt(struct kvm *kvm, gfn_t gfn, } static int tdx_sept_map_nonleaf_spte(struct kvm *kvm, gfn_t gfn, - enum pg_level level, u64 mirror_spte) + enum pg_level level, u64 new_spte) { gpa_t gpa = gfn_to_gpa(gfn); u64 err, entry, level_state; struct page *sept_pt; - sept_pt = tdx_spte_to_sept_pt(kvm, gfn, mirror_spte, level); + sept_pt = tdx_spte_to_sept_pt(kvm, gfn, new_spte, level); if (!sept_pt) return -EIO; @@ -1689,16 +1689,16 @@ static int tdx_sept_map_nonleaf_spte(struct kvm *kvm, gfn_t gfn, } static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level level, - u64 mirror_spte) + u64 new_spte) { struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); - kvm_pfn_t pfn = spte_to_pfn(mirror_spte); + kvm_pfn_t pfn = spte_to_pfn(new_spte); /* TODO: handle large pages. */ if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm)) return -EIO; - WARN_ON_ONCE((mirror_spte & VMX_EPT_RWX_MASK) != VMX_EPT_RWX_MASK); + WARN_ON_ONCE((new_spte & VMX_EPT_RWX_MASK) != VMX_EPT_RWX_MASK); /* * Ensure pre_fault_allowed is read by kvm_arch_vcpu_pre_fault_memory() @@ -1718,16 +1718,16 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve return tdx_mem_page_aug(kvm, gfn, level, pfn); } -static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, - enum pg_level level, u64 mirror_spte) +static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte, + u64 new_spte, enum pg_level level) { - if (KVM_BUG_ON(!is_shadow_present_pte(mirror_spte), kvm)) + if (KVM_BUG_ON(!is_shadow_present_pte(new_spte), kvm)) return -EIO; - if (!is_last_spte(mirror_spte, level)) - return tdx_sept_map_nonleaf_spte(kvm, gfn, level, mirror_spte); + if (!is_last_spte(new_spte, level)) + return tdx_sept_map_nonleaf_spte(kvm, gfn, level, new_spte); - return tdx_sept_map_leaf_spte(kvm, gfn, level, mirror_spte); + return tdx_sept_map_leaf_spte(kvm, gfn, level, new_spte); } /* From 6b1280a738f92542a9c73ce190846f54f55ee021 Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Sat, 9 May 2026 15:55:44 +0800 Subject: [PATCH 13/24] KVM: TDX: Move KVM_BUG_ON()s in __tdp_mmu_set_spte_atomic() to TDX code Drop some KVM_BUG_ON()s that are guarding against TDP MMU attempting to propagate unsupported changes to the external page table through __tdp_mmu_set_spte_atomic(). Have TDX code trigger them instead. Now that TDP MMU logically allows propagating atomic zapping operation to the external page table through the set_external_spte() op in __tdp_mmu_set_spte_atomic(), TDX code will trigger the KVM_BUG_ON() on the atomic zapping request instead. (Note: non-atomic zapping is not propagated via the set_external_spte() op yet). Despite the generic naming, external page table ops are designed completely around TDX. They hook the bare minimum of what is needed, and exclude the operations that are not supported by TDX. To help wrangle which operations are handleable by various operations, warnings and KVM_BUG_ON()s exist in the code. These warnings and KVM_BUG_ON()s put the burden of understanding which operations should be forwarded to TDX code on TDP MMU developers, who often read the code without TDX context. Future changes will transition the encapsulation of this domain knowledge to TDX code by funneling the external page table updates through a central update mechanism. In this paradigm, the central update mechanism can encapsulate the special knowledge, but will not have as much knowledge about what operation is in progress. Suggested-by: Sean Christopherson Signed-off-by: Rick Edgecombe Signed-off-by: Yan Zhao Link: https://patch.msgid.link/20260509075544.4210-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/tdp_mmu.c | 10 ---------- arch/x86/kvm/vmx/tdx.c | 3 +++ 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 4e961b331780..ed806a4768c2 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -610,20 +610,10 @@ static inline int __must_check __tdp_mmu_set_spte_atomic(struct kvm *kvm, WARN_ON_ONCE(iter->yielded || is_frozen_spte(iter->old_spte)); if (is_mirror_sptep(iter->sptep) && !is_frozen_spte(new_spte)) { - bool was_present = is_shadow_present_pte(iter->old_spte); int ret; - KVM_BUG_ON(was_present, kvm); - lockdep_assert_held(&kvm->mmu_lock); - /* - * Users of atomic zapping don't operate on mirror roots, - * so don't handle it and bug the VM if it's seen. - */ - if (KVM_BUG_ON(!is_shadow_present_pte(new_spte), kvm)) - return -EBUSY; - /* * We need to lock out other updates to the SPTE until the external * page table has been modified. Use FROZEN_SPTE similar to diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index c7ca5c79ada2..ea3ad913e750 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1721,6 +1721,9 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte, u64 new_spte, enum pg_level level) { + if (KVM_BUG_ON(is_shadow_present_pte(old_spte), kvm)) + return -EIO; + if (KVM_BUG_ON(!is_shadow_present_pte(new_spte), kvm)) return -EIO; From 40342185fd2b2791a11a0ea4ef9f78171b067dd5 Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Sat, 9 May 2026 15:55:57 +0800 Subject: [PATCH 14/24] KVM: TDX: Move lockdep assert in __tdp_mmu_set_spte_atomic() to TDX code Move the MMU lockdep assert in __tdp_mmu_set_spte_atomic() into the TDX specific op because the assert is TDX specific in intention. The TDP MMU has many lockdep asserts for various scenarios, and in fact the callchains that are used for TDX already have a lockdep assert which covers the case in __tdp_mmu_set_spte_atomic(). However, these asserts are for management of the TDP root owned by KVM. In the __tdp_mmu_set_spte_atomic() assert case, it is helping with a scheme to avoid contention in the TDX module during zap operations. That is very TDX specific. One option would be to just remove the assert in __tdp_mmu_set_spte_atomic() and rely on the other ones in the TDP MMU. But that assert is for a different intention, and too far away from the SEAMCALL that needs it. So just move it to TDX code. Suggested-by: Sean Christopherson Signed-off-by: Rick Edgecombe Signed-off-by: Yan Zhao Link: https://patch.msgid.link/20260509075557.4226-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/tdp_mmu.c | 2 -- arch/x86/kvm/vmx/tdx.c | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index ed806a4768c2..dc455e6e7dc7 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -612,8 +612,6 @@ static inline int __must_check __tdp_mmu_set_spte_atomic(struct kvm *kvm, if (is_mirror_sptep(iter->sptep) && !is_frozen_spte(new_spte)) { int ret; - lockdep_assert_held(&kvm->mmu_lock); - /* * We need to lock out other updates to the SPTE until the external * page table has been modified. Use FROZEN_SPTE similar to diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index ea3ad913e750..2e41bddfb80e 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1721,6 +1721,8 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte, u64 new_spte, enum pg_level level) { + lockdep_assert_held(&kvm->mmu_lock); + if (KVM_BUG_ON(is_shadow_present_pte(old_spte), kvm)) return -EIO; From c1d3af136fbf5a092785aa52f0a310008ccfad69 Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Sat, 9 May 2026 15:56:09 +0800 Subject: [PATCH 15/24] KVM: x86/tdp_mmu: Morph !is_frozen_spte() check into a KVM_MMU_WARN_ON() Remove the conditional logic for handling the setting of mirror page table to frozen in __tdp_mmu_set_spte_atomic() and add it as a warning for both mirror and direct cases. The mirror page table needs to propagate PTE changes to the external page table. This presents a problem for atomic updates which can't update both page tables at once. So a special value, FROZEN_SPTE, is used as a temporary state during these updates to prevent concurrent operations on the PTE. If the TDP MMU tried to install FROZEN_SPTE as a long-term value, it would confuse these updates. On the other hand, it would also confuse other threads if FROZEN_SPTE is installed as a long-term value for direct page tables (e.g., causing another thread working on atomic zap to wait for a !FROZEN_SPTE value endlessly). Therefore, add the warning for installing FROZEN_SPTE as a long-term value in __tdp_mmu_set_spte_atomic() without differentiating whether it's a mirror or direct page table. Suggested-by: Sean Christopherson Signed-off-by: Rick Edgecombe Signed-off-by: Yan Zhao Link: https://patch.msgid.link/20260509075609.4242-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/tdp_mmu.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index dc455e6e7dc7..b30e33dea265 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -609,7 +609,10 @@ static inline int __must_check __tdp_mmu_set_spte_atomic(struct kvm *kvm, */ WARN_ON_ONCE(iter->yielded || is_frozen_spte(iter->old_spte)); - if (is_mirror_sptep(iter->sptep) && !is_frozen_spte(new_spte)) { + /* Should not set FROZEN_SPTE as a long-term value. */ + KVM_MMU_WARN_ON(is_frozen_spte(new_spte)); + + if (is_mirror_sptep(iter->sptep)) { int ret; /* From c1ec7f368d85329ab25407a0b9cf0ec99ff15e22 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 9 May 2026 15:56:22 +0800 Subject: [PATCH 16/24] KVM: x86/mmu: Plumb "sp" _pointer_ into the TDP MMU's handle_changed_spte() Plumb the "sp" pointer into handle_changed_spte() to allow checking of is_mirror_sp(sp) in handle_changed_spte(). This will allow consolidating all S-EPT updates into a single kvm_x86_ops hook. [Yan: Remove unused "as_id" param in tdp_mmu_set_spte() ] Signed-off-by: Yan Zhao Link: https://patch.msgid.link/20260509075622.4258-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/tdp_mmu.c | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index b30e33dea265..4c68d5e03346 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -320,9 +320,9 @@ void kvm_tdp_mmu_alloc_root(struct kvm_vcpu *vcpu, bool mirror) } } -static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, - u64 old_spte, u64 new_spte, int level, - bool shared); +static void handle_changed_spte(struct kvm *kvm, struct kvm_mmu_page *sp, + gfn_t gfn, u64 old_spte, u64 new_spte, + int level, bool shared); static void tdp_account_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp) { @@ -471,8 +471,7 @@ static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared) old_spte = kvm_tdp_mmu_write_spte(sptep, old_spte, FROZEN_SPTE, level); } - handle_changed_spte(kvm, kvm_mmu_page_as_id(sp), gfn, - old_spte, FROZEN_SPTE, level, shared); + handle_changed_spte(kvm, sp, gfn, old_spte, FROZEN_SPTE, level, shared); if (is_mirror_sp(sp)) { KVM_BUG_ON(shared, kvm); @@ -498,7 +497,7 @@ static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared) /** * handle_changed_spte - handle bookkeeping associated with an SPTE change * @kvm: kvm instance - * @as_id: the address space of the paging structure the SPTE was a part of + * @sp: the page table in which the SPTE resides * @gfn: the base GFN that was mapped by the SPTE * @old_spte: The value of the SPTE before the change * @new_spte: The value of the SPTE after the change @@ -511,15 +510,16 @@ static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared) * dirty logging updates are handled in common code, not here (see make_spte() * and fast_pf_fix_direct_spte()). */ -static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, - u64 old_spte, u64 new_spte, int level, - bool shared) +static void handle_changed_spte(struct kvm *kvm, struct kvm_mmu_page *sp, + gfn_t gfn, u64 old_spte, u64 new_spte, + int level, bool shared) { bool was_present = is_shadow_present_pte(old_spte); bool is_present = is_shadow_present_pte(new_spte); bool was_leaf = was_present && is_last_spte(old_spte, level); bool is_leaf = is_present && is_last_spte(new_spte, level); bool pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte); + int as_id = kvm_mmu_page_as_id(sp); WARN_ON_ONCE(level > PT64_ROOT_MAX_LEVEL); WARN_ON_ONCE(level < PG_LEVEL_4K); @@ -668,6 +668,7 @@ static inline int __must_check tdp_mmu_set_spte_atomic(struct kvm *kvm, struct tdp_iter *iter, u64 new_spte) { + struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(iter->sptep)); int ret; lockdep_assert_held_read(&kvm->mmu_lock); @@ -676,7 +677,7 @@ static inline int __must_check tdp_mmu_set_spte_atomic(struct kvm *kvm, if (ret) return ret; - handle_changed_spte(kvm, iter->as_id, iter->gfn, iter->old_spte, + handle_changed_spte(kvm, sp, iter->gfn, iter->old_spte, new_spte, iter->level, true); return 0; @@ -685,7 +686,6 @@ static inline int __must_check tdp_mmu_set_spte_atomic(struct kvm *kvm, /* * tdp_mmu_set_spte - Set a TDP MMU SPTE and handle the associated bookkeeping * @kvm: KVM instance - * @as_id: Address space ID, i.e. regular vs. SMM * @sptep: Pointer to the SPTE * @old_spte: The current value of the SPTE * @new_spte: The new value that will be set for the SPTE @@ -695,9 +695,11 @@ static inline int __must_check tdp_mmu_set_spte_atomic(struct kvm *kvm, * Returns the old SPTE value, which _may_ be different than @old_spte if the * SPTE had voldatile bits. */ -static u64 tdp_mmu_set_spte(struct kvm *kvm, int as_id, tdp_ptep_t sptep, - u64 old_spte, u64 new_spte, gfn_t gfn, int level) +static u64 tdp_mmu_set_spte(struct kvm *kvm, tdp_ptep_t sptep, u64 old_spte, + u64 new_spte, gfn_t gfn, int level) { + struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(sptep)); + lockdep_assert_held_write(&kvm->mmu_lock); /* @@ -711,7 +713,7 @@ static u64 tdp_mmu_set_spte(struct kvm *kvm, int as_id, tdp_ptep_t sptep, old_spte = kvm_tdp_mmu_write_spte(sptep, old_spte, new_spte, level); - handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level, false); + handle_changed_spte(kvm, sp, gfn, old_spte, new_spte, level, false); /* * Users that do non-atomic setting of PTEs don't operate on mirror @@ -729,9 +731,8 @@ static inline void tdp_mmu_iter_set_spte(struct kvm *kvm, struct tdp_iter *iter, u64 new_spte) { WARN_ON_ONCE(iter->yielded); - iter->old_spte = tdp_mmu_set_spte(kvm, iter->as_id, iter->sptep, - iter->old_spte, new_spte, - iter->gfn, iter->level); + iter->old_spte = tdp_mmu_set_spte(kvm, iter->sptep, iter->old_spte, + new_spte, iter->gfn, iter->level); } #define tdp_root_for_each_pte(_iter, _kvm, _root, _start, _end) \ From 28c77a0378044b96f9c82cdd889872ab88191857 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 9 May 2026 15:56:34 +0800 Subject: [PATCH 17/24] KVM: x86/tdp_mmu: Centrally propagate to-present/atomic zap updates to external PTEs Move propagation of to-present changes and atomic zap changes to external PTEs from function __tdp_mmu_set_spte_atomic() to function __handle_changed_spte(), which centrally handles changes of SPTEs. When setting a PTE to present in the mirror page tables, the update needs to be propagated to the external page tables (in TDX parlance, the S-EPT). Today this is handled by special mirror page tables logic/branching in __tdp_mmu_set_spte_atomic(), which is the only place where present PTEs are set for TDX. The current approach obviously works, but is a bit hacked on. The hook for setting present leaf PTEs is added only where TDX happens to need it. For example, TDX does not support any of the operations that use the non-atomic variant, tdp_mmu_set_spte(), to set present PTEs. Since the hook is missing there, it is very hard to understand the code from a non-TDX lens. If the reader doesn't know the TDX specifics it could look like the external SPTE update is missing. In addition to being confusing, it also litters the TDP MMU with "external" update callbacks. This is especially unfortunate because there is already a central place to react to TDP updates, handle_changed_spte(). Begin the process of moving towards a model where all mirror page table updates are forwarded to TDX code where the TDX-specific logic can live with a more proper separation of concerns. Do this by adding a helper __handle_changed_spte() and teaching it how to return error codes, such that it can propagate the failures that may come from TDX external page table updates. Make the original handle_changed_spte() a no-fail version of __handle_changed_spte(), so it handles no-fail changes which are under exclusive mmu_lock or under the no-fail path handle_removed_pt(), triggering KVM_BUG_ON() on error returns. Instead of having __tdp_mmu_set_spte_atomic() do the frozen mirror SPTE dance and trigger propagation to external PTEs, make __tdp_mmu_set_spte_atomic() a simple helper of try_cmpxchg64() and hoist the frozen mirror SPTE dance up a level to tdp_mmu_set_spte_atomic(). Then, the propagation of changes to present to the external PTEs can be centralized to __handle_changed_spte(). Aging external SPTEs is not yet supported for the mirror page table, so just warn on mirror usage in kvm_tdp_mmu_age_spte() and invoke __tdp_mmu_set_spte_atomic() directly without frozen dance. No need to warn on installing FROZEN_SPTE as a long-term value in kvm_tdp_mmu_age_spte() since removing accessed bit is mutually exclusive with installing FROZEN_SPTE (FROZEN_SPTE is with accessed bit in all x86 platforms). Since tdp_mmu_set_spte_atomic() can also be invoked to atomically zap SPTEs (though there's no path to trigger atomic zap on the mirror page table up to now), also leverage set_external_spte() op to propagate the atomic zaps when tdp_mmu_set_spte_atomic() zaps leaf SPTEs directly. (When tdp_mmu_set_spte_atomic() zaps a non-leaf SPTE, zaps of the child leaf SPTEs are propagated via the remove_external_spte() op). Note: tdp_mmu_set_spte_atomic() invokes __handle_changed_spte() to handle changes to new_spte while the mirror SPTE is frozen, so (1) the update of the external PTEs and statistics, or (2) the update of child mirror SPTEs, child external PTEs and corresponding statistics, now occur before the mirror SPTE is actually set to new_spte. (1) is ok since if it fails, the mirror SPTE will be restored to its original value. (2) is also ok since handle_removed_pt() is no-fail. Link: https://lore.kernel.org/lkml/aYYn0nf2cayYu8e7@google.com [Rick: Based on a diff by Sean Chrisopherson] Co-developed-by: Rick Edgecombe Signed-off-by: Rick Edgecombe [Yan: added atomic zap case ] Co-developed-by: Yan Zhao Signed-off-by: Yan Zhao Link: https://patch.msgid.link/20260509075634.4274-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/tdp_mmu.c | 126 ++++++++++++++++++++++++------------- 1 file changed, 81 insertions(+), 45 deletions(-) diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 4c68d5e03346..3c3e73ce8da9 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -495,7 +495,7 @@ static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared) } /** - * handle_changed_spte - handle bookkeeping associated with an SPTE change + * __handle_changed_spte - handle bookkeeping associated with an SPTE change * @kvm: kvm instance * @sp: the page table in which the SPTE resides * @gfn: the base GFN that was mapped by the SPTE @@ -510,9 +510,9 @@ static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared) * dirty logging updates are handled in common code, not here (see make_spte() * and fast_pf_fix_direct_spte()). */ -static void handle_changed_spte(struct kvm *kvm, struct kvm_mmu_page *sp, - gfn_t gfn, u64 old_spte, u64 new_spte, - int level, bool shared) +static int __handle_changed_spte(struct kvm *kvm, struct kvm_mmu_page *sp, + gfn_t gfn, u64 old_spte, u64 new_spte, + int level, bool shared) { bool was_present = is_shadow_present_pte(old_spte); bool is_present = is_shadow_present_pte(new_spte); @@ -549,9 +549,7 @@ static void handle_changed_spte(struct kvm *kvm, struct kvm_mmu_page *sp, } if (old_spte == new_spte) - return; - - trace_kvm_tdp_mmu_spte_changed(as_id, gfn, level, old_spte, new_spte); + return 0; if (is_leaf) check_spte_writable_invariants(new_spte); @@ -578,29 +576,51 @@ static void handle_changed_spte(struct kvm *kvm, struct kvm_mmu_page *sp, "a temporary frozen SPTE.\n" "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d", as_id, gfn, old_spte, new_spte, level); - return; - } - if (is_leaf != was_leaf) - kvm_update_page_stats(kvm, level, is_leaf ? 1 : -1); + trace_kvm_tdp_mmu_spte_changed(as_id, gfn, level, old_spte, new_spte); + return 0; + } /* * Recursively handle child PTs if the change removed a subtree from * the paging structure. Note the WARN on the PFN changing without the * SPTE being converted to a hugepage (leaf) or being zapped. Shadow * pages are kernel allocations and should never be migrated. + * + * For the mirror page table, propagate changes to present or changes of + * leaf SPTEs to !present under shared mmu_lock to the external SPTE via + * set_external_spte() op. */ if (was_present && !was_leaf && - (is_leaf || !is_present || WARN_ON_ONCE(pfn_changed))) + (is_leaf || !is_present || WARN_ON_ONCE(pfn_changed))) { handle_removed_pt(kvm, spte_to_child_pt(old_spte, level), shared); + } else if (is_mirror_sp(sp) && (is_present || shared)) { + int r; + + r = kvm_x86_call(set_external_spte)(kvm, gfn, old_spte, new_spte, level); + if (r) + return r; + } + trace_kvm_tdp_mmu_spte_changed(as_id, gfn, level, old_spte, new_spte); + + if (is_leaf != was_leaf) + kvm_update_page_stats(kvm, level, is_leaf ? 1 : -1); + + return 0; +} + +static void handle_changed_spte(struct kvm *kvm, struct kvm_mmu_page *sp, + gfn_t gfn, u64 old_spte, u64 new_spte, + int level, bool shared) +{ + KVM_BUG_ON(__handle_changed_spte(kvm, sp, gfn, old_spte, new_spte, + level, shared), kvm); } static inline int __must_check __tdp_mmu_set_spte_atomic(struct kvm *kvm, struct tdp_iter *iter, u64 new_spte) { - u64 *raw_sptep = rcu_dereference(iter->sptep); - /* * The caller is responsible for ensuring the old SPTE is not a FROZEN * SPTE. KVM should never attempt to zap or manipulate a FROZEN SPTE, @@ -609,31 +629,6 @@ static inline int __must_check __tdp_mmu_set_spte_atomic(struct kvm *kvm, */ WARN_ON_ONCE(iter->yielded || is_frozen_spte(iter->old_spte)); - /* Should not set FROZEN_SPTE as a long-term value. */ - KVM_MMU_WARN_ON(is_frozen_spte(new_spte)); - - if (is_mirror_sptep(iter->sptep)) { - int ret; - - /* - * We need to lock out other updates to the SPTE until the external - * page table has been modified. Use FROZEN_SPTE similar to - * the zapping case. - */ - if (!try_cmpxchg64(raw_sptep, &iter->old_spte, FROZEN_SPTE)) - return -EBUSY; - - ret = kvm_x86_call(set_external_spte)(kvm, iter->gfn, iter->old_spte, - new_spte, iter->level); - - if (ret) - __kvm_tdp_mmu_write_spte(iter->sptep, iter->old_spte); - else - __kvm_tdp_mmu_write_spte(iter->sptep, new_spte); - - return ret; - } - /* * Note, fast_pf_fix_direct_spte() can also modify TDP MMU SPTEs and * does not hold the mmu_lock. On failure, i.e. if a different logical @@ -641,7 +636,7 @@ static inline int __must_check __tdp_mmu_set_spte_atomic(struct kvm *kvm, * the current value, so the caller operates on fresh data, e.g. if it * retries tdp_mmu_set_spte_atomic(). */ - if (!try_cmpxchg64(raw_sptep, &iter->old_spte, new_spte)) + if (!try_cmpxchg64(rcu_dereference(iter->sptep), &iter->old_spte, new_spte)) return -EBUSY; return 0; @@ -673,14 +668,51 @@ static inline int __must_check tdp_mmu_set_spte_atomic(struct kvm *kvm, lockdep_assert_held_read(&kvm->mmu_lock); - ret = __tdp_mmu_set_spte_atomic(kvm, iter, new_spte); + /* Should not set FROZEN_SPTE as a long-term value. */ + KVM_MMU_WARN_ON(is_frozen_spte(new_spte)); + + /* + * Temporarily freeze the SPTE until the external PTE operation has + * completed, e.g. so that concurrent faults don't attempt to install a + * child PTE in the external page table before the parent PTE has been + * written. + */ + if (is_mirror_sptep(iter->sptep)) + ret = __tdp_mmu_set_spte_atomic(kvm, iter, FROZEN_SPTE); + else + ret = __tdp_mmu_set_spte_atomic(kvm, iter, new_spte); + if (ret) return ret; - handle_changed_spte(kvm, sp, iter->gfn, iter->old_spte, - new_spte, iter->level, true); - - return 0; + /* + * Handle the change from iter->old_spte to new_spte. + * + * Note: for mirror page table, this means the updates of the external + * PTE, statistics, or updates of child SPTEs, child external PTEs and + * corresponding statistics are performed while the mirror SPTE is in + * frozen state (i.e., before the mirror SPTE is set to new_spte). + */ + ret = __handle_changed_spte(kvm, sp, iter->gfn, iter->old_spte, + new_spte, iter->level, true); + /* + * Unfreeze the mirror SPTE. If updating the external SPTE failed, + * restore the old value so that the mirror SPTE isn't frozen in + * perpetuity, otherwise set the mirror SPTE to the new desired value. + */ + if (is_mirror_sptep(iter->sptep)) { + if (ret) + __kvm_tdp_mmu_write_spte(iter->sptep, iter->old_spte); + else + __kvm_tdp_mmu_write_spte(iter->sptep, new_spte); + } else { + /* + * Bug the VM if handling the change failed, as failure is only + * allowed if KVM couldn't update the external SPTE. + */ + KVM_BUG_ON(ret, kvm); + } + return ret; } /* @@ -1334,6 +1366,10 @@ static void kvm_tdp_mmu_age_spte(struct kvm *kvm, struct tdp_iter *iter) { u64 new_spte; + /* TODO: Add support for aging external SPTEs, if necessary. */ + if (WARN_ON_ONCE(is_mirror_sptep(iter->sptep))) + return; + if (spte_ad_enabled(iter->old_spte)) { iter->old_spte = tdp_mmu_clear_spte_bits_atomic(iter->sptep, shadow_accessed_mask); From 41367d77ee4eba4b062835723b1756b0b6a86324 Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Sat, 9 May 2026 15:56:47 +0800 Subject: [PATCH 18/24] KVM: x86/mmu: Drop KVM_BUG_ON() on shared lock to zap child external PTEs Drop the KVM_BUG_ON() in the KVM MMU core before zapping child external PTEs, since requiring zapping PTEs to be protected by exclusive mmu_lock is TDX's specific requirement. No need to plumb the shared/exclusive info into the remove_external_spte() op or move the KVM_BUG_ON() to TDX, because - There's already an assertion of exclusive mmu_lock protection in TDX. - The KVM_BUG_ON() is a bit redundant given that if there's any bug causing zapping of leaf PTEs in S-EPT under shared mmu_lock, SEAMCALL failures due to contention would result in TDX_BUG_ON() in TDX. Link: https://lore.kernel.org/kvm/aYUarHf3KEwHGuJe@google.com/ Suggested-by: Sean Christopherson Signed-off-by: Rick Edgecombe Signed-off-by: Yan Zhao Link: https://patch.msgid.link/20260509075647.4290-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/tdp_mmu.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 3c3e73ce8da9..3ba7556a8d2f 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -473,10 +473,8 @@ static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared) } handle_changed_spte(kvm, sp, gfn, old_spte, FROZEN_SPTE, level, shared); - if (is_mirror_sp(sp)) { - KVM_BUG_ON(shared, kvm); + if (is_mirror_sp(sp)) remove_external_spte(kvm, gfn, old_spte, level); - } } if (is_mirror_sp(sp) && From c74893aa56f4e7515c0877c5230f9f42357ff044 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 9 May 2026 15:56:58 +0800 Subject: [PATCH 19/24] KVM: TDX: Hoist tdx_sept_remove_private_spte() above set_private_spte() Arrange tdx_sept_remove_private_spte() (and its tdx_track() helper) to be above tdx_sept_set_private_spte() in anticipation of routing all S-EPT writes (with the exception of reclaiming non-leaf pages) through the "set" API. No functional change intended. Signed-off-by: Rick Edgecombe Signed-off-by: Yan Zhao Link: https://patch.msgid.link/20260509075658.4306-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 80 +++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 2e41bddfb80e..8991f08ef81c 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1718,23 +1718,6 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve return tdx_mem_page_aug(kvm, gfn, level, pfn); } -static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte, - u64 new_spte, enum pg_level level) -{ - lockdep_assert_held(&kvm->mmu_lock); - - if (KVM_BUG_ON(is_shadow_present_pte(old_spte), kvm)) - return -EIO; - - if (KVM_BUG_ON(!is_shadow_present_pte(new_spte), kvm)) - return -EIO; - - if (!is_last_spte(new_spte, level)) - return tdx_sept_map_nonleaf_spte(kvm, gfn, level, new_spte); - - return tdx_sept_map_leaf_spte(kvm, gfn, level, new_spte); -} - /* * Ensure shared and private EPTs to be flushed on all vCPUs. * tdh_mem_track() is the only caller that increases TD epoch. An increase in @@ -1781,29 +1764,6 @@ static void tdx_track(struct kvm *kvm) kvm_make_all_cpus_request(kvm, KVM_REQ_OUTSIDE_GUEST_MODE); } -static int tdx_sept_free_private_spt(struct kvm *kvm, gfn_t gfn, - enum pg_level level, void *private_spt) -{ - struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); - - /* - * free_external_spt() is only called after hkid is freed when TD is - * tearing down. - * KVM doesn't (yet) zap page table pages in mirror page table while - * TD is active, though guest pages mapped in mirror page table could be - * zapped during TD is active, e.g. for shared <-> private conversion - * and slot move/deletion. - */ - if (KVM_BUG_ON(is_hkid_assigned(kvm_tdx), kvm)) - return -EIO; - - /* - * The HKID assigned to this TD was already freed and cache was - * already flushed. We don't have to flush again. - */ - return tdx_reclaim_page(virt_to_page(private_spt)); -} - static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, enum pg_level level, u64 mirror_spte) { @@ -1854,6 +1814,46 @@ static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, tdx_quirk_reset_paddr(PFN_PHYS(pfn), PAGE_SIZE); } +static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte, + u64 new_spte, enum pg_level level) +{ + lockdep_assert_held(&kvm->mmu_lock); + + if (KVM_BUG_ON(is_shadow_present_pte(old_spte), kvm)) + return -EIO; + + if (KVM_BUG_ON(!is_shadow_present_pte(new_spte), kvm)) + return -EIO; + + if (!is_last_spte(new_spte, level)) + return tdx_sept_map_nonleaf_spte(kvm, gfn, level, new_spte); + + return tdx_sept_map_leaf_spte(kvm, gfn, level, new_spte); +} + +static int tdx_sept_free_private_spt(struct kvm *kvm, gfn_t gfn, + enum pg_level level, void *private_spt) +{ + struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); + + /* + * free_external_spt() is only called after hkid is freed when TD is + * tearing down. + * KVM doesn't (yet) zap page table pages in mirror page table while + * TD is active, though guest pages mapped in mirror page table could be + * zapped during TD is active, e.g. for shared <-> private conversion + * and slot move/deletion. + */ + if (KVM_BUG_ON(is_hkid_assigned(kvm_tdx), kvm)) + return -EIO; + + /* + * The HKID assigned to this TD was already freed and cache was + * already flushed. We don't have to flush again. + */ + return tdx_reclaim_page(virt_to_page(private_spt)); +} + void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, int trig_mode, int vector) { From 6eae9ce9461cc0ba16a779b49a70eaab2271a969 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 9 May 2026 15:57:09 +0800 Subject: [PATCH 20/24] KVM: TDX: Drop kvm_x86_ops.remove_external_spte() Drop kvm_x86_ops.remove_external_spte(), and instead handle the removal of leaf SPTEs in the S-EPT (a.k.a. external page table) in kvm_x86_ops.set_external_spte(). This will also allow extending tdx_sept_set_private_spte() to support splitting a huge S-EPT entry without needing yet another kvm_x86_ops hook. Now all changes for removing leaf mirror SPTEs are propagated through kvm_x86_ops.set_external_spte(). - When removing leaf mirror SPTEs under shared mmu_lock (though currently no path can trigger this scenario and TDX does not support this scenario), tdx_sept_remove_private_spte() may produce a warning due to lockdep_assert_held_write() or may return -EIO and trigger TDX_BUG_ON() due to concurrent BLOCK, TRACK, REMOVE. - When removing leaf mirror SPTEs under exclusive mmu_lock, all errors are unexpected. If any error occurs in this scenario, tdx_sept_remove_private_spte() will return -EIO and trigger KVM_BUG_ON(). A redundant KVM_BUG_ON() call will also be triggered in TDP MMU core in handle_changed_spte(), which is benign (the WARN will fire if and only if the VM isn't already bugged). Signed-off-by: Rick Edgecombe Signed-off-by: Yan Zhao Link: https://patch.msgid.link/20260509075709.4322-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 1 - arch/x86/include/asm/kvm_host.h | 3 --- arch/x86/kvm/mmu/tdp_mmu.c | 37 +++------------------------- arch/x86/kvm/vmx/tdx.c | 39 +++++++++++++++++++++--------- 4 files changed, 31 insertions(+), 49 deletions(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 2cb393000ee9..771d991562ca 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -98,7 +98,6 @@ KVM_X86_OP_OPTIONAL_RET0(tdp_has_smep) KVM_X86_OP(load_mmu_pgd) KVM_X86_OP_OPTIONAL_RET0(set_external_spte) KVM_X86_OP_OPTIONAL_RET0(free_external_spt) -KVM_X86_OP_OPTIONAL(remove_external_spte) KVM_X86_OP(has_wbinvd_exit) KVM_X86_OP(get_l2_tsc_offset) KVM_X86_OP(get_l2_tsc_multiplier) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index b8ca43241b11..dc5625950e5b 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1919,9 +1919,6 @@ struct kvm_x86_ops { int (*free_external_spt)(struct kvm *kvm, gfn_t gfn, enum pg_level level, void *external_spt); - /* Update external page table from spte getting removed, and flush TLB. */ - void (*remove_external_spte)(struct kvm *kvm, gfn_t gfn, enum pg_level level, - u64 mirror_spte); bool (*has_wbinvd_exit)(void); diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 3ba7556a8d2f..72d323f2d0dc 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -359,25 +359,6 @@ static void tdp_mmu_unlink_sp(struct kvm *kvm, struct kvm_mmu_page *sp) spin_unlock(&kvm->arch.tdp_mmu_pages_lock); } -static void remove_external_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte, - int level) -{ - /* - * External (TDX) SPTEs are limited to PG_LEVEL_4K, and external - * PTs are removed in a special order, involving free_external_spt(). - * But remove_external_spte() will be called on non-leaf PTEs via - * __tdp_mmu_zap_root(), so avoid the error the former would return - * in this case. - */ - if (!is_last_spte(old_spte, level)) - return; - - /* Zapping leaf spte is allowed only when write lock is held. */ - lockdep_assert_held_write(&kvm->mmu_lock); - - kvm_x86_call(remove_external_spte)(kvm, gfn, level, old_spte); -} - /** * handle_removed_pt() - handle a page table removed from the TDP structure * @@ -472,9 +453,6 @@ static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared) FROZEN_SPTE, level); } handle_changed_spte(kvm, sp, gfn, old_spte, FROZEN_SPTE, level, shared); - - if (is_mirror_sp(sp)) - remove_external_spte(kvm, gfn, old_spte, level); } if (is_mirror_sp(sp) && @@ -585,14 +563,14 @@ static int __handle_changed_spte(struct kvm *kvm, struct kvm_mmu_page *sp, * SPTE being converted to a hugepage (leaf) or being zapped. Shadow * pages are kernel allocations and should never be migrated. * - * For the mirror page table, propagate changes to present or changes of - * leaf SPTEs to !present under shared mmu_lock to the external SPTE via + * For the mirror page table, propagate all changes to the external SPTE + * (except zapping/promotion of non-leaf SPTEs) via the * set_external_spte() op. */ if (was_present && !was_leaf && (is_leaf || !is_present || WARN_ON_ONCE(pfn_changed))) { handle_removed_pt(kvm, spte_to_child_pt(old_spte, level), shared); - } else if (is_mirror_sp(sp) && (is_present || shared)) { + } else if (is_mirror_sp(sp)) { int r; r = kvm_x86_call(set_external_spte)(kvm, gfn, old_spte, new_spte, level); @@ -745,15 +723,6 @@ static u64 tdp_mmu_set_spte(struct kvm *kvm, tdp_ptep_t sptep, u64 old_spte, handle_changed_spte(kvm, sp, gfn, old_spte, new_spte, level, false); - /* - * Users that do non-atomic setting of PTEs don't operate on mirror - * roots, so don't handle it and bug the VM if it's seen. - */ - if (is_mirror_sptep(sptep)) { - KVM_BUG_ON(is_shadow_present_pte(new_spte), kvm); - remove_external_spte(kvm, gfn, old_spte, level); - } - return old_spte; } diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 8991f08ef81c..54c85b81a09f 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1764,11 +1764,11 @@ static void tdx_track(struct kvm *kvm) kvm_make_all_cpus_request(kvm, KVM_REQ_OUTSIDE_GUEST_MODE); } -static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, - enum pg_level level, u64 mirror_spte) +static int tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, + enum pg_level level, u64 old_spte) { struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); - kvm_pfn_t pfn = spte_to_pfn(mirror_spte); + kvm_pfn_t pfn = spte_to_pfn(old_spte); gpa_t gpa = gfn_to_gpa(gfn); u64 err, entry, level_state; @@ -1780,16 +1780,16 @@ static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, * there can't be anything populated in the private EPT. */ if (KVM_BUG_ON(!is_hkid_assigned(to_kvm_tdx(kvm)), kvm)) - return; + return -EIO; /* TODO: handle large pages. */ if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm)) - return; + return -EIO; err = tdh_do_no_vcpus(tdh_mem_range_block, kvm, &kvm_tdx->td, gpa, level, &entry, &level_state); if (TDX_BUG_ON_2(err, TDH_MEM_RANGE_BLOCK, entry, level_state, kvm)) - return; + return -EIO; /* * TDX requires TLB tracking before dropping private page. Do @@ -1805,22 +1805,40 @@ static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, err = tdh_do_no_vcpus(tdh_mem_page_remove, kvm, &kvm_tdx->td, gpa, level, &entry, &level_state); if (TDX_BUG_ON_2(err, TDH_MEM_PAGE_REMOVE, entry, level_state, kvm)) - return; + return -EIO; err = tdh_phymem_page_wbinvd_hkid((u16)kvm_tdx->hkid, pfn); if (TDX_BUG_ON(err, TDH_PHYMEM_PAGE_WBINVD, kvm)) - return; + return -EIO; tdx_quirk_reset_paddr(PFN_PHYS(pfn), PAGE_SIZE); + return 0; } +/* + * Handle changes for + * (1) leaf SPTEs from non-present to present + * (2) non-leaf SPTEs from non-present to present + * (3) leaf SPTEs from present to non-present + * + * - (1) and (2) must be under shared mmu_lock. If (1) and (2) are under + * exclusive mmu_lock (currently impossible), contention errors may lead to + * KVM_BUG_ON() in handle_changed_spte(), e.g., due to tdx_mem_page_aug(), + * tdx_mem_page_add(), or tdh_mem_sept_add() contending with tdh_vp_enter() + * due to zero-step mitigation or contending with TDCALLs. + * - (3) must be under write mmu_lock. If (3) is under shared mmu_lock + * (currently impossible), warnings will be generated due to + * lockdep_assert_held_write() or TDX_BUG_ON() caused by concurrent BLOCK, + * TRACK, REMOVE. + * - Promotion/demotion is not yet supported. + */ static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte, u64 new_spte, enum pg_level level) { lockdep_assert_held(&kvm->mmu_lock); - if (KVM_BUG_ON(is_shadow_present_pte(old_spte), kvm)) - return -EIO; + if (is_shadow_present_pte(old_spte)) + return tdx_sept_remove_private_spte(kvm, gfn, level, old_spte); if (KVM_BUG_ON(!is_shadow_present_pte(new_spte), kvm)) return -EIO; @@ -3446,7 +3464,6 @@ int __init tdx_hardware_setup(void) vt_x86_ops.set_external_spte = tdx_sept_set_private_spte; vt_x86_ops.free_external_spt = tdx_sept_free_private_spt; - vt_x86_ops.remove_external_spte = tdx_sept_remove_private_spte; vt_x86_ops.protected_apic_has_interrupt = tdx_protected_apic_has_interrupt; return 0; From 0cef26b537ffa963d719c529f8ff604c1db505fd Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Sat, 9 May 2026 15:57:19 +0800 Subject: [PATCH 21/24] KVM: TDX: Rename tdx_sept_remove_private_spte() to show it's for leaf SPTEs Rename tdx_sept_remove_private_spte() to tdx_sept_remove_leaf_spte() to clearly show that this function is for removal of leaf SPTEs. No functional change intended. Signed-off-by: Yan Zhao Link: https://patch.msgid.link/20260509075719.4338-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 54c85b81a09f..b1c273dfbe4c 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1764,8 +1764,8 @@ static void tdx_track(struct kvm *kvm) kvm_make_all_cpus_request(kvm, KVM_REQ_OUTSIDE_GUEST_MODE); } -static int tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, - enum pg_level level, u64 old_spte) +static int tdx_sept_remove_leaf_spte(struct kvm *kvm, gfn_t gfn, + enum pg_level level, u64 old_spte) { struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); kvm_pfn_t pfn = spte_to_pfn(old_spte); @@ -1838,7 +1838,7 @@ static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte, lockdep_assert_held(&kvm->mmu_lock); if (is_shadow_present_pte(old_spte)) - return tdx_sept_remove_private_spte(kvm, gfn, level, old_spte); + return tdx_sept_remove_leaf_spte(kvm, gfn, level, old_spte); if (KVM_BUG_ON(!is_shadow_present_pte(new_spte), kvm)) return -EIO; @@ -2832,7 +2832,7 @@ void tdx_flush_tlb_current(struct kvm_vcpu *vcpu) void tdx_flush_tlb_all(struct kvm_vcpu *vcpu) { /* - * TDX has called tdx_track() in tdx_sept_remove_private_spte() to + * TDX has called tdx_track() in tdx_sept_remove_leaf_spte() to * ensure that private EPT will be flushed on the next TD enter. No need * to call tdx_track() here again even when this callback is a result of * zapping private EPT. From b35bda696e4416a01a064ccb5e67bca03132d8ec Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 9 May 2026 15:57:30 +0800 Subject: [PATCH 22/24] KVM: x86: Move error handling inside free_external_spt() Move the logic for TDX's specific need to leak pages when reclaim fails inside the free_external_spt() op, so this can be done in TDX specific code and not the generic MMU. Do this by passing in "sp" instead of the external page table pointer. This way, TDX code can set sp->external_spt to NULL. Since the error is now handled internally in TDX code (by triggering KVM_BUG_ON() or TDX_BUG_ON_3(), which warn and stop the VM on any error), change the op to return void. This way it also operates like a normal free in that success is guaranteed from the caller's perspective. Opportunistically, drop the unused level and gfn args while adjusting the sp arg. [ Rick: Re-wrote log and massaged op name ] Co-developed-by: Rick Edgecombe Signed-off-by: Rick Edgecombe [ Yan: Updated patch log/function comment, dropped unused param in op ] Co-developed-by: Yan Zhao Signed-off-by: Yan Zhao Link: https://patch.msgid.link/20260509075730.4354-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 2 +- arch/x86/include/asm/kvm_host.h | 3 +-- arch/x86/kvm/mmu/tdp_mmu.c | 13 ++----------- arch/x86/kvm/vmx/tdx.c | 28 ++++++++++++++-------------- 4 files changed, 18 insertions(+), 28 deletions(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 771d991562ca..83dc5086138b 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -97,7 +97,7 @@ KVM_X86_OP_OPTIONAL_RET0(get_mt_mask) KVM_X86_OP_OPTIONAL_RET0(tdp_has_smep) KVM_X86_OP(load_mmu_pgd) KVM_X86_OP_OPTIONAL_RET0(set_external_spte) -KVM_X86_OP_OPTIONAL_RET0(free_external_spt) +KVM_X86_OP_OPTIONAL(free_external_spt) KVM_X86_OP(has_wbinvd_exit) KVM_X86_OP(get_l2_tsc_offset) KVM_X86_OP(get_l2_tsc_multiplier) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index dc5625950e5b..a2ae2d7ca00f 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1916,8 +1916,7 @@ struct kvm_x86_ops { u64 new_spte, enum pg_level level); /* Update external page tables for page table about to be freed. */ - int (*free_external_spt)(struct kvm *kvm, gfn_t gfn, enum pg_level level, - void *external_spt); + void (*free_external_spt)(struct kvm *kvm, struct kvm_mmu_page *sp); bool (*has_wbinvd_exit)(void); diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 72d323f2d0dc..74531e4bbee6 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -455,17 +455,8 @@ static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared) handle_changed_spte(kvm, sp, gfn, old_spte, FROZEN_SPTE, level, shared); } - if (is_mirror_sp(sp) && - WARN_ON(kvm_x86_call(free_external_spt)(kvm, base_gfn, sp->role.level, - sp->external_spt))) { - /* - * Failed to free page table page in mirror page table and - * there is nothing to do further. - * Intentionally leak the page to prevent the kernel from - * accessing the encrypted page. - */ - sp->external_spt = NULL; - } + if (is_mirror_sp(sp)) + kvm_x86_call(free_external_spt)(kvm, sp); call_rcu(&sp->rcu_head, tdp_mmu_free_sp_rcu_callback); } diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index b1c273dfbe4c..b3185cd9c6d6 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1849,27 +1849,27 @@ static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte, return tdx_sept_map_leaf_spte(kvm, gfn, level, new_spte); } -static int tdx_sept_free_private_spt(struct kvm *kvm, gfn_t gfn, - enum pg_level level, void *private_spt) +/* + * Handle changes for non-leaf SPTEs from present to non-present. + * Must be under exclusive mmu_lock and cannot fail. + */ +static void tdx_sept_free_private_spt(struct kvm *kvm, struct kvm_mmu_page *sp) { - struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); - /* - * free_external_spt() is only called after hkid is freed when TD is - * tearing down. * KVM doesn't (yet) zap page table pages in mirror page table while * TD is active, though guest pages mapped in mirror page table could be * zapped during TD is active, e.g. for shared <-> private conversion * and slot move/deletion. + * + * In other words, KVM should only free mirror page tables after the + * TD's hkid is freed, when the TD is being torn down. + * + * If the S-EPT PTE can't be removed for any reason, intentionally leak + * the page to prevent the kernel from accessing the encrypted page. */ - if (KVM_BUG_ON(is_hkid_assigned(kvm_tdx), kvm)) - return -EIO; - - /* - * The HKID assigned to this TD was already freed and cache was - * already flushed. We don't have to flush again. - */ - return tdx_reclaim_page(virt_to_page(private_spt)); + if (KVM_BUG_ON(is_hkid_assigned(to_kvm_tdx(kvm)), kvm) || + tdx_reclaim_page(virt_to_page(sp->external_spt))) + sp->external_spt = NULL; } void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, From 110d4d263450e4172db2f71053d9382320de7e82 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 9 May 2026 15:57:40 +0800 Subject: [PATCH 23/24] KVM: TDX: Move external page table freeing to TDX code Move the freeing of external page tables into the reclaim operation that lives in TDX code. The TDP MMU supports traversing the TDP without holding locks. Page tables need to be freed via RCU to prevent walking one that gets freed. While none of these lockless walk operations actually happen for the mirror page table, the TDP MMU nonetheless frees the mirror page table in the same way, and (because it's a handy place to plug it in) the external page table as well. However, the external page table definitely can't be walked once the page table pages are reclaimed from the TDX module. The TDX module releases the page for the host VMM to use, so this RCU-time free is unnecessary for the external page table. So move the free_page() call to TDX code. Create an tdp_mmu_free_unused_sp() to allow for freeing external page tables that have never left the TDP MMU code (i.e. don't need to be freed in a special way). Link: https://lore.kernel.org/kvm/aYpjNrtGmogNzqwT@google.com [Based on a diff by Sean, added log] Co-developed-by: Rick Edgecombe Signed-off-by: Rick Edgecombe Signed-off-by: Yan Zhao Link: https://patch.msgid.link/20260509075740.4371-1-yan.y.zhao@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/tdp_mmu.c | 16 +++++++++++----- arch/x86/kvm/vmx/tdx.c | 11 ++++++++++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 74531e4bbee6..5b3041138301 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -53,13 +53,18 @@ void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) rcu_barrier(); } -static void tdp_mmu_free_sp(struct kvm_mmu_page *sp) +static void __tdp_mmu_free_sp(struct kvm_mmu_page *sp) { - free_page((unsigned long)sp->external_spt); free_page((unsigned long)sp->spt); kmem_cache_free(mmu_page_header_cache, sp); } +static void tdp_mmu_free_unused_sp(struct kvm_mmu_page *sp) +{ + free_page((unsigned long)sp->external_spt); + __tdp_mmu_free_sp(sp); +} + /* * This is called through call_rcu in order to free TDP page table memory * safely with respect to other kernel threads that may be operating on @@ -73,7 +78,8 @@ static void tdp_mmu_free_sp_rcu_callback(struct rcu_head *head) struct kvm_mmu_page *sp = container_of(head, struct kvm_mmu_page, rcu_head); - tdp_mmu_free_sp(sp); + WARN_ON_ONCE(sp->external_spt); + __tdp_mmu_free_sp(sp); } void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root) @@ -1268,7 +1274,7 @@ int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) * failed, e.g. because a different task modified the SPTE. */ if (r) { - tdp_mmu_free_sp(sp); + tdp_mmu_free_unused_sp(sp); goto retry; } @@ -1579,7 +1585,7 @@ static int tdp_mmu_split_huge_pages_root(struct kvm *kvm, * installs its own sp in place of the last sp we tried to split. */ if (sp) - tdp_mmu_free_sp(sp); + tdp_mmu_free_unused_sp(sp); return 0; } diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index b3185cd9c6d6..54f9eb1a0202 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1869,7 +1869,16 @@ static void tdx_sept_free_private_spt(struct kvm *kvm, struct kvm_mmu_page *sp) */ if (KVM_BUG_ON(is_hkid_assigned(to_kvm_tdx(kvm)), kvm) || tdx_reclaim_page(virt_to_page(sp->external_spt))) - sp->external_spt = NULL; + goto out; + + /* + * Immediately free the S-EPT page because RCU-time free is unnecessary + * after TDH.PHYMEM.PAGE.RECLAIM ensures there are no outstanding + * readers. + */ + free_page((unsigned long)sp->external_spt); +out: + sp->external_spt = NULL; } void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, From 69397c92de77525f70aa43cf3a47256cef409382 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 5 Jun 2026 10:46:10 -0700 Subject: [PATCH 24/24] KVM: x86/mmu: Recursively zap orphaned nested TDP shadow pages on emulated writes Recursively zap orphaned nested TDP shadow pages when emulating a guest write to a shadowed page table, regardless of whether or not the associated (parent) shadow page will be zapped, e.g. due to detected write-flooding. This plugs a hole where KVM fails to reclaim defunct, unsync shadow pages for select L1 hypervisor patterns. Commit 2de4085cccea ("KVM: x86/MMU: Recursively zap nested TDP SPs when zapping last/only parent") modified KVM to recursively zap synchronized shadow pages (KVM already recursively zaps unsync children) when a child is orphaned. But the fix effectively only applied the logic to kvm_mmu_page_unlink_children(), i.e. only performs the recursive zap when KVM is already zapping a parent SP and processing its children. If L1 zaps SPTEs bottom-up (4KiB => 2MiB => ...), as KVM's TDP MMU does with CONFIG_KVM_PROVE_MMU=n since commit 8ca983631f3c ("KVM: x86/mmu: Zap invalidated TDP MMU roots at 4KiB granularity"), then KVM (as L0) will leak upwards of 4 shadow pages per GiB of L2 guest memory. Over hundreds or thousands of L2 boots, if the VM is "lucky" enough to escape write-flooding detection, i.e. not trigger reclaim of the orphaned shadow pages by dumb luck, then it's possible to end up with tens or even hundreds of thousands of unsync shadow pages and associated rmap entries. Polluting the hash table and rmap entries with a horde of stale entries can eventually degrade L2 guest boot time by an order of magnitude, especially if there is any antagonistic activity in the host, i.e. anything that will contend for mmu_lock and/or needs to walk rmaps. With "top"-down zapping, where "top" is 1GiB or above, then L0 KVM is effectively limited to leaking 4 shadow pages per 256 GiB of memory, as KVM's write flooding detection will kick in on the third write to an L1 TDP PUD, and thus recursively zap the entire 256 GiB range of the parent PGD. I.e. even though L1 KVM still recursively zaps 2MiB => 4KiB SPTEs when zapping each 1GiB SPTE, KVM only gets through two of the 1GiB SPTEs before dropping everything. E.g. hacking tracing into L0 KVM's kvm_mmu_track_write(), the top-down zapping of L1's TDP MMU for an L2 with 16GiB of memory leads to: gpa = 107407000, old = 800000010741bd07, new = 8000000000000000, level = 3, flood = 0 gpa = 10741b000, old = 8000000112fb2d07, new = 80000000000001a0, level = 2, flood = 0 gpa = 10741b008, old = 800000012509cd07, new = 80000000000001a0, level = 2, flood = 1 gpa = 10741b010, old = 80000001114b9d07, new = 80000000000001a0, level = 2, flood = 2 gpa = 107407008, old = 8000000112fb5d07, new = 8000000000000000, level = 3, flood = 1 gpa = 112fb5298, old = 8000000106f43d07, new = 80000000000001a0, level = 2, flood = 0 gpa = 112fb52a0, old = 8000000106f4dd07, new = 80000000000001a0, level = 2, flood = 1 gpa = 112fb5ea0, old = 8000000120490d07, new = 80000000000001a0, level = 2, flood = 2 gpa = 107407010, old = 8000000106df2d07, new = 8000000000000000, level = 3, flood = 2 gpa = 107410000, old = 8000000107408d07, new = 8000000000000000, level = 5, flood = 0 gpa = 107408000, old = 8000000107407d07, new = 80000000000001a0, level = 4, flood = 0 Contrast that with a bottom-up zap, which effectively allows all 2MiB SPTEs in L1 to leak their children. gpa = 167939000, old = 800000011c8f4d07, new = 8000000000000000, level = 2, flood = 0 gpa = 167939020, old = 8000000104407d07, new = 8000000000000000, level = 2, flood = 1 gpa = 167939028, old = 800000011ed20d07, new = 8000000000000000, level = 2, flood = 2 gpa = 118c70bb0, old = 8000000167ab9d07, new = 8000000000000000, level = 2, flood = 0 gpa = 118c70bb8, old = 8000000163913d07, new = 8000000000000000, level = 2, flood = 1 gpa = 118c70de8, old = 800000011cc9dd07, new = 8000000000000000, level = 2, flood = 2 gpa = 160be7fb0, old = 800000011d322d07, new = 8000000000000000, level = 2, flood = 1 gpa = 160be7fb8, old = 8000000126b1bd07, new = 8000000000000000, level = 2, flood = 2 gpa = 1634ab000, old = 800000010e984d07, new = 8000000000000000, level = 2, flood = 0 gpa = 1634ab008, old = 800000016879fd07, new = 8000000000000000, level = 2, flood = 1 gpa = 1634ab010, old = 800000016879ed07, new = 8000000000000000, level = 2, flood = 2 gpa = 11e3f1e48, old = 8000000168a33d07, new = 8000000000000000, level = 2, flood = 0 gpa = 11e3f1e50, old = 80000001664dcd07, new = 8000000000000000, level = 2, flood = 1 gpa = 1167eacb8, old = 8000000166544d07, new = 8000000000000000, level = 2, flood = 0 gpa = 1167eacc0, old = 800000015c16bd07, new = 8000000000000000, level = 2, flood = 1 gpa = 1689e89b8, old = 800000015f296d07, new = 8000000000000000, level = 2, flood = 0 gpa = 1689e89c0, old = 8000000167ca8d07, new = 8000000000000000, level = 2, flood = 1 gpa = 107b35eb8, old = 8000000161e71d07, new = 8000000000000000, level = 2, flood = 0 gpa = 107b35ec0, old = 8000000118cf3d07, new = 8000000000000000, level = 2, flood = 1 gpa = 118cf2d48, old = 8000000118cf1d07, new = 8000000000000000, level = 2, flood = 0 gpa = 118cf2d50, old = 8000000118cf0d07, new = 8000000000000000, level = 2, flood = 1 gpa = 118dcb770, old = 8000000118dcad07, new = 8000000000000000, level = 2, flood = 0 gpa = 118dcb778, old = 8000000118dc9d07, new = 8000000000000000, level = 2, flood = 1 gpa = 118dc87e8, old = 8000000126997d07, new = 8000000000000000, level = 2, flood = 0 gpa = 118dc87f0, old = 8000000126996d07, new = 8000000000000000, level = 2, flood = 1 gpa = 126995148, old = 8000000126994d07, new = 8000000000000000, level = 2, flood = 0 gpa = 126995150, old = 8000000103477d07, new = 8000000000000000, level = 2, flood = 1 gpa = 1034764c8, old = 8000000103475d07, new = 8000000000000000, level = 2, flood = 0 gpa = 1034764d0, old = 8000000103474d07, new = 8000000000000000, level = 2, flood = 1 gpa = 10ea4b788, old = 800000010ea4ad07, new = 8000000000000000, level = 2, flood = 0 gpa = 10ea4b790, old = 800000010ea49d07, new = 8000000000000000, level = 2, flood = 1 gpa = 10ea48928, old = 800000011a5bfd07, new = 8000000000000000, level = 2, flood = 0 gpa = 10ea48930, old = 800000011a5bed07, new = 8000000000000000, level = 2, flood = 1 gpa = 11a5bd0d8, old = 800000011a5bcd07, new = 8000000000000000, level = 2, flood = 0 gpa = 11a5bd0e0, old = 800000011d323d07, new = 8000000000000000, level = 2, flood = 1 gpa = 122ce2b40, old = 800000011fe0bd07, new = 8000000000000000, level = 2, flood = 0 gpa = 122ce2b48, old = 800000010e985d07, new = 8000000000000000, level = 2, flood = 1 gpa = 122ce2b50, old = 8000000161c9dd07, new = 8000000000000000, level = 2, flood = 2 gpa = 16864c000, old = 8000000167939d07, new = 8000000000000000, level = 3, flood = 0 gpa = 16864c008, old = 8000000118c70d07, new = 8000000000000000, level = 3, flood = 1 gpa = 16864c010, old = 80000001688a6d07, new = 8000000000000000, level = 3, flood = 2 gpa = 11c8f7000, old = 80000001608a7d07, new = 8000000000000000, level = 5, flood = 0 gpa = 1608a7000, old = 800000016864cd07, new = 80000000000001a0, level = 4, flood = 0 Note, in the shadow MMU, "level" describes the level a shadow page "points" at, not the level of its associated SPTE. I.e. when write-flooding of 1GiB PUD entries is detected, KVM recursively zaps shadow pages covering 256GiB worth of memory. And as shown above, KVM's write-flooding detection operates at all levels, so a single PMD (in L1) can effectively only leak two unsync children (4KiB shadow pages) before it gets recursively zapped. As a result, for the top-down zap, L0 KVM will leak at most 4 unsync shadow pages per 256GiB of L2 memory. The top-down zap also makes it more likely that L1 will self-heal (to some extent), as any shadow pages that are "rediscovered" by future runs of L2 can get reclaimed by a recursive zap, whereas bottom-up zapping orphans shadow pages over and over. Note, in theory, there is some risk of over-zapping, e.g. due to zapping a a large branch of the paging tree that L1 is only temporarily removing. In practice, the usage patterns of hypervisors are highly unlikely to trigger false positives. E.g. temporarily changing paging protections is typically done at the leaf, not on a non-leaf entry. And if the L1 hypervisor is updating large swaths of PTEs, e.g. to (temporarily?) remove chunks of memory from L2, then L0 KVM's write-flooding detection will kick in, and the children would be zapped anyways. Fixes: 2de4085cccea ("KVM: x86/MMU: Recursively zap nested TDP SPs when zapping last/only parent") Cc: Yosry Ahmed Cc: Jim Mattson Cc: James Houghton Reviewed-by: Jim Mattson Reviewed-by: Yosry Ahmed Link: https://patch.msgid.link/20260605174611.2222504-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index f8aa7eda661e..6e881c40f823 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -6357,7 +6357,7 @@ void kvm_mmu_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, while (npte--) { entry = *spte; - mmu_page_zap_pte(vcpu->kvm, sp, spte, NULL); + mmu_page_zap_pte(vcpu->kvm, sp, spte, &invalid_list); if (gentry && sp->role.level != PG_LEVEL_4K) ++vcpu->kvm->stat.mmu_pde_zapped; if (is_shadow_present_pte(entry))