From 42a39ad5d592aec87a70527a4e694f6210694482 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:31 -0700 Subject: [PATCH 01/21] KVM: SEV: Track the GPA of the guest-controlled VMSA used for SNP guests Track the GPA of the guest-provided VMSA used after AP_CREATION events when running SNP guests, instead of simply tracking whether or not the vCPU is using a guest-provided VMSA. KVM needs to know the GPA of the VMSA that's actively being used so that it can react to MMU invalidation events, i.e. so that KVM can drop the VMSA if its backing guest_memfd page is punched out of existence. Opportunistically rename snp_vmsa_gpa to clarify that it tracks the pending VMSA GPA, whereas snp_guest_vmsa_gpa now tracks the in-use VMSA GPA. Note! Take care to track the GPA, not the GFN, as VALID_PAGE() won't behave correctly if an invalid GFN is converted to a GPA for checking. Note #2! Keep snp_has_guest_vmsa so that switching to a guest-provided VMSA is sticky, even if the guest-provided VMSA becomes invalid. No functional change intended. Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 14 +++++++++----- arch/x86/kvm/svm/svm.h | 3 ++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 74fb15551e83..827f5dc06102 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -4003,6 +4003,7 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) /* Clear use of the VMSA */ svm->vmcb->control.vmsa_pa = INVALID_PAGE; + svm->sev_es.snp_guest_vmsa_gpa = INVALID_PAGE; /* * When replacing the VMSA during SEV-SNP AP creation, @@ -4010,11 +4011,11 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) */ vmcb_mark_all_dirty(svm->vmcb); - if (!VALID_PAGE(svm->sev_es.snp_vmsa_gpa)) + if (!VALID_PAGE(svm->sev_es.snp_pending_vmsa_gpa)) return; - gfn = gpa_to_gfn(svm->sev_es.snp_vmsa_gpa); - svm->sev_es.snp_vmsa_gpa = INVALID_PAGE; + gfn = gpa_to_gfn(svm->sev_es.snp_pending_vmsa_gpa); + svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; slot = gfn_to_memslot(vcpu->kvm, gfn); if (!slot) @@ -4039,6 +4040,7 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) svm->sev_es.snp_has_guest_vmsa = true; /* Use the new VMSA */ + svm->sev_es.snp_guest_vmsa_gpa = gfn_to_gpa(gfn); svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); /* Mark the vCPU as runnable */ @@ -4105,10 +4107,10 @@ static int sev_snp_ap_creation(struct vcpu_svm *svm) return -EINVAL; } - target_svm->sev_es.snp_vmsa_gpa = svm->vmcb->control.exit_info_2; + target_svm->sev_es.snp_pending_vmsa_gpa = svm->vmcb->control.exit_info_2; break; case SVM_VMGEXIT_AP_DESTROY: - target_svm->sev_es.snp_vmsa_gpa = INVALID_PAGE; + target_svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; break; default: vcpu_unimpl(vcpu, "vmgexit: invalid AP creation request [%#x] from guest\n", @@ -4791,6 +4793,8 @@ int sev_vcpu_create(struct kvm_vcpu *vcpu) return -ENOMEM; svm->sev_es.vmsa = page_address(vmsa_page); + svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; + svm->sev_es.snp_guest_vmsa_gpa = INVALID_PAGE; vcpu->arch.guest_tsc_protected = snp_is_secure_tsc_enabled(vcpu->kvm); diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 716be21fba33..d077783c287e 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -271,7 +271,8 @@ struct vcpu_sev_es_state { u64 ghcb_registered_gpa; struct mutex snp_vmsa_mutex; /* Used to handle concurrent updates of VMSA. */ - gpa_t snp_vmsa_gpa; + gpa_t snp_pending_vmsa_gpa; + gpa_t snp_guest_vmsa_gpa; bool snp_ap_waiting_for_reset; bool snp_has_guest_vmsa; }; From 0060569e4f18a7dee2dd8728595e909f19a23c24 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:32 -0700 Subject: [PATCH 02/21] KVM: SEV: Extract loading of guest-provided VMSA to a separate helper Extract the loading/retrieval of a guest-provided VMSA to a separate helper so that KVM can reuse the core logic when refreshing the VMSA after an MMU invalidation from guest_memfd. No functional change intended. Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 52 +++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 827f5dc06102..d8ed00f76aa3 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -3979,29 +3979,17 @@ static int snp_begin_psc(struct vcpu_svm *svm) return snp_do_psc(svm); } -/* - * Invoked as part of svm_vcpu_reset() processing of an init event. - */ -static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) +static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) { struct vcpu_svm *svm = to_svm(vcpu); struct kvm_memory_slot *slot; + gfn_t gfn = gpa_to_gfn(gpa); struct page *page; kvm_pfn_t pfn; - gfn_t gfn; - guard(mutex)(&svm->sev_es.snp_vmsa_mutex); + lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex); - if (!svm->sev_es.snp_ap_waiting_for_reset) - return; - - svm->sev_es.snp_ap_waiting_for_reset = false; - - /* Mark the vCPU as offline and not runnable */ - vcpu->arch.pv.pv_unhalted = false; - kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED); - - /* Clear use of the VMSA */ + /* Clear use of the VMSA. */ svm->vmcb->control.vmsa_pa = INVALID_PAGE; svm->sev_es.snp_guest_vmsa_gpa = INVALID_PAGE; @@ -4011,12 +3999,9 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) */ vmcb_mark_all_dirty(svm->vmcb); - if (!VALID_PAGE(svm->sev_es.snp_pending_vmsa_gpa)) + if (!VALID_PAGE(gpa)) return; - gfn = gpa_to_gfn(svm->sev_es.snp_pending_vmsa_gpa); - svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; - slot = gfn_to_memslot(vcpu->kvm, gfn); if (!slot) return; @@ -4040,7 +4025,7 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) svm->sev_es.snp_has_guest_vmsa = true; /* Use the new VMSA */ - svm->sev_es.snp_guest_vmsa_gpa = gfn_to_gpa(gfn); + svm->sev_es.snp_guest_vmsa_gpa = gpa; svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); /* Mark the vCPU as runnable */ @@ -4054,6 +4039,31 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) kvm_release_page_clean(page); } +/* + * Invoked as part of svm_vcpu_reset() processing of an init event. + */ +static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + gpa_t gpa; + + guard(mutex)(&svm->sev_es.snp_vmsa_mutex); + + if (!svm->sev_es.snp_ap_waiting_for_reset) + return; + + svm->sev_es.snp_ap_waiting_for_reset = false; + + /* Mark the vCPU as offline and not runnable */ + vcpu->arch.pv.pv_unhalted = false; + kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED); + + gpa = svm->sev_es.snp_pending_vmsa_gpa; + svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; + + sev_snp_reload_vmsa(vcpu, gpa); +} + static int sev_snp_ap_creation(struct vcpu_svm *svm) { struct kvm_sev_info *sev = to_kvm_sev_info(svm->vcpu.kvm); From 98ade8c48c28c227fe2e80e545ff0c57cd4712a3 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:33 -0700 Subject: [PATCH 03/21] KVM: SEV: Mark vCPU RUNNABLE after AP_CREATE, even if VMSA is unusable Always mark the vCPU as RUNNABLE after responding to AP_CREATE, even if the guest-specified VMSA is unusable, e.g. isn't backed by a memslot or doesn't have a backing guest_memfd page. If the VMSA is unusable, leaving the vCPU in a non-running state will effectively hang the vCPU instead of reporting an error to userspace. This will also allow retrying the VMSA load in the future, to fix a bug where KVM doesn't honor guest_memfd invalidation events, e.g. if AP_CREATION races with PUNCH_HOLE. Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index d8ed00f76aa3..30792adcfc8e 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -4028,9 +4028,6 @@ static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) svm->sev_es.snp_guest_vmsa_gpa = gpa; svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); - /* Mark the vCPU as runnable */ - kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); - /* * gmem pages aren't currently migratable, but if this ever changes * then care should be taken to ensure svm->sev_es.vmsa is pinned @@ -4062,6 +4059,15 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; sev_snp_reload_vmsa(vcpu, gpa); + + /* + * Mark the vCPU as runnable for CREATE requests, indicated by a valid + * VMSA GPA, even if installing the VMSA failed, so that KVM_RUN will + * fail instead of blocking indefinitely and hanging the vCPU, e.g. if + * the backing guest_memfd page is unavailable. + */ + if (VALID_PAGE(gpa)) + kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); } static int sev_snp_ap_creation(struct vcpu_svm *svm) From 01a96ff30dde5127c37497f1e098e639e7ae152f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:34 -0700 Subject: [PATCH 04/21] KVM: SEV: Wire up kvm_x86_ops.gmem_xxx() if and only if CONFIG_KVM_AMD_SEV=y Wire up the SEV-SNP guest_memfd kvm_x86_ops hooks if and only if SEV is actually enabled, and drop the now-unnecessary stubs. Leaving the hooks NULL allows the static call infrastructure to elide the CALL+RET, and more importantly, referencing the hooks if and only if SEV support is enabled will allow conditionally definining the hooks using their corresponding HAVE_KVM_ARCH_GMEM_XXX Kconfig. No functional change intended. Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260709204948.1988414-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/svm.c | 8 ++++---- arch/x86/kvm/svm/svm.h | 10 ---------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index ef69a51ab27f..79c818d91dda 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5448,6 +5448,10 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .vm_copy_enc_context_from = sev_vm_copy_enc_context_from, .vm_move_enc_context_from = sev_vm_move_enc_context_from, + + .gmem_prepare = sev_gmem_prepare, + .gmem_invalidate = sev_gmem_invalidate, + .gmem_max_mapping_level = sev_gmem_max_mapping_level, #endif .check_emulate_instruction = svm_check_emulate_instruction, @@ -5459,10 +5463,6 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector, .vcpu_get_apicv_inhibit_reasons = avic_vcpu_get_apicv_inhibit_reasons, .alloc_apic_backing_page = svm_alloc_apic_backing_page, - - .gmem_prepare = sev_gmem_prepare, - .gmem_invalidate = sev_gmem_invalidate, - .gmem_max_mapping_level = sev_gmem_max_mapping_level, }; /* diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index d077783c287e..effca6372e15 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -1035,16 +1035,6 @@ static inline int sev_cpu_init(struct svm_cpu_data *sd) { return 0; } static inline int sev_dev_get_attr(u32 group, u64 attr, u64 *val) { return -ENXIO; } #define max_sev_asid 0 static inline void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) {} -static inline int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) -{ - return 0; -} -static inline void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) {} -static inline int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private) -{ - return 0; -} - static inline struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu) { return NULL; From ba76b23ed36ab230fc2577aba24f65851114902f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:35 -0700 Subject: [PATCH 05/21] KVM: x86: Serialize writes to disabled_quirks using kvm->lock Protect writes to disabled_quirks with kvm->lock to ensure KVM doesn't clobber state in the unlikely scenario that userspace disables disparate quirks from multiple tasks. More importantly, this will allow wrapping accesses with {READ,WRITE}_ONCE without "needing" to also guard the writer with a useless and confusing READ_ONCE (since the RMW wouldn't be atomic anyways). Ideally, KVM would disallow disabling quirks once quirks are "live", but that would be a potentially breaking userspace ABI change, and while all existing quirks are fully live only after vCPUs have been created, several MMU-related quirks, IGNORE_GUEST_PAT and SLOT_ZAP_ALL, are partially live at all times. Because populating MMUs requires a vCPU, the guest-visible behavior of IGNORE_GUEST_PAT and SLOT_ZAP_ALL requires a vCPU, but for KVM itself, processing the quirk (or not) has functional impact, i.e. for all intents and purposes, KVM can't prevent those quirks from being disabled after they've been consumed. Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-6-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 0626e835e9eb..226c6cfe8062 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3939,7 +3939,9 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, break; fallthrough; case KVM_CAP_DISABLE_QUIRKS: + mutex_lock(&kvm->lock); kvm->arch.disabled_quirks |= cap->args[0] & kvm_caps.supported_quirks; + mutex_unlock(&kvm->lock); r = 0; break; case KVM_CAP_SPLIT_IRQCHIP: { From ed15cb21999217e549414c128b4a0485debf6278 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:36 -0700 Subject: [PATCH 06/21] KVM: x86: Ensure runtime reads of disabled_quirks are resolved once Wrap the sole reader of disabled_quirks with READ_ONCE(), and wrap the post-VM-creation write to disabled_quirks with WRITE_ONCE(), to ensure checking the status of a quirk doesn't re-read disabled_quirks *if* the caller needs such a guarantee. This will allow splitting the "fast" MMU zap into front and back halves, without potentially skipping the back half if SLOT_ZAP_ALL were concurrently disabled (which would be "fine" in the current code base, but far from ideal). Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-7-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 3 ++- arch/x86/kvm/x86.h | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 226c6cfe8062..8abd733d5173 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3940,7 +3940,8 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, fallthrough; case KVM_CAP_DISABLE_QUIRKS: mutex_lock(&kvm->lock); - kvm->arch.disabled_quirks |= cap->args[0] & kvm_caps.supported_quirks; + WRITE_ONCE(kvm->arch.disabled_quirks, + kvm->arch.disabled_quirks | (cap->args[0] & kvm_caps.supported_quirks)); mutex_unlock(&kvm->lock); r = 0; break; diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 8ece468087a8..75f13d88db58 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -304,7 +304,7 @@ static inline bool vcpu_match_mmio_gpa(struct kvm_vcpu *vcpu, gpa_t gpa) static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk) { - return !(kvm->arch.disabled_quirks & quirk); + return !(READ_ONCE(kvm->arch.disabled_quirks) & quirk); } static __always_inline void kvm_request_l1tf_flush_l1d(void) From 06d38eaa78fdac1cc889f261fa420eba8e9caa1a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:37 -0700 Subject: [PATCH 07/21] KVM: x86/mmu: Fold kvm_mmu_zap_memslot() into kvm_arch_flush_shadow_memslot() Fold kvm_mmu_zap_memslot() into its sole caller so that its GFN range structure can be used to trigger guest_memfd invalidations regardless of whether KVM will do a partial or full zap of the MMU. No functional change intended. Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-8-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 6c13da942bfc..223d80b12b9b 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -7560,8 +7560,14 @@ static void kvm_mmu_zap_memslot_pages_and_flush(struct kvm *kvm, kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush); } -static void kvm_mmu_zap_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot) +static inline bool kvm_memslot_flush_zap_all(struct kvm *kvm) +{ + return kvm->arch.vm_type == KVM_X86_DEFAULT_VM && + kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL); +} + +void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) { struct kvm_gfn_range range = { .slot = slot, @@ -7572,25 +7578,14 @@ static void kvm_mmu_zap_memslot(struct kvm *kvm, }; bool flush; - write_lock(&kvm->mmu_lock); - flush = kvm_unmap_gfn_range(kvm, &range); - kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush); - write_unlock(&kvm->mmu_lock); -} - -static inline bool kvm_memslot_flush_zap_all(struct kvm *kvm) -{ - return kvm->arch.vm_type == KVM_X86_DEFAULT_VM && - kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL); -} - -void kvm_arch_flush_shadow_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot) -{ - if (kvm_memslot_flush_zap_all(kvm)) + if (kvm_memslot_flush_zap_all(kvm)) { kvm_mmu_zap_all_fast(kvm); - else - kvm_mmu_zap_memslot(kvm, slot); + } else { + write_lock(&kvm->mmu_lock); + flush = kvm_unmap_gfn_range(kvm, &range); + kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush); + write_unlock(&kvm->mmu_lock); + } } void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) From b27622c4eeb125814081baaefe9175191be5b94d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:38 -0700 Subject: [PATCH 08/21] KVM: x86/mmu: Split kvm_mmu_zap_all_fast() into "front" and "back" halves Split kvm_mmu_zap_all_fast() into a "front half" and a "back half", where the front half is everything that runs with mmu_lock held for write, and the back half is the code that runs outside of mmu_lock. This will allow putting more code inside kvm_arch_flush_shadow_memslot()'s critical section without having to take mmu_lock twice in quick succession. No functional change intended. Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-9-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 223d80b12b9b..a5c2a560a88a 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -6921,20 +6921,11 @@ static void kvm_zap_obsolete_pages(struct kvm *kvm) kvm_mmu_commit_zap_page(kvm, &invalid_list); } -/* - * Fast invalidate all shadow pages and use lock-break technique - * to zap obsolete pages. - * - * It's required when memslot is being deleted or VM is being - * destroyed, in these cases, we should ensure that KVM MMU does - * not use any resource of the being-deleted slot or all slots - * after calling the function. - */ -static void kvm_mmu_zap_all_fast(struct kvm *kvm) +static void __kvm_mmu_zap_all_fast_front_half(struct kvm *kvm) { lockdep_assert_held(&kvm->slots_lock); + lockdep_assert_held_write(&kvm->mmu_lock); - write_lock(&kvm->mmu_lock); trace_kvm_mmu_zap_all_fast(kvm); /* @@ -6971,8 +6962,12 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm) kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS); kvm_zap_obsolete_pages(kvm); +} - write_unlock(&kvm->mmu_lock); +static void __kvm_mmu_zap_all_fast_back_half(struct kvm *kvm) +{ + lockdep_assert_held(&kvm->slots_lock); + lockdep_assert_not_held(&kvm->mmu_lock); /* * Zap the invalidated TDP MMU roots, all SPTEs must be dropped before @@ -6986,6 +6981,24 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm) kvm_tdp_mmu_zap_invalidated_roots(kvm, true); } +/* + * Fast invalidate all shadow pages and use lock-break technique + * to zap obsolete pages. + * + * It's required when memslot is being deleted or VM is being + * destroyed, in these cases, we should ensure that KVM MMU does + * not use any resource of the being-deleted slot or all slots + * after calling the function. + */ +static void kvm_mmu_zap_all_fast(struct kvm *kvm) +{ + write_lock(&kvm->mmu_lock); + __kvm_mmu_zap_all_fast_front_half(kvm); + write_unlock(&kvm->mmu_lock); + + __kvm_mmu_zap_all_fast_back_half(kvm); +} + int kvm_mmu_init_vm(struct kvm *kvm) { int r, i; From db095727ff5739f4f46ee641ee6ef450032886db Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:39 -0700 Subject: [PATCH 09/21] KVM: x86/mmu: Use split "zap all fast" helpers when invalidating memslot Manually invoke the front half and back half of the "zap all fast" flow when invalidating a memslot so that mmu_lock is acquired at function scope in kvm_arch_flush_shadow_memslot(). This will allow putting more code inside the critical section without having to take mmu_lock twice in quick succession. Opportunistically open code checking whether or not to do the fast zap, to discourage removing the local "zap_all" in a future cleanup, i.e. to ensure the SLOT_ZAP_ALL quirk is queried exactly once. Processing the front half but not the back half of the fast zap (if SLOT_ZAP_ALL were disabled concurrently) would result in KVM unnecessarily keeping invalid TDP MMU roots until the VM is destroyed. No functional change intended. Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-10-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index a5c2a560a88a..3eb1f86593b1 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -7573,12 +7573,6 @@ static void kvm_mmu_zap_memslot_pages_and_flush(struct kvm *kvm, kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush); } -static inline bool kvm_memslot_flush_zap_all(struct kvm *kvm) -{ - return kvm->arch.vm_type == KVM_X86_DEFAULT_VM && - kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL); -} - void kvm_arch_flush_shadow_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) { @@ -7589,16 +7583,23 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm, .may_block = true, .attr_filter = KVM_FILTER_PRIVATE | KVM_FILTER_SHARED, }; + bool zap_all = kvm->arch.vm_type == KVM_X86_DEFAULT_VM && + kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL); bool flush; - if (kvm_memslot_flush_zap_all(kvm)) { - kvm_mmu_zap_all_fast(kvm); + write_lock(&kvm->mmu_lock); + + if (zap_all) { + __kvm_mmu_zap_all_fast_front_half(kvm); } else { - write_lock(&kvm->mmu_lock); flush = kvm_unmap_gfn_range(kvm, &range); kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush); - write_unlock(&kvm->mmu_lock); } + + write_unlock(&kvm->mmu_lock); + + if (zap_all) + __kvm_mmu_zap_all_fast_back_half(kvm); } void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) From d1a3c216233413f57f5341a9b878b7e2dde7e785 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:40 -0700 Subject: [PATCH 10/21] KVM: SEV: Forcefully invalidate SNP VMSA if its backing gmem page is zapped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire up a gmem_invalidate_range() call for SNP VMs, and use it to force vCPUs to reload/recheck their guest-provided VMSA if the backing gmem page is being invalidated, e.g. is being PUNCH_HOLE'd. Use the same core logic to handle invalidations as VMX does for the APIC-access page, as the two concepts are nearly identical: shove the physical address of a page into the vCPU's control structure: 1. Snapshot the invalidation sequence counter 2. Grab the pfn (from guest_memfd in this case) 3. Acquire mmu_lock for read 4. Re-request reload if retry is needed, otherwise commit the change. Note, the re-request action in #4 is necessary as KVM's retry logic is fuzzy, i.e. can get false positives. If the guest_memfd page has been dropped, at some point a subsequent reload will fail to get a PFN from guest_memfd, and KVM will fail KVM_RUN. If the retry was due to a false positive, KVM will retry until there are no relevant MMU notifier events (and will retry in the "outer" loop, i.e. will drop locks and resched as needed). Note #2! Take care to invalidate the VMSA when a relevant memslot is DELETED or MOVED, as invalidations in response to PUNCH_HOLE are predicated on memslot bindings (KVM doesn't know what GFN range(s) to invalidate without a binding). And more importantly, the VMSA mapping requires a memslot, i.e. must be invalidated if its memslots disappears, regardless of the state of the underlying guest_memfd inode. Failure to invalidate the vCPU's control.vmsa_pa (which is checked by pre_sev_run()) can prevent KVM from properly freeing the page as firmware will reject the RMPUPDATE to reclaim the page with FAIL_INUSE if the vCPU is actively running, i.e. if VMSA page is in-use. That in turn leads to an RMP #PF on the next use, as the page will still be assigned to the SNP VM. SEV-SNP: RMPUPDATE failed for PFN 78d198, pg_level: 1, ret: 3 SEV-SNP: PFN 0x78d198, RMP entry: [0xfff0000000144001 - 0x000000000000000f] CPU: 3 UID: 0 PID: 31345 Comm: sev_snp_vmsa_pu Tainted: G U O Tainted: [U]=USER, [O]=OOT_MODULE Hardware name: Google, Inc. Arcadia_IT_80/Arcadia_IT_80, BIOS 34.86.0-102 01/25/2026 Call Trace: dump_stack_lvl+0x54/0x70 rmpupdate+0x12c/0x140 rmp_make_shared+0x3b/0x60 sev_gmem_invalidate+0xe0/0x170 [kvm_amd] delete_from_page_cache_batch+0x1d8/0x220 truncate_inode_pages_range+0x120/0x3d0 kvm_gmem_fallocate+0x19a/0x270 [kvm] vfs_fallocate+0x1bc/0x1f0 __x64_sys_fallocate+0x48/0x70 do_syscall_64+0x10a/0x480 entry_SYSCALL_64_after_hwframe+0x4b/0x53 RIP: 0033:0x496c7e ------------[ cut here ]------------ SEV: Failed to update RMP entry for PFN 0x78d198 error -14 WARNING: arch/x86/kvm/svm/sev.c:5160 at sev_gmem_invalidate+0x126/0x170 [kvm_amd], CPU#3: sev_snp_vmsa_pu/31345 CPU: 3 UID: 0 PID: 31345 Comm: sev_snp_vmsa_pu Tainted: G U O Tainted: [U]=USER, [O]=OOT_MODULE Hardware name: Google, Inc. Arcadia_IT_80/Arcadia_IT_80, BIOS 34.86.0-102 01/25/2026 RIP: 0010:sev_gmem_invalidate+0x12b/0x170 [kvm_amd] Call Trace: delete_from_page_cache_batch+0x1d8/0x220 truncate_inode_pages_range+0x120/0x3d0 kvm_gmem_fallocate+0x19a/0x270 [kvm] vfs_fallocate+0x1bc/0x1f0 __x64_sys_fallocate+0x48/0x70 do_syscall_64+0x10a/0x480 entry_SYSCALL_64_after_hwframe+0x4b/0x53 RIP: 0033:0x496c7e irq event stamp: 20689 hardirqs last enabled at (20699): [] __console_unlock+0x5c/0x60 hardirqs last disabled at (20708): [] __console_unlock+0x41/0x60 softirqs last enabled at (20722): [] __irq_exit_rcu+0x7e/0x140 softirqs last disabled at (20717): [] __irq_exit_rcu+0x7e/0x140 ---[ end trace 0000000000000000 ]--- BUG: unable to handle page fault for address: ffff99a64d198000 #PF: supervisor write access in kernel mode #PF: error_code(0x80000003) - RMP violation PGD 13eb001067 P4D 13eb001067 PUD 78d1d1063 PMD 1184e0063 PTE 800000078d198163 SEV-SNP: PFN 0x78d198, RMP entry: [0x6030000000144001 - 0x000000000000000f] Oops: Oops: 0003 [#1] SMP CPU: 3 UID: 0 PID: 31407 Comm: highlanderd_hea Tainted: G U W O Tainted: [U]=USER, [W]=WARN, [O]=OOT_MODULE Hardware name: Google, Inc. Arcadia_IT_80/Arcadia_IT_80, BIOS 34.86.0-102 01/25/2026 RIP: 0010:prep_new_page+0x67/0x220 Call Trace: get_page_from_freelist+0x1c40/0x1c70 __alloc_frozen_pages_noprof+0xca/0x1f0 alloc_pages_mpol+0x10b/0x1b0 alloc_pages_noprof+0x81/0x90 pte_alloc_one+0x1b/0xd0 do_pte_missing+0xdf/0x1020 handle_mm_fault+0x7c7/0xb20 do_user_addr_fault+0x268/0x6b0 exc_page_fault+0x67/0xa0 asm_exc_page_fault+0x26/0x30 RIP: 0033:0x4a6b1e gsmi: Log Shutdown Reason 0x03 CR2: ffff99a64d198000 ---[ end trace 0000000000000000 ]--- RIP: 0010:prep_new_page+0x67/0x220 Drop the pseudo-TODO comment about needing to pin the page if guest_memfd every supports migration, as integrating with invalidations events means KVM will Just Work if/when page migration is ever supported (assuming SNP hardware supports migrating VMSA pages). Note #3, invalidate() and invalidate_range() have _completely_ different semantics; the new invalidate_range() is a true invalidation, whereas the existing invalidate() is really a "make shared" operation. Ignore the confusing naming and poor Kconfig bundling for the moment to minimize the delta for LTS kernels, the mess will be cleaned up shortly. Reported-by: Hyunwoo Kim Closes: https://lore.kernel.org/all/aimMWzAf5b3luM0b@v4bel Fixes: e366f92ea99e ("KVM: SEV: Support SEV-SNP AP Creation NAE event") Cc: stable@vger.kernel.org Cc: Tom Lendacky Cc: Michael Roth Cc: Jörg Rödel Cc: Fuad Tabba Cc: Ackerley Tng Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-11-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 4 ++ arch/x86/include/asm/kvm_host.h | 6 +++ arch/x86/kvm/mmu/mmu.c | 5 ++ arch/x86/kvm/svm/sev.c | 80 ++++++++++++++++++++++++++---- arch/x86/kvm/svm/svm.c | 2 + arch/x86/kvm/svm/svm.h | 2 + arch/x86/kvm/x86.c | 6 +++ include/linux/kvm_host.h | 1 + virt/kvm/guest_memfd.c | 4 ++ 9 files changed, 99 insertions(+), 11 deletions(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 83dc5086138b..ccf23b3f0e1c 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -134,6 +134,7 @@ KVM_X86_OP_OPTIONAL(mem_enc_unregister_region) KVM_X86_OP_OPTIONAL(vm_copy_enc_context_from) KVM_X86_OP_OPTIONAL(vm_move_enc_context_from) KVM_X86_OP_OPTIONAL(guest_memory_reclaimed) +KVM_X86_OP_OPTIONAL(reload_vmsa) KVM_X86_OP(get_feature_msr) KVM_X86_OP(check_emulate_instruction) KVM_X86_OP(apic_init_signal_blocked) @@ -148,6 +149,9 @@ KVM_X86_OP_OPTIONAL(alloc_apic_backing_page) KVM_X86_OP_OPTIONAL_RET0(gmem_prepare) KVM_X86_OP_OPTIONAL_RET0(gmem_max_mapping_level) KVM_X86_OP_OPTIONAL(gmem_invalidate) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE +KVM_X86_OP_OPTIONAL(gmem_invalidate_range) +#endif #endif #undef KVM_X86_OP diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index b517257a6315..1c598b40e0c3 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -122,6 +122,8 @@ KVM_ARCH_REQ_FLAGS(31, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_HV_TLB_FLUSH \ KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) +#define KVM_REQ_VMSA_PAGE_RELOAD \ + KVM_ARCH_REQ_FLAGS(33, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_UPDATE_PROTECTED_GUEST_STATE \ KVM_ARCH_REQ_FLAGS(34, KVM_REQUEST_WAIT) @@ -1878,6 +1880,7 @@ struct kvm_x86_ops { int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd); int (*vm_move_enc_context_from)(struct kvm *kvm, unsigned int source_fd); void (*guest_memory_reclaimed)(struct kvm *kvm); + void (*reload_vmsa)(struct kvm_vcpu *vcpu); int (*get_feature_msr)(u32 msr, u64 *data); @@ -1902,6 +1905,9 @@ struct kvm_x86_ops { void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu); int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); void (*gmem_invalidate)(kvm_pfn_t start, kvm_pfn_t end); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE + void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range); +#endif int (*gmem_max_mapping_level)(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); }; diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 3eb1f86593b1..e2978e9a1731 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -7589,6 +7589,11 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm, write_lock(&kvm->mmu_lock); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE + if (slot->gmem.file) + kvm_arch_gmem_invalidate_range(kvm, &range); +#endif + if (zap_all) { __kvm_mmu_zap_all_fast_front_half(kvm); } else { diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 30792adcfc8e..62c6126d52c0 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -3979,19 +3979,25 @@ static int snp_begin_psc(struct vcpu_svm *svm) return snp_do_psc(svm); } -static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) +static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) { struct vcpu_svm *svm = to_svm(vcpu); struct kvm_memory_slot *slot; + struct kvm *kvm = vcpu->kvm; gfn_t gfn = gpa_to_gfn(gpa); + unsigned long mmu_seq; struct page *page; kvm_pfn_t pfn; lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex); - /* Clear use of the VMSA. */ + /* + * Clear use of the VMSA. Ensure snp_guest_vmsa_gpa is written exactly + * once, as it is read locklessly when responding to gfn invalidations. + * Pairs with the READ_ONCE() in sev_gmem_invalidate_range(). + */ svm->vmcb->control.vmsa_pa = INVALID_PAGE; - svm->sev_es.snp_guest_vmsa_gpa = INVALID_PAGE; + WRITE_ONCE(svm->sev_es.snp_guest_vmsa_gpa, INVALID_PAGE); /* * When replacing the VMSA during SEV-SNP AP creation, @@ -4006,6 +4012,9 @@ static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) if (!slot) return; + mmu_seq = kvm->mmu_invalidate_seq; + smp_rmb(); + /* * The new VMSA will be private memory guest memory, so retrieve the * PFN from the gmem backend. @@ -4024,15 +4033,20 @@ static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) */ svm->sev_es.snp_has_guest_vmsa = true; - /* Use the new VMSA */ - svm->sev_es.snp_guest_vmsa_gpa = gpa; - svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); - + read_lock(&kvm->mmu_lock); /* - * gmem pages aren't currently migratable, but if this ever changes - * then care should be taken to ensure svm->sev_es.vmsa is pinned - * through some other means. + * Save the guest-provided GPA. If retry is needed, then KVM will try + * again with the same GPA. If the VMSA is usable, then KVM needs to + * track the GPA so that the VMSA can be reloaded if the backing page + * for the GPA is invalidated. */ + svm->sev_es.snp_guest_vmsa_gpa = gpa; + if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) + kvm_make_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu); + else + svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); + read_unlock(&kvm->mmu_lock); + kvm_release_page_clean(page); } @@ -4058,7 +4072,7 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) gpa = svm->sev_es.snp_pending_vmsa_gpa; svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; - sev_snp_reload_vmsa(vcpu, gpa); + __sev_snp_reload_vmsa(vcpu, gpa); /* * Mark the vCPU as runnable for CREATE requests, indicated by a valid @@ -4070,6 +4084,15 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); } +void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu) +{ + struct vcpu_sev_es_state *sev_es = &to_svm(vcpu)->sev_es; + + guard(mutex)(&sev_es->snp_vmsa_mutex); + + __sev_snp_reload_vmsa(vcpu, sev_es->snp_guest_vmsa_gpa); +} + static int sev_snp_ap_creation(struct vcpu_svm *svm) { struct kvm_sev_info *sev = to_kvm_sev_info(svm->vcpu.kvm); @@ -5199,6 +5222,41 @@ void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) } } +void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) +{ + struct kvm_vcpu *vcpu; + unsigned long i; + + lockdep_assert_held_write(&kvm->mmu_lock); + + /* + * An unstable result for "is SNP" is a-ok here, thanks to mmu_lock. + * The vCPU's VMSA GPA is invalidated before the vCPU is made visible + * to other tasks, and can only become valid while holding mmu_lock, + * after the VM is fully committed to being an SNP VM. + */ + if (!____sev_snp_guest(kvm)) + return; + + kvm_for_each_vcpu(i, vcpu, kvm) { + /* + * Read snp_guest_vmsa_gpa without taking the vCPU's VMSA mutex + * (or its generic mutex) as mmu_lock is held, i.e. this task + * can't sleep. The VMSA is invalidated outside of mmu_lock, + * but can only become valid inside of mmu_lock, i.e. the below + * can get false positives, but not false negatives. A false + * positive is benign, as a spurious request simply forces the + * vCPU to re-establish its VMSA. + */ + gpa_t gpa = READ_ONCE(to_svm(vcpu)->sev_es.snp_guest_vmsa_gpa); + + if (VALID_PAGE(gpa) && + gpa_to_gfn(gpa) >= range->start && + gpa_to_gfn(gpa) < range->end) + kvm_make_request_and_kick(KVM_REQ_VMSA_PAGE_RELOAD, vcpu); + } +} + int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private) { int level, rc; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 79c818d91dda..dd51df74c2dc 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5445,12 +5445,14 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .mem_enc_register_region = sev_mem_enc_register_region, .mem_enc_unregister_region = sev_mem_enc_unregister_region, .guest_memory_reclaimed = sev_guest_memory_reclaimed, + .reload_vmsa = sev_snp_reload_vmsa, .vm_copy_enc_context_from = sev_vm_copy_enc_context_from, .vm_move_enc_context_from = sev_vm_move_enc_context_from, .gmem_prepare = sev_gmem_prepare, .gmem_invalidate = sev_gmem_invalidate, + .gmem_invalidate_range = sev_gmem_invalidate_range, .gmem_max_mapping_level = sev_gmem_max_mapping_level, #endif .check_emulate_instruction = svm_check_emulate_instruction, diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index effca6372e15..130205defffa 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -996,6 +996,7 @@ static inline struct page *snp_safe_alloc_page(void) { return snp_safe_alloc_page_node(numa_node_id(), GFP_KERNEL_ACCOUNT); } +void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu); int sev_vcpu_create(struct kvm_vcpu *vcpu); void sev_free_vcpu(struct kvm_vcpu *vcpu); @@ -1010,6 +1011,7 @@ extern unsigned int max_sev_asid; void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code); int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); +void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu); void sev_free_decrypted_vmsa(struct kvm_vcpu *vcpu, struct vmcb_save_area *vmsa); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 8abd733d5173..5a5fd6211d23 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8170,6 +8170,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) goto out; } } + if (kvm_check_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu)) + kvm_x86_call(reload_vmsa)(vcpu); } if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win || @@ -10599,6 +10601,10 @@ void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) { kvm_x86_call(gmem_invalidate)(start, end); } +void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) +{ + kvm_x86_call(gmem_invalidate_range)(kvm, range); +} #endif #endif diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index ab8cfaec82d3..c00fc1740ce5 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2608,6 +2608,7 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); +void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); #endif #ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 86690683b2fe..659b8dbe0b30 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -185,6 +185,10 @@ static void __kvm_gmem_invalidate_start(struct gmem_file *f, pgoff_t start, } flush |= kvm_mmu_unmap_gfn_range(kvm, &gfn_range); + +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE + kvm_arch_gmem_invalidate_range(kvm, &gfn_range); +#endif } if (flush) From 98fb11aed12ba8c2edb42428a0188f726a61182f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:41 -0700 Subject: [PATCH 11/21] KVM: SEV: Mark vCPU has having guest-provided VMSA even if its invalid Track the guest as having a guest-provided VMSA as soon as control.vmsa_pa is invalidated, instead of waiting to see if the guest-provided VMSA is usable, so that KVM doesn't switch back to the original VMSA instead of exiting to userspace (due to an invalid VMSA). By the time a vCPU tries to load a guest-provided VMSA, KVM has already communicated "success" for AP creation, i.e. KVM has committed to using the guest-provided VMSA. Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-12-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 62c6126d52c0..a7584b7ed6dc 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -4005,6 +4005,17 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) */ vmcb_mark_all_dirty(svm->vmcb); + /* + * From this point forward, the VMSA will always be a guest-mapped page + * rather than the initial one allocated by KVM in svm->sev_es.vmsa. In + * theory, svm->sev_es.vmsa could be free'd and cleaned up here, but + * that involves cleanups like flushing caches, which would ideally be + * handled during teardown rather than guest boot. Deferring that also + * allows the existing logic for SEV-ES VMSAs to be re-used with + * minimal SNP-specific changes. + */ + svm->sev_es.snp_has_guest_vmsa = true; + if (!VALID_PAGE(gpa)) return; @@ -4022,17 +4033,6 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, &page, NULL)) return; - /* - * From this point forward, the VMSA will always be a guest-mapped page - * rather than the initial one allocated by KVM in svm->sev_es.vmsa. In - * theory, svm->sev_es.vmsa could be free'd and cleaned up here, but - * that involves cleanups like flushing caches, which would ideally be - * handled during teardown rather than guest boot. Deferring that also - * allows the existing logic for SEV-ES VMSAs to be re-used with - * minimal SNP-specific changes. - */ - svm->sev_es.snp_has_guest_vmsa = true; - read_lock(&kvm->mmu_lock); /* * Save the guest-provided GPA. If retry is needed, then KVM will try From bfbf5fa51145cf874fa6a51d123e0d917db6eedc Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:42 -0700 Subject: [PATCH 12/21] KVM: x86: Guard .gmem_prepare() declarations with HAVE_KVM_GMEM_PREPARE=y Wrap the .gmem_prepare() declarations with HAVE_KVM_GMEM_PREPARE so that non-SEV code doesn't try to wire up a callback without doing the necessary enabling. No functional change intended. Fixes: 3bb2531e20bf ("KVM: guest_memfd: Add hook for initializing memory") Reviewed-by: Ackerley Tng Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-13-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 4 +++- arch/x86/include/asm/kvm_host.h | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index ccf23b3f0e1c..736129db272a 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -146,12 +146,14 @@ KVM_X86_OP(vcpu_deliver_sipi_vector) KVM_X86_OP_OPTIONAL_RET0(vcpu_get_apicv_inhibit_reasons); KVM_X86_OP_OPTIONAL(get_untagged_addr) KVM_X86_OP_OPTIONAL(alloc_apic_backing_page) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE KVM_X86_OP_OPTIONAL_RET0(gmem_prepare) -KVM_X86_OP_OPTIONAL_RET0(gmem_max_mapping_level) +#endif KVM_X86_OP_OPTIONAL(gmem_invalidate) #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE KVM_X86_OP_OPTIONAL(gmem_invalidate_range) #endif +KVM_X86_OP_OPTIONAL_RET0(gmem_max_mapping_level) #endif #undef KVM_X86_OP diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 1c598b40e0c3..1f8ef0e1566a 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1903,7 +1903,9 @@ struct kvm_x86_ops { gva_t (*get_untagged_addr)(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); +#endif void (*gmem_invalidate)(kvm_pfn_t start, kvm_pfn_t end); #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range); From 0f0893ac5af3ac9d18440955f302bd20a35ccab2 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:03 -0700 Subject: [PATCH 13/21] KVM: guest_memfd: Pass the number of pages instead of the end pfn into .invalidate() Pass the number of pages to "invalidate", i.e. reclaim, instead of the end pfn, as a first step towards aligning the function prototypes between the de facto "to private" and "to shared" arch hooks. Eventually, the goal is to end up with kvm_gmem_arch_make_{private,shared}(), and in both cases, providing the number of pages makes the call sites slightly nicer, and also avoids any confusion over whether the end pfn is inclusive or exclusive. Opportunistically rename "start" to "pfn", again to align with the expected signature of make_private() (which needs to pass a starting gfn as well, at which point the "start" becomes noise). No functional change intended. Cc: Fuad Tabba Cc: Ackerley Tng Reviewed-by: Xiaoyao Li Reviewed-by: Ackerley Tng Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260723210811.72720-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/svm/sev.c | 8 ++++---- arch/x86/kvm/svm/svm.h | 2 +- arch/x86/kvm/x86.c | 4 ++-- include/linux/kvm_host.h | 2 +- virt/kvm/guest_memfd.c | 6 +----- 6 files changed, 10 insertions(+), 14 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 1f8ef0e1566a..ef20f0e12193 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1906,7 +1906,7 @@ struct kvm_x86_ops { #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); #endif - void (*gmem_invalidate)(kvm_pfn_t start, kvm_pfn_t end); + void (*gmem_invalidate)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range); #endif diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index a7584b7ed6dc..5a5e8342c4fe 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -5159,16 +5159,16 @@ int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) return 0; } -void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) +void sev_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages) { - kvm_pfn_t pfn; + kvm_pfn_t end = pfn + nr_pages; if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) return; - pr_debug("%s: PFN start 0x%llx PFN end 0x%llx\n", __func__, start, end); + pr_debug("%s: PFN start 0x%llx PFN end 0x%llx\n", __func__, pfn, end); - for (pfn = start; pfn < end;) { + while (pfn < end) { bool use_2m_update = false; int rc, rmp_level; bool assigned; diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 130205defffa..2180d03bb0a6 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -1010,7 +1010,7 @@ int sev_dev_get_attr(u32 group, u64 attr, u64 *val); extern unsigned int max_sev_asid; void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code); int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); -void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); +void sev_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 5a5fd6211d23..3519fca53cfc 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10597,9 +10597,9 @@ int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_ord #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE -void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) +void kvm_arch_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages) { - kvm_x86_call(gmem_invalidate)(start, end); + kvm_x86_call(gmem_invalidate)(pfn, nr_pages); } void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) { diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index c00fc1740ce5..79868ebfc113 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2607,7 +2607,7 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE -void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); +void kvm_arch_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); #endif diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 659b8dbe0b30..c96a2ef5f2ce 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -530,11 +530,7 @@ static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *fol #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE static void kvm_gmem_free_folio(struct folio *folio) { - struct page *page = folio_page(folio, 0); - kvm_pfn_t pfn = page_to_pfn(page); - int order = folio_order(folio); - - kvm_arch_gmem_invalidate(pfn, pfn + (1ul << order)); + kvm_arch_gmem_invalidate(folio_file_pfn(folio, 0), folio_nr_pages(folio)); } #endif From 7b8529e70a11ed110a15568f6b391a5f32eef67f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:04 -0700 Subject: [PATCH 14/21] KVM: guest_memfd: Rename invalidate() arch hook to reclaim() and isolate it Rename guest_memfd's invalidate() hook to reclaim() and isolate it via its own RECLAIM Kconfig, as the hook is called when a folio is freed, which is far too late and lacks sufficient information for KVM to actually invalidate its usage of the memory. E.g. SNP uses the hook to convert memory back to SHARED so that it can be safely accessed by the host, there is no invalidation of guest mappings anywhere. Isolating the hook will also allow pKVM on arm64 to opt-in to reclaim() without also having to differentiate between reclaim and conversions to shared for active VMs. Keep guest_memfd's trampoline, even though it would be trivial to wire up .free_folio() directly to an arch callback, to avoid bleeding guest_memfd internals into arch code (specifically, avoid referencing folios in arch code). Leave the kvm_x86_ops hook as-is for the moment, as "reclaim" on SNP is the same as convert-to-shared, i.e. using a different name for the x86 hook will allow reusing it for in-place conversion. Reviewed-by: Xiaoyao Li Reviewed-by: Fuad Tabba Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260723210811.72720-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/Kconfig | 1 + arch/x86/kvm/x86.c | 7 +++++-- include/linux/kvm_host.h | 5 ++++- virt/kvm/Kconfig | 4 ++++ virt/kvm/guest_memfd.c | 6 +++--- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 801bf9e520db..e0e7ad015839 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -161,6 +161,7 @@ config KVM_AMD_SEV select ARCH_HAS_CC_PLATFORM select KVM_GENERIC_MEMORY_ATTRIBUTES select HAVE_KVM_ARCH_GMEM_PREPARE + select HAVE_KVM_ARCH_GMEM_RECLAIM select HAVE_KVM_ARCH_GMEM_INVALIDATE select HAVE_KVM_ARCH_GMEM_POPULATE help diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 3519fca53cfc..befba27672c7 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10596,11 +10596,14 @@ int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_ord } #endif -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE -void kvm_arch_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM +void kvm_arch_gmem_reclaim(kvm_pfn_t pfn, kvm_pfn_t nr_pages) { kvm_x86_call(gmem_invalidate)(pfn, nr_pages); } +#endif + +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) { kvm_x86_call(gmem_invalidate_range)(kvm, range); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 79868ebfc113..46c8d18fd043 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2606,8 +2606,11 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, kvm_gmem_populate_cb post_populate, void *opaque); #endif +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM +void kvm_arch_gmem_reclaim(kvm_pfn_t pfn, kvm_pfn_t nr_pages); +#endif + #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE -void kvm_arch_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); #endif diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index 794976b88c6f..617876993225 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -111,6 +111,10 @@ config HAVE_KVM_ARCH_GMEM_PREPARE bool depends on KVM_GUEST_MEMFD +config HAVE_KVM_ARCH_GMEM_RECLAIM + bool + depends on KVM_GUEST_MEMFD + config HAVE_KVM_ARCH_GMEM_INVALIDATE bool depends on KVM_GUEST_MEMFD diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index c96a2ef5f2ce..c086cfcd0257 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -527,10 +527,10 @@ static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *fol return MF_DELAYED; } -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM static void kvm_gmem_free_folio(struct folio *folio) { - kvm_arch_gmem_invalidate(folio_file_pfn(folio, 0), folio_nr_pages(folio)); + kvm_arch_gmem_reclaim(folio_file_pfn(folio, 0), folio_nr_pages(folio)); } #endif @@ -538,7 +538,7 @@ static const struct address_space_operations kvm_gmem_aops = { .dirty_folio = noop_dirty_folio, .migrate_folio = kvm_gmem_migrate_folio, .error_remove_folio = kvm_gmem_error_folio, -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM .free_folio = kvm_gmem_free_folio, #endif }; From f3c073e25476422ade8ed95e18734c5f3a98ba2b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:05 -0700 Subject: [PATCH 15/21] KVM: x86: Rename kvm_x86_ops' gmem_invalidate() to gmem_make_shared() Rename kvm_x86_ops's gmem_invalidate() hook to gmem_make_shared(), as the hook doesn't invalidate anything, and so that KVM doesn't need to add yet another vendor callback to support "convert to shared" once in-place conversion comes along. Opportunistically wrap the ops declarations with a GMEM_RECLAIM guard so that attempting to wire up a .gmem_make_shared() hook without selecting CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM will result in a build failure. No functional change intended. Reviewed-by: Xiaoyao Li Reviewed-by: Ackerley Tng Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260723210811.72720-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 4 +++- arch/x86/include/asm/kvm_host.h | 4 +++- arch/x86/kvm/svm/sev.c | 2 +- arch/x86/kvm/svm/svm.c | 2 +- arch/x86/kvm/svm/svm.h | 2 +- arch/x86/kvm/x86.c | 2 +- 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 736129db272a..210cb95d0a0b 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -149,7 +149,9 @@ KVM_X86_OP_OPTIONAL(alloc_apic_backing_page) #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE KVM_X86_OP_OPTIONAL_RET0(gmem_prepare) #endif -KVM_X86_OP_OPTIONAL(gmem_invalidate) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM +KVM_X86_OP_OPTIONAL(gmem_make_shared) +#endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE KVM_X86_OP_OPTIONAL(gmem_invalidate_range) #endif diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index ef20f0e12193..693be16abd95 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1906,7 +1906,9 @@ struct kvm_x86_ops { #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); #endif - void (*gmem_invalidate)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM + void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); +#endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range); #endif diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 5a5e8342c4fe..4c46081889a6 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -5159,7 +5159,7 @@ int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) return 0; } -void sev_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages) +void sev_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages) { kvm_pfn_t end = pfn + nr_pages; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index dd51df74c2dc..44c3af6f71d8 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5451,7 +5451,7 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .vm_move_enc_context_from = sev_vm_move_enc_context_from, .gmem_prepare = sev_gmem_prepare, - .gmem_invalidate = sev_gmem_invalidate, + .gmem_make_shared = sev_gmem_make_shared, .gmem_invalidate_range = sev_gmem_invalidate_range, .gmem_max_mapping_level = sev_gmem_max_mapping_level, #endif diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 2180d03bb0a6..51e3494e7802 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -1010,7 +1010,7 @@ int sev_dev_get_attr(u32 group, u64 attr, u64 *val); extern unsigned int max_sev_asid; void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code); int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); -void sev_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages); +void sev_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index befba27672c7..0075815acdd5 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10599,7 +10599,7 @@ int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_ord #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM void kvm_arch_gmem_reclaim(kvm_pfn_t pfn, kvm_pfn_t nr_pages) { - kvm_x86_call(gmem_invalidate)(pfn, nr_pages); + kvm_x86_call(gmem_make_shared)(pfn, nr_pages); } #endif From 0f68fabe68fe138a5c00310ba7935dc849ab111d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:06 -0700 Subject: [PATCH 16/21] KVM: guest_memfd: Drop the redundant printk on arch gmem_prepare() failure Drop guest_memfd's ratelimited printk to log "preparation" failures, as KVM SNP already logs more precise messages in all error paths, and whether or not failure to convert the pfn to private is "unexpected", i.e. warrants logging, is firmly an architecture specific detail. Reviewed-by: Ackerley Tng Reviewed-by: Xiaoyao Li Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260723210811.72720-5-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index c086cfcd0257..3e4273f60a8c 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -66,15 +66,11 @@ static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slo #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE kvm_pfn_t pfn = folio_file_pfn(folio, index); gfn_t gfn = slot->base_gfn + index - slot->gmem.pgoff; - int rc = kvm_arch_gmem_prepare(kvm, gfn, pfn, folio_order(folio)); - if (rc) { - pr_warn_ratelimited("gmem: Failed to prepare folio for index %lx GFN %llx PFN %llx error %d.\n", - index, gfn, pfn, rc); - return rc; - } -#endif + return kvm_arch_gmem_prepare(kvm, gfn, pfn, folio_order(folio)); +#else return 0; +#endif } /* From 19e612cd59bf447df63b77cda7feb7d82ed74f7a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:07 -0700 Subject: [PATCH 17/21] KVM: guest_memfd: Add helpers to query SHARED vs. PRIVATE for a given page Add helpers to check if a given page in a guest_memfd instance is PRIVATE versus SHARED, and use the "is shared" helper instead of an open-coded equivalent in the user pagefault handler. In addition to the immediate usage, providing an "is private" helper will allow cleaning up the so called prepare() code, and eventually will be heavily used once in-place conversion support comes along. No functional change intended. Reviewed-by: Xiaoyao Li Reviewed-by: Ackerley Tng Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260723210811.72720-6-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 3e4273f60a8c..3fa4969b38f9 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -60,6 +60,16 @@ static pgoff_t kvm_gmem_get_index(struct kvm_memory_slot *slot, gfn_t gfn) return gfn - slot->base_gfn + slot->gmem.pgoff; } +static bool kvm_gmem_is_private_mem(struct inode *inode, pgoff_t index) +{ + return !(GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED); +} + +static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index) +{ + return !kvm_gmem_is_private_mem(inode, index); +} + static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, pgoff_t index, struct folio *folio) { @@ -397,7 +407,7 @@ static vm_fault_t kvm_gmem_fault_user_mapping(struct vm_fault *vmf) if (((loff_t)vmf->pgoff << PAGE_SHIFT) >= i_size_read(inode)) return VM_FAULT_SIGBUS; - if (!(GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED)) + if (!kvm_gmem_is_shared_mem(inode, vmf->pgoff)) return VM_FAULT_SIGBUS; folio = kvm_gmem_get_folio(inode, vmf->pgoff); From 3df611b5a04e916f406882a22f33b570831df404 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 23 Jul 2026 14:08:08 -0700 Subject: [PATCH 18/21] KVM: guest_memfd: Only "prepare" folios for private pages When getting a guest_memfd pfn, prepare the folio, i.e. convert its pages to private, if and only if the page is actually private. The misnamed prepare() hook exists specifically to allow x86's SNP to assign pages to the owning VM in the RMP when mapping private memory into a guest. Guarding the call will allow renaming the prepare() hook to better reflect its role, without creating a semantic mess, and will become a hard requirement once in-place conversion is supported, i.e. when CoCo VMs support SHARED guest_memfd pages. For all intents, no functional change intended (the sole arch hook is a nop for SHARED memory). Suggested-by: Michael Roth Reviewed-by: Fuad Tabba [sean: rewrite changelog to fit the context] Signed-off-by: Ackerley Tng Reviewed-by: Xiaoyao Li Link: https://patch.msgid.link/20260723210811.72720-7-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 3fa4969b38f9..25da7778af01 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -814,7 +814,8 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, folio_mark_uptodate(folio); } - r = kvm_gmem_prepare_folio(kvm, slot, gfn, folio); + if (kvm_gmem_is_private_mem(file_inode(file), index)) + r = kvm_gmem_prepare_folio(kvm, slot, gfn, folio); folio_unlock(folio); From 2131c4f763d2e4cbbe5c227a406488c38e23f806 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:09 -0700 Subject: [PATCH 19/21] KVM: guest_memfd: Rename prepare() hook and Kconfig to make_private() / CONVERT Rework guest_memfd's prepare() hook into a more accurate make_private(), and rework its Kconfig from PREPARE to a more generic CONVERT. This will allow x86 to share (pun intended) a kvm_x86_ops.gmem_make_shared() hook between the "convert to shared" and "reclaim" flows, which are one and the same for SNP. No functional change intended. Reviewed-by: Ackerley Tng Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260723210811.72720-8-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 4 ++-- arch/x86/include/asm/kvm_host.h | 4 ++-- arch/x86/kvm/Kconfig | 2 +- arch/x86/kvm/svm/sev.c | 2 +- arch/x86/kvm/svm/svm.c | 2 +- arch/x86/kvm/svm/svm.h | 2 +- arch/x86/kvm/x86.c | 6 +++--- include/linux/kvm_host.h | 5 +++-- virt/kvm/Kconfig | 2 +- virt/kvm/guest_memfd.c | 4 ++-- 10 files changed, 17 insertions(+), 16 deletions(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 210cb95d0a0b..a4d872ddef9d 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -146,8 +146,8 @@ KVM_X86_OP(vcpu_deliver_sipi_vector) KVM_X86_OP_OPTIONAL_RET0(vcpu_get_apicv_inhibit_reasons); KVM_X86_OP_OPTIONAL(get_untagged_addr) KVM_X86_OP_OPTIONAL(alloc_apic_backing_page) -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE -KVM_X86_OP_OPTIONAL_RET0(gmem_prepare) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +KVM_X86_OP_OPTIONAL_RET0(gmem_make_private) #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM KVM_X86_OP_OPTIONAL(gmem_make_shared) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 693be16abd95..ae9a229c6b11 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1903,8 +1903,8 @@ struct kvm_x86_ops { gva_t (*get_untagged_addr)(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu); -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE - int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT + int (*gmem_make_private)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index e0e7ad015839..538ed1e80332 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -160,7 +160,7 @@ config KVM_AMD_SEV depends on CRYPTO_DEV_SP_PSP && !(KVM_AMD=y && CRYPTO_DEV_CCP_DD=m) select ARCH_HAS_CC_PLATFORM select KVM_GENERIC_MEMORY_ATTRIBUTES - select HAVE_KVM_ARCH_GMEM_PREPARE + select HAVE_KVM_ARCH_GMEM_CONVERT select HAVE_KVM_ARCH_GMEM_RECLAIM select HAVE_KVM_ARCH_GMEM_INVALIDATE select HAVE_KVM_ARCH_GMEM_POPULATE diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 4c46081889a6..779889f5e994 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -5112,7 +5112,7 @@ static bool is_large_rmp_possible(struct kvm *kvm, kvm_pfn_t pfn, int order) return false; } -int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) +int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) { struct kvm_sev_info *sev = to_kvm_sev_info(kvm); kvm_pfn_t pfn_aligned; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 44c3af6f71d8..d9f632ea8057 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5450,7 +5450,7 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .vm_copy_enc_context_from = sev_vm_copy_enc_context_from, .vm_move_enc_context_from = sev_vm_move_enc_context_from, - .gmem_prepare = sev_gmem_prepare, + .gmem_make_private = sev_gmem_make_private, .gmem_make_shared = sev_gmem_make_shared, .gmem_invalidate_range = sev_gmem_invalidate_range, .gmem_max_mapping_level = sev_gmem_max_mapping_level, diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 51e3494e7802..b5cd8437988f 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -1009,7 +1009,7 @@ int sev_cpu_init(struct svm_cpu_data *sd); int sev_dev_get_attr(u32 group, u64 attr, u64 *val); extern unsigned int max_sev_asid; void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code); -int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); +int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); void sev_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 0075815acdd5..512bcc35507a 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10589,10 +10589,10 @@ bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm) return !kvm_arch_has_private_mem(kvm); } -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE -int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order) { - return kvm_x86_call(gmem_prepare)(kvm, pfn, gfn, max_order); + return kvm_x86_call(gmem_make_private)(kvm, pfn, gfn, max_order); } #endif diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 46c8d18fd043..ecdda1f00c6c 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2572,8 +2572,9 @@ static inline int kvm_gmem_get_pfn(struct kvm *kvm, } #endif /* CONFIG_KVM_GUEST_MEMFD */ -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE -int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, + int max_order); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index 617876993225..c3c0ee253fc7 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -107,7 +107,7 @@ config KVM_GUEST_MEMFD select XARRAY_MULTI bool -config HAVE_KVM_ARCH_GMEM_PREPARE +config HAVE_KVM_ARCH_GMEM_CONVERT bool depends on KVM_GUEST_MEMFD diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 25da7778af01..6635ed05f411 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -73,11 +73,11 @@ static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index) static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, pgoff_t index, struct folio *folio) { -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT kvm_pfn_t pfn = folio_file_pfn(folio, index); gfn_t gfn = slot->base_gfn + index - slot->gmem.pgoff; - return kvm_arch_gmem_prepare(kvm, gfn, pfn, folio_order(folio)); + return kvm_arch_gmem_make_private(kvm, gfn, pfn, folio_order(folio)); #else return 0; #endif From fb50ca77b672fd359453728fc59730105f2bf7eb Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:10 -0700 Subject: [PATCH 20/21] KVM: guest_memfd: Explicitly pass number of pages to make_private() hook Tweak the guest_memfd make_private() hook to explicitly pass the number of pages to align with the signature of the make_shared() hook, and because the existing code is outright broken if a guest_memfd folio is comprised of more than one page (which can't happen, yet). The SNP code *tries* to create a corresponding huge entry, but if the RMP must use 4KiB entries for whatever reason, KVM will only convert the first pfn, and not the entire range of pfns that will be mapped into the guest. Alternatively, @max_order could simply be repurposed as _the_ @order, but that will fall apart when in-place conversion comes along, at which point KVM will need to deal with conversions that aren't bound 1:1 to a folio. I.e. the number of pages to convert may not be exactly a power-of-2 (and folios don't strictly guarantee power-of-2 pages anyways). WARN in the SNP code if the number of pages to prepare is anything other than '1', i.e. if guest_memfd is trying to prepare/convert more than a single 4KiB page, as sev_gmem_prepare() doesn't actually handle conversion greater than order-0 folios. Opportunistically swap the ordering of @pfn and @gfn params for kvm_x86_ops.gmem_make_private() to match kvm_arch_gmem_make_private(). Fixes: b85524314a3d ("KVM: guest_memfd: delay kvm_gmem_prepare_folio() until the memory is passed to the guest") Reviewed-by: Xiaoyao Li Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260723210811.72720-9-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 3 ++- arch/x86/kvm/svm/sev.c | 27 +++++++++++---------------- arch/x86/kvm/svm/svm.h | 2 +- arch/x86/kvm/x86.c | 5 +++-- include/linux/kvm_host.h | 2 +- virt/kvm/guest_memfd.c | 2 +- 6 files changed, 19 insertions(+), 22 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index ae9a229c6b11..7643e078ba36 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1904,7 +1904,8 @@ struct kvm_x86_ops { gva_t (*get_untagged_addr)(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu); #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT - int (*gmem_make_private)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); + int (*gmem_make_private)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, + kvm_pfn_t nr_pages); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 779889f5e994..5dd45b7a09f2 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -5088,15 +5088,7 @@ static bool is_pfn_range_shared(kvm_pfn_t start, kvm_pfn_t end) return true; } -static u8 max_level_for_order(int order) -{ - if (order >= KVM_HPAGE_GFN_SHIFT(PG_LEVEL_2M)) - return PG_LEVEL_2M; - - return PG_LEVEL_4K; -} - -static bool is_large_rmp_possible(struct kvm *kvm, kvm_pfn_t pfn, int order) +static bool is_large_rmp_possible(kvm_pfn_t pfn, kvm_pfn_t nr_pages) { kvm_pfn_t pfn_aligned = ALIGN_DOWN(pfn, PTRS_PER_PMD); @@ -5105,14 +5097,14 @@ static bool is_large_rmp_possible(struct kvm *kvm, kvm_pfn_t pfn, int order) * PFN is currently shared, then the entire 2M-aligned range can be * set to private via a single 2M RMP entry. */ - if (max_level_for_order(order) > PG_LEVEL_4K && + if (nr_pages >= KVM_PAGES_PER_HPAGE(PG_LEVEL_2M) && is_pfn_range_shared(pfn_aligned, pfn_aligned + PTRS_PER_PMD)) return true; return false; } -int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) +int sev_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages) { struct kvm_sev_info *sev = to_kvm_sev_info(kvm); kvm_pfn_t pfn_aligned; @@ -5123,6 +5115,9 @@ int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_ord if (!sev_snp_guest(kvm)) return 0; + if (WARN_ON_ONCE(nr_pages != 1)) + return -EIO; + rc = snp_lookup_rmpentry(pfn, &assigned, &level); if (rc) { pr_err_ratelimited("SEV: Failed to look up RMP entry: GFN %llx PFN %llx error %d\n", @@ -5131,12 +5126,12 @@ int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_ord } if (assigned) { - pr_debug("%s: already assigned: gfn %llx pfn %llx max_order %d level %d\n", - __func__, gfn, pfn, max_order, level); + pr_debug("%s: already assigned: gfn %llx pfn %llx nr_pages %llx level %d\n", + __func__, gfn, pfn, nr_pages, level); return 0; } - if (is_large_rmp_possible(kvm, pfn, max_order)) { + if (is_large_rmp_possible(pfn, nr_pages)) { level = PG_LEVEL_2M; pfn_aligned = ALIGN_DOWN(pfn, PTRS_PER_PMD); gfn_aligned = ALIGN_DOWN(gfn, PTRS_PER_PMD); @@ -5153,8 +5148,8 @@ int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_ord return -EINVAL; } - pr_debug("%s: updated: gfn %llx pfn %llx pfn_aligned %llx max_order %d level %d\n", - __func__, gfn, pfn, pfn_aligned, max_order, level); + pr_debug("%s: updated: gfn %llx pfn %llx pfn_aligned %llx nr_pages %llx level %d\n", + __func__, gfn, pfn, pfn_aligned, nr_pages, level); return 0; } diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index b5cd8437988f..da4c66eb8d70 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -1009,7 +1009,7 @@ int sev_cpu_init(struct svm_cpu_data *sd); int sev_dev_get_attr(u32 group, u64 attr, u64 *val); extern unsigned int max_sev_asid; void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code); -int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); +int sev_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages); void sev_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 512bcc35507a..9390e0d4c1e5 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10590,9 +10590,10 @@ bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm) } #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT -int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order) +int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, + kvm_pfn_t nr_pages) { - return kvm_x86_call(gmem_make_private)(kvm, pfn, gfn, max_order); + return kvm_x86_call(gmem_make_private)(kvm, gfn, pfn, nr_pages); } #endif diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index ecdda1f00c6c..b24a090eb34d 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2574,7 +2574,7 @@ static inline int kvm_gmem_get_pfn(struct kvm *kvm, #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, - int max_order); + kvm_pfn_t nr_pages); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 6635ed05f411..0c1ee3e49176 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -77,7 +77,7 @@ static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slo kvm_pfn_t pfn = folio_file_pfn(folio, index); gfn_t gfn = slot->base_gfn + index - slot->gmem.pgoff; - return kvm_arch_gmem_make_private(kvm, gfn, pfn, folio_order(folio)); + return kvm_arch_gmem_make_private(kvm, gfn, pfn, folio_nr_pages(folio)); #else return 0; #endif From 2abcdf03fda1380bcbe00417b9ce2b4afb30899b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:11 -0700 Subject: [PATCH 21/21] KVM: guest_memfd: Make private exactly what can be mapped on page fault When calling into arch code to make the underlying memory private, i.e. to assign memory to the VM in SNP's RMP table, assign/convert *exactly* the range of memory that can be mapped into the guest for the current page fault, instead of aggressively converting/assigning the entire folio. For SNP, the mapping size in the stage-2 page tables (Nested Page Tables, NPT) must be at least the size of the corresponding RMP entry, e.g. assigning a 2MiB mapping in the RMP when it can only be mapped at 4KiB granualarity will ultimate result in another page fault (#NPF for SNP) to "smash" the RMP down to the correct mapping size. Assigning the entire folio was necessary back when guest_memfd tracked preparedness, which was done on a per-folio basis. At the time, it made sense to do per-folio tracking/preparation, because tracking per-folio meant guest_memfd didn't need to add a separate data structure to track that information, and doing per-folio tracking only works if the entire folio is prepared (or not). Now that guest_memfd no longer does preparation tracking (see commit 8622ef05709f ("KVM: guest_memfd: Remove preparation tracking")), in favor having SNP query the RMP, per-folio preparation, i.e. per-folio conversions to private, doesn't make any sense. *If* SNP allowed the RMP size to be greater than the NPT size, then per-folio conversion could theoretically provide marginal value, as it would allow KVM to assign a hugepage in the RMP even if it can only be mapped into the NPT with a smaller page, e.g. because of memslot alignment. The documentation of that reasoning would be something like this: /* * If the memory is private from KVM's perspective, and hardware tracks * VM-assigned private memory in a dedicated data structure, i.e. not * in the stage-2 page tables, then call into arch code to assign the * entire folio to the guest. Assigning the entire folio, e.g. instead * of only the memory being mapped into the guest, allows KVM to assign * an entire hugepage of memory in the out-of-band structure even if * KVM can only map a smaller page size into the MMU, e.g. because the * gmem hugepage is spread across multiple memslots. */ But even *if* a future SNP implementation supported that behavior, the value added would be dubious, as having a huge folio that is fully private, but can only be mapped at a smaller granularity, would be rare. E.g. maybe for memory at the top of lower DRAM that has holes for non-RAM assets? So, convert/assign exactly what guest_memfd allows the caller to map to simplify the guest_memfd code and provide a (super) minor performance optimization for SNP. E.g. once hugepage support comes along, guest_memfd will only need a single flow to compute "how much memory can be assigned and at what size". Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260723210811.72720-10-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 53 ++++++------------------------------------ 1 file changed, 7 insertions(+), 46 deletions(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 0c1ee3e49176..fdd26adaa7d4 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -70,50 +70,6 @@ static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index) return !kvm_gmem_is_private_mem(inode, index); } -static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, - pgoff_t index, struct folio *folio) -{ -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT - kvm_pfn_t pfn = folio_file_pfn(folio, index); - gfn_t gfn = slot->base_gfn + index - slot->gmem.pgoff; - - return kvm_arch_gmem_make_private(kvm, gfn, pfn, folio_nr_pages(folio)); -#else - return 0; -#endif -} - -/* - * Process @folio, which contains @gfn, so that the guest can use it. - * The folio must be locked and the gfn must be contained in @slot. - * On successful return the guest sees a zero page so as to avoid - * leaking host data and the up-to-date flag is set. - */ -static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, - gfn_t gfn, struct folio *folio) -{ - pgoff_t index; - - /* - * Preparing huge folios should always be safe, since it should - * be possible to split them later if needed. - * - * Right now the folio order is always going to be zero, but the - * code is ready for huge folios. The only assumption is that - * the base pgoff of memslots is naturally aligned with the - * requested page order, ensuring that huge folios can also use - * huge page table entries for GPA->HPA mapping. - * - * The order will be passed when creating the guest_memfd, and - * checked when creating memslots. - */ - WARN_ON(!IS_ALIGNED(slot->gmem.pgoff, folio_nr_pages(folio))); - index = kvm_gmem_get_index(slot, gfn); - index = ALIGN_DOWN(index, folio_nr_pages(folio)); - - return __kvm_gmem_prepare_folio(kvm, slot, index, folio); -} - /* * Returns a locked folio on success. The caller is responsible for * setting the up-to-date flag before the memory is mapped into the guest. @@ -799,7 +755,9 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, { pgoff_t index = kvm_gmem_get_index(slot, gfn); struct folio *folio; - int r = 0; + int r = 0, __order; + + max_order = max_order ?: &__order; CLASS(gmem_get_file, file)(slot); if (!file) @@ -814,8 +772,11 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, folio_mark_uptodate(folio); } +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT if (kvm_gmem_is_private_mem(file_inode(file), index)) - r = kvm_gmem_prepare_folio(kvm, slot, gfn, folio); + r = kvm_arch_gmem_make_private(kvm, gfn, *pfn, + (kvm_pfn_t)1 << *max_order); +#endif folio_unlock(folio);