From d215f014b3526a9898d87bfb4b866287f0864cc9 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Fri, 28 Aug 2026 13:54:32 +0200 Subject: [PATCH 01/54] KVM: s390: Fix dirty marking in adapter_indicators_set*() When the indicator and/or summary bits are set in the guest, the accessed page was only marked dirty in KVM if the access was performed using the slow path; accesses through the new kvm_arch_set_irq_inatomic fast inject path would not mark the page as dirty. Fix by adding/moving the missing calls to mark_page_dirty(). Note that for the inatomic path set_page_dirty{,_lock}() is not needed as the page stays pinned; the unpin path correctly marks it as dirty. Opportunistically reorder the local variables to be in reverse Christmas tree order and refactor to use guard(). Fixes: 1e95e3bc6b05 ("KVM: s390: Enable adapter_indicators_set to use mapped pages") Signed-off-by: Claudio Imbrenda Message-ID: <20260828115439.145885-2-imbrenda@linux.ibm.com> --- arch/s390/kvm/s390/interrupt.c | 77 +++++++++++++++++----------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/arch/s390/kvm/s390/interrupt.c b/arch/s390/kvm/s390/interrupt.c index 0381ae981703..49b4e233e791 100644 --- a/arch/s390/kvm/s390/interrupt.c +++ b/arch/s390/kvm/s390/interrupt.c @@ -2984,61 +2984,58 @@ static int adapter_indicators_set(struct kvm *kvm, struct s390_io_adapter *adapter, struct kvm_s390_adapter_int *adapter_int) { - unsigned long bit; - int summary_set, idx; struct s390_map_info *ind_info, *summary_info; - void *map; struct page *ind_page, *summary_page; - unsigned long flags; + unsigned long bit; + int summary_set; + void *map; ind_page = NULL; - spin_lock_irqsave(&adapter->maps_lock, flags); - ind_info = get_map_info(adapter, adapter_int->ind_addr); + scoped_guard(spinlock_irqsave, &adapter->maps_lock) { + ind_info = get_map_info(adapter, adapter_int->ind_addr); + if (ind_info) { + map = page_address(ind_info->page); + bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); + set_bit(bit, map); + } + } if (!ind_info) { - spin_unlock_irqrestore(&adapter->maps_lock, flags); ind_page = pin_map_page(kvm, adapter_int->ind_addr, 0); if (!ind_page) return -1; - idx = srcu_read_lock(&kvm->srcu); map = page_address(ind_page); bit = get_ind_bit(adapter_int->ind_addr, adapter_int->ind_offset, adapter->swap); set_bit(bit, map); - mark_page_dirty(kvm, adapter_int->ind_gaddr >> PAGE_SHIFT); set_page_dirty_lock(ind_page); - srcu_read_unlock(&kvm->srcu, idx); unpin_user_page(ind_page); - } else { - map = page_address(ind_info->page); - bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); - set_bit(bit, map); - spin_unlock_irqrestore(&adapter->maps_lock, flags); } + scoped_guard(srcu, &kvm->srcu) + mark_page_dirty(kvm, gpa_to_gfn(adapter_int->ind_gaddr)); - spin_lock_irqsave(&adapter->maps_lock, flags); - summary_info = get_map_info(adapter, adapter_int->summary_addr); + scoped_guard(spinlock_irqsave, &adapter->maps_lock) { + summary_info = get_map_info(adapter, adapter_int->summary_addr); + if (summary_info) { + map = page_address(summary_info->page); + bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, + adapter->swap); + summary_set = test_and_set_bit(bit, map); + } + } if (!summary_info) { - spin_unlock_irqrestore(&adapter->maps_lock, flags); summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0); if (!summary_page) return -1; - idx = srcu_read_lock(&kvm->srcu); map = page_address(summary_page); bit = get_ind_bit(adapter_int->summary_addr, adapter_int->summary_offset, adapter->swap); summary_set = test_and_set_bit(bit, map); - mark_page_dirty(kvm, adapter_int->summary_gaddr >> PAGE_SHIFT); set_page_dirty_lock(summary_page); - srcu_read_unlock(&kvm->srcu, idx); unpin_user_page(summary_page); - } else { - map = page_address(summary_info->page); - bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, - adapter->swap); - summary_set = test_and_set_bit(bit, map); - spin_unlock_irqrestore(&adapter->maps_lock, flags); } + scoped_guard(srcu, &kvm->srcu) + mark_page_dirty(kvm, gpa_to_gfn(adapter_int->summary_gaddr)); return summary_set ? 0 : 1; } @@ -3048,26 +3045,29 @@ static int adapter_indicators_set_fast(struct kvm *kvm, struct kvm_s390_adapter_int *adapter_int, int setbit) { + struct s390_map_info *ind_info, *summary_info; unsigned long bit; int summary_set; - struct s390_map_info *ind_info, *summary_info; void *map; - spin_lock(&adapter->maps_lock); + guard(srcu)(&kvm->srcu); + guard(spinlock)(&adapter->maps_lock); + ind_info = get_map_info(adapter, adapter_int->ind_addr); - if (!ind_info) { - spin_unlock(&adapter->maps_lock); + if (!ind_info) return -EWOULDBLOCK; - } + map = page_address(ind_info->page); bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); - if (setbit) + if (setbit) { set_bit(bit, map); - summary_info = get_map_info(adapter, adapter_int->summary_addr); - if (!summary_info) { - spin_unlock(&adapter->maps_lock); - return -EWOULDBLOCK; + mark_page_dirty(kvm, gpa_to_gfn(adapter_int->ind_gaddr)); } + + summary_info = get_map_info(adapter, adapter_int->summary_addr); + if (!summary_info) + return -EWOULDBLOCK; + map = page_address(summary_info->page); bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, adapter->swap); @@ -3077,7 +3077,8 @@ static int adapter_indicators_set_fast(struct kvm *kvm, summary_set = test_and_set_bit(bit, map); else summary_set = test_and_clear_bit(bit, map); - spin_unlock(&adapter->maps_lock); + mark_page_dirty(kvm, gpa_to_gfn(adapter_int->summary_gaddr)); + return summary_set ? 0 : 1; } From ae12d2f9c119639a142d92c4a37c30277e8576da Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Fri, 28 Aug 2026 13:54:33 +0200 Subject: [PATCH 02/54] KVM: s390: Fix compile warning for kvm_s390_update_cmma_dirty() The parameter "old" should be marked as const, to prevent compile-time warnings. Fixes: d487a24041c2 ("KVM: s390: Prepare gmap for a second KVM implementation") Signed-off-by: Claudio Imbrenda Message-ID: <20260828115439.145885-3-imbrenda@linux.ibm.com> --- arch/s390/kvm/s390/s390.c | 2 +- arch/s390/kvm/s390/s390.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/kvm/s390/s390.c b/arch/s390/kvm/s390/s390.c index b0839e887221..8f7e09d7d049 100644 --- a/arch/s390/kvm/s390/s390.c +++ b/arch/s390/kvm/s390/s390.c @@ -5766,7 +5766,7 @@ static long cmma_d_count_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_ return 0; } -void kvm_s390_update_cmma_dirty(struct kvm *kvm, struct kvm_memory_slot *old) +void kvm_s390_update_cmma_dirty(struct kvm *kvm, const struct kvm_memory_slot *old) { const struct dat_walk_ops ops = { .pte_entry = cmma_d_count_pte, }; diff --git a/arch/s390/kvm/s390/s390.h b/arch/s390/kvm/s390/s390.h index d284a263ba70..aa0d1d062f8d 100644 --- a/arch/s390/kvm/s390/s390.h +++ b/arch/s390/kvm/s390/s390.h @@ -472,7 +472,7 @@ int __kvm_s390_mprotect_many(struct gmap *gmap, gpa_t gpa, u8 npages, unsigned i unsigned long bits); bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu); -void kvm_s390_update_cmma_dirty(struct kvm *kvm, struct kvm_memory_slot *old); +void kvm_s390_update_cmma_dirty(struct kvm *kvm, const struct kvm_memory_slot *old); int kvm_s390_vm_stop_migration(struct kvm *kvm); From faff4c8ff3dbec6d71b89dd281a0d17c4478fa44 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Fri, 28 Aug 2026 13:54:34 +0200 Subject: [PATCH 03/54] KVM: s390: Fix _gaccess_shadow_fault() In some circumstances, it is possible that the page of nested guest memory that is being shadowed is not present at all in the parent guest gmap. dat_entry_walk() will not find any leaf entry and return with -ENOENT, which will erroneously be propagated all the way to userspace. Fix by manually calling gmap_link() on the memory of the nested guest that is being shadowed if the mapping was not already present. Fixes: e38c884df921 ("KVM: s390: Switch to new gmap") Signed-off-by: Claudio Imbrenda Message-ID: <20260828115439.145885-4-imbrenda@linux.ibm.com> --- arch/s390/kvm/s390/gaccess.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/s390/kvm/s390/gaccess.c b/arch/s390/kvm/s390/gaccess.c index e5c064f263df..405345ccc4f1 100644 --- a/arch/s390/kvm/s390/gaccess.c +++ b/arch/s390/kvm/s390/gaccess.c @@ -1589,12 +1589,25 @@ static inline int ___gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg parent = READ_ONCE(sg->parent); if (!parent) return -EAGAIN; +retry: scoped_guard(spinlock, &parent->children_lock) { if (READ_ONCE(sg->parent) != parent) return -EAGAIN; sg->invalidated = false; rc = _gaccess_do_shadow(vcpu->arch.mc, sg, saddr, walk); } + if (rc == -ENOENT) { + struct kvm_memory_slot *slot; + struct guest_fault *entries; + + entries = get_entries(walk); + slot = kvm_vcpu_gfn_to_memslot(vcpu, entries[LEVEL_MEM].gfn); + if (!slot) + return PGM_ADDRESSING; + rc = gmap_link(vcpu->arch.mc, parent, entries + LEVEL_MEM, slot); + if (!rc) + goto retry; + } if (!rc) kvm_s390_release_faultin_array(vcpu->kvm, walk->raw_entries, false); return rc; From 00c0ae5e438615bde828a3b9f33912960f4e6511 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Fri, 28 Aug 2026 13:54:35 +0200 Subject: [PATCH 04/54] KVM: s390: Refactor dat_set_slot() Refactor dat_set_slot(), _dat_slot_pte(), _dat_slot_crste(). Now they only take a struct kvm_s390_mmu_cache as priv. For dat_delete_slot(), mc is NULL, as no allocations should take place. This is needed as a prerequisite to move gmap DAT table setup from kvm_arch_commit_memory_region() to kvm_arch_prepare_memory_region(). Signed-off-by: Claudio Imbrenda Message-ID: <20260828115439.145885-5-imbrenda@linux.ibm.com> --- arch/s390/kvm/gmap/dat.c | 29 +++++++++-------------------- arch/s390/kvm/gmap/dat.h | 10 ++++------ arch/s390/kvm/gmap/kvm_mmu.c | 4 ++-- 3 files changed, 15 insertions(+), 28 deletions(-) diff --git a/arch/s390/kvm/gmap/dat.c b/arch/s390/kvm/gmap/dat.c index 24547e39fab2..dcedd5479d82 100644 --- a/arch/s390/kvm/gmap/dat.c +++ b/arch/s390/kvm/gmap/dat.c @@ -846,19 +846,12 @@ long dat_reset_skeys(union asce asce, gfn_t start) } #endif /* KVM_S390_MANAGES_S390_GUEST */ -struct slot_priv { - unsigned long token; - struct kvm_s390_mmu_cache *mc; -}; - static long _dat_slot_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) { - struct slot_priv *p = walk->priv; - union crste dummy = { .val = p->token }; union pte new_pte, pte = READ_ONCE(*ptep); union pgste pgste; - new_pte = _PTE_TOK(dummy.tok.type, dummy.tok.par); + new_pte = walk->priv ? _PTE_EMPTY : _PTE_TOK(_DAT_TOKEN_PIC, PGM_ADDRESSING); /* Table entry already in the desired state. */ if (pte.val == new_pte.val) @@ -875,10 +868,9 @@ static long _dat_slot_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_wal static long _dat_slot_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) { union crste new_crste, crste = READ_ONCE(*crstep); - struct slot_priv *p = walk->priv; + struct kvm_s390_mmu_cache *mc = walk->priv; - new_crste.val = p->token; - new_crste.h.tt = crste.h.tt; + new_crste = mc ? _CRSTE_EMPTY(crste.h.tt) : _CRSTE_HOLE(crste.h.tt); /* Table entry already in the desired state. */ if (crste.val == new_crste.val) @@ -902,7 +894,10 @@ static long _dat_slot_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct d if (!crste.h.fc && !crste.h.i) return 0; /* Split (install a lower level table), and handle things there. */ - return dat_split_crste(p->mc, crstep, gfn, walk->asce, false); + if (mc) + return dat_split_crste(mc, crstep, gfn, walk->asce, false); + /* A large page should never cross memslots boundaries */ + return -EINVAL; } static const struct dat_walk_ops dat_slot_ops = { @@ -910,16 +905,10 @@ static const struct dat_walk_ops dat_slot_ops = { .crste_ops = { _dat_slot_crste, _dat_slot_crste, _dat_slot_crste, _dat_slot_crste, }, }; -int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end, - u16 type, u16 param) +int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end) { - struct slot_priv priv = { - .token = _CRSTE_TOK(0, type, param).val, - .mc = mc, - }; - return _dat_walk_gfn_range(start, end, asce, &dat_slot_ops, - DAT_WALK_IGN_HOLES | DAT_WALK_ANY, &priv); + DAT_WALK_IGN_HOLES | DAT_WALK_ANY, mc); } static void pgste_set_unlock_multiple(union pte *first, int n, union pgste *pgstes) diff --git a/arch/s390/kvm/gmap/dat.h b/arch/s390/kvm/gmap/dat.h index e452c141b841..90389d47ba4e 100644 --- a/arch/s390/kvm/gmap/dat.h +++ b/arch/s390/kvm/gmap/dat.h @@ -547,8 +547,7 @@ long dat_reset_skeys(union asce asce, gfn_t start); unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param); void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val); -int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end, - u16 type, u16 param); +int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end); #if KVM_S390_MANAGES_S390_GUEST int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn); @@ -973,16 +972,15 @@ static inline int get_level(union crste *crstep, union pte *ptep) return ptep ? TABLE_TYPE_PAGE_TABLE : crstep->h.tt; } -static inline int dat_delete_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, - unsigned long npages) +static inline int dat_delete_slot(union asce asce, gfn_t start, unsigned long npages) { - return dat_set_slot(mc, asce, start, start + npages, _DAT_TOKEN_PIC, PGM_ADDRESSING); + return dat_set_slot(NULL, asce, start, start + npages); } static inline int dat_create_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, unsigned long npages) { - return dat_set_slot(mc, asce, start, start + npages, _DAT_TOKEN_NONE, 0); + return dat_set_slot(mc, asce, start, start + npages); } static inline bool crste_is_ucas(union crste crste) diff --git a/arch/s390/kvm/gmap/kvm_mmu.c b/arch/s390/kvm/gmap/kvm_mmu.c index b08b8229bb6f..4c8054e18490 100644 --- a/arch/s390/kvm/gmap/kvm_mmu.c +++ b/arch/s390/kvm/gmap/kvm_mmu.c @@ -111,10 +111,10 @@ void s390_kvm_mmu_commit_memory_region(struct kvm *kvm, kvm_s390_update_cmma_dirty(kvm, old); switch (change) { case KVM_MR_DELETE: - rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages); + rc = dat_delete_slot(kvm->arch.gmap->asce, old->base_gfn, old->npages); break; case KVM_MR_MOVE: - rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages); + rc = dat_delete_slot(kvm->arch.gmap->asce, old->base_gfn, old->npages); if (rc) break; fallthrough; From 19192a4043277af380f83d550a55b92eeeaac7c6 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Fri, 28 Aug 2026 13:54:36 +0200 Subject: [PATCH 05/54] KVM: s390: Move all code into s390_kvm_mmu_prepare_memory_region() Move all code from s390_kvm_mmu_commit_memory_region() into s390_kvm_mmu_prepare_memory_region(). This allows the function to fail gracefully if needed. The previous behaviour was to print a warning and continue execution with page tables inconsistent with the memslots. Fixes: e38c884df921 ("KVM: s390: Switch to new gmap") Signed-off-by: Claudio Imbrenda Message-ID: <20260828115439.145885-6-imbrenda@linux.ibm.com> --- arch/s390/kvm/gmap/kvm_mmu.c | 89 +++++++++++++++--------------------- arch/s390/kvm/gmap/kvm_mmu.h | 4 -- arch/s390/kvm/s390/s390.c | 1 - 3 files changed, 36 insertions(+), 58 deletions(-) diff --git a/arch/s390/kvm/gmap/kvm_mmu.c b/arch/s390/kvm/gmap/kvm_mmu.c index 4c8054e18490..c2ffb5e59ec6 100644 --- a/arch/s390/kvm/gmap/kvm_mmu.c +++ b/arch/s390/kvm/gmap/kvm_mmu.c @@ -47,6 +47,9 @@ int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *new, enum kvm_mr_change change) { + struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; + int rc = 0; + if (kvm_is_ucontrol(kvm) && new && new->id < KVM_USER_MEM_SLOTS) return -EINVAL; @@ -61,6 +64,10 @@ int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, * and munmap() stuff in this slot after doing this call at any * time. */ + if (change != KVM_MR_MOVE && change != KVM_MR_CREATE) { + WARN(1, "Unknown KVM MR CHANGE: %d\n", change); + return -EINVAL; + } if (new->userspace_addr & ~PAGE_MASK) return -EINVAL; if ((new->base_gfn + new->npages) * PAGE_SIZE > kvm->arch.mem_limit) @@ -69,65 +76,41 @@ int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, return -EINVAL; } - if (!kvm_s390_is_migration_mode(kvm)) - return 0; - - /* - * Turn off migration mode when: - * - userspace creates a new memslot with dirty logging off, - * - userspace modifies an existing memslot (MOVE or FLAGS_ONLY) and - * dirty logging is turned off. - * Migration mode expects dirty page logging being enabled to store - * its dirty bitmap. - */ - if (change != KVM_MR_DELETE && - !(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) - WARN(kvm_s390_vm_stop_migration(kvm), - "Failed to stop migration mode"); - - return 0; -} - -void s390_kvm_mmu_commit_memory_region(struct kvm *kvm, - struct kvm_memory_slot *old, - const struct kvm_memory_slot *new, - enum kvm_mr_change change) -{ - struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; - int rc = 0; - - guard(mutex)(&kvm->slots_arch_lock); + if (kvm->arch.migration_mode) { + /* + * Turn off migration mode when: + * - userspace creates a new memslot with dirty logging off, + * - userspace modifies an existing memslot (MOVE or FLAGS_ONLY) + * and dirty logging is turned off. + * Migration mode expects dirty page logging being enabled to + * store its dirty bitmap. + */ + if (change != KVM_MR_DELETE && + !(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) + WARN(kvm_s390_vm_stop_migration(kvm), + "Failed to stop migration mode"); + } if (change == KVM_MR_FLAGS_ONLY) - return; - - mc = kvm_s390_new_mmu_cache(); - if (!mc) { - rc = -ENOMEM; - goto out; + return 0; + if (change != KVM_MR_DELETE) { + /* Enough capacity to add a new memslot */ + mc = kvm_s390_new_mmu_cache(); + if (!mc) + return -ENOMEM; } - scoped_guard(write_lock, &kvm->mmu_lock) { kvm_s390_update_cmma_dirty(kvm, old); - switch (change) { - case KVM_MR_DELETE: + if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) rc = dat_delete_slot(kvm->arch.gmap->asce, old->base_gfn, old->npages); - break; - case KVM_MR_MOVE: - rc = dat_delete_slot(kvm->arch.gmap->asce, old->base_gfn, old->npages); - if (rc) - break; - fallthrough; - case KVM_MR_CREATE: + if (!rc && (change == KVM_MR_MOVE || change == KVM_MR_CREATE)) rc = dat_create_slot(mc, kvm->arch.gmap->asce, new->base_gfn, new->npages); - break; - case KVM_MR_FLAGS_ONLY: - break; - default: - WARN(1, "Unknown KVM MR CHANGE: %d\n", change); - } } -out: - if (rc) - pr_warn("failed to commit memory region\n"); + /* + * Can only be triggered if dat_{create,delete}_slot() found an + * internal inconsistency or if the mmu cache ran out of memory; + * both should be impossible. + */ + KVM_BUG_ON(rc, kvm); + return rc; } diff --git a/arch/s390/kvm/gmap/kvm_mmu.h b/arch/s390/kvm/gmap/kvm_mmu.h index cdbd390bd33c..43cde61bae03 100644 --- a/arch/s390/kvm/gmap/kvm_mmu.h +++ b/arch/s390/kvm/gmap/kvm_mmu.h @@ -10,9 +10,5 @@ int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, const struct kvm_memory_slot *old, struct kvm_memory_slot *new, enum kvm_mr_change change); -void s390_kvm_mmu_commit_memory_region(struct kvm *kvm, - struct kvm_memory_slot *old, - const struct kvm_memory_slot *new, - enum kvm_mr_change change); #endif /* ARCH_KVM_GMAP_KVM_MMU_H */ diff --git a/arch/s390/kvm/s390/s390.c b/arch/s390/kvm/s390/s390.c index 8f7e09d7d049..eca4a4359ab2 100644 --- a/arch/s390/kvm/s390/s390.c +++ b/arch/s390/kvm/s390/s390.c @@ -5781,7 +5781,6 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { - s390_kvm_mmu_commit_memory_region(kvm, old, new, change); } /** From f3a557067d57ce6ae98d485c8009b223e16f5f36 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Fri, 28 Aug 2026 13:54:37 +0200 Subject: [PATCH 06/54] KVM: s390: Add missing srcu in kvm_s390_set_irq_state() Like kvm_s390_inject_vcpu(), kvm_s390_set_irq_state() also needs the kvm->srcu or the slots lock when performing the Store status operation. Fix by taking kvm->srcu in kvm_s390_set_irq_state(). Fixes: ba5c1e9b6cee ("KVM: s390: interrupt subsystem, cpu timer, waitpsw") Fixes: 062e44a9319f ("KVM: s390: Use srcu in kvm_arch_vcpu_unlocked_ioctl()") Signed-off-by: Claudio Imbrenda Message-ID: <20260828115439.145885-7-imbrenda@linux.ibm.com> --- arch/s390/kvm/s390/interrupt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/kvm/s390/interrupt.c b/arch/s390/kvm/s390/interrupt.c index 49b4e233e791..f892f4307883 100644 --- a/arch/s390/kvm/s390/interrupt.c +++ b/arch/s390/kvm/s390/interrupt.c @@ -3229,9 +3229,9 @@ int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len break; } } - if (storestatus) { - n = kvm_s390_store_status_unloaded(vcpu, KVM_S390_STORE_STATUS_NOADDR); + scoped_guard(srcu, &vcpu->kvm->srcu) + n = kvm_s390_store_status_unloaded(vcpu, KVM_S390_STORE_STATUS_NOADDR); return r ? r : n; } From 27554b9505ddfc0aeab466aeb60929dfa17284c7 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Fri, 28 Aug 2026 13:54:38 +0200 Subject: [PATCH 07/54] KVM: s390: Fix potential races in dat skey functions When dat_cond_set_storage_key() finds a large page, it will conditionally set the storage key in absolute memory using large_crste_to_phys() to get the absolute address. There is a race window between dat_entry_walk() and large_crste_to_phys(): the large page could have been split concurrently, and large_crste_to_phys() might be called with a crste that does not designate a large page, leading to crashes. Similar issues were also present in dat_set_storage_key(). dat_get_storage_key() and dat_reset_reference_bit() did instead check for a potential concurrent splitting of the large page, but then handled it incorrectly. Fix by performing a READ_ONCE on the crste pointer, checking and using the result, instead of dereferencing the pointer again. In case a race is detacted, try dat_entry_walk() again. Fixes: 8e03e8316eb2 ("KVM: s390: KVM page table management functions: storage keys") Signed-off-by: Claudio Imbrenda Message-ID: <20260828115439.145885-8-imbrenda@linux.ibm.com> --- arch/s390/kvm/gmap/dat.c | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/arch/s390/kvm/gmap/dat.c b/arch/s390/kvm/gmap/dat.c index dcedd5479d82..ff80d02c9f56 100644 --- a/arch/s390/kvm/gmap/dat.c +++ b/arch/s390/kvm/gmap/dat.c @@ -621,17 +621,20 @@ int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey) union pte *ptep; int rc; +again: skey->skey = 0; rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); if (rc) return rc; if (!ptep) { - union crste crste; + union crste crste = READ_ONCE(*crstep); - crste = READ_ONCE(*crstep); - if (!crste.h.fc || !crste.s.fc1.pr) + if (!crste_leaf(crste) && !crste.h.i) + goto again; + if (!crste.s.fc1.pr) return 0; + skey->skey = page_get_storage_key(large_crste_to_phys(crste, gfn)); return 0; } @@ -662,13 +665,20 @@ int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gf union pte *ptep; int rc; +again: rc = dat_entry_walk(mc, gfn, asce, DAT_WALK_LEAF_ALLOC, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); if (rc) return rc; if (!ptep) { - page_set_storage_key(large_crste_to_phys(*crstep, gfn), skey.skey, !nq); + union crste crste = READ_ONCE(*crstep); + + /* A large page has been split concurrently, try again */ + if (!crste_leaf(crste)) + goto again; + + page_set_storage_key(large_crste_to_phys(crste, gfn), skey.skey, !nq); return 0; } @@ -718,15 +728,22 @@ int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gf union pte *ptep; int rc; +again: rc = dat_entry_walk(mmc, gfn, asce, DAT_WALK_LEAF_ALLOC, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); if (rc) return rc; if (!ptep) { + union crste crste = READ_ONCE(*crstep); + + /* A large page has been split concurrently, try again */ + if (!crste_leaf(crste)) + goto again; if (!oldkey) oldkey = &prev; - return page_cond_set_storage_key(large_crste_to_phys(*crstep, gfn), skey, oldkey, + + return page_cond_set_storage_key(large_crste_to_phys(crste, gfn), skey, oldkey, nq, mr, mc); } @@ -768,7 +785,7 @@ int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey) int rc; skey->skey = 0; - +again: rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); if (rc) return rc; @@ -776,9 +793,12 @@ int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey) if (!ptep) { union crste crste = READ_ONCE(*crstep); - if (!crste.h.fc || !crste.s.fc1.pr) + /* A large page has been split concurrently, try again */ + if (!crste_leaf(crste) && !crste.h.i) + goto again; + if (!crste.s.fc1.pr) return 0; - skey->skey = page_reset_referenced(large_crste_to_phys(*crstep, gfn)) << 1; + skey->skey = page_reset_referenced(large_crste_to_phys(crste, gfn)) << 1; return 0; } old = pgste_get_lock(ptep); From 4ca00a9154f998116fba9a32cce5bd938d228065 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Fri, 28 Aug 2026 13:54:39 +0200 Subject: [PATCH 08/54] KVM: s390: Fix race in _destroy_pages_crste() Use READ_ONCE() in _destroy_pages_crste() to read the crste, avoid dereferencing the pointer multiple times. Fixes: a2c17f9270cc ("KVM: s390: New gmap code") Signed-off-by: Claudio Imbrenda Message-ID: <20260828115439.145885-9-imbrenda@linux.ibm.com> --- arch/s390/kvm/gmap/gmap.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/s390/kvm/gmap/gmap.c b/arch/s390/kvm/gmap/gmap.c index 4968330e9553..3f3fa864cc36 100644 --- a/arch/s390/kvm/gmap/gmap.c +++ b/arch/s390/kvm/gmap/gmap.c @@ -994,11 +994,13 @@ static long _destroy_pages_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct da static long _destroy_pages_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) { phys_addr_t origin, cur, end; + union crste crste; - if (!crstep->h.fc || !crstep->s.fc1.pr) + crste = READ_ONCE(*crstep); + if (!crste.h.fc || !crste.s.fc1.pr) return 0; - origin = crste_origin_large(*crstep); + origin = crste_origin_large(crste); cur = ((max(gfn, walk->start) - gfn) << PAGE_SHIFT) + origin; end = ((min(next, walk->end) - gfn) << PAGE_SHIFT) + origin; for ( ; cur < end; cur += PAGE_SIZE) From 65e05ec252a9b79e75930d3c4dd42d8877db04c5 Mon Sep 17 00:00:00 2001 From: Anthony Krowiak Date: Tue, 18 Aug 2026 15:33:49 -0400 Subject: [PATCH 09/54] s390/vfio-ap: fix KVM GISC and page leak when queue removed from host config Three related problems exist in the handling of KVM interrupt and page resources when a queue is removed from the host's AP configuration while assigned to a mediated device (mdev). Problem 1: ~~~~~~~~~ AP_RESPONSE_Q_NOT_AVAIL not handled in vfio_ap_mdev_reset_queue() When the AP bus removes a queue device whose adapter or domain has been removed from the host's AP configuration, vfio_ap_mdev_remove_queue() is called. If the queue is still in the host's AP configuration at that point, it calls vfio_ap_mdev_reset_queue(), which issues a PQAP(ZAPQ). Since the adapter is already gone from the host configuration, ap_zapq() returns AP_RESPONSE_Q_NOT_AVAIL (0x01). This response code is not handled in vfio_ap_mdev_reset_queue()'s switch statement and falls through to the default case, which issues a WARN but does not call vfio_ap_free_aqic_resources(). As a result, if IRQ handling was enabled for the queue by the guest, the KVM GISC registration and the pinned guest page holding the notification indicator byte (NIB) are both leaked. This is fixed by adding AP_RESPONSE_Q_NOT_AVAIL to the same case as AP_RESPONSE_DECONFIGURED and AP_RESPONSE_CHECKSTOPPED in vfio_ap_mdev_reset_queue(). Like those response codes, Q_NOT_AVAIL indicates the queue is not operational and no further reset attempts are possible; the correct action is to free the IRQ resources immediately. Problem 2: ~~~~~~~~~ AP_RESPONSE_Q_NOT_AVAIL not handled in apq_status_check() In vfio_ap_mdev_reset_queue(), there are four cases that indicate a queue reset has not yet completed, in which case apq_reset_check() is queued to a work queue to verify completion of the reset operation. This function uses the PQAP(TAPQ) function to get the queue's status and calls apq_status_check() to verify whether the reset has completed, failed or needs to be executed again. As described in Problem #1 above, apq_reset_check() does not specifically check for AP_RESPONSE_Q_NOT_AVAIL, thereby potentially leaking KVM GISC registration and the pinned guest page holding the NIB. This is fixed by adding a case statement for AP_RESPONSE_Q_NOT_AVAIL to apq_status_check() and returning -ENODEV for that case. The caller, apq_reset_check() will then check for this return code and call vfio_ap_free_aqic_resources() to prevent the leak. Problem 3: ~~~~~~~~~ vfio_ap_free_aqic_resources() leaks saved_isc when kvm is NULL vfio_ap_free_aqic_resources() guards the call to kvm_s390_gisc_unregister() with: if (q->saved_isc != VFIO_AP_ISC_INVALID && !WARN_ON(!(q->matrix_mdev && q->matrix_mdev->kvm))) If matrix_mdev->kvm is NULL -- which can happen when vfio_ap_mdev_unset_kvm() has already run and cleared kvm before a subsequent cleanup path reaches this function -- the WARN_ON fires and the entire block is skipped. This leaves q->saved_isc set to a non-invalid value, creating a potential double-free on any subsequent call to this function. When kvm is NULL the KVM guest is already torn down, so kvm_s390_gisc_unregister() need not and cannot be called; however, q->saved_isc must always be cleared. Fix this by separating the kvm_s390_gisc_unregister() call from the q->saved_isc reset. The WARN_ON now guards only the genuinely impossible case of matrix_mdev being NULL. A NULL kvm is handled gracefully by skipping only the unregister call, and q->saved_isc = VFIO_AP_ISC_INVALID is set unconditionally whenever saved_isc was not already invalid. Additionally, add an else clause to the host-config check in vfio_ap_mdev_remove_queue() to call vfio_ap_free_aqic_resources() directly when the queue is not in the host's AP configuration. This serves as a backstop: when the AP bus fires the driver .remove callback after an adapter is removed from the host config, the queue is by definition no longer addressable, so vfio_ap_mdev_reset_queue() would always return Q_NOT_AVAIL. The else clause handles this case directly without the unnecessary ap_zapq() call, and ensures cleanup occurs even if kvm has already been set to NULL by a prior call to vfio_ap_mdev_unset_kvm(). Fixes: b9bd10c43456d ("s390/vfio-ap: do not reset queue removed from host config") Cc: stable@vger.kernel.org Signed-off-by: Anthony Krowiak Reviewed-by: Matthew Rosato Acked-by: Halil Pasic Signed-off-by: Claudio Imbrenda Message-ID: <20260818193349.1877940-2-akrowiak@linux.ibm.com> --- drivers/s390/crypto/vfio_ap_ops.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 940c0ff668be..4db878c18f41 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -277,9 +277,9 @@ static void vfio_ap_free_aqic_resources(struct vfio_ap_queue *q) { if (!q) return; - if (q->saved_isc != VFIO_AP_ISC_INVALID && - !WARN_ON(!(q->matrix_mdev && q->matrix_mdev->kvm))) { - kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc); + if (q->saved_isc != VFIO_AP_ISC_INVALID) { + if (!WARN_ON(!q->matrix_mdev) && q->matrix_mdev->kvm) + kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc); q->saved_isc = VFIO_AP_ISC_INVALID; } if (q->saved_iova && !WARN_ON(!q->matrix_mdev)) { @@ -1935,6 +1935,8 @@ static int apq_status_check(int apqn, struct ap_queue_status *status) * a value indicating a reset needs to be performed again. */ return -EAGAIN; + case AP_RESPONSE_Q_NOT_AVAIL: + return -ENODEV; default: WARN(true, "failed to verify reset of queue %02x.%04x: TAPQ rc=%u\n", @@ -1961,6 +1963,10 @@ static void apq_reset_check(struct work_struct *reset_work) ret = apq_status_check(q->apqn, &status); if (ret == -EIO) return; + if (ret == -ENODEV) { + vfio_ap_free_aqic_resources(q); + return; + } if (ret == -EBUSY) { pr_notice_ratelimited(WAIT_MSG, elapsed, AP_QID_CARD(q->apqn), @@ -2004,6 +2010,7 @@ static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q) break; case AP_RESPONSE_DECONFIGURED: case AP_RESPONSE_CHECKSTOPPED: + case AP_RESPONSE_Q_NOT_AVAIL: vfio_ap_free_aqic_resources(q); break; default: @@ -2528,12 +2535,15 @@ void vfio_ap_mdev_remove_queue(struct ap_device *apdev) /* * If the queue is not in the host's AP configuration, then resetting * it will fail with response code 01, (APQN not valid); so, let's make - * sure it is in the host's config. + * sure it is in the host's config. If it is not, free the KVM GISC + * resources. */ if (test_bit_inv(apid, (unsigned long *)matrix_dev->info.apm) && test_bit_inv(apqi, (unsigned long *)matrix_dev->info.aqm)) { vfio_ap_mdev_reset_queue(q); flush_work(&q->reset_work); + } else { + vfio_ap_free_aqic_resources(q); } done: From d12ce6bce5ec5175c3581e01c71e7a5abb286d9b Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Wed, 12 Aug 2026 17:55:19 +0200 Subject: [PATCH 10/54] s390/uv: Fix loop condition in uv_find_secrets Systems with more than 85 UV secrets got -ENOENT for any secret past the first page. Fix this by setting the start index at the beginning of the loop in uv_find_secret() and not at the end. First test if there are more secrets left by comparing start_idx with list->next_secret_idx, and then set the start index to the next secret index. Fixes: 7c9137af2042 ("s390/uv: Retrieve UV secrets support") Acked-by: Claudio Imbrenda Reviewed-by: Christoph Schlameuss Signed-off-by: Steffen Eiden Signed-off-by: Claudio Imbrenda Message-ID: <20260812-uv_secrets_fix-v3-1-a85bd29e0666@linux.ibm.com> --- arch/s390/kernel/uv.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index dc14ebc0105b..52003c463fad 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -846,11 +846,14 @@ int uv_find_secret(const u8 secret_id[UV_SECRET_ID_LEN], struct uv_secret_list *list, struct uv_secret_list_item_hdr *secret) { - u16 start_idx = 0; + u16 start_idx; u16 list_rc; int ret; + list->next_secret_idx = 0; + do { + start_idx = list->next_secret_idx; uv_list_secrets(list, start_idx, &list_rc, NULL); if (list_rc != UVC_RC_EXECUTED && list_rc != UVC_RC_MORE_DATA) { if (list_rc == UVC_RC_INV_CMD) @@ -861,7 +864,6 @@ int uv_find_secret(const u8 secret_id[UV_SECRET_ID_LEN], ret = find_secret_in_page(secret_id, list, secret); if (ret == 0) return ret; - start_idx = list->next_secret_idx; } while (list_rc == UVC_RC_MORE_DATA && start_idx < list->next_secret_idx); return -ENOENT; From f47190b08b71e8482072978373ee88cb2dfbdaf4 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Wed, 12 Aug 2026 17:55:20 +0200 Subject: [PATCH 11/54] s390/uv: Prevent potential out-of-bounds read When the system has more than 85 secrets, the uv_secret_list struct array only holds up to 85 items per page, resulting in an out of bounds read in find_secret_in_page if the targeted secret is in the next page or not stored at all. Fix this by looping over the number of stored secrets which is the per sub-list count of stored secrets and not the overall count. Fixes: 7c9137af2042 ("s390/uv: Retrieve UV secrets support") Signed-off-by: Steffen Eiden Reviewed-by: Christoph Schlameuss Signed-off-by: Claudio Imbrenda Message-ID: <20260812-uv_secrets_fix-v3-2-a85bd29e0666@linux.ibm.com> --- arch/s390/kernel/uv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index 52003c463fad..8ea9dd7704ff 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -825,7 +825,7 @@ static int find_secret_in_page(const u8 secret_id[UV_SECRET_ID_LEN], { u16 i; - for (i = 0; i < list->total_num_secrets; i++) { + for (i = 0; i < list->num_secr_stored; i++) { if (memcmp(secret_id, list->secrets[i].id, UV_SECRET_ID_LEN) == 0) { *secret = list->secrets[i].hdr; return 0; From 79a71cc2568f4b5d42284da2aa26f3b4f47ce01b Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Wed, 2 Sep 2026 11:47:11 -0700 Subject: [PATCH 12/54] KVM: x86/pmu: Move Intel PMU global MSRs to intel_is_valid_msr() Commit c85cdc1cc1ea ("KVM: x86/pmu: Move handling PERF_GLOBAL_CTRL and friends to common x86") moved the existence check for the following Intel PMU MSRs to kvm_pmu_is_valid_msr(): - MSR_CORE_PERF_GLOBAL_STATUS - MSR_CORE_PERF_GLOBAL_CTRL - MSR_CORE_PERF_GLOBAL_OVF_CTRL That commit deemed these MSRs valid whenever pmu->version > 1. It intended to share the check with AMD PerfMonV2 because both vendor implementations require version 2 or greater for global PMU controls. However, as noted in the commit message, AMD uses different MSR indices for its global PMU registers. Commit 4a2771895ca6 ("KVM: x86/svm/pmu: Add AMD PerfMonV2 support") subsequently added AMD PerfMonV2 support and set pmu->version = 2. Because kvm_pmu_is_valid_msr() validated the Intel MSRs whenever pmu->version > 1, KVM incorrectly permitted AMD guests with PerfMonV2 to access these Intel MSRs without a #GP. Move the validation of these Intel MSRs to intel_is_valid_msr() and remove the common switch statement from kvm_pmu_is_valid_msr(). AMD already validates its own global PMU MSRs in amd_is_valid_msr(). Fixes: 4a2771895ca6 ("KVM: x86/svm/pmu: Add AMD PerfMonV2 support") Signed-off-by: Jim Mattson Reviewed-by: Like Xu Reviewed-by: Sandipan Das Link: https://patch.msgid.link/20260902184711.138538-1-jmattson@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/pmu.c | 8 -------- arch/x86/kvm/vmx/pmu_intel.c | 3 +++ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index a7d60c8785cd..d2fd47ee5ec8 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -823,14 +823,6 @@ void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) { - switch (msr) { - case MSR_CORE_PERF_GLOBAL_STATUS: - case MSR_CORE_PERF_GLOBAL_CTRL: - case MSR_CORE_PERF_GLOBAL_OVF_CTRL: - return kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)); - default: - break; - } return kvm_pmu_call(msr_idx_to_pmc)(vcpu, msr) || kvm_pmu_call(is_valid_msr)(vcpu, msr); } diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index bfa8612fb450..70a8c4816135 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -187,6 +187,9 @@ static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) int ret; switch (msr) { + case MSR_CORE_PERF_GLOBAL_STATUS: + case MSR_CORE_PERF_GLOBAL_CTRL: + case MSR_CORE_PERF_GLOBAL_OVF_CTRL: case MSR_CORE_PERF_FIXED_CTR_CTRL: return kvm_pmu_has_perf_global_ctrl(pmu); case MSR_IA32_PEBS_ENABLE: From f13368e0acffd6d6289629705cb821d921104725 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 09:14:30 -0700 Subject: [PATCH 13/54] KVM: selftests: Use __GLIBC__, not _GNU_SOURCE, to detect actual glibc Use __GLIBC__ in the hardware disable test to detect when selftests are being built/linked against glibc and thus pthread_attr_setaffinity_np() is (hopefully) available. As pointed out by Sashiko and Hisam, _GNU_SOURCE is effectively a "request" macro to enable functionality, whereas __GLIBC__ is an announcement of support and selftests' idiomatic way of guarding code that's specific to glibc. Fixes: 496779b54943 ("KVM: selftests: Pre-set threads affinity in hardware disable test when possible") Reported-by: Sashiko Bot Closes: https://lore.kernel.org/all/20260731201140.5AF0C1F00AC4@smtp.kernel.org Suggested-by: Hisam Mehboob Link: https://patch.msgid.link/20260826161430.714316-1-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/hardware_disable_test.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/kvm/hardware_disable_test.c b/tools/testing/selftests/kvm/hardware_disable_test.c index 43a36ef3ead8..1c20892d6782 100644 --- a/tools/testing/selftests/kvm/hardware_disable_test.c +++ b/tools/testing/selftests/kvm/hardware_disable_test.c @@ -37,7 +37,7 @@ static void *run_vcpu(void *arg) struct kvm_vcpu *vcpu = arg; struct kvm_run *run = vcpu->run; -#ifndef _GNU_SOURCE +#ifndef __GLIBC__ kvm_sched_setaffinity(0, sizeof(cpu_set_t), &threads_cpu_set); #endif @@ -51,7 +51,7 @@ static void *sleeping_thread(void *arg) { int fd; -#ifndef _GNU_SOURCE +#ifndef __GLIBC__ kvm_sched_setaffinity(0, sizeof(cpu_set_t), &threads_cpu_set); #endif @@ -71,7 +71,7 @@ static void run_test(u32 run) u32 i, j; TEST_ASSERT_EQ(pthread_attr_init(&attr), 0); -#ifdef _GNU_SOURCE +#ifdef __GLIBC__ TEST_ASSERT_EQ(pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &threads_cpu_set), 0); #endif From 8cd280282d1239bca68f8c3632ef1ac8556a396b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 6 Aug 2026 14:46:18 -0700 Subject: [PATCH 14/54] KVM: Never clear KVM_REQ_VM_DEAD from a vCPU's requests Use kvm_test_request() instead of kvm_check_request() when querying KVM_REQ_VM_DEAD, i.e. don't clear KVM_REQ_VM_DEAD, as the entire purpose of KVM_REQ_VM_DEAD is to prevent the vCPU from enterring the guest ever again, even if userspace insists on redoing KVM_RUN. Ensuring KVM_REQ_VM_DEAD is never cleared will allow relaxing KVM's rule that ioctls can't be invoked on dead VMs, to only disallow ioctls if the VM is bugged, i.e. if KVM hit a KVM_BUG_ON(). Opportunistically add compile-time assertions to guard against clearing KVM_REQ_VM_DEAD through the standard APIs. Reviewed-by: Kai Huang Acked-by: Marc Zyngier Link: https://patch.msgid.link/20260806214618.82180-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/arm64/kvm/arm.c | 2 +- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/vmx/tdx.c | 2 +- arch/x86/kvm/x86.c | 2 +- include/linux/kvm_host.h | 9 +++++++-- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 8b080804bc90..62c81d6ae8ee 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1129,7 +1129,7 @@ static int kvm_vcpu_suspend(struct kvm_vcpu *vcpu) static int check_vcpu_requests(struct kvm_vcpu *vcpu) { if (kvm_request_pending(vcpu)) { - if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) return -EIO; if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 064ecc33b926..8e62476e477b 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5058,7 +5058,7 @@ static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa, if (signal_pending(current)) return -EINTR; - if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) return -EIO; cond_resched(); diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index b272c20586a7..6c842e9191a5 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1995,7 +1995,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu) if (kvm_vcpu_has_events(vcpu) || signal_pending(current)) break; - if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) { + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) { ret = -EIO; break; } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 79468ddfe473..a137dc6dd8c6 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8060,7 +8060,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) bool req_immediate_exit = false; if (kvm_request_pending(vcpu)) { - if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) { + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) { r = -EIO; goto out; } diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 03bfc92864b6..cf7fe835c4ad 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2324,13 +2324,18 @@ static inline bool kvm_test_request(int req, struct kvm_vcpu *vcpu) return test_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests); } -static inline void kvm_clear_request(int req, struct kvm_vcpu *vcpu) +static __always_inline void kvm_clear_request(int req, struct kvm_vcpu *vcpu) { + BUILD_BUG_ON(req == KVM_REQ_VM_DEAD); + clear_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests); } -static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu) +static __always_inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu) { + /* Once a VM is dead, it needs to stay dead. */ + BUILD_BUG_ON(req == KVM_REQ_VM_DEAD); + if (kvm_test_request(req, vcpu)) { kvm_clear_request(req, vcpu); From aaad136d56d91252517272b68cd533e5714698d5 Mon Sep 17 00:00:00 2001 From: Myeonghun Pak Date: Sat, 1 Aug 2026 01:35:50 +0900 Subject: [PATCH 15/54] RISC-V: KVM: Synchronize hrtimer callback during teardown The non-Sstc hrtimer callback clears next_set before its final uses of the enclosing vCPU. If teardown observes next_set as false while the callback is still running, kvm_riscv_vcpu_timer_cancel() skips hrtimer_cancel() and kvm_destroy_vcpus() can free the vCPU before the callback enters kvm_riscv_vcpu_set_interrupt(). A guest can arm the timer with SBI TIME and request shutdown with SBI legacy shutdown or SRST. A VMM that honors KVM_EXIT_SYSTEM_EVENT and destroys the VM supplies the teardown side of the race; no post-launch host ioctl is needed to arm or request teardown. On upstream master 62cc90241548, generic KASAN reported: BUG: KASAN: slab-use-after-free in do_raw_spin_lock Write of size 4 at addr ff60000005e58898 kvm_riscv_vcpu_set_interrupt kvm_riscv_vcpu_hrtimer_expired __hrtimer_run_queues hrtimer_interrupt The object was allocated by KVM_CREATE_VCPU and freed concurrently by: kvm_destroy_vcpus kvm_arch_destroy_vm kvm_destroy_vm __fput For deterministic validation, I added mdelay(1000) immediately after the existing next_set = false assignment. This only widens the existing post-clear callback window. A no-delay trace build naturally reached the callback-after-teardown-start/before-deinit ordering in 12 of 200 runs, but 1,500 stock-kernel stress iterations did not produce a KASAN report, so natural reproduction is timing-sensitive. Always invoke hrtimer_cancel() for an initialized timer. Preserve the existing -EINVAL result when the timer is no longer set, but only after synchronizing with a running callback. With this patch, hrtimer_cancel() blocked for the full widened callback window before vCPU destruction. KASAN reported no error in 100 fixed-and-widened runs or 200 fix-only timing-sweep runs. Fixes: 3a9f66cb25e1 ("RISC-V: KVM: Add timer functionality") Cc: stable@vger.kernel.org Assisted-by: OpenAI:GPT-5.6 Signed-off-by: Myeonghun Pak Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260731163550.46991-1-mhun512@gmail.com Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_timer.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/riscv/kvm/vcpu_timer.c b/arch/riscv/kvm/vcpu_timer.c index ae53133c7ab0..a2cd277a4059 100644 --- a/arch/riscv/kvm/vcpu_timer.c +++ b/arch/riscv/kvm/vcpu_timer.c @@ -61,10 +61,13 @@ static enum hrtimer_restart kvm_riscv_vcpu_hrtimer_expired(struct hrtimer *h) static int kvm_riscv_vcpu_timer_cancel(struct kvm_vcpu_timer *t) { - if (!t->init_done || !t->next_set) + if (!t->init_done) return -EINVAL; hrtimer_cancel(&t->hrt); + + if (!t->next_set) + return -EINVAL; t->next_set = false; return 0; From 52c6b7d20d3e791a9e75aa2be2a467990154c7cd Mon Sep 17 00:00:00 2001 From: Yicong Yang Date: Tue, 4 Aug 2026 21:40:18 +0800 Subject: [PATCH 16/54] RISC-V: KVM: Fix the conversion between vsip and hvip Per AIA spec 1.0 Section 6.3.2, the interrupt numbers 13-63 shares same bit position between related VS shadow CSRs and hypervisor CSRs. So there's a shift only for SSI, STI and SEI interrupt. Currently the KVM always does a shift for all the interrupts (include LCOFI with number 13) when doing the conversion between vsip and hvip. Fix this by only doing shift the SSI, STI and SEI. Add wrappers for doing the conversion between vsip and hvip. Fixes: 16b0bde9a37c ("RISC-V: KVM: Add perf sampling support for guests") Signed-off-by: Yicong Yang Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260804134018.85497-1-yang.yicong@picoheart.com Signed-off-by: Anup Patel --- arch/riscv/include/asm/csr.h | 20 ++++++++++++++++---- arch/riscv/kvm/vcpu.c | 3 +-- arch/riscv/kvm/vcpu_onereg.c | 8 +++----- tools/arch/riscv/include/asm/csr.h | 20 ++++++++++++++++---- 4 files changed, 36 insertions(+), 15 deletions(-) diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index 6c823361be86..621f84d1898f 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -188,12 +188,24 @@ #define HGATP_MODE_SHIFT HGATP32_MODE_SHIFT #endif -/* VSIP & HVIP relation */ +/* + * VSIP & HVIP relation + * + * The bit positions are same between VSIP and HVIP for interrupt + * numbers 13-63, where there's a shift for the SSI, STI and SEI. + */ #define VSIP_TO_HVIP_SHIFT (IRQ_VS_SOFT - IRQ_S_SOFT) -#define VSIP_VALID_MASK ((_AC(1, UL) << IRQ_S_SOFT) | \ +#define VSIP_BIAS_MASK ((_AC(1, UL) << IRQ_S_SOFT) | \ (_AC(1, UL) << IRQ_S_TIMER) | \ - (_AC(1, UL) << IRQ_S_EXT) | \ - (_AC(1, UL) << IRQ_PMU_OVF)) + (_AC(1, UL) << IRQ_S_EXT)) +#define VSIP_NO_BIAS_MASK (_AC(1, UL) << IRQ_PMU_OVF) +#define VSIP_VALID_MASK (VSIP_BIAS_MASK | VSIP_NO_BIAS_MASK) +#define vsip_to_hvip(_vsip) ((((_vsip) & VSIP_BIAS_MASK) << \ + VSIP_TO_HVIP_SHIFT) | \ + ((_vsip) & VSIP_NO_BIAS_MASK)) +#define hvip_to_vsip(_hvip) ((((_hvip) >> VSIP_TO_HVIP_SHIFT) & \ + VSIP_BIAS_MASK) | \ + ((_hvip) & VSIP_NO_BIAS_MASK)) /* AIA CSR bits */ #define TOPI_IID_SHIFT 16 diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index e062ca19f9d8..7f1636c44d41 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -491,8 +491,7 @@ bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) bool ret; raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); - ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK) - << VSIP_TO_HVIP_SHIFT) & (unsigned long)mask; + ie = vsip_to_hvip(vcpu->arch.guest_csr.vsie) & (unsigned long)mask; ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK & (unsigned long)mask; ret = vcpu->arch.irqs_pending[0] & ie; diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index 841f2cf87168..f16e25098af1 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -272,7 +272,7 @@ static int kvm_riscv_vcpu_general_get_csr(struct kvm_vcpu *vcpu, if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { kvm_riscv_vcpu_flush_interrupts(vcpu); - *out_val = (csr->hvip >> VSIP_TO_HVIP_SHIFT) & VSIP_VALID_MASK; + *out_val = hvip_to_vsip(csr->hvip); *out_val |= csr->hvip & ~IRQ_LOCAL_MASK; } else *out_val = ((unsigned long *)csr)[reg_num]; @@ -293,10 +293,8 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, reg_num = array_index_nospec(reg_num, regs_max); - if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { - reg_val &= VSIP_VALID_MASK; - reg_val <<= VSIP_TO_HVIP_SHIFT; - } + if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) + reg_val = vsip_to_hvip(reg_val); ((unsigned long *)csr)[reg_num] = reg_val; diff --git a/tools/arch/riscv/include/asm/csr.h b/tools/arch/riscv/include/asm/csr.h index 21d8cee04638..8df64314d613 100644 --- a/tools/arch/riscv/include/asm/csr.h +++ b/tools/arch/riscv/include/asm/csr.h @@ -163,12 +163,24 @@ #define HGATP_MODE_SHIFT HGATP32_MODE_SHIFT #endif -/* VSIP & HVIP relation */ +/* + * VSIP & HVIP relation + * + * The bit positions are same between VSIP and HVIP for interrupt + * numbers 13-63, where there's a shift for the SSI, STI and SEI. + */ #define VSIP_TO_HVIP_SHIFT (IRQ_VS_SOFT - IRQ_S_SOFT) -#define VSIP_VALID_MASK ((_AC(1, UL) << IRQ_S_SOFT) | \ +#define VSIP_BIAS_MASK ((_AC(1, UL) << IRQ_S_SOFT) | \ (_AC(1, UL) << IRQ_S_TIMER) | \ - (_AC(1, UL) << IRQ_S_EXT) | \ - (_AC(1, UL) << IRQ_PMU_OVF)) + (_AC(1, UL) << IRQ_S_EXT)) +#define VSIP_NO_BIAS_MASK (_AC(1, UL) << IRQ_PMU_OVF) +#define VSIP_VALID_MASK (VSIP_BIAS_MASK | VSIP_NO_BIAS_MASK) +#define vsip_to_hvip(_vsip) ((((_vsip) & VSIP_BIAS_MASK) << \ + VSIP_TO_HVIP_SHIFT) | \ + ((_vsip) & VSIP_NO_BIAS_MASK)) +#define hvip_to_vsip(_hvip) ((((_hvip) >> VSIP_TO_HVIP_SHIFT) & \ + VSIP_BIAS_MASK) | \ + ((_hvip) & VSIP_NO_BIAS_MASK)) /* AIA CSR bits */ #define TOPI_IID_SHIFT 16 From 8ae12ccaec6ec74945d8c1ef39f2c1b8df779abc Mon Sep 17 00:00:00 2001 From: Xie Bo Date: Mon, 10 Aug 2026 09:21:15 +0800 Subject: [PATCH 17/54] RISC-V: KVM: Serialize IMSIC attributes with vCPU migration KVM device ioctls are not serialized against KVM_RUN. As a result, kvm_riscv_aia_imsic_rw_attr() can snapshot the physical CPU and HGEI of an IMSIC VS-file before a concurrent vCPU migration releases it. The HGEI can then be allocated to another vCPU before imsic_vsfile_rw() uses the stale tuple. A GET or SET attribute may consequently access the new owner's interrupt file. Serialize the entire IMSIC attribute operation with the target vCPU mutex. This prevents the VS-file from being migrated and recycled until the attribute access completes. Acquire the mutex killably so that the device ioctl remains interruptible while waiting for KVM_RUN to finish. Fixes: db8b7e97d613 ("RISC-V: KVM: Add in-kernel virtualization of AIA IMSIC") Cc: stable@vger.kernel.org Signed-off-by: Xie Bo Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260810-imsic-attr-race-v2-1-00ed95ad321e@ultrarisc.com Signed-off-by: Anup Patel --- arch/riscv/kvm/aia_imsic.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/arch/riscv/kvm/aia_imsic.c b/arch/riscv/kvm/aia_imsic.c index c1af23e79ae0..f17dc559a709 100644 --- a/arch/riscv/kvm/aia_imsic.c +++ b/arch/riscv/kvm/aia_imsic.c @@ -965,9 +965,14 @@ int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type, if (!vcpu) return -ENODEV; + if (mutex_lock_killable(&vcpu->mutex)) + return -EINTR; + imsic = vcpu->arch.aia_context.imsic_state; - if (!imsic) - return -ENODEV; + if (!imsic) { + rc = -ENODEV; + goto out_unlock; + } isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); read_lock_irqsave(&imsic->vsfile_lock, flags); @@ -991,6 +996,8 @@ int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type, rc = imsic_vsfile_rw(vsfile_hgei, vsfile_cpu, imsic->nr_eix, isel, write, val); +out_unlock: + mutex_unlock(&vcpu->mutex); return rc; } From ed54fdb460a6e81d5f8f38388d1f10b385dd4a58 Mon Sep 17 00:00:00 2001 From: Xie Bo Date: Mon, 10 Aug 2026 13:15:43 +0800 Subject: [PATCH 18/54] RISC-V: KVM: Release unused page after MMU invalidation If an MMU invalidation races with a G-stage fault, the fault handler skips installing the page but leaves ret set to zero. As a result, kvm_release_faultin_page() treats the page as used and can unnecessarily mark it dirty. Track the invalidation retry separately and release the page as unused, while preserving the existing return value so that the vCPU retries the fault. Fixes: 2ed90cb0938a ("KVM: RISC-V: Retry fault if vma_lookup() results become invalid") Cc: stable@vger.kernel.org Signed-off-by: Xie Bo Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260810051544.3953925-2-xb@ultrarisc.com Signed-off-by: Anup Patel --- arch/riscv/kvm/mmu.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 6035b5ec9503..d189fd58d7bf 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -627,6 +627,7 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, int ret; kvm_pfn_t hfn; bool is_hugetlb; + bool unused = false; bool writable; unsigned int vma_pageshift; gfn_t gfn = gpa >> PAGE_SHIFT; @@ -719,8 +720,10 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, write_lock(&kvm->mmu_lock); - if (mmu_invalidate_retry(kvm, mmu_seq)) + if (mmu_invalidate_retry(kvm, mmu_seq)) { + unused = true; goto out_unlock; + } /* * Check if we are backed by a THP and thus use block mapping if @@ -743,7 +746,8 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, kvm_err("Failed to map in G-stage\n"); out_unlock: - kvm_release_faultin_page(kvm, page, ret && ret != -EEXIST, writable); + kvm_release_faultin_page(kvm, page, + unused || (ret && ret != -EEXIST), writable); write_unlock(&kvm->mmu_lock); return ret; } From f41fb17143df890855d3de980f8eb92dfd595817 Mon Sep 17 00:00:00 2001 From: Xie Bo Date: Mon, 10 Aug 2026 13:15:44 +0800 Subject: [PATCH 19/54] RISC-V: KVM: Propagate interrupted G-stage faults __kvm_faultin_pfn() reports an interrupted host page fault with KVM_PFN_ERR_SIGPENDING. RISC-V currently handles it as a generic error PFN and returns -EFAULT. Return -EINTR for the signal-pending sentinel so callers can distinguish an interrupted fault from an invalid userspace mapping. Do not log the expected interruption as a vCPU exit error. Fixes: 9d05c1fee837 ("RISC-V: KVM: Implement stage2 page table programming") Cc: stable@vger.kernel.org Signed-off-by: Xie Bo Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260810051544.3953925-3-xb@ultrarisc.com Signed-off-by: Anup Patel --- arch/riscv/kvm/mmu.c | 2 ++ arch/riscv/kvm/vcpu_exit.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index d189fd58d7bf..3e955d808743 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -708,6 +708,8 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, vma_pageshift, current); return 0; } + if (is_sigpending_pfn(hfn)) + return -EINTR; if (is_error_noslot_pfn(hfn)) return -EFAULT; diff --git a/arch/riscv/kvm/vcpu_exit.c b/arch/riscv/kvm/vcpu_exit.c index 88e0c369b354..8d36eb8abce1 100644 --- a/arch/riscv/kvm/vcpu_exit.c +++ b/arch/riscv/kvm/vcpu_exit.c @@ -283,7 +283,7 @@ int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, } /* Print details in-case of error */ - if (ret < 0) { + if (ret < 0 && ret != -EINTR) { kvm_err("VCPU exit error %d\n", ret); kvm_err("SEPC=0x%lx SSTATUS=0x%lx HSTATUS=0x%lx\n", vcpu->arch.guest_context.sepc, From b3d346838ec65fac7fd83f5dbcedd13cadfffddb Mon Sep 17 00:00:00 2001 From: Zongmin Zhou Date: Wed, 26 Aug 2026 15:50:09 +0800 Subject: [PATCH 20/54] KVM: riscv: Fix NACL hfence entry update order The SBI v3.0 specification (section 15.1.2) requires a nested HFENCE entry to be populated as follows: 1) find an unused entry with Config.Pending == 0 2) update the Page_Number and Page_Count words 3) update the Config word with Config.Pending set __kvm_riscv_nacl_hfence() writes the Config word first, so the SBI implementation (or NACL hardware) can observe a pending entry with pnum/pcount values left over from the previous use of that entry, resulting in incorrect TLB flush ranges. Write pnum and pcount first and the Config word last. Since the consumer is an external agent on coherent shared memory, use WRITE_ONCE() to stop the compiler from reordering the stores and smp_wmb() to make the parameter words globally visible before the Pending bit is set. Fixes: d466c19cead5 ("RISC-V: KVM: Add common nested acceleration support") Signed-off-by: Zongmin Zhou Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260826075009.68952-1-min_halo@163.com Signed-off-by: Anup Patel --- arch/riscv/kvm/nacl.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/arch/riscv/kvm/nacl.c b/arch/riscv/kvm/nacl.c index 9aff03c4f667..a5cda9a65156 100644 --- a/arch/riscv/kvm/nacl.c +++ b/arch/riscv/kvm/nacl.c @@ -42,12 +42,24 @@ void __kvm_riscv_nacl_hfence(void *shmem, } } - entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_CONFIG(i); - *entp = cpu_to_lelong(control); + /* + * Per SBI v3.0 section 15.1.2, the Page_Number and Page_Count + * words must be updated before the Config word with its Pending + * bit set. WRITE_ONCE() stops the compiler from reordering the + * stores and smp_wmb() makes the parameter words globally + * visible to the SBI implementation (or NACL hardware) before + * the Pending bit is set. + */ entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_PNUM(i); - *entp = cpu_to_lelong(page_num); + WRITE_ONCE(*entp, cpu_to_lelong(page_num)); entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_PCOUNT(i); - *entp = cpu_to_lelong(page_count); + WRITE_ONCE(*entp, cpu_to_lelong(page_count)); + + /* Ensure the parameter words are visible before the Pending bit */ + smp_wmb(); + + entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_CONFIG(i); + WRITE_ONCE(*entp, cpu_to_lelong(control)); } int kvm_riscv_nacl_enable(void) From b7749531a9b195f4fd7db92ca3cc49bda1a4dc8d Mon Sep 17 00:00:00 2001 From: Zongmin Zhou Date: Wed, 26 Aug 2026 14:41:17 +0800 Subject: [PATCH 21/54] RISC-V: KVM: Fix sdata leak and stale snapshot_addr in snapshot_set_shmem A guest may call SBI_PMU_SNAPSHOT_SET_SHMEM repeatedly. Each call overwrites kvpmu->sdata without freeing the old buffer (memory leak), and if a later kvm_vcpu_write_guest() fails, the error path frees sdata but leaves snapshot_addr stale. A subsequent SBI_PMU_COUNTER_START then passes the INVALID_GPA check and crashes the host with a NULL buffer in kvm_vcpu_read_guest(). Fix this by clearing the previously installed snapshot area before installing a new one, which keeps sdata and snapshot_addr consistent. The SBI spec suggests a single invocation but defines no error code for repeated calls, so KVM must tolerate them. Fixes: c2f41ddbcdd756 ("RISC-V: KVM: Implement SBI PMU Snapshot feature") Cc: stable@vger.kernel.org Signed-off-by: Zongmin Zhou Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260826064117.58029-1-min_halo@163.com Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_pmu.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/riscv/kvm/vcpu_pmu.c b/arch/riscv/kvm/vcpu_pmu.c index 6ff741ee7803..cc0c138651bc 100644 --- a/arch/riscv/kvm/vcpu_pmu.c +++ b/arch/riscv/kvm/vcpu_pmu.c @@ -454,6 +454,14 @@ int kvm_riscv_vcpu_pmu_snapshot_set_shmem(struct kvm_vcpu *vcpu, unsigned long s } } + /* + * Clear any previously installed snapshot area to avoid leaking + * the old sdata and to keep sdata/snapshot_addr consistent if + * the re-install fails below. + */ + if (kvpmu->snapshot_addr != INVALID_GPA) + kvm_pmu_clear_snapshot_area(vcpu); + kvpmu->sdata = kzalloc(snapshot_area_size, GFP_ATOMIC | __GFP_ACCOUNT); if (!kvpmu->sdata) { sbiret = SBI_ERR_FAILURE; From 8b3fd1a8b305321171602bfa7c41212441cf69e4 Mon Sep 17 00:00:00 2001 From: SeungJu Cheon Date: Tue, 25 Aug 2026 17:37:17 +0900 Subject: [PATCH 22/54] RISC-V: KVM: Preserve firmware counter value across stop/start Firmware events accumulate in kvpmu->fw_event[].value while running, but counter stop only clears fw_event[].started without saving the value back to pmc->counter_val. A subsequent counter start without SBI_PMU_START_FLAG_SET_INIT_VALUE reloads the stale counter_val into fw_event[].value, losing all events counted so far. Save fw_event[].value into counter_val when actually stopping a running counter, and remove the now redundant synchronization from the snapshot path. Fixes: badc386869e2c ("RISC-V: KVM: Support firmware events") Signed-off-by: SeungJu Cheon Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260825083719.643970-2-suunj1331@gmail.com Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_pmu.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/riscv/kvm/vcpu_pmu.c b/arch/riscv/kvm/vcpu_pmu.c index cc0c138651bc..5dfe2054d12b 100644 --- a/arch/riscv/kvm/vcpu_pmu.c +++ b/arch/riscv/kvm/vcpu_pmu.c @@ -683,10 +683,12 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, goto out; } - if (!kvpmu->fw_event[fevent_code].started) + if (!kvpmu->fw_event[fevent_code].started) { sbiret = SBI_ERR_ALREADY_STOPPED; - - kvpmu->fw_event[fevent_code].started = false; + } else { + kvpmu->fw_event[fevent_code].started = false; + pmc->counter_val = kvpmu->fw_event[fevent_code].value; + } } else if (pmc->perf_event) { if (pmc->started) { /* Stop counting the counter */ @@ -704,9 +706,7 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, } if (snap_flag_set && !sbiret) { - if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) - pmc->counter_val = kvpmu->fw_event[fevent_code].value; - else if (pmc->perf_event) + if (pmc->perf_event) pmc->counter_val += perf_event_read_value(pmc->perf_event, &enabled, &running); /* From 057dd2639ceae79adced5d8fe52c32d562edcb3a Mon Sep 17 00:00:00 2001 From: SeungJu Cheon Date: Tue, 25 Aug 2026 17:37:18 +0900 Subject: [PATCH 23/54] RISC-V: KVM: Report snapshot write failure to the guest If kvm_vcpu_write_guest() fails while updating the PMU snapshot area on counter stop, the guest may receive SBI_SUCCESS without the snapshot being updated, leaving stale data in shared memory. Return SBI_ERR_FAILURE when the snapshot write fails. Fixes: c2f41ddbcdd7 ("RISC-V: KVM: Implement SBI PMU Snapshot feature") Signed-off-by: SeungJu Cheon Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260825083719.643970-3-suunj1331@gmail.com Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_pmu.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/riscv/kvm/vcpu_pmu.c b/arch/riscv/kvm/vcpu_pmu.c index 5dfe2054d12b..8be87f879473 100644 --- a/arch/riscv/kvm/vcpu_pmu.c +++ b/arch/riscv/kvm/vcpu_pmu.c @@ -735,9 +735,10 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, } } - if (shmem_needs_update) - kvm_vcpu_write_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata, - sizeof(struct riscv_pmu_snapshot_data)); + if (shmem_needs_update && + kvm_vcpu_write_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata, + sizeof(struct riscv_pmu_snapshot_data))) + sbiret = SBI_ERR_FAILURE; out: retdata->err_val = sbiret; From c7e2cc38c56142cdab25e6f73602a8222bf9479b Mon Sep 17 00:00:00 2001 From: SeungJu Cheon Date: Tue, 25 Aug 2026 17:37:19 +0900 Subject: [PATCH 24/54] RISC-V: KVM: Fix perf-backed counter accounting across stop and read pmu_ctr_read() adds the event count returned by perf_event_read_value() to counter_val, which can accumulate the same count repeatedly across reads. kvm_riscv_vcpu_pmu_ctr_stop() also leaves counter_val stale by not folding the current event count into it. Make reads of perf-backed counters side-effect free, and use perf_event_pause() when stopping a counter to fold the current event count into counter_val while resetting it. This preserves the counter value across stop/start and lets the snapshot path use counter_val directly. Fixes: 0cb74b65d2e5 ("RISC-V: KVM: Implement perf support without sampling") Signed-off-by: SeungJu Cheon Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260825083719.643970-4-suunj1331@gmail.com Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_pmu.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/riscv/kvm/vcpu_pmu.c b/arch/riscv/kvm/vcpu_pmu.c index 8be87f879473..e095d1ff439f 100644 --- a/arch/riscv/kvm/vcpu_pmu.c +++ b/arch/riscv/kvm/vcpu_pmu.c @@ -270,12 +270,13 @@ static int pmu_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx, return -EINVAL; pmc->counter_val = kvpmu->fw_event[fevent_code].value; + *out_val = pmc->counter_val; } else if (pmc->perf_event) { - pmc->counter_val += perf_event_read_value(pmc->perf_event, &enabled, &running); + *out_val = pmc->counter_val + + perf_event_read_value(pmc->perf_event, &enabled, &running); } else { return -EINVAL; } - *out_val = pmc->counter_val; return 0; } @@ -653,7 +654,6 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, { struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); int i, pmc_index, sbiret = 0; - u64 enabled, running; struct kvm_pmc *pmc; int fevent_code; bool snap_flag_set = flags & SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT; @@ -691,8 +691,11 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, } } else if (pmc->perf_event) { if (pmc->started) { - /* Stop counting the counter */ - perf_event_disable(pmc->perf_event); + /* + * Stop the counter and fold the live count into counter_val. + * Reset the event value to avoid redundant accumulation. + */ + pmc->counter_val += perf_event_pause(pmc->perf_event, true); pmc->started = false; } else { sbiret = SBI_ERR_ALREADY_STOPPED; @@ -706,9 +709,6 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, } if (snap_flag_set && !sbiret) { - if (pmc->perf_event) - pmc->counter_val += perf_event_read_value(pmc->perf_event, - &enabled, &running); /* * The counter and overflow indices in the snapshot region are w.r.to * cbase. Modify the set bit in the counter mask instead of the pmc_index From 41e81f7e3ef96594fb840445343c0ee7723aa550 Mon Sep 17 00:00:00 2001 From: Tan Chi Date: Mon, 14 Sep 2026 11:11:46 +0800 Subject: [PATCH 25/54] RISC-V: KVM: Fix HSM hart status error propagation kvm_sbi_hsm_vcpu_get_status() returns SBI_ERR_INVALID_PARAM when the requested hart does not exist. However, the HART_STATUS case returns from the SBI handler without storing this error in retdata->err_val. As a result, a guest querying the status of a non-existent hart observes SBI_SUCCESS instead of SBI_ERR_INVALID_PARAM. Use the common SBI error handling path for HART_STATUS after saving a valid hart state in retdata->out_val. This preserves the returned error when kvm_sbi_hsm_vcpu_get_status() fails. Fixes: bae0dfd74e01 ("RISC-V: KVM: Modify SBI extension handler to return SBI error code") Cc: stable@vger.kernel.org Assisted-by: LLM Signed-off-by: Tan Chi Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260914031146.446157-1-tanchi25@mails.ucas.ac.cn Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_sbi_hsm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/riscv/kvm/vcpu_sbi_hsm.c b/arch/riscv/kvm/vcpu_sbi_hsm.c index f26207f84bab..06a15629c26b 100644 --- a/arch/riscv/kvm/vcpu_sbi_hsm.c +++ b/arch/riscv/kvm/vcpu_sbi_hsm.c @@ -95,9 +95,9 @@ static int kvm_sbi_ext_hsm_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, ret = kvm_sbi_hsm_vcpu_get_status(vcpu); if (ret >= 0) { retdata->out_val = ret; - retdata->err_val = 0; + ret = 0; } - return 0; + break; case SBI_EXT_HSM_HART_SUSPEND: switch (lower_32_bits(cp->a0)) { case SBI_HSM_SUSPEND_RET_DEFAULT: From 8cd92f77ae4f5371a7d581f8324c24919670b304 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 21 Aug 2026 07:44:42 +0100 Subject: [PATCH 26/54] KVM: arm64: vgic-its: Free the caches when GITS_BASER changes A guest that disables the ITS and re-points or shrinks GITS_BASER with VALID still set keeps the devices and collections it mapped against the old table, as KVM frees them only when VALID is cleared. The contents of the table are IMPLEMENTATION DEFINED, so a write that gives GITS_BASER a different address or size may lose whatever the old value described. Free the list whenever the stored value changes, and drop the translation cache with it. The cache is not empty just because the ITS is disabled: its->enabled is written under the cmd_lock, while vgic_its_resolve_lpi() tests it under the its_lock, so an injection can still cache an entry after the ITS was disabled. Hence the invalidation inside the its_lock section. Test for a change rather than a write: its_restore_enable() rewrites GITS_BASER from its probe-time cache on resume, and KVM reports GITS_TYPER.HCC as 0, so nothing re-maps the boot CPU's collection afterwards. Fixes: 36d6961c2b481 ("KVM: arm/arm64: vgic-its: Free caches when GITS_BASER Valid bit is cleared") Suggested-by: Marc Zyngier Link: https://lore.kernel.org/all/87ecg9owwa.wl-maz@kernel.org/ Signed-off-by: Fuad Tabba Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260821064445.615838-2-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-its.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 9e782a4fea7e..ab89b0138efd 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -1658,7 +1658,7 @@ static void vgic_mmio_write_its_baser(struct kvm *kvm, unsigned long val) { const struct vgic_its_abi *abi = vgic_its_get_abi(its); - u64 entry_size, table_type; + u64 old, entry_size, table_type; u64 reg, *regptr, clearbits = 0; /* When GITS_CTLR.Enable is 1, we ignore write accesses. */ @@ -1681,7 +1681,9 @@ static void vgic_mmio_write_its_baser(struct kvm *kvm, return; } - reg = update_64bit_reg(*regptr, addr & 7, len, val); + old = *regptr; + + reg = update_64bit_reg(old, addr & 7, len, val); reg &= ~GITS_BASER_RO_MASK; reg &= ~clearbits; @@ -1691,7 +1693,8 @@ static void vgic_mmio_write_its_baser(struct kvm *kvm, *regptr = reg; - if (!(reg & GITS_BASER_VALID)) { + /* The ITS driver rewrites an unchanged GITS_BASER on resume. */ + if (reg != old) { /* Take the its_lock to prevent a race with a save/restore */ mutex_lock(&its->its_lock); switch (table_type) { @@ -1702,6 +1705,8 @@ static void vgic_mmio_write_its_baser(struct kvm *kvm, vgic_its_free_collection_list(kvm, its); break; } + /* A concurrent injection may have cached a translation. */ + vgic_its_invalidate_cache(its); mutex_unlock(&its->its_lock); } } From 30908e7272479b6453745022abb9f2eb9c9933f7 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 21 Aug 2026 07:44:43 +0100 Subject: [PATCH 27/54] Revert "KVM: arm64: vgic-its: Don't save collections the table cannot hold" This reverts commit 9b10fb74e4b661543d188701bd4d024fc5c18f58. Freeing the collections when GITS_BASER changes removes the state this check rejected: vgic_its_cmd_handle_mapi(), vgic_its_cmd_handle_mapc() and vgic_its_restore_cte() all validate the ID against the current table before allocating, and the table can no longer change under the list. What remains is a collection whose entry is not backed by a memslot, which the write fails on anyway, so the check costs a save userspace should be able to issue reliably and buys nothing. The reverted commit credited the check with bounding the walk as well. It stays bounded without it: collection IDs are unique and each is below the table's capacity, so the list cannot be longer than the table. Suggested-by: Marc Zyngier Link: https://lore.kernel.org/all/87ecg9owwa.wl-maz@kernel.org/ Signed-off-by: Fuad Tabba Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260821064445.615838-3-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-its.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index ab89b0138efd..313bf9e802bf 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -2546,9 +2546,6 @@ static int vgic_its_save_collection_table(struct vgic_its *its) max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K; list_for_each_entry(collection, &its->collection_list, coll_list) { - if (!vgic_its_check_id(its, baser, collection->collection_id, NULL)) - return -EINVAL; - ret = vgic_its_save_cte(its, collection, gpa); if (ret) return ret; From cc5d96036e01ac330d24b2f0c336d60f82ab4930 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 21 Aug 2026 07:44:44 +0100 Subject: [PATCH 28/54] KVM: arm64: vgic-its: Skip unreachable devices instead of failing the save vgic_its_save_device_tables() aborts with -EINVAL when a device's entry falls outside the device table, which a guest can arrange on its own: an indirect table lets it clear an L1 entry's valid bit without touching GITS_BASER. That fails a save userspace should be able to issue reliably. Skip the device instead, and point the saved DTE chain past it, as commit ad1e686e2378d ("KVM: arm64: vgic-its: Point saved ITEs at the next valid entry") does for ITEs. compute_next_devid_offset() takes the next device off the list whether or not it was saved, so the predecessor would otherwise point at an entry the save never wrote. Restore follows that offset while it stays inside the table being scanned: within an L2 block, or anywhere in a flat table. Both need userspace to remove a memslot under the table, since dropping an L1 entry takes the whole block with it and scan_its_table() stops at the block boundary. Fixes: 57a9a117154c9 ("KVM: arm64: vgic-its: Device table save/restore") Suggested-by: Marc Zyngier Link: https://lore.kernel.org/all/86bjaz5s6v.wl-maz@kernel.org/ Signed-off-by: Fuad Tabba Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260821064445.615838-4-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-its.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 313bf9e802bf..0904ae850c35 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -2024,18 +2024,22 @@ static int vgic_its_attr_regs_access(struct kvm_device *dev, return ret; } -static u32 compute_next_devid_offset(struct list_head *h, +static u32 compute_next_devid_offset(struct vgic_its *its, u64 baser, struct its_device *dev) { - struct its_device *next; - u32 next_offset; + struct its_device *next = dev; - if (list_is_last(&dev->dev_list, h)) - return 0; - next = list_next_entry(dev, dev_list); - next_offset = next->device_id - dev->device_id; + /* + * Point at the next device vgic_its_save_device_tables() saves. It + * sorts device_list first, so the subtraction cannot underflow. + */ + list_for_each_entry_continue(next, &its->device_list, dev_list) { + if (vgic_its_check_id(its, baser, next->device_id, NULL)) + return min_t(u32, next->device_id - dev->device_id, + VITS_DTE_MAX_DEVID_OFFSET); + } - return min_t(u32, next_offset, VITS_DTE_MAX_DEVID_OFFSET); + return 0; } static u32 compute_next_eventid_offset(struct list_head *h, struct its_ite *ite) @@ -2276,17 +2280,18 @@ static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev) * vgic_its_save_dte - Save a device table entry at a given GPA * * @its: ITS handle + * @baser: GITS_BASER the caller is saving against * @dev: ITS device * @ptr: GPA */ -static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev, - gpa_t ptr) +static int vgic_its_save_dte(struct vgic_its *its, u64 baser, + struct its_device *dev, gpa_t ptr) { u64 val, itt_addr_field; u32 next_offset; itt_addr_field = dev->itt_addr >> 8; - next_offset = compute_next_devid_offset(&its->device_list, dev); + next_offset = compute_next_devid_offset(its, baser, dev); val = (1ULL << KVM_ITS_DTE_VALID_SHIFT | ((u64)next_offset << KVM_ITS_DTE_NEXT_SHIFT) | (itt_addr_field << KVM_ITS_DTE_ITTADDR_SHIFT) | @@ -2385,15 +2390,16 @@ static int vgic_its_save_device_tables(struct vgic_its *its) int ret; gpa_t eaddr; + /* Don't fail a save that userspace must be able to issue. */ if (!vgic_its_check_id(its, baser, dev->device_id, &eaddr)) - return -EINVAL; + continue; ret = vgic_its_save_itt(its, dev); if (ret) return ret; - ret = vgic_its_save_dte(its, dev, eaddr); + ret = vgic_its_save_dte(its, baser, dev, eaddr); if (ret) return ret; } From 6f182db39fb00548c26d689163ceb2fe4a802793 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 21 Aug 2026 07:44:45 +0100 Subject: [PATCH 29/54] KVM: arm64: selftests: Add ITS table save tests Cover the two ways a guest can leave a table that KVM_DEV_ARM_ITS_SAVE_TABLES has to cope with: a GITS_BASER write that changes the table, which drops the collections it described, and a device whose L2 block the guest invalidated, which the save skips. Each case then resets and restores, which is what the save exists for. Both fail without the preceding patches. The first save returns -EINVAL, and the second saves a DTE pointing 8192 entries ahead, at an entry the save never wrote. Assisted-by: Antigravity:gemini-3.1-pro Signed-off-by: Fuad Tabba Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260821064445.615838-5-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../selftests/kvm/arm64/vgic_its_save.c | 441 ++++++++++++++++++ 2 files changed, 442 insertions(+) create mode 100644 tools/testing/selftests/kvm/arm64/vgic_its_save.c diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 96bab7002d39..6a1482e3a286 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -189,6 +189,7 @@ TEST_GEN_PROGS_arm64 += arm64/stage2_block_transitions TEST_GEN_PROGS_arm64 += arm64/vcpu_width_config TEST_GEN_PROGS_arm64 += arm64/vgic_init TEST_GEN_PROGS_arm64 += arm64/vgic_irq +TEST_GEN_PROGS_arm64 += arm64/vgic_its_save TEST_GEN_PROGS_arm64 += arm64/vgic_lpi_stress TEST_GEN_PROGS_arm64 += arm64/vgic_v5 TEST_GEN_PROGS_arm64 += arm64/vpmu_counter_access diff --git a/tools/testing/selftests/kvm/arm64/vgic_its_save.c b/tools/testing/selftests/kvm/arm64/vgic_its_save.c new file mode 100644 index 000000000000..864da01539f3 --- /dev/null +++ b/tools/testing/selftests/kvm/arm64/vgic_its_save.c @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * vgic_its_save - KVM_DEV_ARM_ITS_SAVE_TABLES against tables a guest broke. + * + * Both cases are reachable by a guest on its own, and neither may fail a save + * that userspace has to be able to issue: + * + * - Changing GITS_BASER drops the collections it described, so the save + * writes nothing but the terminating invalid entry. + * - A device the device table can no longer address is skipped, and the saved + * DTE chain skips it too rather than pointing at an entry never written. + * + * Both cases then reset and restore, which is what the save exists for. + * + * Copyright (c) 2026 Google LLC + * Author: Fuad Tabba + */ + +#include +#include +#include +#include + +#include "kvm_util.h" +#include "gic.h" +#include "gic_v3.h" +#include "gic_v3_its.h" +#include "processor.h" +#include "ucall.h" +#include "vgic.h" + +#define TEST_MEMSLOT_INDEX 1 + +/* All three ITS table entry sizes are 8 bytes in ABI 0. */ +#define ESZ 8 +#define ENTRIES_PER_PAGE (SZ_64K / ESZ) + +/* CTE and DTE layout, mirroring KVM's KVM_ITS_* in arch/arm64/kvm/vgic/vgic.h */ +#define CTE_VALID_MASK BIT_ULL(63) +#define DTE_VALID_MASK BIT_ULL(63) +#define DTE_NEXT_SHIFT 49 +#define DTE_NEXT_MASK GENMASK_ULL(62, 49) + +/* L1 entry of an indirect table: valid bit plus a 64K aligned L2 address. */ +#define L1E_VALID_MASK BIT_ULL(63) +#define L1E_ADDR_MASK GENMASK_ULL(51, 16) + +#define GITS_BASER_PAGES_MASK GENMASK_ULL(7, 0) + +#define POISON 0xdeadbeefdeadbeefULL + +/* The collection table starts at two pages and is shrunk to one. */ +#define COLL_TBL_PAGES 2 +#define COLL_TBL_SZ (COLL_TBL_PAGES * SZ_64K) + +/* One more collection than the shrunken table can hold. */ +#define NR_COLLECTIONS (ENTRIES_PER_PAGE + 1) + +/* Two devices, one per L2 block of the indirect device table. */ +#define DEVICE_A_ID 0 +#define DEVICE_B_ID ENTRIES_PER_PAGE + +/* + * its_send_mapd_cmd() encodes ilog2(itt_size) - 1 as num_eventid_bits, and + * vgic_its_restore_itt() scans BIT_ULL(num_eventid_bits) * ESZ, so the size + * handed to MAPD has to match the ITT allocated for it. + */ +#define ITT_EVENTID_BITS 13 +#define ITT_MAPD_SIZE BIT_ULL(ITT_EVENTID_BITS + 1) +#define ITT_SZ (BIT_ULL(ITT_EVENTID_BITS) * ESZ) + +static struct kvm_vm *vm; +static struct kvm_vcpu *vcpu; +static int its_fd; +static gpa_t gpa_base; + +static struct test_data { + gpa_t device_table; + gpa_t collection_table; + gpa_t cmdq_base; + void *cmdq_base_va; + + gpa_t lpi_prop_table; + gpa_t lpi_pend_table; + + void *device_l1_va; + gpa_t device_l2[2]; + gpa_t itt_tables; +} test_data; + +static unsigned long its_baser_offset(unsigned int type) +{ + int i; + + for (i = 0; i < GITS_BASER_NR_REGS; i++) { + unsigned long offset = GITS_BASER + (i * sizeof(u64)); + u64 baser = readq_relaxed(GITS_BASE_GVA + offset); + + if (GITS_BASER_TYPE(baser) == type) + return offset; + } + + GUEST_FAIL("Couldn't find an ITS BASER of type %u", type); + return -1; +} + +static void its_set_enable(bool enable) +{ + u32 ctlr = readl_relaxed(GITS_BASE_GVA + GITS_CTLR); + + if (enable) + ctlr |= GITS_CTLR_ENABLE; + else + ctlr &= ~GITS_CTLR_ENABLE; + + writel_relaxed(ctlr, GITS_BASE_GVA + GITS_CTLR); +} + +/* + * Shrink the collection table to a single page, leaving VALID set. BASER + * writes are ignored while the ITS is enabled. + */ +static void guest_shrink_coll_table(void) +{ + unsigned long offset = its_baser_offset(GITS_BASER_TYPE_COLLECTION); + u64 baser; + + its_set_enable(false); + + baser = readq_relaxed(GITS_BASE_GVA + offset); + baser &= ~GITS_BASER_PAGES_MASK; + writeq_relaxed(baser, GITS_BASE_GVA + offset); +} + +static void guest_baser_change(void) +{ + u32 coll_id; + + gic_init(GIC_V3, 1); + gic_rdist_enable_lpis(test_data.lpi_prop_table, SZ_64K, + test_data.lpi_pend_table); + + its_init(test_data.collection_table, COLL_TBL_SZ, + test_data.device_table, SZ_64K, + test_data.cmdq_base, SZ_64K); + + for (coll_id = 0; coll_id < NR_COLLECTIONS; coll_id++) + its_send_mapc_cmd(test_data.cmdq_base_va, 0, coll_id, true); + + guest_shrink_coll_table(); + + GUEST_DONE(); +} + +/* Turn the already installed device table into an indirect one. */ +static void guest_make_device_table_indirect(void) +{ + unsigned long offset = its_baser_offset(GITS_BASER_TYPE_DEVICE); + u64 baser; + + its_set_enable(false); + + baser = readq_relaxed(GITS_BASE_GVA + offset); + writeq_relaxed(baser | GITS_BASER_INDIRECT, GITS_BASE_GVA + offset); + + its_set_enable(true); +} + +static void guest_unreachable_device(void) +{ + u64 *l1; + + gic_init(GIC_V3, 1); + gic_rdist_enable_lpis(test_data.lpi_prop_table, SZ_64K, + test_data.lpi_pend_table); + + its_init(test_data.collection_table, SZ_64K, + test_data.device_table, SZ_64K, + test_data.cmdq_base, SZ_64K); + + guest_make_device_table_indirect(); + + /* Both L2 blocks present, so both MAPDs are in range. */ + l1 = test_data.device_l1_va; + l1[0] = L1E_VALID_MASK | (test_data.device_l2[0] & L1E_ADDR_MASK); + l1[1] = L1E_VALID_MASK | (test_data.device_l2[1] & L1E_ADDR_MASK); + + its_send_mapd_cmd(test_data.cmdq_base_va, DEVICE_A_ID, + test_data.itt_tables, ITT_MAPD_SIZE, true); + its_send_mapd_cmd(test_data.cmdq_base_va, DEVICE_B_ID, + test_data.itt_tables + ITT_SZ, ITT_MAPD_SIZE, true); + + /* + * Drop the block holding device B. No ITS command and no GITS_BASER + * write is involved, so nothing tells KVM the device is now + * unreachable. + */ + l1[1] = 0; + + GUEST_DONE(); +} + +static void run_guest(void) +{ + struct ucall uc; + + vcpu_run(vcpu); + switch (get_ucall(vcpu, &uc)) { + case UCALL_DONE: + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + default: + TEST_FAIL("Unexpected ucall: %lu", uc.cmd); + } +} + +static int save_tables(void) +{ + return __kvm_device_attr_set(its_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_ITS_SAVE_TABLES, NULL); +} + +static u64 its_reg_get(unsigned long offset) +{ + u64 val; + + kvm_device_attr_get(its_fd, KVM_DEV_ARM_VGIC_GRP_ITS_REGS, offset, + &val); + return val; +} + +static void its_reg_set(unsigned long offset, u64 val) +{ + kvm_device_attr_set(its_fd, KVM_DEV_ARM_VGIC_GRP_ITS_REGS, offset, + &val); +} + +/* + * What a migration target does with the saved tables, in the order + * Documentation/virt/kvm/devices/arm-vgic-its.rst gives: the GITS_ registers + * first, then the tables. The reset in between clears GITS_BASER.Valid, + * which is why the registers have to be written back before the restore. + */ +static void reset_and_restore_tables(void) +{ + u64 baser[GITS_BASER_NR_REGS]; + int ret, i; + + for (i = 0; i < GITS_BASER_NR_REGS; i++) + baser[i] = its_reg_get(GITS_BASER + (i * sizeof(u64))); + + ret = __kvm_device_attr_set(its_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_ITS_CTRL_RESET, NULL); + TEST_ASSERT(!ret, "Expected the reset to succeed, got ret %d errno %d", + ret, errno); + + for (i = 0; i < GITS_BASER_NR_REGS; i++) + its_reg_set(GITS_BASER + (i * sizeof(u64)), baser[i]); + + ret = __kvm_device_attr_set(its_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_ITS_RESTORE_TABLES, NULL); + TEST_ASSERT(!ret, "Expected the restore to succeed, got ret %d errno %d", + ret, errno); +} + +static void poison_range(gpa_t base, size_t size) +{ + u64 *entry = addr_gpa2hva(vm, base); + size_t i; + + for (i = 0; i < size / ESZ; i++) + entry[i] = POISON; +} + +static void setup_memslot(size_t sz) +{ + size_t pages = sz / vm->page_size; + + gpa_base = ((vm_compute_max_gfn(vm) + 1) * vm->page_size) - sz; + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, gpa_base, + TEST_MEMSLOT_INDEX, pages, 0); +} + +static gpa_t alloc_64k(size_t nr) +{ + size_t pages_per_64k = vm_calc_num_guest_pages(vm->mode, SZ_64K); + gpa_t gpa = vm_phy_pages_alloc(vm, nr * pages_per_64k, gpa_base, + TEST_MEMSLOT_INDEX); + + TEST_ASSERT(IS_ALIGNED(gpa, SZ_64K), + "Allocation at 0x%lx is not 64K aligned, GITS_BASER cannot address it", + gpa); + return gpa; +} + +static void map_to_guest(gpa_t gpa, size_t nr) +{ + size_t pages_per_64k = vm_calc_num_guest_pages(vm->mode, SZ_64K); + + virt_map(vm, gpa, gpa, nr * pages_per_64k); +} + +static void setup_common(void) +{ + test_data.cmdq_base = alloc_64k(1); + map_to_guest(test_data.cmdq_base, 1); + test_data.cmdq_base_va = (void *)test_data.cmdq_base; + + test_data.lpi_prop_table = alloc_64k(1); + test_data.lpi_pend_table = alloc_64k(1); +} + +static void teardown(void) +{ + close(its_fd); + kvm_vm_free(vm); + memset(&test_data, 0, sizeof(test_data)); +} + +/* + * A GITS_BASER write that changes the table drops the collections it + * described. The save then has an empty list, so it writes the terminating + * invalid entry and nothing else. + */ +static void test_baser_change_drops_collections(void) +{ + u64 *cte; + int ret, i; + + pr_info("Testing that a GITS_BASER change drops the collections\n"); + + vm = vm_create_with_one_vcpu(&vcpu, guest_baser_change); + setup_memslot((4 + COLL_TBL_PAGES) * SZ_64K); + its_fd = vgic_its_setup(vm); + + test_data.device_table = alloc_64k(1); + test_data.collection_table = alloc_64k(COLL_TBL_PAGES); + setup_common(); + + sync_global_to_guest(vm, test_data); + run_guest(); + + /* Anything KVM writes is then the only thing that changed. */ + poison_range(test_data.collection_table, COLL_TBL_SZ); + + ret = save_tables(); + TEST_ASSERT(!ret, "Expected the save to succeed, got %d errno %d", + ret, errno); + + cte = addr_gpa2hva(vm, test_data.collection_table); + + /* + * Finding the terminator at the head of the table is also what proves + * the reads below landed in the saved table rather than elsewhere. + */ + TEST_ASSERT(le64toh(cte[0]) == 0, + "CTE 0: expected the terminating invalid entry, got 0x%llx", + (unsigned long long)le64toh(cte[0])); + + for (i = 1; i < COLL_TBL_SZ / ESZ; i++) + TEST_ASSERT(cte[i] == POISON, + "CTE %d: expected it untouched, got 0x%llx", + i, (unsigned long long)cte[i]); + + reset_and_restore_tables(); + + teardown(); +} + +/* + * A device whose L2 block the guest dropped is skipped by the save, and the + * DTE chain skips it too: left alone, the surviving device would point at an + * entry the save never wrote. + */ +static void test_unreachable_device_skipped(void) +{ + u64 dte; + int ret; + + pr_info("Testing that an unreachable device is skipped by the save\n"); + + vm = vm_create_with_one_vcpu(&vcpu, guest_unreachable_device); + setup_memslot(9 * SZ_64K); + its_fd = vgic_its_setup(vm); + + test_data.device_table = alloc_64k(1); + test_data.collection_table = alloc_64k(1); + test_data.device_l2[0] = alloc_64k(1); + test_data.device_l2[1] = alloc_64k(1); + test_data.itt_tables = alloc_64k(2); + setup_common(); + + map_to_guest(test_data.device_table, 1); + test_data.device_l1_va = (void *)test_data.device_table; + + sync_global_to_guest(vm, test_data); + run_guest(); + + poison_range(test_data.device_l2[0], SZ_64K); + poison_range(test_data.device_l2[1], SZ_64K); + + ret = save_tables(); + TEST_ASSERT(!ret, "Expected the save to succeed, got %d errno %d", + ret, errno); + + dte = le64toh(*(u64 *)addr_gpa2hva(vm, test_data.device_l2[0])); + + /* Device A is still reachable, so it is saved. */ + TEST_ASSERT(dte & DTE_VALID_MASK, + "Device A: expected a valid DTE, got 0x%llx", + (unsigned long long)dte); + + /* + * Device B is the only device after it and was skipped, so nothing + * follows A in the saved chain. + */ + TEST_ASSERT(FIELD_GET(DTE_NEXT_MASK, dte) == 0, + "Device A: expected no next device, got offset %llu", + (unsigned long long)FIELD_GET(DTE_NEXT_MASK, dte)); + + /* And nothing was written into the block the guest dropped. */ + TEST_ASSERT(*(u64 *)addr_gpa2hva(vm, test_data.device_l2[1]) == POISON, + "Device B: expected its entry untouched"); + + reset_and_restore_tables(); + + teardown(); +} + +int main(void) +{ + TEST_REQUIRE(kvm_supports_vgic_v3()); + + test_baser_change_drops_collections(); + test_unreachable_device_skipped(); + + pr_info("All ok!\n"); + return 0; +} From 38b70fc453c3112f1a62583b89903ae41116cc27 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Tue, 1 Sep 2026 18:28:59 +0100 Subject: [PATCH 30/54] KVM: arm64: Fix spurious warning for benign stage 2 teardown race kvmtool was used to establish an L1 guest with 8 CPUs and 8 GiB of RAM, an L2 guest with 4 CPUs and 4 GiB of RAM and an L3 guest with 2 CPUs and 2 GiB of RAM, all of which was then exited. Under memory pressure in the L0 host warnings were observed due to migration triggered by compaction: WARNING: arch/arm64/kvm/mmu.c:336 at __unmap_stage2_range+0x64/0x80, CPU#5: kcompactd0/66 Which was, in turn, triggered by an MMU notifier for the host invalidation: mmu_notifier_invalidate_range_start() -> ... -> kvm_mmu_notifier_invalidate_range_start() -> kvm_mmu_unmap_gfn_range() -> kvm_unmap_gfn_range() -> kvm_nested_s2_unmap() -> kvm_stage2_unmap_range() -> __unmap_stage2_range() -> stage2_apply_range() <- -EINVAL, triggering a WARN_ON() Racing with L0's teardown of stage 2 page tables: exit_mm() -> mmput() -> __mmput() -> exit_mmap() -> mmu_notifier_release() -> ... -> kvm_mmu_notifier_release() -> kvm_flush_shadow_all() -> kvm_arch_flush_shadow_all() -> kvm_free_stage2_pgd() -> [ acquire kvm->mmu_lock for write ] -> mmu->pgt = NULL [ among other tasks ] -> [ release kvm->mmu_lock for write ] It turns out there is a benign race resulting in a spurious warning: Thread A - notify: migration | Thread B - notify: release -------------------------------|--------------------------------- < kvm->mmu_lock held > | stage2_apply_range() | get mmu->pgt, check !NULL | ... | kvm_arch_flush_shadow_all() cond_resched_rwlock_write(); | < contend, sleep kvm->mmu_lock > < drop kvm->mmu_lock > | < acquire kvm->mmu_lock> | ... | kvm_free_stage2_pgd() | mmu->pgt = NULL | < invalidate MMU > | ... | < release kvm->mmu_lock > [ scheduled ] | stage2_apply_range() | < loop to next > | get, mmu->pgt, check !NULL | is NULL, return -EINVAL | __unmap_stage2_range() | WARN_ON(-EINVAL) <--- entirely spurious - the race was handled correctly. Fix the spurious warning by updating stage2_apply_range() to no longer treat concurrent PGT teardown on lock release as an error - whether the walker is tearing down page tables or doing something else this is a legitimate reason to abort the operation without error. This keeps the warning in place for all other circumstances. In practice only __unmap_stage2_range() actually does anything with the error so this only impacts that. Fixes: ec14c272408a ("KVM: arm64: nv: Unmap/flush shadow stage 2 page tables") Cc: stable@vger.kernel.org Reviewed-by: Yuan Yao Reviewed-by: Marc Zyngier Signed-off-by: Lorenzo Stoakes (ARM) Link: https://patch.msgid.link/20260901-kvm-arm-nested-virt-fix-v3-1-b154676f7e4c@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/mmu.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 9ba86450fe4a..2d44cd6a5aed 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -59,27 +59,36 @@ static phys_addr_t stage2_range_addr_end(phys_addr_t addr, phys_addr_t end) * long will also starve other vCPUs. We have to also make sure that the page * tables are not freed while we released the lock. */ -static int stage2_apply_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, +static int stage2_apply_range(struct kvm_s2_mmu *mmu, phys_addr_t start, phys_addr_t end, int (*fn)(struct kvm_pgtable *, u64, u64), bool resched) { struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + bool lock_dropped = false; + phys_addr_t addr = start; int ret; u64 next; do { struct kvm_pgtable *pgt = mmu->pgt; + /* + * We may be raced on PGT teardown when we release the + * kvm->mmu_lock. That's fine as the PGT is legitimately no + * longer present. + */ if (!pgt) - return -EINVAL; + return lock_dropped ? 0 : -EINVAL; next = stage2_range_addr_end(addr, end); ret = fn(pgt, addr, next - addr); if (ret) break; - if (resched && next != end) + if (resched && next != end) { cond_resched_rwlock_write(&kvm->mmu_lock); + lock_dropped = true; + } } while (addr = next, addr != end); return ret; From 4c74e233cdedd11592775fae2a6243e67ca3f891 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Tue, 1 Sep 2026 18:29:00 +0100 Subject: [PATCH 31/54] KVM: arm64: nv: Fix null ptr deref on nested wp/unmap, teardown race Commit 7270cc9157f4 ("KVM: arm64: nv: Handle VNCR_EL2 invalidation from MMU notifiers") introduced VNCR_EL2 invalidation in both kvm_nested_s2_unmap() and kvm_nested_s2_wp(). However at the point of this being performed concurrent stage 2 teardown of a nested guest can cause kvm->arch.mmu.pgt to be set to NULL. This happens in kvm_flush_shadow_all() -> kvm_arch_flush_shadow_all() -> kvm_free_stage2_pgd() and is performed under the kvm->mmu_lock. Commit ec14c272408a ("KVM: arm64: nv: Unmap/flush shadow stage 2 page tables") introduced the teardown of the entire nested MMU range, which then invokes stage2_apply_range() with resched=true: mmu_notifier_invalidate_range_start() -> ... -> kvm_mmu_notifier_invalidate_range_start() -> kvm_mmu_unmap_gfn_range() -> kvm_unmap_gfn_range() -> kvm_nested_s2_unmap() -> kvm_stage2_unmap_range() -> __unmap_stage2_range() -> stage2_apply_range() This means that stage2_apply_range() can drop the kvm->mmu_lock and thus concurrent progress can be made in lockstep with kvm_arch_flush_shadow_all(). If kvm_arch_flush_shadow_all() advances ahead of stage2_apply_range() and completes its operation it guarantees a NULL pointer deref. Since kvm_free_stage2_pgd() is performed under the kvm->mmu_lock this will either be observed NULL or not and serialised against kvm_free_stage2_pgd(). Resolve the issue by abstracting the invalidation to a new function, kvm_invalidate_vncr_ipa_all(), and check that the pgt is non-NULL before dereferencing it. Fixes: 7270cc9157f4 ("KVM: arm64: nv: Handle VNCR_EL2 invalidation from MMU notifiers") Cc: stable@vger.kernel.org Reviewed-by: Marc Zyngier Signed-off-by: Lorenzo Stoakes (ARM) Tested-by: Jonathan Davies Link: https://patch.msgid.link/20260901-kvm-arm-nested-virt-fix-v3-2-b154676f7e4c@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/nested.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 3c4fc566eafc..a0808391a456 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1260,6 +1260,17 @@ void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val) invalidate_vncr_va(vcpu->kvm, &scope); } +static void kvm_invalidate_vncr_ipa_all(struct kvm *kvm) +{ + struct kvm_pgtable *pgt = kvm->arch.mmu.pgt; + + lockdep_assert_held_write(&kvm->mmu_lock); + + /* if the mmu lock was dropped, pgt teardown may have raced. */ + if (pgt) + kvm_invalidate_vncr_ipa(kvm, 0, BIT(pgt->ia_bits)); +} + void kvm_nested_s2_wp(struct kvm *kvm) { int i; @@ -1276,7 +1287,7 @@ void kvm_nested_s2_wp(struct kvm *kvm) kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu)); } - kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits)); + kvm_invalidate_vncr_ipa_all(kvm); } void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) @@ -1295,7 +1306,7 @@ void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block); } - kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits)); + kvm_invalidate_vncr_ipa_all(kvm); } void kvm_nested_s2_flush(struct kvm *kvm) From 2a2eb10795a1e495aebc7f829ccecb72c05b4fd9 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 25 Aug 2026 09:59:45 +0100 Subject: [PATCH 32/54] KVM: arm64: Validate the SVE vector length in pkvm_vcpu_init_sve() pkvm_vcpu_init_sve() clamps only the upper bound of the host-provided sve_max_vl, so an invalid vector length reaches sve_state_size_from_vl() and the WARN_ON() there, which is fatal at EL2. The existing !sve_state_size test rejects such a length, but only after the macro has run. Check sve_vl_valid() before deriving the state size. A valid length cannot yield a zero size, so the !sve_state_size test goes with it. Fixes: 5db1bef93342 ("KVM: arm64: Track SVE state in the hypervisor vcpu structure") Reported-by: Stefan Teodorescu Reviewed-by: Marc Zyngier Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260825085948.1674721-2-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/pkvm.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 459bd9eb7e4b..627f13fc98d4 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -459,14 +459,15 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h /* Limit guest vector length to the maximum supported by the host. */ sve_max_vl = min(READ_ONCE(host_vcpu->arch.sve_max_vl), kvm_host_sve_max_vl); - sve_state_size = sve_state_size_from_vl(sve_max_vl); sve_state = kern_hyp_va(READ_ONCE(host_vcpu->arch.sve_state)); - if (!sve_state || !sve_state_size) { + if (!sve_vl_valid(sve_max_vl) || !sve_state) { ret = -EINVAL; goto err; } + sve_state_size = sve_state_size_from_vl(sve_max_vl); + ret = hyp_pin_shared_mem(sve_state, sve_state + sve_state_size); if (ret) goto err; From a1b3c788ad31837e348075e93dbba3f447492776 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 25 Aug 2026 09:59:46 +0100 Subject: [PATCH 33/54] KVM: arm64: Do not clear VM-wide SVE feature on vCPU init failure pkvm_vcpu_init_sve() clears KVM_ARM_VCPU_SVE in kvm->arch.vcpu_features when it fails, but vcpu_has_sve() tests KVM_ARCH_FLAG_GUEST_HAS_SVE, which is left set. Later vCPUs on that VM then skip the SVE setup and register with a NULL sve_state, which the guest's first FP access hands to sve_load_state(). Return the error without touching vcpu_features. Fixes: 5db1bef93342 ("KVM: arm64: Track SVE state in the hypervisor vcpu structure") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260825085948.1674721-3-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/pkvm.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 627f13fc98d4..4857a11d4292 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -450,7 +450,7 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h unsigned int sve_max_vl; size_t sve_state_size; void *sve_state; - int ret = 0; + int ret; if (!vcpu_has_feature(vcpu, KVM_ARM_VCPU_SVE)) { vcpu_clear_flag(vcpu, VCPU_SVE_FINALIZED); @@ -461,24 +461,19 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h sve_max_vl = min(READ_ONCE(host_vcpu->arch.sve_max_vl), kvm_host_sve_max_vl); sve_state = kern_hyp_va(READ_ONCE(host_vcpu->arch.sve_state)); - if (!sve_vl_valid(sve_max_vl) || !sve_state) { - ret = -EINVAL; - goto err; - } + if (!sve_vl_valid(sve_max_vl) || !sve_state) + return -EINVAL; sve_state_size = sve_state_size_from_vl(sve_max_vl); ret = hyp_pin_shared_mem(sve_state, sve_state + sve_state_size); if (ret) - goto err; + return ret; vcpu->arch.sve_state = sve_state; vcpu->arch.sve_max_vl = sve_max_vl; return 0; -err: - clear_bit(KVM_ARM_VCPU_SVE, vcpu->kvm->arch.vcpu_features); - return ret; } static int vm_copy_id_regs(struct pkvm_hyp_vcpu *hyp_vcpu) From 0d62fbf34d8fe7a3ff56692d39fbb8e6e9bf15fb Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 25 Aug 2026 09:59:47 +0100 Subject: [PATCH 34/54] KVM: arm64: Key unpin_host_sve_state() on the state it unpins unpin_host_sve_state() gates on the VM's SVE feature bit, but what it unpins is the state pkvm_vcpu_init_sve() pinned. A vCPU that completed init has sve_state set exactly when that bit is set, so the two agree. Gate on sve_state, which is what is being unpinned. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260825085948.1674721-4-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/pkvm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 4857a11d4292..453d3fa4a420 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -398,10 +398,10 @@ static void unpin_host_sve_state(struct pkvm_hyp_vcpu *hyp_vcpu) { void *sve_state; - if (!vcpu_has_feature(&hyp_vcpu->vcpu, KVM_ARM_VCPU_SVE)) + sve_state = hyp_vcpu->vcpu.arch.sve_state; + if (!sve_state) return; - sve_state = hyp_vcpu->vcpu.arch.sve_state; hyp_unpin_shared_mem(sve_state, sve_state + vcpu_sve_state_size(&hyp_vcpu->vcpu)); } From 4f16c5fc8dc4c5596e3777ab9f449a54e3f85fd5 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 25 Aug 2026 09:59:48 +0100 Subject: [PATCH 35/54] KVM: arm64: Derive GUEST_HAS_SVE from the SVE feature bit at EL2 pkvm_init_features_from_host() takes KVM_ARCH_FLAG_GUEST_HAS_SVE and KVM_ARM_VCPU_SVE from the host separately, but pkvm_vcpu_init_sve() tests the bit while vcpu_has_sve() reads the flag. A host that sets the flag without the bit gets a vCPU with a NULL sve_state that the world switch loads the guest's SVE state from. Derive the flag from the bit, and drop the protected path's copy of the host's flag, which is dead code since protected VMs are not allowed SVE. Fixes: 41d6028e28bd ("KVM: arm64: Convert the SVE guest vcpu flag to a vm flag") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260825085948.1674721-5-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/pkvm.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 453d3fa4a420..bb3e0dc0676e 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -360,7 +360,7 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc if (test_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &host_arch_flags)) hyp_vm->kvm.arch.midr_el1 = host_kvm->arch.midr_el1; - return; + goto out; } if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_MTE)) @@ -379,13 +379,14 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_GENERIC)) set_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, allowed_features); - if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE)) { + if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE)) set_bit(KVM_ARM_VCPU_SVE, allowed_features); - kvm->arch.flags |= host_arch_flags & BIT(KVM_ARCH_FLAG_GUEST_HAS_SVE); - } bitmap_and(kvm->arch.vcpu_features, host_kvm->arch.vcpu_features, allowed_features, KVM_VCPU_MAX_FEATURES); +out: + __assign_bit(KVM_ARCH_FLAG_GUEST_HAS_SVE, &kvm->arch.flags, + kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_SVE)); } static void unpin_host_vcpu(struct kvm_vcpu *host_vcpu) From 64dc6f1db7e620f2e9337bb181f305fb0561da79 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 29 Aug 2026 07:48:55 +0200 Subject: [PATCH 36/54] KVM: arm64: Return -EINVAL for an empty SMCCC filter range at base 0 kvm_smccc_set_filter() only rejects a range if its inclusive end, base + nr_functions - 1, is below base. That catches an empty range (nr_functions == 0) at every nonzero base, but at base 0 the end wraps to U32_MAX and KVM tries to insert [0, U32_MAX], which overlaps the reserved Arm Architecture Calls ranges. KVM_ARM_VM_SMCCC_FILTER then returns -EEXIST instead of the -EINVAL that the smccc_filter selftest expects for an empty range. Reject a zero function count explicitly. Tested with a userspace reproducer on an arm64 VHE host under QEMU TCG: EEXIST before, EINVAL after. Fixes: 821d935c87bc ("KVM: arm64: Introduce support for userspace SMCCC filtering") Assisted-by: LLM Signed-off-by: Karl Mehltretter Reviewed-by: Steffen Eiden Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260829054856.70549-2-kmehltretter@gmail.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/hypercalls.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index b11b8821c9fb..dfa25bb6f25d 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -185,7 +185,8 @@ static int kvm_smccc_set_filter(struct kvm *kvm, struct kvm_smccc_filter __user start = filter.base; end = start + filter.nr_functions - 1; - if (end < start || filter.action >= NR_SMCCC_FILTER_ACTIONS) + if (!filter.nr_functions || end < start || + filter.action >= NR_SMCCC_FILTER_ACTIONS) return -EINVAL; mutex_lock(&kvm->arch.config_lock); From 0a46eb5719fa57cc9d06025dcd8ba271643dee61 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 29 Aug 2026 07:48:56 +0200 Subject: [PATCH 37/54] KVM: arm64: selftests: Test empty SMCCC filter range at base 0 test_invalid_nr_functions() only checks an empty range at PSCI_0_2_FN64_CPU_ON, which KVM's end < start check happens to catch. It never exercised base 0, where the inclusive end wraps to U32_MAX instead. Add the base 0 case. Without the preceding fix it fails with EEXIST. Assisted-by: LLM Signed-off-by: Karl Mehltretter Reviewed-by: Steffen Eiden Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260829054856.70549-3-kmehltretter@gmail.com Signed-off-by: Oliver Upton --- tools/testing/selftests/kvm/arm64/smccc_filter.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/testing/selftests/kvm/arm64/smccc_filter.c b/tools/testing/selftests/kvm/arm64/smccc_filter.c index 21e41880261b..a41ed3e016ba 100644 --- a/tools/testing/selftests/kvm/arm64/smccc_filter.c +++ b/tools/testing/selftests/kvm/arm64/smccc_filter.c @@ -140,6 +140,10 @@ static void test_invalid_nr_functions(void) TEST_ASSERT(r < 0 && errno == EINVAL, "Attempt to filter 0 functions should return EINVAL"); + r = __set_smccc_filter(vm, 0, 0, KVM_SMCCC_FILTER_DENY); + TEST_ASSERT(r < 0 && errno == EINVAL, + "Attempt to filter 0 functions at base 0 should return EINVAL"); + kvm_vm_free(vm); } From 3a8c562892b96f35bba1e00d5e455a15963bbb92 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 8 Sep 2026 12:07:10 +0100 Subject: [PATCH 38/54] KVM: arm64: Transfer the hyp stack pages out of the host stage-2 fix_host_ownership() walks only the linear-map alias of each memblock region, and the per-CPU hyp stack, mapped in the private VA range for its guard page, has none. Walk each stack's VA range with the same walker. Fixes: 1a919b17ef012 ("KVM: arm64: Add guard pages for pKVM (protected nVHE) hypervisor stack") Reported-by: Hiroyuki Katsura Cc: stable@vger.kernel.org Signed-off-by: Fuad Tabba Reviewed-by: Vincent Donnefort Tested-by: Vincent Donnefort Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260908110713.1540304-2-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/setup.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 75b00c323310..362f2891cb32 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -269,6 +269,16 @@ static int fix_host_ownership(void) return ret; } + /* The stacks sit in the private VA range, not the linear map. */ + for (i = 0; i < hyp_nr_cpus; i++) { + struct kvm_nvhe_init_params *params = per_cpu_ptr(&kvm_init_params, i); + u64 start = params->stack_hyp_va - NVHE_STACK_SIZE; + + ret = kvm_pgtable_walk(&pkvm_pgtable, start, NVHE_STACK_SIZE, &walker); + if (ret) + return ret; + } + return 0; } From 5a8b505ede133fb30ca3b3a19d0db00c08237615 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 8 Sep 2026 12:07:11 +0100 Subject: [PATCH 39/54] KVM: arm64: Match hyp text by physical address in fix_host_ownership() On a non-hVHE host, fix_host_ownership_walker()'s test for PAGE_HYP_EXEC never matches: KVM_PGTABLE_PROT_UX is cleared at map time and only PX is reported on read-back. Hyp text is therefore donated rather than left read-only in the host stage-2, and the instruction dump in nvhe_hyp_panic_handler() reads a page the host has no access to. Match the text by physical address instead, in a helper a later patch reuses. A test on the permissions would leave any other executable mapping host-readable too. Fixes: 80cbfd7174f31 ("KVM: arm64: Honor UX/PX attributes for EL2 S1 mappings") Signed-off-by: Fuad Tabba Reviewed-by: Vincent Donnefort Tested-by: Vincent Donnefort Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260908110713.1540304-3-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/include/nvhe/mem_protect.h | 1 + arch/arm64/kvm/hyp/nvhe/mem_protect.c | 8 ++++++++ arch/arm64/kvm/hyp/nvhe/setup.c | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index 29935c7da1de..cab27f7bd423 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -52,6 +52,7 @@ int __pkvm_host_test_clear_young_guest(u64 gfn, u64 nr_pages, bool mkold, struct int __pkvm_host_mkyoung_guest(u64 gfn, struct pkvm_hyp_vcpu *vcpu); bool addr_is_memory(phys_addr_t phys); +bool addr_is_hyp_text(phys_addr_t phys); int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot prot); int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id); int kvm_host_prepare_stage2(void *pgt_pool_base); diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 39aa8911f62c..d026f446bd8e 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -450,6 +450,14 @@ bool addr_is_memory(phys_addr_t phys) return !!find_mem_range(phys, &range); } +bool addr_is_hyp_text(phys_addr_t phys) +{ + phys_addr_t start = ALIGN_DOWN(__hyp_pa(__hyp_text_start), PAGE_SIZE); + phys_addr_t end = PAGE_ALIGN(__hyp_pa(__hyp_text_end)); + + return phys >= start && phys < end; +} + static bool is_in_mem_range(u64 addr, struct kvm_mem_range *range) { return range->start <= addr && addr < range->end; diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 362f2891cb32..bb667cd7080b 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -217,7 +217,7 @@ static int fix_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx, case PKVM_PAGE_OWNED: set_hyp_state(page, PKVM_PAGE_OWNED); /* hyp text is RO in the host stage-2 to be inspected on panic. */ - if (prot == PAGE_HYP_EXEC) { + if (addr_is_hyp_text(phys)) { set_host_state(page, PKVM_NOPAGE); return host_stage2_idmap_locked(phys, PAGE_SIZE, KVM_PGTABLE_PROT_R); } else { From 2245841401147b8022a5857061b870d82e595352 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 8 Sep 2026 12:07:12 +0100 Subject: [PATCH 40/54] KVM: arm64: Move the private VA allocation cursor to __io_map_next __io_map_base is the start of the private VA range only until the first allocation from it, after which it is the allocation cursor. Keep it as the start and move the cursor to __io_map_next, for the walk of the range the next patch adds. No functional change intended. Signed-off-by: Fuad Tabba Reviewed-by: Vincent Donnefort Tested-by: Vincent Donnefort Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260908110713.1540304-4-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/mm.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 3b0bee496bff..422ee57be956 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -25,6 +25,7 @@ struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS]; unsigned int hyp_memblock_nr; static u64 __io_map_base; +static u64 __io_map_next; struct hyp_fixmap_slot { u64 addr; @@ -50,7 +51,7 @@ static int __pkvm_alloc_private_va_range(unsigned long start, size_t size) hyp_assert_lock_held(&pkvm_pgd_lock); - if (!start || start < __io_map_base) + if (!start || start < __io_map_next) return -EINVAL; /* The allocated size is always a multiple of PAGE_SIZE */ @@ -60,7 +61,7 @@ static int __pkvm_alloc_private_va_range(unsigned long start, size_t size) if (cur > __hyp_vmemmap) return -ENOMEM; - __io_map_base = cur; + __io_map_next = cur; return 0; } @@ -70,7 +71,7 @@ static int __pkvm_alloc_private_va_range(unsigned long start, size_t size) * @size: The size of the VA range to reserve. * @haddr: The hypervisor virtual start address of the allocation. * - * The private virtual address (VA) range is allocated above __io_map_base + * The private virtual address (VA) range is allocated above __io_map_next * and aligned based on the order of @size. * * Return: 0 on success or negative error code on failure. @@ -81,7 +82,7 @@ int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr) int ret; hyp_spin_lock(&pkvm_pgd_lock); - addr = __io_map_base; + addr = __io_map_next; ret = __pkvm_alloc_private_va_range(addr, size); hyp_spin_unlock(&pkvm_pgd_lock); @@ -341,7 +342,7 @@ static int create_fixblock(void) return -EINVAL; hyp_spin_lock(&pkvm_pgd_lock); - addr = ALIGN(__io_map_base, PMD_SIZE); + addr = ALIGN(__io_map_next, PMD_SIZE); ret = __pkvm_alloc_private_va_range(addr, PMD_SIZE); if (ret) goto unlock; @@ -426,6 +427,7 @@ int hyp_create_idmap(u32 hyp_va_bits) */ __io_map_base = start & BIT(hyp_va_bits - 2); __io_map_base ^= BIT(hyp_va_bits - 2); + __io_map_next = __io_map_base; __hyp_vmemmap = __io_map_base | BIT(hyp_va_bits - 3); return __pkvm_create_mappings(start, end - start, start, PAGE_HYP_EXEC); @@ -433,19 +435,19 @@ int hyp_create_idmap(u32 hyp_va_bits) int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr) { - unsigned long addr, prev_base; + unsigned long addr, prev_next; size_t size; int ret; hyp_spin_lock(&pkvm_pgd_lock); - prev_base = __io_map_base; + prev_next = __io_map_next; /* * Efficient stack verification using the NVHE_STACK_SHIFT bit implies * an alignment of our allocation on the order of the size. */ size = NVHE_STACK_SIZE * 2; - addr = ALIGN(__io_map_base, size); + addr = ALIGN(__io_map_next, size); ret = __pkvm_alloc_private_va_range(addr, size); if (!ret) { @@ -461,7 +463,7 @@ int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr) ret = kvm_pgtable_hyp_map(&pkvm_pgtable, addr + NVHE_STACK_SIZE, NVHE_STACK_SIZE, phys, PAGE_HYP); if (ret) - __io_map_base = prev_base; + __io_map_next = prev_next; } hyp_spin_unlock(&pkvm_pgd_lock); From cfe80c3837f93202970b6d8f706f79c5c7396f17 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 8 Sep 2026 12:07:13 +0100 Subject: [PATCH 41/54] KVM: arm64: Check every private mapping is hyp-owned at pKVM init fix_host_ownership() transfers only what it walks, so a hyp mapping outside the linear map is not manipulated by the walk. Walk the quarter of the VA space holding the private range and the vmemmap once the transfer is done, and fail init unless every valid leaf is hyp-owned: in the vmemmap when the page is memory, and in the host stage-2, where hyp text may instead be mapped without write access. A leaf that is not memory has no vmemmap entry and is checked against the host stage-2 alone. Hyp text is matched by physical address, since the only executable mapping in the range is the Spectre-v3a vectors, whose VA is a private allocation, and an executable mapping of anything else must not be host-readable. The vmemmap can be block-mapped, so the walker checks each page of a leaf. Suggested-by: Will Deacon Signed-off-by: Fuad Tabba Reviewed-by: Vincent Donnefort Tested-by: Vincent Donnefort Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260908110713.1540304-5-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/include/nvhe/mem_protect.h | 1 + arch/arm64/kvm/hyp/include/nvhe/mm.h | 1 + arch/arm64/kvm/hyp/nvhe/mem_protect.c | 12 ++++ arch/arm64/kvm/hyp/nvhe/mm.c | 59 +++++++++++++++++++ arch/arm64/kvm/hyp/nvhe/setup.c | 4 ++ 5 files changed, 77 insertions(+) diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index cab27f7bd423..ec85a9547120 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -55,6 +55,7 @@ bool addr_is_memory(phys_addr_t phys); bool addr_is_hyp_text(phys_addr_t phys); int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot prot); int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id); +bool host_stage2_pte_is_hyp_owned(kvm_pte_t pte); int kvm_host_prepare_stage2(void *pgt_pool_base); int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd); void kvm_guest_destroy_stage2(struct pkvm_hyp_vm *vm); diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h index 6e83ce35c2f2..31cae95ddb71 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h @@ -29,6 +29,7 @@ int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, enum kvm_pgtable_prot prot, unsigned long *haddr); int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr); +int pkvm_check_host_ownership(void); int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr); #endif /* __KVM_HYP_MM_H */ diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index d026f446bd8e..a6a47c1e058b 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -641,6 +641,18 @@ int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id) return ret; } +bool host_stage2_pte_is_hyp_owned(kvm_pte_t pte) +{ + if (kvm_pte_valid(pte)) + return false; + + if (FIELD_GET(KVM_INVALID_PTE_TYPE_MASK, pte) != + KVM_HOST_INVALID_PTE_TYPE_DONATION) + return false; + + return FIELD_GET(KVM_HOST_DONATION_PTE_OWNER_MASK, pte) == PKVM_ID_HYP; +} + #define KVM_HOST_PTE_OWNER_GUEST_HANDLE_MASK GENMASK(15, 0) /* We need 40 bits for the GFN to cover a 52-bit IPA with 4k pages and LPA2 */ #define KVM_HOST_PTE_OWNER_GUEST_GFN_MASK GENMASK(55, 16) diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 422ee57be956..29ab5ee9d57f 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -472,6 +472,65 @@ int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr) return ret; } +static int check_page_ownership(phys_addr_t phys) +{ + kvm_pte_t pte; + bool host_ok; + int ret; + + if (addr_is_memory(phys)) { + struct hyp_page *page = hyp_phys_to_page(phys); + + if (get_hyp_state(page) != PKVM_PAGE_OWNED || + get_host_state(page) != PKVM_NOPAGE) + return -EPERM; + } + + ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, NULL); + if (ret) + return ret; + + /* Hyp text may stay host-readable, see fix_host_ownership_walker(). */ + if (kvm_pte_valid(pte) && addr_is_hyp_text(phys)) + host_ok = !(kvm_pgtable_stage2_pte_prot(pte) & KVM_PGTABLE_PROT_W); + else + host_ok = host_stage2_pte_is_hyp_owned(pte); + + return host_ok ? 0 : -EPERM; +} + +static int check_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx, + enum kvm_pgtable_walk_flags visit) +{ + phys_addr_t phys, end; + int ret; + + if (!kvm_pte_valid(ctx->old)) + return 0; + + phys = kvm_pte_to_phys(ctx->old); + end = phys + kvm_granule_size(ctx->level); + for (; phys < end; phys += PAGE_SIZE) { + ret = check_page_ownership(phys); + if (ret) + return ret; + } + + return 0; +} + +int pkvm_check_host_ownership(void) +{ + struct kvm_pgtable_walker walker = { + .cb = check_host_ownership_walker, + .flags = KVM_PGTABLE_WALK_LEAF, + }; + + /* The private range and the vmemmap share one quarter of the VA space. */ + return kvm_pgtable_walk(&pkvm_pgtable, __io_map_base, + BIT(pkvm_pgtable.ia_bits - 2), &walker); +} + static void *admit_host_page(void *arg) { struct kvm_hyp_memcache *host_mc = arg; diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index bb667cd7080b..45ac5f2ba4f7 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -334,6 +334,10 @@ void __noreturn __pkvm_init_finalise(void) if (ret) goto out; + ret = pkvm_check_host_ownership(); + if (ret) + goto out; + ret = hyp_ffa_init(ffa_proxy_pages); if (ret) goto out; From 33346f8960c7bb6a3b4e273b5cfe25c5a8be349f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 11 Sep 2026 17:22:02 +0100 Subject: [PATCH 42/54] KVM: arm64: nv: Fix life cycle of the nested_mmus array The nested_mmus array holds the shadow page tables that are used when a guest is running a nested context. These structures are allocated on VCPU_INIT for whole guest, which implies that they may have to be relocated as the array grows. Should a VCPU_INIT occur whilst a vcpu is actively running an L2 and that the allocation requires relocation, that vcpu will still be running with a pointer to the previous structure, which will have been freed. Fix this by turning the array of structures to an array of pointers, which is now allocated at VM creation, sized to the absolute maximum that KVM can handle. In turn, each VCPU_INIT contributes S2_MMU_PER_VCPU to the pool. No reallocation is ever performed, and the life cycle of each object is much clearer: - the nested_mmus array is allocated in kvm_init_nested(), and freed in kvm_arch_destroy_vm() - s2_mmu structures are allocated in kvm_vcpu_init_nested(), and freed on kvm_arch_flush_shadow_all() Finally, the freeing of vcpu->arch.vncr_array is made consistent rather than being done on some failure paths, but not others. Fixes: 4f128f8e1aaa ("KVM: arm64: nv: Support multiple nested Stage-2 mmu structures") Reported-by: Shen Yongchao Reported-by: Karl Mehltretter Suggested-by: Karl Mehltretter Acked-by: Lorenzo Stoakes (ARM) Link: https://lore.kernel.org/r/20260803224405.41468-1-kmehltretter@gmail.com Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Reviewed-by: Wei-Lin Chang Link: https://patch.msgid.link/20260911162203.1919330-2-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_host.h | 2 +- arch/arm64/include/asm/kvm_nested.h | 2 +- arch/arm64/kvm/arm.c | 8 ++- arch/arm64/kvm/nested.c | 93 +++++++++++++---------------- 4 files changed, 51 insertions(+), 54 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 27fe0cd5b2d7..cd9b9d2462f9 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -322,7 +322,7 @@ struct kvm_arch { * Stage 2 paging state for VMs with nested S2 using a virtual * VMID. */ - struct kvm_s2_mmu *nested_mmus; + struct kvm_s2_mmu **nested_mmus; size_t nested_mmus_size; int nested_mmus_next; diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index 1ed708335809..5b8edb2e8a87 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -66,7 +66,7 @@ static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0) extern bool forward_smc_trap(struct kvm_vcpu *vcpu); extern bool forward_debug_exception(struct kvm_vcpu *vcpu); -extern void kvm_init_nested(struct kvm *kvm); +extern int kvm_init_nested(struct kvm *kvm); extern int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu); extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu); extern struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 8b080804bc90..b53219e048bd 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -236,8 +236,6 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) mutex_unlock(&kvm->lock); #endif - kvm_init_nested(kvm); - ret = kvm_share_hyp(kvm, kvm + 1); if (ret) return ret; @@ -252,6 +250,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) if (ret) goto err_free_cpumask; + ret = kvm_init_nested(kvm); + if (ret) + goto err_uninit_mmu; + if (is_protected_kvm_enabled()) { /* * If any failures occur after this is successful, make sure to @@ -280,6 +282,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) err_uninit_mmu: kvm_uninit_stage2_mmu(kvm); + kvfree(kvm->arch.nested_mmus); err_free_cpumask: free_cpumask_var(kvm->arch.supported_cpus); err_unshare_kvm: @@ -337,6 +340,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm) kvm_unshare_hyp(kvm, kvm + 1); + kvfree(kvm->arch.nested_mmus); kvm_arm_teardown_hypercalls(kvm); } diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index a0808391a456..2571f177e654 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -45,11 +45,15 @@ struct vncr_tlb { */ #define S2_MMU_PER_VCPU 2 -void kvm_init_nested(struct kvm *kvm) +int kvm_init_nested(struct kvm *kvm) { - kvm->arch.nested_mmus = NULL; + kvm->arch.nested_mmus = kvmalloc_objs(struct kvm_s2_mmu *, + KVM_MAX_VCPUS * S2_MMU_PER_VCPU, + GFP_KERNEL_ACCOUNT); kvm->arch.nested_mmus_size = 0; atomic_set(&kvm->arch.vncr_tlb_count, 0); + + return kvm->arch.nested_mmus ? 0 : -ENOMEM; } static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) @@ -70,8 +74,9 @@ static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) { struct kvm *kvm = vcpu->kvm; - struct kvm_s2_mmu *tmp; - int num_mmus, ret = 0; + int num_mmus; + + lockdep_assert_held(&kvm->arch.config_lock); if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features) && !cpus_have_final_cap(ARM64_HAS_HCR_NV1)) @@ -84,51 +89,40 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) if (!vcpu->arch.ctxt.vncr_array) return -ENOMEM; - /* - * Let's treat memory allocation failures as benign: If we fail to - * allocate anything, return an error and keep the allocated array - * alive. Userspace may try to recover by initializing the vcpu - * again, and there is no reason to affect the whole VM for this. - */ num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU; if (num_mmus > kvm->arch.nested_mmus_size) { - tmp = kvzalloc_objs(*tmp, num_mmus, GFP_KERNEL_ACCOUNT); + struct kvm_s2_mmu *tmp; + int i, ret = 0; + + tmp = kvzalloc_objs(*tmp, S2_MMU_PER_VCPU, GFP_KERNEL_ACCOUNT); if (!tmp) - return -ENOMEM; + ret = -ENOMEM; - write_lock(&kvm->mmu_lock); - - if (kvm->arch.nested_mmus_size) { - memcpy(tmp, kvm->arch.nested_mmus, - size_mul(sizeof(*tmp), kvm->arch.nested_mmus_size)); - - for (int i = 0; i < kvm->arch.nested_mmus_size; i++) - tmp[i].pgt->mmu = &tmp[i]; + for (i = 0; !ret && i < S2_MMU_PER_VCPU; i++) { + ret = init_nested_s2_mmu(kvm, &tmp[i]); + if (ret) + break; } - swap(kvm->arch.nested_mmus, tmp); + if (ret) { + while (--i >= 0) + kvm_free_stage2_pgd(&tmp[i]); - write_unlock(&kvm->mmu_lock); + kvfree(tmp); + free_page((unsigned long)vcpu->arch.ctxt.vncr_array); + vcpu->arch.ctxt.vncr_array = NULL; + return ret; + } - kvfree(tmp); + guard(write_lock)(&kvm->mmu_lock); + + for (i = 0; i < S2_MMU_PER_VCPU; i++) + kvm->arch.nested_mmus[i + kvm->arch.nested_mmus_size] = &tmp[i]; + + kvm->arch.nested_mmus_size += S2_MMU_PER_VCPU; } - for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++) - ret = init_nested_s2_mmu(kvm, &kvm->arch.nested_mmus[i]); - - if (ret) { - for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++) - kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]); - - free_page((unsigned long)vcpu->arch.ctxt.vncr_array); - vcpu->arch.ctxt.vncr_array = NULL; - - return ret; - } - - kvm->arch.nested_mmus_size = num_mmus; - return 0; } @@ -742,7 +736,7 @@ void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, write_lock(&kvm->mmu_lock); for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (!kvm_s2_mmu_valid(mmu)) continue; @@ -784,7 +778,7 @@ struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu) * if S2 translation is disabled. */ for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (!kvm_s2_mmu_valid(mmu)) continue; @@ -823,7 +817,7 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) for (i = kvm->arch.nested_mmus_next; i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next); i++) { - s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size]; + s2_mmu = kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size]; if (atomic_read(&s2_mmu->refcnt) == 0) break; @@ -1281,7 +1275,7 @@ void kvm_nested_s2_wp(struct kvm *kvm) return; for (i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (kvm_s2_mmu_valid(mmu)) kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu)); @@ -1300,7 +1294,7 @@ void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) return; for (i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (kvm_s2_mmu_valid(mmu)) kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block); @@ -1319,7 +1313,7 @@ void kvm_nested_s2_flush(struct kvm *kvm) return; for (i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (kvm_s2_mmu_valid(mmu)) kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu)); @@ -1328,16 +1322,15 @@ void kvm_nested_s2_flush(struct kvm *kvm) void kvm_arch_flush_shadow_all(struct kvm *kvm) { - int i; - - for (i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + for (int i = kvm->arch.nested_mmus_size - 1; i >= 0; i--) { + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (!WARN_ON(atomic_read(&mmu->refcnt))) kvm_free_stage2_pgd(mmu); + + if ((i % S2_MMU_PER_VCPU) == 0) + kvfree(mmu); } - kvfree(kvm->arch.nested_mmus); - kvm->arch.nested_mmus = NULL; kvm->arch.nested_mmus_size = 0; kvm_uninit_stage2_mmu(kvm); } From e5843f4effaa2ffac3e789ecd4456403564961d4 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 11 Sep 2026 17:22:03 +0100 Subject: [PATCH 43/54] KVM: arm64: nv: Delay freeing of shadow S2 structures until VM destruction We free the shadow S2 structures from kvm_arch_flush_shadow_all(), which is a Bad Idea(tm). Freeing the page tables is fair game (this is what this callback is for), but freeing the container that could still be referenced by another part of the system is not great. Instead, grow separate destructors that gets called when we tear the VM down for good. From there, we can nuke both the individual MMUs as well as the global array that points to them, safe in the knowledge that the vcpus themselves have been destroyed already. Fixes: 4f128f8e1aaac ("KVM: arm64: nv: Support multiple nested Stage-2 mmu structures") Reviewed-by: Lorenzo Stoakes (ARM) Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Reviewed-by: Wei-Lin Chang Link: https://patch.msgid.link/20260911162203.1919330-3-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_nested.h | 1 + arch/arm64/kvm/arm.c | 4 ++-- arch/arm64/kvm/nested.c | 15 ++++++++++----- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index 5b8edb2e8a87..586026e85903 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -67,6 +67,7 @@ static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0) extern bool forward_smc_trap(struct kvm_vcpu *vcpu); extern bool forward_debug_exception(struct kvm_vcpu *vcpu); extern int kvm_init_nested(struct kvm *kvm); +extern void kvm_destroy_nested(struct kvm *kvm); extern int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu); extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu); extern struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index b53219e048bd..eaf583b77193 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -282,7 +282,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) err_uninit_mmu: kvm_uninit_stage2_mmu(kvm); - kvfree(kvm->arch.nested_mmus); + kvm_destroy_nested(kvm); err_free_cpumask: free_cpumask_var(kvm->arch.supported_cpus); err_unshare_kvm: @@ -340,7 +340,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm) kvm_unshare_hyp(kvm, kvm + 1); - kvfree(kvm->arch.nested_mmus); + kvm_destroy_nested(kvm); kvm_arm_teardown_hypercalls(kvm); } diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 2571f177e654..b191365d97cc 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -56,6 +56,15 @@ int kvm_init_nested(struct kvm *kvm) return kvm->arch.nested_mmus ? 0 : -ENOMEM; } +void kvm_destroy_nested(struct kvm *kvm) +{ + for (int i = 0; i < kvm->arch.nested_mmus_size; i+= S2_MMU_PER_VCPU) + kvfree(kvm->arch.nested_mmus[i]); + + kvm->arch.nested_mmus_size = 0; + kvfree(kvm->arch.nested_mmus); +} + static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) { /* @@ -1322,16 +1331,12 @@ void kvm_nested_s2_flush(struct kvm *kvm) void kvm_arch_flush_shadow_all(struct kvm *kvm) { - for (int i = kvm->arch.nested_mmus_size - 1; i >= 0; i--) { + for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (!WARN_ON(atomic_read(&mmu->refcnt))) kvm_free_stage2_pgd(mmu); - - if ((i % S2_MMU_PER_VCPU) == 0) - kvfree(mmu); } - kvm->arch.nested_mmus_size = 0; kvm_uninit_stage2_mmu(kvm); } From 49d9d295d69d07e850cae35933ba8519e2915f26 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 14 Sep 2026 10:38:38 +0100 Subject: [PATCH 44/54] KVM: arm64: Don't WARN on an unknown VM ioctl in protected mode kvm_pkvm_ioctl_allowed() WARNs when kvm_get_cap_for_kvm_ioctl() doesn't find the ioctl number in vm_ioctl_caps[], and kvm_arch_vm_ioctl() calls it for every number the generic code doesn't handle, so ioctl(vm_fd, 0xdeadbeef) from userspace taints a pKVM host and panics it under panic_on_warn. The lookup is fed userspace input: return false, and userspace gets the -EINVAL kvm_arch_vm_ioctl() returns for that number on a host without pKVM. Fixes: b12b3b04f6ba0 ("KVM: arm64: Check whether a VM IOCTL is allowed in pKVM") Cc: stable@vger.kernel.org Signed-off-by: Fuad Tabba Reviewed-by: Suzuki K Poulose Link: https://patch.msgid.link/20260914093838.1082637-1-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_pkvm.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index beea00e693a0..cad60569f061 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -65,8 +65,7 @@ static inline bool kvm_pkvm_ioctl_allowed(struct kvm *kvm, unsigned int ioctl) int r; r = kvm_get_cap_for_kvm_ioctl(ioctl, &ext); - - if (WARN_ON_ONCE(r < 0)) + if (r < 0) return false; return kvm_pkvm_ext_allowed(kvm, ext); From 96e6757cb0674acb86ee8b558ee6bffe0eec0bb0 Mon Sep 17 00:00:00 2001 From: Sebastian Ott Date: Mon, 14 Sep 2026 15:10:13 +0200 Subject: [PATCH 45/54] KVM: selftests: fix steal_time for arm64 with host page size > 4K Fix the following failure when running with 16K host page size: ==== Test Assertion Failure ==== lib/kvm_util.c:991: vm_adjust_num_guest_pages(vm->mode, npages) == npages pid=873 tid=873 errno=0 - Success 1 0x0000000000405a27: vm_mem_add at kvm_util.c:991 2 0x000000000040241f: check_steal_time_uapi at steal_time.c:223 (discriminator 7) 3 (inlined by) main at steal_time.c:539 (discriminator 7) 4 0x00007fff8b57af3b: ?? ??:0 5 0x00007fff8b57b007: ?? ??:0 6 0x0000000000402b6f: _start at ??:? Number of guest pages is not compatible with the host. Try npages=4 Fixes: fc240715fc50 ("KVM: selftests: arm64: Fix steal_time test after UAPI refactoring") Reported-by: Zenghui Yu Link: https://lore.kernel.org/kvmarm/7575a845-a542-4b16-b512-aec3126f97f3@linux.dev/T/#u Signed-off-by: Sebastian Ott Reviewed-by: Zenghui Yu (Huawei) Link: https://patch.msgid.link/20260914131013.60334-1-sebott@redhat.com Signed-off-by: Oliver Upton --- tools/testing/selftests/kvm/steal_time.c | 30 ++++++++++++++++-------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/tools/testing/selftests/kvm/steal_time.c b/tools/testing/selftests/kvm/steal_time.c index bc3c62b72c58..785d19f9ee8f 100644 --- a/tools/testing/selftests/kvm/steal_time.c +++ b/tools/testing/selftests/kvm/steal_time.c @@ -27,6 +27,9 @@ static void *st_gva[NR_VCPUS]; static u64 guest_stolen_time[NR_VCPUS]; +static struct kvm_vm *vm_create_steal_time(u32 nr_vcpus, void *guest_code, + struct kvm_vcpu *vcpus[]); + #if defined(__x86_64__) /* steal_time must have 64-byte alignment */ @@ -210,17 +213,14 @@ static void check_steal_time_uapi(void) u64 st_ipa; int ret; - vm = vm_create_with_one_vcpu(&vcpu, NULL); - struct kvm_device_attr dev = { .group = KVM_ARM_VCPU_PVTIME_CTRL, .attr = KVM_ARM_VCPU_PVTIME_IPA, .addr = (u64)&st_ipa, }; + vm = vm_create_steal_time(1, NULL, &vcpu); vcpu_ioctl(vcpu, KVM_HAS_DEVICE_ATTR, &dev); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, ST_GPA_BASE, 1, 1, 0); - virt_map(vm, ST_GPA_BASE, ST_GPA_BASE, 1); st_ipa = (ulong)ST_GPA_BASE | 1; ret = __vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev); @@ -500,13 +500,27 @@ static void run_vcpu(struct kvm_vcpu *vcpu) } } +static struct kvm_vm *vm_create_steal_time(u32 nr_vcpus, void *guest_code, + struct kvm_vcpu *vcpus[]) +{ + unsigned int gpages; + struct kvm_vm *vm; + + /* Create a VM and an identity mapped memslot for the steal time structure */ + vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus); + gpages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, STEAL_TIME_SIZE * nr_vcpus); + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, ST_GPA_BASE, 1, gpages, 0); + virt_map(vm, ST_GPA_BASE, ST_GPA_BASE, gpages); + + return vm; +} + int main(int ac, char **av) { struct kvm_vcpu *vcpus[NR_VCPUS]; struct kvm_vm *vm; pthread_t thread; cpu_set_t cpuset; - unsigned int gpages; long stolen_time; long run_delay; bool verbose; @@ -517,11 +531,7 @@ int main(int ac, char **av) /* Set CPU affinity so we can force preemption of the VCPU */ cpu = pin_self_to_any_cpu(); - /* Create a VM and an identity mapped memslot for the steal time structure */ - vm = vm_create_with_vcpus(NR_VCPUS, guest_code, vcpus); - gpages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, STEAL_TIME_SIZE * NR_VCPUS); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, ST_GPA_BASE, 1, gpages, 0); - virt_map(vm, ST_GPA_BASE, ST_GPA_BASE, gpages); + vm = vm_create_steal_time(NR_VCPUS, guest_code, vcpus); ksft_print_header(); TEST_REQUIRE(is_steal_time_supported(vcpus[0])); From 089e4f3c4862ba3f29dff2361caa8084879194fd Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 1 Sep 2026 22:47:00 +0100 Subject: [PATCH 46/54] KVM: arm64: Fix FGT mapping for HFGITR_EL2.nGCSEPP The encoding to trap mapping currently maps a FGT on OP_GCSPOPX to HFGITR_EL2.nGCSEPP but as per DDI0601 2026-06 this FGT controls trapping of GCSPUSHX and GCSPOPCX, and not the separate GCSPOPX instruction. Update the mapping to reflect the architecture. Fixes: 863ac38984a82 ("KVM: arm64: Add missing HFGITR_EL2 FGT entries to nested virt") Reviewed-by: Leonardo Bras Signed-off-by: Mark Brown Reviewed-by: Lorenzo Stoakes (ARM) Link: https://patch.msgid.link/20260901-arm64-gcs-v20-2-f31750bdfadb@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/emulate-nested.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index 625604019fb3..3806ff0920fe 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -1445,7 +1445,7 @@ static const struct encoding_to_trap_config encoding_to_fgt[] __initconst = { SR_FGT(OP_AT_S1E1A, HFGITR, ATS1E1A, 1), SR_FGT(OP_COSP_RCTX, HFGITR, COSPRCTX, 1), SR_FGT(OP_GCSPUSHX, HFGITR, nGCSEPP, 0), - SR_FGT(OP_GCSPOPX, HFGITR, nGCSEPP, 0), + SR_FGT(OP_GCSPOPCX, HFGITR, nGCSEPP, 0), SR_FGT(OP_GCSPUSHM, HFGITR, nGCSPUSHM_EL1, 0), SR_FGT(OP_BRB_IALL, HFGITR, nBRBIALL, 0), SR_FGT(OP_BRB_INJ, HFGITR, nBRBINJ, 0), From 6b1bca1b1ab77f60a62087337bfe6e2f0efb9e6d Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Mon, 10 Aug 2026 02:56:16 +0200 Subject: [PATCH 47/54] KVM: arm64: Fix AArch32 DBGBXVR handling The consolidation of the breakpoint and watchpoint register accessors switched DBGBXVR from trap_bvr() to trap_dbg_wb_reg(). The latter selects backing storage with demux_wb_reg(), which only handles Op2 values 4 through 7. Since DBGBXVR uses Op2 1, an AArch32 guest access hits KVM_BUG_ON() and marks the VM dead. DBGBXVR aliases DBGBVR_EL1[63:32], and its AA32(HI) descriptor already selects the upper half. Map Op2 1 to dbg_bvr[] alongside Op2 4, restoring the pre-regression behavior. Fixes: 3ce9f3357e9e ("KVM: arm64: Fold DBGxVR/DBGxCR accessors into common set") Cc: stable@vger.kernel.org Assisted-by: Claude:claude-opus-5 Signed-off-by: Karl Mehltretter Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260810005616.13227-1-kmehltretter@gmail.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/sys_regs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 44aae52c473d..a2f4e769a428 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -926,6 +926,7 @@ static u64 *demux_wb_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd) struct kvm_guest_debug_arch *dbg = &vcpu->arch.vcpu_debug_state; switch (rd->Op2) { + case 0b001: case 0b100: return &dbg->dbg_bvr[rd->CRm]; case 0b101: From 10180a277549339020b08000206092c07e0bff5a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 14:16:07 -0700 Subject: [PATCH 48/54] KVM: x86: Re-pend GET_NESTED_STATE_PAGES if getting said pages fails Re-pend GET_NESTED_STATE_PAGES before exiting to userspace if getting the nested pages fails in the KVM_RUN path. If userspace re-runs the vCPU, and vmcs02 holds valid PFNs from the *previous* run of L2, then KVM could re-enter L2 with stale, unpinned PFNs mapped into e.g. the vAPIC page. Note, both SVM and VMX (as of commit 11722439fb20 ("KVM: nVMX: Ensure KVM_REQ_GET_NESTED_STATE_PAGES is cleared on VM-Exit") ensure the request is cleared on VM-Exit (including the "forced" case), i.e. there is no risk of double-mapping due to emulated VMLAUNCH/VMRESUME/VMRUN *and* the request trying to map the nested pages. Fixes: 671ddc700fd0 ("KVM: nVMX: Don't leak L1 MMIO regions to L2") Cc: stable@vger.kernel.org Reported-by: Jinwoo Lee Closes: https://lore.kernel.org/all/20260813043932.3214460-1-rkskek9254@gmail.com Reported-by: Stefan Teodorescu Link: https://patch.msgid.link/20260921211608.1030158-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index a137dc6dd8c6..4ec17aaff413 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8072,6 +8072,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { if (unlikely(!kvm_nested_call(get_nested_state_pages)(vcpu))) { + kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); r = 0; goto out; } From c1214f293d77c6e425a379f90d09bdb4815993a8 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 14:16:08 -0700 Subject: [PATCH 49/54] KVM: x86: Fill kvm_run exit fields in common get_nested_state_pages() error paths Fill kvm_run with "internal error, emulation" in the common error handling paths for getting nested state pages, as requiring each check to manually fill kvm_run is error prone and requires a non-trivial amount of copy+paste. Specifically, both SVM and VMX fail to fill kvm_run if load_pdptrs() fails, and SVM fails to fill kvm_run if kvm_hv_verify_vp_assist() fails. If those flows fail, the *best* case scenario is that KVM will exit to userspace with KVM_EXIT_UNKNOWN. The worst case scenario is that KVM exits with a stale exit_reason and confuses userspace. Note, SVM never exits to userspace if something goes sideways when dealing with vmcb12 assets while emulating VMRUN, i.e. lack of SVM-specific code is not a bug. Fixes: 0f85722341b0 ("KVM: nVMX: delay loading of PDPTRs to KVM_REQ_GET_NESTED_STATE_PAGES") Fixes: 232f75d3b4b5 ("KVM: nSVM: call nested_svm_load_cr3 on nested state load") Fixes: 3f4a812edf5c ("KVM: nSVM: hyper-v: Enable L2 TLB flush") Cc: stable@vger.kernel.org Reported-by: Jinwoo Lee Closes: https://lore.kernel.org/all/20260813043932.3214460-1-rkskek9254@gmail.com Reported-by: Stefan Teodorescu Link: https://patch.msgid.link/20260921211608.1030158-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 7 +------ arch/x86/kvm/vmx/nested.c | 15 +++++---------- arch/x86/kvm/x86.c | 3 +++ 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 73f37b050d0a..f9090b601efa 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -2125,13 +2125,8 @@ static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu) return false; } - if (!nested_svm_merge_msrpm(vcpu)) { - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = - KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 0; + if (!nested_svm_merge_msrpm(vcpu)) return false; - } if (kvm_hv_verify_vp_assist(vcpu)) return false; diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 151873407abd..40c1a5f6fa8a 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -3465,10 +3465,6 @@ static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) } else { pr_debug_ratelimited("%s: no backing for APIC-access address in vmcs12\n", __func__); - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = - KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 0; return false; } } @@ -3539,11 +3535,6 @@ static bool vmx_get_nested_state_pages(struct kvm_vcpu *vcpu) if (!nested_get_evmcs_page(vcpu)) { pr_debug_ratelimited("%s: enlightened vmptrld failed\n", __func__); - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = - KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 0; - return false; } #endif @@ -3915,8 +3906,12 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) vmentry_failed: vcpu->arch.nested_run_pending = 0; - if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR) + if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR) { + vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; + vcpu->run->internal.ndata = 0; return 0; + } if (status == NVMX_VMENTRY_VMEXIT) return 1; WARN_ON_ONCE(status != NVMX_VMENTRY_VMFAIL); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 4ec17aaff413..aad065d035fb 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8072,6 +8072,9 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { if (unlikely(!kvm_nested_call(get_nested_state_pages)(vcpu))) { + vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; + vcpu->run->internal.ndata = 0; kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); r = 0; goto out; From 277d3623d99a4fc2623bfb7d191b649ca380605d Mon Sep 17 00:00:00 2001 From: Zeng Chi Date: Mon, 21 Sep 2026 18:24:42 +0800 Subject: [PATCH 50/54] KVM: Don't treat reserved xarray entries as having memory attributes kvm_vm_set_mem_attributes() reserves an xarray entry for every gfn in the range before storing the new attributes, so that the store loop can't fail partway through. If one of the reservations fails, e.g. with -ENOMEM, the entries that were already reserved are left in the array. That is harmless as far as xa_reserve() is concerned, as the reserved entries read back as NULL via xa_load(), but it confuses the "does this range have no attributes at all" check: if (!attrs) return !xas_find(&xas, end - 1); A reserved entry is XA_ZERO_ENTRY, not NULL, and xas_find() returns it as present. So a leftover reservation makes KVM report that a fully shared range has attributes even though kvm_get_memory_attributes() returns none for every gfn in the range. On x86, the next time mixed-attribute tracking is recomputed for the range (memslot creation, or a later attribute change that straddles the 2MiB page), hugepage_has_attrs() treats a fully shared 2MiB range as mixed and refuses to map it with a hugepage, until userspace happens to set attributes on the range again. Drop the shortcut and handle the !attrs case in the per-index loop, using xas_next_entry() to find the next non-NULL entry. xas_next_entry() is essentially an optimized xas_find(), so the effective change is that the !attrs lookup now goes through xas_retry() like the attrs != 0 case, i.e. reserved entries are skipped and retry entries restart the walk. Don't check the index when no entry is found, as the xarray leaves the xas index in a bogus state in that case; no entry simply means the rest of the range has no attributes. KVM never stores a non-NULL entry with a value of zero (clearing stores NULL), but such an entry would be returned by xas_next_entry() and trip the index check, so WARN if one is ever seen. Fixes: 5a475554db1e ("KVM: Introduce per-page memory attributes") Cc: stable@vger.kernel.org Suggested-by: Sean Christopherson Cc: David Ballesteros Signed-off-by: Zeng Chi Link: https://patch.msgid.link/20260921102442.1232375-1-zeng_chi911@163.com [sean: expand comment to elaborate on xarray APIs, split optimization out] Signed-off-by: Sean Christopherson --- virt/kvm/kvm_main.c | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 65eb26a0520d..2a046e95f95e 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2447,14 +2447,36 @@ bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, return (kvm_get_memory_attributes(kvm, start) & mask) == attrs; guard(rcu)(); - if (!attrs) - return !xas_find(&xas, end - 1); + /* + * Lookup the entry for each index instead of iterating over the xarray + * as KVM deletes/nullifies entries to represent "no attributes", and + * the xas index is effectively invalid when no entry is found. I.e. + * matching non-zero attributes for *every* entry effectively requires + * a manually lookup for each index. + * + * Skip pre-allocated, reserved entries, or restart the lookup if the + * xarray was concurrently modified, via xas_retry() ("retry" means the + * entry holds an internal xarray value, i.e. is either invalid or NULL + * from the caller's perspective). + * + * Use xas_next() when looking for non-zero attributes to optimize for + * the case where the start of the range (or the entire range) doesn't + * have any attributes, as xas_next() returns literally the next entry, + * whereas xas_next_entry() returns the next non-NULL entry (bounded by + * a maximum index). + */ for (index = start; index < end; index++) { do { - entry = xas_next(&xas); + entry = attrs ? xas_next(&xas) : + xas_next_entry(&xas, end - 1); } while (xas_retry(&xas, entry)); + if (!entry) + return !attrs; + + WARN_ON_ONCE(!xa_to_value(entry)); + if (xas.xa_index != index || (xa_to_value(entry) & mask) != attrs) return false; From 382e5d514b6f35bdda2ab9044b4eed23d2ec4254 Mon Sep 17 00:00:00 2001 From: David Ballesteros Date: Tue, 15 Sep 2026 17:53:57 +0000 Subject: [PATCH 51/54] KVM: Ensure memory attributes xarray nodes are accounted to the caller's memcg Explicitly instantiate the memory attributes xarray with XA_FLAGS_ACCOUNT to ensure that all allocations are accounted to the memcg. Frustratingly, memory allocations done in the "fastpath" do not honor the passed in gfp, even for an explicit xa_reserve(). Only the rare, slow path __xas_nomem() honors the original gfp. E.g. xa_reserve(..., GFP_KERNEL_ACCOUNT) | -> ... | -> __xa_cmpxchg_raw() | -> xas_store() <== does not take @gfp | -> xas_create() | -> xas_alloc() The bug was confirmed by observing that a process in a cgroup limited to 256 MiB grew radix_tree_node slab by ~512 MiB while its memory.current stayed near 0. Fixes: 5a475554db1e ("KVM: Introduce per-page memory attributes") Cc: stable@vger.kernel.org Assisted-by: Claude-Code:claude-opus-5 Signed-off-by: David Ballesteros Link: https://patch.msgid.link/20260915175335.138547-4-davimaba.v@proton.me [sean: rewrite changelog, tag for stable] Signed-off-by: Sean Christopherson --- virt/kvm/kvm_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 2a046e95f95e..df643ec3ea75 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1117,7 +1117,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) rcuwait_init(&kvm->mn_memslots_update_rcuwait); xa_init(&kvm->vcpu_array); #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES - xa_init(&kvm->mem_attr_array); + xa_init_flags(&kvm->mem_attr_array, XA_FLAGS_ACCOUNT); #endif INIT_LIST_HEAD(&kvm->gpc_list); From 119e9db233ad0f4cbd4cfb9f9385a7bca378b37d Mon Sep 17 00:00:00 2001 From: Zeng Chi Date: Mon, 21 Sep 2026 18:24:42 +0800 Subject: [PATCH 52/54] KVM: Don't pre-reserve xarray entries when storing empty/NULL attributes Skip the xarray reservation loop when clearing all memory attributes, as storing NULL only erases the entry and never needs to allocate, so no reservation (and no cleanup of a failed one) is required in that case. Suggested-by: Sean Christopherson Cc: David Ballesteros Signed-off-by: Zeng Chi Link: https://patch.msgid.link/20260921102442.1232375-1-zeng_chi911@163.com [sean: split to separate patch] Signed-off-by: Sean Christopherson --- virt/kvm/kvm_main.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index df643ec3ea75..85f42289748d 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2593,9 +2593,10 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, /* * Reserve memory ahead of time to avoid having to deal with failures - * partway through setting the new attributes. + * partway through setting the new attributes. Storing NULL never + * allocates, so no reservations are needed when clearing. */ - for (i = start; i < end; i++) { + for (i = start; entry && i < end; i++) { r = xa_reserve(&kvm->mem_attr_array, i, GFP_KERNEL_ACCOUNT); if (r) goto out_unlock; From 12c1f6e03f944e399bd2c88441dca5dc702b95a5 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 23 Sep 2026 09:37:20 -0700 Subject: [PATCH 53/54] KVM: SEV: Free have_run_cpus during VM destruction even if VM is no longer SEV Unconditionally free SEV's "have run CPUs" cpumask in the VM destroy path, i.e. even for what appear to be non-SEV VMs, as an SEV VM becomes a non-SEV VM if its state is intra-host migrated. Alternatively, the mask could be freed in sev_migrate_from() when "converting" the source VM, but that gets annoying because ideally KVM would nullify the mask to guard against UAF, and nullifying the mask would need be conditioned on CPUMASK_OFFSTACK=y. Freeing the mask during sev_migrate_from() is also not robust against other KVM bugs, though that's kind of a moot point since any such bugs would show up even if sev->active is never set. I.e. KVM must get that side of things correct. But, that's not a great reason to add more code just to make things marginally less robust. Fixes: 6f38f8c57464 ("KVM: SVM: Flush cache only on CPUs running SEV guest") Cc: stable@vger.kernel.org Reported-by: Stefan Teodorescu Signed-off-by: Sean Christopherson Message-ID: <20260923163721.1584779-2-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/sev.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 5705723f1f41..cdc1c04f60da 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2980,13 +2980,17 @@ void sev_vm_destroy(struct kvm *kvm) struct list_head *head = &sev->regions_list; struct list_head *pos, *q; + /* + * Free the mask even if the VM is not *currently* an SEV VM, as it may + * have been an SEV VM prior to intra-host migration. + */ + free_cpumask_var(sev->have_run_cpus); + if (!sev_guest(kvm)) return; WARN_ON(!list_empty(&sev->mirror_vms)); - free_cpumask_var(sev->have_run_cpus); - /* * If this is a mirror VM, remove it from the owner's list of a mirrors * and skip ASID cleanup (the ASID is tied to the lifetime of the owner). From 93de2a6a4b91b72607136dd656edf03fb399d27f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 23 Sep 2026 09:37:21 -0700 Subject: [PATCH 54/54] KVM: SEV: Do cache maintenance on the source VM during intra-host migration Manually perform cache maintenance on the source VM during intra-host migration to ensure no stale data is left in CPU caches after the VM is destroyed. Because the source VM is "converted" to a non-SEV VM, KVM's memory reclaim flows won't trigger cache maintenance, e.g. when all guest memory is reclaimed in response to detaching from the mmu_notifier. Note, relying on the destination VM to do cache maintenance isn't an option as KVM doesn't require identical guest memory configurations, i.e. the source VM may have access to memory that the destination VM does not. Enforcing equivalent memory configurations is infeasible, as it would require a *deep* comparison of memslots, e.g. to verify that not only are the memslot identical, but what the memslots point at is also identical. Fixes: b56639318bb2 ("KVM: SEV: Add support for SEV intra host migration") Cc: stable@vger.kernel.org Reported-by: Stefan Teodorescu Signed-off-by: Sean Christopherson Message-ID: <20260923163721.1584779-3-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/sev.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index cdc1c04f60da..63eb2155a774 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2048,6 +2048,12 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) src->pages_locked = 0; src->es_active = false; + /* + * Do cache maintenance on the source VM as it is no longer an SEV VM, + * i.e. memory reclaim flows won't trigger cache maintenance on the VM. + */ + sev_writeback_caches(src_kvm); + list_cut_before(&dst->regions_list, &src->regions_list, &src->regions_list); mutex_lock(&sev_mirror_lock); @@ -2187,6 +2193,10 @@ int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) * the set of CPUs from the source. If a CPU was used to run a vCPU in * the source VM but is never used for the destination VM, then the CPU * can only have cached memory that was accessible to the source VM. + * Furthermore, KVM *must* perform cache maintenance on the source VM, + * as the source VM may have access to memory that the destination VM + * does not, i.e. KVM could skip flushes if memory is reclaimed from + * the old VM but not the new VM. */ if (!zalloc_cpumask_var(&dst_sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) { ret = -ENOMEM;