From 6835fbed39bb329744a3f44e8e6a39e24079af10 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 21 May 2026 15:36:24 +0100 Subject: [PATCH 01/12] KVM: arm64: Reset page order in pKVM hyp_pool When a VM fails to initialise after its stage-2 hyp_pool has been initialised, that stage-2 must be torn down entirely. This requires resetting both the refcount and the order of its pages back to 0. Currently, reclaim_pgtable_pages() implicitly resets the page order by allocating the entire pool with order-0 granularity. However, in the VM initialisation error path, the addresses of the donated memory (the PGD) are already known, making it unnecessary to iterate over all pages in the pool. Since the vmemmap page order is a hyp_pool-specific field, leaving a non-zero order on hyp_pool destruction is harmless until another pool attempts to admit the page. Instead of resetting this field during destruction, reset it during pool initialization in hyp_pool_init(). For 'external' pages, we can't trust the order either as they bypass hyp_pool_init(). Since we never coalesce them, enforce order-0 to ensure safe insertion into the pool. This leaves no vmemmap order users outside of hyp_pool. Fixes: 256b4668cd89 ("KVM: arm64: Introduce separate hypercalls for pKVM VM reservation and initialization") Reported-by: Sashiko Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260521143626.1005660-2-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/mem_protect.c | 2 -- arch/arm64/kvm/hyp/nvhe/page_alloc.c | 21 ++++++++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 28a471d1927c..5c1e1742db4f 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -202,7 +202,6 @@ static void *guest_s2_zalloc_page(void *mc) memset(addr, 0, PAGE_SIZE); p = hyp_virt_to_page(addr); p->refcount = 1; - p->order = 0; return addr; } @@ -307,7 +306,6 @@ void reclaim_pgtable_pages(struct pkvm_hyp_vm *vm, struct kvm_hyp_memcache *mc) while (addr) { page = hyp_virt_to_page(addr); page->refcount = 0; - page->order = 0; push_hyp_memcache(mc, addr, hyp_virt_to_phys); WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(addr), 1)); addr = hyp_alloc_pages(&vm->pool, 0); diff --git a/arch/arm64/kvm/hyp/nvhe/page_alloc.c b/arch/arm64/kvm/hyp/nvhe/page_alloc.c index a1eb27a1a747..57f86aa0f82f 100644 --- a/arch/arm64/kvm/hyp/nvhe/page_alloc.c +++ b/arch/arm64/kvm/hyp/nvhe/page_alloc.c @@ -94,13 +94,22 @@ static void __hyp_attach_page(struct hyp_pool *pool, struct hyp_page *p) { phys_addr_t phys = hyp_page_to_phys(p); - u8 order = p->order; struct hyp_page *buddy; + bool coalesce = true; + u8 order = p->order; - memset(hyp_page_to_virt(p), 0, PAGE_SIZE << p->order); + /* + * 'external' pages are never coalesced and their ->order field + * untrusted as they bypass hyp_pool_init(). Enforce order-0. + */ + if (phys < pool->range_start || phys >= pool->range_end) { + order = 0; + coalesce = false; + } - /* Skip coalescing for 'external' pages being freed into the pool. */ - if (phys < pool->range_start || phys >= pool->range_end) + memset(hyp_page_to_virt(p), 0, PAGE_SIZE << order); + + if (!coalesce) goto insert; /* @@ -237,8 +246,10 @@ int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages, /* Init the vmemmap portion */ p = hyp_phys_to_page(phys); - for (i = 0; i < nr_pages; i++) + for (i = 0; i < nr_pages; i++) { hyp_set_page_refcounted(&p[i]); + p[i].order = 0; + } /* Attach the unused pages to the buddy tree */ for (i = reserved_pages; i < nr_pages; i++) From 20d2753295b1cd3c766199b5990119ca514e1302 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 21 May 2026 15:36:25 +0100 Subject: [PATCH 02/12] KVM: arm64: Fix __pkvm_init_vm error path In the unlikely case where insert_vm_table_entry fails, __pkvm_init_vm release the memory donated by the host for the PGD, but as the stage-2 is still set-up the hypervisor keeps a refcount on those pages, effectively leaking the references. Fix the rollback with the newly added kvm_guest_destroy_stage2(). Fixes: 256b4668cd89 ("KVM: arm64: Introduce separate hypercalls for pKVM VM reservation and initialization") Reported-by: Sashiko Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Signed-off-by: Vincent Donnefort Link: https://patch.msgid.link/20260521143626.1005660-3-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/include/nvhe/mem_protect.h | 1 + arch/arm64/kvm/hyp/nvhe/mem_protect.c | 13 +++++++++---- arch/arm64/kvm/hyp/nvhe/pkvm.c | 4 +++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index 3cbfae0e3dda..4f2b871199cb 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -56,6 +56,7 @@ int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot p int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id); int kvm_host_prepare_stage2(void *pgt_pool_base); int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd); +void kvm_guest_destroy_stage2(struct pkvm_hyp_vm *vm); void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt); int hyp_pin_shared_mem(void *from, void *to); diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 5c1e1742db4f..88cd72332208 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -290,16 +290,21 @@ int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd) return 0; } +void kvm_guest_destroy_stage2(struct pkvm_hyp_vm *vm) +{ + guest_lock_component(vm); + kvm_pgtable_stage2_destroy(&vm->pgt); + vm->kvm.arch.mmu.pgd_phys = 0ULL; + guest_unlock_component(vm); +} + void reclaim_pgtable_pages(struct pkvm_hyp_vm *vm, struct kvm_hyp_memcache *mc) { struct hyp_page *page; void *addr; /* Dump all pgtable pages in the hyp_pool */ - guest_lock_component(vm); - kvm_pgtable_stage2_destroy(&vm->pgt); - vm->kvm.arch.mmu.pgd_phys = 0ULL; - guest_unlock_component(vm); + kvm_guest_destroy_stage2(vm); /* Drain the hyp_pool into the memcache */ addr = hyp_alloc_pages(&vm->pool, 0); diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index e7496eb85628..ff89b30f5c4a 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -839,10 +839,12 @@ int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, /* Must be called last since this publishes the VM. */ ret = insert_vm_table_entry(handle, hyp_vm); if (ret) - goto err_remove_mappings; + goto err_destroy_stage2; return 0; +err_destroy_stage2: + kvm_guest_destroy_stage2(hyp_vm); err_remove_mappings: unmap_donated_memory(hyp_vm, vm_size); unmap_donated_memory(pgd, pgd_size); From 5c30c9fc117cc7d0c1b45741d1127edd4c2ae990 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 21 May 2026 15:36:26 +0100 Subject: [PATCH 03/12] KVM: arm64: Add fail-safe for refcounted pages in __pkvm_hyp_donate_host A previous bug in __pkvm_init_vm error path showed that the hypervisor could leak refcounted pages, (i.e. losing access to a page while its refcount is still elevated). This poses a threat to the pKVM state machine. Address this by introducing a fail-safe in __pkvm_hyp_donate_host. Transitions are not a hot path so added security is worth the extra check. Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Signed-off-by: Vincent Donnefort Link: https://patch.msgid.link/20260521143626.1005660-4-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/mem_protect.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 88cd72332208..3263f6d0a0a4 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -833,6 +833,16 @@ static int __hyp_check_page_state_range(phys_addr_t phys, u64 size, enum pkvm_pa return 0; } +static int __hyp_check_page_count_range(phys_addr_t phys, u64 size) +{ + for_each_hyp_page(page, phys, size) { + if (page->refcount) + return -EBUSY; + } + + return 0; +} + static bool guest_pte_is_poisoned(kvm_pte_t pte) { if (kvm_pte_valid(pte)) @@ -1031,7 +1041,6 @@ int __pkvm_guest_unshare_host(struct pkvm_hyp_vcpu *vcpu, u64 gfn) int __pkvm_host_unshare_hyp(u64 pfn) { u64 phys = hyp_pfn_to_phys(pfn); - u64 virt = (u64)__hyp_va(phys); u64 size = PAGE_SIZE; int ret; @@ -1044,10 +1053,9 @@ int __pkvm_host_unshare_hyp(u64 pfn) ret = __hyp_check_page_state_range(phys, size, PKVM_PAGE_SHARED_BORROWED); if (ret) goto unlock; - if (hyp_page_count((void *)virt)) { - ret = -EBUSY; + ret = __hyp_check_page_count_range(phys, size); + if (ret) goto unlock; - } __hyp_set_page_state_range(phys, size, PKVM_NOPAGE); WARN_ON(__host_set_page_state_range(phys, size, PKVM_PAGE_OWNED)); @@ -1110,6 +1118,10 @@ int __pkvm_hyp_donate_host(u64 pfn, u64 nr_pages) if (ret) goto unlock; + ret = __hyp_check_page_count_range(phys, size); + if (ret) + goto unlock; + __hyp_set_page_state_range(phys, size, PKVM_NOPAGE); WARN_ON(kvm_pgtable_hyp_unmap(&pkvm_pgtable, virt, size) != size); WARN_ON(host_stage2_set_owner_locked(phys, size, PKVM_ID_HOST)); From 3a4f5b96730cb40d5d9b31293fd34e11a10f2d6d Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 1 May 2026 12:21:47 +0100 Subject: [PATCH 04/12] KVM: arm64: Seed pkvm_ownership_selftest vcpu memcache The hypercall handlers call pkvm_refill_memcache() to top up the hyp_vcpu memcache before invoking __pkvm_host_{share,donate}_guest(). pkvm_ownership_selftest invokes those functions directly with a static selftest_vcpu that has an empty memcache. Seed selftest_vcpu's memcache from the prepopulated selftest pages, leaving the remainder for selftest_vm.pool. Required by the memcache-sufficiency pre-check added in the following patches. Assisted-by: Gemini:gemini-3.1-pro review-prompts Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260501112149.2824881-5-tabba@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/pkvm.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index ff89b30f5c4a..3b2c4fbc34d8 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -752,16 +752,30 @@ static struct pkvm_hyp_vcpu selftest_vcpu = { struct pkvm_hyp_vcpu *init_selftest_vm(void *virt) { struct hyp_page *p = hyp_virt_to_page(virt); + unsigned long min_pages, seeded = 0; int i; selftest_vm.kvm.arch.mmu.vtcr = host_mmu.arch.mmu.vtcr; WARN_ON(kvm_guest_prepare_stage2(&selftest_vm, virt)); + /* + * Mirror pkvm_refill_memcache() for the share/donate pre-checks; + * the selftest invokes those functions directly and would + * otherwise see an empty memcache. + */ + min_pages = kvm_mmu_cache_min_pages(&selftest_vm.kvm.arch.mmu); + for (i = 0; i < pkvm_selftest_pages(); i++) { if (p[i].refcount) continue; p[i].refcount = 1; - hyp_put_page(&selftest_vm.pool, hyp_page_to_virt(&p[i])); + if (seeded < min_pages) { + push_hyp_memcache(&selftest_vcpu.vcpu.arch.pkvm_memcache, + hyp_page_to_virt(&p[i]), hyp_virt_to_phys); + seeded++; + } else { + hyp_put_page(&selftest_vm.pool, hyp_page_to_virt(&p[i])); + } } selftest_vm.kvm.arch.pkvm.handle = __pkvm_reserve_vm(); From 8ed0fbe5404616041f6daf1d2fa1824d75602f63 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 1 May 2026 12:21:48 +0100 Subject: [PATCH 05/12] KVM: arm64: Pre-check vcpu memcache for host->guest share __pkvm_host_share_guest() ends with kvm_pgtable_stage2_map() to install the guest stage-2 mapping, after a forward pass that mutates the host vmemmap (sets PKVM_PAGE_SHARED_OWNED and increments host_share_guest_count) for every page in the range. The map's return value is wrapped in WARN_ON() and otherwise discarded, asserting that the call cannot fail. WARN_ON() at nVHE EL2 panics, so this assertion is only correct if the call genuinely cannot fail. kvm_pgtable_stage2_map() can fail with -ENOMEM when the stage-2 walker exhausts the caller's memcache, and the host controls the vcpu memcache via the topup interface, so an under-provisioned share request would otherwise turn a recoverable -ENOMEM into a fatal hyp panic. Bound the worst-case walker allocation in the existing pre-check pass so that kvm_pgtable_stage2_map() cannot fail at the call site, using kvm_mmu_cache_min_pages() -- the same bound host EL1 uses for its own stage-2 maps. If the vcpu memcache holds fewer pages, return -ENOMEM before any state mutation. Fixes: d0bd3e6570ae ("KVM: arm64: Introduce __pkvm_host_share_guest()") Assisted-by: Gemini:gemini-3.1-pro review-prompts Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260501112149.2824881-6-tabba@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/mem_protect.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 3263f6d0a0a4..6af4a1c95aef 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -1384,6 +1384,22 @@ int __pkvm_host_reclaim_page_guest(u64 gfn, struct pkvm_hyp_vm *vm) return ret && ret != -EHWPOISON ? ret : 0; } +/* + * share/donate install at most one stage-2 leaf (PAGE_SIZE, or one + * KVM_PGTABLE_LAST_LEVEL - 1 block for share). kvm_mmu_cache_min_pages() + * bounds the worst-case allocation: exact for the PAGE_SIZE leaf, + * conservative by one for the block. + */ +static int __guest_check_pgtable_memcache(struct pkvm_hyp_vcpu *vcpu) +{ + struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu); + + if (vcpu->vcpu.arch.pkvm_memcache.nr_pages < kvm_mmu_cache_min_pages(vm->pgt.mmu)) + return -ENOMEM; + + return 0; +} + int __pkvm_host_donate_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu) { struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu); @@ -1468,6 +1484,10 @@ int __pkvm_host_share_guest(u64 pfn, u64 gfn, u64 nr_pages, struct pkvm_hyp_vcpu } } + ret = __guest_check_pgtable_memcache(vcpu); + if (ret) + goto unlock; + for_each_hyp_page(page, phys, size) { set_host_state(page, PKVM_PAGE_SHARED_OWNED); page->host_share_guest_count++; From cada2549ca4c934e6fb3801f857c6b4b0c36490b Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 1 May 2026 12:21:49 +0100 Subject: [PATCH 06/12] KVM: arm64: Pre-check vcpu memcache for host->guest donate __pkvm_host_donate_guest() flips the host stage-2 PTE for the donated page to a non-valid annotation via host_stage2_set_owner_metadata_locked() and then calls kvm_pgtable_stage2_map() to install the matching guest stage-2 mapping. The map's return value is wrapped in WARN_ON() and otherwise discarded, asserting that the call cannot fail. WARN_ON() at nVHE EL2 panics, so this assertion is only correct if the call genuinely cannot fail. kvm_pgtable_stage2_map() can fail with -ENOMEM even at PAGE_SIZE granularity: the donate path verifies PKVM_NOPAGE for the guest IPA before the map, so the walker must allocate fresh page-table pages from the vcpu memcache, and the host controls the vcpu memcache via the topup interface. An under-provisioned donation request would otherwise turn a recoverable -ENOMEM into a fatal hyp panic. Bound the worst-case walker allocation alongside the existing __host_check_page_state_range() / __guest_check_page_state_range() pre-checks, using the helper introduced for host->guest share. If the vcpu memcache holds fewer pages than kvm_mmu_cache_min_pages(), return -ENOMEM before any state mutation. Fixes: 1e579adca177 ("KVM: arm64: Introduce __pkvm_host_donate_guest()") Assisted-by: Gemini:gemini-3.1-pro review-prompts Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260501112149.2824881-7-tabba@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/mem_protect.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 6af4a1c95aef..848c5b9dcfe6 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -1419,6 +1419,10 @@ int __pkvm_host_donate_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu) if (ret) goto unlock; + ret = __guest_check_pgtable_memcache(vcpu); + if (ret) + goto unlock; + meta = host_stage2_encode_gfn_meta(vm, gfn); WARN_ON(host_stage2_set_owner_metadata_locked(phys, PAGE_SIZE, PKVM_ID_GUEST, meta)); From e8042f6e1d7befb2fb6b10a75918642bcd0acf9a Mon Sep 17 00:00:00 2001 From: Hyunwoo Kim Date: Sun, 7 Jun 2026 02:56:10 +0900 Subject: [PATCH 07/12] KVM: arm64: Clear __hyp_running_vcpu when flushing the pKVM hyp vCPU flush_hyp_vcpu() copies the host vCPU context into the hyp's private vCPU on every run. ctxt_to_vcpu() expects a guest context to have a NULL __hyp_running_vcpu, which is only ever set on the host context, so that it resolves the vCPU via container_of(). While this is generally the case, flush_hyp_vcpu() copies the context verbatim and does not enforce this, so a value provided by the host is dereferenced at EL2 (host -> EL2). Fix by clearing __hyp_running_vcpu after the copy. Cc: stable@vger.kernel.org Fixes: be66e67f1750 ("KVM: arm64: Use the pKVM hyp vCPU structure in handle___kvm_vcpu_run()") Signed-off-by: Hyunwoo Kim Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260606175614.83273-2-imv4bel@gmail.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 06db299c37a8..02c5d6e5abcb 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -128,6 +128,9 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; + /* __hyp_running_vcpu must be NULL in a guest context. */ + hyp_vcpu->vcpu.arch.ctxt.__hyp_running_vcpu = NULL; + hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2; hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWI | HCR_TWE); hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & From 8cc8bbbfab14c22c5551d0dd19b208a44b141c76 Mon Sep 17 00:00:00 2001 From: Hyunwoo Kim Date: Sun, 7 Jun 2026 02:56:11 +0900 Subject: [PATCH 08/12] KVM: arm64: Bound used_lrs when flushing the pKVM hyp vCPU flush_hyp_vcpu() copies the host vGIC state into the hyp's private vCPU on every run. The vGIC list register save and restore use used_lrs as their loop bound and expect it to stay within the number of implemented list registers. While this is generally the case, flush_hyp_vcpu() copies vgic_v3 verbatim and does not enforce this, so a value provided by the host is used at EL2 to index vgic_lr[] and access ICH_LR_EL2 (host -> EL2). Fix by clamping used_lrs to the number of implemented list registers after the copy, as the trusted path already does in vgic_flush_lr_state(). The number of implemented list registers is constant after init, so it is replicated once from kvm_vgic_global_state.nr_lr into hyp_gicv3_nr_lr rather than read on every entry. Cc: stable@vger.kernel.org Fixes: be66e67f1750 ("KVM: arm64: Use the pKVM hyp vCPU structure in handle___kvm_vcpu_run()") Signed-off-by: Hyunwoo Kim Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260606175614.83273-3-imv4bel@gmail.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_hyp.h | 1 + arch/arm64/kvm/arm.c | 2 ++ arch/arm64/kvm/hyp/nvhe/hyp-main.c | 9 +++++++++ 3 files changed, 12 insertions(+) diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 8d06b62e7188..e9b2b0c40ec6 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -157,5 +157,6 @@ extern unsigned long kvm_nvhe_sym(__icache_flags); extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits); extern unsigned int kvm_nvhe_sym(kvm_host_sve_max_vl); extern unsigned long kvm_nvhe_sym(hyp_nr_cpus); +extern unsigned int kvm_nvhe_sym(hyp_gicv3_nr_lr); #endif /* __ARM64_KVM_HYP_H__ */ diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 8bb2c7422cc8..74965b358cdf 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2423,6 +2423,8 @@ static int __init init_subsystems(void) switch (err) { case 0: vgic_present = true; + if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) + kvm_nvhe_sym(hyp_gicv3_nr_lr) = kvm_vgic_global_state.nr_lr; break; case -ENODEV: case -ENXIO: diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 02c5d6e5abcb..a0da08caa6c2 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -24,6 +24,9 @@ DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); +/* Number of implemented GICv3 LRs. Used by flush_hyp_vcpu(). */ +unsigned int hyp_gicv3_nr_lr; + void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt); static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) @@ -142,6 +145,12 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3 = host_vcpu->arch.vgic_cpu.vgic_v3; + /* Bound used_lrs by the number of implemented list registers. */ + hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3.used_lrs = + min_t(unsigned int, + hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3.used_lrs, + hyp_gicv3_nr_lr); + hyp_vcpu->vcpu.arch.pid = host_vcpu->arch.pid; } From 832dfa237f836549b202d3eebc0bc29b8a719608 Mon Sep 17 00:00:00 2001 From: "tabba@google.com" Date: Sun, 31 May 2026 16:45:48 +0100 Subject: [PATCH 09/12] KVM: arm64: Flush HCR_EL2.VSE to deliver SErrors to pKVM guests With pKVM enabled, the host injects a virtual SError by setting HCR_EL2.VSE on its vCPU copy, but flush_hyp_vcpu() only flows TWI/TWE into the hyp vCPU that runs, so VSE never reaches it and a deferred (masked) SError is never delivered. VSE is a host-owned injection control, not a trap-configuration bit, so restricting the host's trap-register values should not have dropped it. Flow it on entry; sync_hyp_vcpu() already copies hcr_el2 back, so delivery is reflected to the host. THis makes it consistent with the existing forwarding of VSESR_EL2, which qualifies the Serror. Fixes: b56680de9c648 ("KVM: arm64: Initialize trap register values in hyp in pKVM") Reported-by: Sashiko (local):gemini-3.1-pro Signed-off-by: Fuad Tabba Reviewed-by: Oliver Upton Link: https://patch.msgid.link/20260531154548.1505799-1-tabba@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index a0da08caa6c2..1d01c6e547f5 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -135,9 +135,14 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_vcpu->vcpu.arch.ctxt.__hyp_running_vcpu = NULL; hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2; - hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWI | HCR_TWE); + /* + * HCR_EL2.VSE is host-owned (a pending virtual SError to inject), not a + * trap-control bit, so it must flow to the hyp vCPU alongside TWI/TWE + * for the vSError to be delivered. sync_hyp_vcpu() reflects it back. + */ + hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWI | HCR_TWE | HCR_VSE); hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & - (HCR_TWI | HCR_TWE); + (HCR_TWI | HCR_TWE | HCR_VSE); hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags; From 63336d57a26904f58e4ff2cf584ef9958564a7c6 Mon Sep 17 00:00:00 2001 From: "tabba@google.com" Date: Fri, 29 May 2026 13:17:53 +0100 Subject: [PATCH 10/12] KVM: arm64: Free hyp-share tracking node when share hypercall fails share_pfn_hyp() inserts a tracking node into hyp_shared_pfns and then invokes __pkvm_host_share_hyp. If the hypercall rejects the share (page-state mismatch at EL2), the node stays in the tree with refcount 1: a phantom share that leaks the allocation and that a later unshare will trust. Erase the node and free it on hypercall failure. Fixes: a83e2191b7f1 ("KVM: arm64: pkvm: Refcount the pages shared with EL2") Reported-by: Sashiko (local):gemini-3.1-pro Suggested-by: Vincent Donnefort Signed-off-by: Fuad Tabba Reviewed-by: Vincent Donnefort Link: https://patch.msgid.link/20260529121755.2923500-2-tabba@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index d089c107d9b7..0abf3a2d587b 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -501,6 +501,10 @@ static int share_pfn_hyp(u64 pfn) rb_link_node(&this->node, parent, node); rb_insert_color(&this->node, &hyp_shared_pfns); ret = kvm_call_hyp_nvhe(__pkvm_host_share_hyp, pfn); + if (ret) { + rb_erase(&this->node, &hyp_shared_pfns); + kfree(this); + } unlock: mutex_unlock(&hyp_shared_pfns_lock); From bd2618780ab4584a33ab1049338294a50690d149 Mon Sep 17 00:00:00 2001 From: "tabba@google.com" Date: Fri, 29 May 2026 13:17:54 +0100 Subject: [PATCH 11/12] KVM: arm64: Avoid host/hyp share desync on unshare hypercall failure unshare_pfn_hyp() erases the tracking node from hyp_shared_pfns and frees it before invoking __pkvm_host_unshare_hyp. If the hypercall fails (e.g. EL2 refcount still held, or page-state mismatch), the host loses its record while EL2 still holds the share, breaking later share/unshare attempts on the same pfn. Invoke the hypercall first; erase and free only on success. Document at the kvm_unshare_hyp() call site that the WARN_ON() is left non-fatal: a failed unshare leaks the page (it stays shared with the hypervisor) but breaks no isolation guarantee. Fixes: 52b28657ebd7 ("KVM: arm64: pkvm: Unshare guest structs during teardown") Reported-by: Sashiko (local):gemini-3.1-pro Suggested-by: Vincent Donnefort Signed-off-by: Fuad Tabba Reviewed-by: Vincent Donnefort Link: https://patch.msgid.link/20260529121755.2923500-3-tabba@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 0abf3a2d587b..c82d4ececab8 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -524,13 +524,17 @@ static int unshare_pfn_hyp(u64 pfn) goto unlock; } - this->count--; - if (this->count) + if (this->count > 1) { + this->count--; + goto unlock; + } + + ret = kvm_call_hyp_nvhe(__pkvm_host_unshare_hyp, pfn); + if (ret) goto unlock; rb_erase(&this->node, &hyp_shared_pfns); kfree(this); - ret = kvm_call_hyp_nvhe(__pkvm_host_unshare_hyp, pfn); unlock: mutex_unlock(&hyp_shared_pfns_lock); @@ -581,6 +585,11 @@ void kvm_unshare_hyp(void *from, void *to) end = PAGE_ALIGN(__pa(to)); for (cur = start; cur < end; cur += PAGE_SIZE) { pfn = __phys_to_pfn(cur); + /* + * A failed unshare leaks the page: it stays shared with the + * hypervisor and is no longer reusable for pKVM. No isolation + * guarantee is broken, and this is not expected in practice. + */ WARN_ON(unshare_pfn_hyp(pfn)); } } From f4411f9308c0187c211577b7c489545b0bdae455 Mon Sep 17 00:00:00 2001 From: "tabba@google.com" Date: Fri, 29 May 2026 13:17:55 +0100 Subject: [PATCH 12/12] KVM: arm64: Roll back partial shares on kvm_share_hyp() failure kvm_share_hyp() shares a range one page at a time. If share_pfn_hyp() fails partway through, the pages already shared by this call are left shared, while the caller treats the whole range as failed and never unshares them. Unshare those pages before returning the error. If an unshare itself fails the page is leaked: it stays shared with the hypervisor and is no longer reusable for pKVM, but no isolation guarantee is broken, so WARN and continue. Not expected in practice. Fixes: a83e2191b7f1 ("KVM: arm64: pkvm: Refcount the pages shared with EL2") Suggested-by: Vincent Donnefort Signed-off-by: Fuad Tabba Reviewed-by: Vincent Donnefort Link: https://patch.msgid.link/20260529121755.2923500-4-tabba@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index c82d4ececab8..f18b287c98b6 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -544,8 +544,8 @@ static int unshare_pfn_hyp(u64 pfn) int kvm_share_hyp(void *from, void *to) { phys_addr_t start, end, cur; + int ret = 0; u64 pfn; - int ret; if (is_kernel_in_hyp_mode()) return 0; @@ -567,10 +567,24 @@ int kvm_share_hyp(void *from, void *to) pfn = __phys_to_pfn(cur); ret = share_pfn_hyp(pfn); if (ret) - return ret; + break; } - return 0; + if (!ret) + return 0; + + /* + * Roll back the pages shared by this call. A failed unshare leaks + * the page (it stays shared with the hypervisor and is no longer + * reusable for pKVM) but breaks no isolation guarantee, so warn and + * continue. Not expected in practice. + */ + for (end = cur, cur = start; cur < end; cur += PAGE_SIZE) { + pfn = __phys_to_pfn(cur); + WARN_ON(unshare_pfn_hyp(pfn)); + } + + return ret; } void kvm_unshare_hyp(void *from, void *to)