From 7258770e5814f15e8308ebda82ac9acf6964ba8e Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 15 Jun 2026 19:16:25 +0100 Subject: [PATCH 01/48] KVM: arm64: vgic: Handle race between interrupt affinity change and LPI disabling Hyunwoo Kim reports some really bad races should the following situation occur: - LPI-I is pending in vcpu-B's AP list - vcpu-A writes to vcpu-B's RD to disable its LPIs - vcpu-C moves I from B to C If the last two race nicely enough, vgic_prune_ap_list() can drop the irq and AP list locks, reacquire them, and in the interval the irq has been freed. UAF follows. The fix is two-fold: - Before dropping the irq and ap_list locks, take a reference on the irq - Do not try to handle migration of the pending bit: there is no expectation that this state is retained, as per the architecture With that, we're sure that the interrupt is still around, and we safely remove it from the AP list as it has no target at this stage (unless another interrupt fires, but that's another story). Reported-by: Hyunwoo Kim Tested-by: Hyunwoo Kim Link: https://lore.kernel.org/r/ailsCnyoS82r_QRz@v4bel Link: https://patch.msgid.link/20260615181625.3029352-1-maz@kernel.org Fixes: 5dd4b924e390a ("KVM: arm/arm64: vgic: Add refcounting for IRQs") Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org --- arch/arm64/kvm/vgic/vgic.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 5a4768d8cd4f..70a161383e5a 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -203,6 +203,7 @@ void vgic_flush_pending_lpis(struct kvm_vcpu *vcpu) list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) { if (irq_is_lpi(vcpu->kvm, irq->intid)) { raw_spin_lock(&irq->irq_lock); + irq->pending_latch = false; list_del(&irq->ap_list); irq->vcpu = NULL; raw_spin_unlock(&irq->irq_lock); @@ -792,7 +793,11 @@ static void vgic_prune_ap_list(struct kvm_vcpu *vcpu) continue; } - /* This interrupt looks like it has to be migrated. */ + /* + * This interrupt looks like it has to be migrated, + * make sure it is kept alive while locks are dropped. + */ + vgic_get_irq_ref(irq); raw_spin_unlock(&irq->irq_lock); raw_spin_unlock(&vgic_cpu->ap_list_lock); @@ -836,6 +841,8 @@ static void vgic_prune_ap_list(struct kvm_vcpu *vcpu) raw_spin_unlock(&vcpuB->arch.vgic_cpu.ap_list_lock); raw_spin_unlock(&vcpuA->arch.vgic_cpu.ap_list_lock); + deleted_lpis |= vgic_put_irq_norelease(vcpu->kvm, irq); + if (target_vcpu_needs_kick) { kvm_make_request(KVM_REQ_IRQ_PENDING, target_vcpu); kvm_vcpu_kick(target_vcpu); From 0074b82cdfcb5fd13710a0ac308ade68ac6f6fbe Mon Sep 17 00:00:00 2001 From: Hyunwoo Kim Date: Fri, 5 Jun 2026 05:59:15 +0900 Subject: [PATCH 02/48] KVM: arm64: vgic: Check the interrupt is still ours before migrating it vgic_prune_ap_list() drops both ap_list_lock and irq_lock while migrating an interrupt to another vCPU. After reacquiring the locks it only checks that the affinity is unchanged (target_vcpu == vgic_target_oracle(irq)) before moving the interrupt, which assumes that an interrupt whose affinity is preserved is still queued on this vCPU's ap_list. That assumption no longer holds if the interrupt is taken off the ap_list while the locks are dropped. vgic_flush_pending_lpis() removes the interrupt from the list and sets irq->vcpu to NULL, but leaves enabled/pending/target_vcpu untouched. As the interrupt is still enabled and pending, vgic_target_oracle() returns the same target_vcpu, so the affinity check passes and list_del() is run a second time on an entry that has already been removed. Also check that the interrupt is still assigned to this vCPU (irq->vcpu == vcpu) before moving it. Fixes: 0919e84c0fc1 ("KVM: arm/arm64: vgic-new: Add IRQ sync/flush framework") Signed-off-by: Hyunwoo Kim Link: https://patch.msgid.link/aiHnI1mu6SGQrgnz@v4bel Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org --- arch/arm64/kvm/vgic/vgic.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 70a161383e5a..ccb7e3a90cd0 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -820,15 +820,16 @@ static void vgic_prune_ap_list(struct kvm_vcpu *vcpu) raw_spin_lock(&irq->irq_lock); /* - * If the affinity has been preserved, move the - * interrupt around. Otherwise, it means things have - * changed while the interrupt was unlocked, and we - * need to replay this. + * If the interrupt is still ours and its affinity has + * been preserved, move it around. Otherwise, it means + * things have changed while the interrupt was unlocked + * (it may even have been taken off the list with its + * affinity left untouched), and we need to replay this. * * In all cases, we cannot trust the list not to have * changed, so we restart from the beginning. */ - if (target_vcpu == vgic_target_oracle(irq)) { + if (irq->vcpu == vcpu && target_vcpu == vgic_target_oracle(irq)) { struct vgic_cpu *new_cpu = &target_vcpu->arch.vgic_cpu; list_del(&irq->ap_list); From ff1022c3de46753eb7eba2f6efd990569e66ff95 Mon Sep 17 00:00:00 2001 From: Weiming Shi Date: Wed, 17 Jun 2026 12:08:21 +0800 Subject: [PATCH 03/48] KVM: arm64: nv: Fix SPSR_EL2 restore in kvm_hyp_handle_mops() kvm_hyp_handle_mops() resets the single-step state machine as part of rewinding state for a MOPS exception by modifying vcpu_cpsr() and writing the result directly into hardware. In the case of nested virtualization, vcpu_cpsr() is a synthetic value such that the rest of KVM can deal with vEL2 cleanly. That means the value requires translation before being written into hardware, which is unfortunately missing from the MOPS handler. Fix it by directly modifying SPSR_EL2 and avoiding the synthetic state altogether, which will be resynchronized on the next 'full' exit back to KVM. Fixes: 2de451a329cf ("KVM: arm64: Add handler for MOPS exceptions") Reported-by: Zhong Wang Reported-by: Xuanqing Shi Link: https://lore.kernel.org/all/ajE4lHQevXNHpl1M@Air.local/ Cc: stable@vger.kernel.org Signed-off-by: Weiming Shi Link: https://patch.msgid.link/20260617040820.2194831-2-bestswngs@gmail.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/include/hyp/switch.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 161bb2a3e1d9..d56371b189bf 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -446,16 +446,19 @@ static inline bool __populate_fault_info(struct kvm_vcpu *vcpu) static inline bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code) { + u64 spsr; + *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR); arm64_mops_reset_regs(vcpu_gp_regs(vcpu), vcpu->arch.fault.esr_el2); write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); /* * Finish potential single step before executing the prologue - * instruction. + * instruction. Modify the hardware SPSR_EL2 directly, as vcpu_cpsr() + * may hold a synthetic (vEL2) value for a guest hypervisor. */ - *vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS; - write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); + spsr = read_sysreg_el2(SYS_SPSR); + write_sysreg_el2(spsr & ~DBG_SPSR_SS, SYS_SPSR); return true; } From 9f1667098c6ae7ec81a9a56859cfdacb822aa0d0 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Sun, 14 Jun 2026 22:13:24 -0700 Subject: [PATCH 04/48] KVM: arm64: nv: Drop bogus WARN for write to ZCR_EL2 It is entirely possible for a guest to write to the ZCR_EL2 sysreg alias while in a nested context, as it is expected if FEAT_NV2 is advertised to the L1 hypervisor. Get rid of the bogus WARN which, since the hyp vectors were installed at this point, has the effect of a hyp_panic... Cc: stable@vger.kernel.org Fixes: 0cfc85b8f5cf ("KVM: arm64: nv: Load guest FP state for ZCR_EL2 trap") Signed-off-by: Oliver Upton Link: https://patch.msgid.link/20260615051324.830045-1-oupton@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/include/hyp/switch.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index d56371b189bf..ea22d1b50512 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -598,8 +598,6 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) return false; break; case ESR_ELx_EC_SYS64: - if (WARN_ON_ONCE(!is_hyp_ctxt(vcpu))) - return false; fallthrough; case ESR_ELx_EC_SVE: if (!sve_guest) From e2cb1f4578625e71f461d5c1ce70984193389cbb Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 15 Jun 2026 14:11:16 +0100 Subject: [PATCH 05/48] KVM: arm64: nv: Write ESR_EL2 for injected nested SError exceptions kvm_inject_el2_exception() writes ESR_EL2 for synchronous exceptions but not for SError. enter_exception64() does not write ESR_ELx for any exception type, so the constructed syndrome is dropped. A guest L2 hypervisor taking a nested SError observes stale ESR_EL2. This affects both kvm_inject_nested_serror() and the EASE path in kvm_inject_nested_sea(). Write ESR_EL2 for except_type_serror, matching except_type_sync. Fixes: 77ee70a07357 ("KVM: arm64: nv: Honor SError exception routing / masking") Reported-by: sashiko Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260615131116.390977-1-tabba@google.com Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org --- arch/arm64/kvm/emulate-nested.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index e688bc5139c1..76c3e6c1144c 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -2826,6 +2826,7 @@ static void kvm_inject_el2_exception(struct kvm_vcpu *vcpu, u64 esr_el2, break; case except_type_serror: kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SERR); + vcpu_write_sys_reg(vcpu, esr_el2, ESR_EL2); break; default: WARN_ONCE(1, "Unsupported EL2 exception injection %d\n", type); From ec40342aaca8162bc8ab2607076535ebab1838b8 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 12 Jun 2026 12:34:14 +0100 Subject: [PATCH 06/48] KVM: arm64: Sync SPSR_EL1 when injecting an exception into a pVM When pKVM injects a synchronous exception into a protected guest, it re-enters without restoring the guest's EL1 sysregs and writes the EL1 exception registers to hardware by hand: ESR_EL1 and ELR_EL1, but not SPSR_EL1. enter_exception64() sets SPSR_EL1 (the interrupted PSTATE) only in memory, so the guest's handler reads a stale SPSR_EL1 and restores the wrong PSTATE on eret. Write SPSR_EL1 alongside the other exception registers. Fixes: 6c30bfb18d0b ("KVM: arm64: Add handlers for protected VM System Registers") Reported-by: sashiko Signed-off-by: Fuad Tabba Acked-by: Will Deacon Link: https://patch.msgid.link/20260612113414.1022901-1-tabba@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/sys_regs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c index 8c3fbb413a06..1a7d5cd16d72 100644 --- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c +++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c @@ -268,6 +268,7 @@ static void inject_sync64(struct kvm_vcpu *vcpu, u64 esr) write_sysreg_el1(esr, SYS_ESR); write_sysreg_el1(read_sysreg_el2(SYS_ELR), SYS_ELR); + write_sysreg_el1(read_sysreg_el2(SYS_SPSR), SYS_SPSR); write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); } From 0e8e955b9bcf84a70f20079390e19971fec1586d Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 17 Jun 2026 15:49:07 +0100 Subject: [PATCH 07/48] KVM: arm64: nv: Fix PSTATE construction on illegal exception return kvm_check_illegal_exception_return() sourced the flags {N,Z,C,V} and masks {D,A,I,F} of the resulting PSTATE from the current PSTATE, but R_VWJHB takes them from the SPSR being returned to and leaves PSTATE.{EL,SP,nRW} (and EXLOCK when FEAT_GCS) unchanged. PAN, ALLINT and PM were not applied at all. Build the PSTATE by taking those fields from the SPSR while preserving EL, SP, nRW and EXLOCK from the current PSTATE, then set IL. Fixes: 47f3a2fc765a ("KVM: arm64: nv: Support virtual EL2 exceptions") Suggested-by: Marc Zyngier Link: https://lore.kernel.org/all/86wlvxs5r0.wl-maz@kernel.org/ Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260617144907.2972095-1-tabba@google.com [maz: tidied things a bit] Signed-off-by: Marc Zyngier --- arch/arm64/kvm/emulate-nested.c | 36 ++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index 76c3e6c1144c..96ebe7e3b408 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -2746,17 +2746,33 @@ static u64 kvm_check_illegal_exception_return(struct kvm_vcpu *vcpu, u64 spsr) (spsr & PSR_MODE32_BIT) || (vcpu_el2_tge_is_set(vcpu) && (mode == PSR_MODE_EL1t || mode == PSR_MODE_EL1h))) { - /* - * The guest is playing with our nerves. Preserve EL, SP, - * masks, flags from the existing PSTATE, and set IL. - * The HW will then generate an Illegal State Exception - * immediately after ERET. - */ - spsr = *vcpu_cpsr(vcpu); + u64 mask; - spsr &= (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | - PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT | - PSR_MODE_MASK | PSR_MODE32_BIT); + /* + * On an illegal exception return, the flags and masks are + * taken from the SPSR while PSTATE.{EL,SP,nRW} and, if + * FEAT_GCS, PSTATE.EXLOCK are unchanged (R_VWJHB). Set IL + * so the HW generates an Illegal State Exception right + * after ERET. + */ + mask = PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | + PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT; + + if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, PAN, IMP)) + mask |= PSR_PAN_BIT; + if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, NMI, IMP)) + mask |= ALLINT_ALLINT; + /* FEAT_SPE_EXC and FEAT_TRBE_EXC also gate PSTATE.PM one day... */ + if (kvm_has_feat(vcpu->kvm, ID_AA64DFR1_EL1, EBEP, IMP)) + mask |= BIT_ULL(32); /* SPSR_ELx.PM */ + + spsr &= mask; + + mask = PSR_MODE_MASK | PSR_MODE32_BIT; + if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, GCS, IMP)) + mask |= BIT_ULL(34); /* PSTATE.EXLOCK */ + + spsr |= *vcpu_cpsr(vcpu) & mask; spsr |= PSR_IL_BIT; } From fcda30b134d262a1b1826555352bd7349051486e Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 17 Jun 2026 14:12:36 +0100 Subject: [PATCH 08/48] KVM: arm64: Add Fuad Tabba as a reviewer I have been working on KVM/arm64 for a couple of years, mostly on pKVM, and am currently upstreaming protected guest support, with more to come later. I already review KVM/arm64 patches more broadly, and am happy to continue doing so in an official capacity. Add myself to the KVM/arm64 reviewer list. Signed-off-by: Fuad Tabba Acked-by: Oliver Upton Link: https://patch.msgid.link/20260617131237.2842619-1-tabba@google.com Signed-off-by: Marc Zyngier --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index c2c6d79275c6..30cba9f7491e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14060,6 +14060,7 @@ F: virt/kvm/* KERNEL VIRTUAL MACHINE FOR ARM64 (KVM/arm64) M: Marc Zyngier M: Oliver Upton +R: Fuad Tabba R: Joey Gouly R: Steffen Eiden R: Suzuki K Poulose From 2684e02bac41c5220f6c1ab2bdcc957b71812977 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Thu, 18 Jun 2026 16:42:02 -0700 Subject: [PATCH 09/48] KVM: arm64: nv: Respect read-only PFN when mapping L1 VNCR KVM currently maps the L1 VNCR into the host stage-1 by relying entirely on the permissions of the guest stage-1. At the same time, it is entirely possible that the backing PFN is read-only (e.g. RO memslot), meaning that the L1 VNCR should use at most a read-only mapping. Cache the writability of the PFN in the VNCR TLB and use it to constrain the resulting fixmap permissions. Promote VNCR permission faults to an SEA in the case where the guest attempts to write to a read-only endpoint. Conveniently, this also plugs a page leak found by Sashiko [*] resulting from the early return for a read-only PFN. Cc: stable@vger.kernel.org Fixes: 2a359e072596 ("KVM: arm64: nv: Handle mapping of VNCR_EL2 at EL2") Link: https://lore.kernel.org/kvm/20260608082603.16AEC1F00893@smtp.kernel.org/ Signed-off-by: Oliver Upton Link: https://patch.msgid.link/20260618234207.1063941-2-oupton@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/nested.c | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 3a5571c3c114..903ccabca78c 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -24,6 +24,7 @@ struct vncr_tlb { struct s1_walk_result wr; u64 hpa; + bool hpa_writable; /* -1 when not mapped on a CPU */ int cpu; @@ -1401,7 +1402,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) if (!*is_gmem) { pfn = __kvm_faultin_pfn(memslot, gfn, write_fault ? FOLL_WRITE : 0, &writable, &page); - if (is_error_noslot_pfn(pfn) || (write_fault && !writable)) + if (is_error_noslot_pfn(pfn)) return -EFAULT; } else { ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL); @@ -1410,6 +1411,8 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) write_fault, false, false); return ret; } + + writable = !(memslot->flags & KVM_MEM_READONLY); } scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { @@ -1420,28 +1423,41 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) vt->gva = va; vt->hpa = pfn << PAGE_SHIFT; + vt->hpa_writable = writable; vt->valid = true; vt->cpu = -1; kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); - kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw); + kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable); } - if (vt->wr.pw) + if (vt->wr.pw && vt->hpa_writable) mark_page_dirty(vcpu->kvm, gfn); return 0; } -static void inject_vncr_perm(struct kvm_vcpu *vcpu) +static void handle_vncr_perm(struct kvm_vcpu *vcpu) { struct vncr_tlb *vt = vcpu->arch.vncr_tlb; u64 esr = kvm_vcpu_get_esr(vcpu); + u64 fsc; + + /* + * Promote to an external abort if the stage-1 permits writes but the + * HPA is read-only (e.g. RO memslot). + */ + if (kvm_is_write_fault(vcpu) && vt->wr.pw && !vt->hpa_writable) + fsc = ESR_ELx_FSC_EXTABT; + /* + * Otherwise, inject a permission fault using the guest's translation + * level rather than the host's. + */ + else + fsc = ESR_ELx_FSC_PERM_L(vt->wr.level); - /* Adjust the fault level to reflect that of the guest's */ esr &= ~ESR_ELx_FSC; - esr |= FIELD_PREP(ESR_ELx_FSC, - ESR_ELx_FSC_PERM_L(vt->wr.level)); + esr |= FIELD_PREP(ESR_ELx_FSC, fsc); kvm_inject_nested_sync(vcpu, esr); } @@ -1475,7 +1491,7 @@ int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu) return kvm_handle_guest_sea(vcpu); if (esr_fsc_is_permission_fault(esr)) { - inject_vncr_perm(vcpu); + handle_vncr_perm(vcpu); } else if (esr_fsc_is_translation_fault(esr)) { bool valid, is_gmem = false; int ret; @@ -1523,7 +1539,7 @@ int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu) break; case -EPERM: /* Hack to deal with POE until we get kernel support */ - inject_vncr_perm(vcpu); + handle_vncr_perm(vcpu); break; case 0: break; @@ -1567,7 +1583,7 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) vt->cpu = smp_processor_id(); - if (vt->wr.pw && vt->wr.pr) + if (vt->hpa_writable && vt->wr.pw && vt->wr.pr) prot = PAGE_KERNEL; else if (vt->wr.pr) prot = PAGE_KERNEL_RO; From 9f3e83345a56280efffe235c65593c7e544c0fcc Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Thu, 18 Jun 2026 16:42:03 -0700 Subject: [PATCH 10/48] KVM: arm64: nv: Inject SEA if kvm_translate_vncr() can't resolve PFN kvm_handle_vncr_abort() assumes that s1_walk_result conveys an abort when kvm_translate_vncr() returns -EFAULT. This is not always the case as it's possible to encounter 'late' failures on the output of S1 translation, e.g. a GFN outside of the memslots. Fix it by preparing an external abort before returning from kvm_translate_vncr(). Get rid of the BUG_ON() in the fault injection path while at it. Cc: stable@vger.kernel.org Fixes: 2a359e072596 ("KVM: arm64: nv: Handle mapping of VNCR_EL2 at EL2") Signed-off-by: Oliver Upton Link: https://patch.msgid.link/20260618234207.1063941-3-oupton@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_nested.h | 8 ++++++++ arch/arm64/kvm/at.c | 8 -------- arch/arm64/kvm/nested.c | 10 ++++++---- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index dc2957662ff2..cbdaaa2a2903 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -388,6 +388,14 @@ struct s1_walk_result { bool failed; }; +static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) +{ + wr->fst = fst; + wr->ptw = s1ptw; + wr->s2 = s1ptw; + wr->failed = true; +} + int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, struct s1_walk_result *wr, u64 va); int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa, diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c index 30e6fa8ac07c..8263c648207b 100644 --- a/arch/arm64/kvm/at.c +++ b/arch/arm64/kvm/at.c @@ -11,14 +11,6 @@ #include #include -static void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) -{ - wr->fst = fst; - wr->ptw = s1ptw; - wr->s2 = s1ptw; - wr->failed = true; -} - #define S1_MMU_DISABLED (-127) static int get_ia_size(struct s1_walk_info *wi) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 903ccabca78c..53dea9c3f14f 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1395,15 +1395,19 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) gfn = vt->wr.pa >> PAGE_SHIFT; memslot = gfn_to_memslot(vcpu->kvm, gfn); - if (!memslot) + if (!memslot) { + fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); return -EFAULT; + } *is_gmem = kvm_slot_has_gmem(memslot); if (!*is_gmem) { pfn = __kvm_faultin_pfn(memslot, gfn, write_fault ? FOLL_WRITE : 0, &writable, &page); - if (is_error_noslot_pfn(pfn)) + if (is_error_noslot_pfn(pfn)) { + fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); return -EFAULT; + } } else { ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL); if (ret) { @@ -1530,8 +1534,6 @@ int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu) * Translation failed, inject the corresponding * exception back to EL2. */ - BUG_ON(!vt->wr.failed); - esr &= ~ESR_ELx_FSC; esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst); From bb645aa0a4caeaf7f9cd32e9a948594d434c1a8f Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Thu, 18 Jun 2026 16:42:04 -0700 Subject: [PATCH 11/48] KVM: arm64: nv: Re-translate VNCR before injecting abort KVM faults in the VNCR page with FOLL_WRITE whenever the guest aborts for a write, similar to how a regular stage-2 mapping is handled. It is entirely possible that the guest reads from the VNCR before writing to it, in which case the PFN could only be read-only. Invalidate the VNCR TLB and re-fetch the translation upon taking a VNCR abort, allowing the host mapping to be faulted in for write the second time around. Interestingly enough, this also satisfies the ordering requirements of FEAT_ETS2/3 between descriptor updates and MMU faults. Cc: stable@vger.kernel.org Fixes: 2a359e072596 ("KVM: arm64: nv: Handle mapping of VNCR_EL2 at EL2") Reported-by: Sashiko Signed-off-by: Oliver Upton Link: https://patch.msgid.link/20260618234207.1063941-4-oupton@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/nested.c | 115 +++++++++++++++------------------------- 1 file changed, 44 insertions(+), 71 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 53dea9c3f14f..7fffd86eee94 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1466,88 +1466,61 @@ static void handle_vncr_perm(struct kvm_vcpu *vcpu) kvm_inject_nested_sync(vcpu, esr); } -static bool kvm_vncr_tlb_lookup(struct kvm_vcpu *vcpu) -{ - struct vncr_tlb *vt = vcpu->arch.vncr_tlb; - - lockdep_assert_held_read(&vcpu->kvm->mmu_lock); - - if (!vt->valid) - return false; - - if (read_vncr_el2(vcpu) != vt->gva) - return false; - - if (vt->wr.nG) - return get_asid_by_regime(vcpu, TR_EL20) == vt->wr.asid; - - return true; -} - int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu) { struct vncr_tlb *vt = vcpu->arch.vncr_tlb; u64 esr = kvm_vcpu_get_esr(vcpu); + bool is_gmem = false; + bool perm; + int ret; WARN_ON_ONCE(!(esr & ESR_ELx_VNCR)); if (kvm_vcpu_abt_issea(vcpu)) return kvm_handle_guest_sea(vcpu); - if (esr_fsc_is_permission_fault(esr)) { - handle_vncr_perm(vcpu); - } else if (esr_fsc_is_translation_fault(esr)) { - bool valid, is_gmem = false; - int ret; - - scoped_guard(read_lock, &vcpu->kvm->mmu_lock) - valid = kvm_vncr_tlb_lookup(vcpu); - - if (!valid) - ret = kvm_translate_vncr(vcpu, &is_gmem); - else - ret = -EPERM; - - switch (ret) { - case -EAGAIN: - /* Let's try again... */ - break; - case -ENOMEM: - /* - * For guest_memfd, this indicates that it failed to - * create a folio to back the memory. Inform userspace. - */ - if (is_gmem) - return 0; - /* Otherwise, let's try again... */ - break; - case -EFAULT: - case -EIO: - case -EHWPOISON: - if (is_gmem) - return 0; - fallthrough; - case -EINVAL: - case -ENOENT: - case -EACCES: - /* - * Translation failed, inject the corresponding - * exception back to EL2. - */ - esr &= ~ESR_ELx_FSC; - esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst); - - kvm_inject_nested_sync(vcpu, esr); - break; - case -EPERM: - /* Hack to deal with POE until we get kernel support */ - handle_vncr_perm(vcpu); - break; - case 0: - break; - } - } else { + if (!esr_fsc_is_translation_fault(esr) && !esr_fsc_is_permission_fault(esr)) { WARN_ONCE(1, "Unhandled VNCR abort, ESR=%llx\n", esr); + return 1; + } + + ret = kvm_translate_vncr(vcpu, &is_gmem); + switch (ret) { + case -EAGAIN: + /* Let's try again... */ + return 1; + case -ENOMEM: + /* + * For guest_memfd, this indicates that it failed to + * create a folio to back the memory. Inform userspace. + */ + if (is_gmem) + return 0; + /* Otherwise, let's try again... */ + break; + case -EFAULT: + case -EIO: + case -EHWPOISON: + if (is_gmem) + return 0; + fallthrough; + case -EINVAL: + case -ENOENT: + case -EACCES: + /* + * Translation failed, inject the corresponding + * exception back to EL2. + */ + esr &= ~ESR_ELx_FSC; + esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst); + + kvm_inject_nested_sync(vcpu, esr); + break; + case 0: + perm = kvm_is_write_fault(vcpu) ? vt->wr.pw && vt->hpa_writable : vt->wr.pr; + if (!perm) + handle_vncr_perm(vcpu); + break; } return 1; From 4bd7dbe0b2243e6aa735cae4d5e1ff988b30b2a6 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Thu, 18 Jun 2026 16:42:05 -0700 Subject: [PATCH 12/48] KVM: arm64: nv: Inject SEA if guest VNCR isn't normal memory When constructing an L1 VNCR mapping, KVM unconditionally uses cacheable memory attributes, even if the underlying PFN isn't memory. This gets particularly hairy if the endpoint doesn't support cacheable memory attributes, potentially throwing an SError on writeback... While KVM does permit cacheable memory attributes on certain PFNMAP VMAs, kvm_translate_vncr() isn't currently grabbing the VMA. So do the simpler thing for now and just reject everything that isn't memory. Cc: stable@vger.kernel.org Fixes: 2a359e072596 ("KVM: arm64: nv: Handle mapping of VNCR_EL2 at EL2") Signed-off-by: Oliver Upton Link: https://patch.msgid.link/20260618234207.1063941-5-oupton@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/nested.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 7fffd86eee94..d4c9a9b05e3f 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1419,6 +1419,17 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) writable = !(memslot->flags & KVM_MEM_READONLY); } + /* + * FIXME: This check is too restrictive as KVM allows cacheable memory + * attributes for PFNMAP VMAs that have cacheable attributes in host + * stage-1. + */ + if (!pfn_is_map_memory(pfn)) { + kvm_release_faultin_page(vcpu->kvm, page, true, false); + fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); + return -EINVAL; + } + scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) { kvm_release_faultin_page(vcpu->kvm, page, true, false); From 265b58aba51b6aaaad81678fbc57fcdb2d4ed480 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Thu, 18 Jun 2026 16:42:06 -0700 Subject: [PATCH 13/48] KVM: arm64: nv: Mark VM as bugged for unexpected VNCR abort KVM is unlikely to resolve an unexpected VNCR abort, meaning that returning to the guest will likely leave the vCPU stuck in an abort loop. Bug the VM and exit to userspace instead. Signed-off-by: Oliver Upton Link: https://patch.msgid.link/20260618234207.1063941-6-oupton@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/nested.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index d4c9a9b05e3f..94df26de6990 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1491,8 +1491,8 @@ int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu) return kvm_handle_guest_sea(vcpu); if (!esr_fsc_is_translation_fault(esr) && !esr_fsc_is_permission_fault(esr)) { - WARN_ONCE(1, "Unhandled VNCR abort, ESR=%llx\n", esr); - return 1; + KVM_BUG(1, vcpu->kvm, "Unhandled VNCR abort, ESR=%llx\n", esr); + return -EIO; } ret = kvm_translate_vncr(vcpu, &is_gmem); From daa71eca24fdfb43029830bd57ddaddf70c59b23 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 18 Jun 2026 13:16:37 +0100 Subject: [PATCH 14/48] KVM: arm64: Set ESR_ELx.IL for injected undefined exceptions at EL2 inject_undef64() constructs an ESR with EC=0 (Unknown) but does not set IL. The architecture mandates IL=1 for EC=0 unconditionally (ARM DDI 0487, ESR_ELx.IL description), so the injected syndrome is one that conforming hardware cannot produce. Set ESR_ELx_IL in the constructed syndrome. Fixes: e5d40a5a97c1 ("KVM: arm64: pkvm: Add a generic synchronous exception injection primitive") Reported-by: sashiko Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260618121643.4105064-2-tabba@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/sys_regs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c index 1a7d5cd16d72..b1411fb54139 100644 --- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c +++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c @@ -279,7 +279,7 @@ static void inject_sync64(struct kvm_vcpu *vcpu, u64 esr) */ static void inject_undef64(struct kvm_vcpu *vcpu) { - inject_sync64(vcpu, (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT)); + inject_sync64(vcpu, (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT) | ESR_ELx_IL); } static u64 read_id_reg(const struct kvm_vcpu *vcpu, From 1d695dc827957e9570d1b56abac1250d2d13bf0c Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 18 Jun 2026 13:16:38 +0100 Subject: [PATCH 15/48] KVM: arm64: Unconditionally set IL for injected undefined exceptions inject_undef64() derives IL from the triggering trap's instruction length (kvm_vcpu_trap_il_is32bit()), but the IL of the injected exception is fixed by its EC, not by the triggering instruction. The architecture mandates IL=1 for EC=0 (Unknown) unconditionally, so the conditional is wrong. The undef-injection paths are not reached from 16-bit instructions, so there is no functional change today, but the logic should not rely on that. Set ESR_ELx_IL unconditionally. Fixes: aa8eff9bfbd5 ("arm64: KVM: fault injection into a guest") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260618121643.4105064-3-tabba@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/inject_fault.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c index 89982bd3345f..9dfae1bcdf99 100644 --- a/arch/arm64/kvm/inject_fault.c +++ b/arch/arm64/kvm/inject_fault.c @@ -170,14 +170,7 @@ void kvm_inject_sync(struct kvm_vcpu *vcpu, u64 esr) static void inject_undef64(struct kvm_vcpu *vcpu) { - u64 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT); - - /* - * Build an unknown exception, depending on the instruction - * set. - */ - if (kvm_vcpu_trap_il_is32bit(vcpu)) - esr |= ESR_ELx_IL; + u64 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT) | ESR_ELx_IL; kvm_inject_sync(vcpu, esr); } From add40af98b34764ff5603dce297160fde12d784c Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 18 Jun 2026 13:16:39 +0100 Subject: [PATCH 16/48] KVM: arm64: Unconditionally set IL for injected abort exceptions inject_abt64() derives IL from the triggering trap's instruction length (kvm_vcpu_trap_il_is32bit()), but the IL of the injected abort is fixed by its EC, not by the triggering instruction. The architecture mandates IL=1 for Instruction Aborts unconditionally and for Data Aborts with ISV=0, and this function never sets ISV (the FSC is always EXTABT or SEA_TTW). For a 16-bit T32 trap (a 32-bit EL0 task under an AArch64 EL1 guest) the trap has IL=0, so the abort is injected with the wrong IL. Set ESR_ELx_IL unconditionally. Fixes: aa8eff9bfbd5 ("arm64: KVM: fault injection into a guest") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260618121643.4105064-4-tabba@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/inject_fault.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c index 9dfae1bcdf99..444d219b0217 100644 --- a/arch/arm64/kvm/inject_fault.c +++ b/arch/arm64/kvm/inject_fault.c @@ -138,11 +138,10 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr pend_sync_exception(vcpu); /* - * Build an {i,d}abort, depending on the level and the - * instruction set. Report an external synchronous abort. + * Build an {i,d}abort, depending on the level. + * Report an external synchronous abort. */ - if (kvm_vcpu_trap_il_is32bit(vcpu)) - esr |= ESR_ELx_IL; + esr |= ESR_ELx_IL; /* * Here, the guest runs in AArch64 mode when in EL1. If we get From a52d6d68ad30374dd794bff300d8538e35ee49a8 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 18 Jun 2026 13:16:40 +0100 Subject: [PATCH 17/48] KVM: arm64: Set IL for injected FPAC exceptions during ERET emulation The FPAC syndrome constructed during nested ERET emulation does not set IL. For FPAC (EC=0x1C), IL reflects the instruction length. ERET and its authenticated variants are always A64 32-bit instructions, so IL must be 1. Fixes: 213b3d1ea161 ("KVM: arm64: nv: Handle ERETA[AB] instructions") Suggested-by: Marc Zyngier Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260618121643.4105064-5-tabba@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/emulate-nested.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index 96ebe7e3b408..a15b2f41a12c 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -2800,7 +2800,7 @@ void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu) * ERET handling, and the guest will have a little surprise. */ if (kvm_has_pauth(vcpu->kvm, FPACCOMBINE) && !(spsr & PSR_IL_BIT)) { - esr &= ESR_ELx_ERET_ISS_ERETA; + esr &= (ESR_ELx_ERET_ISS_ERETA | ESR_ELx_IL); esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_FPAC); kvm_inject_nested_sync(vcpu, esr); return; From 7514f1785d526207af8512cc6ccb1c35c5c61767 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 18 Jun 2026 13:16:41 +0100 Subject: [PATCH 18/48] KVM: arm64: Set IL for emulated SError injection kvm_inject_serror_esr() constructs an SError syndrome without IL. The architecture mandates IL=1 for SError unconditionally. Fixes: f6e2262dfa1a ("KVM: arm64: Populate ESR_ELx.EC for emulated SError injection") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260618121643.4105064-6-tabba@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/inject_fault.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c index 444d219b0217..d6c4fc16f879 100644 --- a/arch/arm64/kvm/inject_fault.c +++ b/arch/arm64/kvm/inject_fault.c @@ -381,7 +381,7 @@ int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr) */ if (!serror_is_masked(vcpu)) { pend_serror_exception(vcpu); - esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR); + esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR) | ESR_ELx_IL; vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu)); return 1; } From a69412287a33c931dca9e48d30c0dbf8cde0ffe6 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 18 Jun 2026 13:16:42 +0100 Subject: [PATCH 19/48] KVM: arm64: Set IL for nested SError injection kvm_inject_nested_serror() constructs an SError syndrome without IL. The architecture mandates IL=1 for SError unconditionally. Fixes: 77ee70a07357 ("KVM: arm64: nv: Honor SError exception routing / masking") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260618121643.4105064-7-tabba@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/emulate-nested.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index a15b2f41a12c..3c82f392845d 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -2967,6 +2967,6 @@ int kvm_inject_nested_serror(struct kvm_vcpu *vcpu, u64 esr) * vSError injection. Manually populate EC for an emulated SError * exception. */ - esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR); + esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR) | ESR_ELx_IL; return kvm_inject_nested(vcpu, esr, except_type_serror); } From cbe2278aa3dd6832c544782c6cfed1fbc1f71a42 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 18 Jun 2026 13:16:43 +0100 Subject: [PATCH 20/48] KVM: arm64: Set IL in fake ESR for pKVM memory sharing exit __pkvm_memshare_page_req() constructs a fake DABT ESR_EL2 to exit to the host without setting IL. The ESR has ISV=0, so IL must be 1 per the architecture. The host does not read IL on this path, but the constructed syndrome should still be architecturally valid. Set ESR_ELx_IL. Fixes: 03313efed5e2 ("KVM: arm64: Implement the MEM_SHARE hypercall for protected VMs") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260618121643.4105064-8-tabba@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/pkvm.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 3b2c4fbc34d8..24d6f164129a 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -1056,7 +1056,8 @@ static u64 __pkvm_memshare_page_req(struct kvm_vcpu *vcpu, u64 ipa) /* Fake up a data abort (level 3 translation fault on write) */ vcpu->arch.fault.esr_el2 = (ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT) | - ESR_ELx_WNR | ESR_ELx_FSC_FAULT | + ESR_ELx_IL | ESR_ELx_WNR | + ESR_ELx_FSC_FAULT | FIELD_PREP(ESR_ELx_FSC_LEVEL, 3); /* Shuffle the IPA around into the HPFAR */ From d098bb75d14fde2f12155f1a95ec0168160867ce Mon Sep 17 00:00:00 2001 From: Bradley Morgan Date: Sun, 21 Jun 2026 21:31:55 +0000 Subject: [PATCH 21/48] KVM: arm64: account pKVM reclaim against the VM mm Protected guest faults charge long term pins to the VM's mm. Teardown can run later from file release, where current->mm may be unrelated. Drop the charge from kvm->mm instead. Fixes: 4e6e03f9eadd ("KVM: arm64: Hook up reclaim hypercall to pkvm_pgtable_stage2_destroy()") Signed-off-by: Bradley Morgan Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260621213155.6019-1-include@grrlz.net Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org --- arch/arm64/kvm/pkvm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 053e4f733e4b..428723b1b0f5 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -352,7 +352,7 @@ static int __pkvm_pgtable_stage2_reclaim(struct kvm_pgtable *pgt, u64 start, u64 page = pfn_to_page(mapping->pfn); WARN_ON_ONCE(mapping->nr_pages != 1); unpin_user_pages_dirty_lock(&page, 1, true); - account_locked_vm(current->mm, 1, false); + account_locked_vm(kvm->mm, 1, false); pkvm_mapping_remove(mapping, &pgt->pkvm_mappings); kfree(mapping); } From 3a07249981629ace483ebbef81ef6b34c2d2afec Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 27 Jun 2026 11:51:05 +0100 Subject: [PATCH 22/48] KVM: Move kvm_io_bus_get_dev() locking responsibilities to callers kvm_io_bus_get_dev() returns a device that is only matched by the address, and nothing else. This can cause a lifetime issue if the matched device is not the expected type, as by the time the caller can introspect the object, it might be gone (the srcu lock having been dropped). Given that there is only a single user of this helper, the simplest option is to move the locking responsibility to the caller, which can keep the srcu lock held for as long as it wants. Note that this aligns with other kvm_io_bus*() helpers, which already require the srcu lock to be held by the callers. Reported-by: Will Deacon Fixes: 8a39d00670f07 ("KVM: kvm_io_bus: Add kvm_io_bus_get_dev() call") Link: https://lore.kernel.org/all/20260626111344.802555-1-maz@kernel.org Cc: stable@vger.kernel.org Reviewed-by: Oliver Upton Link: https://patch.msgid.link/20260627105105.1005990-1-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-its.c | 2 ++ virt/kvm/kvm_main.c | 16 +++++----------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 67d107e9a77d..c90abde39fb8 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -508,6 +508,8 @@ static struct vgic_its *__vgic_doorbell_to_its(struct kvm *kvm, gpa_t db) struct kvm_io_device *kvm_io_dev; struct vgic_io_device *iodev; + guard(srcu)(&kvm->srcu); + kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, db); if (!kvm_io_dev) return ERR_PTR(-EINVAL); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 89489996fbc1..5788eac0ab81 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -6068,25 +6068,19 @@ struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr) { struct kvm_io_bus *bus; - int dev_idx, srcu_idx; - struct kvm_io_device *iodev = NULL; + int dev_idx; - srcu_idx = srcu_read_lock(&kvm->srcu); + lockdep_assert_held(&kvm->srcu); bus = kvm_get_bus_srcu(kvm, bus_idx); if (!bus) - goto out_unlock; + return NULL; dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1); if (dev_idx < 0) - goto out_unlock; + return NULL; - iodev = bus->range[dev_idx].dev; - -out_unlock: - srcu_read_unlock(&kvm->srcu, srcu_idx); - - return iodev; + return bus->range[dev_idx].dev; } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_io_bus_get_dev); From 100baf0184896f859290a684f864b8200d8ac872 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 1 Jul 2026 16:16:19 -0700 Subject: [PATCH 23/48] KVM: arm64: Ensure level is always initialized when relaxing perms stage2_update_leaf_attrs() returns early before writing to @level if the table walker returned an error. At the same time, kvm_pgtable_stage2_relax_perms() uses the level as a TLBI TTL hint when the error was EAGAIN, indicating the vCPU raced with a table update and the TLB entry it hit is now stale. Fall back to an unknown TTL if none was provided by the walk. Cc: stable@vger.kernel.org Fixes: be097997a273 ("KVM: arm64: Always invalidate TLB for stage-2 permission faults") Signed-off-by: Oliver Upton Reviewed-by: Wei-Lin Chang Link: https://patch.msgid.link/20260701231620.3300204-2-oupton@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/pgtable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 0c1defa5fb0f..386a7468136a 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -1356,7 +1356,7 @@ int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, enum kvm_pgtable_prot prot, enum kvm_pgtable_walk_flags flags) { kvm_pte_t xn = 0, set = 0, clr = 0; - s8 level; + s8 level = TLBI_TTL_UNKNOWN; int ret; if (prot & KVM_PTE_LEAF_ATTR_HI_SW) From f35c08c092505f3a83ce097d94fe51eb8bc9c1b5 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 1 Jul 2026 16:16:20 -0700 Subject: [PATCH 24/48] KVM: arm64: Only update XN attr when requested during S2 relaxation On systems without DIC, KVM lazily grants execute permission to stage-2 translations after taking an instruction abort due to a permission fault, allowing it to defer I-cache invalidations to the point they're absolutely required. If a data abort happens later down the line to such a translation, KVM will not request execute permissions as part of the S2 relaxation on the assumption that kvm_pgtable_stage2_relax_perms() does exactly what the name implies and adds the requested permissions to the pre-existing ones. Avoid taking unintended execute permission faults by only preparing the XN attribute if KVM_PGTABLE_PROT_X is set. Fixes: 2608563b466b ("KVM: arm64: Add support for FEAT_XNX stage-2 permissions") Signed-off-by: Oliver Upton Reviewed-by: Wei-Lin Chang Link: https://patch.msgid.link/20260701231620.3300204-3-oupton@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/pgtable.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 386a7468136a..8754c99c22f2 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -1368,12 +1368,14 @@ int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, if (prot & KVM_PGTABLE_PROT_W) set |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W; - ret = stage2_set_xn_attr(prot, &xn); - if (ret) - return ret; + if (prot & KVM_PGTABLE_PROT_X) { + ret = stage2_set_xn_attr(prot, &xn); + if (ret) + return ret; - set |= xn & KVM_PTE_LEAF_ATTR_HI_S2_XN; - clr |= ~xn & KVM_PTE_LEAF_ATTR_HI_S2_XN; + set |= xn & KVM_PTE_LEAF_ATTR_HI_S2_XN; + clr |= ~xn & KVM_PTE_LEAF_ATTR_HI_S2_XN; + } ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level, flags); if (!ret || ret == -EAGAIN) From 85f56708a443ec02a290878f00d79a1ff5110e41 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 6 Jul 2026 12:55:21 +0100 Subject: [PATCH 25/48] KVM: arm64: Fix sign-extension of MMIO loads A sign-extending load (LDRSB, LDRSH, LDRSW) from MMIO returns a zero-extended value to the guest. The architecture performs such a load as a memory read of the access size, then a sign-extension to the register width. For LDRSH (DDI 0487 M.b C6.2.225, with the Mem accessor at J1.2.3.111): data = Mem{16}(address, accdesc); X{regsize}(t) = SignExtend{regsize}(data); The byte order is handled inside the Mem accessor, keyed on the access size; the register width is separate, applied afterwards by SignExtend(). kvm_handle_mmio_return() runs these in the wrong order: it sign-extends the access-width data, then calls vcpu_data_host_to_guest(), which masks the value back to the access width (the size-keyed byte-order step). The mask drops the sign bits that sign-extension produced. Reorder so vcpu_data_host_to_guest() runs first, with the sign-extension to register width after it. trace_kvm_mmio() moves with it and now logs the access-width data before sign-extension. Fixes: b30070862edbd ("ARM64: KVM: MMIO support BE host running LE code") Reviewed-by: Oliver Upton Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260706115522.954913-2-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmio.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/mmio.c b/arch/arm64/kvm/mmio.c index e2285ed8c91d..d1c3a352d5a2 100644 --- a/arch/arm64/kvm/mmio.c +++ b/arch/arm64/kvm/mmio.c @@ -126,6 +126,10 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu) len = kvm_vcpu_dabt_get_as(vcpu); data = kvm_mmio_read_buf(run->mmio.data, len); + trace_kvm_mmio(KVM_TRACE_MMIO_READ, len, run->mmio.phys_addr, + &data); + data = vcpu_data_host_to_guest(vcpu, data, len); + if (kvm_vcpu_dabt_issext(vcpu) && len < sizeof(unsigned long)) { mask = 1U << ((len * 8) - 1); @@ -135,9 +139,6 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu) if (!kvm_vcpu_dabt_issf(vcpu)) data = data & 0xffffffff; - trace_kvm_mmio(KVM_TRACE_MMIO_READ, len, run->mmio.phys_addr, - &data); - data = vcpu_data_host_to_guest(vcpu, data, len); vcpu_set_reg(vcpu, kvm_vcpu_dabt_get_rd(vcpu), data); } From 65a38ddeaeed1a962bc1b4ca03c98201c877fea5 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 6 Jul 2026 12:55:22 +0100 Subject: [PATCH 26/48] KVM: arm64: selftests: Add MMIO sign-extending load test Add a test for sign-extending MMIO loads (LDRSB, LDRSH, LDRSW) into Xt and Wt destinations, with and without the sign bit set. The host supplies the MMIO data and checks the guest register holds the sign-extended value. Repeat the loads big-endian on a mixed-endian implementation. Issue those at EL0: SCTLR_EL1.EE would make an EL1 load big-endian but also walk the little-endian page tables big-endian, whereas SCTLR_EL1.E0E selects only EL0 data endianness and leaves the walk little-endian. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260706115522.954913-3-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../selftests/kvm/arm64/mmio_sign_ext.c | 255 ++++++++++++++++++ 2 files changed, 256 insertions(+) create mode 100644 tools/testing/selftests/kvm/arm64/mmio_sign_ext.c diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 9118a5a51b89..0f5803a1092e 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -171,6 +171,7 @@ TEST_GEN_PROGS_arm64 += arm64/hello_el2 TEST_GEN_PROGS_arm64 += arm64/host_sve TEST_GEN_PROGS_arm64 += arm64/hypercalls TEST_GEN_PROGS_arm64 += arm64/external_aborts +TEST_GEN_PROGS_arm64 += arm64/mmio_sign_ext TEST_GEN_PROGS_arm64 += arm64/page_fault_test TEST_GEN_PROGS_arm64 += arm64/psci_test TEST_GEN_PROGS_arm64 += arm64/sea_to_user diff --git a/tools/testing/selftests/kvm/arm64/mmio_sign_ext.c b/tools/testing/selftests/kvm/arm64/mmio_sign_ext.c new file mode 100644 index 000000000000..25196f1e6322 --- /dev/null +++ b/tools/testing/selftests/kvm/arm64/mmio_sign_ext.c @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * mmio_sign_ext - Test sign-extending MMIO load emulation (LDRSB/LDRSH/LDRSW) + * + * Copyright (c) 2026 Google LLC + * Author: Fuad Tabba + */ + +#include + +#include "processor.h" +#include "test_util.h" + +#define MMIO_ADDR 0x8000000ULL + +/* AP[1]: allow unprivileged (EL0) access to a mapping. */ +#define PTE_USER BIT(6) + +/* SPSR for ERET to EL0t with DAIF masked. */ +#define SPSR_EL0 (PSR_MODE_EL0t | PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT) + +struct mmio_test { + const char *name; + uint64_t data; /* access-width value, host byte order */ + uint8_t len; + uint64_t expected; /* sign-extended result; same for LE and BE */ +}; + +/* Paired 1:1, in order, with the loads in guest_loads_le() and el0_be_loads. */ +static const struct mmio_test tests[] = { + /* LDRSB Xt: byte sign-extended to 64 bits */ + { "LDRSB Xt 0xFF", 0xFF, 1, 0xFFFFFFFFFFFFFFFFULL }, + { "LDRSB Xt 0x7F", 0x7F, 1, 0x7FULL }, + + /* LDRSB Wt: byte sign-extended to 32 bits, upper 32 bits zeroed */ + { "LDRSB Wt 0xFF", 0xFF, 1, 0xFFFFFFFFULL }, + { "LDRSB Wt 0x7F", 0x7F, 1, 0x7FULL }, + + /* LDRSH Xt: halfword sign-extended to 64 bits */ + { "LDRSH Xt 0x8001", 0x8001, 2, 0xFFFFFFFFFFFF8001ULL }, + { "LDRSH Xt 0x7FFF", 0x7FFF, 2, 0x7FFFULL }, + + /* LDRSH Wt: halfword sign-extended to 32 bits, upper 32 bits zeroed */ + { "LDRSH Wt 0x8001", 0x8001, 2, 0xFFFF8001ULL }, + { "LDRSH Wt 0x7FFF", 0x7FFF, 2, 0x7FFFULL }, + + /* LDRSW Xt: word sign-extended to 64 bits (no Wt form) */ + { "LDRSW Xt 0x80000001", 0x80000001, 4, 0xFFFFFFFF80000001ULL }, + { "LDRSW Xt 0x7FFFFFFF", 0x7FFFFFFF, 4, 0x7FFFFFFFULL }, +}; + +/* Issue one sign-extending load from MMIO and report the result. */ +#define GUEST_LDRS(load) do { \ + uint64_t val; \ + \ + asm volatile(load : "=r"(val) : "r"(MMIO_ADDR) : "memory"); \ + GUEST_SYNC(val); \ +} while (0) + +/* Little-endian pass: loads issued at EL1. */ +static void guest_loads_le(void) +{ + GUEST_LDRS("ldrsb %0, [%1]"); + GUEST_LDRS("ldrsb %0, [%1]"); + GUEST_LDRS("ldrsb %w0, [%1]"); + GUEST_LDRS("ldrsb %w0, [%1]"); + GUEST_LDRS("ldrsh %0, [%1]"); + GUEST_LDRS("ldrsh %0, [%1]"); + GUEST_LDRS("ldrsh %w0, [%1]"); + GUEST_LDRS("ldrsh %w0, [%1]"); + GUEST_LDRS("ldrsw %0, [%1]"); + GUEST_LDRS("ldrsw %0, [%1]"); +} + +/* + * Run the big-endian loads at EL0, where SCTLR_EL1.E0E flips only the data + * endianness; at EL1, SCTLR_EL1.EE would also flip the page-table walk and + * fault on the little-endian tables. x0 holds MMIO_ADDR; results return in + * x19..x28 (tests[] order) via a single SVC. + */ +extern char el0_be_loads[]; +asm( +" .pushsection .text, \"ax\"\n" +" .global el0_be_loads\n" +"el0_be_loads:\n" +" ldrsb x19, [x0]\n" +" ldrsb x20, [x0]\n" +" ldrsb w21, [x0]\n" +" ldrsb w22, [x0]\n" +" ldrsh x23, [x0]\n" +" ldrsh x24, [x0]\n" +" ldrsh w25, [x0]\n" +" ldrsh w26, [x0]\n" +" ldrsw x27, [x0]\n" +" ldrsw x28, [x0]\n" +" svc #0\n" +" .popsection\n" +); + +/* EL1 handler for the EL0 SVC: report the results, then finish. */ +static void el0_svc_handler(struct ex_regs *regs) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(tests); i++) + GUEST_SYNC(regs->regs[19 + i]); + + GUEST_DONE(); +} + +static bool guest_mixed_endian_el0(void) +{ + uint64_t mmfr0 = read_sysreg(id_aa64mmfr0_el1); + + return SYS_FIELD_GET(ID_AA64MMFR0_EL1, BIGEND, mmfr0) || + SYS_FIELD_GET(ID_AA64MMFR0_EL1, BIGENDEL0, mmfr0); +} + +static void guest_code(void) +{ + guest_loads_le(); + + if (guest_mixed_endian_el0()) { + write_sysreg(read_sysreg(sctlr_el1) | SCTLR_EL1_E0E, sctlr_el1); + isb(); + + asm volatile( + " msr elr_el1, %[pc]\n" + " msr spsr_el1, %[spsr]\n" + " mov x0, %[mmio]\n" + " isb\n" + " eret\n" + : + : [pc] "r"(el0_be_loads), + [spsr] "r"((uint64_t)SPSR_EL0), + [mmio] "r"(MMIO_ADDR) + : "x0", "memory"); + __builtin_unreachable(); /* el0_svc_handler ends the test */ + } + + GUEST_DONE(); +} + +static void handle_mmio(struct kvm_run *run, const struct mmio_test *t, bool be) +{ + int i; + + TEST_ASSERT_EQ(run->mmio.phys_addr, MMIO_ADDR); + TEST_ASSERT(!run->mmio.is_write, "Expected MMIO read for %s", t->name); + TEST_ASSERT_EQ(run->mmio.len, t->len); + + memset(run->mmio.data, 0, sizeof(run->mmio.data)); + if (be) { + /* The guest reads the device bytes most-significant first. */ + for (i = 0; i < t->len; i++) + run->mmio.data[i] = t->data >> (8 * (t->len - 1 - i)); + } else { + /* Works because arm64 KVM hosts are always little-endian. */ + memcpy(run->mmio.data, &t->data, t->len); + } +} + +static void expect_sync(struct kvm_vcpu *vcpu, struct ucall *uc, + const struct mmio_test *t) +{ + switch (get_ucall(vcpu, uc)) { + case UCALL_SYNC: + TEST_ASSERT(uc->args[1] == t->expected, + "%s: got %#lx, want %#lx", t->name, + (unsigned long)uc->args[1], (unsigned long)t->expected); + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(*uc); + break; + default: + TEST_FAIL("Unexpected ucall for %s", t->name); + } +} + +/* OR PTE_USER into the leaf descriptors covering [gva, gva + len). */ +static void make_el0_accessible(struct kvm_vm *vm, uint64_t gva, uint64_t len) +{ + uint64_t addr; + + for (addr = gva & ~((uint64_t)vm->page_size - 1); addr < gva + len; + addr += vm->page_size) + *virt_get_pte_hva(vm, addr) |= PTE_USER; +} + +static bool vcpu_mixed_endian_el0(struct kvm_vcpu *vcpu) +{ + uint64_t mmfr0 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64MMFR0_EL1)); + + return SYS_FIELD_GET(ID_AA64MMFR0_EL1, BIGEND, mmfr0) || + SYS_FIELD_GET(ID_AA64MMFR0_EL1, BIGENDEL0, mmfr0); +} + +int main(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + unsigned int i; + bool be; + + vm = vm_create_with_one_vcpu(&vcpu, guest_code); + virt_map(vm, MMIO_ADDR, MMIO_ADDR, 1); + + vm_init_descriptor_tables(vm); + vcpu_init_descriptor_tables(vcpu); + vm_install_sync_handler(vm, VECTOR_SYNC_LOWER_64, ESR_ELx_EC_SVC64, + el0_svc_handler); + + be = vcpu_mixed_endian_el0(vcpu); + if (be) + make_el0_accessible(vm, MMIO_ADDR, vm->page_size); + + ksft_print_header(); + ksft_set_plan(ARRAY_SIZE(tests) * (be ? 2 : 1)); + + /* Little-endian pass: one load and one result per iteration. */ + for (i = 0; i < ARRAY_SIZE(tests); i++) { + const struct mmio_test *t = &tests[i]; + + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_MMIO); + handle_mmio(vcpu->run, t, false); + + vcpu_run(vcpu); + expect_sync(vcpu, &uc, t); + + ksft_test_result_pass("%s\n", t->name); + } + + if (be) { + /* The EL0 stub issues all the loads, then reports the results. */ + for (i = 0; i < ARRAY_SIZE(tests); i++) { + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_MMIO); + handle_mmio(vcpu->run, &tests[i], true); + } + for (i = 0; i < ARRAY_SIZE(tests); i++) { + vcpu_run(vcpu); + expect_sync(vcpu, &uc, &tests[i]); + ksft_test_result_pass("BE %s\n", tests[i].name); + } + } + + vcpu_run(vcpu); + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_DONE, "Expected UCALL_DONE"); + + kvm_vm_free(vm); + + ksft_finished(); +} From 8c6db30d79528279abbeb416e4f533f1f91b8724 Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Thu, 2 Jul 2026 10:38:40 +0000 Subject: [PATCH 27/48] KVM: arm64: Fix bounds checking in do_ffa_mem_reclaim() Sashiko (locally) reports out of bound write possiblity if SPMD returns an invalid data. While SPMD is considered trusted, pKVM does some basic checks, for offset to be less than or equal len. However, that is incorrect as even if the offset is smaller than len pKVM can still access out of bound memory in the next ffa_host_unshare_ranges(). Split this check into 2: 1- Check that the fixed portion of the descriptor fits. 2- After getting reg, check the variable array size addr_range_cnt fits. Also, drop the WARN_ONs as that will panic the kernel and in the next checks there are no WARNs, so that makes it consistent. Fixes: 0a9f15fd5674 ("KVM: arm64: pkvm: Add support for fragmented FF-A descriptors") Signed-off-by: Mostafa Saleh Reviewed-by: Vincent Donnefort Signed-off-by: Sebastian Ene Link: https://patch.msgid.link/20260702103848.1647249-4-sebastianene@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/ffa.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index 1af722771178..41cc4c1bafeb 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -607,8 +607,8 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, * check that we end up with something that doesn't look _completely_ * bogus. */ - if (WARN_ON(offset > len || - fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE)) { + if (offset + CONSTITUENTS_OFFSET(0) > len || + fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) { ret = FFA_RET_ABORTED; ffa_rx_release(res); goto out_unlock; @@ -636,11 +636,16 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, ffa_rx_release(res); } + reg = (void *)buf + offset; + if (offset + CONSTITUENTS_OFFSET(reg->addr_range_cnt) > len) { + ret = FFA_RET_ABORTED; + goto out_unlock; + } + ffa_mem_reclaim(res, handle_lo, handle_hi, flags); if (res->a0 != FFA_SUCCESS) goto out_unlock; - reg = (void *)buf + offset; /* If the SPMD was happy, then we should be too. */ WARN_ON(ffa_host_unshare_ranges(reg->constituents, reg->addr_range_cnt)); From a6b49d27c17909608d54523220bb6f3498d4a1df Mon Sep 17 00:00:00 2001 From: Sebastian Ene Date: Thu, 2 Jul 2026 10:38:41 +0000 Subject: [PATCH 28/48] KVM: arm64: Validate the offset to the mem access descriptor Prevent the pKVM hypervisor from making assumptions that the endpoint memory access descriptor (EMAD) comes right after the FF-A memory region header. Prior to FF-A version 1.1 the header of the memory region didn't contain an offset to the endpoint memory access descriptor. The layout of a memory transaction looks like this from 1.1 onward: Type | Field name | Offset [ Header | ffa_mem_region | 0 EMAD 1 | ffa_mem_region_attributes) | ffa_mem_region.ep_mem_offset ] Verify that the offset to the first endpoint memory access descriptor is within the mailbox buffer bounds. Also, fix one hardcoded sizeof(struct ffa_mem_region_attributes) that should be replaced ffa_emad_size_get() for compatibility with FFA v1.0. Fixes: 42fb33dde42b ("KVM: arm64: Use FF-A 1.1 with pKVM") Signed-off-by: Mostafa Saleh Signed-off-by: Sebastian Ene Reviewed-by: Vincent Donnefort Link: https://patch.msgid.link/20260702103848.1647249-5-sebastianene@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/ffa.c | 27 +++++++++++++++++++-------- include/linux/arm_ffa.h | 7 +++++++ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index 41cc4c1bafeb..2e7ab7e3319d 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -476,11 +476,12 @@ static void __do_ffa_mem_xfer(const u64 func_id, DECLARE_REG(u32, fraglen, ctxt, 2); DECLARE_REG(u64, addr_mbz, ctxt, 3); DECLARE_REG(u32, npages_mbz, ctxt, 4); + u32 offset, nr_ranges, checked_offset, em_mem_access_off; struct ffa_mem_region_attributes *ep_mem_access; struct ffa_composite_mem_region *reg; struct ffa_mem_region *buf; - u32 offset, nr_ranges, checked_offset; int ret = 0; + size_t mem_region_len = FFA_MEM_REGION_SZ(hyp_ffa_version); if (addr_mbz || npages_mbz || fraglen > len || fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) { @@ -488,8 +489,7 @@ static void __do_ffa_mem_xfer(const u64 func_id, goto out; } - if (fraglen < sizeof(struct ffa_mem_region) + - sizeof(struct ffa_mem_region_attributes)) { + if (fraglen < mem_region_len + ffa_emad_size_get(hyp_ffa_version)) { ret = FFA_RET_INVALID_PARAMETERS; goto out; } @@ -508,8 +508,13 @@ static void __do_ffa_mem_xfer(const u64 func_id, buf = hyp_buffers.tx; memcpy(buf, host_buffers.tx, fraglen); - ep_mem_access = (void *)buf + - ffa_mem_desc_offset(buf, 0, hyp_ffa_version); + em_mem_access_off = ffa_mem_desc_offset(buf, 0, hyp_ffa_version); + if ((u64)em_mem_access_off + ffa_emad_size_get(hyp_ffa_version) > fraglen) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out_unlock; + } + + ep_mem_access = (void *)buf + em_mem_access_off; offset = ep_mem_access->composite_off; if (!offset || buf->ep_count != 1 || buf->sender_id != HOST_FFA_ID) { ret = FFA_RET_INVALID_PARAMETERS; @@ -574,9 +579,9 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, DECLARE_REG(u32, handle_lo, ctxt, 1); DECLARE_REG(u32, handle_hi, ctxt, 2); DECLARE_REG(u32, flags, ctxt, 3); + u32 offset, len, fraglen, fragoff, em_mem_access_off; struct ffa_mem_region_attributes *ep_mem_access; struct ffa_composite_mem_region *reg; - u32 offset, len, fraglen, fragoff; struct ffa_mem_region *buf; int ret = 0; u64 handle; @@ -599,8 +604,14 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, len = res->a1; fraglen = res->a2; - ep_mem_access = (void *)buf + - ffa_mem_desc_offset(buf, 0, hyp_ffa_version); + em_mem_access_off = ffa_mem_desc_offset(buf, 0, hyp_ffa_version); + if ((u64)em_mem_access_off + ffa_emad_size_get(hyp_ffa_version) > fraglen) { + ret = FFA_RET_INVALID_PARAMETERS; + ffa_rx_release(res); + goto out_unlock; + } + + ep_mem_access = (void *)buf + em_mem_access_off; offset = ep_mem_access->composite_off; /* * We can trust the SPMD to get this right, but let's at least diff --git a/include/linux/arm_ffa.h b/include/linux/arm_ffa.h index 81e603839c4a..3c91d4c4153c 100644 --- a/include/linux/arm_ffa.h +++ b/include/linux/arm_ffa.h @@ -421,6 +421,13 @@ struct ffa_mem_region { #define FFA_EMAD_HAS_IMPDEF_FIELD(version) ((version) >= FFA_VERSION_1_2) #define FFA_MEM_REGION_HAS_EP_MEM_OFFSET(version) ((version) > FFA_VERSION_1_0) +/* The layout changed from FFA_VERSION_1_0 and the region includes an + * ep_mem_offset. + */ +#define FFA_MEM_REGION_SZ(version) (!FFA_MEM_REGION_HAS_EP_MEM_OFFSET((version)) ?\ + offsetof(struct ffa_mem_region, ep_mem_offset) :\ + sizeof(struct ffa_mem_region)) + static inline u32 ffa_emad_size_get(u32 ffa_version) { u32 sz; From 6a7a181f6921db3d9aed1bba7e15547fefd7eedc Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Thu, 2 Jul 2026 10:38:42 +0000 Subject: [PATCH 29/48] KVM: arm64: Ensure FFA ranges are page aligned Harden the check for the constituent memory region page alignment to prevent over-sharing when the negotiated FFA_PAGE_SIZE size is smaller than the system PAGE_SIZE. At the moment we only check that the size of the range is page aligned, and truncate the address to the page boundary which can annotate more memory than needed as being used by the FF-A. Fixes: 436090001776 ("KVM: arm64: Handle FFA_MEM_SHARE calls from the host") Signed-off-by: Mostafa Saleh Reviewed-by: Vincent Donnefort Signed-off-by: Sebastian Ene Link: https://patch.msgid.link/20260702103848.1647249-6-sebastianene@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/ffa.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index 2e7ab7e3319d..9c96e72e522e 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -352,7 +352,7 @@ static u32 __ffa_host_share_ranges(struct ffa_mem_region_addr_range *ranges, u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE; u64 pfn = hyp_phys_to_pfn(range->address); - if (!PAGE_ALIGNED(sz)) + if (!PAGE_ALIGNED(sz | range->address)) break; if (__pkvm_host_share_ffa(pfn, sz / PAGE_SIZE)) @@ -372,7 +372,7 @@ static u32 __ffa_host_unshare_ranges(struct ffa_mem_region_addr_range *ranges, u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE; u64 pfn = hyp_phys_to_pfn(range->address); - if (!PAGE_ALIGNED(sz)) + if (!PAGE_ALIGNED(sz | range->address)) break; if (__pkvm_host_unshare_ffa(pfn, sz / PAGE_SIZE)) From 2bd3c6c702f3a9e2bcb3b536b0fbbaa645005d71 Mon Sep 17 00:00:00 2001 From: Sebastian Ene Date: Thu, 2 Jul 2026 10:38:43 +0000 Subject: [PATCH 30/48] KVM: arm64: Zero out the stack initialized data in the FFA handler Don't leak hypervisor stack data when using the FFA_VERSION call. When the compiler doesn't support -ftrivial-auto-var-init=zero option we need to zero out the stack initialized variable before returning data to the host caller. Closes: https://lore.kernel.org/all/20260616160016.C62C81F000E9@smtp.kernel.org/ Reported-by: Sashiko AI Fixes: c9c012625e12 ("KVM: arm64: Trap FFA_VERSION host call in pKVM") Reviewed-by: Vincent Donnefort Link: https://lore.kernel.org/all/20260616160016.C62C81F000E9@smtp.kernel.org/ Signed-off-by: Sebastian Ene Link: https://patch.msgid.link/20260702103848.1647249-7-sebastianene@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/ffa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index 9c96e72e522e..a327c2bbb6b6 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -880,7 +880,7 @@ static void do_ffa_part_get(struct arm_smccc_1_2_regs *res, bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) { - struct arm_smccc_1_2_regs res; + struct arm_smccc_1_2_regs res = {0}; /* * There's no way we can tell what a non-standard SMC call might From 3383ffb7ef937317361713ffcc21921a7848511a Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Thu, 2 Jul 2026 10:38:38 +0000 Subject: [PATCH 31/48] firmware: arm_ffa: Fix out-of-bound writes in ffa_setup_and_transmit() Sashiko (locally) reports multiple out-of-bound issues in ffa_setup_and_transmit: 1) Writing ep_mem_access->reserved can write out of bounds for FFA versions < 1.2 as ffa_emad_size_get() returns 16 bytes in that case while reserved has an offset of 24. Instead of zeroing fields, memset the struct to zero first based on the FFA version. 2) Make sure there is enough size to write constituents. While at it, convert the only sizeof() in the driver that uses a type instead of variable. Reviewed-by: Sudeep Holla Fixes: 111a833dc5cb ("firmware: arm_ffa: Set reserved/MBZ fields to zero in the memory descriptors") Signed-off-by: Mostafa Saleh Signed-off-by: Sebastian Ene Link: https://patch.msgid.link/20260702103848.1647249-2-sebastianene@google.com Signed-off-by: Marc Zyngier --- drivers/firmware/arm_ffa/driver.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c index eb2782848283..b700b2e93e72 100644 --- a/drivers/firmware/arm_ffa/driver.c +++ b/drivers/firmware/arm_ffa/driver.c @@ -697,11 +697,10 @@ ffa_setup_and_transmit(u32 func_id, void *buffer, u32 max_fragsize, for (idx = 0; idx < args->nattrs; idx++) { ep_mem_access = buffer + ffa_mem_desc_offset(buffer, idx, drv_info->version); + memset(ep_mem_access, 0, ffa_emad_size_get(drv_info->version)); ep_mem_access->receiver = args->attrs[idx].receiver; ep_mem_access->attrs = args->attrs[idx].attrs; ep_mem_access->composite_off = composite_offset; - ep_mem_access->flag = 0; - ep_mem_access->reserved = 0; ffa_emad_impdef_value_init(drv_info->version, ep_mem_access->impdef_val, args->attrs[idx].impdef_val); @@ -741,7 +740,7 @@ ffa_setup_and_transmit(u32 func_id, void *buffer, u32 max_fragsize, constituents = buffer; } - if ((void *)constituents - buffer > max_fragsize) { + if ((void *)constituents + sizeof(*constituents) - buffer > max_fragsize) { pr_err("Memory Region Fragment > Tx Buffer size\n"); return -EFAULT; } @@ -750,7 +749,7 @@ ffa_setup_and_transmit(u32 func_id, void *buffer, u32 max_fragsize, constituents->pg_cnt = args->sg->length / FFA_PAGE_SIZE; constituents->reserved = 0; constituents++; - frag_len += sizeof(struct ffa_mem_region_addr_range); + frag_len += sizeof(*constituents); } while ((args->sg = sg_next(args->sg))); return ffa_transmit_fragment(func_id, addr, buf_sz, frag_len, From b4d961351aa84fdf0148783fb1f3a1391b8a0adb Mon Sep 17 00:00:00 2001 From: Sebastian Ene Date: Thu, 2 Jul 2026 10:38:39 +0000 Subject: [PATCH 32/48] firmware: arm_ffa: Fix Endpoint Memory Access Descriptor offset calculation Use the descriptor's `ep_mem_offset` to calculate the start of the endpoint memory access array and to comply with the FF-A spec instead of defaulting to `sizeof(struct ffa_mem_region)`. This requires moving `ffa_mem_region_additional_setup()` earlier in the setup flow. Also, add sanity checks to ensure the calculated descriptor offsets do not exceed `max_fragsize`. Fixes: 113580530ee7 ("firmware: arm_ffa: Update memory descriptor to support v1.1 format") Reviewed-by: Sudeep Holla Signed-off-by: Mostafa Saleh Signed-off-by: Sebastian Ene Link: https://patch.msgid.link/20260702103848.1647249-3-sebastianene@google.com Signed-off-by: Marc Zyngier --- drivers/firmware/arm_ffa/driver.c | 20 +++++++++++++++----- include/linux/arm_ffa.h | 2 +- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c index b700b2e93e72..0ad323decada 100644 --- a/drivers/firmware/arm_ffa/driver.c +++ b/drivers/firmware/arm_ffa/driver.c @@ -685,19 +685,30 @@ ffa_setup_and_transmit(u32 func_id, void *buffer, u32 max_fragsize, struct ffa_composite_mem_region *composite; struct ffa_mem_region_addr_range *constituents; struct ffa_mem_region_attributes *ep_mem_access; - u32 idx, frag_len, length, buf_sz = 0, num_entries = sg_nents(args->sg); + u32 idx, frag_len, length, buf_sz = 0, num_entries = sg_nents(args->sg), ep_offset; + u32 emad_end, emad_size = ffa_emad_size_get(drv_info->version); mem_region->tag = args->tag; mem_region->flags = args->flags; mem_region->sender_id = drv_info->vm_id; mem_region->attributes = ffa_memory_attributes_get(func_id); + + ffa_mem_region_additional_setup(drv_info->version, mem_region); composite_offset = ffa_mem_desc_offset(buffer, args->nattrs, drv_info->version); + if (composite_offset + sizeof(*composite) > max_fragsize) + return -ENXIO; for (idx = 0; idx < args->nattrs; idx++) { - ep_mem_access = buffer + - ffa_mem_desc_offset(buffer, idx, drv_info->version); - memset(ep_mem_access, 0, ffa_emad_size_get(drv_info->version)); + ep_offset = ffa_mem_desc_offset(buffer, idx, drv_info->version); + if (check_add_overflow(ep_offset, emad_size, &emad_end)) + return -ENXIO; + + if (emad_end > max_fragsize) + return -ENXIO; + + ep_mem_access = buffer + ep_offset; + memset(ep_mem_access, 0, emad_size); ep_mem_access->receiver = args->attrs[idx].receiver; ep_mem_access->attrs = args->attrs[idx].attrs; ep_mem_access->composite_off = composite_offset; @@ -707,7 +718,6 @@ ffa_setup_and_transmit(u32 func_id, void *buffer, u32 max_fragsize, } mem_region->handle = 0; mem_region->ep_count = args->nattrs; - ffa_mem_region_additional_setup(drv_info->version, mem_region); composite = buffer + composite_offset; composite->total_pg_cnt = ffa_get_num_pages_sg(args->sg); diff --git a/include/linux/arm_ffa.h b/include/linux/arm_ffa.h index 3c91d4c4153c..c6a9ffd564f7 100644 --- a/include/linux/arm_ffa.h +++ b/include/linux/arm_ffa.h @@ -452,7 +452,7 @@ ffa_mem_desc_offset(struct ffa_mem_region *buf, int count, u32 ffa_version) if (!FFA_MEM_REGION_HAS_EP_MEM_OFFSET(ffa_version)) offset += offsetof(struct ffa_mem_region, ep_mem_offset); else - offset += sizeof(struct ffa_mem_region); + offset += buf->ep_mem_offset; return offset; } From 8d187d4b33c262c0f3e44842553521151d8629e8 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 7 Jul 2026 17:29:35 +0100 Subject: [PATCH 33/48] KVM: arm64: Fix propagation of TLBI level in kvm_pgtable_stage2_relax_perms() Assigning the invalidation level (an s8 value) with TLBI_TTL_UNKNOWN (a 32bit signed value) is not ideal, to say the least. Instead of this, only pass TLBI_TTL_UNKNOWN to __kvm_tlb_flush_vmid_ipa_nsh() when we know for sure that we don't have a provided level. Fixes: 100baf0184896 ("KVM: arm64: Ensure level is always initialized when relaxing perms") Reported-by: Mark Brown Reviewed-by: Oliver Upton Link: https://lore.kernel.org/r/akztC7H2IsEKaq4i@sirena.org.uk Link: https://patch.msgid.link/20260707162935.1900874-1-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/pgtable.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 8754c99c22f2..70dceb20dfad 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -1356,7 +1356,7 @@ int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, enum kvm_pgtable_prot prot, enum kvm_pgtable_walk_flags flags) { kvm_pte_t xn = 0, set = 0, clr = 0; - s8 level = TLBI_TTL_UNKNOWN; + s8 level; int ret; if (prot & KVM_PTE_LEAF_ATTR_HI_SW) @@ -1379,7 +1379,8 @@ int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level, flags); if (!ret || ret == -EAGAIN) - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr, level); + kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr, + (ret == -EAGAIN) ? TLBI_TTL_UNKNOWN : level); return ret; } From ed446e8aa894883c08892cfee69782fdf8f6c3ca Mon Sep 17 00:00:00 2001 From: leixiang Date: Mon, 22 Jun 2026 15:51:01 +0800 Subject: [PATCH 34/48] KVM: x86: Nullify irqfd->producer if updating IRTE for bypass fails Nullify irqfd->producer if updating the IRTE for bypass fails, as leaving a dangling pointer will result in a use-after-free if the irqfd is reachable through KVM's routing, but the producer is freed separately. E.g. for VFIO PCI, the producer is embedded in struct "vfio_pci_irq_ctx" and freed when the vector is disabled, which can happen independent of routing updates. Fixes: 77e1b8332d1d ("KVM: x86: Decouple device assignment from IRQ bypass") Cc: stable@vger.kernel.org Signed-off-by: leixiang Link: https://patch.msgid.link/1782119051448443.14545.seg@mailgw.kylinos.cn [sean: drop PPC change, massage changelog] Signed-off-by: Sean Christopherson --- arch/x86/kvm/irq.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c index 8c62c6d4d5c1..cb8ac4b9b0d7 100644 --- a/arch/x86/kvm/irq.c +++ b/arch/x86/kvm/irq.c @@ -488,8 +488,10 @@ int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, if (irqfd->irq_entry.type == KVM_IRQ_ROUTING_MSI) { ret = kvm_pi_update_irte(irqfd, &irqfd->irq_entry); - if (ret) + if (ret) { kvm->arch.nr_possible_bypass_irqs--; + irqfd->producer = NULL; + } } spin_unlock_irq(&kvm->irqfds.lock); From 9285e4070df2c40585c3d7ec9571faa7a2b97e17 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 24 Jun 2026 15:05:16 -0700 Subject: [PATCH 35/48] KVM: x86: Ignore pending PV EOI if the vCPU has since disabled PV EOIs Ignore KVM's internal "service pending PV EOI" request if the vCPU has disabled PV EOIs since the request was made. Asserting that PV EOIs are enabled can fail if reading guest memory in pv_eoi_get_user() fails, i.e. if pv_eoi_test_and_clr_pending() bails early, *and* the vCPU also disables PV EOIs. kernel BUG at arch/x86/kvm/lapic.c:3338! Oops: invalid opcode: 0000 [#1] SMP CPU: 4 UID: 1000 PID: 890 Comm: pv_eoi_test Not tainted 7.0.0-d585aa5894d8-vm #337 PREEMPT Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 0.0.0 02/06/2015 RIP: 0010:kvm_lapic_sync_from_vapic+0x12b/0x140 [kvm] Call Trace: kvm_arch_vcpu_ioctl_run+0x1075/0x1c30 [kvm] kvm_vcpu_ioctl+0x2d5/0x980 [kvm] __x64_sys_ioctl+0x8a/0xd0 do_syscall_64+0xb5/0xb40 entry_SYSCALL_64_after_hwframe+0x4b/0x53 Modules linked in: kvm_intel kvm irqbypass ---[ end trace 0000000000000000 ]--- Fixes: ae7a2a3fb6f8 ("KVM: host side for eoi optimization") Cc: stable@vger.kernel.org Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260624220516.3033391-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/lapic.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 6f30bbdddb5a..38bba9a1114c 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -3371,6 +3371,12 @@ static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, struct kvm_lapic *apic) { int vector; + + if (unlikely(!pv_eoi_enabled(vcpu))) { + __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); + return; + } + /* * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host * and KVM_PV_EOI_ENABLED in guest memory as follows: @@ -3382,8 +3388,6 @@ static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset: * -> host enabled PV EOI, guest executed EOI. */ - BUG_ON(!pv_eoi_enabled(vcpu)); - if (pv_eoi_test_and_clr_pending(vcpu)) return; vector = apic_set_eoi(apic); From ebdac7554abb347ca4197be241116842161acd9b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 12 Jun 2026 07:56:41 -0700 Subject: [PATCH 36/48] KVM: nVMX: Move vTPR vs. TPR Threshold consistency check into "normal" checks Move the off-by-default consistency check for vmcs12.tpr_threshold vs. the virtual APIC vTPR into the "normal" controls checks, as waiting until KVM has loaded some amount of state is unnecessary and actively dangerous. Specifically, failure to unwind vmcs01.GUEST_CR3 to KVM's value when EPT is disabled results in KVM running L1 with an L1-controlled CR3, not with KVM's CR3! Alternatively, KVM could simply reset the MMU to force a reload of vmcs01.GUEST_CR3, but the _only_ reason the check was shoved into a "late" flow was to wait until the vmcs12 pages were retrieved. Rather than build up more crusty code, simply access vTPR using a regular guest memory access (performance isn't a concern). To circumvent the restrictions that led to KVM deferring nested_get_vmcs12_pages(), (a) use a VM-scoped API to read guest memory so that it always hits non-SMM memslots (for RSM), and (b) skip the check (since its off-by-default anyways) when the vCPU doesn't want to run, i.e. when userspace is restoring/stuffing state. If reading guest memory fails, simply skip the consistency check, as KVM's de facto ABI is that VMX instruction accesses to non-existent memory get PCI Bus Error semantics, where reads return 0xFFs. And if vTPR=0xFF, then the vTPR is guaranteed to be greater than or equal to TPR_THRESHOLD. Fixes: 1100e4910ad2 ("KVM: nVMX: Add an off-by-default module param to WARN on missed consistency checks") Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260612145642.452392-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/nested.c | 66 +++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 6957bb6f5cf7..4fc4349810a3 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -582,6 +582,9 @@ static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { + gpa_t vtpr_gpa = vmcs12->virtual_apic_page_addr + APIC_TASKPRI; + u32 vtpr; + if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) return 0; @@ -591,6 +594,32 @@ static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, if (CC(!nested_cpu_has_vid(vmcs12) && vmcs12->tpr_threshold >> 4)) return -EINVAL; + /* + * Do the illegal vTPR vs. TPR Threshold consistency check if and only + * if KVM is configured to WARN on missed consistency checks, otherwise + * it's a waste of time. KVM needs to rely on hardware to fully detect + * an illegal combination due to the vTPR being writable by L1 at all + * times (it's an in-memory value, not a VMCS field). I.e. even if the + * check passes now, it might fail at the actual VM-Enter. + * + * If reading guest memory fails, skip the check as KVM's de facto ABI + * for VMX instruction accesses to non-existent memory is to provide + * PCI Bus Error semantics (reads return 0xFFs), in which case the vTPR + * is guaranteed to greater than or equal to the threshold. + * + * Note! Deliberately use the VM-scoped API when reading guest memory, + * to ensure the read doesn't hit SMRAM when restoring L2 state on RSM, + * and only perform the check when in KVM_RUN, to avoid a false failure + * if userspace hasn't yet configured memslots during state restore. + */ + if (warn_on_missed_cc && vcpu->wants_to_run && + nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW) && + !nested_cpu_has_vid(vmcs12) && + !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && + !kvm_read_guest(vcpu->kvm, vtpr_gpa, &vtpr, sizeof(vtpr)) && + CC((vmcs12->tpr_threshold & GENMASK(3, 0)) > ((vtpr >> 4) & GENMASK(3, 0)))) + return -EINVAL; + return 0; } @@ -3104,38 +3133,6 @@ static int nested_vmx_check_controls(struct kvm_vcpu *vcpu, return 0; } -static int nested_vmx_check_controls_late(struct kvm_vcpu *vcpu, - struct vmcs12 *vmcs12) -{ - void *vapic = to_vmx(vcpu)->nested.virtual_apic_map.hva; - u32 vtpr = vapic ? (*(u32 *)(vapic + APIC_TASKPRI)) >> 4 : 0; - - /* - * Don't bother with the consistency checks if KVM isn't configured to - * WARN on missed consistency checks, as KVM needs to rely on hardware - * to fully detect an illegal vTPR vs. TRP Threshold combination due to - * the vTPR being writable by L1 at all times (it's an in-memory value, - * not a VMCS field). I.e. even if the check passes now, it might fail - * at the actual VM-Enter. - * - * Keying off the module param also allows treating an invalid vAPIC - * mapping as a consistency check failure without increasing the risk - * of breaking a "real" VM. - */ - if (!warn_on_missed_cc) - return 0; - - if ((exec_controls_get(to_vmx(vcpu)) & CPU_BASED_TPR_SHADOW) && - nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW) && - !nested_cpu_has_vid(vmcs12) && - !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && - (CC(!vapic) || - CC((vmcs12->tpr_threshold & GENMASK(3, 0)) > (vtpr & GENMASK(3, 0))))) - return -EINVAL; - - return 0; -} - static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { @@ -3685,11 +3682,6 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, return NVMX_VMENTRY_KVM_INTERNAL_ERROR; } - if (nested_vmx_check_controls_late(vcpu, vmcs12)) { - vmx_switch_vmcs(vcpu, &vmx->vmcs01); - return NVMX_VMENTRY_VMFAIL; - } - if (nested_vmx_check_guest_state(vcpu, vmcs12, &entry_failure_code)) { exit_reason.basic = EXIT_REASON_INVALID_STATE; From 3e6dd2b9b7b2884743ed7a0873b8743cc0df6d40 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 12 Jun 2026 07:56:42 -0700 Subject: [PATCH 37/48] KVM: nVMX: Don't use vmcs01.GUEST_CR3 to snapshot L1's CR3 when EPT is disabled Add a dedicated field in "struct nested_vmx" to track L1's pre-VM-Enter CR3 instead of using vmcs01.GUEST_CR3, which isn't anywhere near as safe as the comment purports it to be. E.g. in addition to the warn_on_missed_cc bug (that was fixed by relocating the consistency check), if getting vmcs12 pages (during actual nested VM-Entry) fails and EPT is disabled (in KVM), KVM will return control to userspace with vmcs01.GUEST_CR3 holding a guest- controlled value. Alternatively, KVM could force a reload of vmcs01.GUEST_CR3 by resetting the MMU context in the error path, but as above, the safety of the vmcs01 approach is extremely questionable, e.g. it took all of ~4 months for the code to break. Fixes: 671ddc700fd0 ("KVM: nVMX: Don't leak L1 MMIO regions to L2") Cc: stable@vger.kernel.org Cc: Jim Mattson Link: https://patch.msgid.link/20260612145642.452392-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/nested.c | 21 ++++++++------------- arch/x86/kvm/vmx/vmx.h | 7 +++++++ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 4fc4349810a3..bb0eb40b4448 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -3658,19 +3658,14 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, &vmx->nested.pre_vmenter_ssp_tbl); /* - * Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled. In the - * event of a "late" VM-Fail, i.e. a VM-Fail detected by hardware but - * not KVM, KVM must unwind its software model to the pre-VM-Entry host - * state. When EPT is disabled, GUEST_CR3 holds KVM's shadow CR3, not - * L1's "real" CR3, which causes nested_vmx_restore_host_state() to - * corrupt vcpu->arch.cr3. Stuffing vmcs01.GUEST_CR3 results in the - * unwind naturally setting arch.cr3 to the correct value. Smashing - * vmcs01.GUEST_CR3 is safe because nested VM-Exits, and the unwind, - * reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is guaranteed to be - * overwritten with a shadow CR3 prior to re-entering L1. + * Stash L1's CR3, so that in the event of a "late" VM-Fail, i.e. a + * VM-Fail detected by hardware but not KVM, KVM can unwind its + * software model to the pre-VM-Entry host state. When EPT is + * disabled, GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3, + * and so simply restoring from vmcs01.GUEST_CR3 would corrupt + * vcpu->arch.cr3. */ - if (!enable_ept) - vmcs_writel(GUEST_CR3, vcpu->arch.cr3); + vmx->nested.pre_vmenter_cr3 = kvm_read_cr3(vcpu); vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02); @@ -4982,7 +4977,7 @@ static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW)); nested_ept_uninit_mmu_context(vcpu); - vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); + vcpu->arch.cr3 = vmx->nested.pre_vmenter_cr3; kvm_register_mark_available(vcpu, VCPU_REG_CR3); /* diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index de9de0d2016c..dc8517f15bc4 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -159,6 +159,13 @@ struct nested_vmx { bool has_preemption_timer_deadline; bool preemption_timer_expired; + /* + * Used to restore L1's CR3 if hardware detects a VM-Fail Consistency + * Check that KVM does not, in which case KVM needs to unwind CR3 back + * to its pre-VM-Enter state, NOT to vmcs01.HOST_CR3. + */ + unsigned long pre_vmenter_cr3; + /* * Used to snapshot MSRs that are conditionally loaded on VM-Enter in * order to propagate the guest's pre-VM-Enter value into vmcs02. For From 7b69729046a4c58f4cb457184e5ac4aaa179bff4 Mon Sep 17 00:00:00 2001 From: Haoxiang Li Date: Wed, 24 Jun 2026 14:19:10 +0800 Subject: [PATCH 38/48] KVM: s390: pci: Fix GISC refcount leak on AIF enable failure kvm_s390_gisc_register() registers the guest ISC before pinning the guest interrupt forwarding pages and allocating the AISB bit. If any of the later setup steps fails, the function unwinds the pinned pages and other local state, but does not unregister the GISC reference. Add the missing kvm_s390_gisc_unregister() to the error unwind path. Fixes: 3c5a1b6f0a18 ("KVM: s390: pci: provide routines for enabling/disabling interrupt forwarding") Cc: stable@vger.kernel.org Signed-off-by: Haoxiang Li Reviewed-by: Matthew Rosato Tested-by: Matthew Rosato Acked-by: Claudio Imbrenda Reviewed-by: Christian Borntraeger Signed-off-by: Claudio Imbrenda Message-ID: <20260624061910.2794734-1-haoxiang_li2024@163.com> Signed-off-by: Christian Borntraeger --- arch/s390/kvm/pci.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index 5b075c38998e..686113be0530 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -328,6 +328,7 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, unpin1: unpin_user_page(aibv_page); out: + kvm_s390_gisc_unregister(kvm, fib->fmt0.isc); return rc; } From 866d03de6def89c386cdfd457b28a1f566e02565 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Thu, 2 Jul 2026 17:23:59 +0200 Subject: [PATCH 39/48] KVM: s390: vsie: Avoid potential deadlock with real spaces The natural lock ordering is mmu_lock -> children_lock, but in gmap_create_shadow() the reverse order is used when handling shadowing of real address spaces. Convert the inner locking of kvm->mmu_lock to a trylock; return -EAGAIN if the lock is busy, and let the caller try again. This path is not expected to happen in real-life scenarios, so its performance is not important. Fixes: a2c17f9270cc ("KVM: s390: New gmap code") Signed-off-by: Claudio Imbrenda Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/kvm/gmap.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c index 298fbaecec28..8abb4f55b306 100644 --- a/arch/s390/kvm/gmap.c +++ b/arch/s390/kvm/gmap.c @@ -1374,8 +1374,13 @@ struct gmap *gmap_create_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *pare /* Only allow one real-space gmap shadow. */ list_for_each_entry(sg, &parent->children, list) { if (sg->guest_asce.r) { - scoped_guard(write_lock, &parent->kvm->mmu_lock) + if (write_trylock(&parent->kvm->mmu_lock)) { gmap_unshadow(sg); + write_unlock(&parent->kvm->mmu_lock); + } else { + gmap_put(new); + return ERR_PTR(-EAGAIN); + } break; } } From 4d4a21e38f1b87a76b3e63d4f837ff4e9b52d5a6 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Thu, 2 Jul 2026 17:24:05 +0200 Subject: [PATCH 40/48] KVM: s390: Fix dat_crste_walk_range() early return If a walk entry handler for a lower level returns a value, dat_crste_walk_range() will not return immediately, but instead loop again and move to the next entry. This means that some entries are potentially skipped, and early return is ignored. Skipped entries might lead to all kinds of issues, given that the caller expects them to not be skipped. Early return is often used to interrupt a walk when a rescheduling is needed; if it is ignored it can lead to stalls. Fix by breaking from the loop immediately if the walk to a lower level returned non-zero. Fixes: 2db149a0a6c5 ("KVM: s390: KVM page table management functions: walks") Signed-off-by: Claudio Imbrenda Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/kvm/dat.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c index 5f1960ec982d..ed4259d17629 100644 --- a/arch/s390/kvm/dat.c +++ b/arch/s390/kvm/dat.c @@ -570,6 +570,8 @@ static long dat_crste_walk_range(gfn_t start, gfn_t end, struct crst_table *tabl else if (walk->ops->pte_entry) rc = dat_pte_walk_range(max(start, cur), min(end, next), dereference_pmd(crste.pmd), walk); + if (rc) + break; } } return rc; From 9489220fe0e69d2ca141e5062dd3ef3e2e55959f Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Thu, 2 Jul 2026 17:24:06 +0200 Subject: [PATCH 41/48] KVM: s390: Improve kvm_s390_vm_stop_migration() There is no need to clear cmma-dirty state if the VM is not using CMMA. Skip the CMMA-related code if CMMA is not in use. Fixes: 6cfd47f91f6a ("KVM: s390: Fix cmma dirty tracking") Fixes: 190df4a212a7 ("KVM: s390: CMMA tracking, ESSA emulation, migration mode") Signed-off-by: Claudio Imbrenda Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/kvm/kvm-s390.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 23c817595e28..150b5dd2170e 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -1280,8 +1280,10 @@ static int kvm_s390_vm_stop_migration(struct kvm *kvm) * PGSTEs might have cmma_d set. */ WRITE_ONCE(kvm->arch.migration_mode, 0); - if (kvm->arch.use_cmma) - kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION); + if (!kvm->arch.use_cmma) + return 0; + + kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION); /* Clear cmma_d on all existing PGSTEs and set cmma_dirty_pages to 0. */ gmap_set_cmma_all_clean(kvm->arch.gmap); atomic64_set(&kvm->arch.cmma_dirty_pages, 0); From 3e3aa6da87d30a0064a17b836685cd43c90a3572 Mon Sep 17 00:00:00 2001 From: Matthew Rosato Date: Thu, 9 Jul 2026 09:54:04 -0400 Subject: [PATCH 42/48] KVM: s390: pci: Fix handling of AIF enable without AISB When a guest seeks to register IRQs without a summary bit specified, ensure that the associated GAITE then stores 0 for the guest AISB location instead of virt_to_phys(page_address(NULL)). Fixes: 3c5a1b6f0a18 ("KVM: s390: pci: provide routines for enabling/disabling interrupt forwarding") Cc: stable@vger.kernel.org Reviewed-by: Farhan Ali Signed-off-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- arch/s390/kvm/pci.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index 686113be0530..720bb58cabe2 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -300,9 +300,14 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, gaite->gisc = fib->fmt0.isc; gaite->count++; - gaite->aisbo = fib->fmt0.aisbo; - gaite->aisb = virt_to_phys(page_address(aisb_page) + (fib->fmt0.aisb & - ~PAGE_MASK)); + if (fib->fmt0.sum == 1) { + gaite->aisbo = fib->fmt0.aisbo; + gaite->aisb = virt_to_phys(page_address(aisb_page) + + (fib->fmt0.aisb & ~PAGE_MASK)); + } else { + gaite->aisbo = 0; + gaite->aisb = 0; + } aift->kzdev[zdev->aisb] = zdev->kzdev; spin_unlock_irq(&aift->gait_lock); From 6ee4140788234a6fabf59e6a50e38cdb936008cd Mon Sep 17 00:00:00 2001 From: Atish Patra Date: Tue, 2 Jun 2026 15:36:32 -0700 Subject: [PATCH 43/48] KVM: SEV: Do not allow intra-host migration/mirroring of SNP VMs The intra-host migration/mirroring feature is not fully implemented for SEV-SNP VMs. The proper migration requires additional SNP-specific state such as guest_req_mutex, guest_req_buf, and guest_resp_buf to be transferred or initialized on the destination. The SNP VM mirroring requires vmsa features to be copied as well otherwise ASID would be bound to SNP range while VM is detected as a SEV VM. Reject SNP source VMs in migration/mirroring until proper SNP state transfer is implemented. Fixes: 1dfe571c12cf ("KVM: SEV: Add initial SEV-SNP support") Reported-by: Chris Mason Reported-by: Sashiko Assisted-by: Claude:claude-opus-4-6 Reviewed-by: Tom Lendacky Signed-off-by: Atish Patra Link: https://patch.msgid.link/20260602-sev_snp_fixes-v3-1-24bfd3ae047c@meta.com Cc: stable@vger.kernel.org [sean: let lines poke past 80 chars, tag for stable] Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 427229347876..944aaea6501f 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2129,8 +2129,9 @@ int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) if (ret) return ret; + /* Do not allow SNP VM migration until additional state transfer is implemented */ if (kvm->arch.vm_type != source_kvm->arch.vm_type || - sev_guest(kvm) || !sev_guest(source_kvm)) { + sev_guest(kvm) || !sev_guest(source_kvm) || sev_snp_guest(source_kvm)) { ret = -EINVAL; goto out_unlock; } @@ -2851,8 +2852,9 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) * disallow out-of-band SEV/SEV-ES init if the target is already an * SEV guest, or if vCPUs have been created. KVM relies on vCPUs being * created after SEV/SEV-ES initialization, e.g. to init intercepts. + * Also do not allow SNP VM mirroring until additional state transfer is implemented. */ - if (sev_guest(kvm) || !sev_guest(source_kvm) || + if (sev_guest(kvm) || !sev_guest(source_kvm) || sev_snp_guest(source_kvm) || is_mirroring_enc_context(source_kvm) || kvm->created_vcpus) { ret = -EINVAL; goto e_unlock; From df371f2c62446c5fc5c5c1c91b47c219bb0d2100 Mon Sep 17 00:00:00 2001 From: Atish Patra Date: Tue, 2 Jun 2026 15:36:33 -0700 Subject: [PATCH 44/48] KVM: selftests: Verify SNP VMs are rejected from migration and mirroring Migration and mirroring of SEV-SNP VMs are not supported yet. Add two selftests that verify KVM rejects intra-host migration and mirroring when the source VM is an SNP VM, so the restriction stays enforced until proper SNP state transfer is implemented. Signed-off-by: Atish Patra Link: https://patch.msgid.link/20260602-sev_snp_fixes-v3-2-24bfd3ae047c@meta.com Signed-off-by: Sean Christopherson --- .../selftests/kvm/x86/sev_migrate_tests.c | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tools/testing/selftests/kvm/x86/sev_migrate_tests.c b/tools/testing/selftests/kvm/x86/sev_migrate_tests.c index 42bc023d5193..d59abb198d86 100644 --- a/tools/testing/selftests/kvm/x86/sev_migrate_tests.c +++ b/tools/testing/selftests/kvm/x86/sev_migrate_tests.c @@ -313,6 +313,49 @@ static void test_sev_mirror_parameters(void) kvm_vm_free(vm_no_vcpu); } +static void test_sev_snp_migrate_reject(void) +{ + struct kvm_vm *src_vm, *dst_vm; + int ret; + + src_vm = vm_create_barebones_type(KVM_X86_SNP_VM); + snp_vm_init(src_vm); + __vm_vcpu_add(src_vm, 0); + vm_sev_launch(src_vm, snp_default_policy(), NULL); + + dst_vm = vm_create_barebones_type(KVM_X86_SNP_VM); + __vm_vcpu_add(dst_vm, 0); + + ret = __sev_migrate_from(dst_vm, src_vm); + TEST_ASSERT(ret == -1 && errno == EINVAL, + "SNP VM migration should be rejected. ret: %d, errno: %d", + ret, errno); + + kvm_vm_free(src_vm); + kvm_vm_free(dst_vm); +} + +static void test_sev_snp_mirror_reject(void) +{ + struct kvm_vm *src_vm, *dst_vm; + int ret; + + src_vm = vm_create_barebones_type(KVM_X86_SNP_VM); + snp_vm_init(src_vm); + __vm_vcpu_add(src_vm, 0); + vm_sev_launch(src_vm, snp_default_policy(), NULL); + + dst_vm = aux_vm_create(false); + + ret = __sev_mirror_create(dst_vm, src_vm); + TEST_ASSERT(ret == -1 && errno == EINVAL, + "SNP VM mirroring should be rejected. ret: %d, errno: %d", + ret, errno); + + kvm_vm_free(src_vm); + kvm_vm_free(dst_vm); +} + static void test_sev_move_copy(void) { struct kvm_vm *dst_vm, *dst2_vm, *dst3_vm, *sev_vm, *mirror_vm, @@ -384,12 +427,16 @@ int main(int argc, char *argv[]) test_sev_migrate_parameters(); if (kvm_has_cap(KVM_CAP_VM_COPY_ENC_CONTEXT_FROM)) test_sev_move_copy(); + if (kvm_cpu_has(X86_FEATURE_SEV_SNP)) + test_sev_snp_migrate_reject(); } if (kvm_has_cap(KVM_CAP_VM_COPY_ENC_CONTEXT_FROM)) { test_sev_mirror(/* es= */ false); if (have_sev_es) test_sev_mirror(/* es= */ true); test_sev_mirror_parameters(); + if (kvm_cpu_has(X86_FEATURE_SEV_SNP)) + test_sev_snp_mirror_reject(); } return 0; } From cfbebb55e5127dc162e73fa8956000055a78606c Mon Sep 17 00:00:00 2001 From: Binbin Wu Date: Fri, 10 Jul 2026 11:53:23 +0800 Subject: [PATCH 45/48] KVM: TDX: Reject concurrent change to CPUID entry count Reject KVM_TDX_INIT_VM if userspace changes cpuid.nent between the initial read and the subsequent copy of the initialization data. tdx_td_init() first reads user_data->cpuid.nent to size the flexible kvm_tdx_init_vm copy. The copied structure also contains cpuid.nent, and that field can differ from the value used to size the allocation if userspace modifies the input concurrently. setup_tdparams_cpuids() later passes init_vm->cpuid.nent to kvm_find_cpuid_entry2(), which uses it as the array bound for the copied entries. Require the copied count to match the value used to size the allocation so that CPUID parsing cannot access beyond the entries actually copied. Fixes: 0bd0a4a1428b ("KVM: TDX: Replace kmalloc + copy_from_user with memdup_user in tdx_td_init()") Reported-by: Sashiko:gemini-3.1-pro-preview Cc: Signed-off-by: Binbin Wu Reviewed-by: Xiaoyao Li Reviewed-by: Thorsten Blum Link: https://patch.msgid.link/20260710035324.3170534-1-binbin.wu@linux.intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 989ab29b8c6f..545b03d9d10b 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2797,7 +2797,11 @@ static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) goto out; } - if (init_vm->cpuid.padding) { + /* + * Reject the request if userspace changes cpuid.nent between the + * initial read and the subsequent copy. + */ + if (init_vm->cpuid.padding || init_vm->cpuid.nent != nr_user_entries) { ret = -EINVAL; goto out; } From 9c66085e6dcfb44b70970aa4e323003fc7f2b738 Mon Sep 17 00:00:00 2001 From: Daniel Paziyski Date: Fri, 10 Jul 2026 15:40:54 +0200 Subject: [PATCH 46/48] KVM: x86: Fix null pointer deref due to dummy array in trace_kvm_inj_exception() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The trace_kvm_inj_exception tracepoint takes as arguments the exception vector, whether the exception has an error code (and subsequently, the error code), and whether it is being reinjected. Because '0' is a valid error code, KVM uses __print_symbolic() to format the error code as a string to avoid printing the error code entirely if the exception doesn't have an error code (see commit 21d4c575eb4a ("KVM: x86: Print error code in exception injection tracepoint iff valid"). KVM's abuse of __print_symbolic() was all fine and dandy, until commit 754e38d2d1ae ("tracing: Use explicit array size instead of sentinel elements in symbol printing") reworked the printing to avoid terminating the arrays with NULL/0 values, and missed KVM's clever use of not-quite empty array of symbols. BUG: kernel NULL pointer dereference, address: 0000000000000000 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page PGD 0 P4D 0 Oops: Oops: 0000 [#1] SMP CPU: 20 UID: 0 PID: 791 Comm: less Not tainted 7.2.0-rc2 #401 PREEMPT Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 0.0.0 02/06/2015 RIP: 0010:strlen+0x0/0x20 Call Trace: trace_seq_puts+0x18/0x80 trace_print_symbols_seq+0x68/0xa0 trace_raw_output_kvm_inj_exception+0x64/0xf0 [kvm] s_show+0x47/0x110 seq_read_iter+0x2a5/0x4c0 seq_read+0xfd/0x130 vfs_read+0xb6/0x330 ? vfs_write+0x2f2/0x3f0 ksys_read+0x61/0xd0 do_syscall_64+0xb7/0x570 entry_SYSCALL_64_after_hwframe+0x4b/0x53 RIP: 0033:0x7ff283714862 Simply drop the dummy array entirely, so that __print_symbolic() generates a truly empty array. Signed-off-by: Daniel Paziyski Fixes: 754e38d2d1ae ("tracing: Use explicit array size instead of sentinel elements in symbol printing") Reviewed-by: Thomas Weißschuh Link: https://patch.msgid.link/20260710134055.16432-1-danielpaziyski@gmail.com [sean: massage changelog, add splat, add Fixes, cc stable] Signed-off-by: Sean Christopherson --- arch/x86/kvm/trace.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index 0db25bba17f6..93de876c318c 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -490,7 +490,7 @@ TRACE_EVENT(kvm_inj_exception, TP_printk("%s%s%s%s%s", __print_symbolic(__entry->exception, kvm_trace_sym_exc), !__entry->has_error ? "" : " (", - !__entry->has_error ? "" : __print_symbolic(__entry->error_code, { }), + !__entry->has_error ? "" : __print_symbolic(__entry->error_code), !__entry->has_error ? "" : ")", __entry->reinjected ? " [reinjected]" : "") ); From 2f2312c422fd2695da772cecb30c69994b795964 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 14 Jul 2026 09:03:06 -0700 Subject: [PATCH 47/48] KVM: nVMX: Put vmcs12 pages if nested VM-Enter fails due to invalid guest state Put all vmcs12 pages if KVM synthesizes a nested VM-Exit due to invalid guest while emulating VMLAUNCH or VMRESUME. The invalid guest state path doesn't use nested_vmx_vmexit() as that API is intended to be used if and only if L2 is active, and the open coded equivalent neglects to put the vmcs12 pages. Failure to put the vmcs12 pages leaks any pinned pages (and/or mappings) if L1 retries VMLAUNCH/VMRESUME. Note, the !from_vmenter scenario doesn't suffer the same problem, as vmx_get_nested_state_pages() only gets/pins/maps the vmcs12 pages if L2 is active, i.e. if a "full" VM-Exit is guaranteed before KVM will retry getting vmcs12 pages. Fixes: 96c66e87deee ("KVM/nVMX: Use kvm_vcpu_map when mapping the virtual APIC page") Fixes: 3278e0492554 ("KVM/nVMX: Use kvm_vcpu_map when mapping the posted interrupt descriptor table") Fixes: fe1911aa443e ("KVM: nVMX: Use kvm_vcpu_map() to get/pin vmcs12's APIC-access page") Reported-by: Minh Nguyen Cc: stable@vger.kernel.org Signed-off-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index bb0eb40b4448..220d42ebc82e 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -3761,6 +3761,8 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, if (!from_vmentry) return NVMX_VMENTRY_VMEXIT; + nested_put_vmcs12_pages(vcpu); + load_vmcs12_host_state(vcpu, vmcs12); vmcs12->vm_exit_reason = exit_reason.full; if (enable_shadow_vmcs || nested_vmx_is_evmptr12_valid(vmx)) From 25f744ffa0c8e799e06250ce2e618367b166b0d4 Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Wed, 15 Jul 2026 06:35:06 +0000 Subject: [PATCH 48/48] KVM: SVM: Bump asid_generation on CPU online to avoid ASID collision after hotplug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a vCPU stays scheduled out (or blocked) while the last pCPU it ran on goes through a hotplug cycle (online->offline->online), and the vCPU then resumes execution on the same pCPU, then it is possible for it to run with an ASID that has now been assigned to a different vCPU, resulting in stale TLB translations being used. svm_enable_virtualization_cpu() resets asid_generation to 1 and sets next_asid to max_asid + 1 on every CPU online event, including hotplug cycles. Because next_asid starts beyond the pool boundary, the first call to new_asid() after an online event always wraps the pool, incrementing asid_generation to 2 and assigning ASIDs starting from min_asid. Consider two vCPUs from different VMs, vCPU-A pinned to CPU-X holding asid_generation=2 and ASID=N from before the hotplug event: 1. CPU-X goes offline and back online: asid_generation resets to 1, next_asid = max_asid + 1. 2. One or more vCPUs migrate to CPU-X and call new_asid(), wrapping the pool and consuming ASIDs starting from min_asid. Eventually vCPU-B from a different VM is assigned asid_generation=2, ASID=N — the same ASID that vCPU-A held before the hotplug. 3. vCPU-A enters pre_svm_run() on CPU-X: current_vmcb->cpu is unchanged so the migration branch is skipped. Its saved asid_generation=2 matches sd->asid_generation=2, so the generation check silently passes and vCPU-A continues running with ASID=N — the same ASID just freshly assigned to vCPU-B. Both vCPUs from different VMs now run on CPU-X with the same ASID, causing them to share NPT TLB entries and producing stale translations. The collision manifests as a KVM internal error (Suberror: 1, emulation failure). The NPT page fault reports a faulting GPA far outside the VM's physical memory range — a sign of stale TLB translations being used. KVM falls back to instruction emulation, which fails on FPU/XSave instructions (XRSTOR, STMXCSR) that the emulator does not implement. Fix this by incrementing asid_generation instead of resetting it to 1 in svm_enable_virtualization_cpu(). On module load, asid_generation starts at 0 (memset) and the increment produces 1, identical to the old behaviour. On subsequent hotplug cycles the generation advances beyond any value a vCPU previously observed on this CPU, so the generation check in pre_svm_run() reliably forces new_asid() on every vCPU after every hotplug cycle. Fixes: 774c47f1d78e ("[PATCH] KVM: cpu hotplug support") Reported-by: Chandrakanth Silveru Tested-by: Srikanth Aithal Reviewed-by: K Prateek Nayak Reviewed-by: Tom Lendacky Signed-off-by: Nikunj A Dadhania Message-ID: <20260715063506.672432-1-nikunj@amd.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 4d2bacd00ec4..d0971685034b 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -571,7 +571,12 @@ static int svm_enable_virtualization_cpu(void) return r; sd = per_cpu_ptr(&svm_data, me); - sd->asid_generation = 1; + /* + * Bump the current asid_generation value to ensure any vCPU that + * previously ran on this CPU sees a stale generation and is forced + * to acquire a new ASID, preventing a latent ASID collision. + */ + sd->asid_generation++; sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; sd->next_asid = sd->max_asid + 1; sd->min_asid = max_sev_asid + 1;