From c55bc773b6e814406658fae7dc5c15f639ed816e Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 6 Aug 2026 10:10:19 +0100 Subject: [PATCH 1/9] KVM: arm64: Remove VM-wide VNCR mapping counter The global VNCR mapping counter is used to decide whether an L1 provided VNCR page is mapped in L0 on any CPU at the point of dealing with a TLB invalidation. It is incremented when a mapping is made in the fixmap, and decremented when unmapped. As it turns out, this tracking has several flaws: - we are trying to invalidate TLBs, and the mapping is only an opportunistic consequence of the TLB. Checking this counter to decide whether a TLB needs to be invalidated may result in missed invalidations. - an L1 vcpu invalidating its own TLB (a very likely case) will not succeed in invalidating the VNCR pseudo TLB because that page is not mapped in L0 at this stage. Given that this tracking fails at delivering the minimum guarantees that are required and is only a performance optimisation, remove it completely. Fixes: 4ffa72ad8f37e ("KVM: arm64: nv: Add S1 TLB invalidation primitive for VNCR_EL2") Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Acked-by: Lorenzo Stoakes (ARM) Link: https://patch.msgid.link/20260806091026.620700-2-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_host.h | 3 --- arch/arm64/kvm/hyp/vhe/switch.c | 3 +-- arch/arm64/kvm/nested.c | 3 --- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index bae2c4f92ef5..ac16f96c878d 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -411,9 +411,6 @@ struct kvm_arch { /* Masks for VNCR-backed and general EL2 sysregs */ struct kvm_sysreg_masks *sysreg_masks; - /* Count the number of VNCR_EL2 currently mapped */ - atomic_t vncr_map_count; - /* * For an untrusted host VM, 'pkvm.handle' is used to lookup * the associated pKVM instance in the hypervisor. diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index bbe9cebd3d9d..c09b1d411c58 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -427,8 +427,7 @@ static bool kvm_hyp_handle_tlbi_el2(struct kvm_vcpu *vcpu, u64 *exit_code) * If we have to check for any VNCR mapping being invalidated, * go back to the slow path for further processing. */ - if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu) && - atomic_read(&vcpu->kvm->arch.vncr_map_count)) + if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu)) return false; __kvm_skip_instr(vcpu); diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index dfb96edbdc43..f3c75954cf36 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -48,7 +48,6 @@ void kvm_init_nested(struct kvm *kvm) { kvm->arch.nested_mmus = NULL; kvm->arch.nested_mmus_size = 0; - atomic_set(&kvm->arch.vncr_map_count, 0); } static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) @@ -890,7 +889,6 @@ static void this_cpu_reset_vncr_fixmap(struct kvm_vcpu *vcpu) clear_fixmap(vncr_fixmap(vcpu->arch.vncr_tlb->cpu)); vcpu->arch.vncr_tlb->cpu = -1; host_data_clear_flag(L1_VNCR_MAPPED); - atomic_dec(&vcpu->kvm->arch.vncr_map_count); } void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu) @@ -1592,7 +1590,6 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) { __set_fixmap(vncr_fixmap(vt->cpu), vt->hpa, prot); host_data_set_flag(L1_VNCR_MAPPED); - atomic_inc(&vcpu->kvm->arch.vncr_map_count); } } From 8053393680d4fa3eb962667d2be95dd39f0940e5 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 6 Aug 2026 10:10:20 +0100 Subject: [PATCH 2/9] KVM: arm64: Handle negative S1 walk levels in VNCR TLB size evaluation Computing the effects of a TLB invalidation involves looking at the size of the mapping cached by the TLB. For S1 mappings such as VNCR, this is deducted from the combination of the base granule size and the mapping level. However, this implies that the S1 MMU is *on*. When the MMU is off, we indicate this with the level being set to a "creative" value of -127 (S1_MMU_DISABLED). This ends-up being misinterpreted by pgshift_level_to_ttl() as it doesn't handle negative levels at all (the level is immediately cast to a u8 and only the bottom two bits considered), leading to an invalidation size of 0. Not helpful. Tidy-up pgshift_level_to_ttl() to handle these negative levels, and ttl_to_size() to always return SZ_1G when no valid TTL is present. This allows the removal of open-coded checks for similar situations. Note that the check for a negative value not explicitely checking for S1_MMU_DISABLED is deliberate, so that actual negative levels introduced with LVA2 and D128 can take the same path if we ever support them. Fixes: 7270cc9157f47 ("KVM: arm64: nv: Handle VNCR_EL2 invalidation from MMU notifiers") Reported-by: Hyunwoo Kim Link: https://lore.kernel.org/r/ameGoxbn2wzBq2kL@v4bel Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260806091026.620700-3-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/nested.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index f3c75954cf36..035cda256e2a 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -505,7 +505,7 @@ int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, return ret; } -static unsigned int ttl_to_size(u8 ttl) +static unsigned int __ttl_to_size(u8 ttl) { int level = ttl & 3; int gran = (ttl >> 2) & 3; @@ -561,10 +561,22 @@ static unsigned int ttl_to_size(u8 ttl) return max_size; } -static u8 pgshift_level_to_ttl(u16 shift, u8 level) +static unsigned int ttl_to_size(u8 ttl) +{ + return __ttl_to_size(ttl) ?: SZ_1G; +} + +static u8 pgshift_level_to_ttl(u16 shift, s8 level) { u8 ttl; + /* + * If we don't have a proper level, fallback to the maximum + * size. + */ + if (level < 0) + return 0; + switch(shift) { case 12: ttl = TLBI_TTL_TG_4K; @@ -675,7 +687,11 @@ unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val) ttl = get_guest_mapping_ttl(mmu, addr); } - max_size = ttl_to_size(ttl); + /* + * Don't use the default 1GB fallback, as we can adapt to the + * max mapping size we allow at S2. + */ + max_size = __ttl_to_size(ttl); if (!max_size) { /* Compute the maximum extent of the invalidation */ @@ -1124,8 +1140,6 @@ static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val, case OP_TLBI_VALE1OSNXS: scope->type = TLBI_VA; scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); - if (!scope->size) - scope->size = SZ_1G; scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1); scope->asid = FIELD_GET(TLBIR_ASID_MASK, val); break; @@ -1152,8 +1166,6 @@ static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val, case OP_TLBI_VAALE1OSNXS: scope->type = TLBI_VAA; scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); - if (!scope->size) - scope->size = SZ_1G; scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1); break; case OP_TLBI_RVAE2: From 8c774604b6ecaca495fa3d27c21593269627f48f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 6 Aug 2026 10:10:21 +0100 Subject: [PATCH 3/9] KVM: arm64: Consider SCTLR_EL2.M when mapping the L1 VNCR page We record a VNCR TLB even when SCTLR_EL2.M is 0 in order to make our life easier. But this is not something that the architecture anticipate. As a consequence, a hypervisor is free to set VNCR_EL2 to some PA when SCTLR_EL2.M==0, use it to run a guest which indirectly accesses the VNCR page, then eventually set SCTLR_EL2.M==1 with the same VA. Yes, this is odd, but apparently legal. A common trick in HW is to invalidate the TLBs on SCTLR_ELx.M being flipped. But doing this is a not a good idea for us (we'd need to trap SCTLR accesses), and wouldn't scale as we nest deeper. Instead, use the fact that the S1 MMU being off at the point of translation is cached in our TLB, and if it doesn't match the current MMU state, leave the VNCR unmapped. Fixes: 2a359e072596f ("KVM: arm64: nv: Handle mapping of VNCR_EL2 at EL2") Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260806091026.620700-4-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_nested.h | 7 +++++++ arch/arm64/kvm/at.c | 2 -- arch/arm64/kvm/nested.c | 4 ++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index 012d711034d1..bfed664d823b 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -388,6 +388,8 @@ struct s1_walk_result { bool failed; }; +#define S1_MMU_DISABLED (-127) + static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) { wr->fst = fst; @@ -396,6 +398,11 @@ static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) wr->failed = true; } +static inline bool s1_walk_translated(struct s1_walk_result *wr) +{ + return wr->level != S1_MMU_DISABLED; +} + int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, struct s1_walk_result *wr, u64 va); int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa, diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c index 640f2dc00a8b..0926426b8798 100644 --- a/arch/arm64/kvm/at.c +++ b/arch/arm64/kvm/at.c @@ -11,8 +11,6 @@ #include #include -#define S1_MMU_DISABLED (-127) - static int get_ia_size(struct s1_walk_info *wi) { return 64 - wi->txsz; diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 035cda256e2a..27bc7ee4b338 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1578,6 +1578,10 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) if (!vt->valid) return; + /* We cache the MMU state in the TLB. Check that it matches. */ + if (!!(vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_M) != s1_walk_translated(&vt->wr)) + return; + if (read_vncr_el2(vcpu) != vt->gva) return; From 34af2c3e31f91a739dc175459fdbd99ed952b457 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 6 Aug 2026 10:10:22 +0100 Subject: [PATCH 4/9] KVM: arm64: Correctly handle end of VA space TLBI invalidation Our TLB invalidation by VA code is based on comparing two ranges, one defined by the TLB, and one defined by the TLBI instruction. Each range is defined by a start and a size. However, the way the comparison is done doesn't account for address rollover, as it compares an address with (base + size). This works nicely until this expression represent the last page/block in the TTBR1 VA space, as the result is a big fat 0. And a failed TLB invalidation. Rewrite the comparison in a way that is immune to the address rollover (making the end address inclusive instead of exclusive), and move this into a common helper that is used by both VA and IPA invalidations, as suggested by Hyunwoo Kim (although the IPA version didn't suffer from this particular problem, obviously). Fixes: 4ffa72ad8f37e ("KVM: arm64: nv: Add S1 TLB invalidation primitive for VNCR_EL2") Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260806091026.620700-5-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/nested.c | 43 ++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 27bc7ee4b338..8a602d074dbb 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -999,6 +999,20 @@ static void invalidate_vncr(struct vncr_tlb *vt) clear_fixmap(vncr_fixmap(vt->cpu)); } +static bool vncr_tlb_intersects(struct vncr_tlb *vt, u64 addr, + u64 scope_start, u64 scope_size) +{ + u64 tlb_size, tlb_start, tlb_end, scope_end; + + tlb_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, vt->wr.level)); + + tlb_start = addr & ~(tlb_size - 1); + tlb_end = tlb_start + tlb_size - 1; + scope_end = scope_start + scope_size - 1; + + return !(tlb_end < scope_start || tlb_start > scope_end); +} + /* * VNCR TLB invalidation occurs from MMU notifiers or TLBI instructions, and * either can race against a vcpu not being onlined yet (no pseudo-TLB @@ -1021,19 +1035,9 @@ static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) return; - kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { - u64 ipa_start, ipa_end, ipa_size; - - ipa_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, - vt->wr.level)); - ipa_start = vt->wr.pa & ~(ipa_size - 1); - ipa_end = ipa_start + ipa_size; - - if (ipa_end <= start || ipa_start >= end) - continue; - - invalidate_vncr(vt); - } + kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) + if (vncr_tlb_intersects(vt, vt->wr.pa, start, end - start)) + invalidate_vncr(vt); } struct s1e2_tlbi_scope { @@ -1059,28 +1063,19 @@ static void invalidate_vncr_va(struct kvm *kvm, lockdep_assert_held_write(&kvm->mmu_lock); kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { - u64 va_start, va_end, va_size; - - va_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, - vt->wr.level)); - va_start = vt->gva & ~(va_size - 1); - va_end = va_start + va_size; - switch (scope->type) { case TLBI_ALL: break; case TLBI_VA: - if (va_end <= scope->va || - va_start >= (scope->va + scope->size)) + if (!vncr_tlb_intersects(vt, vt->gva, scope->va, scope->size)) continue; if (vt->wr.nG && vt->wr.asid != scope->asid) continue; break; case TLBI_VAA: - if (va_end <= scope->va || - va_start >= (scope->va + scope->size)) + if (!vncr_tlb_intersects(vt, vt->gva, scope->va, scope->size)) continue; break; From 38640bc32be3fcf9526d477155bc19d3f146231f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 6 Aug 2026 10:10:23 +0100 Subject: [PATCH 5/9] KVM: arm64: Handle VNCR TLB invalidation race with vcpu_put() VNCR unmapping While VNCR TLB invalidation always occurs under the MMU lock, vcpu_put() doesn't, while it unmaps the VNCR page. The problem is that the invalidation evaluates vncr_tlb::cpu to decide whether an unmapping needs to take place (cpu != -1) before performing it. On the other hand, this_cpu_reset_vncr_fixmap() unconditionally unmaps if L1_VNCR_MAPPED is set. These two obviously can race, with a TOCTOU pattern on the TLBI path, and a BUG_ON() on the vcpu_put() path. And the two can end-up calling vncr_fixmap(-1), with extra lethal effects. Move the reset of vncr_tlb::cpu to -1 to a common function, and make this update atomic so that only a single thread can reset the field and perform the corresponding unmap. The vcpu_put() still need to unconditionally unmap the current VNCR to close another ugly race. Finally, the assignment of vncr_tlb::cpu is moved to be kept in sync with the actual mapping, similar to L1_VNCR_MAPPED being set. Fixes: 7270cc9157f47 ("KVM: arm64: nv: Handle VNCR_EL2 invalidation from MMU notifiers") Reported-by: sashiko-bot@kernel.org Link: https://lore.kernel.org/r/20260801130237.0FD8F1F00ACA@smtp.kernel.org Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Reviewed-by: Yuan Yao Link: https://patch.msgid.link/20260806091026.620700-6-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/nested.c | 42 +++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 8a602d074dbb..cf0d45059edb 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -27,7 +27,7 @@ struct vncr_tlb { bool hpa_writable; /* -1 when not mapped on a CPU */ - int cpu; + atomic_t cpu; /* * true if the TLB is valid. Can only be changed with the @@ -894,16 +894,40 @@ void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) } } +/* + * Unmapping an L1 VNCR can happen concurrently without the mmu lock being + * effective (vcpu_put() vs TLBI handling). The atomic_xchg below ensures + * that only one CPU sets it to -1 while getting a valid CPU number back. + */ +static int unmap_l1_vncr(struct vncr_tlb *vt) +{ + int cpu = atomic_xchg_relaxed(&vt->cpu, -1); + + if (cpu != -1) + clear_fixmap(vncr_fixmap(cpu)); + + return cpu; +} + static void this_cpu_reset_vncr_fixmap(struct kvm_vcpu *vcpu) { if (!host_data_test_flag(L1_VNCR_MAPPED)) return; - BUG_ON(vcpu->arch.vncr_tlb->cpu != smp_processor_id()); BUG_ON(is_hyp_ctxt(vcpu)); - clear_fixmap(vncr_fixmap(vcpu->arch.vncr_tlb->cpu)); - vcpu->arch.vncr_tlb->cpu = -1; + /* + * Unconditionally unmap the local VNCR if we have lost the race + * against a concurrent TLBI. Otherwise we could end-up running + * another vcpu with VNCR still mapped if the TLBI thread is + * preempted between the exchange and the clear_fixmap(). + * + * Note that we do not care about the TLBI nuking the fixmap behind + * the back of an running vcpu. This will only generate a fault and + * possibly a retranslation. + */ + if (unmap_l1_vncr(vcpu->arch.vncr_tlb) == -1) + clear_fixmap(vncr_fixmap(smp_processor_id())); host_data_clear_flag(L1_VNCR_MAPPED); } @@ -995,8 +1019,7 @@ u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime) static void invalidate_vncr(struct vncr_tlb *vt) { vt->valid = false; - if (vt->cpu != -1) - clear_fixmap(vncr_fixmap(vt->cpu)); + unmap_l1_vncr(vt); } static bool vncr_tlb_intersects(struct vncr_tlb *vt, u64 addr, @@ -1452,7 +1475,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) vt->hpa = pfn << PAGE_SHIFT; vt->hpa_writable = writable; vt->valid = true; - vt->cpu = -1; + atomic_set(&vt->cpu, -1); kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable); @@ -1583,8 +1606,6 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) if (vt->wr.nG && get_asid_by_regime(vcpu, TR_EL20) != vt->wr.asid) return; - vt->cpu = smp_processor_id(); - if (vt->hpa_writable && vt->wr.pw && vt->wr.pr) prot = PAGE_KERNEL; else if (vt->wr.pr) @@ -1599,7 +1620,8 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) * FIXME: WO doesn't work at all, need POE support in the kernel. */ if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) { - __set_fixmap(vncr_fixmap(vt->cpu), vt->hpa, prot); + atomic_set(&vt->cpu, smp_processor_id()); + __set_fixmap(vncr_fixmap(atomic_read(&vt->cpu)), vt->hpa, prot); host_data_set_flag(L1_VNCR_MAPPED); } } From 2393470085649f0b973ecceb26fe8fc71edde0c1 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 6 Aug 2026 10:10:24 +0100 Subject: [PATCH 6/9] KVM: arm64: Sign-extend VA for range-based TLBI invalidation When the decode_range_tlbi() helper was moved to be used for S1 TLBIs, the required sign extension was omitted. Add it. As a result, special care must be taken to not overflow PA bits when this is used for S2 invalidation. Fixes: 85bba00425ae0 ("KVM: arm64: nv: Move TLBI range decoding to a helper") Reported-by: sashiko-bot@kernel.org Link: https://lore.kernel.org/r/20260801130337.EB2BA1F00AC4@smtp.kernel.org Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260806091026.620700-7-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_nested.h | 7 +++++++ arch/arm64/kvm/sys_regs.c | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index bfed664d823b..c83be6d0e79a 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -291,6 +291,13 @@ static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid) base = (val & GENMASK(36, 0)) << shift; + /* + * We only deal with at most 48bit VA/IPA, so 48 is where we + * sign-extend from. Should we support FEAT_L{VP}A* at some point, + * this will need to be revisited. + */ + base = (u64)sign_extend64(base, 48); + if (asid) *asid = FIELD_GET(TLBIR_ASID_MASK, val); diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 5d5c579d4579..797e888bf939 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -4057,6 +4057,7 @@ static bool handle_ripas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2); u64 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); u64 base, range; + int pa_bits; if (!kvm_supported_tlbi_ipas2_op(vcpu, sys_encoding)) return undef_access(vcpu, p, r); @@ -4068,6 +4069,16 @@ static bool handle_ripas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, */ base = decode_range_tlbi(p->regval, &range, NULL); + /* + * Ignore TLBIs that start out of PA_bits range, and cap the + * invalidation to the [base:bit(PA_bits)] interval. + */ + pa_bits = kvm_get_pa_bits(vcpu->kvm); + if (fls64(base) > pa_bits) + return true; + + range = min(range, BIT_ULL(pa_bits) - base); + kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr), &(union tlbi_info) { .range = { From 2b7324f3a0c1072b9d578b8d42f199506753f26e Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 6 Aug 2026 10:10:25 +0100 Subject: [PATCH 7/9] KVM: arm64: Make VNCR invalidation participate in MMU invalidation retry A VNCR TLB invalidation can occur on one vcpu while another vcpu is faulting in this same page. Without correctly handling this, we can end up with the following scenario: - vcpu A walks the PTs to translate VNCR - before vcpu A is able to grab the MMU lock to insert the TLB, vcpu B updates the S1 PTs with an invalid entry, and issues a TLBI S1E2 for this VA - vcpu A inserts the TLB for something that is now invalid This isn't a new problem, and we manage S2 by having the MMU notifier to bump up mmu_invalidate_seq on invalidation so that the fault can be replayed. We can perform something similar here, and extend invalidate_vncr_va() to update the same counter, clearly indicating that the context has changed under our feet. This is safe as the invalidation always happen while holding the MMU lock for write, and that we sample the sequence number before walking S1. Fixes: 4ffa72ad8f37e ("KVM: arm64: nv: Add S1 TLB invalidation primitive for VNCR_EL2") Reported-by: sashiko-bot@kernel.org Link: https://lore.kernel.org/r/20260801130454.5D9F11F00AC4@smtp.kernel.org Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260806091026.620700-8-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/nested.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index cf0d45059edb..550c9bd3dbe7 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1058,6 +1058,12 @@ static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) return; + /* + * Note that invalidating the VNCR on the back of an MMU notifier + * doesn't require messing with the invalidation counter for a + * parallel walk. The notifier itself will have bumped the counter, + * making sure we rewalk. + */ kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) if (vncr_tlb_intersects(vt, vt->wr.pa, start, end - start)) invalidate_vncr(vt); @@ -1085,6 +1091,15 @@ static void invalidate_vncr_va(struct kvm *kvm, lockdep_assert_held_write(&kvm->mmu_lock); + /* + * We might be performing a parallel S1 walk, so bump up the + * invalidation counter even in the absence of an actual VNCR TLB + * invalidation, as this could indicate that the guest has gone + * through a BBM sequence. + */ + kvm->mmu_invalidate_seq++; + smp_wmb(); + kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { switch (scope->type) { case TLBI_ALL: @@ -1419,15 +1434,15 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) va = read_vncr_el2(vcpu); + mmu_seq = vcpu->kvm->mmu_invalidate_seq; + smp_rmb(); + ret = __kvm_translate_va(vcpu, &vt->wi, &vt->wr, va); if (ret) return ret; write_fault = kvm_is_write_fault(vcpu); - mmu_seq = vcpu->kvm->mmu_invalidate_seq; - smp_rmb(); - gfn = vt->wr.pa >> PAGE_SHIFT; memslot = gfn_to_memslot(vcpu->kvm, gfn); if (!memslot) { From 62e3bb894a6e926a26132bc4f558d80c2997d249 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 6 Aug 2026 10:10:26 +0100 Subject: [PATCH 8/9] KVM: arm64: Add VNCR TLB tracking again Having established that our VNCR TLB tracking was flawed and having dropped it from KVM, it is time to replace it with something that actually works in order to get some performance back, as the TLBI overhead is otherwise pretty high. The goal of such tracking is to hit the TLBI slow path if there are any VNCR TLBs in the guest, irrespective of their mapping state. For this purpose, we introduce an VM wide counter (vncr_tlb_count) that tracks how many valid VNCR TLB are present. This means that creating such TLB must increment the counter, and invalidation decrement it, and both these operations must be done with the MMU lock held for write. On TLBI handling affecting EL2 S1, a non-zero counter forces the handling to take the slow path to consider the VNCR TLBs. Note that the bumping up is done "speculatively" in order to make sure that a concurrent invalidation will be taken on the slow path. Not exactly rocket science. Hopefully I got it right this time. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260806091026.620700-9-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_host.h | 3 +++ arch/arm64/kvm/hyp/vhe/switch.c | 11 +++++++--- arch/arm64/kvm/nested.c | 36 +++++++++++++++++++++++++------ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index ac16f96c878d..108966a9db12 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -411,6 +411,9 @@ struct kvm_arch { /* Masks for VNCR-backed and general EL2 sysregs */ struct kvm_sysreg_masks *sysreg_masks; + /* Count the number of VNCR_EL2 TLBs */ + atomic_t vncr_tlb_count; + /* * For an untrusted host VM, 'pkvm.handle' is used to lookup * the associated pKVM instance in the hypervisor. diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index c09b1d411c58..fa776b18a32c 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -424,10 +424,15 @@ static bool kvm_hyp_handle_tlbi_el2(struct kvm_vcpu *vcpu, u64 *exit_code) return false; /* - * If we have to check for any VNCR mapping being invalidated, - * go back to the slow path for further processing. + * If we have to check for any VNCR TLB being invalidated, go back + * to the slow path for further processing. + * + * The synchronisation betweem TLBI and walk is provided by the + * speculative increment of the TLB counter on walk, and the + * invalidation counter. Yes, this is fiddly. */ - if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu)) + if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu) && + atomic_read(&vcpu->kvm->arch.vncr_tlb_count)) return false; __kvm_skip_instr(vcpu); diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 550c9bd3dbe7..20af94197a8a 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -48,6 +48,7 @@ void kvm_init_nested(struct kvm *kvm) { kvm->arch.nested_mmus = NULL; kvm->arch.nested_mmus_size = 0; + atomic_set(&kvm->arch.vncr_tlb_count, 0); } static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) @@ -1016,10 +1017,12 @@ u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime) return asid; } -static void invalidate_vncr(struct vncr_tlb *vt) +static void invalidate_vncr(struct kvm *kvm, struct vncr_tlb *vt) { + BUG_ON(!vt->valid); vt->valid = false; unmap_l1_vncr(vt); + atomic_dec(&kvm->arch.vncr_tlb_count); } static bool vncr_tlb_intersects(struct vncr_tlb *vt, u64 addr, @@ -1066,7 +1069,7 @@ static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) */ kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) if (vncr_tlb_intersects(vt, vt->wr.pa, start, end - start)) - invalidate_vncr(vt); + invalidate_vncr(kvm, vt); } struct s1e2_tlbi_scope { @@ -1123,7 +1126,7 @@ static void invalidate_vncr_va(struct kvm *kvm, break; } - invalidate_vncr(vt); + invalidate_vncr(kvm, vt); } } @@ -1359,13 +1362,20 @@ void kvm_arch_flush_shadow_all(struct kvm *kvm) * intersects with the TLBI request, invalidate it, and unmap the page * from the fixmap. Because we need to look at all the vcpu-private TLBs, * this requires some wide-ranging locking to ensure that nothing races - * against it. This may require some refcounting to avoid the search when - * no such TLB is present. + * against it. This requires some refcounting to avoid the search when + * no such TLB is present (see below). * * - On MMU notifiers, we must invalidate our TLB in a similar way, but * looking at the IPA instead. The funny part is that there may not be a * stage-2 mapping for this page if L1 hasn't accessed it using LD/ST * instructions. + * + * - vncr_tlb_count tracks the number of valid VNCR TLBs VM-wide. This isn't + * the number of *mapped* L1 VNCR pages, which is likely be a subset (and + * by definition, a TLBI handled from L1 runs with the canonical VNCR + * page, not the L1's). The innermost trap handling code checks this to + * find out whether to return to the guest ASAP (no L1 TLBs) or to visit + * this part of the world for some extra invalidation work. */ int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu) @@ -1420,7 +1430,8 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) */ scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { this_cpu_reset_vncr_fixmap(vcpu); - vt->valid = false; + if (vt->valid) + invalidate_vncr(vcpu->kvm, vt); vt->wi = (struct s1_walk_info) { .regime = TR_EL20, @@ -1545,7 +1556,20 @@ int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu) return -EIO; } + /* + * Speculatively increment the TLB count to make sure concurrent + * TLBIs will take the slow path, and will interact with the retry + * mechanism. Drop it again on error. + */ + atomic_inc(&vcpu->kvm->arch.vncr_tlb_count); + smp_mb__after_atomic(); + ret = kvm_translate_vncr(vcpu, &is_gmem); + if (ret) { + smp_mb__before_atomic(); + atomic_dec(&vcpu->kvm->arch.vncr_tlb_count); + } + switch (ret) { case -EAGAIN: /* Let's try again... */ From 69a598288195947a1662b53de702eb6976af96b7 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 10 Aug 2026 18:06:16 +0100 Subject: [PATCH 9/9] KVM: arm64: Correctly cap TLBI Range to the architural limit TLB Invalidation by Range has a fairly powerful way of encoding pretty large ranges in a small number of bits. This range can be based on an arbitrary VA, which means it is pretty easy for a guest to generate an overflow should the hypervisor be naive enough to add the range to the base... Make sure the range is capped to the limit dictated by the address bit that determines the VA range. For an IPA invalidation, this is further corrected down the line to ignore the upper range. Fixes: 4ffa72ad8f37e ("KVM: arm64: nv: Add S1 TLB invalidation primitive for VNCR_EL2") Reported-by: Wei-Lin Chang Link: https://lore.kernel.org/r/yifz3wn5gk5sr6mapi32trgk5m5kp33bquctsjmkifebnsnndt@fix6u4rthx4g Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Reviewed-by: Wei-Lin Chang Link: https://patch.msgid.link/20260810170616.746100-1-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_nested.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index c83be6d0e79a..1ed708335809 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -305,6 +305,12 @@ static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid) num = FIELD_GET(GENMASK(43, 39), val); *range = __TLBI_RANGE_PAGES(num, scale) << shift; + /* Cap the range to the correct half of the address space */ + if (!(base & BIT(48))) + *range = min(*range, (BIT(48) - base)); + else + *range = min(*range, ~base + 1); + return base; }