From 978cd6b2ad036168712aad8fca213385a5b15e2d Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 2 Jun 2026 16:54:27 +0100 Subject: [PATCH 01/11] KVM: arm64: Key CPTR_EL2.E0POE propagation on FEAT_S1POE We propagate CPTR_EL2.E0POE from a L1 into the L0 configuration, but we key this on the L1 guest supporting FEAT_S2POE. This is obviously wrong, as this bit is solely concerned with Stage-1 translation. Fix this by making the update depend on FEAT_S1POE. Fixes: cd931bd6093cb ("KVM: arm64: nv: Add additional trap setup for CPTR_EL2") Reviewed-by: Joey Gouly Reviewed-by: Oliver Upton Link: https://patch.msgid.link/20260602155430.2088142-2-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/include/hyp/switch.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 98b2976837b1..4d814ae90613 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -141,7 +141,7 @@ static inline void __activate_cptr_traps_vhe(struct kvm_vcpu *vcpu) if (!(SYS_FIELD_GET(CPACR_EL1, ZEN, cptr) & BIT(0))) val &= ~CPACR_EL1_ZEN; - if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR3_EL1, S2POE, IMP)) + if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR3_EL1, S1POE, IMP)) val |= cptr & CPACR_EL1_E0POE; val |= cptr & CPTR_EL2_TCPAC; From f41b481548cc263112b6da4a3b4869fcd35b4e45 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 2 Jun 2026 16:54:28 +0100 Subject: [PATCH 02/11] KVM: arm64: Wire AT S1E1A in the system instruction handling table Despite having handling code for AT S1E1A, the instruction was never plugged into the system instruction table, leading to an exception being injected in the guest. If the guest is Linux and using the __kvm_at() helper, the exception is actually handled in the helper, and KVM continues more or less silently by reentering the guest. Not exactly what you'd expect. Fix this by plugging the emulation code where required. Fixes: ff987ffc0c18c ("KVM: arm64: nv: Add support for FEAT_ATS1A") Reviewed-by: Joey Gouly Reviewed-by: Oliver Upton Link: https://patch.msgid.link/20260602155430.2088142-3-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/sys_regs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 148fc3400ea8..753fe30d322c 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -4217,6 +4217,7 @@ static struct sys_reg_desc sys_insn_descs[] = { SYS_INSN(AT_S1E0W, handle_at_s1e01), SYS_INSN(AT_S1E1RP, handle_at_s1e01), SYS_INSN(AT_S1E1WP, handle_at_s1e01), + SYS_INSN(AT_S1E1A, handle_at_s1e01), { SYS_DESC(SYS_DC_CSW), access_dcsw }, { SYS_DESC(SYS_DC_CGSW), access_dcgsw }, From a62b4226ae47202eb00306b576859131c4c7196e Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 2 Jun 2026 16:54:29 +0100 Subject: [PATCH 03/11] arm64: cpufeature: Expose ID_AA64ISAR2_EL1.ATS1A to KVM KVM needs to know if the HW implements FEAT_ATS1A in order to correctly sanitise HFGITR_EL2.ATS1E1A, which otherwise defaults to RES0 and AT S1E1A traps are handled as UNDEF. Solves this by exposing ID_AA64ISAR2_EL1.ATS1A to the rest of the kernel. Fixes: ff987ffc0c18c ("KVM: arm64: nv: Add support for FEAT_ATS1A") Reviewed-by: Joey Gouly Reviewed-by: Oliver Upton Link: https://patch.msgid.link/20260602155430.2088142-4-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kernel/cpufeature.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 6d53bb15cf7b..62b0d77217ee 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -266,6 +266,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { }; static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_ATS1A_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_LUT_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_RPRFM_SHIFT, 4, 0), From 9f76b039a72d7e06374aa96862f0232ed53f7787 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Tue, 2 Jun 2026 16:54:46 -0700 Subject: [PATCH 04/11] KVM: arm64: Don't leak PFN when kvm_translate_vncr() races MMU notifier In the case that kvm_translate_vncr() races with an MMU notifier the early return does not release a reference on the faulted in PFN. Add the necessary call to kvm_release_faultin_page() for the unused PFN. Cc: stable@vger.kernel.org Fixes: 069a05e535496 ("KVM: arm64: nv: Handle VNCR_EL2-triggered faults") Reported-by: Sashiko (local):gemini-3.1-pro Signed-off-by: Oliver Upton Link: https://patch.msgid.link/20260602235450.103057-2-oupton@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/nested.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 883b6c1008fb..4fa82e96454d 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1326,8 +1326,10 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) } scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { - if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) + if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) { + kvm_release_faultin_page(vcpu->kvm, page, true, false); return -EAGAIN; + } vt->gva = va; vt->hpa = pfn << PAGE_SHIFT; From 5949004d7032767e8fde1e8c986a33f241b2a192 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Tue, 2 Jun 2026 16:54:47 -0700 Subject: [PATCH 05/11] KVM: arm64: nv: Fully update VNCR fixmap state in kvm_translate_vncr() kvm_translate_vncr() first invalidates the pseudo-TLB entry and corresponding fixmap in anticipation of installing a new translation. While the fixmap invalidation does clear the mapping from host stage-1, it does not clear the L1_VNCR_MAPPED flag. Depending on the state of the VNCR TLB at vcpu_put(), this could potentially precipitate a BUG_ON() if vt->cpu is reset. Share a helper with kvm_vcpu_put_hw_mmu(), ensuring that KVM's view of the VNCR fixmap is in sync with the state of the VNCR TLB. Give it a slightly verbose name to make it obvious that it is meant to be used local to a CPU, unlike other VNCR TLB maintenance. Fixes: 069a05e535496 ("KVM: arm64: nv: Handle VNCR_EL2-triggered faults") Signed-off-by: Oliver Upton Link: https://patch.msgid.link/20260602235450.103057-3-oupton@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/nested.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 4fa82e96454d..d0545144eaac 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -797,18 +797,24 @@ void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) } } +static void this_cpu_reset_vncr_fixmap(struct kvm_vcpu *vcpu) +{ + if (!host_data_test_flag(L1_VNCR_MAPPED)) + return; + + BUG_ON(vcpu->arch.vncr_tlb->cpu != smp_processor_id()); + BUG_ON(is_hyp_ctxt(vcpu)); + + clear_fixmap(vncr_fixmap(vcpu->arch.vncr_tlb->cpu)); + vcpu->arch.vncr_tlb->cpu = -1; + host_data_clear_flag(L1_VNCR_MAPPED); + atomic_dec(&vcpu->kvm->arch.vncr_map_count); +} + void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu) { /* Unconditionally drop the VNCR mapping if we have one */ - if (host_data_test_flag(L1_VNCR_MAPPED)) { - BUG_ON(vcpu->arch.vncr_tlb->cpu != smp_processor_id()); - BUG_ON(is_hyp_ctxt(vcpu)); - - clear_fixmap(vncr_fixmap(vcpu->arch.vncr_tlb->cpu)); - vcpu->arch.vncr_tlb->cpu = -1; - host_data_clear_flag(L1_VNCR_MAPPED); - atomic_dec(&vcpu->kvm->arch.vncr_map_count); - } + this_cpu_reset_vncr_fixmap(vcpu); /* * Keep a reference on the associated stage-2 MMU if the vCPU is @@ -1282,7 +1288,8 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) * We also prepare the next walk wilst we're at it. */ scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { - invalidate_vncr(vt); + this_cpu_reset_vncr_fixmap(vcpu); + vt->valid = false; vt->wi = (struct s1_walk_info) { .regime = TR_EL20, From efa871f4a2517385295de2e3f786e4ae4ffa6e77 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Tue, 2 Jun 2026 16:54:48 -0700 Subject: [PATCH 06/11] KVM: arm64: nv: Inject SEA TTW when desc update can't write to GPA Similar to the handling of descriptor reads, inject an SEA during TTW when the descriptor access fails for reasons other than a race, such as a read-only memslot or a bad HVA. Fixes: bff8aa213dee ("KVM: arm64: Implement HW access flag management in stage-1 SW PTW") Fixes: e4c7dfac2f1a ("KVM: arm64: nv: Implement HW access flag management in stage-2 SW PTW") Signed-off-by: Oliver Upton Link: https://patch.msgid.link/20260602235450.103057-4-oupton@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/at.c | 6 +++++- arch/arm64/kvm/nested.c | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c index 9f8f0ae8e86e..119a603e636e 100644 --- a/arch/arm64/kvm/at.c +++ b/arch/arm64/kvm/at.c @@ -521,8 +521,12 @@ static int walk_s1(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, } ret = kvm_swap_s1_desc(vcpu, ipa, desc, new_desc, wi); - if (ret) + if (ret == -EAGAIN) return ret; + if (ret) { + fail_s1_walk(wr, ESR_ELx_FSC_SEA_TTW(level), false); + return ret; + } desc = new_desc; } diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index d0545144eaac..f8d3f3a72328 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -352,8 +352,13 @@ static int walk_nested_s2_pgd(struct kvm_vcpu *vcpu, phys_addr_t ipa, if (new_desc != desc) { ret = swap_guest_s2_desc(vcpu, paddr, desc, new_desc, wi); - if (ret) + if (ret == -EAGAIN) return ret; + if (ret) { + out->esr = ESR_ELx_FSC_SEA_TTW(level); + out->desc = desc; + return 1; + } desc = new_desc; } From 699a2cc7f608145d55621e57828ccf6bfcb8d906 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Tue, 2 Jun 2026 16:54:49 -0700 Subject: [PATCH 07/11] KVM: arm64: Restart instruction upon race in __kvm_at_s12() __kvm_at_s*() are expected to return -EAGAIN if the page table walk raced with a concurrent update to a page table descriptor, which is interpreted as a signal to restart the trapping instruction. While this mostly works, __kvm_at_s12() silently eats the return from __kvm_at_s1e01() and consumes an uninitialized PAR value. Propagate the nonzero return instead. Fixes: 92c6443222ca ("KVM: arm64: Propagate PTW errors up to AT emulation") Signed-off-by: Oliver Upton Link: https://patch.msgid.link/20260602235450.103057-5-oupton@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/at.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c index 119a603e636e..6cc5892023dd 100644 --- a/arch/arm64/kvm/at.c +++ b/arch/arm64/kvm/at.c @@ -1557,7 +1557,10 @@ int __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) return 0; } - __kvm_at_s1e01(vcpu, op, vaddr); + ret = __kvm_at_s1e01(vcpu, op, vaddr); + if (ret) + return ret; + par = vcpu_read_sys_reg(vcpu, PAR_EL1); if (par & SYS_PAR_EL1_F) return 0; From d8839941df7de41fd4b02b7b7cdd0c46e5ba501e Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Tue, 2 Jun 2026 16:54:50 -0700 Subject: [PATCH 08/11] KVM: arm64: nv: Restart stage-1 walk if stage-2 desc update fails kvm_walk_nested_s2() returns -EAGAIN as an indication that an underlying descriptor update fails due to a race. The expectation is that the caller restart translation, yet walk_s1() actually synthesizes an abort. Propagate the -EAGAIN return out of walk_s1(), relying on callers to restart the translation fetch. Fixes: e4c7dfac2f1a ("KVM: arm64: nv: Implement HW access flag management in stage-2 SW PTW") Signed-off-by: Oliver Upton Link: https://patch.msgid.link/20260602235450.103057-6-oupton@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/at.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c index 6cc5892023dd..4d4285e60fce 100644 --- a/arch/arm64/kvm/at.c +++ b/arch/arm64/kvm/at.c @@ -423,6 +423,9 @@ static int walk_s1(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, if (wi->s2) { ret = kvm_walk_nested_s2(vcpu, ipa, &s2_trans); + if (ret == -EAGAIN) + return ret; + if (ret) { fail_s1_walk(wr, (s2_trans.esr & ~ESR_ELx_FSC_LEVEL) | level, From 6bef47288ce1cb8302c84753164b8f8f6d63e0b3 Mon Sep 17 00:00:00 2001 From: Wei-Lin Chang Date: Fri, 5 Jun 2026 19:52:55 +0100 Subject: [PATCH 09/11] KVM: arm64: Fix block mapping validity check in stage-1 walker For the 64K granule size, FEAT_LPA determines whether a level 1 mapping is allowed. Using the result of has_52bit_pa() is too restrictive, as it also checks the selected output addressi size in TCR.(I)PS. Fix it by only checking FEAT_LPA. Fixes: 5da3a3b27a01 ("KVM: arm64: Expand valid block mappings to FEAT_LPA/LPA2 support") Signed-off-by: Wei-Lin Chang Link: https://patch.msgid.link/20260605185255.2431996-1-weilin.chang@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/at.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c index 4d4285e60fce..7663df5e03b7 100644 --- a/arch/arm64/kvm/at.c +++ b/arch/arm64/kvm/at.c @@ -495,15 +495,18 @@ static int walk_s1(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, /* Block mapping, check the validity of the level */ if (!(desc & BIT(1))) { bool valid_block = false; + bool lpa = kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, PARANGE, 52); switch (BIT(wi->pgshift)) { case SZ_4K: valid_block = level == 1 || level == 2 || (wi->pa52bit && level == 0); break; case SZ_16K: - case SZ_64K: valid_block = level == 2 || (wi->pa52bit && level == 1); break; + case SZ_64K: + valid_block = level == 2 || (lpa && level == 1); + break; } if (!valid_block) From 4be6cbeb93d26994bd1827ddbce391e3c4395c8f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sun, 7 Jun 2026 18:57:45 +0100 Subject: [PATCH 10/11] KVM: arm64: nv: Avoid dereferencing NULL VNCR pseudo-TLB VNCR TLB invalidation occurs from MMU notifiers or TLBI instructions, and either can race against a vcpu not being onlined yet (no pseudo-TLB allocated). Similarly, the TLB might be invalid, and the invalidation should be skipped in this case. Both kvm_invalidate_vncr_ipa() and kvm_invalidate_vncr_va() are expected to perform the same checks, except that the latter doesn't check for the allocation and blindly dereferences the pointer. Solve this by introducing a new iterator built on top of the usual kvm_for_each_vcpu() that checks for both of the above conditions, and convert the two users to it. Reported-by: Hyunwoo Kim Link: https://lore.kernel.org/r/aiUvSbrWndQeUPc8@v4bel Fixes: 4ffa72ad8f37 ("KVM: arm64: nv: Add S1 TLB invalidation primitive for VNCR_EL2") Cc: stable@vger.kernel.org Reviewed-by: Oliver Upton Link: https://patch.msgid.link/20260607175745.297793-1-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/nested.c | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index f8d3f3a72328..690b8e856416 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -908,9 +908,21 @@ static void invalidate_vncr(struct vncr_tlb *vt) clear_fixmap(vncr_fixmap(vt->cpu)); } +/* + * VNCR TLB invalidation occurs from MMU notifiers or TLBI instructions, and + * either can race against a vcpu not being onlined yet (no pseudo-TLB + * allocated). Similarly, the TLB might be invalid. Skip those, as they + * obviously don't participate in the invalidation at this stage. + */ +#define kvm_for_each_vncr_tlb(idx, vcpup, tlbp, kvm) \ + kvm_for_each_vcpu(idx, vcpup, kvm) \ + if (((tlbp) = vcpup->arch.vncr_tlb) && \ + (tlbp)->valid) + static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) { struct kvm_vcpu *vcpu; + struct vncr_tlb *vt; unsigned long i; lockdep_assert_held_write(&kvm->mmu_lock); @@ -918,24 +930,9 @@ static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) return; - kvm_for_each_vcpu(i, vcpu, kvm) { - struct vncr_tlb *vt = vcpu->arch.vncr_tlb; + kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { u64 ipa_start, ipa_end, ipa_size; - /* - * Careful here: We end-up here from an MMU notifier, - * and this can race against a vcpu not being onlined - * yet, without the pseudo-TLB being allocated. - * - * Skip those, as they obviously don't participate in - * the invalidation at this stage. - */ - if (!vt) - continue; - - if (!vt->valid) - continue; - ipa_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, vt->wr.level)); ipa_start = vt->wr.pa & ~(ipa_size - 1); @@ -965,17 +962,14 @@ static void invalidate_vncr_va(struct kvm *kvm, struct s1e2_tlbi_scope *scope) { struct kvm_vcpu *vcpu; + struct vncr_tlb *vt; unsigned long i; lockdep_assert_held_write(&kvm->mmu_lock); - kvm_for_each_vcpu(i, vcpu, kvm) { - struct vncr_tlb *vt = vcpu->arch.vncr_tlb; + kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { u64 va_start, va_end, va_size; - if (!vt->valid) - continue; - va_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, vt->wr.level)); va_start = vt->gva & ~(va_size - 1); From 4b54e2374d1bd82031cef9784e125a7100a32499 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 8 Jun 2026 09:11:08 +0100 Subject: [PATCH 11/11] KVM: arm64: nv: Hold kvm->mmu_lock while initialising vcpu->arch.vncr_tlb Sashiko reports that there is a race between initialising vncr_tlb and making use of it, as we don't hold the mmu_lock at this point. Additionally, it identifies a memory leak, should userspace repeatedly invokes the KVM_RUN ioctl after a failure of kvm_arch_vcpu_run_pid_change(), as we assign vncr_tlb blindly on first run, irrespective of prior allocations. Slap the two bugs in one go by taking the kvm->mmu_lock on assigning vncr_tlb, preventing the race for good, and by checking that vncr_tlb is indeed NULL prior to allocation. Reported-by: Sashiko Link: https://lore.kernel.org/r/20260607180815.85FBC1F00893@smtp.kernel.org Reviewed-by: Oliver Upton Link: https://patch.msgid.link/20260608081108.2244133-1-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/nested.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 690b8e856416..326adf404d98 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1253,8 +1253,20 @@ int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu) if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) return 0; - vcpu->arch.vncr_tlb = kzalloc_obj(*vcpu->arch.vncr_tlb, - GFP_KERNEL_ACCOUNT); + if (!vcpu->arch.vncr_tlb) { + struct vncr_tlb *vt = kzalloc_obj(*vcpu->arch.vncr_tlb, + GFP_KERNEL_ACCOUNT); + + /* + * Taking the lock on assignment ensures that the TLB is + * seen as initialised when following the pointer (release + * semantics of the unlock), and avoids having acquires on + * each user which already take the lock. + */ + scoped_guard(write_lock, &vcpu->kvm->mmu_lock) + vcpu->arch.vncr_tlb = vt; + } + if (!vcpu->arch.vncr_tlb) return -ENOMEM;