diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index ac16f96c878d..108966a9db12 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -411,6 +411,9 @@ struct kvm_arch { /* Masks for VNCR-backed and general EL2 sysregs */ struct kvm_sysreg_masks *sysreg_masks; + /* Count the number of VNCR_EL2 TLBs */ + atomic_t vncr_tlb_count; + /* * For an untrusted host VM, 'pkvm.handle' is used to lookup * the associated pKVM instance in the hypervisor. diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index c09b1d411c58..fa776b18a32c 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -424,10 +424,15 @@ static bool kvm_hyp_handle_tlbi_el2(struct kvm_vcpu *vcpu, u64 *exit_code) return false; /* - * If we have to check for any VNCR mapping being invalidated, - * go back to the slow path for further processing. + * If we have to check for any VNCR TLB being invalidated, go back + * to the slow path for further processing. + * + * The synchronisation betweem TLBI and walk is provided by the + * speculative increment of the TLB counter on walk, and the + * invalidation counter. Yes, this is fiddly. */ - if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu)) + if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu) && + atomic_read(&vcpu->kvm->arch.vncr_tlb_count)) return false; __kvm_skip_instr(vcpu); diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 550c9bd3dbe7..20af94197a8a 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -48,6 +48,7 @@ void kvm_init_nested(struct kvm *kvm) { kvm->arch.nested_mmus = NULL; kvm->arch.nested_mmus_size = 0; + atomic_set(&kvm->arch.vncr_tlb_count, 0); } static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) @@ -1016,10 +1017,12 @@ u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime) return asid; } -static void invalidate_vncr(struct vncr_tlb *vt) +static void invalidate_vncr(struct kvm *kvm, struct vncr_tlb *vt) { + BUG_ON(!vt->valid); vt->valid = false; unmap_l1_vncr(vt); + atomic_dec(&kvm->arch.vncr_tlb_count); } static bool vncr_tlb_intersects(struct vncr_tlb *vt, u64 addr, @@ -1066,7 +1069,7 @@ static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) */ kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) if (vncr_tlb_intersects(vt, vt->wr.pa, start, end - start)) - invalidate_vncr(vt); + invalidate_vncr(kvm, vt); } struct s1e2_tlbi_scope { @@ -1123,7 +1126,7 @@ static void invalidate_vncr_va(struct kvm *kvm, break; } - invalidate_vncr(vt); + invalidate_vncr(kvm, vt); } } @@ -1359,13 +1362,20 @@ void kvm_arch_flush_shadow_all(struct kvm *kvm) * intersects with the TLBI request, invalidate it, and unmap the page * from the fixmap. Because we need to look at all the vcpu-private TLBs, * this requires some wide-ranging locking to ensure that nothing races - * against it. This may require some refcounting to avoid the search when - * no such TLB is present. + * against it. This requires some refcounting to avoid the search when + * no such TLB is present (see below). * * - On MMU notifiers, we must invalidate our TLB in a similar way, but * looking at the IPA instead. The funny part is that there may not be a * stage-2 mapping for this page if L1 hasn't accessed it using LD/ST * instructions. + * + * - vncr_tlb_count tracks the number of valid VNCR TLBs VM-wide. This isn't + * the number of *mapped* L1 VNCR pages, which is likely be a subset (and + * by definition, a TLBI handled from L1 runs with the canonical VNCR + * page, not the L1's). The innermost trap handling code checks this to + * find out whether to return to the guest ASAP (no L1 TLBs) or to visit + * this part of the world for some extra invalidation work. */ int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu) @@ -1420,7 +1430,8 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) */ scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { this_cpu_reset_vncr_fixmap(vcpu); - vt->valid = false; + if (vt->valid) + invalidate_vncr(vcpu->kvm, vt); vt->wi = (struct s1_walk_info) { .regime = TR_EL20, @@ -1545,7 +1556,20 @@ int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu) return -EIO; } + /* + * Speculatively increment the TLB count to make sure concurrent + * TLBIs will take the slow path, and will interact with the retry + * mechanism. Drop it again on error. + */ + atomic_inc(&vcpu->kvm->arch.vncr_tlb_count); + smp_mb__after_atomic(); + ret = kvm_translate_vncr(vcpu, &is_gmem); + if (ret) { + smp_mb__before_atomic(); + atomic_dec(&vcpu->kvm->arch.vncr_tlb_count); + } + switch (ret) { case -EAGAIN: /* Let's try again... */