github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-24 06:24:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

KVM: arm64: Handle VNCR TLB invalidation race with vcpu_put() VNCR unmapping

Browse Source 
While VNCR TLB invalidation always occurs under the MMU lock,
vcpu_put() doesn't, while it unmaps the VNCR page.

The problem is that the invalidation evaluates vncr_tlb::cpu to
decide whether an unmapping needs to take place (cpu != -1) before
performing it. On the other hand, this_cpu_reset_vncr_fixmap()
unconditionally unmaps if L1_VNCR_MAPPED is set.

These two obviously can race, with a TOCTOU pattern on the TLBI
path, and a BUG_ON() on the vcpu_put() path. And the two can end-up
calling vncr_fixmap(-1), with extra lethal effects.

Move the reset of vncr_tlb::cpu to -1 to a common function, and make
this update atomic so that only a single thread can reset the field
and perform the corresponding unmap. The vcpu_put() still need to
unconditionally unmap the current VNCR to close another ugly race.

Finally, the assignment of vncr_tlb::cpu is moved to be kept in sync
with the actual mapping, similar to L1_VNCR_MAPPED being set.

Fixes: 7270cc9157 ("KVM: arm64: nv: Handle VNCR_EL2 invalidation from MMU notifiers")
Reported-by: sashiko-bot@kernel.org
Link: https://lore.kernel.org/r/20260801130237.0FD8F1F00ACA@smtp.kernel.org
Signed-off-by: Marc Zyngier <maz@kernel.org>
Cc: stable@vger.kernel.org
Reviewed-by: Yuan Yao <yaoyuan@linux.alibaba.com>
Link: https://patch.msgid.link/20260806091026.620700-6-maz@kernel.org
Signed-off-by: Oliver Upton <oupton@kernel.org>
This commit is contained in:
Marc Zyngier 2026-08-06 10:10:23 +01:00 committed by Oliver Upton
parent 34af2c3e31
commit 38640bc32b
1 changed files with 32 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
arch/arm64/kvm/nested.c
View File

@ -27,7 +27,7 @@ struct vncr_tlb {
bool hpa_writable;
/* -1 when not mapped on a CPU */
int cpu;
atomic_t cpu;
/*
* true if the TLB is valid. Can only be changed with the
@ -894,16 +894,40 @@ void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu)
}
}
/*
* Unmapping an L1 VNCR can happen concurrently without the mmu lock being
* effective (vcpu_put() vs TLBI handling). The atomic_xchg below ensures
* that only one CPU sets it to -1 while getting a valid CPU number back.
*/
static int unmap_l1_vncr(struct vncr_tlb *vt)
{
int cpu = atomic_xchg_relaxed(&vt->cpu, -1);
if (cpu != -1)
clear_fixmap(vncr_fixmap(cpu));
return cpu;
}
static void this_cpu_reset_vncr_fixmap(struct kvm_vcpu *vcpu)
{
if (!host_data_test_flag(L1_VNCR_MAPPED))
return;
BUG_ON(vcpu->arch.vncr_tlb->cpu != smp_processor_id());
BUG_ON(is_hyp_ctxt(vcpu));
clear_fixmap(vncr_fixmap(vcpu->arch.vncr_tlb->cpu));
vcpu->arch.vncr_tlb->cpu = -1;
/*
* Unconditionally unmap the local VNCR if we have lost the race
* against a concurrent TLBI. Otherwise we could end-up running
* another vcpu with VNCR still mapped if the TLBI thread is
* preempted between the exchange and the clear_fixmap().
*
* Note that we do not care about the TLBI nuking the fixmap behind
* the back of an running vcpu. This will only generate a fault and
* possibly a retranslation.
*/
if (unmap_l1_vncr(vcpu->arch.vncr_tlb) == -1)
clear_fixmap(vncr_fixmap(smp_processor_id()));
host_data_clear_flag(L1_VNCR_MAPPED);
}
@ -995,8 +1019,7 @@ u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime)
static void invalidate_vncr(struct vncr_tlb *vt)
{
vt->valid = false;
if (vt->cpu != -1)
clear_fixmap(vncr_fixmap(vt->cpu));
unmap_l1_vncr(vt);
}
static bool vncr_tlb_intersects(struct vncr_tlb *vt, u64 addr,
@ -1452,7 +1475,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem)
vt->hpa = pfn << PAGE_SHIFT;
vt->hpa_writable = writable;
vt->valid = true;
vt->cpu = -1;
atomic_set(&vt->cpu, -1);
kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu);
kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable);
@ -1583,8 +1606,6 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu)
if (vt->wr.nG && get_asid_by_regime(vcpu, TR_EL20) != vt->wr.asid)
return;
vt->cpu = smp_processor_id();
if (vt->hpa_writable && vt->wr.pw && vt->wr.pr)
prot = PAGE_KERNEL;
else if (vt->wr.pr)
@ -1599,7 +1620,8 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu)
* FIXME: WO doesn't work at all, need POE support in the kernel.
*/
if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) {
__set_fixmap(vncr_fixmap(vt->cpu), vt->hpa, prot);
atomic_set(&vt->cpu, smp_processor_id());
__set_fixmap(vncr_fixmap(atomic_read(&vt->cpu)), vt->hpa, prot);
host_data_set_flag(L1_VNCR_MAPPED);
}
}