github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-06 21:58:01 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch kvm-arm64/cross-vcpu-sync into kvmarm-master/fixes

Browse Source 
* kvm-arm64/cross-vcpu-sync:
  : \
  : Fix a number of issues related to publication of state when a vcpu
  : is forced out of the guest, address other problems related to the
  : nesting and overhead of pausing the VM, and apply all of that to
  : some of the nasty corners of the VGIC handling.
  :
  : This eventually simplifies the handling of operations that are
  : inherently racy by pausing the involved vcpus.
  : /
  KVM: arm64: vgic-its: Remove the pending state of a discarded LPI
  KVM: arm64: vgic-its: Don't map an LPI from more than one event
  KVM: arm64: vgic-its: Stop the VM when handling MOVALL
  KVM: arm64: vgic: Stop the target vcpu when disabling LPIs
  KVM: arm64: Add helpers to halt a vCPU
  KVM: arm64: Turn vcpu->arch.pause into a counter
  KVM: arm64: Move IN_GUEST_MODE ahead of state being loaded
  KVM: arm64: Move OUTSIDE_GUEST_MODE publication past context being saved
  KVM: arm64: nv: Move L2->L1 switch on interrupt early

Signed-off-by: Marc Zyngier <maz@kernel.org>
This commit is contained in:
Marc Zyngier 2026-10-26 07:47:18 +00:00
commit af3a65d20d
7 changed files with 135 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
arch/arm64/include/asm/kvm_host.h
View File

@ -965,11 +965,10 @@ struct kvm_vcpu_arch {
/*
* Don't run the guest (internal implementation need).
*
* Contrary to the flags above, this is set/cleared outside of
* a vcpu context, and thus cannot be mixed with the flags
* themselves (or the flag accesses need to be made atomic).
* Contrary to the flags above, this is updated outside of
* a vcpu context, and thus cannot be mixed with the flags.
*/
bool pause;
atomic_t pause;
/*
* We maintain more than a single set of debug registers to support
@ -1353,6 +1352,8 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
void kvm_arm_halt_guest(struct kvm *kvm);
void kvm_arm_resume_guest(struct kvm *kvm);
void kvm_pause_vcpu(struct kvm_vcpu *vcpu);
void kvm_resume_vcpu(struct kvm_vcpu *vcpu);
#define vcpu_has_run_once(vcpu) (!!READ_ONCE((vcpu)->pid))

70
arch/arm64/kvm/arm.c
View File

@ -590,6 +590,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
kvm_arm_pvtime_vcpu_init(&vcpu->arch);
vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
atomic_set(&vcpu->arch.pause, 0);
/*
* This vCPU may have been created after mpidr_data was initialized.
@ -954,6 +955,11 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
return ret;
}
static bool vcpu_can_run(struct kvm_vcpu *vcpu)
{
return !kvm_arm_vcpu_stopped(vcpu) && !atomic_read(&vcpu->arch.pause);
}
/**
* kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
* @v: The VCPU pointer
@ -969,8 +975,7 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
(kvm_timer_should_notify_user(v) ||
kvm_pmu_should_notify_user(v)));
return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
&& !kvm_arm_vcpu_stopped(v) && !v->arch.pause);
return ((irq_lines || kvm_vgic_vcpu_pending_irq(v)) && vcpu_can_run(v));
}
bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
@ -1127,13 +1132,28 @@ bool kvm_arch_intc_initialized(struct kvm *kvm)
return vgic_initialized(kvm);
}
void kvm_pause_vcpu(struct kvm_vcpu *vcpu)
{
DECLARE_BITMAP(vcpu_bitmap, KVM_MAX_VCPUS) = { };
atomic_inc(&vcpu->arch.pause);
__set_bit(vcpu->vcpu_idx, vcpu_bitmap);
kvm_make_vcpus_request_mask(vcpu->kvm, KVM_REQ_SLEEP, vcpu_bitmap);
}
void kvm_resume_vcpu(struct kvm_vcpu *vcpu)
{
if (atomic_dec_and_test(&vcpu->arch.pause))
__kvm_vcpu_wake_up(vcpu);
}
void kvm_arm_halt_guest(struct kvm *kvm)
{
unsigned long i;
struct kvm_vcpu *vcpu;
kvm_for_each_vcpu(i, vcpu, kvm)
vcpu->arch.pause = true;
atomic_inc(&vcpu->arch.pause);
kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP);
}
@ -1142,21 +1162,17 @@ void kvm_arm_resume_guest(struct kvm *kvm)
unsigned long i;
struct kvm_vcpu *vcpu;
kvm_for_each_vcpu(i, vcpu, kvm) {
vcpu->arch.pause = false;
__kvm_vcpu_wake_up(vcpu);
}
kvm_for_each_vcpu(i, vcpu, kvm)
kvm_resume_vcpu(vcpu);
}
static void kvm_vcpu_sleep(struct kvm_vcpu *vcpu)
{
struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
rcuwait_wait_event(wait,
(!kvm_arm_vcpu_stopped(vcpu)) && (!vcpu->arch.pause),
TASK_INTERRUPTIBLE);
rcuwait_wait_event(wait, vcpu_can_run(vcpu), TASK_INTERRUPTIBLE);
if (kvm_arm_vcpu_stopped(vcpu) || vcpu->arch.pause) {
if (!vcpu_can_run(vcpu)) {
/* Awaken to handle a signal, request we sleep again later. */
kvm_make_request(KVM_REQ_SLEEP, vcpu);
}
@ -1437,38 +1453,40 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
*/
preempt_disable();
kvm_nested_flush_hwstate(vcpu);
if (kvm_vcpu_has_pmu(vcpu))
kvm_pmu_flush_hwstate(vcpu);
kvm_nested_flush_hwstate(vcpu);
local_irq_disable();
kvm_vgic_flush_hwstate(vcpu);
kvm_pmu_update_vcpu_events(vcpu);
/*
* Ensure we set mode to IN_GUEST_MODE after we disable
* interrupts and before the final VCPU requests check.
* See the comment in kvm_vcpu_exiting_guest_mode() and
* Documentation/virt/kvm/vcpu-requests.rst
*
* Requests to self must be posted before this point, or
* won't be taken into account until the next exit. No state
* loading other than what happens at vcpu_load() time can
* be done before we have checked for pending requests.
*/
smp_store_mb(vcpu->mode, IN_GUEST_MODE);
if (ret <= 0 || kvm_vcpu_exit_request(vcpu, &ret)) {
vcpu->mode = OUTSIDE_GUEST_MODE;
isb(); /* Ensure work in x_flush_hwstate is committed */
if (kvm_vcpu_has_pmu(vcpu))
kvm_pmu_sync_hwstate(vcpu);
if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
kvm_timer_sync_user(vcpu);
kvm_vgic_sync_hwstate(vcpu);
smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);
local_irq_enable();
preempt_enable();
continue;
}
/* From here onwards, we are commited to entering the guest */
kvm_vgic_flush_hwstate(vcpu);
kvm_pmu_update_vcpu_events(vcpu);
kvm_arch_vcpu_ctxflush_fp(vcpu);
/**************************************************************
@ -1479,7 +1497,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
ret = kvm_arm_vcpu_enter_exit(vcpu);
vcpu->mode = OUTSIDE_GUEST_MODE;
vcpu->stat.exits++;
/*
* Back from guest
@ -1513,6 +1530,13 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
kvm_arch_vcpu_ctxsync_fp(vcpu);
/*
* The state that was pushed into the CPU on entering
* the guest has been synchronised back, let's
* advertise we're outside of the guest.
*/
smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);
/*
* We must ensure that any pending interrupts are taken before
* we exit guest timing so that timer ticks are accounted as

8
arch/arm64/kvm/nested.c
View File

@ -2181,6 +2181,9 @@ void check_nested_vcpu_requests(struct kvm_vcpu *vcpu)
*
* What follows is some slop to (1) react to exception routing / masking and (2)
* preserve the pending SError state across translation regimes.
*
* Also generate the maintenance interrupt state, and finally check that if
* we need to take an interrupt targetting the guest hypervisor.
*/
void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu)
{
@ -2189,6 +2192,11 @@ void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu)
if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING)))
kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu));
vgic_v3_nested_update_mi(vcpu);
if (vgic_state_is_nested(vcpu) && kvm_vgic_vcpu_pending_irq(vcpu))
kvm_make_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu);
}
void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu)

70
arch/arm64/kvm/vgic/vgic-its.c
View File

@ -776,6 +776,7 @@ static void its_free_ite(struct kvm *kvm, struct its_ite *ite)
its_unmap_vlpi(ite->irq->host_irq);
irq->hw = false;
irq->mapped = false;
}
vgic_put_irq(kvm, ite->irq);
@ -784,6 +785,23 @@ static void its_free_ite(struct kvm *kvm, struct its_ite *ite)
kfree(ite);
}
/* Mapping one LPI from two events is UNPREDICTABLE: the first ITE keeps it. */
static bool vgic_its_map_lpi(struct kvm *kvm, struct its_ite *ite,
struct vgic_irq *irq, struct kvm_vcpu *vcpu)
{
scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
if (!irq->mapped) {
irq->mapped = true;
irq->target_vcpu = vcpu;
ite->irq = irq;
return true;
}
}
vgic_put_irq(kvm, irq);
return false;
}
static u64 its_cmd_mask_field(u64 *its_cmd, int word, int shift, int size)
{
return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1);
@ -815,13 +833,36 @@ static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
struct its_device *device = find_its_device(its, device_id);
int ite_esz = vgic_its_get_abi(its)->ite_esz;
gpa_t gpa = device->itt_addr + ite->event_id * ite_esz;
/*
* Though the spec talks about removing the pending state, we
* don't bother here since we clear the ITTE anyway and the
* pending state is a property of the ITTE struct.
*/
struct vgic_irq *irq = ite->irq;
struct kvm_vcpu *vcpu, *target;
vgic_its_invalidate_cache(its);
/*
* Only an LPI on a vCPU's AP list can be in its list registers,
* which the exit path folds back, and that exit moves the LPI
* to its target vCPU if the target has changed: pause both.
*/
scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
vcpu = irq->vcpu;
target = irq->target_vcpu;
}
if (vcpu) {
if (target != vcpu)
kvm_pause_vcpu(target);
kvm_pause_vcpu(vcpu);
}
scoped_guard(raw_spinlock_irqsave, &irq->irq_lock)
irq->pending_latch = false;
if (vcpu) {
kvm_resume_vcpu(vcpu);
if (target != vcpu)
kvm_resume_vcpu(target);
}
its_free_ite(kvm, ite);
return vgic_its_write_entry_lock(its, gpa, 0ULL, ite);
@ -1093,13 +1134,14 @@ static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
vcpu = collection_to_vcpu(kvm, collection);
irq = vgic_add_lpi(kvm, lpi_nr, vcpu);
if (!IS_ERR(irq) && !vgic_its_map_lpi(kvm, ite, irq, vcpu))
irq = ERR_PTR(-EEXIST);
if (IS_ERR(irq)) {
if (new_coll)
vgic_its_free_collection(its, coll_id);
its_free_ite(kvm, ite);
return PTR_ERR(irq);
}
ite->irq = irq;
return 0;
}
@ -1382,6 +1424,15 @@ static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
if (vcpu1 == vcpu2)
return 0;
/*
* Bulk operations such as MOVALL are a pain, as they can clash
* badly locking-wise with other vcpus entering and exiting the
* guest, should they be affected by it. Stopping the guest is a
* safer bet to ensure uncontended access and ultimately forward
* progress. Yeah...
*/
kvm_arm_halt_guest(kvm);
xa_for_each(&dist->lpi_xa, intid, irq) {
irq = vgic_get_irq(kvm, intid);
if (!irq)
@ -1394,6 +1445,8 @@ static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
vgic_its_invalidate_cache(its);
kvm_arm_resume_guest(kvm);
return 0;
}
@ -2221,7 +2274,10 @@ static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,
its_free_ite(kvm, ite);
return PTR_ERR(irq);
}
ite->irq = irq;
/* Drop an ITE whose LPI another ITE maps, as MAPTI does. */
if (!vgic_its_map_lpi(kvm, ite, irq, vcpu))
its_free_ite(kvm, ite);
return offset;
}

10
arch/arm64/kvm/vgic/vgic-mmio-v3.c
View File

@ -300,9 +300,19 @@ static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu,
if (ctlr != GICR_CTLR_ENABLE_LPIS)
return;
/*
* Yes, disabling LPIs is painful, since it can be done from
* a *remote* vcpu! So let's not take any chance, and make
* sure that the target vCPU has written their LRs back to the
* irq structures, and release any reference they would have.
*
* If it hurts, don't do it.
*/
kvm_pause_vcpu(vcpu);
vgic_flush_pending_lpis(vcpu);
vgic_its_invalidate_all_caches(vcpu->kvm);
atomic_set_release(&vgic_cpu->ctlr, 0);
kvm_resume_vcpu(vcpu);
} else {
ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr, 0,
GICR_CTLR_ENABLE_LPIS);

8
arch/arm64/kvm/vgic/vgic.c
View File

@ -1167,7 +1167,7 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
*
* We do that by placing a request to ourselves which will
* abort the entry procedure and inject the exception at the
* beginning of the run loop.
* beginning of the run loop (see kvm_nested_flush_hwstate()).
*
* - Otherwise, do exactly *NOTHING* apart from enabling the virtual
* CPU interface. The guest state is already loaded, and we can
@ -1177,16 +1177,10 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
* maintenance interrupt state, as it may fire.
*/
if (vgic_state_is_nested(vcpu)) {
if (kvm_vgic_vcpu_pending_irq(vcpu))
kvm_make_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu);
vgic_v3_flush_nested(vcpu);
return;
}
if (vcpu_has_nv(vcpu))
vgic_v3_nested_update_mi(vcpu);
DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
vgic_flush_state(vcpu);

1
include/kvm/arm_vgic.h
View File

@ -292,6 +292,7 @@ struct vgic_irq {
bool active:1;
bool hw:1; /* Tied to HW IRQ */
bool on_lr:1; /* Present in a CPU LR */
bool mapped:1; /* LPIs: mapped by an ITE */
refcount_t refcount; /* Used for LPIs */
u32 hwintid; /* HW INTID number */
unsigned int host_irq; /* linux irq corresponding to hwintid */