Merge branch kvm-arm64/no-lazy-vgic-init into kvmarm-master/next

* kvm-arm64/no-lazy-vgic-init:
  : .
  : Fix an ugly situation where the vgic lazy init could happen in
  : non-preemtible contexts such as vcpu reset, resulting in lockdep
  : splats.
  :
  : This requires revamping the way in-kernel emulation of devices
  : (timers, PMU) are presenting their interrupt to the vgic, and
  : make sure there is no need to init the vgic on the back of that.
  : .
  KVM: arm64: vgic-v2: Don't init the vgic on in-kernel interrupt injection
  KVM: arm64: vgic-v2: Force vgic init on injection outside the run loop
  KVM: arm64: pmu: Kill the PMU interrupt level cache
  KVM: arm64: timer: Kill the per-timer irq level cache
  KVM: arm64: Simplify userspace notification of interrupt state
  KVM: arm64: timer: Repaint kvm_timer_{should,irq_can}_fire() to kvm_timer_{pending,enabled}()

Signed-off-by: Marc Zyngier <maz@kernel.org>
This commit is contained in:
Marc Zyngier 2026-06-12 09:03:24 +01:00
commit a1b504eda4
7 changed files with 94 additions and 106 deletions

View File

@ -39,10 +39,9 @@ static const u8 default_ppi[] = {
[TIMER_HVTIMER] = 28,
};
static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx);
static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
struct arch_timer_context *timer_ctx);
static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx);
static bool kvm_timer_pending(struct arch_timer_context *timer_ctx);
static void kvm_arm_timer_write(struct kvm_vcpu *vcpu,
struct arch_timer_context *timer,
enum kvm_arch_timer_regs treg,
@ -224,7 +223,7 @@ static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
else
ctx = map.direct_ptimer;
if (kvm_timer_should_fire(ctx))
if (kvm_timer_pending(ctx))
kvm_timer_update_irq(vcpu, true, ctx);
if (userspace_irqchip(vcpu->kvm) &&
@ -257,7 +256,7 @@ static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx)
return kvm_counter_compute_delta(timer_ctx, timer_get_cval(timer_ctx));
}
static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx)
static bool kvm_timer_enabled(struct arch_timer_context *timer_ctx)
{
WARN_ON(timer_ctx && timer_ctx->loaded);
return timer_ctx &&
@ -294,7 +293,7 @@ static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu)
struct arch_timer_context *ctx = &vcpu->arch.timer_cpu.timers[i];
WARN(ctx->loaded, "timer %d loaded\n", i);
if (kvm_timer_irq_can_fire(ctx))
if (kvm_timer_enabled(ctx))
min_delta = min(min_delta, kvm_timer_compute_delta(ctx));
}
@ -358,7 +357,7 @@ static enum hrtimer_restart kvm_hrtimer_expire(struct hrtimer *hrt)
return HRTIMER_NORESTART;
}
static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx)
static bool kvm_timer_pending(struct arch_timer_context *timer_ctx)
{
enum kvm_arch_timers index;
u64 cval, now;
@ -391,7 +390,7 @@ static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx)
!(cnt_ctl & ARCH_TIMER_CTRL_IT_MASK);
}
if (!kvm_timer_irq_can_fire(timer_ctx))
if (!kvm_timer_enabled(timer_ctx))
return false;
cval = timer_get_cval(timer_ctx);
@ -405,22 +404,30 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
return vcpu_has_wfit_active(vcpu) && wfit_delay_ns(vcpu) == 0;
}
static u64 kvm_timer_needs_notify(struct kvm_vcpu *vcpu)
{
u64 v = vcpu->run->s.regs.device_irq_level;
v ^= kvm_timer_pending(vcpu_vtimer(vcpu)) ? KVM_ARM_DEV_EL1_VTIMER : 0;
v ^= kvm_timer_pending(vcpu_ptimer(vcpu)) ? KVM_ARM_DEV_EL1_PTIMER : 0;
return v & (KVM_ARM_DEV_EL1_VTIMER | KVM_ARM_DEV_EL1_PTIMER);
}
bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
{
return !!kvm_timer_needs_notify(vcpu);
}
/*
* Reflect the timer output level into the kvm_run structure
*/
void kvm_timer_update_run(struct kvm_vcpu *vcpu)
bool kvm_timer_update_run(struct kvm_vcpu *vcpu)
{
struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
struct kvm_sync_regs *regs = &vcpu->run->s.regs;
/* Populate the device bitmap with the timer states */
regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER |
KVM_ARM_DEV_EL1_PTIMER);
if (kvm_timer_should_fire(vtimer))
regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER;
if (kvm_timer_should_fire(ptimer))
regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER;
u64 mask = kvm_timer_needs_notify(vcpu);
if (mask)
vcpu->run->s.regs.device_irq_level ^= mask;
return !!mask;
}
static void kvm_timer_update_status(struct arch_timer_context *ctx, bool level)
@ -446,9 +453,8 @@ static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
{
kvm_timer_update_status(timer_ctx, new_level);
timer_ctx->irq.level = new_level;
trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_irq(timer_ctx),
timer_ctx->irq.level);
new_level);
if (userspace_irqchip(vcpu->kvm))
return;
@ -466,28 +472,25 @@ static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
kvm_vgic_inject_irq(vcpu->kvm, vcpu,
timer_irq(timer_ctx),
timer_ctx->irq.level,
new_level,
timer_ctx);
}
/* Only called for a fully emulated timer */
static void timer_emulate(struct arch_timer_context *ctx)
{
bool should_fire = kvm_timer_should_fire(ctx);
bool pending = kvm_timer_pending(ctx);
trace_kvm_timer_emulate(ctx, should_fire);
trace_kvm_timer_emulate(ctx, pending);
if (should_fire != ctx->irq.level)
kvm_timer_update_irq(timer_context_to_vcpu(ctx), should_fire, ctx);
kvm_timer_update_status(ctx, should_fire);
kvm_timer_update_irq(timer_context_to_vcpu(ctx), pending, ctx);
/*
* If the timer can fire now, we don't need to have a soft timer
* scheduled for the future. If the timer cannot fire at all,
* then we also don't need a soft timer.
* If the timer is pending, we don't need to have a soft timer
* scheduled for the future. If the timer is disabled, then
* we don't need a soft timer either.
*/
if (should_fire || !kvm_timer_irq_can_fire(ctx))
if (pending || !kvm_timer_enabled(ctx))
return;
soft_timer_start(&ctx->hrtimer, kvm_timer_compute_delta(ctx));
@ -594,10 +597,10 @@ static void kvm_timer_blocking(struct kvm_vcpu *vcpu)
* If no timers are capable of raising interrupts (disabled or
* masked), then there's no more work for us to do.
*/
if (!kvm_timer_irq_can_fire(map.direct_vtimer) &&
!kvm_timer_irq_can_fire(map.direct_ptimer) &&
!kvm_timer_irq_can_fire(map.emul_vtimer) &&
!kvm_timer_irq_can_fire(map.emul_ptimer) &&
if (!kvm_timer_enabled(map.direct_vtimer) &&
!kvm_timer_enabled(map.direct_ptimer) &&
!kvm_timer_enabled(map.emul_vtimer) &&
!kvm_timer_enabled(map.emul_ptimer) &&
!vcpu_has_wfit_active(vcpu))
return;
@ -677,6 +680,7 @@ static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, boo
static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx)
{
struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctx);
bool pending = kvm_timer_pending(ctx);
bool phys_active = false;
/*
@ -685,12 +689,12 @@ static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx)
* this point and the register restoration, we'll take the
* interrupt anyway.
*/
kvm_timer_update_irq(vcpu, kvm_timer_should_fire(ctx), ctx);
kvm_timer_update_irq(vcpu, pending, ctx);
if (irqchip_in_kernel(vcpu->kvm))
phys_active = kvm_vgic_map_is_active(vcpu, timer_irq(ctx));
phys_active |= ctx->irq.level;
phys_active |= pending;
phys_active |= vgic_is_v5(vcpu->kvm);
set_timer_irq_phys_active(ctx, phys_active);
@ -699,6 +703,7 @@ static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx)
static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu)
{
struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
bool pending = kvm_timer_pending(vtimer);
/*
* Update the timer output so that it is likely to match the
@ -706,7 +711,7 @@ static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu)
* this point and the register restoration, we'll take the
* interrupt anyway.
*/
kvm_timer_update_irq(vcpu, kvm_timer_should_fire(vtimer), vtimer);
kvm_timer_update_irq(vcpu, pending, vtimer);
/*
* When using a userspace irqchip with the architected timers and a
@ -718,7 +723,7 @@ static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu)
* being de-asserted, we unmask the interrupt again so that we exit
* from the guest when the timer fires.
*/
if (vtimer->irq.level)
if (pending)
disable_percpu_irq(host_vtimer_irq);
else
enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
@ -904,23 +909,6 @@ void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu)
timer_set_traps(vcpu, &map);
}
bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
{
struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
bool vlevel, plevel;
if (likely(irqchip_in_kernel(vcpu->kvm)))
return false;
vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER;
plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER;
return kvm_timer_should_fire(vtimer) != vlevel ||
kvm_timer_should_fire(ptimer) != plevel;
}
void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
{
struct arch_timer_cpu *timer = vcpu_timer(vcpu);
@ -1006,7 +994,7 @@ static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu)
{
struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
if (!kvm_timer_should_fire(vtimer)) {
if (!kvm_timer_pending(vtimer)) {
kvm_timer_update_irq(vcpu, false, vtimer);
if (static_branch_likely(&has_gic_active_state))
set_timer_irq_phys_active(vtimer, false);
@ -1579,7 +1567,7 @@ static bool kvm_arch_timer_get_input_level(int vintid)
ctx = vcpu_get_timer(vcpu, i);
if (timer_irq(ctx) == vintid)
return kvm_timer_should_fire(ctx);
return kvm_timer_pending(ctx);
}
/* A timer IRQ has fired, but no matching timer was found? */

View File

@ -52,6 +52,7 @@
#include <linux/irqchip/arm-gic-v5.h>
#include "vgic/vgic.h"
#include "sys_regs.h"
static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT;
@ -1164,6 +1165,15 @@ static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu)
return !kvm_supports_32bit_el0();
}
static bool kvm_irq_update_run(struct kvm_vcpu *vcpu)
{
bool r;
r = kvm_timer_update_run(vcpu);
r |= kvm_pmu_update_run(vcpu);
return r;
}
/**
* kvm_vcpu_exit_request - returns true if the VCPU should *not* enter the guest
* @vcpu: The VCPU pointer
@ -1185,13 +1195,11 @@ static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret)
/*
* If we're using a userspace irqchip, then check if we need
* to tell a userspace irqchip about timer or PMU level
* changes and if so, exit to userspace (the actual level
* state gets updated in kvm_timer_update_run and
* kvm_pmu_update_run below).
* changes and if so, exit to userspace while updating the run
* state.
*/
if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
if (kvm_timer_should_notify_user(vcpu) ||
kvm_pmu_should_notify_user(vcpu)) {
if (unlikely(kvm_irq_update_run(vcpu))) {
*ret = -EINTR;
run->exit_reason = KVM_EXIT_INTR;
return true;
@ -1406,11 +1414,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
ret = handle_exit(vcpu, ret);
}
/* Tell userspace about in-kernel device output levels */
if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
kvm_timer_update_run(vcpu);
kvm_pmu_update_run(vcpu);
}
if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
kvm_irq_update_run(vcpu);
kvm_sigset_deactivate(vcpu);
@ -1494,8 +1499,13 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
return vcpu_interrupt_line(vcpu, irq_num, level);
case KVM_ARM_IRQ_TYPE_PPI:
if (!irqchip_in_kernel(kvm))
if (irqchip_in_kernel(kvm)) {
int ret = vgic_lazy_init(kvm);
if (ret)
return ret;
} else {
return -ENXIO;
}
vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
if (!vcpu)
@ -1522,8 +1532,13 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
return kvm_vgic_inject_irq(kvm, vcpu, irq_num, level, NULL);
case KVM_ARM_IRQ_TYPE_SPI:
if (!irqchip_in_kernel(kvm))
if (irqchip_in_kernel(kvm)) {
int ret = vgic_lazy_init(kvm);
if (ret)
return ret;
} else {
return -ENXIO;
}
if (vgic_is_v5(kvm)) {
/* Build a GICv5-style IntID here */

View File

@ -396,44 +396,31 @@ static bool kvm_pmu_overflow_status(struct kvm_vcpu *vcpu)
static void kvm_pmu_update_state(struct kvm_vcpu *vcpu)
{
struct kvm_pmu *pmu = &vcpu->arch.pmu;
bool overflow;
overflow = kvm_pmu_overflow_status(vcpu);
if (pmu->irq_level == overflow)
if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
return;
pmu->irq_level = overflow;
if (likely(irqchip_in_kernel(vcpu->kvm))) {
int ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu,
pmu->irq_num, overflow, pmu);
WARN_ON(ret);
}
WARN_ON(kvm_vgic_inject_irq(vcpu->kvm, vcpu, pmu->irq_num,
kvm_pmu_overflow_status(vcpu), pmu));
}
bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu)
{
struct kvm_pmu *pmu = &vcpu->arch.pmu;
struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
bool run_level = sregs->device_irq_level & KVM_ARM_DEV_PMU;
if (likely(irqchip_in_kernel(vcpu->kvm)))
return false;
return pmu->irq_level != run_level;
return kvm_pmu_overflow_status(vcpu) != run_level;
}
/*
* Reflect the PMU overflow interrupt output level into the kvm_run structure
*/
void kvm_pmu_update_run(struct kvm_vcpu *vcpu)
bool kvm_pmu_update_run(struct kvm_vcpu *vcpu)
{
struct kvm_sync_regs *regs = &vcpu->run->s.regs;
/* Populate the timer bitmap for user space */
regs->device_irq_level &= ~KVM_ARM_DEV_PMU;
if (vcpu->arch.pmu.irq_level)
regs->device_irq_level |= KVM_ARM_DEV_PMU;
bool update = kvm_pmu_should_notify_user(vcpu);
if (update)
vcpu->run->s.regs.device_irq_level ^= KVM_ARM_DEV_PMU;
return update;
}
/**

View File

@ -20,9 +20,15 @@ static int vgic_irqfd_set_irq(struct kvm_kernel_irq_routing_entry *e,
int level, bool line_status)
{
unsigned int spi_id = e->irqchip.pin + VGIC_NR_PRIVATE_IRQS;
int ret;
if (!vgic_valid_spi(kvm, spi_id))
return -EINVAL;
ret = vgic_lazy_init(kvm);
if (ret)
return ret;
return kvm_vgic_inject_irq(kvm, NULL, spi_id, level, NULL);
}

View File

@ -534,11 +534,9 @@ int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
{
struct vgic_irq *irq;
unsigned long flags;
int ret;
ret = vgic_lazy_init(kvm);
if (ret)
return ret;
if (unlikely(!vgic_initialized(kvm)))
return 0;
if (!vcpu && irq_is_private(kvm, intid))
return -EINVAL;

View File

@ -66,11 +66,6 @@ struct arch_timer_context {
*/
bool loaded;
/* Output level of the timer IRQ */
struct {
bool level;
} irq;
/* Who am I? */
enum kvm_arch_timers timer_id;
@ -104,7 +99,7 @@ void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu);
void kvm_timer_sync_nested(struct kvm_vcpu *vcpu);
void kvm_timer_sync_user(struct kvm_vcpu *vcpu);
bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu);
void kvm_timer_update_run(struct kvm_vcpu *vcpu);
bool kvm_timer_update_run(struct kvm_vcpu *vcpu);
void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu);
void kvm_timer_init_vm(struct kvm *kvm);

View File

@ -32,7 +32,6 @@ struct kvm_pmu {
struct kvm_pmc pmc[KVM_ARMV8_PMU_MAX_COUNTERS];
int irq_num;
bool created;
bool irq_level;
};
struct arm_pmu_entry {
@ -54,7 +53,7 @@ void kvm_pmu_reprogram_counter_mask(struct kvm_vcpu *vcpu, u64 val);
void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu);
void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu);
bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu);
void kvm_pmu_update_run(struct kvm_vcpu *vcpu);
bool kvm_pmu_update_run(struct kvm_vcpu *vcpu);
void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val);
void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val);
void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,
@ -131,7 +130,7 @@ static inline bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu)
{
return false;
}
static inline void kvm_pmu_update_run(struct kvm_vcpu *vcpu) {}
static inline bool kvm_pmu_update_run(struct kvm_vcpu *vcpu) { return false; }
static inline void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val) {}
static inline void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val) {}
static inline void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu,