diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index cbea4d9ee955..c3b8257888e8 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -39,10 +39,9 @@ static const u8 default_ppi[] = { [TIMER_HVTIMER] = 28, }; -static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx); static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level, struct arch_timer_context *timer_ctx); -static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx); +static bool kvm_timer_pending(struct arch_timer_context *timer_ctx); static void kvm_arm_timer_write(struct kvm_vcpu *vcpu, struct arch_timer_context *timer, enum kvm_arch_timer_regs treg, @@ -224,7 +223,7 @@ static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id) else ctx = map.direct_ptimer; - if (kvm_timer_should_fire(ctx)) + if (kvm_timer_pending(ctx)) kvm_timer_update_irq(vcpu, true, ctx); if (userspace_irqchip(vcpu->kvm) && @@ -257,7 +256,7 @@ static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx) return kvm_counter_compute_delta(timer_ctx, timer_get_cval(timer_ctx)); } -static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx) +static bool kvm_timer_enabled(struct arch_timer_context *timer_ctx) { WARN_ON(timer_ctx && timer_ctx->loaded); return timer_ctx && @@ -294,7 +293,7 @@ static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu) struct arch_timer_context *ctx = &vcpu->arch.timer_cpu.timers[i]; WARN(ctx->loaded, "timer %d loaded\n", i); - if (kvm_timer_irq_can_fire(ctx)) + if (kvm_timer_enabled(ctx)) min_delta = min(min_delta, kvm_timer_compute_delta(ctx)); } @@ -358,7 +357,7 @@ static enum hrtimer_restart kvm_hrtimer_expire(struct hrtimer *hrt) return HRTIMER_NORESTART; } -static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx) +static bool kvm_timer_pending(struct arch_timer_context *timer_ctx) { enum kvm_arch_timers index; u64 cval, now; @@ -391,7 +390,7 @@ static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx) !(cnt_ctl & ARCH_TIMER_CTRL_IT_MASK); } - if (!kvm_timer_irq_can_fire(timer_ctx)) + if (!kvm_timer_enabled(timer_ctx)) return false; cval = timer_get_cval(timer_ctx); @@ -405,22 +404,30 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) return vcpu_has_wfit_active(vcpu) && wfit_delay_ns(vcpu) == 0; } +static u64 kvm_timer_needs_notify(struct kvm_vcpu *vcpu) +{ + u64 v = vcpu->run->s.regs.device_irq_level; + + v ^= kvm_timer_pending(vcpu_vtimer(vcpu)) ? KVM_ARM_DEV_EL1_VTIMER : 0; + v ^= kvm_timer_pending(vcpu_ptimer(vcpu)) ? KVM_ARM_DEV_EL1_PTIMER : 0; + + return v & (KVM_ARM_DEV_EL1_VTIMER | KVM_ARM_DEV_EL1_PTIMER); +} + +bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu) +{ + return !!kvm_timer_needs_notify(vcpu); +} + /* * Reflect the timer output level into the kvm_run structure */ -void kvm_timer_update_run(struct kvm_vcpu *vcpu) +bool kvm_timer_update_run(struct kvm_vcpu *vcpu) { - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); - struct kvm_sync_regs *regs = &vcpu->run->s.regs; - - /* Populate the device bitmap with the timer states */ - regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER | - KVM_ARM_DEV_EL1_PTIMER); - if (kvm_timer_should_fire(vtimer)) - regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER; - if (kvm_timer_should_fire(ptimer)) - regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER; + u64 mask = kvm_timer_needs_notify(vcpu); + if (mask) + vcpu->run->s.regs.device_irq_level ^= mask; + return !!mask; } static void kvm_timer_update_status(struct arch_timer_context *ctx, bool level) @@ -446,9 +453,8 @@ static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level, { kvm_timer_update_status(timer_ctx, new_level); - timer_ctx->irq.level = new_level; trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_irq(timer_ctx), - timer_ctx->irq.level); + new_level); if (userspace_irqchip(vcpu->kvm)) return; @@ -466,28 +472,25 @@ static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level, kvm_vgic_inject_irq(vcpu->kvm, vcpu, timer_irq(timer_ctx), - timer_ctx->irq.level, + new_level, timer_ctx); } /* Only called for a fully emulated timer */ static void timer_emulate(struct arch_timer_context *ctx) { - bool should_fire = kvm_timer_should_fire(ctx); + bool pending = kvm_timer_pending(ctx); - trace_kvm_timer_emulate(ctx, should_fire); + trace_kvm_timer_emulate(ctx, pending); - if (should_fire != ctx->irq.level) - kvm_timer_update_irq(timer_context_to_vcpu(ctx), should_fire, ctx); - - kvm_timer_update_status(ctx, should_fire); + kvm_timer_update_irq(timer_context_to_vcpu(ctx), pending, ctx); /* - * If the timer can fire now, we don't need to have a soft timer - * scheduled for the future. If the timer cannot fire at all, - * then we also don't need a soft timer. + * If the timer is pending, we don't need to have a soft timer + * scheduled for the future. If the timer is disabled, then + * we don't need a soft timer either. */ - if (should_fire || !kvm_timer_irq_can_fire(ctx)) + if (pending || !kvm_timer_enabled(ctx)) return; soft_timer_start(&ctx->hrtimer, kvm_timer_compute_delta(ctx)); @@ -594,10 +597,10 @@ static void kvm_timer_blocking(struct kvm_vcpu *vcpu) * If no timers are capable of raising interrupts (disabled or * masked), then there's no more work for us to do. */ - if (!kvm_timer_irq_can_fire(map.direct_vtimer) && - !kvm_timer_irq_can_fire(map.direct_ptimer) && - !kvm_timer_irq_can_fire(map.emul_vtimer) && - !kvm_timer_irq_can_fire(map.emul_ptimer) && + if (!kvm_timer_enabled(map.direct_vtimer) && + !kvm_timer_enabled(map.direct_ptimer) && + !kvm_timer_enabled(map.emul_vtimer) && + !kvm_timer_enabled(map.emul_ptimer) && !vcpu_has_wfit_active(vcpu)) return; @@ -677,6 +680,7 @@ static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, boo static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx) { struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctx); + bool pending = kvm_timer_pending(ctx); bool phys_active = false; /* @@ -685,12 +689,12 @@ static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx) * this point and the register restoration, we'll take the * interrupt anyway. */ - kvm_timer_update_irq(vcpu, kvm_timer_should_fire(ctx), ctx); + kvm_timer_update_irq(vcpu, pending, ctx); if (irqchip_in_kernel(vcpu->kvm)) phys_active = kvm_vgic_map_is_active(vcpu, timer_irq(ctx)); - phys_active |= ctx->irq.level; + phys_active |= pending; phys_active |= vgic_is_v5(vcpu->kvm); set_timer_irq_phys_active(ctx, phys_active); @@ -699,6 +703,7 @@ static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx) static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu) { struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); + bool pending = kvm_timer_pending(vtimer); /* * Update the timer output so that it is likely to match the @@ -706,7 +711,7 @@ static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu) * this point and the register restoration, we'll take the * interrupt anyway. */ - kvm_timer_update_irq(vcpu, kvm_timer_should_fire(vtimer), vtimer); + kvm_timer_update_irq(vcpu, pending, vtimer); /* * When using a userspace irqchip with the architected timers and a @@ -718,7 +723,7 @@ static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu) * being de-asserted, we unmask the interrupt again so that we exit * from the guest when the timer fires. */ - if (vtimer->irq.level) + if (pending) disable_percpu_irq(host_vtimer_irq); else enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags); @@ -904,23 +909,6 @@ void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu) timer_set_traps(vcpu, &map); } -bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu) -{ - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); - struct kvm_sync_regs *sregs = &vcpu->run->s.regs; - bool vlevel, plevel; - - if (likely(irqchip_in_kernel(vcpu->kvm))) - return false; - - vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER; - plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER; - - return kvm_timer_should_fire(vtimer) != vlevel || - kvm_timer_should_fire(ptimer) != plevel; -} - void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) { struct arch_timer_cpu *timer = vcpu_timer(vcpu); @@ -1006,7 +994,7 @@ static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu) { struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - if (!kvm_timer_should_fire(vtimer)) { + if (!kvm_timer_pending(vtimer)) { kvm_timer_update_irq(vcpu, false, vtimer); if (static_branch_likely(&has_gic_active_state)) set_timer_irq_phys_active(vtimer, false); @@ -1579,7 +1567,7 @@ static bool kvm_arch_timer_get_input_level(int vintid) ctx = vcpu_get_timer(vcpu, i); if (timer_irq(ctx) == vintid) - return kvm_timer_should_fire(ctx); + return kvm_timer_pending(ctx); } /* A timer IRQ has fired, but no matching timer was found? */ diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 34c9950884d5..d4efda7b75a3 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -52,6 +52,7 @@ #include +#include "vgic/vgic.h" #include "sys_regs.h" static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT; @@ -1164,6 +1165,15 @@ static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu) return !kvm_supports_32bit_el0(); } +static bool kvm_irq_update_run(struct kvm_vcpu *vcpu) +{ + bool r; + + r = kvm_timer_update_run(vcpu); + r |= kvm_pmu_update_run(vcpu); + return r; +} + /** * kvm_vcpu_exit_request - returns true if the VCPU should *not* enter the guest * @vcpu: The VCPU pointer @@ -1185,13 +1195,11 @@ static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret) /* * If we're using a userspace irqchip, then check if we need * to tell a userspace irqchip about timer or PMU level - * changes and if so, exit to userspace (the actual level - * state gets updated in kvm_timer_update_run and - * kvm_pmu_update_run below). + * changes and if so, exit to userspace while updating the run + * state. */ if (unlikely(!irqchip_in_kernel(vcpu->kvm))) { - if (kvm_timer_should_notify_user(vcpu) || - kvm_pmu_should_notify_user(vcpu)) { + if (unlikely(kvm_irq_update_run(vcpu))) { *ret = -EINTR; run->exit_reason = KVM_EXIT_INTR; return true; @@ -1406,11 +1414,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) ret = handle_exit(vcpu, ret); } - /* Tell userspace about in-kernel device output levels */ - if (unlikely(!irqchip_in_kernel(vcpu->kvm))) { - kvm_timer_update_run(vcpu); - kvm_pmu_update_run(vcpu); - } + if (unlikely(!irqchip_in_kernel(vcpu->kvm))) + kvm_irq_update_run(vcpu); kvm_sigset_deactivate(vcpu); @@ -1494,8 +1499,13 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, return vcpu_interrupt_line(vcpu, irq_num, level); case KVM_ARM_IRQ_TYPE_PPI: - if (!irqchip_in_kernel(kvm)) + if (irqchip_in_kernel(kvm)) { + int ret = vgic_lazy_init(kvm); + if (ret) + return ret; + } else { return -ENXIO; + } vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); if (!vcpu) @@ -1522,8 +1532,13 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, return kvm_vgic_inject_irq(kvm, vcpu, irq_num, level, NULL); case KVM_ARM_IRQ_TYPE_SPI: - if (!irqchip_in_kernel(kvm)) + if (irqchip_in_kernel(kvm)) { + int ret = vgic_lazy_init(kvm); + if (ret) + return ret; + } else { return -ENXIO; + } if (vgic_is_v5(kvm)) { /* Build a GICv5-style IntID here */ diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index e1860acae641..edb21239478a 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -396,44 +396,31 @@ static bool kvm_pmu_overflow_status(struct kvm_vcpu *vcpu) static void kvm_pmu_update_state(struct kvm_vcpu *vcpu) { struct kvm_pmu *pmu = &vcpu->arch.pmu; - bool overflow; - overflow = kvm_pmu_overflow_status(vcpu); - if (pmu->irq_level == overflow) + if (unlikely(!irqchip_in_kernel(vcpu->kvm))) return; - pmu->irq_level = overflow; - - if (likely(irqchip_in_kernel(vcpu->kvm))) { - int ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu, - pmu->irq_num, overflow, pmu); - WARN_ON(ret); - } + WARN_ON(kvm_vgic_inject_irq(vcpu->kvm, vcpu, pmu->irq_num, + kvm_pmu_overflow_status(vcpu), pmu)); } bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu) { - struct kvm_pmu *pmu = &vcpu->arch.pmu; struct kvm_sync_regs *sregs = &vcpu->run->s.regs; bool run_level = sregs->device_irq_level & KVM_ARM_DEV_PMU; - if (likely(irqchip_in_kernel(vcpu->kvm))) - return false; - - return pmu->irq_level != run_level; + return kvm_pmu_overflow_status(vcpu) != run_level; } /* * Reflect the PMU overflow interrupt output level into the kvm_run structure */ -void kvm_pmu_update_run(struct kvm_vcpu *vcpu) +bool kvm_pmu_update_run(struct kvm_vcpu *vcpu) { - struct kvm_sync_regs *regs = &vcpu->run->s.regs; - - /* Populate the timer bitmap for user space */ - regs->device_irq_level &= ~KVM_ARM_DEV_PMU; - if (vcpu->arch.pmu.irq_level) - regs->device_irq_level |= KVM_ARM_DEV_PMU; + bool update = kvm_pmu_should_notify_user(vcpu); + if (update) + vcpu->run->s.regs.device_irq_level ^= KVM_ARM_DEV_PMU; + return update; } /** diff --git a/arch/arm64/kvm/vgic/vgic-irqfd.c b/arch/arm64/kvm/vgic/vgic-irqfd.c index b9b86e3a6c86..19a1094536e6 100644 --- a/arch/arm64/kvm/vgic/vgic-irqfd.c +++ b/arch/arm64/kvm/vgic/vgic-irqfd.c @@ -20,9 +20,15 @@ static int vgic_irqfd_set_irq(struct kvm_kernel_irq_routing_entry *e, int level, bool line_status) { unsigned int spi_id = e->irqchip.pin + VGIC_NR_PRIVATE_IRQS; + int ret; if (!vgic_valid_spi(kvm, spi_id)) return -EINVAL; + + ret = vgic_lazy_init(kvm); + if (ret) + return ret; + return kvm_vgic_inject_irq(kvm, NULL, spi_id, level, NULL); } diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 1e9fe8764584..9e29f03d3463 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -534,11 +534,9 @@ int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu, { struct vgic_irq *irq; unsigned long flags; - int ret; - ret = vgic_lazy_init(kvm); - if (ret) - return ret; + if (unlikely(!vgic_initialized(kvm))) + return 0; if (!vcpu && irq_is_private(kvm, intid)) return -EINVAL; diff --git a/include/kvm/arm_arch_timer.h b/include/kvm/arm_arch_timer.h index bf8cc9589bd0..15a4f97f8105 100644 --- a/include/kvm/arm_arch_timer.h +++ b/include/kvm/arm_arch_timer.h @@ -66,11 +66,6 @@ struct arch_timer_context { */ bool loaded; - /* Output level of the timer IRQ */ - struct { - bool level; - } irq; - /* Who am I? */ enum kvm_arch_timers timer_id; @@ -104,7 +99,7 @@ void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu); void kvm_timer_sync_nested(struct kvm_vcpu *vcpu); void kvm_timer_sync_user(struct kvm_vcpu *vcpu); bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu); -void kvm_timer_update_run(struct kvm_vcpu *vcpu); +bool kvm_timer_update_run(struct kvm_vcpu *vcpu); void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu); void kvm_timer_init_vm(struct kvm *kvm); diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h index 0a36a3d5c894..b5e5942204fc 100644 --- a/include/kvm/arm_pmu.h +++ b/include/kvm/arm_pmu.h @@ -32,7 +32,6 @@ struct kvm_pmu { struct kvm_pmc pmc[KVM_ARMV8_PMU_MAX_COUNTERS]; int irq_num; bool created; - bool irq_level; }; struct arm_pmu_entry { @@ -54,7 +53,7 @@ void kvm_pmu_reprogram_counter_mask(struct kvm_vcpu *vcpu, u64 val); void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu); void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu); bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu); -void kvm_pmu_update_run(struct kvm_vcpu *vcpu); +bool kvm_pmu_update_run(struct kvm_vcpu *vcpu); void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val); void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val); void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data, @@ -131,7 +130,7 @@ static inline bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu) { return false; } -static inline void kvm_pmu_update_run(struct kvm_vcpu *vcpu) {} +static inline bool kvm_pmu_update_run(struct kvm_vcpu *vcpu) { return false; } static inline void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val) {} static inline void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val) {} static inline void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu,