diff --git a/arch/riscv/include/asm/kvm_aia.h b/arch/riscv/include/asm/kvm_aia.h index c67ec5ac0a14..ab03f853e507 100644 --- a/arch/riscv/include/asm/kvm_aia.h +++ b/arch/riscv/include/asm/kvm_aia.h @@ -165,6 +165,9 @@ int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner, void __iomem **hgei_va, phys_addr_t *hgei_pa); void kvm_riscv_aia_free_hgei(int cpu, int hgei); +void kvm_riscv_aia_pm_exit(void); +void kvm_riscv_aia_pm_enter(void); + void kvm_riscv_aia_enable(void); void kvm_riscv_aia_disable(void); int kvm_riscv_aia_init(void); diff --git a/arch/riscv/kvm/aia.c b/arch/riscv/kvm/aia.c index 9a653b4ad40a..b3b466e4db33 100644 --- a/arch/riscv/kvm/aia.c +++ b/arch/riscv/kvm/aia.c @@ -25,6 +25,7 @@ struct aia_hgei_control { unsigned long free_bitmap; struct kvm_vcpu *owners[BITS_PER_LONG]; unsigned int nr_hgei; + unsigned long saved_hgeie; }; static DEFINE_PER_CPU(struct aia_hgei_control, aia_hgei); static int hgei_parent_irq; @@ -553,6 +554,47 @@ static void aia_hgei_exit(void) free_percpu_irq(hgei_parent_irq, &aia_hgei); } +void kvm_riscv_aia_pm_exit(void) +{ + struct aia_hgei_control *hgctrl; + + if (!kvm_riscv_aia_available()) + return; + + hgctrl = this_cpu_ptr(&aia_hgei); + csr_write(CSR_HGEIE, hgctrl->saved_hgeie); + + csr_write(CSR_HVICTL, aia_hvictl_value(false)); + csr_write(CSR_HVIPRIO1, 0x0); + csr_write(CSR_HVIPRIO2, 0x0); +#ifdef CONFIG_32BIT + csr_write(CSR_HVIPH, 0x0); + csr_write(CSR_HIDELEGH, 0x0); + csr_write(CSR_HVIPRIO1H, 0x0); + csr_write(CSR_HVIPRIO2H, 0x0); +#endif + csr_set(CSR_HIE, BIT(IRQ_S_GEXT)); + /* Enable IRQ filtering for overflow interrupt only if sscofpmf is present */ + if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF)) + csr_set(CSR_HVIEN, BIT(IRQ_PMU_OVF)); +} + +void kvm_riscv_aia_pm_enter(void) +{ + struct aia_hgei_control *hgctrl; + + if (!kvm_riscv_aia_available()) + return; + + if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF)) + csr_clear(CSR_HVIEN, BIT(IRQ_PMU_OVF)); + + csr_write(CSR_HVICTL, aia_hvictl_value(false)); + + hgctrl = this_cpu_ptr(&aia_hgei); + hgctrl->saved_hgeie = csr_read(CSR_HGEIE); +} + void kvm_riscv_aia_enable(void) { const struct imsic_global_config *gc; diff --git a/arch/riscv/kvm/main.c b/arch/riscv/kvm/main.c index 350e4f097d6e..20b516e84e60 100644 --- a/arch/riscv/kvm/main.c +++ b/arch/riscv/kvm/main.c @@ -10,11 +10,14 @@ #include #include #include +#include #include #include #include #include +static DEFINE_PER_CPU(bool, kvm_riscv_virtualization_enabled); + DEFINE_STATIC_KEY_FALSE(kvm_riscv_vsstage_tlb_no_gpa); static void kvm_riscv_setup_vendor_features(void) @@ -33,14 +36,9 @@ long kvm_arch_dev_ioctl(struct file *filp, return -EINVAL; } -int kvm_arch_enable_virtualization_cpu(void) +/* Initialize hypervisor CSRs - called during CPU online and non-retention idle resume */ +static void kvm_riscv_csr_init(void) { - int rc; - - rc = kvm_riscv_nacl_enable(); - if (rc) - return rc; - csr_write(CSR_HEDELEG, 0); csr_write(CSR_HIDELEG, 0); @@ -48,16 +46,11 @@ int kvm_arch_enable_virtualization_cpu(void) csr_write(CSR_HCOUNTEREN, 0x02); csr_write(CSR_HVIP, 0); - - kvm_riscv_aia_enable(); - - return 0; } -void kvm_arch_disable_virtualization_cpu(void) +/* Clear hypervisor CSRs - called during CPU offline and non-retention idle entry */ +static void kvm_riscv_csr_cleanup(void) { - kvm_riscv_aia_disable(); - /* * After clearing the hideleg CSR, the host kernel will receive * spurious interrupts if hvip CSR has pending interrupts and the @@ -70,10 +63,69 @@ void kvm_arch_disable_virtualization_cpu(void) csr_write(CSR_HIDELEG, 0); kvm_riscv_clear_former_vcpu(); - - kvm_riscv_nacl_disable(); } +int kvm_arch_enable_virtualization_cpu(void) +{ + int rc; + + rc = kvm_riscv_nacl_enable(); + if (rc) + return rc; + + kvm_riscv_csr_init(); + kvm_riscv_aia_enable(); + + __this_cpu_write(kvm_riscv_virtualization_enabled, true); + + return 0; +} + +void kvm_arch_disable_virtualization_cpu(void) +{ + kvm_riscv_aia_disable(); + kvm_riscv_csr_cleanup(); + kvm_riscv_nacl_disable(); + + __this_cpu_write(kvm_riscv_virtualization_enabled, false); +} + +static int kvm_riscv_cpu_pm_notifier(struct notifier_block *self, unsigned long cmd, void *v) +{ + switch (cmd) { + case CPU_PM_EXIT: + case CPU_PM_ENTER_FAILED: + /* + * Only restore hypervisor state if KVM virtualization is + * enabled on this CPU. This prevents unintentional re-enabling + * of virtualization after it has been explicitly disabled. + */ + if (__this_cpu_read(kvm_riscv_virtualization_enabled)) { + kvm_riscv_csr_init(); + kvm_riscv_aia_pm_exit(); + } + return NOTIFY_OK; + case CPU_PM_ENTER: + /* + * Only save and clear hypervisor state if KVM virtualization + * is enabled on this CPU. + */ + if (__this_cpu_read(kvm_riscv_virtualization_enabled)) { + kvm_riscv_aia_pm_enter(); + kvm_riscv_csr_cleanup(); + } + return NOTIFY_OK; + default: + break; + } + + return NOTIFY_DONE; +} + +static struct notifier_block kvm_riscv_cpu_pm_nb = { + .notifier_call = kvm_riscv_cpu_pm_notifier, +}; + static void kvm_riscv_teardown(void) { kvm_riscv_aia_exit(); @@ -174,17 +226,31 @@ static int __init riscv_kvm_init(void) kvm_register_perf_callbacks(); - rc = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); - if (rc) { - kvm_riscv_teardown(); - return rc; + /* Register CPU PM notifier for CPU idle non-retention states */ + if (IS_ENABLED(CONFIG_CPU_PM)) { + rc = cpu_pm_register_notifier(&kvm_riscv_cpu_pm_nb); + if (rc) { + kvm_err("Failed to register CPU PM notifier: %d\n", rc); + goto err_teardown; + } } + rc = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); + if (rc) + goto err_unregister_cpu_pm; + if (kvm_riscv_aia_available()) kvm_info("AIA available with %d guest external interrupts\n", atomic_read(&kvm_riscv_aia_nr_hgei)); return 0; + +err_unregister_cpu_pm: + if (IS_ENABLED(CONFIG_CPU_PM)) + cpu_pm_unregister_notifier(&kvm_riscv_cpu_pm_nb); +err_teardown: + kvm_riscv_teardown(); + return rc; } module_init(riscv_kvm_init); @@ -192,6 +258,10 @@ static void __exit riscv_kvm_exit(void) { kvm_exit(); + /* Unregister CPU PM notifier */ + if (IS_ENABLED(CONFIG_CPU_PM)) + cpu_pm_unregister_notifier(&kvm_riscv_cpu_pm_nb); + kvm_riscv_teardown(); } module_exit(riscv_kvm_exit);