diff --git a/arch/riscv/include/asm/kvm_vcpu_vector.h b/arch/riscv/include/asm/kvm_vcpu_vector.h index 57a798a4cb0d..6371d5ea5392 100644 --- a/arch/riscv/include/asm/kvm_vcpu_vector.h +++ b/arch/riscv/include/asm/kvm_vcpu_vector.h @@ -35,6 +35,22 @@ void kvm_riscv_vcpu_host_vector_save(struct kvm_cpu_context *cntx); void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx); int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu); +void kvm_riscv_register_vctx_callback(void (*func)(void)); +void kvm_riscv_unregister_vctx_callback(void); +void kvm_riscv_vcpu_flush_vector(void); + +static inline void kvm_riscv_v_init(void) +{ + if (has_vector()) + kvm_riscv_register_vctx_callback(&kvm_riscv_vcpu_flush_vector); +} + +static inline void kvm_riscv_v_exit(void) +{ + if (has_vector()) + kvm_riscv_unregister_vctx_callback(); +} + #else struct kvm_cpu_context; @@ -69,6 +85,14 @@ static inline int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu) static inline void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu) { } + +static inline void kvm_riscv_v_init(void) +{ +} + +static inline void kvm_riscv_v_exit(void) +{ +} #endif int kvm_riscv_vcpu_get_reg_vector(struct kvm_vcpu *vcpu, diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index 812517b2cec1..a6a0c3d5a913 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -67,6 +67,12 @@ struct pt_regs; * - bit 0: indicates whether the in-kernel Vector context is active. The * activation of this state disables the preemption. On a non-RT kernel, it * also disable bh. + * - bit 1: tells kvm that the vcpu process has guest context saved in vcpu's + * context memory and need to be restore upon returing back to the guest. + * - bit 2: represents that the vector context has now loaded and belongs to + * the guest kernel. Any non-scheduler context saving routing needs to save + * the register file to vcpu's context memory. The bit is set upon returing + * back to the guest and cleared after loading the host's vector context. * - bits 8: is used for tracking preemptible kernel-mode Vector, when * RISCV_ISA_V_PREEMPTIVE is enabled. Calling kernel_vector_begin() does not * disable the preemption if the thread's kernel_vstate.datap is allocated. @@ -97,6 +103,8 @@ struct pt_regs; #define RISCV_V_CTX_UNIT_DEPTH 0x00010000 #define RISCV_KERNEL_MODE_V 0x00000001 +#define RISCV_V_VCPU_NEED_RESTORE 0x00000002 +#define RISCV_V_VCPU_CTX 0x00000004 #define RISCV_PREEMPT_V 0x00000100 #define RISCV_PREEMPT_V_DIRTY 0x80000000 #define RISCV_PREEMPT_V_NEED_RESTORE 0x40000000 diff --git a/arch/riscv/include/asm/vector.h b/arch/riscv/include/asm/vector.h index 00cb9c0982b1..fffe72a77208 100644 --- a/arch/riscv/include/asm/vector.h +++ b/arch/riscv/include/asm/vector.h @@ -58,6 +58,11 @@ static inline u32 riscv_v_flags(void) return READ_ONCE(current->thread.riscv_v_flags); } +static inline void riscv_v_flags_set(u32 flags) +{ + WRITE_ONCE(current->thread.riscv_v_flags, flags); +} + static __always_inline bool has_vector(void) { return riscv_has_extension_unlikely(RISCV_ISA_EXT_ZVE32X); diff --git a/arch/riscv/kernel/kernel_mode_vector.c b/arch/riscv/kernel/kernel_mode_vector.c index 965c8edbe984..77e98b504485 100644 --- a/arch/riscv/kernel/kernel_mode_vector.c +++ b/arch/riscv/kernel/kernel_mode_vector.c @@ -13,16 +13,31 @@ #include #include +#include #include #include #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE #include #endif -static inline void riscv_v_flags_set(u32 flags) +static void (* __rcu kvm_flush_vector_ctx_callback)(void); + +void kvm_riscv_register_vctx_callback(void (*func)(void)) { - WRITE_ONCE(current->thread.riscv_v_flags, flags); + if (WARN_ON_ONCE(rcu_access_pointer(kvm_flush_vector_ctx_callback))) + return; + + rcu_assign_pointer(kvm_flush_vector_ctx_callback, func); } +EXPORT_SYMBOL_GPL(kvm_riscv_register_vctx_callback); + +void kvm_riscv_unregister_vctx_callback(void) +{ + rcu_assign_pointer(kvm_flush_vector_ctx_callback, NULL); + synchronize_rcu(); +} +EXPORT_SYMBOL_GPL(kvm_riscv_unregister_vctx_callback); + static inline void riscv_v_start(u32 flags) { @@ -87,6 +102,22 @@ void put_cpu_vector_context(void) } EXPORT_SYMBOL_FOR_KVM(put_cpu_vector_context); +static void __riscv_flush_vector_context(void) +{ + void (*vcpu_flush_v_callback)(void); + + if (riscv_v_flags() & RISCV_V_VCPU_CTX) { + rcu_read_lock(); + vcpu_flush_v_callback = rcu_dereference(kvm_flush_vector_ctx_callback); + vcpu_flush_v_callback(); + rcu_read_unlock(); + return; + } + + riscv_v_vstate_save(¤t->thread.vstate, task_pt_regs(current)); + riscv_v_vstate_set_restore(current, task_pt_regs(current)); +} + #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE static __always_inline u32 *riscv_v_flags_ptr(void) { @@ -130,7 +161,7 @@ static int riscv_v_stop_kernel_context(void) static int riscv_v_start_kernel_context(void) { - struct __riscv_v_ext_state *kvstate, *uvstate; + struct __riscv_v_ext_state *kvstate; kvstate = ¤t->thread.kernel_vstate; if (!kvstate->datap) @@ -148,13 +179,18 @@ static int riscv_v_start_kernel_context(void) } /* Transfer the ownership of V from user to kernel, then save */ - riscv_v_start(RISCV_PREEMPT_V | RISCV_PREEMPT_V_DIRTY); - if (__riscv_v_vstate_check(task_pt_regs(current)->status, DIRTY)) { - uvstate = ¤t->thread.vstate; - __riscv_v_vstate_save(uvstate, uvstate->datap); - } - riscv_preempt_v_clear_dirty(current); - riscv_v_vstate_set_restore(current, task_pt_regs(current)); + get_cpu_vector_context(); + __riscv_flush_vector_context(); + put_cpu_vector_context(); + /* + * A voluntary context switch caused by put_cpu_vector_context() can + * raise the NEED_RESTORE flag if preempt_v starts too early due to a + * failed risv_v_is_on() check. + * + * This causes the next context_nesting_end pollute the v-reg from + * the stale context memory in kernel-mode vector. + */ + riscv_v_start(RISCV_PREEMPT_V); return 0; } @@ -220,8 +256,7 @@ void kernel_vector_begin(void) if (riscv_v_start_kernel_context()) { get_cpu_vector_context(); - riscv_v_vstate_save(¤t->thread.vstate, task_pt_regs(current)); - riscv_v_vstate_set_restore(current, task_pt_regs(current)); + __riscv_flush_vector_context(); } riscv_v_enable(); diff --git a/arch/riscv/kvm/main.c b/arch/riscv/kvm/main.c index 20b516e84e60..85e772588fad 100644 --- a/arch/riscv/kvm/main.c +++ b/arch/riscv/kvm/main.c @@ -15,6 +15,7 @@ #include #include #include +#include static DEFINE_PER_CPU(bool, kvm_riscv_virtualization_enabled); @@ -130,6 +131,7 @@ static void kvm_riscv_teardown(void) { kvm_riscv_aia_exit(); kvm_riscv_nacl_exit(); + kvm_riscv_v_exit(); kvm_unregister_perf_callbacks(); } @@ -224,6 +226,8 @@ static int __init riscv_kvm_init(void) kvm_riscv_setup_vendor_features(); + kvm_riscv_v_init(); + kvm_register_perf_callbacks(); /* Register CPU PM notifier for CPU idle non-retention states */ diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index 1b4416b20665..e062ca19f9d8 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -647,9 +647,11 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context); kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context, vcpu->arch.isa); + get_cpu_vector_context(); kvm_riscv_vcpu_host_vector_save(&vcpu->arch.host_context); kvm_riscv_vcpu_guest_vector_restore(&vcpu->arch.guest_context, vcpu->arch.isa); + put_cpu_vector_context(); kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); @@ -670,9 +672,11 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) kvm_riscv_vcpu_host_fp_restore(&vcpu->arch.host_context); kvm_riscv_vcpu_timer_save(vcpu); + get_cpu_vector_context(); kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context, vcpu->arch.isa); kvm_riscv_vcpu_host_vector_restore(&vcpu->arch.host_context); + put_cpu_vector_context(); if (kvm_riscv_nacl_available()) { nsh = nacl_shmem(); @@ -815,6 +819,14 @@ static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu, kvm_riscv_vcpu_swap_in_guest_state(vcpu); guest_state_enter_irqoff(); + /* sstatus.VS != SR_VS_OFF is guaranteed when NEED_RESTORE is set */ + if (current->thread.riscv_v_flags & RISCV_V_VCPU_NEED_RESTORE) { + current->thread.riscv_v_flags &= ~RISCV_V_VCPU_NEED_RESTORE; + current->thread.riscv_v_flags |= RISCV_V_VCPU_CTX; + __kvm_riscv_vector_restore(gcntx); + gcntx->sstatus = (gcntx->sstatus & ~SR_VS) | SR_VS_CLEAN; + } + if (kvm_riscv_nacl_sync_sret_available()) { nsh = nacl_shmem(); diff --git a/arch/riscv/kvm/vcpu_vector.c b/arch/riscv/kvm/vcpu_vector.c index e87f49e06f14..bf5c94d8786b 100644 --- a/arch/riscv/kvm/vcpu_vector.c +++ b/arch/riscv/kvm/vcpu_vector.c @@ -57,8 +57,7 @@ void kvm_riscv_vcpu_guest_vector_restore(struct kvm_cpu_context *cntx, { if ((cntx->sstatus & SR_VS) != SR_VS_OFF) { if (riscv_isa_extension_available(isa, v)) - __kvm_riscv_vector_restore(cntx); - kvm_riscv_vcpu_vector_clean(cntx); + riscv_v_flags_set(riscv_v_flags() | RISCV_V_VCPU_NEED_RESTORE); } } @@ -73,6 +72,7 @@ void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx) { if (!kvm_riscv_isa_check_host(V)) __kvm_riscv_vector_restore(cntx); + riscv_v_flags_set(riscv_v_flags() & ~(RISCV_V_VCPU_CTX | RISCV_V_VCPU_NEED_RESTORE)); } int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu) @@ -96,6 +96,24 @@ void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu) kfree(vcpu->arch.guest_context.vector.datap); kfree(vcpu->arch.host_context.vector.datap); } + +void kvm_riscv_vcpu_flush_vector(void) +{ + struct kvm_vcpu *vcpu = *this_cpu_ptr(kvm_get_running_vcpus()); + + /* + * Only reached from __riscv_flush_vector_context() when RISCV_V_VCPU_CTX is set, which + * always have kvm_get_running_vcpus non-NULL. + */ + if (WARN_ON_ONCE(!vcpu)) + return; + + kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context, vcpu->arch.isa); + + if ((vcpu->arch.guest_context.sstatus & SR_VS) != SR_VS_OFF) + riscv_v_flags_set(riscv_v_flags() | RISCV_V_VCPU_NEED_RESTORE); +} + #endif static int kvm_riscv_vcpu_vreg_addr(struct kvm_vcpu *vcpu,