From 27ae400e6e888153ded1ad807a94a94e506dd2df Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 11 May 2026 11:46:11 +0100 Subject: [PATCH 1/2] KVM: arm64: nv: Track L2 to L1 exception emulation While we currently track that we are emulating a nested ERET from L1 to L2, we currently don't track the reverse direction (an exception going from L2 to L1). Add a new vcpu state flag for this purpose, which will see some use shortly. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260520085036.541666-2-maz@kernel.org --- arch/arm64/include/asm/kvm_host.h | 3 ++- arch/arm64/kvm/emulate-nested.c | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 65eead8362e0..c79747d5f4dd 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1112,7 +1112,8 @@ struct kvm_vcpu_arch { #define IN_NESTED_ERET __vcpu_single_flag(sflags, BIT(7)) /* SError pending for nested guest */ #define NESTED_SERROR_PENDING __vcpu_single_flag(sflags, BIT(8)) - +/* KVM is currently emulating an L2 to L1 exception */ +#define IN_NESTED_EXCEPTION __vcpu_single_flag(sflags, BIT(9)) /* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */ #define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) + \ diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index dba7ced74ca5..15c691a6266d 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -2862,6 +2862,8 @@ static int kvm_inject_nested(struct kvm_vcpu *vcpu, u64 esr_el2, preempt_disable(); + vcpu_set_flag(vcpu, IN_NESTED_EXCEPTION); + /* * We may have an exception or PC update in the EL0/EL1 context. * Commit it before entering EL2. @@ -2884,6 +2886,8 @@ static int kvm_inject_nested(struct kvm_vcpu *vcpu, u64 esr_el2, __kvm_adjust_pc(vcpu); kvm_arch_vcpu_load(vcpu, smp_processor_id()); + vcpu_clear_flag(vcpu, IN_NESTED_EXCEPTION); + preempt_enable(); if (kvm_vcpu_has_pmu(vcpu)) From 435c466196148ae116f616e6cda97c33281defc2 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 11 May 2026 11:46:41 +0100 Subject: [PATCH 2/2] KVM: arm64: nv: Don't save/restore FP register during a nested ERET or exception When switching between L1 and L2, we save the old state using kvm_arch_vcpu_put(), mutate the state in memory, then load the new state using kvm_arch_vcpu_load(). Any live FPSIMD/SVE state is saved and unbound, such that it can be lazily restored on a subsequent trap. The FPSIMD/SVE state is shared by exception levels, and only a handful of related control registers need to be changed when transitioning between L1 and L2. The save/restore of the common state is needless overhead, especially as trapping becomes exponentially more expensive with nesting. Avoid this overhead by leaving the common FPSIMD/SVE state live on the CPU, and only switching the state that is distinct for L1 and L2: - the trap controls: the effective values are recomputed on each entry into the guest to take the EL into account and merge the L0 and L1 configuration if in a nested context, or directly use the L0 configuration in non-nested context (see __activate_traps()). - the VL settings: the effective values are are also recomputed on each entry into the guest (see fpsimd_lazy_switch_to_guest()). Since we appear to cover all bases, use the vcpu flags indicating the handling of a nested ERET or exception delivery to avoid the whole FP save/restore shenanigans. SME will have to be similarly dealt with when it eventually gets supported. For an EL1 L3 guest where L1 and L2 have this optimisation, this results in at least a 10% wall clock reduction when running an I/O heavy workload, generating a high rate of nested exceptions. Reviewed-by: Joey Gouly Acked-by: Mark Rutland Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260520085036.541666-3-maz@kernel.org --- arch/arm64/kvm/fpsimd.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c index 15e17aca1dec..3f6b1e29cd6b 100644 --- a/arch/arm64/kvm/fpsimd.c +++ b/arch/arm64/kvm/fpsimd.c @@ -28,6 +28,20 @@ void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu) if (!system_supports_fpsimd()) return; + /* + * Avoid needless save/restore of the guest's common + * FPSIMD/SVE/SME regs during transitions between L1/L2. + * + * These transitions only happens in a non-preemptible context + * where the host regs have already been saved and unbound. The + * live registers are either free or owned by the guest. + */ + if (vcpu_get_flag(vcpu, IN_NESTED_ERET) || + vcpu_get_flag(vcpu, IN_NESTED_EXCEPTION)) { + WARN_ON_ONCE(host_owns_fp_regs()); + return; + } + /* * Ensure that any host FPSIMD/SVE/SME state is saved and unbound such * that the host kernel is responsible for restoring this state upon @@ -102,6 +116,18 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) { unsigned long flags; + /* + * See comment in kvm_arch_vcpu_load_fp(). Note that we also rely on + * the guest's max VL to have been set by fpsimd_lazy_switch_to_host() + * so that any intervening kernel-mode SIMD (NEON or otherwise) + * operation sees the full guest state that needs saving. + */ + if (vcpu_get_flag(vcpu, IN_NESTED_ERET) || + vcpu_get_flag(vcpu, IN_NESTED_EXCEPTION)) { + WARN_ON_ONCE(host_owns_fp_regs()); + return; + } + local_irq_save(flags); if (guest_owns_fp_regs()) {