github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-28 10:09:10 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

arm64: fpsimd: Split FPSR/FPCR from SVE save/restore

Browse Source 
Regardless of whether the vector registers are saved in FPSIMD or SVE
format, we store FPSR and FPCR in user_fpsimd_state::{fpsr,fpcr}.

For historical reasons, the functions which save/restore SVE context
take a pointer to user_fpsimd_state::fpsr, and use this to access both
user_fpsimd_state::fpsr and user_fpsimd_state::fpcr. This is
unnecessarily fragile.

Move the save/restore of FPSR and FPCR into separate helper functions
which take a pointer to user_fpsimd_state. I've used read_sysreg_s() and
write_sysreg_s() as contemporary versions of LLVM will refuse to
directly assemble accesses to FPCR or FPSR unless the "fp" arch
extension is enabled.

For the moment, fpsimd_save_state() and fpsimd_load_state() are left
as-is with their own logic to save/restore FPSR and FPCR. This will be
unified in subsequent patches.

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Vladimir Murzin <vladimir.murzin@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Fuad Tabba <tabba@google.com>
Cc: James Morse <james.morse@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Oliver Upton <oupton@kernel.org>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Will Deacon <will@kernel.org>
This commit is contained in:
Mark Rutland 2026-06-03 12:06:23 +01:00 committed by Will Deacon
parent 36a1d17266
commit 1277531fca
8 changed files with 32 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
arch/arm64/include/asm/fpsimd.h
View File

@ -74,6 +74,18 @@ static inline void cpacr_restore(unsigned long cpacr)
struct task_struct;
static inline void fpsimd_save_common(struct user_fpsimd_state *state)
{
state->fpsr = read_sysreg_s(SYS_FPSR);
state->fpcr = read_sysreg_s(SYS_FPCR);
}
static inline void fpsimd_load_common(const struct user_fpsimd_state *state)
{
write_sysreg_s(state->fpsr, SYS_FPSR);
write_sysreg_s(state->fpcr, SYS_FPCR);
}
extern void fpsimd_save_state(struct user_fpsimd_state *state);
extern void fpsimd_load_state(struct user_fpsimd_state *state);
@ -157,9 +169,8 @@ static inline unsigned int sve_get_vl(void)
return vl;
}
extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr);
extern void sve_load_state(void const *state, u32 const *pfpsr,
int restore_ffr);
extern void sve_save_state(void *state, int save_ffr);
extern void sve_load_state(void const *state, int restore_ffr);
extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
extern void sme_save_state(void *state, int zt);
extern void sme_load_state(void const *state, int zt);

13
arch/arm64/include/asm/fpsimdmacros.h
View File

@ -236,7 +236,7 @@
_sve_wrffr 0
.endm
.macro sve_save nxbase, xpfpsr, save_ffr, nxtmp
.macro sve_save nxbase, save_ffr
_for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34
_for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16
cbz \save_ffr, 921f
@ -247,24 +247,15 @@
922:
_sve_str_p 0, \nxbase
_sve_ldr_p 0, \nxbase, -16
mrs x\nxtmp, fpsr
str w\nxtmp, [\xpfpsr]
mrs x\nxtmp, fpcr
str w\nxtmp, [\xpfpsr, #4]
.endm
.macro sve_load nxbase, xpfpsr, restore_ffr, nxtmp
.macro sve_load nxbase, restore_ffr
_for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34
cbz \restore_ffr, 921f
_sve_ldr_p 0, \nxbase
_sve_wrffr 0
921:
_for n, 0, 15, _sve_ldr_p \n, \nxbase, \n - 16
ldr w\nxtmp, [\xpfpsr]
msr fpsr, x\nxtmp
ldr w\nxtmp, [\xpfpsr, #4]
msr fpcr, x\nxtmp
.endm
.macro sme_save_za nxbase, xvl, nw

4
arch/arm64/include/asm/kvm_hyp.h
View File

@ -123,8 +123,8 @@ void __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu);
void __fpsimd_save_state(struct user_fpsimd_state *fp_regs);
void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs);
void __sve_save_state(void *sve_pffr, u32 *fpsr, int save_ffr);
void __sve_restore_state(void *sve_pffr, u32 *fpsr, int restore_ffr);
void __sve_save_state(void *sve, int save_ffr);
void __sve_restore_state(void *sve, int restore_ffr);
u64 __guest_enter(struct kvm_vcpu *vcpu);

10
arch/arm64/kernel/entry-fpsimd.S
View File

@ -37,11 +37,10 @@ SYM_FUNC_END(fpsimd_load_state)
* Save the SVE state
*
* x0 - pointer to buffer for state
* x1 - pointer to storage for FPSR
* w2 - Save FFR if non-zero
* w1 - Save FFR if non-zero
*/
SYM_FUNC_START(sve_save_state)
sve_save 0, x1, w2, 3
sve_save 0, w1
ret
SYM_FUNC_END(sve_save_state)
@ -49,11 +48,10 @@ SYM_FUNC_END(sve_save_state)
* Load the SVE state
*
* x0 - pointer to buffer for state
* x1 - pointer to storage for FPSR
* w2 - Restore FFR if non-zero
* w1 - Restore FFR if non-zero
*/
SYM_FUNC_START(sve_load_state)
sve_load 0, x1, w2, 4
sve_load 0, w1
ret
SYM_FUNC_END(sve_load_state)

5
arch/arm64/kernel/fpsimd.c
View File

@ -426,8 +426,8 @@ static void task_fpsimd_load(void)
if (restore_sve_regs) {
WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE);
sve_load_state(sve_pffr(&current->thread),
&current->thread.uw.fpsimd_state.fpsr,
restore_ffr);
fpsimd_load_common(&current->thread.uw.fpsimd_state);
} else {
WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD);
fpsimd_load_state(&current->thread.uw.fpsimd_state);
@ -509,7 +509,8 @@ static void fpsimd_save_user_state(void)
sve_save_state((char *)last->sve_state +
sve_ffr_offset(vl),
&last->st->fpsr, save_ffr);
save_ffr);
fpsimd_save_common(last->st);
*last->fp_type = FP_STATE_SVE;
} else {
fpsimd_save_state(last->st);

4
arch/arm64/kvm/hyp/fpsimd.S
View File

@ -21,11 +21,11 @@ SYM_FUNC_START(__fpsimd_restore_state)
SYM_FUNC_END(__fpsimd_restore_state)
SYM_FUNC_START(__sve_restore_state)
sve_load 0, x1, w2, 3
sve_load 0, w1
ret
SYM_FUNC_END(__sve_restore_state)
SYM_FUNC_START(__sve_save_state)
sve_save 0, x1, w2, 3
sve_save 0, w1
ret
SYM_FUNC_END(__sve_save_state)

4
arch/arm64/kvm/hyp/include/hyp/switch.h
View File

@ -468,8 +468,8 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
*/
sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
__sve_restore_state(vcpu_sve_pffr(vcpu),
&vcpu->arch.ctxt.fp_regs.fpsr,
true);
fpsimd_load_common(&vcpu->arch.ctxt.fp_regs);
/*
* The effective VL for a VM could differ from the max VL when running a
@ -490,8 +490,8 @@ static inline void __hyp_sve_save_host(void)
ctxt_sys_reg(hctxt, ZCR_EL1) = read_sysreg_el1(SYS_ZCR);
write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2);
__sve_save_state(sve_regs + sve_ffr_offset(kvm_host_sve_max_vl),
&hctxt->fp_regs.fpsr,
true);
fpsimd_save_common(&hctxt->fp_regs);
}
static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu)

5
arch/arm64/kvm/hyp/nvhe/hyp-main.c
View File

@ -35,7 +35,8 @@ static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu)
* on the VL, so use a consistent (i.e., the maximum) guest VL.
*/
sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
__sve_save_state(vcpu_sve_pffr(vcpu), &vcpu->arch.ctxt.fp_regs.fpsr, true);
__sve_save_state(vcpu_sve_pffr(vcpu), true);
fpsimd_save_common(&vcpu->arch.ctxt.fp_regs);
write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2);
}
@ -55,8 +56,8 @@ static void __hyp_sve_restore_host(void)
*/
write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2);
__sve_restore_state(sve_regs + sve_ffr_offset(kvm_host_sve_max_vl),
&hctxt->fp_regs.fpsr,
true);
fpsimd_load_common(&hctxt->fp_regs);
write_sysreg_el1(ctxt_sys_reg(hctxt, ZCR_EL1), SYS_ZCR);
}