github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-28 01:55:51 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

arm64: fpsimd: Use opaque type for SVE state

Browse Source 
As the SVE state size can vary at runtime, we don't have a concrete type
for the in-memory SVE state, and pass this around using a pointer to
void. The functions which save/restore the SVE state have a very unusual
calling convention, expecting a pointer to the FFR *in the middle of*
the in-memory SVE state, which is also passed as a pointer to void.
Passing a pointer to the FFR also requires that callers find the live VL
and perform some arithmetic, which callers implement differently.

Using pointer to void means that it's very easy to introduce errors that
cannot be caught by the compiler (e.g. as 'void **' can be assigned to
'void *'). In general this is unnecessarily confusing and fragile.

Improve this by adding an opaque 'struct arm64_sve_state', and
consistently passing a pointer to this, performing the necessary
offsetting *within* the save/restore functions.

For the moment, the offsetting is performed in a new '_sve_pffr'
assembly macro, using the ADDVL and ADDPL instructions. These add a
multiple of the live vector length and predicate length respectively.
The ADDVL immediate range cannot encode 32, so this is split into two
increments of 16.

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Vladimir Murzin <vladimir.murzin@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Fuad Tabba <tabba@google.com>
Cc: James Morse <james.morse@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Oliver Upton <oupton@kernel.org>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Will Deacon <will@kernel.org>
This commit is contained in:
Mark Rutland 2026-06-03 12:06:25 +01:00 committed by Will Deacon
parent 890712d450
commit e1b163e405
11 changed files with 40 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
arch/arm64/include/asm/fpsimd.h
View File

@ -162,7 +162,7 @@ extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
struct cpu_fp_state {
struct user_fpsimd_state *st;
void *sve_state;
struct arm64_sve_state *sve_state;
void *sme_state;
u64 *svcr;
u64 *fpmr;
@ -195,24 +195,6 @@ extern void task_smstop_sm(struct task_struct *task);
/* Maximum VL that SVE/SME VL-agnostic software can transparently support */
#define VL_ARCH_MAX 0x100
/* Offset of FFR in the SVE register dump */
static inline size_t sve_ffr_offset(int vl)
{
return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
}
static inline void *sve_pffr(struct thread_struct *thread)
{
unsigned int vl;
if (system_supports_sme() && thread_sm_enabled(thread))
vl = thread_get_sme_vl(thread);
else
vl = thread_get_sve_vl(thread);
return (char *)thread->sve_state + sve_ffr_offset(vl);
}
static inline void *thread_zt_state(struct thread_struct *thread)
{
/* The ZT register state is stored immediately after the ZA state */
@ -233,8 +215,8 @@ static inline unsigned int sve_get_vl(void)
return vl;
}
extern void sve_save_state(void *state, int save_ffr);
extern void sve_load_state(void const *state, int restore_ffr);
extern void sve_save_state(struct arm64_sve_state *state, int save_ffr);
extern void sve_load_state(const struct arm64_sve_state *state, int restore_ffr);
extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
extern void sme_save_state(void *state, int zt);
extern void sme_load_state(void const *state, int zt);

9
arch/arm64/include/asm/fpsimdmacros.h
View File

@ -177,7 +177,15 @@
_sve_wrffr 0
.endm
.macro _sve_pffr ptr
.arch_extension sve
addvl \ptr, \ptr, #16
addvl \ptr, \ptr, #16
addpl \ptr, \ptr, #16
.endm
.macro sve_save nxbase, save_ffr
_sve_pffr x\nxbase
_for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34
_for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16
cbz \save_ffr, 921f
@ -191,6 +199,7 @@
.endm
.macro sve_load nxbase, restore_ffr
_sve_pffr x\nxbase
_for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34
cbz \restore_ffr, 921f
_sve_ldr_p 0, \nxbase

8
arch/arm64/include/asm/kvm_host.h
View File

@ -762,7 +762,7 @@ struct kvm_host_data {
* Hyp VA.
* sve_regs is only used in pKVM and if system_supports_sve().
*/
u8 *sve_regs;
struct arm64_sve_state *sve_regs;
/* Ownership of the FP regs */
enum {
@ -856,7 +856,7 @@ struct kvm_vcpu_arch {
* floating point code saves the register state of a task it
* records which view it saved in fp_type.
*/
void *sve_state;
struct arm64_sve_state *sve_state;
enum fp_type fp_type;
unsigned int sve_max_vl;
@ -1100,10 +1100,6 @@ struct kvm_vcpu_arch {
#define NESTED_SERROR_PENDING __vcpu_single_flag(sflags, BIT(8))
/* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */
#define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) + \
sve_ffr_offset((vcpu)->arch.sve_max_vl))
#define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl)
#define vcpu_sve_zcr_elx(vcpu) \

4
arch/arm64/include/asm/kvm_hyp.h
View File

@ -121,8 +121,8 @@ void __debug_save_host_buffers_nvhe(struct kvm_vcpu *vcpu);
void __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu);
#endif
void __sve_save_state(void *sve, int save_ffr);
void __sve_restore_state(void *sve, int restore_ffr);
void __sve_save_state(struct arm64_sve_state *sve, int save_ffr);
void __sve_restore_state(struct arm64_sve_state *sve, int restore_ffr);
u64 __guest_enter(struct kvm_vcpu *vcpu);

4
arch/arm64/include/asm/processor.h
View File

@ -130,6 +130,8 @@ enum fp_type {
FP_STATE_SVE,
};
struct arm64_sve_state; /* Opaque type */
struct cpu_context {
unsigned long x19;
unsigned long x20;
@ -164,7 +166,7 @@ struct thread_struct {
enum fp_type fp_type; /* registers FPSIMD or SVE? */
unsigned int fpsimd_cpu;
void *sve_state; /* SVE registers, if any */
struct arm64_sve_state *sve_state; /* SVE registers, if any */
void *sme_state; /* ZA and ZT state, if any */
unsigned int vl[ARM64_VEC_MAX]; /* vector length */
unsigned int vl_onexec[ARM64_VEC_MAX]; /* vl after next exec */

21
arch/arm64/kernel/fpsimd.c
View File

@ -425,8 +425,7 @@ static void task_fpsimd_load(void)
if (restore_sve_regs) {
WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE);
sve_load_state(sve_pffr(&current->thread),
restore_ffr);
sve_load_state(current->thread.sve_state, restore_ffr);
fpsimd_load_common(&current->thread.uw.fpsimd_state);
} else {
WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD);
@ -507,9 +506,7 @@ static void fpsimd_save_user_state(void)
return;
}
sve_save_state((char *)last->sve_state +
sve_ffr_offset(vl),
save_ffr);
sve_save_state(last->sve_state, save_ffr);
fpsimd_save_common(last->st);
*last->fp_type = FP_STATE_SVE;
} else {
@ -641,7 +638,8 @@ static __uint128_t arm64_cpu_to_le128(__uint128_t x)
#define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x)
static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst,
static void __fpsimd_to_sve(struct arm64_sve_state *sst,
struct user_fpsimd_state const *fst,
unsigned int vq)
{
unsigned int i;
@ -668,7 +666,7 @@ static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst,
static inline void fpsimd_to_sve(struct task_struct *task)
{
unsigned int vq;
void *sst = task->thread.sve_state;
struct arm64_sve_state *sst = task->thread.sve_state;
struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
if (!system_supports_sve() && !system_supports_sme())
@ -692,7 +690,7 @@ static inline void fpsimd_to_sve(struct task_struct *task)
static inline void sve_to_fpsimd(struct task_struct *task)
{
unsigned int vq, vl;
void const *sst = task->thread.sve_state;
const struct arm64_sve_state *sst = task->thread.sve_state;
struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;
unsigned int i;
__uint128_t const *p;
@ -791,7 +789,7 @@ void fpsimd_sync_from_effective_state(struct task_struct *task)
void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task)
{
unsigned int vq;
void *sst = task->thread.sve_state;
struct arm64_sve_state *sst = task->thread.sve_state;
struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
if (task->thread.fp_type != FP_STATE_SVE)
@ -809,7 +807,8 @@ static int change_live_vector_length(struct task_struct *task,
{
unsigned int sve_vl = task_get_sve_vl(task);
unsigned int sme_vl = task_get_sme_vl(task);
void *sve_state = NULL, *sme_state = NULL;
struct arm64_sve_state *sve_state = NULL;
void *sme_state = NULL;
if (type == ARM64_VEC_SME)
sme_vl = vl;
@ -1645,7 +1644,7 @@ static void fpsimd_flush_thread_vl(enum vec_type type)
void fpsimd_flush_thread(void)
{
void *sve_state = NULL;
struct arm64_sve_state *sve_state = NULL;
void *sme_state = NULL;
if (!system_supports_fpsimd())

4
arch/arm64/kvm/arm.c
View File

@ -2495,7 +2495,7 @@ static void __init teardown_hyp_mode(void)
continue;
if (free_sve) {
u8 *sve_regs;
struct arm64_sve_state *sve_regs;
sve_regs = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs;
free_pages((unsigned long) sve_regs, pkvm_host_sve_state_order());
@ -2644,7 +2644,7 @@ static void finalize_init_hyp_mode(void)
if (system_supports_sve() && is_protected_kvm_enabled()) {
for_each_possible_cpu(cpu) {
u8 *sve_regs;
struct arm64_sve_state *sve_regs;
sve_regs = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs;
per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs =

4
arch/arm64/kvm/guest.c
View File

@ -500,7 +500,7 @@ static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
if (!kvm_arm_vcpu_sve_finalized(vcpu))
return -EPERM;
if (copy_to_user(uptr, vcpu->arch.sve_state + region.koffset,
if (copy_to_user(uptr, (void *)vcpu->arch.sve_state + region.koffset,
region.klen) ||
clear_user(uptr + region.klen, region.upad))
return -EFAULT;
@ -526,7 +526,7 @@ static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
if (!kvm_arm_vcpu_sve_finalized(vcpu))
return -EPERM;
if (copy_from_user(vcpu->arch.sve_state + region.koffset, uptr,
if (copy_from_user((void *)vcpu->arch.sve_state + region.koffset, uptr,
region.klen))
return -EFAULT;

8
arch/arm64/kvm/hyp/include/hyp/switch.h
View File

@ -467,8 +467,7 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
* vCPU. Start off with the max VL so we can load the SVE state.
*/
sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
__sve_restore_state(vcpu_sve_pffr(vcpu),
true);
__sve_restore_state(kern_hyp_va(vcpu->arch.sve_state), true);
fpsimd_load_common(&vcpu->arch.ctxt.fp_regs);
/*
@ -485,12 +484,11 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
static inline void __hyp_sve_save_host(void)
{
struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt);
u8 *sve_regs = *host_data_ptr(sve_regs);
struct arm64_sve_state *sve_regs = *host_data_ptr(sve_regs);
ctxt_sys_reg(hctxt, ZCR_EL1) = read_sysreg_el1(SYS_ZCR);
write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2);
__sve_save_state(sve_regs + sve_ffr_offset(kvm_host_sve_max_vl),
true);
__sve_save_state(sve_regs, true);
fpsimd_save_common(&hctxt->fp_regs);
}

7
arch/arm64/kvm/hyp/nvhe/hyp-main.c
View File

@ -35,7 +35,7 @@ static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu)
* on the VL, so use a consistent (i.e., the maximum) guest VL.
*/
sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
__sve_save_state(vcpu_sve_pffr(vcpu), true);
__sve_save_state(kern_hyp_va(vcpu->arch.sve_state), true);
fpsimd_save_common(&vcpu->arch.ctxt.fp_regs);
write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2);
}
@ -43,7 +43,7 @@ static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu)
static void __hyp_sve_restore_host(void)
{
struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt);
u8 *sve_regs = *host_data_ptr(sve_regs);
struct arm64_sve_state *sve_regs = *host_data_ptr(sve_regs);
/*
* On saving/restoring host sve state, always use the maximum VL for
@ -55,8 +55,7 @@ static void __hyp_sve_restore_host(void)
* need to be revisited.
*/
write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2);
__sve_restore_state(sve_regs + sve_ffr_offset(kvm_host_sve_max_vl),
true);
__sve_restore_state(sve_regs, true);
fpsimd_load_common(&hctxt->fp_regs);
write_sysreg_el1(ctxt_sys_reg(hctxt, ZCR_EL1), SYS_ZCR);
}

2
arch/arm64/kvm/hyp/nvhe/setup.c
View File

@ -82,7 +82,7 @@ static int pkvm_create_host_sve_mappings(void)
for (i = 0; i < hyp_nr_cpus; i++) {
struct kvm_host_data *host_data = per_cpu_ptr(&kvm_host_data, i);
u8 *sve_regs = host_data->sve_regs;
struct arm64_sve_state *sve_regs = host_data->sve_regs;
start = kern_hyp_va(sve_regs);
end = start + PAGE_ALIGN(pkvm_host_sve_state_size());