From 01ab991fc0ee5019aecc4ca461311fc8aa75ece5 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 7 Nov 2022 18:24:00 +0100 Subject: [PATCH 1/7] arm64: Enable data independent timing (DIT) in the kernel The ARM architecture revision v8.4 introduces a data independent timing control (DIT) which can be set at any exception level, and instructs the CPU to avoid optimizations that may result in a correlation between the execution time of certain instructions and the value of the data they operate on. The DIT bit is part of PSTATE, and is therefore context switched as usual, given that it becomes part of the saved program state (SPSR) when taking an exception. We have also defined a hwcap for DIT, and so user space can discover already whether or nor DIT is available. This means that, as far as user space is concerned, DIT is wired up and fully functional. In the kernel, however, we never bothered with DIT: we disable at it boot (i.e., INIT_PSTATE_EL1 has DIT cleared) and ignore the fact that we might run with DIT enabled if user space happened to set it. Currently, we have no idea whether or not running privileged code with DIT disabled on a CPU that implements support for it may result in a side channel that exposes privileged data to unprivileged user space processes, so let's be cautious and just enable DIT while running in the kernel if supported by all CPUs. Cc: Catalin Marinas Cc: Will Deacon Cc: Mark Rutland Cc: Marc Zyngier Cc: Eric Biggers Cc: Jason A. Donenfeld Cc: Kees Cook Cc: Suzuki K Poulose Cc: Adam Langley Link: https://lore.kernel.org/all/YwgCrqutxmX0W72r@gmail.com/ Signed-off-by: Ard Biesheuvel Acked-by: Mark Rutland Link: https://lore.kernel.org/r/20221107172400.1851434-1-ardb@kernel.org [will: Removed cpu_has_dit() as per Mark's suggestion on the list] Signed-off-by: Will Deacon --- arch/arm64/include/asm/sysreg.h | 12 ++++++++---- arch/arm64/kernel/cpufeature.c | 17 +++++++++++++++++ arch/arm64/kernel/entry.S | 3 +++ arch/arm64/kernel/suspend.c | 2 ++ arch/arm64/tools/cpucaps | 1 + 5 files changed, 31 insertions(+), 4 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 7d301700d1a9..1f3f52ce407f 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -90,20 +90,24 @@ */ #define pstate_field(op1, op2) ((op1) << Op1_shift | (op2) << Op2_shift) #define PSTATE_Imm_shift CRm_shift +#define SET_PSTATE(x, r) __emit_inst(0xd500401f | PSTATE_ ## r | ((!!x) << PSTATE_Imm_shift)) #define PSTATE_PAN pstate_field(0, 4) #define PSTATE_UAO pstate_field(0, 3) #define PSTATE_SSBS pstate_field(3, 1) +#define PSTATE_DIT pstate_field(3, 2) #define PSTATE_TCO pstate_field(3, 4) -#define SET_PSTATE_PAN(x) __emit_inst(0xd500401f | PSTATE_PAN | ((!!x) << PSTATE_Imm_shift)) -#define SET_PSTATE_UAO(x) __emit_inst(0xd500401f | PSTATE_UAO | ((!!x) << PSTATE_Imm_shift)) -#define SET_PSTATE_SSBS(x) __emit_inst(0xd500401f | PSTATE_SSBS | ((!!x) << PSTATE_Imm_shift)) -#define SET_PSTATE_TCO(x) __emit_inst(0xd500401f | PSTATE_TCO | ((!!x) << PSTATE_Imm_shift)) +#define SET_PSTATE_PAN(x) SET_PSTATE((x), PAN) +#define SET_PSTATE_UAO(x) SET_PSTATE((x), UAO) +#define SET_PSTATE_SSBS(x) SET_PSTATE((x), SSBS) +#define SET_PSTATE_DIT(x) SET_PSTATE((x), DIT) +#define SET_PSTATE_TCO(x) SET_PSTATE((x), TCO) #define set_pstate_pan(x) asm volatile(SET_PSTATE_PAN(x)) #define set_pstate_uao(x) asm volatile(SET_PSTATE_UAO(x)) #define set_pstate_ssbs(x) asm volatile(SET_PSTATE_SSBS(x)) +#define set_pstate_dit(x) asm volatile(SET_PSTATE_DIT(x)) #define __SYS_BARRIER_INSN(CRm, op2, Rt) \ __emit_inst(0xd5000000 | sys_insn(0, 3, 3, (CRm), (op2)) | ((Rt) & 0x1f)) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index b3f37e2209ad..3086304652fd 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2101,6 +2101,11 @@ static void cpu_trap_el0_impdef(const struct arm64_cpu_capabilities *__unused) sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_TIDCP); } +static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused) +{ + set_pstate_dit(1); +} + /* Internal helper functions to match cpu capability type */ static bool cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap) @@ -2664,6 +2669,18 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .cpu_enable = cpu_trap_el0_impdef, }, + { + .desc = "Data independent timing control (DIT)", + .capability = ARM64_HAS_DIT, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .sys_reg = SYS_ID_AA64PFR0_EL1, + .sign = FTR_UNSIGNED, + .field_pos = ID_AA64PFR0_EL1_DIT_SHIFT, + .field_width = 4, + .min_field_value = ID_AA64PFR0_EL1_DIT_IMP, + .matches = has_cpuid_feature, + .cpu_enable = cpu_enable_dit, + }, {}, }; diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index e28137d64b76..11cb99c4d298 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -197,6 +197,9 @@ alternative_cb_end .endm .macro kernel_entry, el, regsize = 64 + .if \el == 0 + alternative_insn nop, SET_PSTATE_DIT(1), ARM64_HAS_DIT + .endif .if \regsize == 32 mov w0, w0 // zero upper 32 bits of x0 .endif diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index 8b02d310838f..e7163f31f716 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -60,6 +60,8 @@ void notrace __cpu_suspend_exit(void) * PSTATE was not saved over suspend/resume, re-enable any detected * features that might not have been set correctly. */ + if (cpus_have_const_cap(ARM64_HAS_DIT)) + set_pstate_dit(1); __uaccess_enable_hw_pan(); /* diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index f1c0347ec31a..a86ee376920a 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -20,6 +20,7 @@ HAS_CNP HAS_CRC32 HAS_DCPODP HAS_DCPOP +HAS_DIT HAS_E0PD HAS_ECV HAS_EPAN From 95aa6860d60881e8f08956e02199f7d65c9cbf7b Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 17 Oct 2022 16:25:15 +0100 Subject: [PATCH 2/7] arm64/hwcap: Add support for FEAT_CSSC FEAT_CSSC adds a number of new instructions usable to optimise common short sequences of instructions, add a hwcap indicating that the feature is available and can be used by userspace. Signed-off-by: Mark Brown Reviewed-by: Catalin Marinas Link: https://lore.kernel.org/r/20221017152520.1039165-2-broonie@kernel.org Signed-off-by: Will Deacon --- Documentation/arm64/elf_hwcaps.rst | 3 +++ arch/arm64/include/asm/hwcap.h | 1 + arch/arm64/include/uapi/asm/hwcap.h | 1 + arch/arm64/kernel/cpufeature.c | 2 ++ arch/arm64/kernel/cpuinfo.c | 1 + arch/arm64/tools/sysreg | 7 ++++++- 6 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst index bb34287c8e01..58197e9ccb6d 100644 --- a/Documentation/arm64/elf_hwcaps.rst +++ b/Documentation/arm64/elf_hwcaps.rst @@ -275,6 +275,9 @@ HWCAP2_EBF16 HWCAP2_SVE_EBF16 Functionality implied by ID_AA64ZFR0_EL1.BF16 == 0b0010. +HWCAP2_CSSC + Functionality implied by ID_AA64ISAR2_EL1.CSSC == 0b0001. + 4. Unused AT_HWCAP bits ----------------------- diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index 298b386d3ebe..a0e080df9a62 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -120,6 +120,7 @@ #define KERNEL_HWCAP_WFXT __khwcap2_feature(WFXT) #define KERNEL_HWCAP_EBF16 __khwcap2_feature(EBF16) #define KERNEL_HWCAP_SVE_EBF16 __khwcap2_feature(SVE_EBF16) +#define KERNEL_HWCAP_CSSC __khwcap2_feature(CSSC) /* * This yields a mask that user programs can use to figure out what diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 9b245da6f507..a43dddd94b4a 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -93,5 +93,6 @@ #define HWCAP2_WFXT (1UL << 31) #define HWCAP2_EBF16 (1UL << 32) #define HWCAP2_SVE_EBF16 (1UL << 33) +#define HWCAP2_CSSC (1UL << 34) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 3086304652fd..23d1d6b01c1f 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -212,6 +212,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { }; static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { + ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_EL1_BC_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH), FTR_STRICT, FTR_EXACT, ID_AA64ISAR2_EL1_APA3_SHIFT, 4, 0), @@ -2815,6 +2816,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { #endif /* CONFIG_ARM64_MTE */ HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_EL1_ECV_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV), HWCAP_CAP(SYS_ID_AA64MMFR1_EL1, ID_AA64MMFR1_EL1_AFP_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AFP), + HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, FTR_UNSIGNED, ID_AA64ISAR2_EL1_CSSC_IMP, CAP_HWCAP, KERNEL_HWCAP_CSSC), HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_RPRES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RPRES), HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_WFxT_SHIFT, 4, FTR_UNSIGNED, ID_AA64ISAR2_EL1_WFxT_IMP, CAP_HWCAP, KERNEL_HWCAP_WFXT), #ifdef CONFIG_ARM64_SME diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 28d4f442b0bc..3160550c0cc9 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -116,6 +116,7 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_WFXT] = "wfxt", [KERNEL_HWCAP_EBF16] = "ebf16", [KERNEL_HWCAP_SVE_EBF16] = "sveebf16", + [KERNEL_HWCAP_CSSC] = "cssc", }; #ifdef CONFIG_COMPAT diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 384757a7eda9..629d119151bf 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -484,7 +484,12 @@ EndEnum EndSysreg Sysreg ID_AA64ISAR2_EL1 3 0 0 6 2 -Res0 63:28 +Res0 63:56 +Enum 55:52 CSSC + 0b0000 NI + 0b0001 IMP +EndEnum +Res0 51:28 Enum 27:24 PAC_frac 0b0000 NI 0b0001 IMP From b0ab73a5479fdf6b42babaccf22b4fa88f5a20a6 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 17 Oct 2022 16:25:16 +0100 Subject: [PATCH 3/7] kselftest/arm64: Add FEAT_CSSC to the hwcap selftest Add FEAT_CSSC to the set of features checked by the hwcap selftest. Signed-off-by: Mark Brown Reviewed-by: Catalin Marinas Link: https://lore.kernel.org/r/20221017152520.1039165-3-broonie@kernel.org Signed-off-by: Will Deacon --- tools/testing/selftests/arm64/abi/hwcap.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c index 9f1a7b5c6193..c7a6b327a7d0 100644 --- a/tools/testing/selftests/arm64/abi/hwcap.c +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -33,6 +33,12 @@ */ typedef void (*sigill_fn)(void); +static void cssc_sigill(void) +{ + /* CNT x0, x0 */ + asm volatile(".inst 0xdac01c00" : : : "x0"); +} + static void rng_sigill(void) { asm volatile("mrs x0, S3_3_C2_C4_0" : : : "x0"); @@ -118,6 +124,13 @@ static const struct hwcap_data { sigill_fn sigill_fn; bool sigill_reliable; } hwcaps[] = { + { + .name = "CSSC", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_CSSC, + .cpuinfo = "cssc", + .sigill_fn = cssc_sigill, + }, { .name = "RNG", .at_hwcap = AT_HWCAP2, From 939e4649d4fd54b622d08cd57100828df7a82074 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 17 Oct 2022 16:25:17 +0100 Subject: [PATCH 4/7] arm64/hwcap: Add support for FEAT_RPRFM FEAT_RPRFM adds a new range prefetch hint within the existing PRFM space for range prefetch hinting. Add a new hwcap to allow userspace to discover support for the new instruction. Signed-off-by: Mark Brown Reviewed-by: Catalin Marinas Link: https://lore.kernel.org/r/20221017152520.1039165-4-broonie@kernel.org Signed-off-by: Will Deacon --- Documentation/arm64/elf_hwcaps.rst | 3 +++ arch/arm64/include/asm/hwcap.h | 1 + arch/arm64/include/uapi/asm/hwcap.h | 1 + arch/arm64/kernel/cpufeature.c | 2 ++ arch/arm64/kernel/cpuinfo.c | 1 + arch/arm64/tools/sysreg | 6 +++++- 6 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst index 58197e9ccb6d..a82b2cdff680 100644 --- a/Documentation/arm64/elf_hwcaps.rst +++ b/Documentation/arm64/elf_hwcaps.rst @@ -278,6 +278,9 @@ HWCAP2_SVE_EBF16 HWCAP2_CSSC Functionality implied by ID_AA64ISAR2_EL1.CSSC == 0b0001. +HWCAP2_RPRFM + Functionality implied by ID_AA64ISAR2_EL1.RPRFM == 0b0001. + 4. Unused AT_HWCAP bits ----------------------- diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index a0e080df9a62..605ec55cee70 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -121,6 +121,7 @@ #define KERNEL_HWCAP_EBF16 __khwcap2_feature(EBF16) #define KERNEL_HWCAP_SVE_EBF16 __khwcap2_feature(SVE_EBF16) #define KERNEL_HWCAP_CSSC __khwcap2_feature(CSSC) +#define KERNEL_HWCAP_RPRFM __khwcap2_feature(RPRFM) /* * This yields a mask that user programs can use to figure out what diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index a43dddd94b4a..063cc6ea560f 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -94,5 +94,6 @@ #define HWCAP2_EBF16 (1UL << 32) #define HWCAP2_SVE_EBF16 (1UL << 33) #define HWCAP2_CSSC (1UL << 34) +#define HWCAP2_RPRFM (1UL << 35) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 23d1d6b01c1f..6e0837212e97 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -213,6 +213,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_RPRFM_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_EL1_BC_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH), FTR_STRICT, FTR_EXACT, ID_AA64ISAR2_EL1_APA3_SHIFT, 4, 0), @@ -2817,6 +2818,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_EL1_ECV_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV), HWCAP_CAP(SYS_ID_AA64MMFR1_EL1, ID_AA64MMFR1_EL1_AFP_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AFP), HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, FTR_UNSIGNED, ID_AA64ISAR2_EL1_CSSC_IMP, CAP_HWCAP, KERNEL_HWCAP_CSSC), + HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_RPRFM_SHIFT, 4, FTR_UNSIGNED, ID_AA64ISAR2_EL1_RPRFM_IMP, CAP_HWCAP, KERNEL_HWCAP_RPRFM), HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_RPRES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RPRES), HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_WFxT_SHIFT, 4, FTR_UNSIGNED, ID_AA64ISAR2_EL1_WFxT_IMP, CAP_HWCAP, KERNEL_HWCAP_WFXT), #ifdef CONFIG_ARM64_SME diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 3160550c0cc9..85108832d86e 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -117,6 +117,7 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_EBF16] = "ebf16", [KERNEL_HWCAP_SVE_EBF16] = "sveebf16", [KERNEL_HWCAP_CSSC] = "cssc", + [KERNEL_HWCAP_RPRFM] = "rprfm", }; #ifdef CONFIG_COMPAT diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 629d119151bf..a2b2e4c1c3f2 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -489,7 +489,11 @@ Enum 55:52 CSSC 0b0000 NI 0b0001 IMP EndEnum -Res0 51:28 +Enum 51:48 RPRFM + 0b0000 NI + 0b0001 IMP +EndEnum +Res0 47:28 Enum 27:24 PAC_frac 0b0000 NI 0b0001 IMP From 989d37fc3d976f6bfa3f1da484f8f22c57c21c0c Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 17 Oct 2022 16:25:18 +0100 Subject: [PATCH 5/7] kselftest/arm64: Add FEAT_RPRFM to the hwcap test Since the newly added instruction is in the HINT space we can't reasonably test for it actually being present. Signed-off-by: Mark Brown Reviewed-by: Catalin Marinas Link: https://lore.kernel.org/r/20221017152520.1039165-5-broonie@kernel.org Signed-off-by: Will Deacon --- tools/testing/selftests/arm64/abi/hwcap.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c index c7a6b327a7d0..30f87dfd634e 100644 --- a/tools/testing/selftests/arm64/abi/hwcap.c +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -138,6 +138,12 @@ static const struct hwcap_data { .cpuinfo = "rng", .sigill_fn = rng_sigill, }, + { + .name = "RPRFM", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_RPRFM, + .cpuinfo = "rprfm", + }, { .name = "SME", .at_hwcap = AT_HWCAP2, From d12aada8dfb030f7df8ec0bb0ce4bb01a2ab3944 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 17 Oct 2022 16:25:19 +0100 Subject: [PATCH 6/7] arm64/hwcap: Add support for SVE 2.1 FEAT_SVE2p1 introduces a number of new SVE instructions. Since there is no new architectural state added kernel support is simply a new hwcap which lets userspace know that the feature is supported. Signed-off-by: Mark Brown Reviewed-by: Catalin Marinas Link: https://lore.kernel.org/r/20221017152520.1039165-6-broonie@kernel.org Signed-off-by: Will Deacon --- Documentation/arm64/elf_hwcaps.rst | 3 +++ Documentation/arm64/sve.rst | 1 + arch/arm64/include/asm/hwcap.h | 1 + arch/arm64/include/uapi/asm/hwcap.h | 1 + arch/arm64/kernel/cpufeature.c | 1 + arch/arm64/kernel/cpuinfo.c | 1 + arch/arm64/tools/sysreg | 1 + 7 files changed, 9 insertions(+) diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst index a82b2cdff680..6fed84f935df 100644 --- a/Documentation/arm64/elf_hwcaps.rst +++ b/Documentation/arm64/elf_hwcaps.rst @@ -281,6 +281,9 @@ HWCAP2_CSSC HWCAP2_RPRFM Functionality implied by ID_AA64ISAR2_EL1.RPRFM == 0b0001. +HWCAP2_SVE2P1 + Functionality implied by ID_AA64ZFR0_EL1.SVEver == 0b0010. + 4. Unused AT_HWCAP bits ----------------------- diff --git a/Documentation/arm64/sve.rst b/Documentation/arm64/sve.rst index f338ee2df46d..c7a356bf4e8f 100644 --- a/Documentation/arm64/sve.rst +++ b/Documentation/arm64/sve.rst @@ -52,6 +52,7 @@ model features for SVE is included in Appendix A. HWCAP2_SVEBITPERM HWCAP2_SVESHA3 HWCAP2_SVESM4 + HWCAP2_SVE2P1 This list may be extended over time as the SVE architecture evolves. diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index 605ec55cee70..06dd12c514e6 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -122,6 +122,7 @@ #define KERNEL_HWCAP_SVE_EBF16 __khwcap2_feature(SVE_EBF16) #define KERNEL_HWCAP_CSSC __khwcap2_feature(CSSC) #define KERNEL_HWCAP_RPRFM __khwcap2_feature(RPRFM) +#define KERNEL_HWCAP_SVE2P1 __khwcap2_feature(SVE2P1) /* * This yields a mask that user programs can use to figure out what diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 063cc6ea560f..b713d30544f1 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -95,5 +95,6 @@ #define HWCAP2_SVE_EBF16 (1UL << 33) #define HWCAP2_CSSC (1UL << 34) #define HWCAP2_RPRFM (1UL << 35) +#define HWCAP2_SVE2P1 (1UL << 36) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 6e0837212e97..3e3af6b8cc85 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2791,6 +2791,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(SYS_ID_AA64MMFR2_EL1, ID_AA64MMFR2_EL1_AT_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_USCAT), #ifdef CONFIG_ARM64_SVE HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_EL1_SVE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR0_EL1_SVE_IMP, CAP_HWCAP, KERNEL_HWCAP_SVE), + HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_SVEver_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_SVEver_SVE2p1, CAP_HWCAP, KERNEL_HWCAP_SVE2P1), HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_SVEver_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_SVEver_SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2), HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_AES_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_AES_IMP, CAP_HWCAP, KERNEL_HWCAP_SVEAES), HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_AES_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_AES_PMULL128, CAP_HWCAP, KERNEL_HWCAP_SVEPMULL), diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 85108832d86e..379695262b77 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -118,6 +118,7 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_SVE_EBF16] = "sveebf16", [KERNEL_HWCAP_CSSC] = "cssc", [KERNEL_HWCAP_RPRFM] = "rprfm", + [KERNEL_HWCAP_SVE2P1] = "sve2p1", }; #ifdef CONFIG_COMPAT diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index a2b2e4c1c3f2..b2782b8faa01 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -210,6 +210,7 @@ EndEnum Enum 3:0 SVEver 0b0000 IMP 0b0001 SVE2 + 0b0010 SVE2p1 EndEnum EndSysreg From c5195b027d29bbaae0c9668704ab996773788d23 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 17 Oct 2022 16:25:20 +0100 Subject: [PATCH 7/7] kselftest/arm64: Add SVE 2.1 to hwcap test Add coverage for FEAT_SVE2p1. Signed-off-by: Mark Brown Reviewed-by: Catalin Marinas Link: https://lore.kernel.org/r/20221017152520.1039165-7-broonie@kernel.org Signed-off-by: Will Deacon --- tools/testing/selftests/arm64/abi/hwcap.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c index 30f87dfd634e..9f255bc5f31c 100644 --- a/tools/testing/selftests/arm64/abi/hwcap.c +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -62,6 +62,12 @@ static void sve2_sigill(void) asm volatile(".inst 0x4408A000" : : : "z0"); } +static void sve2p1_sigill(void) +{ + /* BFADD Z0.H, Z0.H, Z0.H */ + asm volatile(".inst 0x65000000" : : : "z0"); +} + static void sveaes_sigill(void) { /* AESD z0.b, z0.b, z0.b */ @@ -167,6 +173,13 @@ static const struct hwcap_data { .cpuinfo = "sve2", .sigill_fn = sve2_sigill, }, + { + .name = "SVE 2.1", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SVE2P1, + .cpuinfo = "sve2p1", + .sigill_fn = sve2p1_sigill, + }, { .name = "SVE AES", .at_hwcap = AT_HWCAP2,