From dbd7d67b13184854f483433b1e12d29ad96715c6 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Thu, 16 Apr 2026 10:51:10 +0100 Subject: [PATCH 01/73] MAINTAINERS: Update HiSilicon PMU driver maintainer to Yushan Wang Replace myself with Yushan Wang who is very familiar with the HiSilicon PMU drivers. Signed-off-by: Jonathan Cameron Acked-by: Jie Zhan Acked-by: Yushan Wang Signed-off-by: Will Deacon --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 2fb1c75afd16..2875706afb94 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11625,7 +11625,7 @@ F: Documentation/devicetree/bindings/net/hisilicon*.txt F: drivers/net/ethernet/hisilicon/ HISILICON PMU DRIVER -M: Jonathan Cameron +M: Yushan Wang S: Supported W: http://www.hisilicon.com F: Documentation/admin-guide/perf/hisi-pcie-pmu.rst From 72f7be0c2e303be591ace9312c51d22d7c2d6882 Mon Sep 17 00:00:00 2001 From: eillon Date: Wed, 25 Feb 2026 12:04:17 +0800 Subject: [PATCH 02/73] arm64/sysreg: Add HDBSS related register information The ARM architecture added the HDBSS feature and descriptions of related registers (HDBSSBR/HDBSSPROD) in the DDI0601(ID121123) version, add them to Linux. Signed-off-by: eillon Signed-off-by: Tian Zheng Signed-off-by: Will Deacon --- arch/arm64/include/asm/esr.h | 2 ++ arch/arm64/tools/sysreg | 29 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index 7e86d400864e..81c17320a588 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -160,6 +160,8 @@ #define ESR_ELx_CM (UL(1) << ESR_ELx_CM_SHIFT) /* ISS2 field definitions for Data Aborts */ +#define ESR_ELx_HDBSSF_SHIFT (11) +#define ESR_ELx_HDBSSF (UL(1) << ESR_ELx_HDBSSF_SHIFT) #define ESR_ELx_TnD_SHIFT (10) #define ESR_ELx_TnD (UL(1) << ESR_ELx_TnD_SHIFT) #define ESR_ELx_TagAccess_SHIFT (9) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 6c3ff14e561e..4516971cea06 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -4597,6 +4597,35 @@ Sysreg GCSPR_EL2 3 4 2 5 1 Fields GCSPR_ELx EndSysreg +Sysreg HDBSSBR_EL2 3 4 2 3 2 +Res0 63:56 +Field 55:12 BADDR +Res0 11:4 +Enum 3:0 SZ + 0b0000 4KB + 0b0001 8KB + 0b0010 16KB + 0b0011 32KB + 0b0100 64KB + 0b0101 128KB + 0b0110 256KB + 0b0111 512KB + 0b1000 1MB + 0b1001 2MB +EndEnum +EndSysreg + +Sysreg HDBSSPROD_EL2 3 4 2 3 3 +Res0 63:32 +Enum 31:26 FSC + 0b000000 OK + 0b010000 ExternalAbort + 0b101000 GPF +EndEnum +Res0 25:19 +Field 18:0 INDEX +EndSysreg + Sysreg DACR32_EL2 3 4 3 0 0 Res0 63:32 Field 31:30 D15 From 3ef020a3c9ebd1e3814d900e996fa29ce678c0d0 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Mon, 27 Apr 2026 09:00:57 +0200 Subject: [PATCH 03/73] perf: qcom: Unify user-visible "Qualcomm" name Various names for Qualcomm as a company are used in user-visible config options: QCOM, Qualcomm and Qualcomm Technologies. Switch to unified "Qualcomm" so it will be easier for users to identify the options when for example running menuconfig. Signed-off-by: Krzysztof Kozlowski Signed-off-by: Will Deacon --- drivers/perf/Kconfig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig index ab90932fc2d0..245e7bb763b9 100644 --- a/drivers/perf/Kconfig +++ b/drivers/perf/Kconfig @@ -188,7 +188,7 @@ config FUJITSU_UNCORE_PMU monitoring Uncore events. config QCOM_L2_PMU - bool "Qualcomm Technologies L2-cache PMU" + bool "Qualcomm L2-cache PMU" depends on ARCH_QCOM && ARM64 && ACPI select QCOM_KRYO_L2_ACCESSORS help @@ -198,7 +198,7 @@ config QCOM_L2_PMU monitoring L2 cache events. config QCOM_L3_PMU - bool "Qualcomm Technologies L3-cache PMU" + bool "Qualcomm L3-cache PMU" depends on ARCH_QCOM && ARM64 && ACPI select QCOM_IRQ_COMBINER help From 827ce94e0897a70241abf810b1d3d7d083053a39 Mon Sep 17 00:00:00 2001 From: Leonardo Bras Date: Mon, 27 Apr 2026 15:01:26 +0100 Subject: [PATCH 04/73] arm64/daifflags: Make local_daif_*() helpers __always_inline Make sure those helpers are always inlined and instrumentation safe. Suggested-by: Mark Rutland Signed-off-by: Leonardo Bras Signed-off-by: Will Deacon --- arch/arm64/include/asm/daifflags.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/daifflags.h b/arch/arm64/include/asm/daifflags.h index 5fca48009043..795b35128467 100644 --- a/arch/arm64/include/asm/daifflags.h +++ b/arch/arm64/include/asm/daifflags.h @@ -19,7 +19,7 @@ /* mask/save/unmask/restore all exceptions, including interrupts. */ -static inline void local_daif_mask(void) +static __always_inline void local_daif_mask(void) { WARN_ON(system_has_prio_mask_debugging() && (read_sysreg_s(SYS_ICC_PMR_EL1) == (GIC_PRIO_IRQOFF | @@ -38,7 +38,7 @@ static inline void local_daif_mask(void) trace_hardirqs_off(); } -static inline unsigned long local_daif_save_flags(void) +static __always_inline unsigned long local_daif_save_flags(void) { unsigned long flags; @@ -53,7 +53,7 @@ static inline unsigned long local_daif_save_flags(void) return flags; } -static inline unsigned long local_daif_save(void) +static __always_inline unsigned long local_daif_save(void) { unsigned long flags; @@ -64,7 +64,7 @@ static inline unsigned long local_daif_save(void) return flags; } -static inline void local_daif_restore(unsigned long flags) +static __always_inline void local_daif_restore(unsigned long flags) { bool irq_disabled = flags & PSR_I_BIT; @@ -124,7 +124,7 @@ static inline void local_daif_restore(unsigned long flags) * Called by synchronous exception handlers to restore the DAIF bits that were * modified by taking an exception. */ -static inline void local_daif_inherit(struct pt_regs *regs) +static __always_inline void local_daif_inherit(struct pt_regs *regs) { unsigned long flags = regs->pstate & DAIF_MASK; From 13d0fdc0901614f0d6c9066193bca6fe63a70c6e Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Thu, 30 Apr 2026 06:38:59 +0100 Subject: [PATCH 05/73] arm64/mm: Replace BUG_ON() with VM_WARN_ON_ONCE() Avoid BUG_ON() while checking for inconsistent page table state conditions and instead replace them with VM_WARN_ON_ONCE(). Cc: Catalin Marinas Cc: Will Deacon Cc: Ryan Roberts Cc: David Hildenbrand Cc: linux-kernel@vger.kernel.org Cc: linux-arm-kernel@lists.infradead.org Acked-by: David Hildenbrand (Arm) Suggested-by: David Hildenbrand (Arm) Signed-off-by: Anshuman Khandual Signed-off-by: Will Deacon --- arch/arm64/include/asm/pgtable.h | 4 ++-- arch/arm64/mm/mmu.c | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 4dfa42b7d053..c9e4e00a9af2 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -1007,7 +1007,7 @@ static inline pud_t *p4d_pgtable(p4d_t p4d) static inline phys_addr_t pud_offset_phys(p4d_t *p4dp, unsigned long addr) { - BUG_ON(!pgtable_l4_enabled()); + VM_WARN_ON_ONCE(!pgtable_l4_enabled()); return p4d_page_paddr(READ_ONCE(*p4dp)) + pud_index(addr) * sizeof(pud_t); } @@ -1130,7 +1130,7 @@ static inline p4d_t *pgd_to_folded_p4d(pgd_t *pgdp, unsigned long addr) static inline phys_addr_t p4d_offset_phys(pgd_t *pgdp, unsigned long addr) { - BUG_ON(!pgtable_l5_enabled()); + VM_WARN_ON_ONCE(!pgtable_l5_enabled()); return pgd_page_paddr(READ_ONCE(*pgdp)) + p4d_index(addr) * sizeof(p4d_t); } diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index dd85e093ffdb..4c8959153ac4 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -273,8 +273,8 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, if (ret) return ret; - BUG_ON(pmd_val(old_pmd) != 0 && - pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); + VM_WARN_ON_ONCE(pmd_val(old_pmd) != 0 && + pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); } phys += next - addr; } while (pmdp++, addr = next, addr != end); @@ -394,8 +394,8 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, if (ret) goto out; - BUG_ON(pud_val(old_pud) != 0 && - pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); + VM_WARN_ON_ONCE(pud_val(old_pud) != 0 && + pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); } phys += next - addr; } while (pudp++, addr = next, addr != end); @@ -445,8 +445,8 @@ static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end, if (ret) goto out; - BUG_ON(p4d_val(old_p4d) != 0 && - p4d_val(old_p4d) != READ_ONCE(p4d_val(*p4dp))); + VM_WARN_ON_ONCE(p4d_val(old_p4d) != 0 && + p4d_val(old_p4d) != READ_ONCE(p4d_val(*p4dp))); phys += next - addr; } while (p4dp++, addr = next, addr != end); From c364aa56d6738c8759e88941d7a45a1d6b4c52d0 Mon Sep 17 00:00:00 2001 From: Kevin Brodsky Date: Mon, 27 Apr 2026 13:03:34 +0100 Subject: [PATCH 06/73] selftests/mm: Fix resv_sz when parsing arm64 signal frame get_header() wants the size of the reserved area in struct sigcontext, but instead we pass it the size of the entire struct. This could in theory result in an out-of-bounds read (if the signal frame is malformed). Fix this using one of the existing macros from tools/testing/selftests/arm64/signal/testcases/testcases.h. This issue was reported by Sashiko on a patch that copied this portion of the code. Link: https://sashiko.dev/#/patchset/20260421144252.1440365-1-kevin.brodsky%40arm.com Fixes: f5b5ea51f78f ("selftests: mm: make protection_keys test work on arm64") Signed-off-by: Kevin Brodsky Reviewed-by: Mark Brown Signed-off-by: Will Deacon --- tools/testing/selftests/mm/pkey-arm64.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/mm/pkey-arm64.h b/tools/testing/selftests/mm/pkey-arm64.h index 8e9685e03c44..c5a78a2f211d 100644 --- a/tools/testing/selftests/mm/pkey-arm64.h +++ b/tools/testing/selftests/mm/pkey-arm64.h @@ -130,9 +130,10 @@ static inline u64 get_pkey_bits(u64 reg, int pkey) static inline void aarch64_write_signal_pkey(ucontext_t *uctxt, u64 pkey) { struct _aarch64_ctx *ctx = GET_UC_RESV_HEAD(uctxt); + size_t resv_size = GET_UCP_RESV_SIZE(uctxt); struct poe_context *poe_ctx = (struct poe_context *) get_header(ctx, POE_MAGIC, - sizeof(uctxt->uc_mcontext), NULL); + resv_size, NULL); if (poe_ctx) poe_ctx->por_el0 = pkey; } From 42c21954063e76701b5025664631f553cef16f83 Mon Sep 17 00:00:00 2001 From: Kevin Brodsky Date: Mon, 27 Apr 2026 13:03:35 +0100 Subject: [PATCH 07/73] kselftest/arm64: Add POE as a feature in the signal tests Add the POE feature to the signal tests framework, to allow tests to require it. Reviewed-by: Mark Brown Signed-off-by: Kevin Brodsky Signed-off-by: Will Deacon --- tools/testing/selftests/arm64/signal/test_signals.h | 2 ++ tools/testing/selftests/arm64/signal/test_signals_utils.c | 3 +++ 2 files changed, 5 insertions(+) diff --git a/tools/testing/selftests/arm64/signal/test_signals.h b/tools/testing/selftests/arm64/signal/test_signals.h index ee75a2c25ce7..c7c343494cb8 100644 --- a/tools/testing/selftests/arm64/signal/test_signals.h +++ b/tools/testing/selftests/arm64/signal/test_signals.h @@ -36,6 +36,7 @@ enum { FSME_FA64_BIT, FSME2_BIT, FGCS_BIT, + FPOE_BIT, FMAX_END }; @@ -45,6 +46,7 @@ enum { #define FEAT_SME_FA64 (1UL << FSME_FA64_BIT) #define FEAT_SME2 (1UL << FSME2_BIT) #define FEAT_GCS (1UL << FGCS_BIT) +#define FEAT_POE (1UL << FPOE_BIT) /* * A descriptor used to describe and configure a test case. diff --git a/tools/testing/selftests/arm64/signal/test_signals_utils.c b/tools/testing/selftests/arm64/signal/test_signals_utils.c index 5d3621921cfe..4b12dbd7669d 100644 --- a/tools/testing/selftests/arm64/signal/test_signals_utils.c +++ b/tools/testing/selftests/arm64/signal/test_signals_utils.c @@ -31,6 +31,7 @@ static char const *const feats_names[FMAX_END] = { " FA64 ", " SME2 ", " GCS ", + " POE ", }; #define MAX_FEATS_SZ 128 @@ -341,6 +342,8 @@ int test_init(struct tdescr *td) td->feats_supported |= FEAT_SME2; if (getauxval(AT_HWCAP) & HWCAP_GCS) td->feats_supported |= FEAT_GCS; + if (getauxval(AT_HWCAP2) & HWCAP2_POE) + td->feats_supported |= FEAT_POE; if (feats_ok(td)) { if (td->feats_required & td->feats_supported) fprintf(stderr, From 925a082ec2a05593966ab93fd478153ae6465f18 Mon Sep 17 00:00:00 2001 From: Kevin Brodsky Date: Mon, 27 Apr 2026 13:03:36 +0100 Subject: [PATCH 08/73] kselftest/arm64: Move/add POE helpers to test_signals_utils.h In preparation to adding further POE signal tests, move get_por_el0() to test_signals_utils.h and add set_por_el0(). Reviewed-by: Mark Brown Signed-off-by: Kevin Brodsky Signed-off-by: Will Deacon --- .../selftests/arm64/signal/test_signals_utils.h | 16 ++++++++++++++++ .../arm64/signal/testcases/poe_siginfo.c | 15 --------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tools/testing/selftests/arm64/signal/test_signals_utils.h b/tools/testing/selftests/arm64/signal/test_signals_utils.h index 36fc12b3cd60..2c7b8c64a35a 100644 --- a/tools/testing/selftests/arm64/signal/test_signals_utils.h +++ b/tools/testing/selftests/arm64/signal/test_signals_utils.h @@ -57,6 +57,22 @@ static inline __attribute__((always_inline)) uint64_t get_gcspr_el0(void) return val; } +#define SYS_POR_EL0 "S3_3_C10_C2_4" + +static inline uint64_t get_por_el0(void) +{ + uint64_t val; + + asm volatile("mrs %0, " SYS_POR_EL0 "\n" : "=r"(val)); + + return val; +} + +static inline void set_por_el0(uint64_t val) +{ + asm volatile("msr " SYS_POR_EL0 ", %0\n" :: "r"(val)); +} + static inline bool feats_ok(struct tdescr *td) { if (td->feats_incompatible & td->feats_supported) diff --git a/tools/testing/selftests/arm64/signal/testcases/poe_siginfo.c b/tools/testing/selftests/arm64/signal/testcases/poe_siginfo.c index 36bd9940ee05..e15fedf4da6e 100644 --- a/tools/testing/selftests/arm64/signal/testcases/poe_siginfo.c +++ b/tools/testing/selftests/arm64/signal/testcases/poe_siginfo.c @@ -21,21 +21,6 @@ static union { char buf[1024 * 128]; } context; -#define SYS_POR_EL0 "S3_3_C10_C2_4" - -static uint64_t get_por_el0(void) -{ - uint64_t val; - - asm volatile( - "mrs %0, " SYS_POR_EL0 "\n" - : "=r"(val) - : - : ); - - return val; -} - int poe_present(struct tdescr *td, siginfo_t *si, ucontext_t *uc) { struct _aarch64_ctx *head = GET_BUF_RESV_HEAD(context); From f2db075234c870b4c10673445d8b3fbf19d799cb Mon Sep 17 00:00:00 2001 From: Kevin Brodsky Date: Mon, 27 Apr 2026 13:03:37 +0100 Subject: [PATCH 09/73] kselftest/arm64: Add tests for POR_EL0 save/reset/restore POR_EL0 is expected to be: - Saved in the poe_context record - Reset to POR_EL0_INIT when invoking the signal handler - Restored from poe_context when returning from the signal handler Add a new test, poe_restore, to check that the save/reset/restore mechanism is working as intended. See commit 2e8a1acea859 ("arm64: signal: Improve POR_EL0 handling to avoid uaccess failures") for more details. This commit did not handle the case where poe_context is missing correctly. This was recently fixed; add a new test, poe_missing_poe_context, to check this case. Note: td->pass is only set to true at the very end, as an unexpected signal may occur in case of failure (especially in poe_missing_poe_context if POR_EL0 is restored to an invalid value). Failures are tracked with a global, failed_check. Signed-off-by: Kevin Brodsky Signed-off-by: Will Deacon --- .../testcases/poe_missing_poe_context.c | 73 +++++++++++++++++++ .../arm64/signal/testcases/poe_restore.c | 64 ++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 tools/testing/selftests/arm64/signal/testcases/poe_missing_poe_context.c create mode 100644 tools/testing/selftests/arm64/signal/testcases/poe_restore.c diff --git a/tools/testing/selftests/arm64/signal/testcases/poe_missing_poe_context.c b/tools/testing/selftests/arm64/signal/testcases/poe_missing_poe_context.c new file mode 100644 index 000000000000..3f22d8cf6106 --- /dev/null +++ b/tools/testing/selftests/arm64/signal/testcases/poe_missing_poe_context.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026 Arm Ltd + * + * Verify that the POR_EL0 register is left untouched on sigreturn if the + * POE frame record is missing. + */ + +#include + +#include "test_signals_utils.h" +#include "testcases.h" + +#define POR_EL0_INIT 0x07ul +#define POR_EL0_CUSTOM 0x77ul + +static bool failed_check; + +static bool modify_por_el0(struct tdescr *td) +{ + set_por_el0(POR_EL0_CUSTOM); + + return true; +} + +static int signal_remove_poe_context(struct tdescr *td, siginfo_t *si, + ucontext_t *uc) +{ + struct _aarch64_ctx *ctx = GET_UC_RESV_HEAD(uc); + size_t resv_size = GET_UCP_RESV_SIZE(uc); + struct _aarch64_ctx *poe_ctx_head; + + poe_ctx_head = get_header(ctx, POE_MAGIC, resv_size, NULL); + if (!poe_ctx_head) { + fprintf(stderr, "Missing poe_context record\n"); + failed_check = true; + return 0; + } + + /* + * Actually removing the record would require moving down the next + * records. An easier option is to turn it into an ESR record, which is + * ignored by sigreturn(). + */ + poe_ctx_head->magic = ESR_MAGIC; + + return 0; +} + +static void check_por_el0_preserved(struct tdescr *td) +{ + uint64_t por_el0 = get_por_el0(); + + if (por_el0 == POR_EL0_INIT) { + fprintf(stderr, "POR_EL0 preserved\n"); + } else { + fprintf(stderr, "POR_EL0 unexpectedly set to %lx\n", por_el0); + failed_check = true; + } + + td->pass = !failed_check; +} + +struct tdescr tde = { + .name = "POR_EL0 missing poe_context", + .descr = "Remove poe_context record and check POR_EL0 is preserved", + .feats_required = FEAT_POE, + .timeout = 3, + .sig_trig = SIGUSR1, + .init = modify_por_el0, + .run = signal_remove_poe_context, + .check_result = check_por_el0_preserved, +}; diff --git a/tools/testing/selftests/arm64/signal/testcases/poe_restore.c b/tools/testing/selftests/arm64/signal/testcases/poe_restore.c new file mode 100644 index 000000000000..9f9a61a4214d --- /dev/null +++ b/tools/testing/selftests/arm64/signal/testcases/poe_restore.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026 Arm Ltd + * + * Verify that the POR_EL0 register is saved and restored as expected on signal + * entry/return. + */ + +#include + +#include "test_signals_utils.h" +#include "testcases.h" + +#define POR_EL0_INIT 0x07ul +#define POR_EL0_CUSTOM 0x77ul + +static bool failed_check; + +static bool modify_por_el0(struct tdescr *td) +{ + set_por_el0(POR_EL0_CUSTOM); + + return true; +} + +static int signal_check_por_el0_reset(struct tdescr *td, siginfo_t *si, + ucontext_t *uc) +{ + uint64_t signal_por_el0 = get_por_el0(); + + if (signal_por_el0 != POR_EL0_INIT) { + fprintf(stderr, "POR_EL0 is %lx in signal handler (expected %lx)\n", + signal_por_el0, POR_EL0_INIT); + failed_check = true; + } + + return 0; +} + +static void check_por_el0_restored(struct tdescr *td) +{ + uint64_t por_el0 = get_por_el0(); + + if (por_el0 == POR_EL0_CUSTOM) { + fprintf(stderr, "POR_EL0 restored\n"); + } else { + fprintf(stderr, "POR_EL0 was %lx but is now %lx\n", + POR_EL0_CUSTOM, por_el0); + failed_check = true; + } + + td->pass = !failed_check; +} + +struct tdescr tde = { + .name = "POR_EL0 restore", + .descr = "Validate that POR_EL0 is saved/restored on signal entry/return", + .feats_required = FEAT_POE, + .timeout = 3, + .sig_trig = SIGUSR1, + .init = modify_por_el0, + .run = signal_check_por_el0_reset, + .check_result = check_por_el0_restored, +}; From ede94377d855cff9c729539ef1e0ef525f7318cd Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 1 May 2026 18:52:28 +0100 Subject: [PATCH 10/73] arm64: errata: Reformat table for IDs We have some inconsistency where multiple errata for the same component share the same Kconfig workaround; some are one ID per line, some are smooshed together, and some are entirely separate entries. Standardise on the single entry, one ID per line format so that things render nice and consistently in the HTML docs, and it's simple and clear to add new IDs to existing workarounds without churning the table too much. Acked-by: Catalin Marinas Signed-off-by: Robin Murphy Tested-by: Randy Dunlap Signed-off-by: Will Deacon --- Documentation/arch/arm64/silicon-errata.rst | 47 +++++++++++---------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 211119ce7adc..046a7fa47063 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -116,7 +116,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A73 | #858921 | ARM64_ERRATUM_858921 | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | Cortex-A76 | #1188873,1418040| ARM64_ERRATUM_1418040 | +| ARM | Cortex-A76 | #1188873, | ARM64_ERRATUM_1418040 | +| | | #1418040 | | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A76 | #1165522 | ARM64_ERRATUM_1165522 | +----------------+-----------------+-----------------+-----------------------------+ @@ -136,7 +137,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A78 | #3324344 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | Cortex-A78C | #3324346,3324347| ARM64_ERRATUM_3194386 | +| ARM | Cortex-A78C | #3324346, | ARM64_ERRATUM_3194386 | +| | | #3324347 | | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #2119858 | ARM64_ERRATUM_2119858 | +----------------+-----------------+-----------------+-----------------------------+ @@ -172,11 +174,11 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X925 | #3324334 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | Neoverse-N1 | #1188873,1418040| ARM64_ERRATUM_1418040 | +| ARM | Neoverse-N1 | #1188873, | ARM64_ERRATUM_1418040 | +| | | #1418040 | | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | Neoverse-N1 | #1349291 | N/A | -+----------------+-----------------+-----------------+-----------------------------+ -| ARM | Neoverse-N1 | #1490853 | N/A | +| ARM | Neoverse-N1 | #1349291, | N/A | +| | | #1490853 | | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1542419 | ARM64_ERRATUM_1542419 | +----------------+-----------------+-----------------+-----------------------------+ @@ -204,10 +206,13 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | C1-Pro | #4193714 | ARM64_ERRATUM_4193714 | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | MMU-500 | #841119,826419 | ARM_SMMU_MMU_500_CPRE_ERRATA| -| | | #562869,1047329 | | +| ARM | MMU-500 | #562869, | ARM_SMMU_MMU_500_CPRE_ERRATA| +| | | #841119, | | +| | | #826419, | | +| | | #1047329 | | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | MMU-600 | #1076982,1209401| N/A | +| ARM | MMU-600 | #1076982, | N/A | +| | | #1209401 | | +----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-700 | #2133013, | N/A | | | | #2268618, | | @@ -230,11 +235,13 @@ stable kernels. | Broadcom | Brahma-B53 | N/A | ARM64_ERRATUM_843419 | +----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ -| Cavium | ThunderX ITS | #22375,24313 | CAVIUM_ERRATUM_22375 | +| Cavium | ThunderX ITS | #22375, | CAVIUM_ERRATUM_22375 | +| | | #24313 | | +----------------+-----------------+-----------------+-----------------------------+ | Cavium | ThunderX ITS | #23144 | CAVIUM_ERRATUM_23144 | +----------------+-----------------+-----------------+-----------------------------+ -| Cavium | ThunderX GICv3 | #23154,38545 | CAVIUM_ERRATUM_23154 | +| Cavium | ThunderX GICv3 | #23154, | CAVIUM_ERRATUM_23154 | +| | | #38545 | | +----------------+-----------------+-----------------+-----------------------------+ | Cavium | ThunderX GICv3 | #38539 | N/A | +----------------+-----------------+-----------------+-----------------------------+ @@ -244,9 +251,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | Cavium | ThunderX SMMUv2 | #27704 | N/A | +----------------+-----------------+-----------------+-----------------------------+ -| Cavium | ThunderX2 SMMUv3| #74 | N/A | -+----------------+-----------------+-----------------+-----------------------------+ -| Cavium | ThunderX2 SMMUv3| #126 | N/A | +| Cavium | ThunderX2 SMMUv3| #74, | N/A | +| | | #126 | | +----------------+-----------------+-----------------+-----------------------------+ | Cavium | ThunderX2 Core | #219 | CAVIUM_TX2_ERRATUM_219 | +----------------+-----------------+-----------------+-----------------------------+ @@ -258,11 +264,9 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | T241 GICv3/4.x | T241-FABRIC-4 | N/A | +----------------+-----------------+-----------------+-----------------------------+ -| NVIDIA | T241 MPAM | T241-MPAM-1 | N/A | -+----------------+-----------------+-----------------+-----------------------------+ -| NVIDIA | T241 MPAM | T241-MPAM-4 | N/A | -+----------------+-----------------+-----------------+-----------------------------+ -| NVIDIA | T241 MPAM | T241-MPAM-6 | N/A | +| NVIDIA | T241 MPAM | T241-MPAM-1, | N/A | +| | | T241-MPAM-4, | | +| | | T241-MPAM-6 | | +----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ | Freescale/NXP | LS2080A/LS1043A | A-008585 | FSL_ERRATUM_A008585 | @@ -270,9 +274,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | Hisilicon | Hip0{5,6,7} | #161010101 | HISILICON_ERRATUM_161010101 | +----------------+-----------------+-----------------+-----------------------------+ -| Hisilicon | Hip0{6,7} | #161010701 | N/A | -+----------------+-----------------+-----------------+-----------------------------+ -| Hisilicon | Hip0{6,7} | #161010803 | N/A | +| Hisilicon | Hip0{6,7} | #161010701, | N/A | +| | | #161010803 | | +----------------+-----------------+-----------------+-----------------------------+ | Hisilicon | Hip07 | #161600802 | HISILICON_ERRATUM_161600802 | +----------------+-----------------+-----------------+-----------------------------+ From 243c1d75a01e5fc6cea213e52ea871155cf4946e Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Wed, 13 May 2026 13:57:55 +0200 Subject: [PATCH 11/73] arm64: proton-pack: use sysfs_emit in sysfs show functions Replace sprintf() with sysfs_emit() in sysfs show functions, which is preferred for formatting sysfs output because it provides safer bounds checking. While the current code only emits fixed strings that fit easily within PAGE_SIZE, use sysfs_emit() to follow secure coding best practices. Signed-off-by: Thorsten Blum Signed-off-by: Will Deacon --- arch/arm64/kernel/proton-pack.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index b3801f532b10..7bb6553fec08 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -61,7 +62,7 @@ static void update_mitigation_state(enum mitigation_state *oldp, ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "Mitigation: __user pointer sanitization\n"); + return sysfs_emit(buf, "Mitigation: __user pointer sanitization\n"); } /* @@ -126,7 +127,7 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, switch (spectre_v2_state) { case SPECTRE_UNAFFECTED: if (bhb_state == SPECTRE_UNAFFECTED) - return sprintf(buf, "Not affected\n"); + return sysfs_emit(buf, "Not affected\n"); /* * Platforms affected by Spectre-BHB can't report @@ -136,13 +137,13 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, fallthrough; case SPECTRE_MITIGATED: if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled()) - return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n"); + return sysfs_emit(buf, "Vulnerable: Unprivileged eBPF enabled\n"); - return sprintf(buf, "Mitigation: %s%s\n", v2_str, bhb_str); + return sysfs_emit(buf, "Mitigation: %s%s\n", v2_str, bhb_str); case SPECTRE_VULNERABLE: fallthrough; default: - return sprintf(buf, "Vulnerable\n"); + return sysfs_emit(buf, "Vulnerable\n"); } } @@ -438,13 +439,13 @@ ssize_t cpu_show_spec_store_bypass(struct device *dev, { switch (spectre_v4_state) { case SPECTRE_UNAFFECTED: - return sprintf(buf, "Not affected\n"); + return sysfs_emit(buf, "Not affected\n"); case SPECTRE_MITIGATED: - return sprintf(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n"); + return sysfs_emit(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n"); case SPECTRE_VULNERABLE: fallthrough; default: - return sprintf(buf, "Vulnerable\n"); + return sysfs_emit(buf, "Vulnerable\n"); } } From e7cec385d2c6726e692d057f91f20b08c2981af3 Mon Sep 17 00:00:00 2001 From: Pengjie Zhang Date: Wed, 6 May 2026 17:08:51 +0800 Subject: [PATCH 12/73] arm64: smp: Do not mark secondary CPUs possible under nosmp Under nosmp (maxcpus=0), arm64 never brings up secondary CPUs. smp_prepare_cpus() already treats this as a UP-mandated boot and returns before marking secondary CPUs present. However, smp_init_cpus() may still enumerate firmware-described secondary CPUs and mark them possible before that point. Avoid marking secondary CPUs possible when nosmp/maxcpus=0 is in effect, so that cpu_possible_mask reflects the nosmp boot policy for this boot. Suggested-by: Catalin Marinas Reviewed-by: Catalin Marinas Signed-off-by: Pengjie Zhang Signed-off-by: Will Deacon --- arch/arm64/kernel/smp.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 1aa324104afb..1b63846f646a 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -745,15 +745,21 @@ void __init smp_init_cpus(void) else acpi_parse_and_init_cpus(); - if (cpu_count > nr_cpu_ids) - pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n", - cpu_count, nr_cpu_ids); - if (!bootcpu_valid) { pr_err("missing boot CPU MPIDR, not enabling secondaries\n"); return; } + /* + * For the nosmp/maxcpus=0 case, do not mark the secondary CPUs + * possible. + */ + if (!setup_max_cpus) + return; + + if (cpu_count > nr_cpu_ids) + pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n", + cpu_count, nr_cpu_ids); /* * We need to set the cpu_logical_map entries before enabling * the cpus so that cpu processor description entries (DT cpu nodes From 7dc6922f7fdd3496de4e7d8fb99284fc08f98003 Mon Sep 17 00:00:00 2001 From: Osama Abdelkader Date: Thu, 26 Mar 2026 23:57:52 +0100 Subject: [PATCH 13/73] arm64: panic from init_IRQ if IRQ handler stacks cannot be allocated init_irq_stacks() and init_irq_scs() may fail when arch_alloc_vmap_stack or scs_alloc return NULL. Return -ENOMEM from both and call panic() once from init_IRQ(), covering per-CPU IRQ stacks and shadow IRQ stacks consistently. Signed-off-by: Osama Abdelkader Signed-off-by: Will Deacon --- arch/arm64/kernel/irq.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c index 15dedb385b9e..9fafd826002b 100644 --- a/arch/arm64/kernel/irq.c +++ b/arch/arm64/kernel/irq.c @@ -10,6 +10,7 @@ * Copyright (C) 2012 ARM Ltd. */ +#include #include #include #include @@ -32,34 +33,43 @@ DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts); DEFINE_PER_CPU(unsigned long *, irq_stack_ptr); - DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr); #ifdef CONFIG_SHADOW_CALL_STACK DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr); #endif -static void init_irq_scs(void) +static int __init init_irq_scs(void) { int cpu; + void *s; if (!scs_is_enabled()) - return; + return 0; - for_each_possible_cpu(cpu) - per_cpu(irq_shadow_call_stack_ptr, cpu) = - scs_alloc(early_cpu_to_node(cpu)); + for_each_possible_cpu(cpu) { + s = scs_alloc(early_cpu_to_node(cpu)); + if (!s) + return -ENOMEM; + per_cpu(irq_shadow_call_stack_ptr, cpu) = s; + } + + return 0; } -static void __init init_irq_stacks(void) +static int __init init_irq_stacks(void) { int cpu; unsigned long *p; for_each_possible_cpu(cpu) { p = arch_alloc_vmap_stack(IRQ_STACK_SIZE, early_cpu_to_node(cpu)); + if (!p) + return -ENOMEM; per_cpu(irq_stack_ptr, cpu) = p; } + + return 0; } #ifdef CONFIG_SOFTIRQ_ON_OWN_STACK @@ -109,8 +119,9 @@ int __init set_handle_fiq(void (*handle_fiq)(struct pt_regs *)) void __init init_IRQ(void) { - init_irq_stacks(); - init_irq_scs(); + if (init_irq_stacks() || init_irq_scs()) + panic("Failed to allocate IRQ stack resources\n"); + irqchip_init(); if (system_uses_irq_prio_masking()) { From d54e4fde9de2b8670788fd3bb4daf31cf1cb1622 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Mon, 11 May 2026 22:16:59 +0200 Subject: [PATCH 14/73] arm64: Implement _THIS_IP_ using inline asm Both GCC [1] and Clang [2] consider the generic version of _THIS_IP_ to be broken: #define _THIS_IP_ ({ __label__ __here; __here: (unsigned long)&&__here; }) In particular, the address of a label is only expected to be used with a computed goto. While the generic version more or less works today, it is known to be brittle and may break with current and future optimizations. For example, Clang -O2 always returns 1 when this function is inlined: static inline unsigned long get_ip(void) { return ({ __label__ __here; __here: (unsigned long)&&__here; }); } Fix it by overriding _THIS_IP_ in (which is included by ) using an architecture-specific inline asm version. Additionally, avoiding taking the address of a label prevents compilers from emitting spurious indirect branch targets (e.g. ENDBR or BTI) under control-flow integrity schemes. Link: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=120071 [1] Link: https://github.com/llvm/llvm-project/issues/138272 [2] Signed-off-by: Marco Elver Signed-off-by: Will Deacon --- arch/arm64/include/asm/linkage.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/linkage.h b/arch/arm64/include/asm/linkage.h index 40bd17add539..73eabc82a6bb 100644 --- a/arch/arm64/include/asm/linkage.h +++ b/arch/arm64/include/asm/linkage.h @@ -43,4 +43,6 @@ SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ bti c ; +#define _THIS_IP_ ({ unsigned long __ip; asm volatile("adr %0, ." : "=r" (__ip)); __ip; }) + #endif From 5302bc1493be7e00367bd9aab0cdcb40bf073011 Mon Sep 17 00:00:00 2001 From: Zeng Heng Date: Fri, 8 May 2026 17:23:40 +0100 Subject: [PATCH 15/73] arm64: cpufeature: Add support for the MPAM v0.1 architecture version According to the MPAM spec [1], the supported architecture versions are v1.0, v1.1 and v0.1. MPAM versions v0.1 and v1.1 are functionally identical, but v0.1 additionally supports the FORCE_NS feature. ID_AA64PR | ID_AA64PR | MPAM Extension | Notes F0_EL1. | F1_EL1. | Architecture | MPAM | MPAM_frac | version | --------------------------------------------------------------------------- 0b0000 | 0b0001 | v0.1 | MPAM v0.1 is implemented. | | | MPAM v0.1 is the same as MPAM v1.1 | | | with FORCE_NS which is | | | incompatible with MPAM v1.0. --------------------------------------------------------------------------- 0b0001 | 0b0000 | v1.0 | MPAM v1.0 is implemented. --------------------------------------------------------------------------- 0b0001 | 0b0001 | v1.1 | MPAM v1.1 is implemented. | | | MPAM v1.1 includes all features of | | | MPAM v1.0. | | | It must not include FORCE_NS. FORCE_NS is a feature that operates in EL3 mode. Consequently, the current Linux MPAM driver is also compatible with MPAM v0.1. To support v0.1, the existing driver which only checks ID_AA64PFR0_EL1.MPAM for the major version needs to examine ID_AA64PFR1_EL1.MPAM_frac for the minor version as well. [1] https://developer.arm.com/documentation/ddi0598/db/?lang=en Signed-off-by: Zeng Heng Reviewed-by: James Morse Signed-off-by: James Morse Acked-by: Catalin Marinas Signed-off-by: Will Deacon --- arch/arm64/include/asm/cpufeature.h | 7 +++++++ arch/arm64/include/asm/el2_setup.h | 4 +++- arch/arm64/kernel/cpufeature.c | 15 +++++++++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 4de51f8d92cb..a57870fa96db 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -620,6 +620,13 @@ static inline bool id_aa64pfr0_mpam(u64 pfr0) return val > 0; } +static inline bool id_aa64pfr1_mpamfrac(u64 pfr1) +{ + u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_EL1_MPAM_frac_SHIFT); + + return val > 0; +} + static inline bool id_aa64pfr1_mte(u64 pfr1) { u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_EL1_MTE_SHIFT); diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index 587507a9980e..aa8ec9df8024 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -510,7 +510,9 @@ #endif .macro finalise_el2_state - check_override id_aa64pfr0, ID_AA64PFR0_EL1_MPAM_SHIFT, .Linit_mpam_\@, .Lskip_mpam_\@, x1, x2 + check_override id_aa64pfr0, ID_AA64PFR0_EL1_MPAM_SHIFT, .Linit_mpam_\@, .Lmpam_minor_\@, x1, x2 +.Lmpam_minor_\@: + check_override id_aa64pfr1, ID_AA64PFR1_EL1_MPAM_frac_SHIFT, .Linit_mpam_\@, .Lskip_mpam_\@, x1, x2 .Linit_mpam_\@: mov x0, #MPAM2_EL2_EnMPAMSM_MASK diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 6d53bb15cf7b..0a1d198b3995 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1164,6 +1164,14 @@ static __init void detect_system_supports_pseudo_nmi(void) static inline void detect_system_supports_pseudo_nmi(void) { } #endif +static bool detect_ftr_has_mpam(void) +{ + u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); + u64 pfr1 = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1); + + return id_aa64pfr0_mpam(pfr0) || id_aa64pfr1_mpamfrac(pfr1); +} + void __init init_cpu_features(struct cpuinfo_arm64 *info) { /* Before we start using the tables, make sure it is sorted */ @@ -1211,7 +1219,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) cpacr_restore(cpacr); } - if (id_aa64pfr0_mpam(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) { + if (detect_ftr_has_mpam()) { info->reg_mpamidr = read_cpuid(MPAMIDR_EL1); init_cpu_ftr_reg(SYS_MPAMIDR_EL1, info->reg_mpamidr); } @@ -1467,7 +1475,7 @@ void update_cpu_features(int cpu, cpacr_restore(cpacr); } - if (id_aa64pfr0_mpam(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) { + if (detect_ftr_has_mpam()) { info->reg_mpamidr = read_cpuid(MPAMIDR_EL1); taint |= check_update_ftr_reg(SYS_MPAMIDR_EL1, cpu, info->reg_mpamidr, boot->reg_mpamidr); @@ -2486,7 +2494,7 @@ cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap) static bool test_has_mpam(const struct arm64_cpu_capabilities *entry, int scope) { - if (!has_cpuid_feature(entry, scope)) + if (!detect_ftr_has_mpam()) return false; /* Check firmware actually enabled MPAM on this cpu. */ @@ -3093,7 +3101,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .capability = ARM64_MPAM, .matches = test_has_mpam, .cpu_enable = cpu_enable_mpam, - ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, MPAM, 1) }, { .desc = "Memory Partitioning And Monitoring Virtualisation", From 50a42e03cdbd77d93366d301db3d367dce78eda6 Mon Sep 17 00:00:00 2001 From: Zeng Heng Date: Fri, 8 May 2026 17:23:41 +0100 Subject: [PATCH 16/73] arm_mpam: Update architecture version check for MPAM MSC In addition to updating the CPU MPAM version check, the MPAM MSC version check also need to be updated. mpam_msc_check_aidr() is added to check the MSC AIDR register, ensuring that both the major and minor version numbers fall within the supported range of the MPAM architecture version. Signed-off-by: Zeng Heng [ morse: changed mpam_msc_check_aidr() to accept versions like v1.2 ] Signed-off-by: James Morse Reviewed-by: Ben Horgan Signed-off-by: Will Deacon --- drivers/resctrl/mpam_devices.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 41b14344b16f..83ddbddb2782 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -218,6 +218,24 @@ static inline void _mpam_write_monsel_reg(struct mpam_msc *msc, u16 reg, u32 val #define mpam_write_monsel_reg(msc, reg, val) _mpam_write_monsel_reg(msc, MSMON_##reg, val) +static bool mpam_msc_check_aidr(struct mpam_msc *msc) +{ + u32 aidr = __mpam_read_reg(msc, MPAMF_AIDR); + u32 major = FIELD_GET(MPAMF_AIDR_ARCH_MAJOR_REV, aidr); + u32 minor = FIELD_GET(MPAMF_AIDR_ARCH_MINOR_REV, aidr); + + /* + * v0.0 and >v2.x aren't supported, but anything else should be backward + * compatible to v0.1 or v1.0. + */ + if (!major && !minor) + return false; + if (major > 1) + return false; + + return true; +} + static u64 mpam_msc_read_idr(struct mpam_msc *msc) { u64 idr_high = 0, idr_low; @@ -945,9 +963,8 @@ static int mpam_msc_hw_probe(struct mpam_msc *msc) lockdep_assert_held(&msc->probe_lock); - idr = __mpam_read_reg(msc, MPAMF_AIDR); - if ((idr & MPAMF_AIDR_ARCH_MAJOR_REV) != MPAM_ARCHITECTURE_V1) { - dev_err_once(dev, "MSC does not match MPAM architecture v1.x\n"); + if (!mpam_msc_check_aidr(msc)) { + dev_err_once(dev, "MSC does not match architecture v1.x\n"); return -EIO; } From abca5e69ab6268cbe1913b19da5a98c3383f8bb3 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 18 May 2026 16:07:29 +0100 Subject: [PATCH 17/73] arm64/cpufeature: Define hwcaps for 2025 dpISA features The features added by the 2025 dpISA are all straightforward instruction only features so there is no state to manage, we can just expose hwcaps to let userspace know they are available. F16MM is slightly odd in that the feature is FEAT_F16MM but it is discovered via ID_AA64FPFR0_EL1.F16MM2. We follow the feature name. Signed-off-by: Mark Brown Signed-off-by: Will Deacon --- Documentation/arch/arm64/elf_hwcaps.rst | 24 ++++++++++++++++++++++++ arch/arm64/include/uapi/asm/hwcap.h | 8 ++++++++ arch/arm64/kernel/cpufeature.c | 11 +++++++++++ arch/arm64/kernel/cpuinfo.c | 8 ++++++++ 4 files changed, 51 insertions(+) diff --git a/Documentation/arch/arm64/elf_hwcaps.rst b/Documentation/arch/arm64/elf_hwcaps.rst index 97315ae6c0da..07ff9ea1d605 100644 --- a/Documentation/arch/arm64/elf_hwcaps.rst +++ b/Documentation/arch/arm64/elf_hwcaps.rst @@ -451,6 +451,30 @@ HWCAP3_LS64 of CPU. User should only use ld64b/st64b on supported target (device) memory location, otherwise fallback to the non-atomic alternatives. +HWCAP3_SVE_B16MM + Functionality implied by ID_AA64ZFR0_EL1.B16B16 == 0b0011 + +HWCAP3_SVE2P3 + Functionality implied by ID_AA64ZFR0_EL1.SVEver == 0b0100 + +HWCAP3_SME_LUT6 + Functionality implied by ID_AA64SMFR0_EL1.LUT6 == 0b1 + +HWCAP3_SME2P3 + Functionality implied by ID_AA64SMFR0_EL1.SMEver == 0b0100 + +HWCAP3_F16MM + Functionality implied by ID_AA64FPFR0_EL1.F16MM2 == 0b1 + +HWCAP3_F16F32DOT + Functionality implied by ID_AA64ISAR0_EL1.FHM == 0b0010 + +HWCAP3_F16F32MM + Functionality implied by ID_AA64ISAR0_EL1.FHM == 0b0011 + +HWCAP3_SVE_LUT6 + Functionality implied by ID_AA64ISAR2_EL1.LUT == 0b0010 and + ID_AA64PFR0_EL1.SVE == 0b0001. 4. Unused AT_HWCAP bits ----------------------- diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 06f83ca8de56..10272ddb4d6f 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -147,5 +147,13 @@ #define HWCAP3_MTE_STORE_ONLY (1UL << 1) #define HWCAP3_LSFE (1UL << 2) #define HWCAP3_LS64 (1UL << 3) +#define HWCAP3_SVE_B16MM (1UL << 4) +#define HWCAP3_SVE2P3 (1UL << 5) +#define HWCAP3_SME_LUT6 (1UL << 6) +#define HWCAP3_SME2P3 (1UL << 7) +#define HWCAP3_F16MM (1UL << 8) +#define HWCAP3_F16F32DOT (1UL << 9) +#define HWCAP3_F16F32MM (1UL << 10) +#define HWCAP3_SVE_LUT6 (1UL << 11) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 6d53bb15cf7b..96de16582fca 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -365,6 +365,8 @@ static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = { static const struct arm64_ftr_bits ftr_id_aa64smfr0[] = { ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_FA64_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), + FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_LUT6_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_LUTv2_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), @@ -419,6 +421,7 @@ static const struct arm64_ftr_bits ftr_id_aa64fpfr0[] = { ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8DP2_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8MM8_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8MM4_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F16MM2_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E4M3_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E5M2_SHIFT, 1, 0), ARM64_FTR_END, @@ -3284,6 +3287,8 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64ISAR0_EL1, SM4, IMP, CAP_HWCAP, KERNEL_HWCAP_SM4), HWCAP_CAP(ID_AA64ISAR0_EL1, DP, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDDP), HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDFHM), + HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, F16F32DOT, CAP_HWCAP, KERNEL_HWCAP_F16F32DOT), + HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, F16F32MM, CAP_HWCAP, KERNEL_HWCAP_F16F32MM), HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM, CAP_HWCAP, KERNEL_HWCAP_FLAGM), HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM2, CAP_HWCAP, KERNEL_HWCAP_FLAGM2), HWCAP_CAP(ID_AA64ISAR0_EL1, RNDR, IMP, CAP_HWCAP, KERNEL_HWCAP_RNG), @@ -3313,7 +3318,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64ISAR3_EL1, LSFE, IMP, CAP_HWCAP, KERNEL_HWCAP_LSFE), HWCAP_CAP(ID_AA64MMFR2_EL1, AT, IMP, CAP_HWCAP, KERNEL_HWCAP_USCAT), #ifdef CONFIG_ARM64_SVE + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ISAR2_EL1, LUT, LUT6, CAP_HWCAP, KERNEL_HWCAP_SVE_LUT6), HWCAP_CAP(ID_AA64PFR0_EL1, SVE, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE), + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p3, CAP_HWCAP, KERNEL_HWCAP_SVE2P3), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p2, CAP_HWCAP, KERNEL_HWCAP_SVE2P2), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p1, CAP_HWCAP, KERNEL_HWCAP_SVE2P1), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2), @@ -3323,6 +3330,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BitPerm, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBITPERM), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE_B16B16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, BFSCALE, CAP_HWCAP, KERNEL_HWCAP_SVE_BFSCALE), + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, B16MM, CAP_HWCAP, KERNEL_HWCAP_SVE_B16MM), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBF16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, EBF16, CAP_HWCAP, KERNEL_HWCAP_SVE_EBF16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SHA3, IMP, CAP_HWCAP, KERNEL_HWCAP_SVESHA3), @@ -3362,7 +3370,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { #ifdef CONFIG_ARM64_SME HWCAP_CAP(ID_AA64PFR1_EL1, SME, IMP, CAP_HWCAP, KERNEL_HWCAP_SME), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, FA64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, LUT6, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_LUT6), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, LUTv2, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_LUTV2), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p3, CAP_HWCAP, KERNEL_HWCAP_SME2P3), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p2, CAP_HWCAP, KERNEL_HWCAP_SME2P2), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p1, CAP_HWCAP, KERNEL_HWCAP_SME2P1), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2, CAP_HWCAP, KERNEL_HWCAP_SME2), @@ -3393,6 +3403,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64FPFR0_EL1, F8DP2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8DP2), HWCAP_CAP(ID_AA64FPFR0_EL1, F8MM8, IMP, CAP_HWCAP, KERNEL_HWCAP_F8MM8), HWCAP_CAP(ID_AA64FPFR0_EL1, F8MM4, IMP, CAP_HWCAP, KERNEL_HWCAP_F8MM4), + HWCAP_CAP(ID_AA64FPFR0_EL1, F16MM2, IMP, CAP_HWCAP, KERNEL_HWCAP_F16MM), HWCAP_CAP(ID_AA64FPFR0_EL1, F8E4M3, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E4M3), HWCAP_CAP(ID_AA64FPFR0_EL1, F8E5M2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E5M2), #ifdef CONFIG_ARM64_POE diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 6149bc91251d..d50e2a9b066b 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -164,6 +164,14 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_MTE_FAR] = "mtefar", [KERNEL_HWCAP_MTE_STORE_ONLY] = "mtestoreonly", [KERNEL_HWCAP_LSFE] = "lsfe", + [KERNEL_HWCAP_SVE_B16MM] = "sveb16mm", + [KERNEL_HWCAP_SVE2P3] = "sve2p3", + [KERNEL_HWCAP_SME_LUT6] = "smelut6", + [KERNEL_HWCAP_SME2P3] = "sme2p3", + [KERNEL_HWCAP_F16MM] = "f16mm", + [KERNEL_HWCAP_F16F32DOT] = "f16f32dot", + [KERNEL_HWCAP_F16F32MM] = "f16f32mm", + [KERNEL_HWCAP_SVE_LUT6] = "svelut6", }; #ifdef CONFIG_COMPAT From 7da70f41da81ebe624ac4c222792693be7c395d2 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 18 May 2026 16:07:30 +0100 Subject: [PATCH 18/73] kselftest/arm64: Add 2025 dpISA coverage to hwcaps Add coverage of the new hwcaps to the test program, encodings cross checked against LLVM 22. Signed-off-by: Mark Brown Signed-off-by: Will Deacon --- tools/testing/selftests/arm64/abi/hwcap.c | 116 ++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c index e22703d6b97c..19fca95f7c22 100644 --- a/tools/testing/selftests/arm64/abi/hwcap.c +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -108,6 +108,24 @@ static void f8mm8_sigill(void) asm volatile(".inst 0x6e80ec00"); } +static void f16f32dot_sigill(void) +{ + /* FDOT V0.2S, V0.4H, V0.2H[0] */ + asm volatile(".inst 0xf409000"); +} + +static void f16f32mm_sigill(void) +{ + /* FMMLA V0.4S, V0.8H, V0.8H */ + asm volatile(".inst 0x4e40ec00"); +} + +static void f16mm_sigill(void) +{ + /* FMMLA V0.8H, V0.8H, V0.8H */ + asm volatile(".inst 0x4ec0ec00"); +} + static void faminmax_sigill(void) { /* FAMIN V0.4H, V0.4H, V0.4H */ @@ -191,6 +209,12 @@ static void lut_sigill(void) asm volatile(".inst 0x4e801000"); } +static void sve_lut6_sigill(void) +{ + /* LUTI6 Z0.H, { Z0.H, Z1.H }, Z0[0] */ + asm volatile(".inst 0x4560ac00"); +} + static void mops_sigill(void) { char dst[1], src[1]; @@ -282,6 +306,18 @@ static void sme2p2_sigill(void) asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); } +static void sme2p3_sigill(void) +{ + /* SMSTART SM */ + asm volatile("msr S0_3_C4_C3_3, xzr" : : : ); + + /* ADDQP Z0.B, Z0.B, Z0.B */ + asm volatile(".inst 0x4207800" : : : "z0"); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + static void sme_aes_sigill(void) { /* SMSTART SM */ @@ -378,6 +414,18 @@ static void smef8f32_sigill(void) asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); } +static void smelut6_sigill(void) +{ + /* SMSTART */ + asm volatile("msr S0_3_C4_C7_3, xzr" : : : ); + + /* LUTI6 { Z0.B-Z3.B }, ZT0, { Z0-Z2 } */ + asm volatile(".inst 0xc08a0000" : : : ); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + static void smelutv2_sigill(void) { /* SMSTART */ @@ -486,6 +534,12 @@ static void sve2p2_sigill(void) asm volatile(".inst 0x4cea000" : : : "z0"); } +static void sve2p3_sigill(void) +{ + /* ADDQP Z0.B, Z0.B, Z0.B */ + asm volatile(".inst 0x4207800" : : : "z0"); +} + static void sveaes_sigill(void) { /* AESD z0.b, z0.b, z0.b */ @@ -504,6 +558,12 @@ static void sveb16b16_sigill(void) asm volatile(".inst 0x65000000" : : : ); } +static void sveb16mm_sigill(void) +{ + /* BFMMLA Z0.H, Z0.H, Z0.H */ + asm volatile(".inst 0x64e0e000" : : : ); +} + static void svebfscale_sigill(void) { /* BFSCALE Z0.H, P0/M, Z0.H, Z0.H */ @@ -729,6 +789,27 @@ static const struct hwcap_data { .cpuinfo = "f8mm4", .sigill_fn = f8mm4_sigill, }, + { + .name = "F16MM", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_F16MM, + .cpuinfo = "f16mm", + .sigill_fn = f16mm_sigill, + }, + { + .name = "F16F32DOT", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_F16F32DOT, + .cpuinfo = "f16f32dot", + .sigill_fn = f16f32dot_sigill, + }, + { + .name = "F16F32MM", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_F16F32MM, + .cpuinfo = "f16f32mm", + .sigill_fn = f16f32mm_sigill, + }, { .name = "FAMINMAX", .at_hwcap = AT_HWCAP2, @@ -918,6 +999,13 @@ static const struct hwcap_data { .cpuinfo = "sme2p2", .sigill_fn = sme2p2_sigill, }, + { + .name = "SME 2.3", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_SME2P3, + .cpuinfo = "sme2p3", + .sigill_fn = sme2p3_sigill, + }, { .name = "SME AES", .at_hwcap = AT_HWCAP, @@ -967,6 +1055,13 @@ static const struct hwcap_data { .cpuinfo = "smef8f32", .sigill_fn = smef8f32_sigill, }, + { + .name = "SME LUT6", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_SME_LUT6, + .cpuinfo = "smelut6", + .sigill_fn = smelut6_sigill, + }, { .name = "SME LUTV2", .at_hwcap = AT_HWCAP2, @@ -1052,6 +1147,13 @@ static const struct hwcap_data { .cpuinfo = "sve2p2", .sigill_fn = sve2p2_sigill, }, + { + .name = "SVE 2.3", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_SVE2P3, + .cpuinfo = "sve2p3", + .sigill_fn = sve2p3_sigill, + }, { .name = "SVE AES", .at_hwcap = AT_HWCAP2, @@ -1066,6 +1168,13 @@ static const struct hwcap_data { .cpuinfo = "sveaes2", .sigill_fn = sveaes2_sigill, }, + { + .name = "SVE B16MM", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_SVE_B16MM, + .cpuinfo = "sveb16mm", + .sigill_fn = sveb16mm_sigill, + }, { .name = "SVE BFSCALE", .at_hwcap = AT_HWCAP, @@ -1087,6 +1196,13 @@ static const struct hwcap_data { .cpuinfo = "svef16mm", .sigill_fn = svef16mm_sigill, }, + { + .name = "SVE_LUT6", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_SVE_LUT6, + .cpuinfo = "svelut6", + .sigill_fn = sve_lut6_sigill, + }, { .name = "SVE2 B16B16", .at_hwcap = AT_HWCAP2, From 32e4c96d6a7121cb378ebf9ce2167bb1918eeaa5 Mon Sep 17 00:00:00 2001 From: Ethan Nelson-Moore Date: Sat, 16 May 2026 19:53:42 -0700 Subject: [PATCH 19/73] ARM64: remove unnecessary architecture-specific arch/arm64/include/asm/device.h is identical to include/asm-generic/device.h, and therefore the ARM64-specific version is unnecessary. Remove it. Signed-off-by: Ethan Nelson-Moore Signed-off-by: Will Deacon --- arch/arm64/include/asm/device.h | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 arch/arm64/include/asm/device.h diff --git a/arch/arm64/include/asm/device.h b/arch/arm64/include/asm/device.h deleted file mode 100644 index 996498751318..000000000000 --- a/arch/arm64/include/asm/device.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 ARM Ltd. - */ -#ifndef __ASM_DEVICE_H -#define __ASM_DEVICE_H - -struct dev_archdata { -}; - -struct pdev_archdata { -}; - -#endif From 36ca7f4be8094d1441fc35e00dd581467badf298 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:01:52 +0200 Subject: [PATCH 20/73] arm64: mm: Remove bogus stop condition from map_mem() loop The memblock API guarantees that start is not greater than or equal to end, so there is no need to test it. And if it were, it is doubtful that breaking out of the loop would be a reasonable course of action here (rather than attempting to map the remaining regions) So let's drop this check. Reviewed-by: Ryan Roberts Reviewed-by: Kevin Brodsky Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 4c8959153ac4..96ab9e822bb1 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1173,8 +1173,6 @@ static void __init map_mem(pgd_t *pgdp) /* map all the memory banks */ for_each_mem_range(i, &start, &end) { - if (start >= end) - break; /* * The linear map must allow allocation tags reading/writing * if MTE is present. Otherwise, it has the same attributes as From 2e527667a3b9b17833ea82c81bb9da649eb3e50e Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:01:53 +0200 Subject: [PATCH 21/73] arm64: mm: Drop redundant pgd_t* argument from map_mem() __map_memblock() and map_mem() always operate on swapper_pg_dir, so there is no need to pass around a pgd_t pointer between them. Reviewed-by: Ryan Roberts Reviewed-by: Kevin Brodsky Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 96ab9e822bb1..6cc7f9a51634 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1035,11 +1035,11 @@ static void update_mapping_prot(phys_addr_t phys, unsigned long virt, flush_tlb_kernel_range(virt, virt + size); } -static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start, - phys_addr_t end, pgprot_t prot, int flags) +static void __init __map_memblock(phys_addr_t start, phys_addr_t end, + pgprot_t prot, int flags) { - early_create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, - prot, early_pgtable_alloc, flags); + early_create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start), + end - start, prot, early_pgtable_alloc, flags); } void __init mark_linear_text_alias_ro(void) @@ -1087,13 +1087,13 @@ static phys_addr_t __init arm64_kfence_alloc_pool(void) return kfence_pool; } -static void __init arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) +static void __init arm64_kfence_map_pool(phys_addr_t kfence_pool) { if (!kfence_pool) return; /* KFENCE pool needs page-level mapping. */ - __map_memblock(pgdp, kfence_pool, kfence_pool + KFENCE_POOL_SIZE, + __map_memblock(kfence_pool, kfence_pool + KFENCE_POOL_SIZE, pgprot_tagged(PAGE_KERNEL), NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS); memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE); @@ -1129,11 +1129,11 @@ bool arch_kfence_init_pool(void) #else /* CONFIG_KFENCE */ static inline phys_addr_t arm64_kfence_alloc_pool(void) { return 0; } -static inline void arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) { } +static inline void arm64_kfence_map_pool(phys_addr_t kfence_pool) { } #endif /* CONFIG_KFENCE */ -static void __init map_mem(pgd_t *pgdp) +static void __init map_mem(void) { static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN); phys_addr_t kernel_start = __pa_symbol(_text); @@ -1178,7 +1178,7 @@ static void __init map_mem(pgd_t *pgdp) * if MTE is present. Otherwise, it has the same attributes as * PAGE_KERNEL. */ - __map_memblock(pgdp, start, end, pgprot_tagged(PAGE_KERNEL), + __map_memblock(start, end, pgprot_tagged(PAGE_KERNEL), flags); } @@ -1192,10 +1192,9 @@ static void __init map_mem(pgd_t *pgdp) * Note that contiguous mappings cannot be remapped in this way, * so we should avoid them here. */ - __map_memblock(pgdp, kernel_start, kernel_end, - PAGE_KERNEL, NO_CONT_MAPPINGS); + __map_memblock(kernel_start, kernel_end, PAGE_KERNEL, NO_CONT_MAPPINGS); memblock_clear_nomap(kernel_start, kernel_end - kernel_start); - arm64_kfence_map_pool(early_kfence_pool, pgdp); + arm64_kfence_map_pool(early_kfence_pool); } void mark_rodata_ro(void) @@ -1417,7 +1416,7 @@ static void __init create_idmap(void) void __init paging_init(void) { - map_mem(swapper_pg_dir); + map_mem(); memblock_allow_resize(); From 8dd640d9233dd1fdd649920fe1f7fce78a8b40fa Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:01:54 +0200 Subject: [PATCH 22/73] arm64: mm: Check for pud_/pmd_set_huge() failures on kernel mappings Sashiko reports: | If pmd_set_huge() rejects an unsafe page table transition (such as | mapping a different physical address over an existing block mapping), | it returns 0 and leaves the page table entry unmodified. | | Because *pmdp remains unmodified, READ_ONCE(pmd_val(*pmdp)) will equal | pmd_val(old_pmd). The transition from old_pmd to old_pmd is evaluated | as safe by pgattr_change_is_safe(), so the BUG_ON never triggers. | | This allows invalid and unsafe mapping updates to be silently dropped | instead of panicking, leaving stale memory mappings active while the | caller assumes the update was successful. The same applies to pud_set_huge() in alloc_init_pud(). Given how it is generally preferred to limp on rather than blow up the system if an unexpected condition such as this one occurs, and the fact that there are no known cases where this disparity results in real problems, let's WARN on these failures rather than BUG, allowing the system to survive to the point where it can actually report them. Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 6cc7f9a51634..3b302a0e8f34 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -257,7 +257,7 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, /* try section mapping first */ if (((addr | next | phys) & ~PMD_MASK) == 0 && (flags & NO_BLOCK_MAPPINGS) == 0) { - pmd_set_huge(pmdp, phys, prot); + WARN_ON(!pmd_set_huge(pmdp, phys, prot)); /* * After the PMD entry has been populated once, we @@ -380,7 +380,7 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, if (pud_sect_supported() && ((addr | next | phys) & ~PUD_MASK) == 0 && (flags & NO_BLOCK_MAPPINGS) == 0) { - pud_set_huge(pudp, phys, prot); + WARN_ON(!pud_set_huge(pudp, phys, prot)); /* * After the PUD entry has been populated once, we From a64293e993f6ee6b5dcf107d3a7aa8c6ccca029c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:01:55 +0200 Subject: [PATCH 23/73] arm64: mm: Preserve existing table mappings when mapping DRAM Instead of blindly overwriting an existing table entry when mapping DRAM regions, take care not to replace a pre-existing table entry with a block entry. This permits the logic of mapping the kernel's linear alias to be simplified in a subsequent patch. Reviewed-by: Ryan Roberts Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 3b302a0e8f34..1db44adc8717 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -256,7 +256,8 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, /* try section mapping first */ if (((addr | next | phys) & ~PMD_MASK) == 0 && - (flags & NO_BLOCK_MAPPINGS) == 0) { + (flags & NO_BLOCK_MAPPINGS) == 0 && + !pmd_table(old_pmd)) { WARN_ON(!pmd_set_huge(pmdp, phys, prot)); /* @@ -379,7 +380,8 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, */ if (pud_sect_supported() && ((addr | next | phys) & ~PUD_MASK) == 0 && - (flags & NO_BLOCK_MAPPINGS) == 0) { + (flags & NO_BLOCK_MAPPINGS) == 0 && + !pud_table(old_pud)) { WARN_ON(!pud_set_huge(pudp, phys, prot)); /* From ecda73ae92cab57611037cec8d29dd6f2ca68fe2 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:01:56 +0200 Subject: [PATCH 24/73] arm64: mm: Preserve non-contiguous descriptors when mapping DRAM Instead of blindly overwriting existing live entries regardless of the value of their contiguous bit when mapping DRAM regions at contiguous-hint granularity, check whether the contiguous region in question contains any valid descriptors that have the contiguous bit cleared, and in that case, leave the contiguous bit unset on the entire region. This permits the logic of mapping the kernel's linear alias to be simplified in a subsequent patch. Note that this can only result in a misprogrammed contiguous bit (as per ARM ARM RNGLXZ) if the region in question already contains a mix of valid contiguous and valid non-contiguous descriptors, in which case it was already misprogrammed to begin with. Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/include/asm/pgtable.h | 4 ++++ arch/arm64/mm/mmu.c | 22 ++++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index c9e4e00a9af2..491ba0a6492d 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -181,6 +181,10 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) * Returns true if the pte is valid and has the contiguous bit set. */ #define pte_valid_cont(pte) (pte_valid(pte) && pte_cont(pte)) +/* + * Returns true if the pte is valid and has the contiguous bit cleared. + */ +#define pte_valid_noncont(pte) (pte_valid(pte) && !pte_cont(pte)) /* * Could the pte be present in the TLB? We must check mm_tlb_flush_pending * so that we don't erroneously return false for pages that have been diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 1db44adc8717..1f2ec06a8736 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -187,6 +187,14 @@ static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end, } while (ptep++, addr += PAGE_SIZE, addr != end); } +static bool pte_range_has_valid_noncont(pte_t *ptep) +{ + for (int i = 0; i < CONT_PTES; i++) + if (pte_valid_noncont(__ptep_get(&ptep[i]))) + return true; + return false; +} + static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, @@ -224,7 +232,8 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, /* use a contiguous mapping if the range is suitably aligned */ if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) && - (flags & NO_CONT_MAPPINGS) == 0) + (flags & NO_CONT_MAPPINGS) == 0 && + !pte_range_has_valid_noncont(ptep)) __prot = __pgprot(pgprot_val(prot) | PTE_CONT); init_pte(ptep, addr, next, phys, __prot); @@ -283,6 +292,14 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, return 0; } +static bool pmd_range_has_valid_noncont(pmd_t *pmdp) +{ + for (int i = 0; i < CONT_PMDS; i++) + if (pte_valid_noncont(pmd_pte(READ_ONCE(pmdp[i])))) + return true; + return false; +} + static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, @@ -324,7 +341,8 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, /* use a contiguous mapping if the range is suitably aligned */ if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) && - (flags & NO_CONT_MAPPINGS) == 0) + (flags & NO_CONT_MAPPINGS) == 0 && + !pmd_range_has_valid_noncont(pmdp)) __prot = __pgprot(pgprot_val(prot) | PTE_CONT); ret = init_pmd(pmdp, addr, next, phys, __prot, pgtable_alloc, flags); From 05c5c31e9d8d3930d3852018de154bcdb9eeb817 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:01:57 +0200 Subject: [PATCH 25/73] arm64: mm: Permit contiguous descriptors to be manipulated Currently, pgattr_change_is_safe() is overly pedantic when it comes to descriptors with the contiguous hint attribute set, as it rejects assignments even if the old and the new value are the same. In fact, as per ARM ARM RJQQTC, manipulating descriptors with the contiguous bit set is safe as long as the bit itself does not change value, in the sense that no TLB conflict aborts or other exceptions may be raised as a result. Inconsistent permission attributes within the contiguous region may result in any of the alternatives to be taken to apply to the entire region, which might be a programming error, but it does not constitute an unsafe manipulation in terms of what pgattr_change_is_safe() is intended to detect. So drop the special PTE_CONT check, but still omit PTE_CONT from 'mask' so that modifying the bit is still regarded as unsafe. Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 1f2ec06a8736..da81444015dc 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -134,10 +134,6 @@ bool pgattr_change_is_safe(pteval_t old, pteval_t new) if (pte_pfn(__pte(old)) != pte_pfn(__pte(new))) return false; - /* live contiguous mappings may not be manipulated at all */ - if ((old | new) & PTE_CONT) - return false; - /* Transitioning from Non-Global to Global is unsafe */ if (old & ~new & PTE_NG) return false; From dfd73e574d389f3df4e6e39157b22e71a978db34 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:01:58 +0200 Subject: [PATCH 26/73] arm64: kfence: Avoid NOMAP tricks when mapping the early pool Now that the map_mem() routines respect existing page mappings and contiguous granule sized blocks with the contiguous bit cleared, there is no longer a reason to play tricks with the memblock NOMAP attribute. Instead, the kfence pool can be allocated and mapped with page granularity first, and this granularity will be respected when the rest of DRAM is mapped later, even if block and contiguous mappings are allowed for the remainder of those mappings. Add the NO_EXEC_MAPPINGS flag to ensure that hierarchical XN attributes are set on the intermediate page tables that are allocated when mapping the pool. Signed-off-by: Ard Biesheuvel Reviewed-by: Kevin Brodsky Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index da81444015dc..661ff3c1fe10 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1083,36 +1083,24 @@ static int __init parse_kfence_early_init(char *arg) } early_param("kfence.sample_interval", parse_kfence_early_init); -static phys_addr_t __init arm64_kfence_alloc_pool(void) +static void __init arm64_kfence_map_pool(void) { phys_addr_t kfence_pool; if (!kfence_early_init) - return 0; + return; kfence_pool = memblock_phys_alloc(KFENCE_POOL_SIZE, PAGE_SIZE); if (!kfence_pool) { pr_err("failed to allocate kfence pool\n"); kfence_early_init = false; - return 0; - } - - /* Temporarily mark as NOMAP. */ - memblock_mark_nomap(kfence_pool, KFENCE_POOL_SIZE); - - return kfence_pool; -} - -static void __init arm64_kfence_map_pool(phys_addr_t kfence_pool) -{ - if (!kfence_pool) return; + } /* KFENCE pool needs page-level mapping. */ __map_memblock(kfence_pool, kfence_pool + KFENCE_POOL_SIZE, pgprot_tagged(PAGE_KERNEL), - NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS); - memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE); + NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS | NO_EXEC_MAPPINGS); __kfence_pool = phys_to_virt(kfence_pool); } @@ -1144,8 +1132,7 @@ bool arch_kfence_init_pool(void) } #else /* CONFIG_KFENCE */ -static inline phys_addr_t arm64_kfence_alloc_pool(void) { return 0; } -static inline void arm64_kfence_map_pool(phys_addr_t kfence_pool) { } +static inline void arm64_kfence_map_pool(void) { } #endif /* CONFIG_KFENCE */ @@ -1155,7 +1142,6 @@ static void __init map_mem(void) phys_addr_t kernel_start = __pa_symbol(_text); phys_addr_t kernel_end = __pa_symbol(__init_begin); phys_addr_t start, end; - phys_addr_t early_kfence_pool; int flags = NO_EXEC_MAPPINGS; u64 i; @@ -1172,7 +1158,7 @@ static void __init map_mem(void) BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end) && pgd_index(_PAGE_OFFSET(VA_BITS_MIN)) != PTRS_PER_PGD - 1); - early_kfence_pool = arm64_kfence_alloc_pool(); + arm64_kfence_map_pool(); linear_map_requires_bbml2 = !force_pte_mapping() && can_set_direct_map(); @@ -1210,7 +1196,6 @@ static void __init map_mem(void) */ __map_memblock(kernel_start, kernel_end, PAGE_KERNEL, NO_CONT_MAPPINGS); memblock_clear_nomap(kernel_start, kernel_end - kernel_start); - arm64_kfence_map_pool(early_kfence_pool); } void mark_rodata_ro(void) From 28becb2c1d741259b2f12737e0a3828fe59700d9 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:01:59 +0200 Subject: [PATCH 27/73] arm64: mm: Permit contiguous attribute for preliminary mappings There are a few cases where we omit the contiguous hint for mappings that start out as read-write and are remapped read-only later, on the basis that manipulating live descriptors with the PTE_CONT attribute set is unsafe. When support for the contiguous hint was added to the code, the ARM ARM was ambiguous about this, and so we erred on the side of caution. In the meantime, this has been clarified [0], and regions that will be remapped in their entirety, retaining the contiguous bit on all entries, can use the contiguous hint both in the initial mapping as well as the one that replaces it. Note that this requires that the logic that may be called to remap overlapping regions respects existing valid descriptors that have the contiguous bit cleared. So omit the NO_CONT_MAPPINGS flag in places where it is unneeded. [0] RJQQTC For a TLB lookup in a contiguous region mapped by translation table entries that have consistent values for the Contiguous bit, but have the OA, attributes, or permissions misprogrammed, that TLB lookup is permitted to produce an OA, access permissions, and memory attributes that are consistent with any one of the programmed translation table values. Reviewed-by: Kevin Brodsky Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 661ff3c1fe10..ff9e74045308 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1016,8 +1016,7 @@ void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, &phys, virt); return; } - early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, - NO_CONT_MAPPINGS); + early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 0); } void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, @@ -1044,8 +1043,7 @@ static void update_mapping_prot(phys_addr_t phys, unsigned long virt, return; } - early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, - NO_CONT_MAPPINGS); + early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 0); /* flush the TLBs after updating live kernel mappings */ flush_tlb_kernel_range(virt, virt + size); @@ -1191,10 +1189,8 @@ static void __init map_mem(void) * alternative patching has completed). This makes the contents * of the region accessible to subsystems such as hibernate, * but protects it from inadvertent modification or execution. - * Note that contiguous mappings cannot be remapped in this way, - * so we should avoid them here. */ - __map_memblock(kernel_start, kernel_end, PAGE_KERNEL, NO_CONT_MAPPINGS); + __map_memblock(kernel_start, kernel_end, PAGE_KERNEL, 0); memblock_clear_nomap(kernel_start, kernel_end - kernel_start); } From 382a03e12ebad387fad616da78b99720ea3ee683 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:02:00 +0200 Subject: [PATCH 28/73] arm64: Move fixmap and kasan page tables to end of kernel image Move the fixmap and kasan page tables out of the BSS section, and place them at the end of the image, right before the init_pg_dir section where some of the other statically allocated page tables live. These page tables are currently the only data objects in vmlinux that are meant to be accessed via the kernel image's linear alias, and so placing them together allows the remainder of the data/bss section to be remapped read-only or unmapped entirely. Reviewed-by: Kevin Brodsky Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/include/asm/mmu.h | 2 ++ arch/arm64/kernel/vmlinux.lds.S | 8 +++++++- arch/arm64/mm/fixmap.c | 6 +++--- arch/arm64/mm/kasan_init.c | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 5e1211c540ab..fb95754f2876 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -13,6 +13,8 @@ #ifndef __ASSEMBLER__ +#define __pgtbl_bss __section(".pgdir.bss") __aligned(PAGE_SIZE) + #include #include diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index e1ac876200a3..2b0ebfb30c63 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -349,9 +349,15 @@ SECTIONS _edata = .; /* start of zero-init region */ - BSS_SECTION(SBSS_ALIGN, 0, 0) + BSS_SECTION(SBSS_ALIGN, 0, PAGE_SIZE) __pi___bss_start = __bss_start; + /* fixmap BSS starts here - preceding data/BSS is omitted from the linear map */ + .pgdir.bss (NOLOAD) : ALIGN(PAGE_SIZE) { + *(.pgdir.bss) + } + ASSERT(ADDR(.pgdir.bss) == __bss_stop, ".pgdir.bss must follow BSS") + . = ALIGN(PAGE_SIZE); __pi_init_pg_dir = .; . += INIT_DIR_SIZE; diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c index c5c5425791da..1a3bbd67dd76 100644 --- a/arch/arm64/mm/fixmap.c +++ b/arch/arm64/mm/fixmap.c @@ -31,9 +31,9 @@ static_assert(NR_BM_PMD_TABLES == 1); #define BM_PTE_TABLE_IDX(addr) __BM_TABLE_IDX(addr, PMD_SHIFT) -static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __page_aligned_bss; -static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; -static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; +static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __pgtbl_bss; +static pmd_t bm_pmd[PTRS_PER_PMD] __pgtbl_bss __maybe_unused; +static pud_t bm_pud[PTRS_PER_PUD] __pgtbl_bss __maybe_unused; static inline pte_t *fixmap_pte(unsigned long addr) { diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c index abeb81bf6ebd..dbf22cae82ee 100644 --- a/arch/arm64/mm/kasan_init.c +++ b/arch/arm64/mm/kasan_init.c @@ -214,7 +214,7 @@ asmlinkage void __init kasan_early_init(void) * shadow pud_t[]/p4d_t[], which could end up getting corrupted * when the linear region is mapped. */ - static pte_t tbl[PTRS_PER_PTE] __page_aligned_bss; + static pte_t tbl[PTRS_PER_PTE] __pgtbl_bss; pgd_t *pgdp = pgd_offset_k(KASAN_SHADOW_START); set_pgd(pgdp, __pgd(__pa_symbol(tbl) | PGD_TYPE_TABLE)); From d672a4b72c9542911244b0bc5d5c14c77251159c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:02:01 +0200 Subject: [PATCH 29/73] arm64: mm: Don't abuse memblock NOMAP to check for overlaps Now that the linear region mapping routines respect existing table mappings and contiguous block and page mappings, it is no longer needed to fiddle with the memblock tables to set and clear the NOMAP attribute in order to omit text and rodata when creating the linear map. Instead, map the kernel text and rodata alias first with the desired initial attributes and granularity, so that the loop iterating over the memblocks will not remap it in a manner that prevents it from being remapped with updated attributes later. Signed-off-by: Ard Biesheuvel Reviewed-by: Kevin Brodsky Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index ff9e74045308..6b6b2dac41f5 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1164,12 +1164,17 @@ static void __init map_mem(void) flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; /* - * Take care not to create a writable alias for the - * read-only text and rodata sections of the kernel image. - * So temporarily mark them as NOMAP to skip mappings in - * the following for-loop + * Map the linear alias of the [_text, __init_begin) interval first + * so that its write permissions can be removed later without the need + * to split any block mappings created by the loop below. + * + * Write permissions are needed for alternatives patching, and will be + * removed later by mark_linear_text_alias_ro() above. This makes the + * contents of the region accessible to subsystems such as hibernate, + * but protects it from inadvertent modification or execution. */ - memblock_mark_nomap(kernel_start, kernel_end - kernel_start); + __map_memblock(kernel_start, kernel_end, pgprot_tagged(PAGE_KERNEL), + flags); /* map all the memory banks */ for_each_mem_range(i, &start, &end) { @@ -1181,17 +1186,6 @@ static void __init map_mem(void) __map_memblock(start, end, pgprot_tagged(PAGE_KERNEL), flags); } - - /* - * Map the linear alias of the [_text, __init_begin) interval - * as non-executable now, and remove the write permission in - * mark_linear_text_alias_ro() below (which will be called after - * alternative patching has completed). This makes the contents - * of the region accessible to subsystems such as hibernate, - * but protects it from inadvertent modification or execution. - */ - __map_memblock(kernel_start, kernel_end, PAGE_KERNEL, 0); - memblock_clear_nomap(kernel_start, kernel_end - kernel_start); } void mark_rodata_ro(void) From c0693153fb17069f536548396c5a7946692b8948 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:02:02 +0200 Subject: [PATCH 30/73] powerpc/code-patching: Avoid r/w mapping of the zero page The only remaining use of map_patch_area() is mapping the zero page, and immediately unmapping it again so that the intermediate page table levels are all guaranteed to be populated. The use of the zero page here is completely arbitrary, and not harmful per se, but currently, it creates a writable mapping, and does so in a manner that requires that the empty_zero_page[] symbol is not const-qualified. Given that this is about to change, and that map_patch_area() now never maps anything other than the zero page, let's simplify the code and - remove the helpers and call [un]map_kernel_page() directly - take the PA of empty_zero_page directly - create a read-only temporary mapping. This allows empty_zero_page[] to be repainted as const u8[] in a subsequent patch, without making substantial changes to this code patching logic. Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Christophe Leroy (CS GROUP) Link: https://lore.kernel.org/all/20260520085423.485402-1-ardb@kernel.org/ Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/powerpc/lib/code-patching.c | 52 ++------------------------------ 1 file changed, 2 insertions(+), 50 deletions(-) diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c index f84e0337cc02..44ff9f684bef 100644 --- a/arch/powerpc/lib/code-patching.c +++ b/arch/powerpc/lib/code-patching.c @@ -60,9 +60,6 @@ struct patch_context { static DEFINE_PER_CPU(struct patch_context, cpu_patching_context); -static int map_patch_area(void *addr, unsigned long text_poke_addr); -static void unmap_patch_area(unsigned long addr); - static bool mm_patch_enabled(void) { return IS_ENABLED(CONFIG_SMP) && radix_enabled(); @@ -117,11 +114,11 @@ static int text_area_cpu_up(unsigned int cpu) // Map/unmap the area to ensure all page tables are pre-allocated addr = (unsigned long)area->addr; - err = map_patch_area(empty_zero_page, addr); + err = map_kernel_page(addr, __pa_symbol(empty_zero_page), PAGE_KERNEL_RO); if (err) return err; - unmap_patch_area(addr); + unmap_kernel_page(addr); this_cpu_write(cpu_patching_context.area, area); this_cpu_write(cpu_patching_context.addr, addr); @@ -233,51 +230,6 @@ static unsigned long get_patch_pfn(void *addr) return __pa_symbol(addr) >> PAGE_SHIFT; } -/* - * This can be called for kernel text or a module. - */ -static int map_patch_area(void *addr, unsigned long text_poke_addr) -{ - unsigned long pfn = get_patch_pfn(addr); - - return map_kernel_page(text_poke_addr, (pfn << PAGE_SHIFT), PAGE_KERNEL); -} - -static void unmap_patch_area(unsigned long addr) -{ - pte_t *ptep; - pmd_t *pmdp; - pud_t *pudp; - p4d_t *p4dp; - pgd_t *pgdp; - - pgdp = pgd_offset_k(addr); - if (WARN_ON(pgd_none(*pgdp))) - return; - - p4dp = p4d_offset(pgdp, addr); - if (WARN_ON(p4d_none(*p4dp))) - return; - - pudp = pud_offset(p4dp, addr); - if (WARN_ON(pud_none(*pudp))) - return; - - pmdp = pmd_offset(pudp, addr); - if (WARN_ON(pmd_none(*pmdp))) - return; - - ptep = pte_offset_kernel(pmdp, addr); - if (WARN_ON(pte_none(*ptep))) - return; - - /* - * In hash, pte_clear flushes the tlb, in radix, we have to - */ - pte_clear(&init_mm, addr, ptep); - flush_tlb_kernel_range(addr, addr + PAGE_SIZE); -} - static int __do_patch_mem_mm(void *addr, unsigned long val, bool is_dword) { int err; From 99bad3e992e26addd2a7d779cfbe42c048abd808 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:02:03 +0200 Subject: [PATCH 31/73] sh: Drop cache flush of the zero page at boot SuperH performs cache maintenance on the zero page during boot, presumably because before commit 6215d9f4470f ("arch, mm: consolidate empty_zero_page") the zero page did double duty as a boot params region, and was cleared separately, as it was not part of BSS. The memset() in question was dropped by that commit, but the __flush_wback_region() call remained. As empty_zero_page[] has been moved to BSS, it can be treated as any other BSS memory, and so the cache flush can be dropped. Cc: Yoshinori Sato Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Mike Rapoport Cc: Geert Uytterhoeven Signed-off-by: Ard Biesheuvel Reviewed-by: Geert Uytterhoeven Acked-by: Mike Rapoport (Microsoft) Signed-off-by: Will Deacon --- arch/sh/mm/init.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index 4e40d5e96be9..110308bdef01 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -331,9 +331,6 @@ void __init mem_init(void) /* Set this up early, so we can take care of the zero page */ cpu_cache_init(); - /* clear the zero-page */ - __flush_wback_region(empty_zero_page, PAGE_SIZE); - vsyscall_init(); pr_info("virtual kernel memory layout:\n" From 0aae825f1ed7ce3eedfadc54684aa86bbbe188b0 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:02:04 +0200 Subject: [PATCH 32/73] mm: Make empty_zero_page[] const The empty zero page is used to back any kernel or user space mapping that is supposed to remain cleared, and so the page itself is never supposed to be modified. So mark it as const, which moves it into .rodata rather than .bss: on most architectures, this ensures that both the kernel's mapping of it and any aliases that are accessible via the kernel direct (linear) map are mapped read-only, and cannot be used (inadvertently or maliciously) to corrupt the contents of the zero page. Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Kevin Brodsky Acked-by: David Hildenbrand (Arm) Reviewed-by: Jann Horn Reviewed-by: Feng Tang Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- include/linux/pgtable.h | 2 +- mm/mm_init.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index cdd68ed3ae1a..67aa23814010 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1993,7 +1993,7 @@ static inline unsigned long zero_pfn(unsigned long addr) return zero_page_pfn; } -extern uint8_t empty_zero_page[PAGE_SIZE]; +extern const uint8_t empty_zero_page[PAGE_SIZE]; extern struct page *__zero_page; static inline struct page *_zero_page(unsigned long addr) diff --git a/mm/mm_init.c b/mm/mm_init.c index f9f8e1af921c..46cf001238c5 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -57,7 +57,7 @@ unsigned long zero_page_pfn __ro_after_init; EXPORT_SYMBOL(zero_page_pfn); #ifndef __HAVE_COLOR_ZERO_PAGE -uint8_t empty_zero_page[PAGE_SIZE] __page_aligned_bss; +const uint8_t empty_zero_page[PAGE_SIZE] __aligned(PAGE_SIZE); EXPORT_SYMBOL(empty_zero_page); struct page *__zero_page __ro_after_init; From f2ba877402e5f74b27d9dbc2c8d059e7e9daf500 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:02:05 +0200 Subject: [PATCH 33/73] arm64: mm: Map the kernel data/bss read-only in the linear map On systems where the bootloader adheres to the original arm64 boot protocol, the placement of the kernel in the physical address space is highly predictable, and this makes the placement of its linear alias in the kernel virtual address space equally predictable, given the lack of randomization of the linear map. The linear aliases of the kernel text and rodata regions are already mapped read-only, but the kernel data and bss are mapped read-write in this region. This is not needed, so map them read-only as well. Note that the statically allocated kernel page tables do need to be modifiable via the linear map, so leave these mapped read-write. Reviewed-by: Kevin Brodsky Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 6b6b2dac41f5..7f7693a5dfff 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1138,7 +1138,9 @@ static void __init map_mem(void) { static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN); phys_addr_t kernel_start = __pa_symbol(_text); - phys_addr_t kernel_end = __pa_symbol(__init_begin); + phys_addr_t init_begin = __pa_symbol(__init_begin); + phys_addr_t init_end = __pa_symbol(__init_end); + phys_addr_t kernel_end = __pa_symbol(__bss_stop); phys_addr_t start, end; int flags = NO_EXEC_MAPPINGS; u64 i; @@ -1173,7 +1175,11 @@ static void __init map_mem(void) * contents of the region accessible to subsystems such as hibernate, * but protects it from inadvertent modification or execution. */ - __map_memblock(kernel_start, kernel_end, pgprot_tagged(PAGE_KERNEL), + __map_memblock(kernel_start, init_begin, pgprot_tagged(PAGE_KERNEL), + flags); + + /* Map the kernel data/bss so it can be remapped later */ + __map_memblock(init_end, kernel_end, pgprot_tagged(PAGE_KERNEL), flags); /* map all the memory banks */ @@ -1186,6 +1192,11 @@ static void __init map_mem(void) __map_memblock(start, end, pgprot_tagged(PAGE_KERNEL), flags); } + + /* Map the kernel data/bss read-only in the linear map */ + __map_memblock(init_end, kernel_end, PAGE_KERNEL_RO, flags); + flush_tlb_kernel_range((unsigned long)lm_alias(__init_end), + (unsigned long)lm_alias(__bss_stop)); } void mark_rodata_ro(void) From 63e0b6a5b6934d6a919d1c65ea185303200a1874 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:02:06 +0200 Subject: [PATCH 34/73] arm64: mm: Unmap kernel data/bss entirely from the linear map The linear aliases of the kernel text and rodata are also mapped read-only in the linear map. Given that the contents of these regions are mostly identical to the version in the loadable image, mapping them read-only and leaving their contents visible is a reasonable hardening measure. Data and bss, however, are now also mapped read-only but the contents of these regions are more likely to contain data that we'd rather not leak. So let's unmap these entirely in the linear map when the kernel is running normally. When going into hibernation or waking up from it, these regions need to be mapped, so map the region initially, and toggle the valid bit so map/unmap the region as needed. Doing so is required because pages covering the kernel image are marked as PageReserved, and therefore disregarded for snapshotting by the hibernate logic unless they are mapped. Signed-off-by: Ard Biesheuvel Reviewed-by: Kevin Brodsky Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 45 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 7f7693a5dfff..be51f6cac86f 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -1056,6 +1057,29 @@ static void __init __map_memblock(phys_addr_t start, phys_addr_t end, end - start, prot, early_pgtable_alloc, flags); } +static void mark_linear_data_alias_valid(bool valid) +{ + set_memory_valid((unsigned long)lm_alias(__init_end), + (unsigned long)(__bss_stop - __init_end) / PAGE_SIZE, + valid); +} + +static int arm64_hibernate_pm_notify(struct notifier_block *nb, + unsigned long mode, void *unused) +{ + switch (mode) { + default: + break; + case PM_POST_HIBERNATION: + mark_linear_data_alias_valid(false); + break; + case PM_HIBERNATION_PREPARE: + mark_linear_data_alias_valid(true); + break; + } + return 0; +} + void __init mark_linear_text_alias_ro(void) { /* @@ -1064,6 +1088,21 @@ void __init mark_linear_text_alias_ro(void) update_mapping_prot(__pa_symbol(_text), (unsigned long)lm_alias(_text), (unsigned long)__init_begin - (unsigned long)_text, PAGE_KERNEL_RO); + + /* + * Register a PM notifier to remap the linear alias of data/bss as + * valid read-only before hibernation. This is needed because the + * snapshot logic disregards PageReserved pages (such as the ones + * covering the kernel image) unless they are mapped in the linear + * map. + */ + if (IS_ENABLED(CONFIG_HIBERNATION)) { + static struct notifier_block nb = { + .notifier_call = arm64_hibernate_pm_notify + }; + + register_pm_notifier(&nb); + } } #ifdef CONFIG_KFENCE @@ -1193,10 +1232,8 @@ static void __init map_mem(void) flags); } - /* Map the kernel data/bss read-only in the linear map */ - __map_memblock(init_end, kernel_end, PAGE_KERNEL_RO, flags); - flush_tlb_kernel_range((unsigned long)lm_alias(__init_end), - (unsigned long)lm_alias(__bss_stop)); + /* Map the kernel data/bss as invalid in the linear map */ + mark_linear_data_alias_valid(false); } void mark_rodata_ro(void) From 25996982ebcf8242c9e44589f8dd1ed760df8ac0 Mon Sep 17 00:00:00 2001 From: Zeng Heng Date: Wed, 3 Jun 2026 14:20:24 +0800 Subject: [PATCH 35/73] arm64: cpufeature: Add WORKAROUND_DISABLE_CNP capability The NVIDIA Carmel CNP erratum is not the only case requiring CNP to be disabled. Abstract this into a common WORKAROUND_DISABLE_CNP capability to facilitate adding errata for future chips and reduce duplicate checks in has_useable_cnp(). This serves as a prerequisite for the subsequent Hisilicon erratum 162100125. Suggested-by: Vladimir Murzin Signed-off-by: Zeng Heng Reviewed-by: Vladimir Murzin Acked-by: Wei Xu Signed-off-by: Will Deacon --- arch/arm64/Kconfig | 4 ++++ arch/arm64/include/asm/cpucaps.h | 4 ++-- arch/arm64/kernel/cpu_errata.c | 15 ++++++++++++--- arch/arm64/kernel/cpufeature.c | 2 +- arch/arm64/tools/cpucaps | 2 +- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index fe60738e5943..f297517a83b9 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1315,9 +1315,13 @@ config QCOM_FALKOR_ERRATUM_E1041 If unsure, say Y. +config ARM64_WORKAROUND_DISABLE_CNP + bool + config NVIDIA_CARMEL_CNP_ERRATUM bool "NVIDIA Carmel CNP: CNP on Carmel semantically different than ARM cores" default y + select ARM64_WORKAROUND_DISABLE_CNP help If CNP is enabled on Carmel cores, non-sharable TLBIs on a core will not invalidate shared TLB entries installed by a different core, as it would diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index d0d3cdd5763c..25c61cda901c 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -58,8 +58,8 @@ cpucap_is_possible(const unsigned int cap) return IS_ENABLED(CONFIG_ARM64_ERRATUM_2658417); case ARM64_WORKAROUND_CAVIUM_23154: return IS_ENABLED(CONFIG_CAVIUM_ERRATUM_23154); - case ARM64_WORKAROUND_NVIDIA_CARMEL_CNP: - return IS_ENABLED(CONFIG_NVIDIA_CARMEL_CNP_ERRATUM); + case ARM64_WORKAROUND_DISABLE_CNP: + return IS_ENABLED(CONFIG_ARM64_WORKAROUND_DISABLE_CNP); case ARM64_WORKAROUND_REPEAT_TLBI: return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI); case ARM64_WORKAROUND_SPECULATIVE_SSBS: diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 5377e4c2eba2..08eb9d6545d1 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -608,6 +608,15 @@ static const struct midr_range erratum_ac04_cpu_23_list[] = { }; #endif +#ifdef CONFIG_ARM64_WORKAROUND_DISABLE_CNP +static const struct midr_range cnp_erratum_cpus[] = { +#ifdef CONFIG_NVIDIA_CARMEL_CNP_ERRATUM + MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL), +#endif + {}, +}; +#endif + const struct arm64_cpu_capabilities arm64_errata[] = { #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE { @@ -801,12 +810,12 @@ const struct arm64_cpu_capabilities arm64_errata[] = { 1, 0), }, #endif -#ifdef CONFIG_NVIDIA_CARMEL_CNP_ERRATUM +#ifdef CONFIG_ARM64_WORKAROUND_DISABLE_CNP { /* NVIDIA Carmel */ .desc = "NVIDIA Carmel CNP erratum", - .capability = ARM64_WORKAROUND_NVIDIA_CARMEL_CNP, - ERRATA_MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL), + .capability = ARM64_WORKAROUND_DISABLE_CNP, + ERRATA_MIDR_RANGE_LIST(cnp_erratum_cpus), }, #endif #ifdef CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 6d53bb15cf7b..20c5f24f74a9 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1785,7 +1785,7 @@ has_useable_cnp(const struct arm64_cpu_capabilities *entry, int scope) if (is_kdump_kernel()) return false; - if (cpus_have_cap(ARM64_WORKAROUND_NVIDIA_CARMEL_CNP)) + if (cpus_have_cap(ARM64_WORKAROUND_DISABLE_CNP)) return false; return has_cpuid_feature(entry, scope); diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 811c2479e82d..9b85a84f6fd4 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -120,7 +120,7 @@ WORKAROUND_CAVIUM_TX2_219_PRFM WORKAROUND_CAVIUM_TX2_219_TVM WORKAROUND_CLEAN_CACHE WORKAROUND_DEVICE_LOAD_ACQUIRE -WORKAROUND_NVIDIA_CARMEL_CNP +WORKAROUND_DISABLE_CNP WORKAROUND_PMUV3_IMPDEF_TRAPS WORKAROUND_QCOM_FALKOR_E1003 WORKAROUND_QCOM_ORYON_CNTVOFF From f64328ecf4bf354d1ddf88542dc21a151ce5a2c4 Mon Sep 17 00:00:00 2001 From: Zeng Heng Date: Wed, 3 Jun 2026 14:20:25 +0800 Subject: [PATCH 36/73] arm64: kernel: Disable CNP on HiSilicon HIP09 HiSilicon HIP09 implements TLB entry matching behavior that deviates from the ARM architecture specification when the CNP (Common not Private) bit is set in TTBRx_ELx. When TTBRx.CNP=1, TLB entries may be incorrectly shared between CPU cores, leading to TLB conflicts and stale mappings. This affects coherency and can result in incorrect translations. Add the hardware erratum workaround (Hisilicon erratum 162100125) to disable CNP on affected HIP09 cores. Co-developed-by: Tong Tiangen Signed-off-by: Tong Tiangen Signed-off-by: Zeng Heng Reviewed-by: Vladimir Murzin Acked-by: Wei Xu Signed-off-by: Will Deacon --- Documentation/arch/arm64/silicon-errata.rst | 2 ++ arch/arm64/Kconfig | 16 ++++++++++++++++ arch/arm64/kernel/cpu_errata.c | 6 ++++-- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 046a7fa47063..61c2fd7ef644 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -287,6 +287,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | Hisilicon | Hip09 | #162100801 | HISILICON_ERRATUM_162100801 | +----------------+-----------------+-----------------+-----------------------------+ +| Hisilicon | Hip09 | #162100125 | HISILICON_ERRATUM_162100125 | ++----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ | Qualcomm Tech. | Kryo/Falkor v1 | E1003 | QCOM_FALKOR_ERRATUM_1003 | +----------------+-----------------+-----------------+-----------------------------+ diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index f297517a83b9..75638e37883d 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1273,6 +1273,22 @@ config HISILICON_ERRATUM_162100801 If unsure, say Y. +config HISILICON_ERRATUM_162100125 + bool "Hisilicon erratum 162100125" + default y + select ARM64_WORKAROUND_DISABLE_CNP + help + On HiSilicon HIP09, TLB entry matching behavior when CNP + (TTBRx.CNP=1) is enabled differs from the ARM architecture + specification. + + TLB entries may be incorrectly shared between CPUs, potentially + causing TLB conflicts and stale mappings. + + Disable CNP support for affected HiSilicon HIP09 cores. + + If unsure, say Y. + config QCOM_FALKOR_ERRATUM_1003 bool "Falkor E1003: Incorrect translation due to ASID change" default y diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 08eb9d6545d1..310e6f120992 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -612,6 +612,9 @@ static const struct midr_range erratum_ac04_cpu_23_list[] = { static const struct midr_range cnp_erratum_cpus[] = { #ifdef CONFIG_NVIDIA_CARMEL_CNP_ERRATUM MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL), +#endif +#ifdef CONFIG_HISILICON_ERRATUM_162100125 + MIDR_ALL_VERSIONS(MIDR_HISI_HIP09), #endif {}, }; @@ -812,8 +815,7 @@ const struct arm64_cpu_capabilities arm64_errata[] = { #endif #ifdef CONFIG_ARM64_WORKAROUND_DISABLE_CNP { - /* NVIDIA Carmel */ - .desc = "NVIDIA Carmel CNP erratum", + .desc = "NVIDIA Carmel CNP erratum, or Hisilicon erratum 162100125", .capability = ARM64_WORKAROUND_DISABLE_CNP, ERRATA_MIDR_RANGE_LIST(cnp_erratum_cpus), }, From f20f8fe086b2487ff70562866c4a30b1d9a66a37 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Fri, 22 May 2026 18:50:28 +0100 Subject: [PATCH 37/73] arm64: Document SVE constraints on new hwcaps Two of the SVE hwcaps added for the SVE features in the 2025 dpISA did not explicitly call out their dependency on SVE in the ABI documentation. Do so. While we're here reorder the SVE and fature specific ID registers for HWCAP3_SVE_LUT6 which did have the SVE dependency but listed it second unlike the other SVE specific ID registers. Fixes: abca5e69ab62 ("arm64/cpufeature: Define hwcaps for 2025 dpISA features") Reported-by: Will Deacon Signed-off-by: Mark Brown Signed-off-by: Will Deacon --- Documentation/arch/arm64/elf_hwcaps.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Documentation/arch/arm64/elf_hwcaps.rst b/Documentation/arch/arm64/elf_hwcaps.rst index 07ff9ea1d605..f60ca5612daa 100644 --- a/Documentation/arch/arm64/elf_hwcaps.rst +++ b/Documentation/arch/arm64/elf_hwcaps.rst @@ -452,10 +452,12 @@ HWCAP3_LS64 memory location, otherwise fallback to the non-atomic alternatives. HWCAP3_SVE_B16MM - Functionality implied by ID_AA64ZFR0_EL1.B16B16 == 0b0011 + Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and + ID_AA64ZFR0_EL1.B16B16 == 0b0011 HWCAP3_SVE2P3 - Functionality implied by ID_AA64ZFR0_EL1.SVEver == 0b0100 + Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and + ID_AA64ZFR0_EL1.SVEver == 0b0100 HWCAP3_SME_LUT6 Functionality implied by ID_AA64SMFR0_EL1.LUT6 == 0b1 @@ -473,8 +475,9 @@ HWCAP3_F16F32MM Functionality implied by ID_AA64ISAR0_EL1.FHM == 0b0011 HWCAP3_SVE_LUT6 - Functionality implied by ID_AA64ISAR2_EL1.LUT == 0b0010 and - ID_AA64PFR0_EL1.SVE == 0b0001. + Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and + ID_AA64ISAR2_EL1.LUT == 0b0010. + 4. Unused AT_HWCAP bits ----------------------- From ae24f6b06e90681ec36b9c21c3f5c09618350f5a Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:11 +0100 Subject: [PATCH 38/73] arm64: fpsimd: Fix type mismatch in sve_{save,load}_state() The sve_save_state() and sve_load_state() functions take a 32-bit int argument that describes whether to save/restore the FFR. Their assembly implementations consume the entire 64-bit register containing this 32-bit value, and will attempt to save/restore the FFR if any bit of that 64-bit register is non-zero. Per the AAPCS64 parameter passing rules, the callee is responsible for any necessary widening, and the upper 32-bits are permitted to contain arbitrary values. If the upper 32 bits are non-zero, this could result in an unexpected attempt to save/restore the FFR, and consequently could lead to unexpected traps/undefs/faults. In practice compilers are very unlikely to generate code where the upper 32-bits would be non-zero, but they are permitted to do so. Fix this by only consuming the low 32 bits of the register, and update comments accordingly. The hyp code __sve_save_state() and __sve_restore_state() functions don't have the same latent bug as they override the full 64-bit register containing the argument. Fixes: 9f5848665788 ("arm64/sve: Make access to FFR optional") Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Mark Brown Cc: Oliver Upton Cc: Vladimir Murzin Cc: Will Deacon Cc: stable@vger.kernel.org Signed-off-by: Will Deacon --- arch/arm64/kernel/entry-fpsimd.S | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S index 6325db1a2179..cb08d879cb4d 100644 --- a/arch/arm64/kernel/entry-fpsimd.S +++ b/arch/arm64/kernel/entry-fpsimd.S @@ -38,10 +38,10 @@ SYM_FUNC_END(fpsimd_load_state) * * x0 - pointer to buffer for state * x1 - pointer to storage for FPSR - * x2 - Save FFR if non-zero + * w2 - Save FFR if non-zero */ SYM_FUNC_START(sve_save_state) - sve_save 0, x1, x2, 3 + sve_save 0, x1, w2, 3 ret SYM_FUNC_END(sve_save_state) @@ -50,10 +50,10 @@ SYM_FUNC_END(sve_save_state) * * x0 - pointer to buffer for state * x1 - pointer to storage for FPSR - * x2 - Restore FFR if non-zero + * w2 - Restore FFR if non-zero */ SYM_FUNC_START(sve_load_state) - sve_load 0, x1, x2, 4 + sve_load 0, x1, w2, 4 ret SYM_FUNC_END(sve_load_state) From 247bd153905085c18ff9006cca1ccb96dfd18e7f Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:12 +0100 Subject: [PATCH 39/73] arm64: fpsimd: Fix type mismatch in sme_{save,load}_state() The sme_save_state() and sme_load_state() functions take a 32-bit int argument that describes whether to save/restore ZT0. Their assembly implementations consume the entire 64-bit register containing this 32-bit value, and will attempt to save/restore ZT0 if any bit of that 64-bit register is non-zero. Per the AAPCS64 parameter passing rules, the callee is responsible for any necessary widening, and the upper 32-bits are permitted to contain arbitrary values. If the upper 32 bits are non-zero, this could result in an unexpected attempt to save/restore ZT0, and consequently could lead to unexpected traps/undefs/faults. In practice compilers are very unlikely to generate code where the upper 32-bits would be non-zero, but they are permitted to do so. Fix this by only consuming the low 32 bits of the register, and update comments accordingly. Fixes: 95fcec713259 ("arm64/sme: Implement context switching for ZT0") Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Mark Brown Cc: Oliver Upton Cc: Vladimir Murzin Cc: Will Deacon Cc: stable@vger.kernel.org Signed-off-by: Will Deacon --- arch/arm64/kernel/entry-fpsimd.S | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S index cb08d879cb4d..28af4bcd97a5 100644 --- a/arch/arm64/kernel/entry-fpsimd.S +++ b/arch/arm64/kernel/entry-fpsimd.S @@ -103,13 +103,13 @@ SYM_FUNC_END(sme_set_vq) * Save the ZA and ZT state * * x0 - pointer to buffer for state - * x1 - number of ZT registers to save + * w1 - number of ZT registers to save */ SYM_FUNC_START(sme_save_state) _sme_rdsvl 2, 1 // x2 = VL/8 sme_save_za 0, x2, 12 // Leaves x0 pointing to the end of ZA - cbz x1, 1f + cbz w1, 1f _str_zt 0 1: ret @@ -119,13 +119,13 @@ SYM_FUNC_END(sme_save_state) * Load the ZA and ZT state * * x0 - pointer to buffer for state - * x1 - number of ZT registers to save + * w1 - number of ZT registers to save */ SYM_FUNC_START(sme_load_state) _sme_rdsvl 2, 1 // x2 = VL/8 sme_load_za 0, x2, 12 // Leaves x0 pointing to the end of ZA - cbz x1, 1f + cbz w1, 1f _ldr_zt 0 1: ret From 79e66bb7e8b48f953c97022998b25734a5d09651 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:13 +0100 Subject: [PATCH 40/73] KVM: arm64: Don't include There's no need for hyp/entry.S to include . The fpsimd macros have never been used by code in hyp/entry.S, and were instead used by code in hyp/fpsimd.S. Remove the unnecessary include. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/kvm/hyp/entry.S | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S index 11a10d8f5beb..308100ed25de 100644 --- a/arch/arm64/kvm/hyp/entry.S +++ b/arch/arm64/kvm/hyp/entry.S @@ -8,7 +8,6 @@ #include #include -#include #include #include #include From dc233762588051bfd28f03e848aa2015c9b1dbd2 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:14 +0100 Subject: [PATCH 41/73] KVM: arm64: Don't override FFR save/restore argument The __sve_save_state() and __sve_restore_state() functions take a parameter describing whether to save/restore the FFR, but both functions silently override this with '1'. This has always been benign (and callers have all passed 'true' since the parameter was introduced), but clearly this is not intentional. Historically, the functions always saved/restored the FFR, and there was no parameter to control this. In v5.16, the sve_save and sve_load assembly macros used by __sve_save_state() and __sve_restore_state() were changed to make saving/restoring FFR optional. The implementations of __sve_save_state() and __sve_restore_state() were changed to pass '1' to their respective macros, and the prototypes of __sve_save_state() and __sve_restore_state() were unchanged. See commit: 9f5848665788 ("arm64/sve: Make access to FFR optional") In v6.10, the prototypes of __sve_save_state() and __sve_restore_state() were changed to add 'save_ffr' and 'restore_ffr' parameters respectively, but the implementations were not changed to stop passing 1 to their respective macros. All callers were changed to pass 'true' to __sve_save_state() and __sve_restore_state(). See commit: 45f4ea9bcfe9 ("KVM: arm64: Fix prototype for __sve_save_state/__sve_restore_state") This is all benign, but clearly unintentional, and it gets in the way of cleaning up the FPSIMD/SVE/SME code. Remove the unnecessary overriding. The 'save_ffr' and 'restore_ffr' parameters are 32-bit ints, and per the AAPCS64 parameter passing rules, the upper 32 bits of the register holding these arguments might contain arbitrary values. Thus it is necessary to pass 'w2' rather than 'x2' to the sve_load and save_save macros, such that the upper 32 bits are ignored when deciding whether to save/restore the FFR. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/kvm/hyp/fpsimd.S | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kvm/hyp/fpsimd.S b/arch/arm64/kvm/hyp/fpsimd.S index e950875e31ce..30507c50942e 100644 --- a/arch/arm64/kvm/hyp/fpsimd.S +++ b/arch/arm64/kvm/hyp/fpsimd.S @@ -21,13 +21,11 @@ SYM_FUNC_START(__fpsimd_restore_state) SYM_FUNC_END(__fpsimd_restore_state) SYM_FUNC_START(__sve_restore_state) - mov x2, #1 - sve_load 0, x1, x2, 3 + sve_load 0, x1, w2, 3 ret SYM_FUNC_END(__sve_restore_state) SYM_FUNC_START(__sve_save_state) - mov x2, #1 - sve_save 0, x1, x2, 3 + sve_save 0, x1, w2, 3 ret SYM_FUNC_END(__sve_save_state) From da20bb4bc5e6cc2028d61ba5dceaf9147348f617 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:15 +0100 Subject: [PATCH 42/73] KVM: arm64: pkvm: Save host FPMR in host cpu context Protected KVM stores most of the host's system register state in kvm_host_data::host_ctxt, which is an instance of struct kvm_cpu_context. As kvm_cpu_context::sys_regs[] has a slot for FPMR, we can store the host's FPMR there. Do so, and remove kvm_host_data::fpmr. Signed-off-by: Mark Rutland Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Mark Brown Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/kvm_host.h | 3 --- arch/arm64/kvm/hyp/include/hyp/switch.h | 6 ++++-- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 5 +++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 851f6171751c..0595644d6a3a 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -778,9 +778,6 @@ struct kvm_host_data { */ struct cpu_sve_state *sve_state; - /* Used by pKVM only. */ - u64 fpmr; - /* Ownership of the FP regs */ enum { FP_STATE_FREE, diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 98b2976837b1..cc4d011a2b38 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -554,6 +554,8 @@ static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu) static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu) { + struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); + /* * Non-protected kvm relies on the host restoring its sve state. * Protected kvm restores the host's sve state as not to reveal that @@ -562,11 +564,11 @@ static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu) if (system_supports_sve()) { __hyp_sve_save_host(); } else { - __fpsimd_save_state(host_data_ptr(host_ctxt.fp_regs)); + __fpsimd_save_state(&hctxt->fp_regs); } if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm))) - *host_data_ptr(fpmr) = read_sysreg_s(SYS_FPMR); + ctxt_sys_reg(hctxt, FPMR) = read_sysreg_s(SYS_FPMR); } diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 73f2e0221e70..3b2c00ca9408 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -66,6 +66,7 @@ static void fpsimd_sve_flush(void) static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) { + struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); bool has_fpmr; if (!guest_owns_fp_regs()) @@ -89,10 +90,10 @@ static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) if (system_supports_sve()) __hyp_sve_restore_host(); else - __fpsimd_restore_state(host_data_ptr(host_ctxt.fp_regs)); + __fpsimd_restore_state(&hctxt->fp_regs); if (has_fpmr) - write_sysreg_s(*host_data_ptr(fpmr), SYS_FPMR); + write_sysreg_s(ctxt_sys_reg(hctxt, FPMR), SYS_FPMR); *host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED; } From afd7af2b56ec7553e4bdc6387bb923b01d2ad2b6 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:16 +0100 Subject: [PATCH 43/73] KVM: arm64: pkvm: Remove struct cpu_sve_state There's no need for struct cpu_sve_state. Code would be simpler and more robust without it, and removing it will simplify further cleanups (e.g. adding an opaque type for the sve register state). Protected KVM stores most of the host's system register state in kvm_host_data::host_ctxt, which is an instance of struct kvm_cpu_context. As kvm_cpu_context::sys_regs[] has a slot for ZCR_EL1, we can store the host's ZCR_EL1 there. While kvm_cpu_context::sys_regs doesn't have slots for FPSR and FPCR, these are usually expected to be stored in struct user_fpsimd_state. For historical reasons, __sve_save_state and __sve_restore_state() expect a pointer to fpsr *within* struct user_fpsimd_state, assuming the fpcr will immediately follow, as per the order within struct user_fpsimd_state. We currently match this ordering in struct cpu_sve_state, but it would be simpler and more robust to use struct user_fpsimd_state directly. After moving ZCR_EL1, FPSR, and FPCR out of struct cpu_sve_state, all that's left is sve_regs, which can be represented as a pointer without need for a container struct. This is kept as a pointer to u8 (matching the array type), as this permits the compiler to catch unbalanced referencing/dereferencing, which is not possible for pointers to void. Apply the above changes, and remove cpu_sve_state. I've dropped the comment regarding buffer alignment as AFAICT this was never necessary. The LDR/STR (vector) instructions only require this alignment when SCTLR_ELx.A==1, which is not the case for the kernel or hyp code. Nothing else depends on the alignment. Signed-off-by: Mark Rutland Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Mark Brown Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/kvm_host.h | 18 ++---------------- arch/arm64/include/asm/kvm_pkvm.h | 3 +-- arch/arm64/kvm/arm.c | 16 ++++++++-------- arch/arm64/kvm/hyp/include/hyp/switch.h | 9 +++++---- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 9 +++++---- arch/arm64/kvm/hyp/nvhe/setup.c | 4 ++-- 6 files changed, 23 insertions(+), 36 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 0595644d6a3a..ae98375a8d32 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -735,20 +735,6 @@ struct kvm_cpu_context { u64 *vncr_array; }; -struct cpu_sve_state { - __u64 zcr_el1; - - /* - * Ordering is important since __sve_save_state/__sve_restore_state - * relies on it. - */ - __u32 fpsr; - __u32 fpcr; - - /* Must be SVE_VQ_BYTES (128 bit) aligned. */ - __u8 sve_regs[]; -}; - /* * This structure is instantiated on a per-CPU basis, and contains * data that is: @@ -774,9 +760,9 @@ struct kvm_host_data { /* * Hyp VA. - * sve_state is only used in pKVM and if system_supports_sve(). + * sve_regs is only used in pKVM and if system_supports_sve(). */ - struct cpu_sve_state *sve_state; + u8 *sve_regs; /* Ownership of the FP regs */ enum { diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index 2954b311128c..74fedd9c5ff0 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -188,8 +188,7 @@ static inline size_t pkvm_host_sve_state_size(void) if (!system_supports_sve()) return 0; - return size_add(sizeof(struct cpu_sve_state), - SVE_SIG_REGS_SIZE(sve_vq_from_vl(kvm_host_sve_max_vl))); + return SVE_SIG_REGS_SIZE(sve_vq_from_vl(kvm_host_sve_max_vl)); } struct pkvm_mapping { diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 176cbe8baad3..d51586b74915 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2495,10 +2495,10 @@ static void __init teardown_hyp_mode(void) continue; if (free_sve) { - struct cpu_sve_state *sve_state; + u8 *sve_regs; - sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state; - free_pages((unsigned long) sve_state, pkvm_host_sve_state_order()); + sve_regs = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs; + free_pages((unsigned long) sve_regs, pkvm_host_sve_state_order()); } free_pages(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu], nvhe_percpu_order()); @@ -2623,7 +2623,7 @@ static int init_pkvm_host_sve_state(void) if (!page) return -ENOMEM; - per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = page_address(page); + per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs = page_address(page); } /* @@ -2644,11 +2644,11 @@ static void finalize_init_hyp_mode(void) if (system_supports_sve() && is_protected_kvm_enabled()) { for_each_possible_cpu(cpu) { - struct cpu_sve_state *sve_state; + u8 *sve_regs; - sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state; - per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = - kern_hyp_va(sve_state); + sve_regs = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs; + per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs = + kern_hyp_va(sve_regs); } } } diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index cc4d011a2b38..6512dd3f75ae 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -484,12 +484,13 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) static inline void __hyp_sve_save_host(void) { - struct cpu_sve_state *sve_state = *host_data_ptr(sve_state); + struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); + u8 *sve_regs = *host_data_ptr(sve_regs); - sve_state->zcr_el1 = read_sysreg_el1(SYS_ZCR); + ctxt_sys_reg(hctxt, ZCR_EL1) = read_sysreg_el1(SYS_ZCR); write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); - __sve_save_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_sve_max_vl), - &sve_state->fpsr, + __sve_save_state(sve_regs + sve_ffr_offset(kvm_host_sve_max_vl), + &hctxt->fp_regs.fpsr, true); } diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 3b2c00ca9408..31b0245a7e64 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -41,7 +41,8 @@ static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) static void __hyp_sve_restore_host(void) { - struct cpu_sve_state *sve_state = *host_data_ptr(sve_state); + struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); + u8 *sve_regs = *host_data_ptr(sve_regs); /* * On saving/restoring host sve state, always use the maximum VL for @@ -53,10 +54,10 @@ static void __hyp_sve_restore_host(void) * need to be revisited. */ write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); - __sve_restore_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_sve_max_vl), - &sve_state->fpsr, + __sve_restore_state(sve_regs + sve_ffr_offset(kvm_host_sve_max_vl), + &hctxt->fp_regs.fpsr, true); - write_sysreg_el1(sve_state->zcr_el1, SYS_ZCR); + write_sysreg_el1(ctxt_sys_reg(hctxt, ZCR_EL1), SYS_ZCR); } static void fpsimd_sve_flush(void) diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index d8e5b563fd3d..399608694920 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -82,9 +82,9 @@ static int pkvm_create_host_sve_mappings(void) for (i = 0; i < hyp_nr_cpus; i++) { struct kvm_host_data *host_data = per_cpu_ptr(&kvm_host_data, i); - struct cpu_sve_state *sve_state = host_data->sve_state; + u8 *sve_regs = host_data->sve_regs; - start = kern_hyp_va(sve_state); + start = kern_hyp_va(sve_regs); end = start + PAGE_ALIGN(pkvm_host_sve_state_size()); ret = pkvm_create_mappings(start, end, PAGE_HYP); if (ret) From 3efb6c7f22c60a1440a80ddfa3d30b9406f01ccc Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:17 +0100 Subject: [PATCH 44/73] arm64: fpsimd: Fold sve_init_regs() into do_sve_acc() For historical reasons, do_sve_acc() is structurally different from do_sme_acc(), and the logic to convert the task from FPSIMD to SVE is out-of-line in sve_init_regs(). We only use sve_init_regs() within do_sve_acc(), so it's not necessary for this to be a separate function. Fold sve_init_regs() into do_sve_acc(), and simplify the associated comments. This makes do_sve_acc() structurally similar to do_sme_acc(), making it easier to see similarities and differences. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/kernel/fpsimd.c | 48 ++++++++++++++------------------------ 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 60a45d600b46..a8395cb30334 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1293,31 +1293,6 @@ void sme_suspend_exit(void) #endif /* CONFIG_ARM64_SME */ -static void sve_init_regs(void) -{ - /* - * Convert the FPSIMD state to SVE, zeroing all the state that - * is not shared with FPSIMD. If (as is likely) the current - * state is live in the registers then do this there and - * update our metadata for the current task including - * disabling the trap, otherwise update our in-memory copy. - * We are guaranteed to not be in streaming mode, we can only - * take a SVE trap when not in streaming mode and we can't be - * in streaming mode when taking a SME trap. - */ - if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { - unsigned long vq_minus_one = - sve_vq_from_vl(task_get_sve_vl(current)) - 1; - sve_set_vq(vq_minus_one); - sve_flush_live(true, vq_minus_one); - fpsimd_bind_task_to_cpu(); - } else { - fpsimd_to_sve(current); - current->thread.fp_type = FP_STATE_SVE; - fpsimd_flush_task_state(current); - } -} - /* * Trapped SVE access * @@ -1349,13 +1324,24 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs) WARN_ON(1); /* SVE access shouldn't have trapped */ /* - * Even if the task can have used streaming mode we can only - * generate SVE access traps in normal SVE mode and - * transitioning out of streaming mode may discard any - * streaming mode state. Always clear the high bits to avoid - * any potential errors tracking what is properly initialised. + * Convert the FPSIMD state to SVE. Stale SVE state can be present in + * registers or memory, so we must zero all state that is not shared + * with FPSIMD. + * + * SVE traps cannot be taken from streaming mode, so there cannot be + * any effective streaming mode SVE state. */ - sve_init_regs(); + if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { + unsigned long vq_minus_one = + sve_vq_from_vl(task_get_sve_vl(current)) - 1; + sve_set_vq(vq_minus_one); + sve_flush_live(true, vq_minus_one); + fpsimd_bind_task_to_cpu(); + } else { + fpsimd_to_sve(current); + current->thread.fp_type = FP_STATE_SVE; + fpsimd_flush_task_state(current); + } put_cpu_fpsimd_context(); } From e0cde2d2bb1bced57079fb45a323b65759a78962 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:18 +0100 Subject: [PATCH 45/73] arm64: fpsimd: Remove sve_set_vq() and sme_set_vq() The sve_set_vq() and sme_set_vq() assembly functions (and the sve_load_vq and sme_load_vq macros they use) are open-coded forms of sysreg_clear_set*(). There's no need for these to be implemented out-of-line in assembly, and the 'vq_minus_1' argument is unusual and confusing. Use sysreg_clear_set_s() directly, where the necessary 'vq - 1' encoding is more obviously part of encoding the register value. For now, sve_flush_live() is left with the unusual vq_minus_1 argument. This will be addressed in subsequent patches. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/fpsimd.h | 2 -- arch/arm64/include/asm/fpsimdmacros.h | 22 ---------------------- arch/arm64/kernel/entry-fpsimd.S | 10 ---------- arch/arm64/kernel/fpsimd.c | 24 +++++++++++++----------- 4 files changed, 13 insertions(+), 45 deletions(-) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index d9d00b45ab11..8efa3c0402a7 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -146,8 +146,6 @@ extern void sve_load_state(void const *state, u32 const *pfpsr, int restore_ffr); extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1); extern unsigned int sve_get_vl(void); -extern void sve_set_vq(unsigned long vq_minus_1); -extern void sme_set_vq(unsigned long vq_minus_1); extern void sme_save_state(void *state, int zt); extern void sme_load_state(void const *state, int zt); diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h index cda81d009c9b..adf33d2da40c 100644 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ b/arch/arm64/include/asm/fpsimdmacros.h @@ -265,28 +265,6 @@ .purgem _for__body .endm -/* Update ZCR_EL1.LEN with the new VQ */ -.macro sve_load_vq xvqminus1, xtmp, xtmp2 - mrs_s \xtmp, SYS_ZCR_EL1 - bic \xtmp2, \xtmp, ZCR_ELx_LEN_MASK - orr \xtmp2, \xtmp2, \xvqminus1 - cmp \xtmp2, \xtmp - b.eq 921f - msr_s SYS_ZCR_EL1, \xtmp2 //self-synchronising -921: -.endm - -/* Update SMCR_EL1.LEN with the new VQ */ -.macro sme_load_vq xvqminus1, xtmp, xtmp2 - mrs_s \xtmp, SYS_SMCR_EL1 - bic \xtmp2, \xtmp, SMCR_ELx_LEN_MASK - orr \xtmp2, \xtmp2, \xvqminus1 - cmp \xtmp2, \xtmp - b.eq 921f - msr_s SYS_SMCR_EL1, \xtmp2 //self-synchronising -921: -.endm - /* Preserve the first 128-bits of Znz and zero the rest. */ .macro _sve_flush_z nz _sve_check_zreg \nz diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S index 28af4bcd97a5..bfa73f26653f 100644 --- a/arch/arm64/kernel/entry-fpsimd.S +++ b/arch/arm64/kernel/entry-fpsimd.S @@ -62,11 +62,6 @@ SYM_FUNC_START(sve_get_vl) ret SYM_FUNC_END(sve_get_vl) -SYM_FUNC_START(sve_set_vq) - sve_load_vq x0, x1, x2 - ret -SYM_FUNC_END(sve_set_vq) - /* * Zero all SVE registers but the first 128-bits of each vector * @@ -94,11 +89,6 @@ SYM_FUNC_START(sme_get_vl) ret SYM_FUNC_END(sme_get_vl) -SYM_FUNC_START(sme_set_vq) - sme_load_vq x0, x1, x2 - ret -SYM_FUNC_END(sme_set_vq) - /* * Save the ZA and ZT state * diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index a8395cb30334..2578c2372c89 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -377,8 +377,10 @@ static void task_fpsimd_load(void) if (!thread_sm_enabled(¤t->thread)) WARN_ON_ONCE(!test_and_set_thread_flag(TIF_SVE)); - if (test_thread_flag(TIF_SVE)) - sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1); + if (test_thread_flag(TIF_SVE)) { + unsigned long vq = sve_vq_from_vl(task_get_sve_vl(current)); + sysreg_clear_set_s(SYS_ZCR_EL1, ZCR_ELx_LEN, vq - 1); + } restore_sve_regs = true; restore_ffr = true; @@ -403,8 +405,10 @@ static void task_fpsimd_load(void) unsigned long sme_vl = task_get_sme_vl(current); /* Ensure VL is set up for restoring data */ - if (test_thread_flag(TIF_SME)) - sme_set_vq(sve_vq_from_vl(sme_vl) - 1); + if (test_thread_flag(TIF_SME)) { + unsigned long vq = sve_vq_from_vl(sme_vl); + sysreg_clear_set_s(SYS_SMCR_EL1, SMCR_ELx_LEN, vq - 1); + } write_sysreg_s(current->thread.svcr, SYS_SVCR); @@ -1332,10 +1336,9 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs) * any effective streaming mode SVE state. */ if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { - unsigned long vq_minus_one = - sve_vq_from_vl(task_get_sve_vl(current)) - 1; - sve_set_vq(vq_minus_one); - sve_flush_live(true, vq_minus_one); + unsigned long vq = sve_vq_from_vl(task_get_sve_vl(current)); + sysreg_clear_set_s(SYS_ZCR_EL1, ZCR_ELx_LEN, vq - 1); + sve_flush_live(true, vq - 1); fpsimd_bind_task_to_cpu(); } else { fpsimd_to_sve(current); @@ -1465,9 +1468,8 @@ void do_sme_acc(unsigned long esr, struct pt_regs *regs) WARN_ON(1); if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { - unsigned long vq_minus_one = - sve_vq_from_vl(task_get_sme_vl(current)) - 1; - sme_set_vq(vq_minus_one); + unsigned long vq = sve_vq_from_vl(task_get_sme_vl(current)); + sysreg_clear_set_s(SYS_SMCR_EL1, SMCR_ELx_LEN, vq - 1); fpsimd_bind_task_to_cpu(); } else { From f27fe9aa2d06e01687c3871fb28492ccbab11e4c Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:19 +0100 Subject: [PATCH 46/73] arm64: fpsimd: Use assembler for SVE instructions Historically we supported assemblers which could not assemble SVE instructions. We dropped support for such assemblers in commit: 118c40b7b503 ("kbuild: require gcc-8 and binutils-2.30") Since that commit, all supported assemblers (binutils and LLVM) are capable of assembling SVE instructions, and there's no need for us to manually encode SVE instructions. Rely on the assembler to encode SVE instructions, and remove the manual encoding. The various _sve_ macros are kept for now, and will be cleaned up in subsequent patches. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/fpsimdmacros.h | 65 +++++++-------------------- 1 file changed, 17 insertions(+), 48 deletions(-) diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h index adf33d2da40c..4ba23e493cb6 100644 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ b/arch/arm64/include/asm/fpsimdmacros.h @@ -99,85 +99,54 @@ .endif .endm -/* SVE instruction encodings for non-SVE-capable assemblers */ -/* (pre binutils 2.28, all kernel capable clang versions support SVE) */ +/* Deprecated macros for SVE instructions */ /* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */ .macro _sve_str_v nz, nxbase, offset=0 - _sve_check_zreg \nz - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0xe5804000 \ - | (\nz) \ - | ((\nxbase) << 5) \ - | (((\offset) & 7) << 10) \ - | (((\offset) & 0x1f8) << 13) + .arch_extension sve + str z\nz, [X\nxbase, #\offset, MUL VL] .endm /* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */ .macro _sve_ldr_v nz, nxbase, offset=0 - _sve_check_zreg \nz - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0x85804000 \ - | (\nz) \ - | ((\nxbase) << 5) \ - | (((\offset) & 7) << 10) \ - | (((\offset) & 0x1f8) << 13) + .arch_extension sve + ldr z\nz, [X\nxbase, #\offset, MUL VL] .endm /* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */ .macro _sve_str_p np, nxbase, offset=0 - _sve_check_preg \np - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0xe5800000 \ - | (\np) \ - | ((\nxbase) << 5) \ - | (((\offset) & 7) << 10) \ - | (((\offset) & 0x1f8) << 13) + .arch_extension sve + str p\np, [X\nxbase, #\offset, MUL VL] .endm /* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */ .macro _sve_ldr_p np, nxbase, offset=0 - _sve_check_preg \np - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0x85800000 \ - | (\np) \ - | ((\nxbase) << 5) \ - | (((\offset) & 7) << 10) \ - | (((\offset) & 0x1f8) << 13) + .arch_extension sve + ldr p\np, [x\nxbase, #\offset, MUL VL] .endm /* RDVL X\nx, #\imm */ .macro _sve_rdvl nx, imm - _check_general_reg \nx - _check_num (\imm), -0x20, 0x1f - .inst 0x04bf5000 \ - | (\nx) \ - | (((\imm) & 0x3f) << 5) + .arch_extension sve + rdvl x\nx, #\imm .endm /* RDFFR (unpredicated): RDFFR P\np.B */ .macro _sve_rdffr np - _sve_check_preg \np - .inst 0x2519f000 \ - | (\np) + .arch_extension sve + rdffr p\np\().b .endm /* WRFFR P\np.B */ .macro _sve_wrffr np - _sve_check_preg \np - .inst 0x25289000 \ - | ((\np) << 5) + .arch_extension sve + wrffr p\np\().b .endm /* PFALSE P\np.B */ .macro _sve_pfalse np - _sve_check_preg \np - .inst 0x2518e400 \ - | (\np) + .arch_extension sve + pfalse p\np\().b .endm /* SME instruction encodings for non-SME-capable assemblers */ From db9d63eafebac4712ef2602599965caa66c74228 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:20 +0100 Subject: [PATCH 47/73] arm64: fpsimd: Use assembler for baseline SME instructions We currently support assemblers which do not support SME instructions, and have macros to manually encode SME instructions. This was necessary historically as SME support was developed before assembler support was widely available, but things have changed: * All currently supported versions of LLVM support baseline SME instructions. Building the kernel requires LLVM 15+, while LLVM 13+ supports SME. * GNU binutils has supported baseline SME instructions since 2.38, which was released on 09 February 2022. Toolchains using this or later are widely available. For example Debian 12 (released on 10 June 2023) provides binutils 2.40. Toolchains provided kernel.org provide binutils 2.38+ since the GCC 12.1.0 release (released between 06 May 2022 and 17 August 2022). * For various reasons, SME support was marked as BROKEN, and re-enabled in v6.16 (released on 27 July 2025). The earliest support LTS kernel with SME support is v6.18.y, v6.18 was tagged on 30 November 2025, and contemporary toolchains (GCC 15.2 and binutils 2.45) supported baseline SME instructions. * Any distribution which intends to support SME will presumably have a toolchain that supports baseline SME instructions such that userspace can be built. Considering the above, there's no practical benefit to allowing SME to be built when the toolchain doesn't support baseline SME instructions. Make CONFIG_ARM64_SME depend on assembler support for SME, and remove the manual encoding of SME instructions. The various _sme_ macros are kept for now, and will be cleaned up in subsequent patches. A couple of SME2 instructions require a more recent toolchain, and are left as-is for now. I've looked through releases of binutils and LLVM to find when support was added, and noted this in a comment. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/Kconfig | 5 ++++ arch/arm64/include/asm/fpsimdmacros.h | 38 +++++++++++---------------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index fe60738e5943..378e50fef247 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2247,10 +2247,15 @@ config ARM64_SVE booting the kernel. If unsure and you are not observing these symptoms, you should assume that it is safe to say Y. +config AS_HAS_SME + # Supported by LLVM 13+ and binutils 2.38+ + def_bool $(as-instr,.arch_extension sme) + config ARM64_SME bool "ARM Scalable Matrix Extension support" default y depends on ARM64_SVE + depends on AS_HAS_SME help The Scalable Matrix Extension (SME) is an extension to the AArch64 execution state which utilises a substantial subset of the SVE diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h index 4ba23e493cb6..4a9bf46e5291 100644 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ b/arch/arm64/include/asm/fpsimdmacros.h @@ -149,46 +149,38 @@ pfalse p\np\().b .endm -/* SME instruction encodings for non-SME-capable assemblers */ -/* (pre binutils 2.38/LLVM 13) */ +/* Deprecated macros for SME instructions */ /* RDSVL X\nx, #\imm */ .macro _sme_rdsvl nx, imm - _check_general_reg \nx - _check_num (\imm), -0x20, 0x1f - .inst 0x04bf5800 \ - | (\nx) \ - | (((\imm) & 0x3f) << 5) + .arch_extension sme + rdsvl x\nx, #\imm .endm /* * STR (vector from ZA array): - * STR ZA[\nw, #\offset], [X\nxbase, #\offset, MUL VL] + * STR ZA[W\nw, #\offset], [X\nxbase, #\offset, MUL VL] */ .macro _sme_str_zav nw, nxbase, offset=0 - _sme_check_wv \nw - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0xe1200000 \ - | (((\nw) & 3) << 13) \ - | ((\nxbase) << 5) \ - | ((\offset) & 7) + .arch_extension sme + str za[w\nw, #\offset], [x\nxbase, #\offset, MUL VL] .endm /* * LDR (vector to ZA array): - * LDR ZA[\nw, #\offset], [X\nxbase, #\offset, MUL VL] + * LDR ZA[w\nw, #\offset], [X\nxbase, #\offset, MUL VL] */ .macro _sme_ldr_zav nw, nxbase, offset=0 - _sme_check_wv \nw - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0xe1000000 \ - | (((\nw) & 3) << 13) \ - | ((\nxbase) << 5) \ - | ((\offset) & 7) + .arch_extension sme + ldr za[w\nw, #\offset], [x\nxbase, #\offset, MUL VL] .endm +/* + * SME2 instruction encodings for older assemblers. + * Supported by binutils 2.41+. + * Supported by LLVM 16+ + */ + /* * LDR (ZT0) * From 3f26d7c6544cae590d8aa380aac62fe455dac2c3 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:21 +0100 Subject: [PATCH 48/73] arm64: fpsimd: Move sve_get_vl() and sme_get_vl() inline The sve_get_vl() and sme_get_vl() functions are wrappers for the RDVL and RDSVL instructions respectively. There's no need for those to be out-of-line. Replace the out-of-line assembly functions with equivalent inline functions. The _sve_rdvl assembly macro is unused, and so it is removed. The _sme_rdsvl assembly macro is still used elsewhere, and so is kept for now. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/fpsimd.h | 31 +++++++++++++++++++++++++-- arch/arm64/include/asm/fpsimdmacros.h | 6 ------ arch/arm64/kernel/entry-fpsimd.S | 10 --------- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 8efa3c0402a7..36cf528e6497 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -22,6 +22,9 @@ #include #include +#define __SVE_PREAMBLE ".arch_extension sve\n" +#define __SME_PREAMBLE ".arch_extension sme\n" + /* Masks for extracting the FPSR and FPCR from the FPSCR */ #define VFP_FPSCR_STAT_MASK 0xf800009f #define VFP_FPSCR_CTRL_MASK 0x07f79f00 @@ -141,11 +144,23 @@ static inline void *thread_zt_state(struct thread_struct *thread) return thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq); } +static inline unsigned int sve_get_vl(void) +{ + unsigned int vl; + + asm volatile( + __SVE_PREAMBLE + " rdvl %x[vl], #1\n" + : [vl] "=r" (vl) + ); + + return vl; +} + extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr); extern void sve_load_state(void const *state, u32 const *pfpsr, int restore_ffr); extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1); -extern unsigned int sve_get_vl(void); extern void sme_save_state(void *state, int zt); extern void sme_load_state(void const *state, int zt); @@ -400,8 +415,20 @@ static inline int sme_max_virtualisable_vl(void) return vec_max_virtualisable_vl(ARM64_VEC_SME); } +static inline unsigned int sme_get_vl(void) +{ + unsigned int vl; + + asm volatile( + __SME_PREAMBLE + " rdsvl %x[vl], #1\n" + : [vl] "=r" (vl) + ); + + return vl; +} + extern void sme_alloc(struct task_struct *task, bool flush); -extern unsigned int sme_get_vl(void); extern int sme_set_current_vl(unsigned long arg); extern int sme_get_current_vl(void); extern void sme_suspend_exit(void); diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h index 4a9bf46e5291..c724fcad7ee0 100644 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ b/arch/arm64/include/asm/fpsimdmacros.h @@ -125,12 +125,6 @@ ldr p\np, [x\nxbase, #\offset, MUL VL] .endm -/* RDVL X\nx, #\imm */ -.macro _sve_rdvl nx, imm - .arch_extension sve - rdvl x\nx, #\imm -.endm - /* RDFFR (unpredicated): RDFFR P\np.B */ .macro _sve_rdffr np .arch_extension sve diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S index bfa73f26653f..a124c6d3766c 100644 --- a/arch/arm64/kernel/entry-fpsimd.S +++ b/arch/arm64/kernel/entry-fpsimd.S @@ -57,11 +57,6 @@ SYM_FUNC_START(sve_load_state) ret SYM_FUNC_END(sve_load_state) -SYM_FUNC_START(sve_get_vl) - _sve_rdvl 0, 1 - ret -SYM_FUNC_END(sve_get_vl) - /* * Zero all SVE registers but the first 128-bits of each vector * @@ -84,11 +79,6 @@ SYM_FUNC_END(sve_flush_live) #ifdef CONFIG_ARM64_SME -SYM_FUNC_START(sme_get_vl) - _sme_rdsvl 0, 1 - ret -SYM_FUNC_END(sme_get_vl) - /* * Save the ZA and ZT state * From 36a1d1726634f2e53eefaec32e116fad465b89cb Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:22 +0100 Subject: [PATCH 49/73] arm64: sysreg: Add FPCR and FPSR Add sysreg definitions for FPCR and FPSR. Some versions of LLVM will refuse to assemble accesses to FPCR and FPSR unless the "fp" arch extension is enabled, which we don't currently do for read_sysreg() and write_sysreg(). In general, handling feature dependencies would complicate read_sysreg() and write_sysreg(), and it's simpler to use read_sysreg_s() and write_sysreg_s() instead, requiring sysreg definitions. The values used can be found in ARM ARM issue M.b: https://developer.arm.com/documentation/ddi0487/mb/ ... in sections: * C5.2.8 ("FPCR, Floating-point Control Register") * C5.2.10 ("FPSR, Floating-point Status Register") Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/tools/sysreg | 45 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 6c3ff14e561e..8b219d656660 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -3790,6 +3790,51 @@ Field 1 ZA Field 0 SM EndSysreg +Sysreg FPCR 3 3 4 4 0 +Res0 63:27 +Field 26 AHP +Field 25 DN +Field 24 FZ +Enum 23:22 RMode + 0b00 RN + 0b01 RP + 0b10 RM + 0b11 RZ +EndEnum +Field 21:20 Stride +Field 19 FZ16 +Field 18:16 Len +Field 15 IDE +Res0 14 +Field 13 EBF +Field 12 IXE +Field 11 UFE +Field 10 OFE +Field 9 DZE +Field 8 IOE +Res0 7:3 +Field 2 NEP +Field 1 AH +Field 0 FIZ +EndSysreg + +Sysreg FPSR 3 3 4 4 1 +Res0 63:32 +Field 31 N +Field 30 Z +Field 29 C +Field 28 V +Field 27 QC +Res0 26:8 +Field 7 IDC +Res0 6:5 +Field 4 IXC +Field 3 UFC +Field 2 OFC +Field 1 DZC +Field 0 IOC +EndSysreg + Sysreg FPMR 3 3 4 4 2 Res0 63:38 Field 37:32 LSCALE2 From 1277531fca43de81962812d28fb126775a5a509e Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:23 +0100 Subject: [PATCH 50/73] arm64: fpsimd: Split FPSR/FPCR from SVE save/restore Regardless of whether the vector registers are saved in FPSIMD or SVE format, we store FPSR and FPCR in user_fpsimd_state::{fpsr,fpcr}. For historical reasons, the functions which save/restore SVE context take a pointer to user_fpsimd_state::fpsr, and use this to access both user_fpsimd_state::fpsr and user_fpsimd_state::fpcr. This is unnecessarily fragile. Move the save/restore of FPSR and FPCR into separate helper functions which take a pointer to user_fpsimd_state. I've used read_sysreg_s() and write_sysreg_s() as contemporary versions of LLVM will refuse to directly assemble accesses to FPCR or FPSR unless the "fp" arch extension is enabled. For the moment, fpsimd_save_state() and fpsimd_load_state() are left as-is with their own logic to save/restore FPSR and FPCR. This will be unified in subsequent patches. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/fpsimd.h | 17 ++++++++++++++--- arch/arm64/include/asm/fpsimdmacros.h | 13 ++----------- arch/arm64/include/asm/kvm_hyp.h | 4 ++-- arch/arm64/kernel/entry-fpsimd.S | 10 ++++------ arch/arm64/kernel/fpsimd.c | 5 +++-- arch/arm64/kvm/hyp/fpsimd.S | 4 ++-- arch/arm64/kvm/hyp/include/hyp/switch.h | 4 ++-- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 5 +++-- 8 files changed, 32 insertions(+), 30 deletions(-) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 36cf528e6497..6fd5cdf5e5f1 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -74,6 +74,18 @@ static inline void cpacr_restore(unsigned long cpacr) struct task_struct; +static inline void fpsimd_save_common(struct user_fpsimd_state *state) +{ + state->fpsr = read_sysreg_s(SYS_FPSR); + state->fpcr = read_sysreg_s(SYS_FPCR); +} + +static inline void fpsimd_load_common(const struct user_fpsimd_state *state) +{ + write_sysreg_s(state->fpsr, SYS_FPSR); + write_sysreg_s(state->fpcr, SYS_FPCR); +} + extern void fpsimd_save_state(struct user_fpsimd_state *state); extern void fpsimd_load_state(struct user_fpsimd_state *state); @@ -157,9 +169,8 @@ static inline unsigned int sve_get_vl(void) return vl; } -extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr); -extern void sve_load_state(void const *state, u32 const *pfpsr, - int restore_ffr); +extern void sve_save_state(void *state, int save_ffr); +extern void sve_load_state(void const *state, int restore_ffr); extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1); extern void sme_save_state(void *state, int zt); extern void sme_load_state(void const *state, int zt); diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h index c724fcad7ee0..1f32e0967dcd 100644 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ b/arch/arm64/include/asm/fpsimdmacros.h @@ -236,7 +236,7 @@ _sve_wrffr 0 .endm -.macro sve_save nxbase, xpfpsr, save_ffr, nxtmp +.macro sve_save nxbase, save_ffr _for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34 _for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16 cbz \save_ffr, 921f @@ -247,24 +247,15 @@ 922: _sve_str_p 0, \nxbase _sve_ldr_p 0, \nxbase, -16 - mrs x\nxtmp, fpsr - str w\nxtmp, [\xpfpsr] - mrs x\nxtmp, fpcr - str w\nxtmp, [\xpfpsr, #4] .endm -.macro sve_load nxbase, xpfpsr, restore_ffr, nxtmp +.macro sve_load nxbase, restore_ffr _for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34 cbz \restore_ffr, 921f _sve_ldr_p 0, \nxbase _sve_wrffr 0 921: _for n, 0, 15, _sve_ldr_p \n, \nxbase, \n - 16 - - ldr w\nxtmp, [\xpfpsr] - msr fpsr, x\nxtmp - ldr w\nxtmp, [\xpfpsr, #4] - msr fpcr, x\nxtmp .endm .macro sme_save_za nxbase, xvl, nw diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 8d06b62e7188..0030cc1b5219 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -123,8 +123,8 @@ void __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu); void __fpsimd_save_state(struct user_fpsimd_state *fp_regs); void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs); -void __sve_save_state(void *sve_pffr, u32 *fpsr, int save_ffr); -void __sve_restore_state(void *sve_pffr, u32 *fpsr, int restore_ffr); +void __sve_save_state(void *sve, int save_ffr); +void __sve_restore_state(void *sve, int restore_ffr); u64 __guest_enter(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S index a124c6d3766c..1a581597037c 100644 --- a/arch/arm64/kernel/entry-fpsimd.S +++ b/arch/arm64/kernel/entry-fpsimd.S @@ -37,11 +37,10 @@ SYM_FUNC_END(fpsimd_load_state) * Save the SVE state * * x0 - pointer to buffer for state - * x1 - pointer to storage for FPSR - * w2 - Save FFR if non-zero + * w1 - Save FFR if non-zero */ SYM_FUNC_START(sve_save_state) - sve_save 0, x1, w2, 3 + sve_save 0, w1 ret SYM_FUNC_END(sve_save_state) @@ -49,11 +48,10 @@ SYM_FUNC_END(sve_save_state) * Load the SVE state * * x0 - pointer to buffer for state - * x1 - pointer to storage for FPSR - * w2 - Restore FFR if non-zero + * w1 - Restore FFR if non-zero */ SYM_FUNC_START(sve_load_state) - sve_load 0, x1, w2, 4 + sve_load 0, w1 ret SYM_FUNC_END(sve_load_state) diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 2578c2372c89..9806fea8fea7 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -426,8 +426,8 @@ static void task_fpsimd_load(void) if (restore_sve_regs) { WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE); sve_load_state(sve_pffr(¤t->thread), - ¤t->thread.uw.fpsimd_state.fpsr, restore_ffr); + fpsimd_load_common(¤t->thread.uw.fpsimd_state); } else { WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD); fpsimd_load_state(¤t->thread.uw.fpsimd_state); @@ -509,7 +509,8 @@ static void fpsimd_save_user_state(void) sve_save_state((char *)last->sve_state + sve_ffr_offset(vl), - &last->st->fpsr, save_ffr); + save_ffr); + fpsimd_save_common(last->st); *last->fp_type = FP_STATE_SVE; } else { fpsimd_save_state(last->st); diff --git a/arch/arm64/kvm/hyp/fpsimd.S b/arch/arm64/kvm/hyp/fpsimd.S index 30507c50942e..3d5b5e93ee5e 100644 --- a/arch/arm64/kvm/hyp/fpsimd.S +++ b/arch/arm64/kvm/hyp/fpsimd.S @@ -21,11 +21,11 @@ SYM_FUNC_START(__fpsimd_restore_state) SYM_FUNC_END(__fpsimd_restore_state) SYM_FUNC_START(__sve_restore_state) - sve_load 0, x1, w2, 3 + sve_load 0, w1 ret SYM_FUNC_END(__sve_restore_state) SYM_FUNC_START(__sve_save_state) - sve_save 0, x1, w2, 3 + sve_save 0, w1 ret SYM_FUNC_END(__sve_save_state) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 6512dd3f75ae..eb76a863ebb8 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -468,8 +468,8 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) */ sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2); __sve_restore_state(vcpu_sve_pffr(vcpu), - &vcpu->arch.ctxt.fp_regs.fpsr, true); + fpsimd_load_common(&vcpu->arch.ctxt.fp_regs); /* * The effective VL for a VM could differ from the max VL when running a @@ -490,8 +490,8 @@ static inline void __hyp_sve_save_host(void) ctxt_sys_reg(hctxt, ZCR_EL1) = read_sysreg_el1(SYS_ZCR); write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); __sve_save_state(sve_regs + sve_ffr_offset(kvm_host_sve_max_vl), - &hctxt->fp_regs.fpsr, true); + fpsimd_save_common(&hctxt->fp_regs); } static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 31b0245a7e64..2ed4720e4f70 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -35,7 +35,8 @@ static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) * on the VL, so use a consistent (i.e., the maximum) guest VL. */ sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2); - __sve_save_state(vcpu_sve_pffr(vcpu), &vcpu->arch.ctxt.fp_regs.fpsr, true); + __sve_save_state(vcpu_sve_pffr(vcpu), true); + fpsimd_save_common(&vcpu->arch.ctxt.fp_regs); write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); } @@ -55,8 +56,8 @@ static void __hyp_sve_restore_host(void) */ write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); __sve_restore_state(sve_regs + sve_ffr_offset(kvm_host_sve_max_vl), - &hctxt->fp_regs.fpsr, true); + fpsimd_load_common(&hctxt->fp_regs); write_sysreg_el1(ctxt_sys_reg(hctxt, ZCR_EL1), SYS_ZCR); } From 890712d4507b8950bd5fa005077a9178ddde95e6 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:24 +0100 Subject: [PATCH 51/73] arm64: fpsimd: Move fpsimd save/restore inline Currently the FPSIMD register save/restore sequences are written in out-of-line assembly routines. While this works, it's somewhat painful: * As KVM needs to be able to use the sequences in hyp code, separate assembly files are used for the regular kernel and KVM code. While the common logic is shared in assembly macros, this still requires some duplication, and has lead to some trivial divergence. * For historical reasons, the assembly macros take some register arguments as numerical indices (e.g. "fpsimd_save x0, 8" uses x0 and x8), which is simply confusing. * For historical reasons, the SVE save/restore code and FPSIMD save/restore code have distinct sequences for FPSR and FPCR. Ideally this logic would be shared. * The assembly sequences can't be instrumented, and so it's harder than necessary to catch memory safety issues. To handle the above, move the FPSIMD register save/restore sequences to inline assembly, and share the FPSR+FPCR save/restore with SVE. Neither GCC nor LLVM instrument memory arguments to inline assembly, so explicit instrumentation is added in the same manner as other assembly routines. This instrumentation is implicitly disabled by Kbuild for nVHE hyp code. I've used the SVE sequence for restoring FPCR, which uses an unconditional write to FPCR, rather than the conditional write used by the FPSIMD assembly sequence. I believe that in practice, this doesn't matter to a real workload, and given it's possible for the mis-predicted branch to cost more than the necessary micro-architectural synchronization, I strongly suspect any performance impact is within the noise. Looking at the history, the FPSIMD assembly sequence was changed to use a conditional write to FPCR since 2014 in commit: 5959e25729a5 ("arm64: fpsimd: avoid restoring fpcr if the contents haven't change") ... as described in the commit message, this was based on an expectation of implementation style, and was not based on benchmarking. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/fpsimd.h | 68 ++++++++++++++++++++++++- arch/arm64/include/asm/fpsimdmacros.h | 59 --------------------- arch/arm64/include/asm/kvm_hyp.h | 2 - arch/arm64/kernel/entry-fpsimd.S | 20 -------- arch/arm64/kvm/hyp/fpsimd.S | 10 ---- arch/arm64/kvm/hyp/include/hyp/switch.h | 4 +- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 4 +- 7 files changed, 70 insertions(+), 97 deletions(-) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 6fd5cdf5e5f1..19b373ad0ebf 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -22,6 +22,8 @@ #include #include +#define __FPSIMD_PREAMBLE ".arch_extension fp\n" \ + ".arch_extension simd\n" #define __SVE_PREAMBLE ".arch_extension sve\n" #define __SME_PREAMBLE ".arch_extension sme\n" @@ -86,8 +88,70 @@ static inline void fpsimd_load_common(const struct user_fpsimd_state *state) write_sysreg_s(state->fpcr, SYS_FPCR); } -extern void fpsimd_save_state(struct user_fpsimd_state *state); -extern void fpsimd_load_state(struct user_fpsimd_state *state); +static inline void fpsimd_save_vregs(struct user_fpsimd_state *state) +{ + instrument_write(state->vregs, sizeof(state->vregs)); + asm volatile( + __FPSIMD_PREAMBLE + " stp q0, q1, [%[vregs], #16 * 0]\n" + " stp q2, q3, [%[vregs], #16 * 2]\n" + " stp q4, q5, [%[vregs], #16 * 4]\n" + " stp q6, q7, [%[vregs], #16 * 6]\n" + " stp q8, q9, [%[vregs], #16 * 8]\n" + " stp q10, q11, [%[vregs], #16 * 10]\n" + " stp q12, q13, [%[vregs], #16 * 12]\n" + " stp q14, q15, [%[vregs], #16 * 14]\n" + " stp q16, q17, [%[vregs], #16 * 16]\n" + " stp q18, q19, [%[vregs], #16 * 18]\n" + " stp q20, q21, [%[vregs], #16 * 20]\n" + " stp q22, q23, [%[vregs], #16 * 22]\n" + " stp q24, q25, [%[vregs], #16 * 24]\n" + " stp q26, q27, [%[vregs], #16 * 26]\n" + " stp q28, q29, [%[vregs], #16 * 28]\n" + " stp q30, q31, [%[vregs], #16 * 30]\n" + : "=Q" (state->vregs) + : [vregs] "r" (state->vregs) + ); +} + +static inline void fpsimd_load_vregs(const struct user_fpsimd_state *state) +{ + instrument_read(state->vregs, sizeof(state->vregs)); + asm volatile( + __FPSIMD_PREAMBLE + " ldp q0, q1, [%[vregs], #16 * 0]\n" + " ldp q2, q3, [%[vregs], #16 * 2]\n" + " ldp q4, q5, [%[vregs], #16 * 4]\n" + " ldp q6, q7, [%[vregs], #16 * 6]\n" + " ldp q8, q9, [%[vregs], #16 * 8]\n" + " ldp q10, q11, [%[vregs], #16 * 10]\n" + " ldp q12, q13, [%[vregs], #16 * 12]\n" + " ldp q14, q15, [%[vregs], #16 * 14]\n" + " ldp q16, q17, [%[vregs], #16 * 16]\n" + " ldp q18, q19, [%[vregs], #16 * 18]\n" + " ldp q20, q21, [%[vregs], #16 * 20]\n" + " ldp q22, q23, [%[vregs], #16 * 22]\n" + " ldp q24, q25, [%[vregs], #16 * 24]\n" + " ldp q26, q27, [%[vregs], #16 * 26]\n" + " ldp q28, q29, [%[vregs], #16 * 28]\n" + " ldp q30, q31, [%[vregs], #16 * 30]\n" + : + : "Q" (state->vregs), + [vregs] "r" (state->vregs) + ); +} + +static inline void fpsimd_save_state(struct user_fpsimd_state *state) +{ + fpsimd_save_vregs(state); + fpsimd_save_common(state); +} + +static inline void fpsimd_load_state(const struct user_fpsimd_state *state) +{ + fpsimd_load_vregs(state); + fpsimd_load_common(state); +} extern void fpsimd_thread_switch(struct task_struct *next); extern void fpsimd_flush_thread(void); diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h index 1f32e0967dcd..b486c6399bb4 100644 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ b/arch/arm64/include/asm/fpsimdmacros.h @@ -8,65 +8,6 @@ #include -.macro fpsimd_save state, tmpnr - stp q0, q1, [\state, #16 * 0] - stp q2, q3, [\state, #16 * 2] - stp q4, q5, [\state, #16 * 4] - stp q6, q7, [\state, #16 * 6] - stp q8, q9, [\state, #16 * 8] - stp q10, q11, [\state, #16 * 10] - stp q12, q13, [\state, #16 * 12] - stp q14, q15, [\state, #16 * 14] - stp q16, q17, [\state, #16 * 16] - stp q18, q19, [\state, #16 * 18] - stp q20, q21, [\state, #16 * 20] - stp q22, q23, [\state, #16 * 22] - stp q24, q25, [\state, #16 * 24] - stp q26, q27, [\state, #16 * 26] - stp q28, q29, [\state, #16 * 28] - stp q30, q31, [\state, #16 * 30]! - mrs x\tmpnr, fpsr - str w\tmpnr, [\state, #16 * 2] - mrs x\tmpnr, fpcr - str w\tmpnr, [\state, #16 * 2 + 4] -.endm - -.macro fpsimd_restore_fpcr state, tmp - /* - * Writes to fpcr may be self-synchronising, so avoid restoring - * the register if it hasn't changed. - */ - mrs \tmp, fpcr - cmp \tmp, \state - b.eq 9999f - msr fpcr, \state -9999: -.endm - -/* Clobbers \state */ -.macro fpsimd_restore state, tmpnr - ldp q0, q1, [\state, #16 * 0] - ldp q2, q3, [\state, #16 * 2] - ldp q4, q5, [\state, #16 * 4] - ldp q6, q7, [\state, #16 * 6] - ldp q8, q9, [\state, #16 * 8] - ldp q10, q11, [\state, #16 * 10] - ldp q12, q13, [\state, #16 * 12] - ldp q14, q15, [\state, #16 * 14] - ldp q16, q17, [\state, #16 * 16] - ldp q18, q19, [\state, #16 * 18] - ldp q20, q21, [\state, #16 * 20] - ldp q22, q23, [\state, #16 * 22] - ldp q24, q25, [\state, #16 * 24] - ldp q26, q27, [\state, #16 * 26] - ldp q28, q29, [\state, #16 * 28] - ldp q30, q31, [\state, #16 * 30]! - ldr w\tmpnr, [\state, #16 * 2] - msr fpsr, x\tmpnr - ldr w\tmpnr, [\state, #16 * 2 + 4] - fpsimd_restore_fpcr x\tmpnr, \state -.endm - /* Sanity-check macros to help avoid encoding garbage instructions */ .macro _check_general_reg nr diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 0030cc1b5219..8c4602c8f435 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -121,8 +121,6 @@ void __debug_save_host_buffers_nvhe(struct kvm_vcpu *vcpu); void __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu); #endif -void __fpsimd_save_state(struct user_fpsimd_state *fp_regs); -void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs); void __sve_save_state(void *sve, int save_ffr); void __sve_restore_state(void *sve, int restore_ffr); diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S index 1a581597037c..66668bfca5ae 100644 --- a/arch/arm64/kernel/entry-fpsimd.S +++ b/arch/arm64/kernel/entry-fpsimd.S @@ -11,26 +11,6 @@ #include #include -/* - * Save the FP registers. - * - * x0 - pointer to struct fpsimd_state - */ -SYM_FUNC_START(fpsimd_save_state) - fpsimd_save x0, 8 - ret -SYM_FUNC_END(fpsimd_save_state) - -/* - * Load the FP registers. - * - * x0 - pointer to struct fpsimd_state - */ -SYM_FUNC_START(fpsimd_load_state) - fpsimd_restore x0, 8 - ret -SYM_FUNC_END(fpsimd_load_state) - #ifdef CONFIG_ARM64_SVE /* diff --git a/arch/arm64/kvm/hyp/fpsimd.S b/arch/arm64/kvm/hyp/fpsimd.S index 3d5b5e93ee5e..00c56e31484a 100644 --- a/arch/arm64/kvm/hyp/fpsimd.S +++ b/arch/arm64/kvm/hyp/fpsimd.S @@ -10,16 +10,6 @@ .text -SYM_FUNC_START(__fpsimd_save_state) - fpsimd_save x0, 1 - ret -SYM_FUNC_END(__fpsimd_save_state) - -SYM_FUNC_START(__fpsimd_restore_state) - fpsimd_restore x0, 1 - ret -SYM_FUNC_END(__fpsimd_restore_state) - SYM_FUNC_START(__sve_restore_state) sve_load 0, w1 ret diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index eb76a863ebb8..aaa43554fd8e 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -565,7 +565,7 @@ static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu) if (system_supports_sve()) { __hyp_sve_save_host(); } else { - __fpsimd_save_state(&hctxt->fp_regs); + fpsimd_save_state(&hctxt->fp_regs); } if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm))) @@ -625,7 +625,7 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) if (sve_guest) __hyp_sve_restore_guest(vcpu); else - __fpsimd_restore_state(&vcpu->arch.ctxt.fp_regs); + fpsimd_load_state(&vcpu->arch.ctxt.fp_regs); if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm))) write_sysreg_s(__vcpu_sys_reg(vcpu, FPMR), SYS_FPMR); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 2ed4720e4f70..71fcfe8928f0 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -83,7 +83,7 @@ static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) if (vcpu_has_sve(vcpu)) __hyp_sve_save_guest(vcpu); else - __fpsimd_save_state(&vcpu->arch.ctxt.fp_regs); + fpsimd_save_state(&vcpu->arch.ctxt.fp_regs); has_fpmr = kvm_has_fpmr(kern_hyp_va(vcpu->kvm)); if (has_fpmr) @@ -92,7 +92,7 @@ static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) if (system_supports_sve()) __hyp_sve_restore_host(); else - __fpsimd_restore_state(&hctxt->fp_regs); + fpsimd_load_state(&hctxt->fp_regs); if (has_fpmr) write_sysreg_s(ctxt_sys_reg(hctxt, FPMR), SYS_FPMR); From e1b163e405534bcec5026596d90aecd21b766004 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:25 +0100 Subject: [PATCH 52/73] arm64: fpsimd: Use opaque type for SVE state As the SVE state size can vary at runtime, we don't have a concrete type for the in-memory SVE state, and pass this around using a pointer to void. The functions which save/restore the SVE state have a very unusual calling convention, expecting a pointer to the FFR *in the middle of* the in-memory SVE state, which is also passed as a pointer to void. Passing a pointer to the FFR also requires that callers find the live VL and perform some arithmetic, which callers implement differently. Using pointer to void means that it's very easy to introduce errors that cannot be caught by the compiler (e.g. as 'void **' can be assigned to 'void *'). In general this is unnecessarily confusing and fragile. Improve this by adding an opaque 'struct arm64_sve_state', and consistently passing a pointer to this, performing the necessary offsetting *within* the save/restore functions. For the moment, the offsetting is performed in a new '_sve_pffr' assembly macro, using the ADDVL and ADDPL instructions. These add a multiple of the live vector length and predicate length respectively. The ADDVL immediate range cannot encode 32, so this is split into two increments of 16. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/fpsimd.h | 24 +++--------------------- arch/arm64/include/asm/fpsimdmacros.h | 9 +++++++++ arch/arm64/include/asm/kvm_host.h | 8 ++------ arch/arm64/include/asm/kvm_hyp.h | 4 ++-- arch/arm64/include/asm/processor.h | 4 +++- arch/arm64/kernel/fpsimd.c | 21 ++++++++++----------- arch/arm64/kvm/arm.c | 4 ++-- arch/arm64/kvm/guest.c | 4 ++-- arch/arm64/kvm/hyp/include/hyp/switch.h | 8 +++----- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 7 +++---- arch/arm64/kvm/hyp/nvhe/setup.c | 2 +- 11 files changed, 40 insertions(+), 55 deletions(-) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 19b373ad0ebf..581661a759a3 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -162,7 +162,7 @@ extern void fpsimd_update_current_state(struct user_fpsimd_state const *state); struct cpu_fp_state { struct user_fpsimd_state *st; - void *sve_state; + struct arm64_sve_state *sve_state; void *sme_state; u64 *svcr; u64 *fpmr; @@ -195,24 +195,6 @@ extern void task_smstop_sm(struct task_struct *task); /* Maximum VL that SVE/SME VL-agnostic software can transparently support */ #define VL_ARCH_MAX 0x100 -/* Offset of FFR in the SVE register dump */ -static inline size_t sve_ffr_offset(int vl) -{ - return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET; -} - -static inline void *sve_pffr(struct thread_struct *thread) -{ - unsigned int vl; - - if (system_supports_sme() && thread_sm_enabled(thread)) - vl = thread_get_sme_vl(thread); - else - vl = thread_get_sve_vl(thread); - - return (char *)thread->sve_state + sve_ffr_offset(vl); -} - static inline void *thread_zt_state(struct thread_struct *thread) { /* The ZT register state is stored immediately after the ZA state */ @@ -233,8 +215,8 @@ static inline unsigned int sve_get_vl(void) return vl; } -extern void sve_save_state(void *state, int save_ffr); -extern void sve_load_state(void const *state, int restore_ffr); +extern void sve_save_state(struct arm64_sve_state *state, int save_ffr); +extern void sve_load_state(const struct arm64_sve_state *state, int restore_ffr); extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1); extern void sme_save_state(void *state, int zt); extern void sme_load_state(void const *state, int zt); diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h index b486c6399bb4..e613dc94dc35 100644 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ b/arch/arm64/include/asm/fpsimdmacros.h @@ -177,7 +177,15 @@ _sve_wrffr 0 .endm +.macro _sve_pffr ptr + .arch_extension sve + addvl \ptr, \ptr, #16 + addvl \ptr, \ptr, #16 + addpl \ptr, \ptr, #16 +.endm + .macro sve_save nxbase, save_ffr + _sve_pffr x\nxbase _for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34 _for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16 cbz \save_ffr, 921f @@ -191,6 +199,7 @@ .endm .macro sve_load nxbase, restore_ffr + _sve_pffr x\nxbase _for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34 cbz \restore_ffr, 921f _sve_ldr_p 0, \nxbase diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index ae98375a8d32..639f9dd967f9 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -762,7 +762,7 @@ struct kvm_host_data { * Hyp VA. * sve_regs is only used in pKVM and if system_supports_sve(). */ - u8 *sve_regs; + struct arm64_sve_state *sve_regs; /* Ownership of the FP regs */ enum { @@ -856,7 +856,7 @@ struct kvm_vcpu_arch { * floating point code saves the register state of a task it * records which view it saved in fp_type. */ - void *sve_state; + struct arm64_sve_state *sve_state; enum fp_type fp_type; unsigned int sve_max_vl; @@ -1100,10 +1100,6 @@ struct kvm_vcpu_arch { #define NESTED_SERROR_PENDING __vcpu_single_flag(sflags, BIT(8)) -/* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */ -#define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) + \ - sve_ffr_offset((vcpu)->arch.sve_max_vl)) - #define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl) #define vcpu_sve_zcr_elx(vcpu) \ diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 8c4602c8f435..190c256e34c0 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -121,8 +121,8 @@ void __debug_save_host_buffers_nvhe(struct kvm_vcpu *vcpu); void __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu); #endif -void __sve_save_state(void *sve, int save_ffr); -void __sve_restore_state(void *sve, int restore_ffr); +void __sve_save_state(struct arm64_sve_state *sve, int save_ffr); +void __sve_restore_state(struct arm64_sve_state *sve, int restore_ffr); u64 __guest_enter(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index e30c4c8e3a7a..96c0c30eac50 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -130,6 +130,8 @@ enum fp_type { FP_STATE_SVE, }; +struct arm64_sve_state; /* Opaque type */ + struct cpu_context { unsigned long x19; unsigned long x20; @@ -164,7 +166,7 @@ struct thread_struct { enum fp_type fp_type; /* registers FPSIMD or SVE? */ unsigned int fpsimd_cpu; - void *sve_state; /* SVE registers, if any */ + struct arm64_sve_state *sve_state; /* SVE registers, if any */ void *sme_state; /* ZA and ZT state, if any */ unsigned int vl[ARM64_VEC_MAX]; /* vector length */ unsigned int vl_onexec[ARM64_VEC_MAX]; /* vl after next exec */ diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 9806fea8fea7..0da2f75de5dd 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -425,8 +425,7 @@ static void task_fpsimd_load(void) if (restore_sve_regs) { WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE); - sve_load_state(sve_pffr(¤t->thread), - restore_ffr); + sve_load_state(current->thread.sve_state, restore_ffr); fpsimd_load_common(¤t->thread.uw.fpsimd_state); } else { WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD); @@ -507,9 +506,7 @@ static void fpsimd_save_user_state(void) return; } - sve_save_state((char *)last->sve_state + - sve_ffr_offset(vl), - save_ffr); + sve_save_state(last->sve_state, save_ffr); fpsimd_save_common(last->st); *last->fp_type = FP_STATE_SVE; } else { @@ -641,7 +638,8 @@ static __uint128_t arm64_cpu_to_le128(__uint128_t x) #define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x) -static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst, +static void __fpsimd_to_sve(struct arm64_sve_state *sst, + struct user_fpsimd_state const *fst, unsigned int vq) { unsigned int i; @@ -668,7 +666,7 @@ static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst, static inline void fpsimd_to_sve(struct task_struct *task) { unsigned int vq; - void *sst = task->thread.sve_state; + struct arm64_sve_state *sst = task->thread.sve_state; struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; if (!system_supports_sve() && !system_supports_sme()) @@ -692,7 +690,7 @@ static inline void fpsimd_to_sve(struct task_struct *task) static inline void sve_to_fpsimd(struct task_struct *task) { unsigned int vq, vl; - void const *sst = task->thread.sve_state; + const struct arm64_sve_state *sst = task->thread.sve_state; struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state; unsigned int i; __uint128_t const *p; @@ -791,7 +789,7 @@ void fpsimd_sync_from_effective_state(struct task_struct *task) void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task) { unsigned int vq; - void *sst = task->thread.sve_state; + struct arm64_sve_state *sst = task->thread.sve_state; struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; if (task->thread.fp_type != FP_STATE_SVE) @@ -809,7 +807,8 @@ static int change_live_vector_length(struct task_struct *task, { unsigned int sve_vl = task_get_sve_vl(task); unsigned int sme_vl = task_get_sme_vl(task); - void *sve_state = NULL, *sme_state = NULL; + struct arm64_sve_state *sve_state = NULL; + void *sme_state = NULL; if (type == ARM64_VEC_SME) sme_vl = vl; @@ -1645,7 +1644,7 @@ static void fpsimd_flush_thread_vl(enum vec_type type) void fpsimd_flush_thread(void) { - void *sve_state = NULL; + struct arm64_sve_state *sve_state = NULL; void *sme_state = NULL; if (!system_supports_fpsimd()) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index d51586b74915..d84e80b2ce54 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2495,7 +2495,7 @@ static void __init teardown_hyp_mode(void) continue; if (free_sve) { - u8 *sve_regs; + struct arm64_sve_state *sve_regs; sve_regs = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs; free_pages((unsigned long) sve_regs, pkvm_host_sve_state_order()); @@ -2644,7 +2644,7 @@ static void finalize_init_hyp_mode(void) if (system_supports_sve() && is_protected_kvm_enabled()) { for_each_possible_cpu(cpu) { - u8 *sve_regs; + struct arm64_sve_state *sve_regs; sve_regs = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs; per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs = diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 332c453b87cf..b01d6622b872 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -500,7 +500,7 @@ static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) if (!kvm_arm_vcpu_sve_finalized(vcpu)) return -EPERM; - if (copy_to_user(uptr, vcpu->arch.sve_state + region.koffset, + if (copy_to_user(uptr, (void *)vcpu->arch.sve_state + region.koffset, region.klen) || clear_user(uptr + region.klen, region.upad)) return -EFAULT; @@ -526,7 +526,7 @@ static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) if (!kvm_arm_vcpu_sve_finalized(vcpu)) return -EPERM; - if (copy_from_user(vcpu->arch.sve_state + region.koffset, uptr, + if (copy_from_user((void *)vcpu->arch.sve_state + region.koffset, uptr, region.klen)) return -EFAULT; diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index aaa43554fd8e..ee366a536c77 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -467,8 +467,7 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) * vCPU. Start off with the max VL so we can load the SVE state. */ sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2); - __sve_restore_state(vcpu_sve_pffr(vcpu), - true); + __sve_restore_state(kern_hyp_va(vcpu->arch.sve_state), true); fpsimd_load_common(&vcpu->arch.ctxt.fp_regs); /* @@ -485,12 +484,11 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) static inline void __hyp_sve_save_host(void) { struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); - u8 *sve_regs = *host_data_ptr(sve_regs); + struct arm64_sve_state *sve_regs = *host_data_ptr(sve_regs); ctxt_sys_reg(hctxt, ZCR_EL1) = read_sysreg_el1(SYS_ZCR); write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); - __sve_save_state(sve_regs + sve_ffr_offset(kvm_host_sve_max_vl), - true); + __sve_save_state(sve_regs, true); fpsimd_save_common(&hctxt->fp_regs); } diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 71fcfe8928f0..dee9fedd9592 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -35,7 +35,7 @@ static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) * on the VL, so use a consistent (i.e., the maximum) guest VL. */ sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2); - __sve_save_state(vcpu_sve_pffr(vcpu), true); + __sve_save_state(kern_hyp_va(vcpu->arch.sve_state), true); fpsimd_save_common(&vcpu->arch.ctxt.fp_regs); write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); } @@ -43,7 +43,7 @@ static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) static void __hyp_sve_restore_host(void) { struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); - u8 *sve_regs = *host_data_ptr(sve_regs); + struct arm64_sve_state *sve_regs = *host_data_ptr(sve_regs); /* * On saving/restoring host sve state, always use the maximum VL for @@ -55,8 +55,7 @@ static void __hyp_sve_restore_host(void) * need to be revisited. */ write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); - __sve_restore_state(sve_regs + sve_ffr_offset(kvm_host_sve_max_vl), - true); + __sve_restore_state(sve_regs, true); fpsimd_load_common(&hctxt->fp_regs); write_sysreg_el1(ctxt_sys_reg(hctxt, ZCR_EL1), SYS_ZCR); } diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 399608694920..e704199048c4 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -82,7 +82,7 @@ static int pkvm_create_host_sve_mappings(void) for (i = 0; i < hyp_nr_cpus; i++) { struct kvm_host_data *host_data = per_cpu_ptr(&kvm_host_data, i); - u8 *sve_regs = host_data->sve_regs; + struct arm64_sve_state *sve_regs = host_data->sve_regs; start = kern_hyp_va(sve_regs); end = start + PAGE_ALIGN(pkvm_host_sve_state_size()); From eb1a68a00c0afc88ec60bc96b45d4bfb478ab716 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:26 +0100 Subject: [PATCH 53/73] arm64: fpsimd: Use opaque type for SME state As the SME state size can vary at runtime, we don't have a concrete type for the in-memory SME state, and pass this around using a pointer to void. Using pointer to void means that it's very easy to introduce errors that cannot be caught by the compiler (e.g. as 'void **' can be assigned to 'void *'). Improve this by adding an opaque 'struct arm64_sme_state', and consistently passing a pointer to this. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/fpsimd.h | 8 ++++---- arch/arm64/include/asm/processor.h | 3 ++- arch/arm64/kernel/fpsimd.c | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 581661a759a3..fff6d54afd9f 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -163,7 +163,7 @@ extern void fpsimd_update_current_state(struct user_fpsimd_state const *state); struct cpu_fp_state { struct user_fpsimd_state *st; struct arm64_sve_state *sve_state; - void *sme_state; + struct arm64_sme_state *sme_state; u64 *svcr; u64 *fpmr; unsigned int sve_vl; @@ -199,7 +199,7 @@ static inline void *thread_zt_state(struct thread_struct *thread) { /* The ZT register state is stored immediately after the ZA state */ unsigned int sme_vq = sve_vq_from_vl(thread_get_sme_vl(thread)); - return thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq); + return (void *)thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq); } static inline unsigned int sve_get_vl(void) @@ -218,8 +218,8 @@ static inline unsigned int sve_get_vl(void) extern void sve_save_state(struct arm64_sve_state *state, int save_ffr); extern void sve_load_state(const struct arm64_sve_state *state, int restore_ffr); extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1); -extern void sme_save_state(void *state, int zt); -extern void sme_load_state(void const *state, int zt); +extern void sme_save_state(struct arm64_sme_state *state, int zt); +extern void sme_load_state(const struct arm64_sme_state *state, int zt); struct arm64_cpu_capabilities; extern void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__unused); diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 96c0c30eac50..c2a627f39314 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -131,6 +131,7 @@ enum fp_type { }; struct arm64_sve_state; /* Opaque type */ +struct arm64_sme_state; /* Opaque type */ struct cpu_context { unsigned long x19; @@ -167,7 +168,7 @@ struct thread_struct { enum fp_type fp_type; /* registers FPSIMD or SVE? */ unsigned int fpsimd_cpu; struct arm64_sve_state *sve_state; /* SVE registers, if any */ - void *sme_state; /* ZA and ZT state, if any */ + struct arm64_sme_state *sme_state; /* ZA and ZT state, if any */ unsigned int vl[ARM64_VEC_MAX]; /* vector length */ unsigned int vl_onexec[ARM64_VEC_MAX]; /* vl after next exec */ unsigned long fault_address; /* fault info */ diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 0da2f75de5dd..b9506422d29c 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -808,7 +808,7 @@ static int change_live_vector_length(struct task_struct *task, unsigned int sve_vl = task_get_sve_vl(task); unsigned int sme_vl = task_get_sme_vl(task); struct arm64_sve_state *sve_state = NULL; - void *sme_state = NULL; + struct arm64_sme_state *sme_state = NULL; if (type == ARM64_VEC_SME) sme_vl = vl; @@ -1645,7 +1645,7 @@ static void fpsimd_flush_thread_vl(enum vec_type type) void fpsimd_flush_thread(void) { struct arm64_sve_state *sve_state = NULL; - void *sme_state = NULL; + struct arm64_sme_state *sme_state = NULL; if (!system_supports_fpsimd()) return; From 2768101b397658352f2aade711135528c3568fb4 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:27 +0100 Subject: [PATCH 54/73] arm64: fpsimd: Move SVE save/restore inline Currently the SVE register save/restore sequences are written in out-of-line assembly routines. While this works, it's somewhat painful: * As KVM needs to be able to use the sequences in hyp code, separate assembly files are used for the regular kernel and KVM code. While the common logic is shared in assembly macros, this still requires some duplication, and has lead to some trivial divergence. * As the SVE LDR/STR instrucitons have limited addressing modes, the assembly macros use an awkward pattern requiring negative offsets. This could be written more clearly with addresses being generated in C code. * As the FFR does not always exist in streaming mode, some awkward conditional branching has been written in assembly which could be clearer in C (and would permit the compiler to optimize out unnecessary branches in some cases). * For historical reasons, the assembly macros take some register arguments as numerical indices (e.g. "sve_save 0, x1" uses x0 and x1), which is simply confusing. * For historical reasons, the SVE save/restore code and FPSIMD save/restore code have a distinct sequences for FPSR and FPCR. Ideally this logic would be shared. * The assembly sequences can't be instrumented, and so it's harder than necessary to catch memory safety issues. To handle the above, move the SVE register save/restore sequences to inline assembly. Neither GCC nor LLVM instrument memory arguments to inline assembly, so explicit instrumentation is added in the same manner as other assembly routines. This instrumentation is implicitly disabled by Kbuild for nVHE hyp code. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/fpsimd.h | 119 +++++++++++++++++++++++- arch/arm64/include/asm/fpsimdmacros.h | 61 ------------ arch/arm64/include/asm/kvm_hyp.h | 3 - arch/arm64/kernel/entry-fpsimd.S | 22 ----- arch/arm64/kvm/hyp/fpsimd.S | 21 ----- arch/arm64/kvm/hyp/include/hyp/switch.h | 4 +- arch/arm64/kvm/hyp/nvhe/Makefile | 2 +- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 4 +- arch/arm64/kvm/hyp/vhe/Makefile | 2 +- 9 files changed, 123 insertions(+), 115 deletions(-) delete mode 100644 arch/arm64/kvm/hyp/fpsimd.S diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index fff6d54afd9f..8f1b844f000f 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -215,8 +215,123 @@ static inline unsigned int sve_get_vl(void) return vl; } -extern void sve_save_state(struct arm64_sve_state *state, int save_ffr); -extern void sve_load_state(const struct arm64_sve_state *state, int restore_ffr); +#define FOR_EACH_Z_REG(idx_str, asm_str) \ + " .irp " idx_str ",0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31\n" \ + asm_str "\n" \ + " .endr\n" + +#define FOR_EACH_P_REG(idx_str, asm_str) \ + " .irp " idx_str ",0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \ + asm_str "\n" \ + " .endr\n" + +static inline void __sve_save_z(struct arm64_sve_state *state, unsigned long vl) +{ + instrument_write(state, SVE_NUM_ZREGS * vl); + asm volatile( + __SVE_PREAMBLE + FOR_EACH_Z_REG("n", "str z\\n, [%[zregs], #\\n, MUL VL]") + : + : [zregs] "r" (state) + : "memory" + ); +} + +static inline void __sve_load_z(const struct arm64_sve_state *state, unsigned long vl) +{ + instrument_read(state, SVE_NUM_ZREGS * vl); + asm volatile( + __SVE_PREAMBLE + FOR_EACH_Z_REG("n", "ldr z\\n, [%[zregs], #\\n, MUL VL]") + : + : [zregs] "r" (state) + : "memory" + ); +} + +static inline void __sve_save_p(struct arm64_sve_state *state, unsigned long vl, bool ffr) +{ + void *pregs = (void *)state + SVE_NUM_ZREGS * vl; + unsigned long pl = vl / 8; + void *pffr = pregs + SVE_NUM_PREGS * pl; + + instrument_write(pregs, SVE_NUM_PREGS * pl); + asm volatile( + __SVE_PREAMBLE + FOR_EACH_P_REG("n", "str p\\n, [%[pregs], #\\n, MUL VL]\n") + : + : [pregs] "r" (pregs) + : "memory" + ); + + instrument_write(pffr, pl); + if (ffr) { + asm volatile( + __SVE_PREAMBLE + " rdffr p0.b\n" + " str p0, [%[pffr]]\n" + " ldr p0, [%[pregs]]\n" + : + : [pregs] "r" (pregs), + [pffr] "r" (pffr) + : "memory" + ); + } else { + asm volatile( + __SVE_PREAMBLE + " pfalse p0.b\n" + " str p0, [%[pffr]]\n" + " ldr p0, [%[pregs]]\n" + : + : [pregs] "r" (pregs), + [pffr] "r" (pffr) + : "memory" + ); + } +} + +static inline void __sve_load_p(const struct arm64_sve_state *state, unsigned long vl, bool ffr) +{ + const void *pregs = (const void *)state + SVE_NUM_ZREGS * vl; + unsigned long pl = vl / 8; + const void *pffr = pregs + SVE_NUM_PREGS * pl; + + if (ffr) { + instrument_read(pffr, pl); + asm volatile( + __SVE_PREAMBLE + " ldr p0, [%[pffr]]\n" + " wrffr p0.b\n" + : + : [pffr] "r" (pffr) + : "memory" + ); + } + + instrument_read(pregs, SVE_NUM_PREGS * pl); + asm volatile( + __SVE_PREAMBLE + FOR_EACH_P_REG("n", "ldr p\\n, [%[pregs], #\\n, MUL VL]\n") + : + : [pregs] "r" (pregs) + : "memory" + ); +} + +static inline void sve_save_state(struct arm64_sve_state *state, bool ffr) +{ + unsigned long vl = sve_get_vl(); + __sve_save_z(state, vl); + __sve_save_p(state, vl, ffr); +} + +static inline void sve_load_state(const struct arm64_sve_state *state, bool ffr) +{ + unsigned long vl = sve_get_vl(); + __sve_load_z(state, vl); + __sve_load_p(state, vl, ffr); +} + extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1); extern void sme_save_state(struct arm64_sme_state *state, int zt); extern void sme_load_state(const struct arm64_sme_state *state, int zt); diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h index e613dc94dc35..5f03fe51d0bf 100644 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ b/arch/arm64/include/asm/fpsimdmacros.h @@ -42,36 +42,6 @@ /* Deprecated macros for SVE instructions */ -/* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */ -.macro _sve_str_v nz, nxbase, offset=0 - .arch_extension sve - str z\nz, [X\nxbase, #\offset, MUL VL] -.endm - -/* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */ -.macro _sve_ldr_v nz, nxbase, offset=0 - .arch_extension sve - ldr z\nz, [X\nxbase, #\offset, MUL VL] -.endm - -/* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */ -.macro _sve_str_p np, nxbase, offset=0 - .arch_extension sve - str p\np, [X\nxbase, #\offset, MUL VL] -.endm - -/* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */ -.macro _sve_ldr_p np, nxbase, offset=0 - .arch_extension sve - ldr p\np, [x\nxbase, #\offset, MUL VL] -.endm - -/* RDFFR (unpredicated): RDFFR P\np.B */ -.macro _sve_rdffr np - .arch_extension sve - rdffr p\np\().b -.endm - /* WRFFR P\np.B */ .macro _sve_wrffr np .arch_extension sve @@ -177,37 +147,6 @@ _sve_wrffr 0 .endm -.macro _sve_pffr ptr - .arch_extension sve - addvl \ptr, \ptr, #16 - addvl \ptr, \ptr, #16 - addpl \ptr, \ptr, #16 -.endm - -.macro sve_save nxbase, save_ffr - _sve_pffr x\nxbase - _for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34 - _for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16 - cbz \save_ffr, 921f - _sve_rdffr 0 - b 922f -921: - _sve_pfalse 0 // Zero out FFR -922: - _sve_str_p 0, \nxbase - _sve_ldr_p 0, \nxbase, -16 -.endm - -.macro sve_load nxbase, restore_ffr - _sve_pffr x\nxbase - _for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34 - cbz \restore_ffr, 921f - _sve_ldr_p 0, \nxbase - _sve_wrffr 0 -921: - _for n, 0, 15, _sve_ldr_p \n, \nxbase, \n - 16 -.endm - .macro sme_save_za nxbase, xvl, nw mov w\nw, #0 diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 190c256e34c0..ad19de1d0654 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -121,9 +121,6 @@ void __debug_save_host_buffers_nvhe(struct kvm_vcpu *vcpu); void __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu); #endif -void __sve_save_state(struct arm64_sve_state *sve, int save_ffr); -void __sve_restore_state(struct arm64_sve_state *sve, int restore_ffr); - u64 __guest_enter(struct kvm_vcpu *vcpu); bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt, u32 func_id); diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S index 66668bfca5ae..f95753635625 100644 --- a/arch/arm64/kernel/entry-fpsimd.S +++ b/arch/arm64/kernel/entry-fpsimd.S @@ -13,28 +13,6 @@ #ifdef CONFIG_ARM64_SVE -/* - * Save the SVE state - * - * x0 - pointer to buffer for state - * w1 - Save FFR if non-zero - */ -SYM_FUNC_START(sve_save_state) - sve_save 0, w1 - ret -SYM_FUNC_END(sve_save_state) - -/* - * Load the SVE state - * - * x0 - pointer to buffer for state - * w1 - Restore FFR if non-zero - */ -SYM_FUNC_START(sve_load_state) - sve_load 0, w1 - ret -SYM_FUNC_END(sve_load_state) - /* * Zero all SVE registers but the first 128-bits of each vector * diff --git a/arch/arm64/kvm/hyp/fpsimd.S b/arch/arm64/kvm/hyp/fpsimd.S deleted file mode 100644 index 00c56e31484a..000000000000 --- a/arch/arm64/kvm/hyp/fpsimd.S +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2015 - ARM Ltd - * Author: Marc Zyngier - */ - -#include - -#include - - .text - -SYM_FUNC_START(__sve_restore_state) - sve_load 0, w1 - ret -SYM_FUNC_END(__sve_restore_state) - -SYM_FUNC_START(__sve_save_state) - sve_save 0, w1 - ret -SYM_FUNC_END(__sve_save_state) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index ee366a536c77..1f12c4ba295a 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -467,7 +467,7 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) * vCPU. Start off with the max VL so we can load the SVE state. */ sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2); - __sve_restore_state(kern_hyp_va(vcpu->arch.sve_state), true); + sve_load_state(kern_hyp_va(vcpu->arch.sve_state), true); fpsimd_load_common(&vcpu->arch.ctxt.fp_regs); /* @@ -488,7 +488,7 @@ static inline void __hyp_sve_save_host(void) ctxt_sys_reg(hctxt, ZCR_EL1) = read_sysreg_el1(SYS_ZCR); write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); - __sve_save_state(sve_regs, true); + sve_save_state(sve_regs, true); fpsimd_save_common(&hctxt->fp_regs); } diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index 62cdfbff7562..f57450ebcb49 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile @@ -26,7 +26,7 @@ hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o hyp-main.o hyp-smp.o psci-relay.o early_alloc.o page_alloc.o \ cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ - ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o ../vgic-v5-sr.o + ../hyp-entry.o ../exception.o ../pgtable.o ../vgic-v5-sr.o hyp-obj-y += ../../../kernel/smccc-call.o hyp-obj-$(CONFIG_LIST_HARDENED) += list_debug.o hyp-obj-$(CONFIG_NVHE_EL2_TRACING) += clock.o trace.o events.o diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index dee9fedd9592..676f756e084d 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -35,7 +35,7 @@ static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) * on the VL, so use a consistent (i.e., the maximum) guest VL. */ sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2); - __sve_save_state(kern_hyp_va(vcpu->arch.sve_state), true); + sve_save_state(kern_hyp_va(vcpu->arch.sve_state), true); fpsimd_save_common(&vcpu->arch.ctxt.fp_regs); write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); } @@ -55,7 +55,7 @@ static void __hyp_sve_restore_host(void) * need to be revisited. */ write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); - __sve_restore_state(sve_regs, true); + sve_load_state(sve_regs, true); fpsimd_load_common(&hctxt->fp_regs); write_sysreg_el1(ctxt_sys_reg(hctxt, ZCR_EL1), SYS_ZCR); } diff --git a/arch/arm64/kvm/hyp/vhe/Makefile b/arch/arm64/kvm/hyp/vhe/Makefile index 9695328bbd96..d6b3475145c0 100644 --- a/arch/arm64/kvm/hyp/vhe/Makefile +++ b/arch/arm64/kvm/hyp/vhe/Makefile @@ -10,4 +10,4 @@ CFLAGS_switch.o += -Wno-override-init obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ - ../fpsimd.o ../hyp-entry.o ../exception.o ../vgic-v5-sr.o + ../hyp-entry.o ../exception.o ../vgic-v5-sr.o From 18618d9ea1fbc251970fe8a072f6fe10266320e0 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:28 +0100 Subject: [PATCH 55/73] arm64: fpsimd: Move sve_flush_live() inline Currently sve_flush_live() is written in out-of-line assembly. It would be nice if we could move it inline such that control flow can be written more clearly in C, and to permit the removal of otherwise unused assembly macros. The 'flush_ffr' argument is redundant as sve_flush_live() is always called from non-streaming mode, and all callers pass 'true'. Remove the argument and make it a requirement that the function is called from non-streaming mode. The 'vq_minus_1' argument is unnecessary, as sve_flush_live() can read the live VL directly using the RDVL instruction (wrapped by the sve_get_vl() helper function). Move the function to C, with the simplifications above. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/fpsimd.h | 25 +++++++++++++++++++++- arch/arm64/include/asm/fpsimdmacros.h | 30 --------------------------- arch/arm64/kernel/entry-common.c | 8 ++----- arch/arm64/kernel/entry-fpsimd.S | 22 -------------------- arch/arm64/kernel/fpsimd.c | 2 +- 5 files changed, 27 insertions(+), 60 deletions(-) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 8f1b844f000f..9dfe53204ebf 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -332,7 +332,30 @@ static inline void sve_load_state(const struct arm64_sve_state *state, bool ffr) __sve_load_p(state, vl, ffr); } -extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1); +/* + * Zero all SVE registers except for the first 128 bits of each vector. + * + * The caller must ensure that the VL has been configured and the CPU must be + * in non-streaming mode. + */ +static inline void sve_flush_live(void) +{ + unsigned long vl = sve_get_vl(); + + if (vl > sizeof(__uint128_t)) { + asm volatile( + __FPSIMD_PREAMBLE + FOR_EACH_Z_REG("n", "mov v\\n\\().16b, v\\n\\().16b") + ); + } + + asm volatile( + __SVE_PREAMBLE + FOR_EACH_P_REG("n", "pfalse p\\n\\().b") + " wrffr p0.b\n" + ); +} + extern void sme_save_state(struct arm64_sme_state *state, int zt); extern void sme_load_state(const struct arm64_sme_state *state, int zt); diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h index 5f03fe51d0bf..9e352b5c6b76 100644 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ b/arch/arm64/include/asm/fpsimdmacros.h @@ -40,20 +40,6 @@ .endif .endm -/* Deprecated macros for SVE instructions */ - -/* WRFFR P\np.B */ -.macro _sve_wrffr np - .arch_extension sve - wrffr p\np\().b -.endm - -/* PFALSE P\np.B */ -.macro _sve_pfalse np - .arch_extension sve - pfalse p\np\().b -.endm - /* Deprecated macros for SME instructions */ /* RDSVL X\nx, #\imm */ @@ -131,22 +117,6 @@ .purgem _for__body .endm -/* Preserve the first 128-bits of Znz and zero the rest. */ -.macro _sve_flush_z nz - _sve_check_zreg \nz - mov v\nz\().16b, v\nz\().16b -.endm - -.macro sve_flush_z - _for n, 0, 31, _sve_flush_z \n -.endm -.macro sve_flush_p - _for n, 0, 15, _sve_pfalse \n -.endm -.macro sve_flush_ffr - _sve_wrffr 0 -.endm - .macro sme_save_za nxbase, xvl, nw mov w\nw, #0 diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index cb54335465f6..2352297330e1 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -237,12 +237,8 @@ static inline void fpsimd_syscall_enter(void) if (!system_supports_sve()) return; - if (test_thread_flag(TIF_SVE)) { - unsigned int sve_vq_minus_one; - - sve_vq_minus_one = sve_vq_from_vl(task_get_sve_vl(current)) - 1; - sve_flush_live(true, sve_vq_minus_one); - } + if (test_thread_flag(TIF_SVE)) + sve_flush_live(); /* * Any live non-FPSIMD SVE state has been zeroed. Allow diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S index f95753635625..2a4755113b99 100644 --- a/arch/arm64/kernel/entry-fpsimd.S +++ b/arch/arm64/kernel/entry-fpsimd.S @@ -11,28 +11,6 @@ #include #include -#ifdef CONFIG_ARM64_SVE - -/* - * Zero all SVE registers but the first 128-bits of each vector - * - * VQ must already be configured by caller, any further updates of VQ - * will need to ensure that the register state remains valid. - * - * x0 = include FFR? - * x1 = VQ - 1 - */ -SYM_FUNC_START(sve_flush_live) - cbz x1, 1f // A VQ-1 of 0 is 128 bits so no extra Z state - sve_flush_z -1: sve_flush_p - tbz x0, #0, 2f - sve_flush_ffr -2: ret -SYM_FUNC_END(sve_flush_live) - -#endif /* CONFIG_ARM64_SVE */ - #ifdef CONFIG_ARM64_SME /* diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index b9506422d29c..25dc5afe9ba0 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1338,7 +1338,7 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs) if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { unsigned long vq = sve_vq_from_vl(task_get_sve_vl(current)); sysreg_clear_set_s(SYS_ZCR_EL1, ZCR_ELx_LEN, vq - 1); - sve_flush_live(true, vq - 1); + sve_flush_live(); fpsimd_bind_task_to_cpu(); } else { fpsimd_to_sve(current); From bfdfafd907204dd1ca7c59e175c2d636ff2361c4 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:29 +0100 Subject: [PATCH 56/73] arm64: fpsimd: Move SME save/restore inline Currently the SVE register save/restore sequences are written in out-of-line assembly routines. While this works, it's somewhat painful: * For KVM to use the sequences, portions of the logic will need to be duplicated in KVM hyp code. While the common logic can be shared in assembly macros, this is very likely to lead to unnecessary divergence and be a maintenance burden. * For historical reasons, the assembly macros take some register arguments as numerical indices (e.g. "sme_save_za 0, x2, 12" uses x0, x1, and x12), which is simply confusing. * Address generation and control flow are far clearer in C than in assembly. * The assembly sequences can't be instrumented, and so it's harder than necessary to catch memory safety issues. To handle the above, move the SME register save/restore sequences to inline assembly. Neither GCC nor LLVM instrument memory arguments to inline assembly, so explicit instrumentation is added in the same manner as other assembly routines. This instrumentation is implicitly disabled by Kbuild for nVHE hyp code. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/fpsimd.h | 106 +++++++++++++++++++++++++- arch/arm64/include/asm/fpsimdmacros.h | 76 ------------------ arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/entry-fpsimd.S | 48 ------------ 4 files changed, 104 insertions(+), 128 deletions(-) delete mode 100644 arch/arm64/kernel/entry-fpsimd.S diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 9dfe53204ebf..a67d5774e672 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -356,9 +356,6 @@ static inline void sve_flush_live(void) ); } -extern void sme_save_state(struct arm64_sme_state *state, int zt); -extern void sme_load_state(const struct arm64_sme_state *state, int zt); - struct arm64_cpu_capabilities; extern void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__unused); extern void cpu_enable_sve(const struct arm64_cpu_capabilities *__unused); @@ -638,6 +635,106 @@ static inline size_t __sme_state_size(unsigned int sme_vl) return size; } +static inline void __sme_save_za(struct arm64_sme_state *state, unsigned long svl) +{ + /* + * The argument to LDR/STR (array vector) can only encode W12-W15. + * The "Ucj" constraint exists for this, but is only supported by GCC + * 14.1.0+ and LLVM 18.1.0+. + */ + register unsigned int v asm ("w12"); + + instrument_write(state, svl * svl); + for (v = 0; v < svl; v++) { + void *pav = (void *)state + v * svl; + + asm volatile( + __SME_PREAMBLE + " str za[%w[v], #0], [%[pav]]\n" + : + : [v] "r" (v), + [pav] "r" (pav) + : "memory" + ); + } +} + +static inline void __sme_load_za(const struct arm64_sme_state *state, unsigned long svl) +{ + /* See comment in __sme_save_za */ + register unsigned int v asm ("w12"); + + instrument_read(state, svl * svl); + for (v = 0; v < svl; v++) { + void *pav = (void *)state + v * svl; + + asm volatile( + __SME_PREAMBLE + " ldr za[%w[v], #0], [%[pav]]\n" + : + : [v] "r" (v), + [pav] "r" (pav) + : "memory" + ); + } +} + +static inline void __sme_save_zt(struct arm64_sme_state *state, unsigned long svl) +{ + void *pzt = (void *)state + svl * svl; + + instrument_write(pzt, 64); + asm volatile( + __DEFINE_ASM_GPR_NUMS + /* + * STR ZT0, [] + * Supported by binutils 2.41+. + * Supported by LLVM 16+ + */ + " .inst 0xe13f8000 | ((.L__gpr_num_%[pzt]) << 5)\n" + : + : [pzt] "r" (pzt) + : "memory" + ); +} + +static inline void __sme_load_zt(const struct arm64_sme_state *state, unsigned long svl) +{ + void *pzt = (void *)state + svl * svl; + + instrument_read(pzt, 64); + asm volatile( + __DEFINE_ASM_GPR_NUMS + /* + * LDR ZT0, [] + * Supported by binutils 2.41+. + * Supported by LLVM 16+ + */ + " .inst 0xe11f8000 | ((.L__gpr_num_%[pzt]) << 5)\n" + : + : [pzt] "r" (pzt) + : "memory" + ); +} + +static inline void sme_save_state(struct arm64_sme_state *state, bool zt) +{ + unsigned long svl = sme_get_vl(); + + __sme_save_za(state, svl); + if (zt) + __sme_save_zt(state, svl); +} + +static inline void sme_load_state(const struct arm64_sme_state *state, bool zt) +{ + unsigned long svl = sme_get_vl(); + + __sme_load_za(state, svl); + if (zt) + __sme_load_zt(state, svl); +} + /* * Return how many bytes of memory are required to store the full SME * specific state for task, given task's currently configured vector @@ -694,6 +791,9 @@ static inline size_t sme_state_size(struct task_struct const *task) return 0; } +static inline void sme_save_state(struct arm64_sme_state *state, bool zt) { BUILD_BUG(); } +static inline void sme_load_state(const struct arm64_sme_state *state, bool zt) { BUILD_BUG(); } + static inline void sme_enter_from_user_mode(void) { } static inline void sme_exit_to_user_mode(void) { } diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h index 9e352b5c6b76..a763fd03ffef 100644 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ b/arch/arm64/include/asm/fpsimdmacros.h @@ -40,60 +40,6 @@ .endif .endm -/* Deprecated macros for SME instructions */ - -/* RDSVL X\nx, #\imm */ -.macro _sme_rdsvl nx, imm - .arch_extension sme - rdsvl x\nx, #\imm -.endm - -/* - * STR (vector from ZA array): - * STR ZA[W\nw, #\offset], [X\nxbase, #\offset, MUL VL] - */ -.macro _sme_str_zav nw, nxbase, offset=0 - .arch_extension sme - str za[w\nw, #\offset], [x\nxbase, #\offset, MUL VL] -.endm - -/* - * LDR (vector to ZA array): - * LDR ZA[w\nw, #\offset], [X\nxbase, #\offset, MUL VL] - */ -.macro _sme_ldr_zav nw, nxbase, offset=0 - .arch_extension sme - ldr za[w\nw, #\offset], [x\nxbase, #\offset, MUL VL] -.endm - -/* - * SME2 instruction encodings for older assemblers. - * Supported by binutils 2.41+. - * Supported by LLVM 16+ - */ - -/* - * LDR (ZT0) - * - * LDR ZT0, nx - */ -.macro _ldr_zt nx - _check_general_reg \nx - .inst 0xe11f8000 \ - | (\nx << 5) -.endm - -/* - * STR (ZT0) - * - * STR ZT0, nx - */ -.macro _str_zt nx - _check_general_reg \nx - .inst 0xe13f8000 \ - | (\nx << 5) -.endm - .macro __for from:req, to:req .if (\from) == (\to) _for__body %\from @@ -116,25 +62,3 @@ .purgem _for__body .endm - -.macro sme_save_za nxbase, xvl, nw - mov w\nw, #0 - -423: - _sme_str_zav \nw, \nxbase - add x\nxbase, x\nxbase, \xvl - add x\nw, x\nw, #1 - cmp \xvl, x\nw - bne 423b -.endm - -.macro sme_load_za nxbase, xvl, nw - mov w\nw, #0 - -423: - _sme_ldr_zav \nw, \nxbase - add x\nxbase, x\nxbase, \xvl - add x\nw, x\nw, #1 - cmp \xvl, x\nw - bne 423b -.endm diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 74b76bb70452..d2690c3ec528 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -27,7 +27,7 @@ KCOV_INSTRUMENT_idle.o := n # Object file lists. obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ - entry-common.o entry-fpsimd.o process.o ptrace.o \ + entry-common.o process.o ptrace.o \ setup.o signal.o sys.o stacktrace.o time.o traps.o \ io.o vdso.o hyp-stub.o psci.o cpu_ops.o \ return_address.o cpuinfo.o cpu_errata.o \ diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S deleted file mode 100644 index 2a4755113b99..000000000000 --- a/arch/arm64/kernel/entry-fpsimd.S +++ /dev/null @@ -1,48 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * FP/SIMD state saving and restoring - * - * Copyright (C) 2012 ARM Ltd. - * Author: Catalin Marinas - */ - -#include - -#include -#include - -#ifdef CONFIG_ARM64_SME - -/* - * Save the ZA and ZT state - * - * x0 - pointer to buffer for state - * w1 - number of ZT registers to save - */ -SYM_FUNC_START(sme_save_state) - _sme_rdsvl 2, 1 // x2 = VL/8 - sme_save_za 0, x2, 12 // Leaves x0 pointing to the end of ZA - - cbz w1, 1f - _str_zt 0 -1: - ret -SYM_FUNC_END(sme_save_state) - -/* - * Load the ZA and ZT state - * - * x0 - pointer to buffer for state - * w1 - number of ZT registers to save - */ -SYM_FUNC_START(sme_load_state) - _sme_rdsvl 2, 1 // x2 = VL/8 - sme_load_za 0, x2, 12 // Leaves x0 pointing to the end of ZA - - cbz w1, 1f - _ldr_zt 0 -1: - ret -SYM_FUNC_END(sme_load_state) - -#endif /* CONFIG_ARM64_SME */ From 987ec51e18419cc0ebf6f6fa1cfbfd149eca443d Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 3 Jun 2026 12:06:30 +0100 Subject: [PATCH 57/73] arm64: fpsimd: Remove We no longer need any of the remaining macros in . Remove all of it. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Vladimir Murzin Cc: Catalin Marinas Cc: Fuad Tabba Cc: James Morse Cc: Marc Zyngier Cc: Oliver Upton Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/fpsimdmacros.h | 64 --------------------------- 1 file changed, 64 deletions(-) delete mode 100644 arch/arm64/include/asm/fpsimdmacros.h diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h deleted file mode 100644 index a763fd03ffef..000000000000 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ /dev/null @@ -1,64 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * FP/SIMD state saving and restoring macros - * - * Copyright (C) 2012 ARM Ltd. - * Author: Catalin Marinas - */ - -#include - -/* Sanity-check macros to help avoid encoding garbage instructions */ - -.macro _check_general_reg nr - .if (\nr) < 0 || (\nr) > 30 - .error "Bad register number \nr." - .endif -.endm - -.macro _sve_check_zreg znr - .if (\znr) < 0 || (\znr) > 31 - .error "Bad Scalable Vector Extension vector register number \znr." - .endif -.endm - -.macro _sve_check_preg pnr - .if (\pnr) < 0 || (\pnr) > 15 - .error "Bad Scalable Vector Extension predicate register number \pnr." - .endif -.endm - -.macro _check_num n, min, max - .if (\n) < (\min) || (\n) > (\max) - .error "Number \n out of range [\min,\max]" - .endif -.endm - -.macro _sme_check_wv v - .if (\v) < 12 || (\v) > 15 - .error "Bad vector select register \v." - .endif -.endm - -.macro __for from:req, to:req - .if (\from) == (\to) - _for__body %\from - .else - __for %\from, %((\from) + ((\to) - (\from)) / 2) - __for %((\from) + ((\to) - (\from)) / 2 + 1), %\to - .endif -.endm - -.macro _for var:req, from:req, to:req, insn:vararg - .macro _for__body \var:req - .noaltmacro - \insn - .altmacro - .endm - - .altmacro - __for \from, \to - .noaltmacro - - .purgem _for__body -.endm From 5936245125f78d896fdb1bbc2ae79213e28a6579 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 29 May 2026 15:33:45 +0100 Subject: [PATCH 58/73] perf/arm-cmn: Fix DVM node events The new DVM node events added in CMN-700 also apply to CMN S3; fix the model encoding so that we can expose the aliases and handle occupancy filtering on newer CMNs too. Cc: stable@vger.kernel.org Fixes: 0dc2f4963f7e ("perf/arm-cmn: Support CMN S3") Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/perf/arm-cmn.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index f5305c8fdca4..6e5cc4086a9e 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -197,13 +197,14 @@ enum cmn_model { CMN600 = 1, CMN650 = 2, - CMN700 = 4, - CI700 = 8, + CI700 = 4, + CMN700 = 8, CMNS3 = 16, /* ...and then we can use bitmap tricks for commonality */ CMN_ANY = -1, NOT_CMN600 = -2, - CMN_650ON = CMN650 | CMN700 | CMNS3, + CMN_700ON = ~(CMN700 - 1), + CMN_650ON = CMN_700ON | CMN650, }; /* Actual part numbers and revision IDs defined by the hardware */ @@ -919,14 +920,14 @@ static struct attribute *arm_cmn_event_attrs[] = { CMN_EVENT_DVM(NOT_CMN600, txsnp_stall, 0x0a), CMN_EVENT_DVM(NOT_CMN600, trkfull, 0x0b), CMN_EVENT_DVM_OCC(NOT_CMN600, trk_occupancy, 0x0c), - CMN_EVENT_DVM_OCC(CMN700, trk_occupancy_cxha, 0x0d), - CMN_EVENT_DVM_OCC(CMN700, trk_occupancy_pdn, 0x0e), - CMN_EVENT_DVM(CMN700, trk_alloc, 0x0f), - CMN_EVENT_DVM(CMN700, trk_cxha_alloc, 0x10), - CMN_EVENT_DVM(CMN700, trk_pdn_alloc, 0x11), - CMN_EVENT_DVM(CMN700, txsnp_stall_limit, 0x12), - CMN_EVENT_DVM(CMN700, rxsnp_stall_starv, 0x13), - CMN_EVENT_DVM(CMN700, txsnp_sync_stall_op, 0x14), + CMN_EVENT_DVM_OCC(CMN_700ON, trk_occupancy_cxha, 0x0d), + CMN_EVENT_DVM_OCC(CMN_700ON, trk_occupancy_pdn, 0x0e), + CMN_EVENT_DVM(CMN_700ON, trk_alloc, 0x0f), + CMN_EVENT_DVM(CMN_700ON, trk_cxha_alloc, 0x10), + CMN_EVENT_DVM(CMN_700ON, trk_pdn_alloc, 0x11), + CMN_EVENT_DVM(CMN_700ON, txsnp_stall_limit, 0x12), + CMN_EVENT_DVM(CMN_700ON, rxsnp_stall_starv, 0x13), + CMN_EVENT_DVM(CMN_700ON, txsnp_sync_stall_op, 0x14), CMN_EVENT_HNF(CMN_ANY, cache_miss, 0x01), CMN_EVENT_HNF(CMN_ANY, slc_sf_cache_access, 0x02), From 6c2ccb4979b2c8aa83cbea827a714a74db7d2398 Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Mon, 1 Jun 2026 00:08:16 +0200 Subject: [PATCH 59/73] arm64: patching: replace min_t with min in __text_poke Use the simpler min() macro since both values are unsigned and compatible. Signed-off-by: Thorsten Blum Signed-off-by: Will Deacon --- arch/arm64/kernel/patching.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/arm64/kernel/patching.c b/arch/arm64/kernel/patching.c index 1041bc67a3ee..09f019c6547a 100644 --- a/arch/arm64/kernel/patching.c +++ b/arch/arm64/kernel/patching.c @@ -116,8 +116,7 @@ static void *__text_poke(text_poke_f func, void *addr, void *src, size_t len) while (patched < len) { ptr = addr + patched; - size = min_t(size_t, PAGE_SIZE - offset_in_page(ptr), - len - patched); + size = min(PAGE_SIZE - offset_in_page(ptr), len - patched); waddr = patch_map(ptr, FIX_TEXT_POKE0); func(waddr, src, patched, size); From 9c401fa7398fc2362b57e7d8b2ea1ab440d8b39f Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 4 Jun 2026 17:11:53 +0200 Subject: [PATCH 60/73] arm64: Rename page table BSS section to .bss..pgtbl Rename the .pgdir.bss section to .bss..pgtbl so that the compiler will notice the leading ".bss" and mark it as NOBITS by default (rather than PROGBITS, which would take up space in Image binary, forcing all of the preceding BSS to be emitted into the image as well). This supersedes the NOLOAD linker directive, which achieves the same thing, and can be therefore be dropped. Also, rename .pgdir to .pgtbl to be more generic, as page tables of various levels will reside here. Signed-off-by: Ard Biesheuvel Tested-by: Mark Brown Signed-off-by: Will Deacon --- arch/arm64/include/asm/linkage.h | 2 ++ arch/arm64/include/asm/mmu.h | 2 -- arch/arm64/kernel/vmlinux.lds.S | 8 ++++---- arch/arm64/mm/fixmap.c | 6 +++--- arch/arm64/mm/kasan_init.c | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/arm64/include/asm/linkage.h b/arch/arm64/include/asm/linkage.h index 40bd17add539..8637f667667c 100644 --- a/arch/arm64/include/asm/linkage.h +++ b/arch/arm64/include/asm/linkage.h @@ -43,4 +43,6 @@ SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ bti c ; +#define __bss_pgtbl __section(".bss..pgtbl") __aligned(PAGE_SIZE) + #endif diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index fb95754f2876..5e1211c540ab 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -13,8 +13,6 @@ #ifndef __ASSEMBLER__ -#define __pgtbl_bss __section(".pgdir.bss") __aligned(PAGE_SIZE) - #include #include diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 2b0ebfb30c63..d3ed59abab38 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -352,11 +352,11 @@ SECTIONS BSS_SECTION(SBSS_ALIGN, 0, PAGE_SIZE) __pi___bss_start = __bss_start; - /* fixmap BSS starts here - preceding data/BSS is omitted from the linear map */ - .pgdir.bss (NOLOAD) : ALIGN(PAGE_SIZE) { - *(.pgdir.bss) + /* page table BSS starts here - preceding data/BSS is omitted from the linear map */ + .pgtbl : ALIGN(PAGE_SIZE) { + *(.bss..pgtbl) } - ASSERT(ADDR(.pgdir.bss) == __bss_stop, ".pgdir.bss must follow BSS") + ASSERT(ADDR(.pgtbl) == __bss_stop, ".pgtbl must follow BSS") . = ALIGN(PAGE_SIZE); __pi_init_pg_dir = .; diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c index 1a3bbd67dd76..f66a0016dd02 100644 --- a/arch/arm64/mm/fixmap.c +++ b/arch/arm64/mm/fixmap.c @@ -31,9 +31,9 @@ static_assert(NR_BM_PMD_TABLES == 1); #define BM_PTE_TABLE_IDX(addr) __BM_TABLE_IDX(addr, PMD_SHIFT) -static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __pgtbl_bss; -static pmd_t bm_pmd[PTRS_PER_PMD] __pgtbl_bss __maybe_unused; -static pud_t bm_pud[PTRS_PER_PUD] __pgtbl_bss __maybe_unused; +static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __bss_pgtbl; +static pmd_t bm_pmd[PTRS_PER_PMD] __bss_pgtbl __maybe_unused; +static pud_t bm_pud[PTRS_PER_PUD] __bss_pgtbl __maybe_unused; static inline pte_t *fixmap_pte(unsigned long addr) { diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c index dbf22cae82ee..3fcad956fdf7 100644 --- a/arch/arm64/mm/kasan_init.c +++ b/arch/arm64/mm/kasan_init.c @@ -214,7 +214,7 @@ asmlinkage void __init kasan_early_init(void) * shadow pud_t[]/p4d_t[], which could end up getting corrupted * when the linear region is mapped. */ - static pte_t tbl[PTRS_PER_PTE] __pgtbl_bss; + static pte_t tbl[PTRS_PER_PTE] __bss_pgtbl; pgd_t *pgdp = pgd_offset_k(KASAN_SHADOW_START); set_pgd(pgdp, __pgd(__pa_symbol(tbl) | PGD_TYPE_TABLE)); From 9f7f685758c6988a6076d4a494d20e99337b79ed Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 4 Jun 2026 17:11:54 +0200 Subject: [PATCH 61/73] kasan: Move generic KASAN page tables out of BSS too Make sure that all KASAN page tables are emitted into the .pgtbl section (provided that the arch has one - otherwise, fall back to page aligned BSS) This is needed because BSS itself is no longer accessible via the linear map on arm64. Cc: Andrey Ryabinin Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Dmitry Vyukov Cc: Vincenzo Frascino Cc: kasan-dev@googlegroups.com Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- include/linux/linkage.h | 4 ++++ mm/kasan/init.c | 10 +++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/include/linux/linkage.h b/include/linux/linkage.h index b11660b706c5..53fe1f48fd28 100644 --- a/include/linux/linkage.h +++ b/include/linux/linkage.h @@ -39,6 +39,10 @@ #define __page_aligned_data __section(".data..page_aligned") __aligned(PAGE_SIZE) #define __page_aligned_bss __section(".bss..page_aligned") __aligned(PAGE_SIZE) +#ifndef __bss_pgtbl +#define __bss_pgtbl __page_aligned_bss +#endif + /* * For assembly routines. * diff --git a/mm/kasan/init.c b/mm/kasan/init.c index 9c880f607c6a..66a883887987 100644 --- a/mm/kasan/init.c +++ b/mm/kasan/init.c @@ -26,10 +26,10 @@ * - Latter it reused it as zero shadow to cover large ranges of memory * that allowed to access, but not handled by kasan (vmalloc/vmemmap ...). */ -unsigned char kasan_early_shadow_page[PAGE_SIZE] __page_aligned_bss; +unsigned char kasan_early_shadow_page[PAGE_SIZE] __bss_pgtbl; #if CONFIG_PGTABLE_LEVELS > 4 -p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D] __page_aligned_bss; +p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D] __bss_pgtbl; static inline bool kasan_p4d_table(pgd_t pgd) { return pgd_page(pgd) == virt_to_page(lm_alias(kasan_early_shadow_p4d)); @@ -41,7 +41,7 @@ static inline bool kasan_p4d_table(pgd_t pgd) } #endif #if CONFIG_PGTABLE_LEVELS > 3 -pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __page_aligned_bss; +pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __bss_pgtbl; static inline bool kasan_pud_table(p4d_t p4d) { return p4d_page(p4d) == virt_to_page(lm_alias(kasan_early_shadow_pud)); @@ -53,7 +53,7 @@ static inline bool kasan_pud_table(p4d_t p4d) } #endif #if CONFIG_PGTABLE_LEVELS > 2 -pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __page_aligned_bss; +pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __bss_pgtbl; static inline bool kasan_pmd_table(pud_t pud) { return pud_page(pud) == virt_to_page(lm_alias(kasan_early_shadow_pmd)); @@ -65,7 +65,7 @@ static inline bool kasan_pmd_table(pud_t pud) } #endif pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS] - __page_aligned_bss; + __bss_pgtbl; static inline bool kasan_pte_table(pmd_t pmd) { From 568def8e87fc666682ec5a74713983ac32335213 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 4 Jun 2026 17:11:55 +0200 Subject: [PATCH 62/73] arm64: Avoid double evaluation of __ptep_get() Sashiko warns that the new pte_valid_noncont() macro is used in a manner where the argument (which performs a READ_ONCE() of the descriptor) is evaluated twice. Drop the macro that we just added, and move the check into the newly added users. Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/include/asm/pgtable.h | 4 ---- arch/arm64/mm/mmu.c | 14 ++++++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 491ba0a6492d..c9e4e00a9af2 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -181,10 +181,6 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) * Returns true if the pte is valid and has the contiguous bit set. */ #define pte_valid_cont(pte) (pte_valid(pte) && pte_cont(pte)) -/* - * Returns true if the pte is valid and has the contiguous bit cleared. - */ -#define pte_valid_noncont(pte) (pte_valid(pte) && !pte_cont(pte)) /* * Could the pte be present in the TLB? We must check mm_tlb_flush_pending * so that we don't erroneously return false for pages that have been diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index be51f6cac86f..d68e691c093a 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -186,9 +186,12 @@ static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end, static bool pte_range_has_valid_noncont(pte_t *ptep) { - for (int i = 0; i < CONT_PTES; i++) - if (pte_valid_noncont(__ptep_get(&ptep[i]))) + for (int i = 0; i < CONT_PTES; i++) { + pte_t pte = __ptep_get(&ptep[i]); + + if (pte_valid(pte) && !pte_cont(pte)) return true; + } return false; } @@ -291,9 +294,12 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, static bool pmd_range_has_valid_noncont(pmd_t *pmdp) { - for (int i = 0; i < CONT_PMDS; i++) - if (pte_valid_noncont(pmd_pte(READ_ONCE(pmdp[i])))) + for (int i = 0; i < CONT_PMDS; i++) { + pte_t pte = pmd_pte(READ_ONCE(pmdp[i])); + + if (pte_valid(pte) && !pte_cont(pte)) return true; + } return false; } From 2986a625740599fe6e7635b0586fed2a95bcd1f7 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 4 Jun 2026 17:11:56 +0200 Subject: [PATCH 63/73] KVM: arm64: Omit tag sync on stage-2 mappings of the zero page Commit f620d66af316 ("arm64: mte: Do not flag the zero page as PG_mte_tagged") removed the PG_mte_tagged flag from the zero page, but missed a KVM code path that may set this flag on the zero page when it is used in a stage-2 CoW mapping of anonymous memory. So disregard the zero page explicitly in sanitise_mte_tags(). Fixes: f620d66af316 ("arm64: mte: Do not flag the zero page as PG_mte_tagged") Cc: stable@vger.kernel.org # 5.10.x Suggested-by: Catalin Marinas Signed-off-by: Ard Biesheuvel Reviewed-by: Catalin Marinas Signed-off-by: Will Deacon --- arch/arm64/kvm/mmu.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index d089c107d9b7..445d6cf035c9 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1479,6 +1479,11 @@ static void sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn, if (!kvm_has_mte(kvm)) return; + if (is_zero_pfn(pfn)) { + WARN_ON_ONCE(nr_pages != 1); + return; + } + if (folio_test_hugetlb(folio)) { /* Hugetlb has MTE flags set on head page only */ if (folio_try_hugetlb_mte_tagging(folio)) { From 53205d56212cbff880a77497e25a0e44036d490a Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 4 Jun 2026 17:11:57 +0200 Subject: [PATCH 64/73] arm64: mm: Defer remap of linear alias of data/bss Marking the linear alias of data/bss invalid involves calling set_memory_valid(), which calls split_kernel_leaf_mapping() under the hood. On BBML2_NOABORT capable systems, this may result in the need to allocate page tables at a time when the generic memory allocation APIs are not yet available, resulting in a splat like WARNING: arch/arm64/mm/mmu.c:821 at split_kernel_leaf_mapping+0x15c/0x170, CPU#0: swapper/0 Modules linked in: CPU: 0 UID: 0 PID: 0 Comm: swapper Not tainted 7.1.0-rc6 #1 PREEMPT(undef) pstate: a04000c9 (NzCv daIF +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : split_kernel_leaf_mapping+0x15c/0x170 lr : update_range_prot+0x40/0x128 sp : ffffc99ad3863c80 ... Call trace: split_kernel_leaf_mapping+0x15c/0x170 (P) update_range_prot+0x40/0x128 set_memory_valid+0x94/0xe0 mark_linear_data_alias_valid+0x54/0x68 map_mem+0x1fc/0x240 paging_init+0x48/0x210 setup_arch+0x274/0x338 start_kernel+0x98/0x538 __primary_switched+0x88/0x98 as reported by CKI automated testing. So defer the boot-time call to mark_linear_data_alias_valid() to a later time when page allocations can be made normally. Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index d68e691c093a..3134f1c1097c 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1095,6 +1095,9 @@ void __init mark_linear_text_alias_ro(void) (unsigned long)__init_begin - (unsigned long)_text, PAGE_KERNEL_RO); + /* Map the kernel data/bss as invalid in the linear map */ + mark_linear_data_alias_valid(false); + /* * Register a PM notifier to remap the linear alias of data/bss as * valid read-only before hibernation. This is needed because the @@ -1237,9 +1240,6 @@ static void __init map_mem(void) __map_memblock(start, end, pgprot_tagged(PAGE_KERNEL), flags); } - - /* Map the kernel data/bss as invalid in the linear map */ - mark_linear_data_alias_valid(false); } void mark_rodata_ro(void) From c34d4d48c4a1427c4af888b02d0ca21fde3e299f Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 20 May 2026 07:34:17 +0100 Subject: [PATCH 65/73] arm64/mm: Rename ptdesc_t ptdesc_t sounds very similar to the core MM struct ptdesc which is actually the memory descriptor for page table allocations. Hence rename this typedef element as ptval_t instead for better clarity and separation. Cc: Catalin Marinas Cc: Will Deacon Cc: David Hildenbrand Cc: Mike Rapoport Cc: linux-efi@vger.kernel.org Cc: linux-kernel@vger.kernel.org Cc: linux-arm-kernel@lists.infradead.org Acked-by: Mike Rapoport (Microsoft) Acked-by: David Hildenbrand (Arm) Suggested-by: David Hildenbrand (Arm) Signed-off-by: Anshuman Khandual Signed-off-by: Will Deacon --- arch/arm64/include/asm/io.h | 2 +- arch/arm64/include/asm/pgtable-types.h | 14 +++++++------- arch/arm64/include/asm/ptdump.h | 8 ++++---- arch/arm64/include/asm/tlbflush.h | 4 ++-- arch/arm64/kernel/efi.c | 4 ++-- arch/arm64/kernel/pi/map_kernel.c | 2 +- arch/arm64/kernel/pi/map_range.c | 4 ++-- arch/arm64/kernel/pi/pi.h | 2 +- arch/arm64/mm/mmap.c | 4 ++-- arch/arm64/mm/pageattr.c | 2 +- arch/arm64/mm/ptdump.c | 2 +- 11 files changed, 24 insertions(+), 24 deletions(-) diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 8cbd1e96fd50..21c8e400107c 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -270,7 +270,7 @@ static inline void __iomem *ioremap_prot(phys_addr_t phys, size_t size, pgprot_t user_prot) { pgprot_t prot; - ptdesc_t user_prot_val = pgprot_val(user_prot); + ptval_t user_prot_val = pgprot_val(user_prot); if (WARN_ON_ONCE(!(user_prot_val & PTE_USER))) return NULL; diff --git a/arch/arm64/include/asm/pgtable-types.h b/arch/arm64/include/asm/pgtable-types.h index 265e8301d7ba..2f2f5527930f 100644 --- a/arch/arm64/include/asm/pgtable-types.h +++ b/arch/arm64/include/asm/pgtable-types.h @@ -17,13 +17,13 @@ * Generic page table descriptor format from which * all level specific descriptors can be derived. */ -typedef u64 ptdesc_t; +typedef u64 ptval_t; -typedef ptdesc_t pteval_t; -typedef ptdesc_t pmdval_t; -typedef ptdesc_t pudval_t; -typedef ptdesc_t p4dval_t; -typedef ptdesc_t pgdval_t; +typedef ptval_t pteval_t; +typedef ptval_t pmdval_t; +typedef ptval_t pudval_t; +typedef ptval_t p4dval_t; +typedef ptval_t pgdval_t; /* * These are used to make use of C type-checking.. @@ -54,7 +54,7 @@ typedef struct { pgdval_t pgd; } pgd_t; #define pgd_val(x) ((x).pgd) #define __pgd(x) ((pgd_t) { (x) } ) -typedef struct { ptdesc_t pgprot; } pgprot_t; +typedef struct { ptval_t pgprot; } pgprot_t; #define pgprot_val(x) ((x).pgprot) #define __pgprot(x) ((pgprot_t) { (x) } ) diff --git a/arch/arm64/include/asm/ptdump.h b/arch/arm64/include/asm/ptdump.h index baff24004459..5b374a6ab34a 100644 --- a/arch/arm64/include/asm/ptdump.h +++ b/arch/arm64/include/asm/ptdump.h @@ -26,8 +26,8 @@ struct ptdump_info { }; struct ptdump_prot_bits { - ptdesc_t mask; - ptdesc_t val; + ptval_t mask; + ptval_t val; const char *set; const char *clear; }; @@ -36,7 +36,7 @@ struct ptdump_pg_level { const struct ptdump_prot_bits *bits; char name[4]; int num; - ptdesc_t mask; + ptval_t mask; }; /* @@ -53,7 +53,7 @@ struct ptdump_pg_state { const struct mm_struct *mm; unsigned long start_address; int level; - ptdesc_t current_prot; + ptval_t current_prot; bool check_wx; unsigned long wx_pages; unsigned long uxn_pages; diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index c0bf5b398041..d52ac8c17190 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -725,9 +725,9 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b sme_dvmsync_add_pending(batch, mm); } -static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval) +static inline bool __pte_flags_need_flush(ptval_t oldval, ptval_t newval) { - ptdesc_t diff = oldval ^ newval; + ptval_t diff = oldval ^ newval; /* invalid to valid transition requires no flush */ if (!(oldval & PTE_VALID)) diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index a81cb4aa4738..30cd7f804398 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -31,7 +31,7 @@ static bool region_is_misaligned(const efi_memory_desc_t *md) * executable, everything else can be mapped with the XN bits * set. Also take the new (optional) RO/XP bits into account. */ -static __init ptdesc_t create_mapping_protection(efi_memory_desc_t *md) +static __init ptval_t create_mapping_protection(efi_memory_desc_t *md) { u64 attr = md->attribute; u32 type = md->type; @@ -85,7 +85,7 @@ static __init ptdesc_t create_mapping_protection(efi_memory_desc_t *md) int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) { - ptdesc_t prot_val = create_mapping_protection(md); + ptval_t prot_val = create_mapping_protection(md); bool page_mappings_only = (md->type == EFI_RUNTIME_SERVICES_CODE || md->type == EFI_RUNTIME_SERVICES_DATA); diff --git a/arch/arm64/kernel/pi/map_kernel.c b/arch/arm64/kernel/pi/map_kernel.c index a852264958c3..fb44cbdd2f29 100644 --- a/arch/arm64/kernel/pi/map_kernel.c +++ b/arch/arm64/kernel/pi/map_kernel.c @@ -165,7 +165,7 @@ static void noinline __section(".idmap.text") set_ttbr0_for_lpa2(phys_addr_t ttb static void __init remap_idmap_for_lpa2(void) { /* clear the bits that change meaning once LPA2 is turned on */ - ptdesc_t mask = PTE_SHARED; + ptval_t mask = PTE_SHARED; /* * We have to clear bits [9:8] in all block or page descriptors in the diff --git a/arch/arm64/kernel/pi/map_range.c b/arch/arm64/kernel/pi/map_range.c index de52cd85c691..761b14893f74 100644 --- a/arch/arm64/kernel/pi/map_range.c +++ b/arch/arm64/kernel/pi/map_range.c @@ -31,7 +31,7 @@ void __init map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, u64 va_offset) { u64 cmask = (level == 3) ? CONT_PTE_SIZE - 1 : U64_MAX; - ptdesc_t protval = pgprot_val(prot) & ~PTE_TYPE_MASK; + ptval_t protval = pgprot_val(prot) & ~PTE_TYPE_MASK; int lshift = (3 - level) * PTDESC_TABLE_SHIFT; u64 lmask = (PAGE_SIZE << lshift) - 1; @@ -88,7 +88,7 @@ void __init map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, } } -asmlinkage phys_addr_t __init create_init_idmap(pgd_t *pg_dir, ptdesc_t clrmask) +asmlinkage phys_addr_t __init create_init_idmap(pgd_t *pg_dir, ptval_t clrmask) { phys_addr_t ptep = (phys_addr_t)pg_dir + PAGE_SIZE; /* MMU is off */ pgprot_t text_prot = PAGE_KERNEL_ROX; diff --git a/arch/arm64/kernel/pi/pi.h b/arch/arm64/kernel/pi/pi.h index aec3172d4003..5dfd8484d200 100644 --- a/arch/arm64/kernel/pi/pi.h +++ b/arch/arm64/kernel/pi/pi.h @@ -35,4 +35,4 @@ void map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, asmlinkage void early_map_kernel(u64 boot_status, phys_addr_t fdt); -asmlinkage phys_addr_t create_init_idmap(pgd_t *pgd, ptdesc_t clrmask); +asmlinkage phys_addr_t create_init_idmap(pgd_t *pgd, ptval_t clrmask); diff --git a/arch/arm64/mm/mmap.c b/arch/arm64/mm/mmap.c index 92b2f5097a96..32e0771d6477 100644 --- a/arch/arm64/mm/mmap.c +++ b/arch/arm64/mm/mmap.c @@ -34,7 +34,7 @@ static pgprot_t protection_map[16] __ro_after_init = { [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_SHARED_EXEC }; -static ptdesc_t gcs_page_prot __ro_after_init = _PAGE_GCS_RO; +static ptval_t gcs_page_prot __ro_after_init = _PAGE_GCS_RO; /* * You really shouldn't be using read() or write() on /dev/mem. This might go @@ -87,7 +87,7 @@ arch_initcall(adjust_protection_map); pgprot_t vm_get_page_prot(vm_flags_t vm_flags) { - ptdesc_t prot; + ptval_t prot; /* Short circuit GCS to avoid bloating the table. */ if (system_supports_gcs() && (vm_flags & VM_SHADOW_STACK)) { diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index ce035e1b4eaf..bbe98ac9ad8c 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -21,7 +21,7 @@ struct page_change_data { pgprot_t clear_mask; }; -static ptdesc_t set_pageattr_masks(ptdesc_t val, struct mm_walk *walk) +static ptval_t set_pageattr_masks(ptval_t val, struct mm_walk *walk) { struct page_change_data *masks = walk->private; diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c index ab9899ca1e5f..1c20144700d7 100644 --- a/arch/arm64/mm/ptdump.c +++ b/arch/arm64/mm/ptdump.c @@ -194,7 +194,7 @@ void note_page(struct ptdump_state *pt_st, unsigned long addr, int level, struct ptdump_pg_state *st = container_of(pt_st, struct ptdump_pg_state, ptdump); struct ptdump_pg_level *pg_level = st->pg_level; static const char units[] = "KMGTPE"; - ptdesc_t prot = 0; + ptval_t prot = 0; /* check if the current level has been folded dynamically */ if (st->mm && ((level == 1 && mm_p4d_folded(st->mm)) || From 11c33ffb3a4e65e775e9bd814eb71df06ea7c847 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Sat, 23 May 2026 12:59:26 -0400 Subject: [PATCH 66/73] arm64: arch_timer: reuse arch_timer_read_cnt{p,v}ct_el0() helpers __arch_counter_get_cntpct() and __arch_counter_get_cntvct() open-code the same ECV-aware ALTERNATIVE block that arch_timer_read_cntpct_el0() and arch_timer_read_cntvct_el0() already provide in the same header. The two pairs are byte-for-byte identical except for the trailing arch_counter_enforce_ordering() the __arch_counter_get_* variants add. Replace the duplicated inline assembly in __arch_counter_get_cntpct() and __arch_counter_get_cntvct() with calls to the corresponding helpers. This mirrors commit 00b39d150986 ("arm64: vdso: Use __arch_counter_get_cntvct()"), which removed similar duplication from the vDSO, and keeps the system-counter read sequence in a single place, reducing assembly code in the kernell No functional change: the resulting inline assembly, alternatives, and clobbers are unchanged; only the source-level expression of the read moves into the existing helper. Verified by rebuilding the consumers of these helpers before and after the change and comparing the resulting disassembly: - arch/arm64/kernel/vdso/vdso.so (final linked vDSO): bit-identical (same sha256 across rebuilds) - arch/arm64/kernel/vdso/vgettimeofday.o: identical disassembly - arch/arm64/lib/delay.o: identical disassembly - drivers/clocksource/arm_arch_timer.o: same 50 functions with byte-identical instruction streams; only difference is function ordering inside .text and NOP padding, with no opcodes added or removed. Signed-off-by: Breno Leitao Signed-off-by: Will Deacon --- arch/arm64/include/asm/arch_timer.h | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/arch/arm64/include/asm/arch_timer.h b/arch/arm64/include/asm/arch_timer.h index f5794d50f51d..6717209df05b 100644 --- a/arch/arm64/include/asm/arch_timer.h +++ b/arch/arm64/include/asm/arch_timer.h @@ -178,12 +178,8 @@ static __always_inline u64 __arch_counter_get_cntpct_stable(void) static __always_inline u64 __arch_counter_get_cntpct(void) { - u64 cnt; + u64 cnt = arch_timer_read_cntpct_el0(); - asm volatile(ALTERNATIVE("isb\n mrs %0, cntpct_el0", - "nop\n" __mrs_s("%0", SYS_CNTPCTSS_EL0), - ARM64_HAS_ECV) - : "=r" (cnt)); arch_counter_enforce_ordering(cnt); return cnt; } @@ -199,12 +195,8 @@ static __always_inline u64 __arch_counter_get_cntvct_stable(void) static __always_inline u64 __arch_counter_get_cntvct(void) { - u64 cnt; + u64 cnt = arch_timer_read_cntvct_el0(); - asm volatile(ALTERNATIVE("isb\n mrs %0, cntvct_el0", - "nop\n" __mrs_s("%0", SYS_CNTVCTSS_EL0), - ARM64_HAS_ECV) - : "=r" (cnt)); arch_counter_enforce_ordering(cnt); return cnt; } From 81479b6888f8ff7099103c08efab35f4817976d5 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 10 Jun 2026 11:34:39 +0100 Subject: [PATCH 67/73] Revert "arm64: mm: Defer remap of linear alias of data/bss" This reverts commit 53205d56212cbff880a77497e25a0e44036d490a. Unmapping the kernel '.bss' appears to break KVM initialisation on some devices, breaking the boot on popular platforms such as RaspberryPi3 and 4. Revert this change for now so that we can revisit it in future. Reported-by: Mark Brown Reported-by: Marek Szyprowski Link: https://lore.kernel.org/all/aicVyebkEMs6w6UV@sirena.co.uk Link: https://lore.kernel.org/r/a1b27e97-182c-485d-a448-56c19c5de2c2@samsung.com Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 3134f1c1097c..d68e691c093a 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1095,9 +1095,6 @@ void __init mark_linear_text_alias_ro(void) (unsigned long)__init_begin - (unsigned long)_text, PAGE_KERNEL_RO); - /* Map the kernel data/bss as invalid in the linear map */ - mark_linear_data_alias_valid(false); - /* * Register a PM notifier to remap the linear alias of data/bss as * valid read-only before hibernation. This is needed because the @@ -1240,6 +1237,9 @@ static void __init map_mem(void) __map_memblock(start, end, pgprot_tagged(PAGE_KERNEL), flags); } + + /* Map the kernel data/bss as invalid in the linear map */ + mark_linear_data_alias_valid(false); } void mark_rodata_ro(void) From f102131c842d1e1d95bc7b818ab93944195da7e9 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 10 Jun 2026 11:40:23 +0100 Subject: [PATCH 68/73] Revert "arm64: mm: Unmap kernel data/bss entirely from the linear map" This reverts commit 63e0b6a5b6934d6a919d1c65ea185303200a1874. Unmapping the kernel '.bss' appears to break KVM initialisation on some devices, breaking the boot on popular platforms such as RaspberryPi3 and 4. Revert this change for now so that we can revisit it in future. Reported-by: Mark Brown Reported-by: Marek Szyprowski Link: https://lore.kernel.org/all/aicVyebkEMs6w6UV@sirena.co.uk Link: https://lore.kernel.org/r/a1b27e97-182c-485d-a448-56c19c5de2c2@samsung.com Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 45 ++++----------------------------------------- 1 file changed, 4 insertions(+), 41 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index d68e691c093a..354bedf0e638 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -1063,29 +1062,6 @@ static void __init __map_memblock(phys_addr_t start, phys_addr_t end, end - start, prot, early_pgtable_alloc, flags); } -static void mark_linear_data_alias_valid(bool valid) -{ - set_memory_valid((unsigned long)lm_alias(__init_end), - (unsigned long)(__bss_stop - __init_end) / PAGE_SIZE, - valid); -} - -static int arm64_hibernate_pm_notify(struct notifier_block *nb, - unsigned long mode, void *unused) -{ - switch (mode) { - default: - break; - case PM_POST_HIBERNATION: - mark_linear_data_alias_valid(false); - break; - case PM_HIBERNATION_PREPARE: - mark_linear_data_alias_valid(true); - break; - } - return 0; -} - void __init mark_linear_text_alias_ro(void) { /* @@ -1094,21 +1070,6 @@ void __init mark_linear_text_alias_ro(void) update_mapping_prot(__pa_symbol(_text), (unsigned long)lm_alias(_text), (unsigned long)__init_begin - (unsigned long)_text, PAGE_KERNEL_RO); - - /* - * Register a PM notifier to remap the linear alias of data/bss as - * valid read-only before hibernation. This is needed because the - * snapshot logic disregards PageReserved pages (such as the ones - * covering the kernel image) unless they are mapped in the linear - * map. - */ - if (IS_ENABLED(CONFIG_HIBERNATION)) { - static struct notifier_block nb = { - .notifier_call = arm64_hibernate_pm_notify - }; - - register_pm_notifier(&nb); - } } #ifdef CONFIG_KFENCE @@ -1238,8 +1199,10 @@ static void __init map_mem(void) flags); } - /* Map the kernel data/bss as invalid in the linear map */ - mark_linear_data_alias_valid(false); + /* Map the kernel data/bss read-only in the linear map */ + __map_memblock(init_end, kernel_end, PAGE_KERNEL_RO, flags); + flush_tlb_kernel_range((unsigned long)lm_alias(__init_end), + (unsigned long)lm_alias(__bss_stop)); } void mark_rodata_ro(void) From 60349e64a6c65f9f0aa118af711b3c7e137f07ff Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 9 Jun 2026 11:12:01 +0100 Subject: [PATCH 69/73] arm64: cputype: Add C1-Ultra definitions Add cputype definitions for C1-Ultra. These will be used for errata detection in subsequent patches. These values can be found in the C1-Ultra TRM: https://developer.arm.com/documentation/108014/0100/ ... in section A.5.1 ("MIDR_EL1, Main ID Register"). Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/cputype.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 7b518e81dd15..3e223a778186 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -97,6 +97,7 @@ #define ARM_CPU_PART_CORTEX_X925 0xD85 #define ARM_CPU_PART_CORTEX_A725 0xD87 #define ARM_CPU_PART_CORTEX_A720AE 0xD89 +#define ARM_CPU_PART_C1_ULTRA 0xD8C #define ARM_CPU_PART_NEOVERSE_N3 0xD8E #define ARM_CPU_PART_C1_PRO 0xD8B @@ -189,6 +190,7 @@ #define MIDR_CORTEX_X925 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X925) #define MIDR_CORTEX_A725 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A725) #define MIDR_CORTEX_A720AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A720AE) +#define MIDR_C1_ULTRA MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_ULTRA) #define MIDR_NEOVERSE_N3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N3) #define MIDR_C1_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PRO) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) From d28413bfc5a255957241f1df5d7fd0c2cd74fe18 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 9 Jun 2026 11:12:02 +0100 Subject: [PATCH 70/73] arm64: cputype: Add C1-Premium definitions Add cputype definitions for C1-Premium. These will be used for errata detection in subsequent patches. These values can be found in the C1-Premium TRM: https://developer.arm.com/documentation/109416/0100/ ... in section A.5.1 ("MIDR_EL1, Main ID Register"). Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/cputype.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 3e223a778186..1b9f0cda1336 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -100,6 +100,7 @@ #define ARM_CPU_PART_C1_ULTRA 0xD8C #define ARM_CPU_PART_NEOVERSE_N3 0xD8E #define ARM_CPU_PART_C1_PRO 0xD8B +#define ARM_CPU_PART_C1_PREMIUM 0xD90 #define APM_CPU_PART_XGENE 0x000 #define APM_CPU_VAR_POTENZA 0x00 @@ -193,6 +194,7 @@ #define MIDR_C1_ULTRA MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_ULTRA) #define MIDR_NEOVERSE_N3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N3) #define MIDR_C1_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PRO) +#define MIDR_C1_PREMIUM MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PREMIUM) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) From cfd391e74134db664feb499d43af286380b10ba8 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 9 Jun 2026 11:12:03 +0100 Subject: [PATCH 71/73] arm64: errata: Mitigate TLBI errata on various Arm CPUs A number of CPUs developed by Arm suffer from errata whereby a broadcast TLBI;DSB sequence may complete before the global observation of writes which are translated by an affected TLB entry. These errata ONLY affect the completion of memory accesses which have been translated by an invalidated TLB entry, and these errata DO NOT affect the actual invalidation of TLB entries. TLB entries are removed correctly. This issue has been assigned CVE ID CVE-2025-10263. To mitigate this issue, Arm recommends that software follows any affected TLBI;DSB sequence with an additional TLBI;DSB, which will ensure that all memory write effects affected by the first TLBI have been globally observed. The additional TLBI can use any operation that is broadcast to affected CPUs, and the additional DSB can use any option that is sufficient to complete the additional TLBI. The ARM64_WORKAROUND_REPEAT_TLBI workaround is sufficient to mitigate the issue. Enable this workaround for affected CPUs, and update the silicon errata documentation accordingly. Note that due to the manner in which Arm develops IP and tracks errata, some CPUs share a common erratum number. Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Will Deacon Signed-off-by: Will Deacon --- Documentation/arch/arm64/silicon-errata.rst | 42 +++++++++++++++++++++ arch/arm64/Kconfig | 36 ++++++++++++++++++ arch/arm64/kernel/cpu_errata.c | 32 +++++++++++++++- 3 files changed, 108 insertions(+), 2 deletions(-) diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 61c2fd7ef644..6f4a93602abc 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -129,17 +129,29 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A76 | #3324349 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A76 | #4193800 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A76AE | #4193801 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #1491015 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #1508412 | ARM64_ERRATUM_1508412 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #3324348 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A77 | #4193798 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A78 | #3324344 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78 | #4193791 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78AE | #4193793 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A78C | #3324346, | ARM64_ERRATUM_3194386 | | | | #3324347 | | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78C | #4193794 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #2119858 | ARM64_ERRATUM_2119858 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #2054223 | ARM64_ERRATUM_2054223 | @@ -148,6 +160,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #3324338 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A710 | #4193788 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A715 | #2645198 | ARM64_ERRATUM_2645198 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A715 | #3456084 | ARM64_ERRATUM_3194386 | @@ -160,20 +174,32 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X1 | #3324344 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X1 | #4193791 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X1C | #3324346 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X1C | #4193792 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #2119858 | ARM64_ERRATUM_2119858 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #2224489 | ARM64_ERRATUM_2224489 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #3324338 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X2 | #4193788 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X3 | #3324335 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X3 | #4193786 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X4 | #3194386 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X4 | #4118414 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X925 | #3324334 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X925 | #4193781 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1188873, | ARM64_ERRATUM_1418040 | | | | #1418040 | | +----------------+-----------------+-----------------+-----------------------------+ @@ -184,6 +210,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #3324349 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-N1 | #4193800 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #2139208 | ARM64_ERRATUM_2139208 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #2067961 | ARM64_ERRATUM_2067961 | @@ -192,20 +220,34 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #3324339 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-N2 | #4193789 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N3 | #3456111 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V1 | #1619801 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V1 | #3324341 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V1 | #4193790 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V2 | #3324336 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V2 | #4193787 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3 | #3312417 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V3 | #4193784 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3AE | #3312417 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V3AE | #4193784 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | C1-Premium | #4193780 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | C1-Pro | #4193714 | ARM64_ERRATUM_4193714 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | C1-Ultra | #4193780 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-500 | #562869, | ARM_SMMU_MMU_500_CPRE_ERRATA| | | | #841119, | | | | | #826419, | | diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 75638e37883d..2b17033d9662 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1154,6 +1154,42 @@ config ARM64_ERRATUM_4193714 If unsure, say Y. +config ARM64_ERRATUM_4118414 + bool "Cortex-*/Neoverse-*/C1-*: Completion of affected memory accesses might not be guaranteed by completion of a TLBI" + default y + select ARM64_WORKAROUND_REPEAT_TLBI + help + This option adds a workaround for the following errata: + + * ARM C1-Premium erratum 4193780 + * ARM C1-Ultra erratum 4193780 + * ARM Cortex-A76 erratum 4193800 + * ARM Cortex-A76AE erratum 4193801 + * ARM Cortex-A77 erratum 4193798 + * ARM Cortex-A78 erratum 4193791 + * ARM Cortex-A78AE erratum 4193793 + * ARM Cortex-A78C erratum 4193794 + * ARM Cortex-A710 erratum 4193788 + * ARM Cortex-X1 erratum 4193791 + * ARM Cortex-X1C erratum 4193792 + * ARM Cortex-X2 erratum 4193788 + * ARM Cortex-X3 erratum 4193786 + * ARM Cortex-X4 erratum 4118414 + * ARM Cortex-X925 erratum 4193781 + * ARM Neoverse-N1 erratum 4193800 + * ARM Neoverse-N2 erratum 4193789 + * ARM Neoverse-V1 erratum 4193790 + * ARM Neoverse-V2 erratum 4193787 + * ARM Neoverse-V3 erratum 4193784 + * ARM Neoverse-V3AE erratum 4193784 + + On affected cores, some memory accesses might not be completed by + broadcast TLB invalidation. + + This issue is also known as CVE-2025-10263. + + If unsure, say Y. + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 310e6f120992..95e3be145fb1 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -340,7 +340,35 @@ static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { ERRATA_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1), }, #endif - {}, +#ifdef CONFIG_ARM64_ERRATUM_4118414 + { + ERRATA_MIDR_RANGE_LIST(((const struct midr_range[]) { + MIDR_ALL_VERSIONS(MIDR_C1_PREMIUM), + MIDR_ALL_VERSIONS(MIDR_C1_ULTRA), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A77), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X3), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X4), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X925), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3AE), + {} + })), + }, +#endif + {} }; #endif @@ -705,7 +733,7 @@ const struct arm64_cpu_capabilities arm64_errata[] = { #endif #ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI { - .desc = "Qualcomm erratum 1009, or ARM erratum 1286807, 2441009", + .desc = "Broken broadcast TLBI completion", .capability = ARM64_WORKAROUND_REPEAT_TLBI, .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, .matches = cpucap_multi_entry_cap_matches, From ec7216f92e4ebd485b1c6dc6aa3f6064b71a5768 Mon Sep 17 00:00:00 2001 From: Shanker Donthineni Date: Tue, 9 Jun 2026 18:40:44 -0500 Subject: [PATCH 72/73] arm64: errata: Mitigate TLBI errata on NVIDIA Olympus CPU NVIDIA Olympus cores are affected by the TLBI completion issue tracked as CVE-2025-10263. The existing ARM64_ERRATUM_4118414 handling already uses ARM64_WORKAROUND_REPEAT_TLBI to issue an additional broadcast TLBI;DSB sequence and ensure affected memory write effects are globally observed. Add MIDR_NVIDIA_OLYMPUS to the repeat-TLBI match list so the same mitigation is enabled on affected Olympus systems. Also document the NVIDIA Olympus erratum in the arm64 silicon errata table and list it in the Kconfig help text. Signed-off-by: Shanker Donthineni Cc: Catalin Marinas Cc: Will Deacon Cc: Mark Rutland Acked-by: Mark Rutland Signed-off-by: Will Deacon --- Documentation/arch/arm64/silicon-errata.rst | 2 ++ arch/arm64/Kconfig | 3 ++- arch/arm64/kernel/cpu_errata.c | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 6f4a93602abc..dc4de8f2dce3 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -304,6 +304,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | Carmel Core | N/A | NVIDIA_CARMEL_CNP_ERRATUM | +----------------+-----------------+-----------------+-----------------------------+ +| NVIDIA | Olympus core | T410-OLY-1029 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | T241 GICv3/4.x | T241-FABRIC-4 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | T241 MPAM | T241-MPAM-1, | N/A | diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 2b17033d9662..eac9a02b5f62 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1155,7 +1155,7 @@ config ARM64_ERRATUM_4193714 If unsure, say Y. config ARM64_ERRATUM_4118414 - bool "Cortex-*/Neoverse-*/C1-*: Completion of affected memory accesses might not be guaranteed by completion of a TLBI" + bool "Various: Completion of affected memory accesses might not be guaranteed by completion of a TLBI" default y select ARM64_WORKAROUND_REPEAT_TLBI help @@ -1182,6 +1182,7 @@ config ARM64_ERRATUM_4118414 * ARM Neoverse-V2 erratum 4193787 * ARM Neoverse-V3 erratum 4193784 * ARM Neoverse-V3AE erratum 4193784 + * NVIDIA Olympus erratum T410-OLY-1029 On affected cores, some memory accesses might not be completed by broadcast TLB invalidation. diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 95e3be145fb1..476b0108057f 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -364,6 +364,7 @@ static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3AE), + MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), {} })), }, From 1940e70a8144bf75e6df26bf6f600862ea7f7ea1 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 10 Jun 2026 12:00:21 +0100 Subject: [PATCH 73/73] arm64: errata: Mitigate TLBI errata on Microsoft Azure Cobalt 100 CPU Commit fb091ff39479 ("arm64: Subscribe Microsoft Azure Cobalt 100 to ARM Neoverse N2 errata") states that Microsoft Azure Cobalt 100 CPU "is a Microsoft implemented CPU based on r0p0 of the ARM Neoverse N2 CPU, and therefore suffers from all the same errata.". So enable the workaround for the latest broadcast TLB invalidation bug on these parts. Signed-off-by: Will Deacon --- Documentation/arch/arm64/silicon-errata.rst | 2 ++ arch/arm64/Kconfig | 1 + arch/arm64/kernel/cpu_errata.c | 1 + 3 files changed, 4 insertions(+) diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index dc4de8f2dce3..014aa1c215a1 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -372,3 +372,5 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | Microsoft | Azure Cobalt 100| #3324339 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| Microsoft | Azure Cobalt 100| #4193789 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index eac9a02b5f62..c053f012c6a6 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1182,6 +1182,7 @@ config ARM64_ERRATUM_4118414 * ARM Neoverse-V2 erratum 4193787 * ARM Neoverse-V3 erratum 4193784 * ARM Neoverse-V3AE erratum 4193784 + * Microsoft Azure Cobalt 100 4193789 * NVIDIA Olympus erratum T410-OLY-1029 On affected cores, some memory accesses might not be completed by diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 476b0108057f..1995e1198648 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -365,6 +365,7 @@ static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3AE), MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), + MIDR_ALL_VERSIONS(MIDR_MICROSOFT_AZURE_COBALT_100), {} })), },