diff --git a/Documentation/arch/arm64/elf_hwcaps.rst b/Documentation/arch/arm64/elf_hwcaps.rst index 97315ae6c0da..f60ca5612daa 100644 --- a/Documentation/arch/arm64/elf_hwcaps.rst +++ b/Documentation/arch/arm64/elf_hwcaps.rst @@ -451,6 +451,33 @@ HWCAP3_LS64 of CPU. User should only use ld64b/st64b on supported target (device) memory location, otherwise fallback to the non-atomic alternatives. +HWCAP3_SVE_B16MM + Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and + ID_AA64ZFR0_EL1.B16B16 == 0b0011 + +HWCAP3_SVE2P3 + Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and + ID_AA64ZFR0_EL1.SVEver == 0b0100 + +HWCAP3_SME_LUT6 + Functionality implied by ID_AA64SMFR0_EL1.LUT6 == 0b1 + +HWCAP3_SME2P3 + Functionality implied by ID_AA64SMFR0_EL1.SMEver == 0b0100 + +HWCAP3_F16MM + Functionality implied by ID_AA64FPFR0_EL1.F16MM2 == 0b1 + +HWCAP3_F16F32DOT + Functionality implied by ID_AA64ISAR0_EL1.FHM == 0b0010 + +HWCAP3_F16F32MM + Functionality implied by ID_AA64ISAR0_EL1.FHM == 0b0011 + +HWCAP3_SVE_LUT6 + Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and + ID_AA64ISAR2_EL1.LUT == 0b0010. + 4. Unused AT_HWCAP bits ----------------------- diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 211119ce7adc..014aa1c215a1 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -116,7 +116,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A73 | #858921 | ARM64_ERRATUM_858921 | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | Cortex-A76 | #1188873,1418040| ARM64_ERRATUM_1418040 | +| ARM | Cortex-A76 | #1188873, | ARM64_ERRATUM_1418040 | +| | | #1418040 | | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A76 | #1165522 | ARM64_ERRATUM_1165522 | +----------------+-----------------+-----------------+-----------------------------+ @@ -128,15 +129,28 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A76 | #3324349 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A76 | #4193800 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A76AE | #4193801 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #1491015 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #1508412 | ARM64_ERRATUM_1508412 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #3324348 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A77 | #4193798 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A78 | #3324344 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | Cortex-A78C | #3324346,3324347| ARM64_ERRATUM_3194386 | +| ARM | Cortex-A78 | #4193791 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78AE | #4193793 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78C | #3324346, | ARM64_ERRATUM_3194386 | +| | | #3324347 | | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78C | #4193794 | ARM64_ERRATUM_4118414 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #2119858 | ARM64_ERRATUM_2119858 | +----------------+-----------------+-----------------+-----------------------------+ @@ -146,6 +160,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #3324338 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A710 | #4193788 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A715 | #2645198 | ARM64_ERRATUM_2645198 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A715 | #3456084 | ARM64_ERRATUM_3194386 | @@ -158,30 +174,44 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X1 | #3324344 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X1 | #4193791 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X1C | #3324346 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X1C | #4193792 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #2119858 | ARM64_ERRATUM_2119858 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #2224489 | ARM64_ERRATUM_2224489 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #3324338 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X2 | #4193788 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X3 | #3324335 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X3 | #4193786 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X4 | #3194386 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X4 | #4118414 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X925 | #3324334 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | Neoverse-N1 | #1188873,1418040| ARM64_ERRATUM_1418040 | +| ARM | Cortex-X925 | #4193781 | ARM64_ERRATUM_4118414 | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | Neoverse-N1 | #1349291 | N/A | +| ARM | Neoverse-N1 | #1188873, | ARM64_ERRATUM_1418040 | +| | | #1418040 | | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | Neoverse-N1 | #1490853 | N/A | +| ARM | Neoverse-N1 | #1349291, | N/A | +| | | #1490853 | | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1542419 | ARM64_ERRATUM_1542419 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #3324349 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-N1 | #4193800 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #2139208 | ARM64_ERRATUM_2139208 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #2067961 | ARM64_ERRATUM_2067961 | @@ -190,24 +220,41 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #3324339 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-N2 | #4193789 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N3 | #3456111 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V1 | #1619801 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V1 | #3324341 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V1 | #4193790 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V2 | #3324336 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V2 | #4193787 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3 | #3312417 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V3 | #4193784 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3AE | #3312417 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V3AE | #4193784 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | C1-Premium | #4193780 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | C1-Pro | #4193714 | ARM64_ERRATUM_4193714 | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | MMU-500 | #841119,826419 | ARM_SMMU_MMU_500_CPRE_ERRATA| -| | | #562869,1047329 | | +| ARM | C1-Ultra | #4193780 | ARM64_ERRATUM_4118414 | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | MMU-600 | #1076982,1209401| N/A | +| ARM | MMU-500 | #562869, | ARM_SMMU_MMU_500_CPRE_ERRATA| +| | | #841119, | | +| | | #826419, | | +| | | #1047329 | | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | MMU-600 | #1076982, | N/A | +| | | #1209401 | | +----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-700 | #2133013, | N/A | | | | #2268618, | | @@ -230,11 +277,13 @@ stable kernels. | Broadcom | Brahma-B53 | N/A | ARM64_ERRATUM_843419 | +----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ -| Cavium | ThunderX ITS | #22375,24313 | CAVIUM_ERRATUM_22375 | +| Cavium | ThunderX ITS | #22375, | CAVIUM_ERRATUM_22375 | +| | | #24313 | | +----------------+-----------------+-----------------+-----------------------------+ | Cavium | ThunderX ITS | #23144 | CAVIUM_ERRATUM_23144 | +----------------+-----------------+-----------------+-----------------------------+ -| Cavium | ThunderX GICv3 | #23154,38545 | CAVIUM_ERRATUM_23154 | +| Cavium | ThunderX GICv3 | #23154, | CAVIUM_ERRATUM_23154 | +| | | #38545 | | +----------------+-----------------+-----------------+-----------------------------+ | Cavium | ThunderX GICv3 | #38539 | N/A | +----------------+-----------------+-----------------+-----------------------------+ @@ -244,9 +293,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | Cavium | ThunderX SMMUv2 | #27704 | N/A | +----------------+-----------------+-----------------+-----------------------------+ -| Cavium | ThunderX2 SMMUv3| #74 | N/A | -+----------------+-----------------+-----------------+-----------------------------+ -| Cavium | ThunderX2 SMMUv3| #126 | N/A | +| Cavium | ThunderX2 SMMUv3| #74, | N/A | +| | | #126 | | +----------------+-----------------+-----------------+-----------------------------+ | Cavium | ThunderX2 Core | #219 | CAVIUM_TX2_ERRATUM_219 | +----------------+-----------------+-----------------+-----------------------------+ @@ -256,13 +304,13 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | Carmel Core | N/A | NVIDIA_CARMEL_CNP_ERRATUM | +----------------+-----------------+-----------------+-----------------------------+ +| NVIDIA | Olympus core | T410-OLY-1029 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | T241 GICv3/4.x | T241-FABRIC-4 | N/A | +----------------+-----------------+-----------------+-----------------------------+ -| NVIDIA | T241 MPAM | T241-MPAM-1 | N/A | -+----------------+-----------------+-----------------+-----------------------------+ -| NVIDIA | T241 MPAM | T241-MPAM-4 | N/A | -+----------------+-----------------+-----------------+-----------------------------+ -| NVIDIA | T241 MPAM | T241-MPAM-6 | N/A | +| NVIDIA | T241 MPAM | T241-MPAM-1, | N/A | +| | | T241-MPAM-4, | | +| | | T241-MPAM-6 | | +----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ | Freescale/NXP | LS2080A/LS1043A | A-008585 | FSL_ERRATUM_A008585 | @@ -270,9 +318,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | Hisilicon | Hip0{5,6,7} | #161010101 | HISILICON_ERRATUM_161010101 | +----------------+-----------------+-----------------+-----------------------------+ -| Hisilicon | Hip0{6,7} | #161010701 | N/A | -+----------------+-----------------+-----------------+-----------------------------+ -| Hisilicon | Hip0{6,7} | #161010803 | N/A | +| Hisilicon | Hip0{6,7} | #161010701, | N/A | +| | | #161010803 | | +----------------+-----------------+-----------------+-----------------------------+ | Hisilicon | Hip07 | #161600802 | HISILICON_ERRATUM_161600802 | +----------------+-----------------+-----------------+-----------------------------+ @@ -284,6 +331,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | Hisilicon | Hip09 | #162100801 | HISILICON_ERRATUM_162100801 | +----------------+-----------------+-----------------+-----------------------------+ +| Hisilicon | Hip09 | #162100125 | HISILICON_ERRATUM_162100125 | ++----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ | Qualcomm Tech. | Kryo/Falkor v1 | E1003 | QCOM_FALKOR_ERRATUM_1003 | +----------------+-----------------+-----------------+-----------------------------+ @@ -323,3 +372,5 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | Microsoft | Azure Cobalt 100| #3324339 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| Microsoft | Azure Cobalt 100| #4193789 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ diff --git a/MAINTAINERS b/MAINTAINERS index dbd4552236e6..a37cfdccf524 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11635,7 +11635,7 @@ F: Documentation/devicetree/bindings/net/hisilicon*.txt F: drivers/net/ethernet/hisilicon/ HISILICON PMU DRIVER -M: Jonathan Cameron +M: Yushan Wang S: Supported W: http://www.hisilicon.com F: Documentation/admin-guide/perf/hisi-pcie-pmu.rst diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 7e331b4f480a..5f5597141c15 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1153,6 +1153,44 @@ config ARM64_ERRATUM_4193714 If unsure, say Y. +config ARM64_ERRATUM_4118414 + bool "Various: Completion of affected memory accesses might not be guaranteed by completion of a TLBI" + default y + select ARM64_WORKAROUND_REPEAT_TLBI + help + This option adds a workaround for the following errata: + + * ARM C1-Premium erratum 4193780 + * ARM C1-Ultra erratum 4193780 + * ARM Cortex-A76 erratum 4193800 + * ARM Cortex-A76AE erratum 4193801 + * ARM Cortex-A77 erratum 4193798 + * ARM Cortex-A78 erratum 4193791 + * ARM Cortex-A78AE erratum 4193793 + * ARM Cortex-A78C erratum 4193794 + * ARM Cortex-A710 erratum 4193788 + * ARM Cortex-X1 erratum 4193791 + * ARM Cortex-X1C erratum 4193792 + * ARM Cortex-X2 erratum 4193788 + * ARM Cortex-X3 erratum 4193786 + * ARM Cortex-X4 erratum 4118414 + * ARM Cortex-X925 erratum 4193781 + * ARM Neoverse-N1 erratum 4193800 + * ARM Neoverse-N2 erratum 4193789 + * ARM Neoverse-V1 erratum 4193790 + * ARM Neoverse-V2 erratum 4193787 + * ARM Neoverse-V3 erratum 4193784 + * ARM Neoverse-V3AE erratum 4193784 + * Microsoft Azure Cobalt 100 4193789 + * NVIDIA Olympus erratum T410-OLY-1029 + + On affected cores, some memory accesses might not be completed by + broadcast TLB invalidation. + + This issue is also known as CVE-2025-10263. + + If unsure, say Y. + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y @@ -1272,6 +1310,22 @@ config HISILICON_ERRATUM_162100801 If unsure, say Y. +config HISILICON_ERRATUM_162100125 + bool "Hisilicon erratum 162100125" + default y + select ARM64_WORKAROUND_DISABLE_CNP + help + On HiSilicon HIP09, TLB entry matching behavior when CNP + (TTBRx.CNP=1) is enabled differs from the ARM architecture + specification. + + TLB entries may be incorrectly shared between CPUs, potentially + causing TLB conflicts and stale mappings. + + Disable CNP support for affected HiSilicon HIP09 cores. + + If unsure, say Y. + config QCOM_FALKOR_ERRATUM_1003 bool "Falkor E1003: Incorrect translation due to ASID change" default y @@ -1314,9 +1368,13 @@ config QCOM_FALKOR_ERRATUM_E1041 If unsure, say Y. +config ARM64_WORKAROUND_DISABLE_CNP + bool + config NVIDIA_CARMEL_CNP_ERRATUM bool "NVIDIA Carmel CNP: CNP on Carmel semantically different than ARM cores" default y + select ARM64_WORKAROUND_DISABLE_CNP help If CNP is enabled on Carmel cores, non-sharable TLBIs on a core will not invalidate shared TLB entries installed by a different core, as it would @@ -2246,10 +2304,15 @@ config ARM64_SVE booting the kernel. If unsure and you are not observing these symptoms, you should assume that it is safe to say Y. +config AS_HAS_SME + # Supported by LLVM 13+ and binutils 2.38+ + def_bool $(as-instr,.arch_extension sme) + config ARM64_SME bool "ARM Scalable Matrix Extension support" default y depends on ARM64_SVE + depends on AS_HAS_SME help The Scalable Matrix Extension (SME) is an extension to the AArch64 execution state which utilises a substantial subset of the SVE diff --git a/arch/arm64/include/asm/arch_timer.h b/arch/arm64/include/asm/arch_timer.h index f5794d50f51d..6717209df05b 100644 --- a/arch/arm64/include/asm/arch_timer.h +++ b/arch/arm64/include/asm/arch_timer.h @@ -178,12 +178,8 @@ static __always_inline u64 __arch_counter_get_cntpct_stable(void) static __always_inline u64 __arch_counter_get_cntpct(void) { - u64 cnt; + u64 cnt = arch_timer_read_cntpct_el0(); - asm volatile(ALTERNATIVE("isb\n mrs %0, cntpct_el0", - "nop\n" __mrs_s("%0", SYS_CNTPCTSS_EL0), - ARM64_HAS_ECV) - : "=r" (cnt)); arch_counter_enforce_ordering(cnt); return cnt; } @@ -199,12 +195,8 @@ static __always_inline u64 __arch_counter_get_cntvct_stable(void) static __always_inline u64 __arch_counter_get_cntvct(void) { - u64 cnt; + u64 cnt = arch_timer_read_cntvct_el0(); - asm volatile(ALTERNATIVE("isb\n mrs %0, cntvct_el0", - "nop\n" __mrs_s("%0", SYS_CNTVCTSS_EL0), - ARM64_HAS_ECV) - : "=r" (cnt)); arch_counter_enforce_ordering(cnt); return cnt; } diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index d0d3cdd5763c..25c61cda901c 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -58,8 +58,8 @@ cpucap_is_possible(const unsigned int cap) return IS_ENABLED(CONFIG_ARM64_ERRATUM_2658417); case ARM64_WORKAROUND_CAVIUM_23154: return IS_ENABLED(CONFIG_CAVIUM_ERRATUM_23154); - case ARM64_WORKAROUND_NVIDIA_CARMEL_CNP: - return IS_ENABLED(CONFIG_NVIDIA_CARMEL_CNP_ERRATUM); + case ARM64_WORKAROUND_DISABLE_CNP: + return IS_ENABLED(CONFIG_ARM64_WORKAROUND_DISABLE_CNP); case ARM64_WORKAROUND_REPEAT_TLBI: return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI); case ARM64_WORKAROUND_SPECULATIVE_SSBS: diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 4de51f8d92cb..a57870fa96db 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -620,6 +620,13 @@ static inline bool id_aa64pfr0_mpam(u64 pfr0) return val > 0; } +static inline bool id_aa64pfr1_mpamfrac(u64 pfr1) +{ + u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_EL1_MPAM_frac_SHIFT); + + return val > 0; +} + static inline bool id_aa64pfr1_mte(u64 pfr1) { u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_EL1_MTE_SHIFT); diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 7b518e81dd15..1b9f0cda1336 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -97,8 +97,10 @@ #define ARM_CPU_PART_CORTEX_X925 0xD85 #define ARM_CPU_PART_CORTEX_A725 0xD87 #define ARM_CPU_PART_CORTEX_A720AE 0xD89 +#define ARM_CPU_PART_C1_ULTRA 0xD8C #define ARM_CPU_PART_NEOVERSE_N3 0xD8E #define ARM_CPU_PART_C1_PRO 0xD8B +#define ARM_CPU_PART_C1_PREMIUM 0xD90 #define APM_CPU_PART_XGENE 0x000 #define APM_CPU_VAR_POTENZA 0x00 @@ -189,8 +191,10 @@ #define MIDR_CORTEX_X925 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X925) #define MIDR_CORTEX_A725 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A725) #define MIDR_CORTEX_A720AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A720AE) +#define MIDR_C1_ULTRA MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_ULTRA) #define MIDR_NEOVERSE_N3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N3) #define MIDR_C1_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PRO) +#define MIDR_C1_PREMIUM MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PREMIUM) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) diff --git a/arch/arm64/include/asm/daifflags.h b/arch/arm64/include/asm/daifflags.h index 5fca48009043..795b35128467 100644 --- a/arch/arm64/include/asm/daifflags.h +++ b/arch/arm64/include/asm/daifflags.h @@ -19,7 +19,7 @@ /* mask/save/unmask/restore all exceptions, including interrupts. */ -static inline void local_daif_mask(void) +static __always_inline void local_daif_mask(void) { WARN_ON(system_has_prio_mask_debugging() && (read_sysreg_s(SYS_ICC_PMR_EL1) == (GIC_PRIO_IRQOFF | @@ -38,7 +38,7 @@ static inline void local_daif_mask(void) trace_hardirqs_off(); } -static inline unsigned long local_daif_save_flags(void) +static __always_inline unsigned long local_daif_save_flags(void) { unsigned long flags; @@ -53,7 +53,7 @@ static inline unsigned long local_daif_save_flags(void) return flags; } -static inline unsigned long local_daif_save(void) +static __always_inline unsigned long local_daif_save(void) { unsigned long flags; @@ -64,7 +64,7 @@ static inline unsigned long local_daif_save(void) return flags; } -static inline void local_daif_restore(unsigned long flags) +static __always_inline void local_daif_restore(unsigned long flags) { bool irq_disabled = flags & PSR_I_BIT; @@ -124,7 +124,7 @@ static inline void local_daif_restore(unsigned long flags) * Called by synchronous exception handlers to restore the DAIF bits that were * modified by taking an exception. */ -static inline void local_daif_inherit(struct pt_regs *regs) +static __always_inline void local_daif_inherit(struct pt_regs *regs) { unsigned long flags = regs->pstate & DAIF_MASK; diff --git a/arch/arm64/include/asm/device.h b/arch/arm64/include/asm/device.h deleted file mode 100644 index 996498751318..000000000000 --- a/arch/arm64/include/asm/device.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 ARM Ltd. - */ -#ifndef __ASM_DEVICE_H -#define __ASM_DEVICE_H - -struct dev_archdata { -}; - -struct pdev_archdata { -}; - -#endif diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index 587507a9980e..aa8ec9df8024 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -510,7 +510,9 @@ #endif .macro finalise_el2_state - check_override id_aa64pfr0, ID_AA64PFR0_EL1_MPAM_SHIFT, .Linit_mpam_\@, .Lskip_mpam_\@, x1, x2 + check_override id_aa64pfr0, ID_AA64PFR0_EL1_MPAM_SHIFT, .Linit_mpam_\@, .Lmpam_minor_\@, x1, x2 +.Lmpam_minor_\@: + check_override id_aa64pfr1, ID_AA64PFR1_EL1_MPAM_frac_SHIFT, .Linit_mpam_\@, .Lskip_mpam_\@, x1, x2 .Linit_mpam_\@: mov x0, #MPAM2_EL2_EnMPAMSM_MASK diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index 7e86d400864e..81c17320a588 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -160,6 +160,8 @@ #define ESR_ELx_CM (UL(1) << ESR_ELx_CM_SHIFT) /* ISS2 field definitions for Data Aborts */ +#define ESR_ELx_HDBSSF_SHIFT (11) +#define ESR_ELx_HDBSSF (UL(1) << ESR_ELx_HDBSSF_SHIFT) #define ESR_ELx_TnD_SHIFT (10) #define ESR_ELx_TnD (UL(1) << ESR_ELx_TnD_SHIFT) #define ESR_ELx_TagAccess_SHIFT (9) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index d9d00b45ab11..a67d5774e672 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -22,6 +22,11 @@ #include #include +#define __FPSIMD_PREAMBLE ".arch_extension fp\n" \ + ".arch_extension simd\n" +#define __SVE_PREAMBLE ".arch_extension sve\n" +#define __SME_PREAMBLE ".arch_extension sme\n" + /* Masks for extracting the FPSR and FPCR from the FPSCR */ #define VFP_FPSCR_STAT_MASK 0xf800009f #define VFP_FPSCR_CTRL_MASK 0x07f79f00 @@ -71,8 +76,82 @@ static inline void cpacr_restore(unsigned long cpacr) struct task_struct; -extern void fpsimd_save_state(struct user_fpsimd_state *state); -extern void fpsimd_load_state(struct user_fpsimd_state *state); +static inline void fpsimd_save_common(struct user_fpsimd_state *state) +{ + state->fpsr = read_sysreg_s(SYS_FPSR); + state->fpcr = read_sysreg_s(SYS_FPCR); +} + +static inline void fpsimd_load_common(const struct user_fpsimd_state *state) +{ + write_sysreg_s(state->fpsr, SYS_FPSR); + write_sysreg_s(state->fpcr, SYS_FPCR); +} + +static inline void fpsimd_save_vregs(struct user_fpsimd_state *state) +{ + instrument_write(state->vregs, sizeof(state->vregs)); + asm volatile( + __FPSIMD_PREAMBLE + " stp q0, q1, [%[vregs], #16 * 0]\n" + " stp q2, q3, [%[vregs], #16 * 2]\n" + " stp q4, q5, [%[vregs], #16 * 4]\n" + " stp q6, q7, [%[vregs], #16 * 6]\n" + " stp q8, q9, [%[vregs], #16 * 8]\n" + " stp q10, q11, [%[vregs], #16 * 10]\n" + " stp q12, q13, [%[vregs], #16 * 12]\n" + " stp q14, q15, [%[vregs], #16 * 14]\n" + " stp q16, q17, [%[vregs], #16 * 16]\n" + " stp q18, q19, [%[vregs], #16 * 18]\n" + " stp q20, q21, [%[vregs], #16 * 20]\n" + " stp q22, q23, [%[vregs], #16 * 22]\n" + " stp q24, q25, [%[vregs], #16 * 24]\n" + " stp q26, q27, [%[vregs], #16 * 26]\n" + " stp q28, q29, [%[vregs], #16 * 28]\n" + " stp q30, q31, [%[vregs], #16 * 30]\n" + : "=Q" (state->vregs) + : [vregs] "r" (state->vregs) + ); +} + +static inline void fpsimd_load_vregs(const struct user_fpsimd_state *state) +{ + instrument_read(state->vregs, sizeof(state->vregs)); + asm volatile( + __FPSIMD_PREAMBLE + " ldp q0, q1, [%[vregs], #16 * 0]\n" + " ldp q2, q3, [%[vregs], #16 * 2]\n" + " ldp q4, q5, [%[vregs], #16 * 4]\n" + " ldp q6, q7, [%[vregs], #16 * 6]\n" + " ldp q8, q9, [%[vregs], #16 * 8]\n" + " ldp q10, q11, [%[vregs], #16 * 10]\n" + " ldp q12, q13, [%[vregs], #16 * 12]\n" + " ldp q14, q15, [%[vregs], #16 * 14]\n" + " ldp q16, q17, [%[vregs], #16 * 16]\n" + " ldp q18, q19, [%[vregs], #16 * 18]\n" + " ldp q20, q21, [%[vregs], #16 * 20]\n" + " ldp q22, q23, [%[vregs], #16 * 22]\n" + " ldp q24, q25, [%[vregs], #16 * 24]\n" + " ldp q26, q27, [%[vregs], #16 * 26]\n" + " ldp q28, q29, [%[vregs], #16 * 28]\n" + " ldp q30, q31, [%[vregs], #16 * 30]\n" + : + : "Q" (state->vregs), + [vregs] "r" (state->vregs) + ); +} + +static inline void fpsimd_save_state(struct user_fpsimd_state *state) +{ + fpsimd_save_vregs(state); + fpsimd_save_common(state); +} + +static inline void fpsimd_load_state(const struct user_fpsimd_state *state) +{ + fpsimd_load_vregs(state); + fpsimd_load_common(state); +} extern void fpsimd_thread_switch(struct task_struct *next); extern void fpsimd_flush_thread(void); @@ -83,8 +162,8 @@ extern void fpsimd_update_current_state(struct user_fpsimd_state const *state); struct cpu_fp_state { struct user_fpsimd_state *st; - void *sve_state; - void *sme_state; + struct arm64_sve_state *sve_state; + struct arm64_sme_state *sme_state; u64 *svcr; u64 *fpmr; unsigned int sve_vl; @@ -116,40 +195,166 @@ extern void task_smstop_sm(struct task_struct *task); /* Maximum VL that SVE/SME VL-agnostic software can transparently support */ #define VL_ARCH_MAX 0x100 -/* Offset of FFR in the SVE register dump */ -static inline size_t sve_ffr_offset(int vl) -{ - return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET; -} - -static inline void *sve_pffr(struct thread_struct *thread) -{ - unsigned int vl; - - if (system_supports_sme() && thread_sm_enabled(thread)) - vl = thread_get_sme_vl(thread); - else - vl = thread_get_sve_vl(thread); - - return (char *)thread->sve_state + sve_ffr_offset(vl); -} - static inline void *thread_zt_state(struct thread_struct *thread) { /* The ZT register state is stored immediately after the ZA state */ unsigned int sme_vq = sve_vq_from_vl(thread_get_sme_vl(thread)); - return thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq); + return (void *)thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq); } -extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr); -extern void sve_load_state(void const *state, u32 const *pfpsr, - int restore_ffr); -extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1); -extern unsigned int sve_get_vl(void); -extern void sve_set_vq(unsigned long vq_minus_1); -extern void sme_set_vq(unsigned long vq_minus_1); -extern void sme_save_state(void *state, int zt); -extern void sme_load_state(void const *state, int zt); +static inline unsigned int sve_get_vl(void) +{ + unsigned int vl; + + asm volatile( + __SVE_PREAMBLE + " rdvl %x[vl], #1\n" + : [vl] "=r" (vl) + ); + + return vl; +} + +#define FOR_EACH_Z_REG(idx_str, asm_str) \ + " .irp " idx_str ",0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31\n" \ + asm_str "\n" \ + " .endr\n" + +#define FOR_EACH_P_REG(idx_str, asm_str) \ + " .irp " idx_str ",0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \ + asm_str "\n" \ + " .endr\n" + +static inline void __sve_save_z(struct arm64_sve_state *state, unsigned long vl) +{ + instrument_write(state, SVE_NUM_ZREGS * vl); + asm volatile( + __SVE_PREAMBLE + FOR_EACH_Z_REG("n", "str z\\n, [%[zregs], #\\n, MUL VL]") + : + : [zregs] "r" (state) + : "memory" + ); +} + +static inline void __sve_load_z(const struct arm64_sve_state *state, unsigned long vl) +{ + instrument_read(state, SVE_NUM_ZREGS * vl); + asm volatile( + __SVE_PREAMBLE + FOR_EACH_Z_REG("n", "ldr z\\n, [%[zregs], #\\n, MUL VL]") + : + : [zregs] "r" (state) + : "memory" + ); +} + +static inline void __sve_save_p(struct arm64_sve_state *state, unsigned long vl, bool ffr) +{ + void *pregs = (void *)state + SVE_NUM_ZREGS * vl; + unsigned long pl = vl / 8; + void *pffr = pregs + SVE_NUM_PREGS * pl; + + instrument_write(pregs, SVE_NUM_PREGS * pl); + asm volatile( + __SVE_PREAMBLE + FOR_EACH_P_REG("n", "str p\\n, [%[pregs], #\\n, MUL VL]\n") + : + : [pregs] "r" (pregs) + : "memory" + ); + + instrument_write(pffr, pl); + if (ffr) { + asm volatile( + __SVE_PREAMBLE + " rdffr p0.b\n" + " str p0, [%[pffr]]\n" + " ldr p0, [%[pregs]]\n" + : + : [pregs] "r" (pregs), + [pffr] "r" (pffr) + : "memory" + ); + } else { + asm volatile( + __SVE_PREAMBLE + " pfalse p0.b\n" + " str p0, [%[pffr]]\n" + " ldr p0, [%[pregs]]\n" + : + : [pregs] "r" (pregs), + [pffr] "r" (pffr) + : "memory" + ); + } +} + +static inline void __sve_load_p(const struct arm64_sve_state *state, unsigned long vl, bool ffr) +{ + const void *pregs = (const void *)state + SVE_NUM_ZREGS * vl; + unsigned long pl = vl / 8; + const void *pffr = pregs + SVE_NUM_PREGS * pl; + + if (ffr) { + instrument_read(pffr, pl); + asm volatile( + __SVE_PREAMBLE + " ldr p0, [%[pffr]]\n" + " wrffr p0.b\n" + : + : [pffr] "r" (pffr) + : "memory" + ); + } + + instrument_read(pregs, SVE_NUM_PREGS * pl); + asm volatile( + __SVE_PREAMBLE + FOR_EACH_P_REG("n", "ldr p\\n, [%[pregs], #\\n, MUL VL]\n") + : + : [pregs] "r" (pregs) + : "memory" + ); +} + +static inline void sve_save_state(struct arm64_sve_state *state, bool ffr) +{ + unsigned long vl = sve_get_vl(); + __sve_save_z(state, vl); + __sve_save_p(state, vl, ffr); +} + +static inline void sve_load_state(const struct arm64_sve_state *state, bool ffr) +{ + unsigned long vl = sve_get_vl(); + __sve_load_z(state, vl); + __sve_load_p(state, vl, ffr); +} + +/* + * Zero all SVE registers except for the first 128 bits of each vector. + * + * The caller must ensure that the VL has been configured and the CPU must be + * in non-streaming mode. + */ +static inline void sve_flush_live(void) +{ + unsigned long vl = sve_get_vl(); + + if (vl > sizeof(__uint128_t)) { + asm volatile( + __FPSIMD_PREAMBLE + FOR_EACH_Z_REG("n", "mov v\\n\\().16b, v\\n\\().16b") + ); + } + + asm volatile( + __SVE_PREAMBLE + FOR_EACH_P_REG("n", "pfalse p\\n\\().b") + " wrffr p0.b\n" + ); +} struct arm64_cpu_capabilities; extern void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__unused); @@ -402,8 +607,20 @@ static inline int sme_max_virtualisable_vl(void) return vec_max_virtualisable_vl(ARM64_VEC_SME); } +static inline unsigned int sme_get_vl(void) +{ + unsigned int vl; + + asm volatile( + __SME_PREAMBLE + " rdsvl %x[vl], #1\n" + : [vl] "=r" (vl) + ); + + return vl; +} + extern void sme_alloc(struct task_struct *task, bool flush); -extern unsigned int sme_get_vl(void); extern int sme_set_current_vl(unsigned long arg); extern int sme_get_current_vl(void); extern void sme_suspend_exit(void); @@ -418,6 +635,106 @@ static inline size_t __sme_state_size(unsigned int sme_vl) return size; } +static inline void __sme_save_za(struct arm64_sme_state *state, unsigned long svl) +{ + /* + * The argument to LDR/STR (array vector) can only encode W12-W15. + * The "Ucj" constraint exists for this, but is only supported by GCC + * 14.1.0+ and LLVM 18.1.0+. + */ + register unsigned int v asm ("w12"); + + instrument_write(state, svl * svl); + for (v = 0; v < svl; v++) { + void *pav = (void *)state + v * svl; + + asm volatile( + __SME_PREAMBLE + " str za[%w[v], #0], [%[pav]]\n" + : + : [v] "r" (v), + [pav] "r" (pav) + : "memory" + ); + } +} + +static inline void __sme_load_za(const struct arm64_sme_state *state, unsigned long svl) +{ + /* See comment in __sme_save_za */ + register unsigned int v asm ("w12"); + + instrument_read(state, svl * svl); + for (v = 0; v < svl; v++) { + void *pav = (void *)state + v * svl; + + asm volatile( + __SME_PREAMBLE + " ldr za[%w[v], #0], [%[pav]]\n" + : + : [v] "r" (v), + [pav] "r" (pav) + : "memory" + ); + } +} + +static inline void __sme_save_zt(struct arm64_sme_state *state, unsigned long svl) +{ + void *pzt = (void *)state + svl * svl; + + instrument_write(pzt, 64); + asm volatile( + __DEFINE_ASM_GPR_NUMS + /* + * STR ZT0, [] + * Supported by binutils 2.41+. + * Supported by LLVM 16+ + */ + " .inst 0xe13f8000 | ((.L__gpr_num_%[pzt]) << 5)\n" + : + : [pzt] "r" (pzt) + : "memory" + ); +} + +static inline void __sme_load_zt(const struct arm64_sme_state *state, unsigned long svl) +{ + void *pzt = (void *)state + svl * svl; + + instrument_read(pzt, 64); + asm volatile( + __DEFINE_ASM_GPR_NUMS + /* + * LDR ZT0, [] + * Supported by binutils 2.41+. + * Supported by LLVM 16+ + */ + " .inst 0xe11f8000 | ((.L__gpr_num_%[pzt]) << 5)\n" + : + : [pzt] "r" (pzt) + : "memory" + ); +} + +static inline void sme_save_state(struct arm64_sme_state *state, bool zt) +{ + unsigned long svl = sme_get_vl(); + + __sme_save_za(state, svl); + if (zt) + __sme_save_zt(state, svl); +} + +static inline void sme_load_state(const struct arm64_sme_state *state, bool zt) +{ + unsigned long svl = sme_get_vl(); + + __sme_load_za(state, svl); + if (zt) + __sme_load_zt(state, svl); +} + /* * Return how many bytes of memory are required to store the full SME * specific state for task, given task's currently configured vector @@ -474,6 +791,9 @@ static inline size_t sme_state_size(struct task_struct const *task) return 0; } +static inline void sme_save_state(struct arm64_sme_state *state, bool zt) { BUILD_BUG(); } +static inline void sme_load_state(const struct arm64_sme_state *state, bool zt) { BUILD_BUG(); } + static inline void sme_enter_from_user_mode(void) { } static inline void sme_exit_to_user_mode(void) { } diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h deleted file mode 100644 index cda81d009c9b..000000000000 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ /dev/null @@ -1,357 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * FP/SIMD state saving and restoring macros - * - * Copyright (C) 2012 ARM Ltd. - * Author: Catalin Marinas - */ - -#include - -.macro fpsimd_save state, tmpnr - stp q0, q1, [\state, #16 * 0] - stp q2, q3, [\state, #16 * 2] - stp q4, q5, [\state, #16 * 4] - stp q6, q7, [\state, #16 * 6] - stp q8, q9, [\state, #16 * 8] - stp q10, q11, [\state, #16 * 10] - stp q12, q13, [\state, #16 * 12] - stp q14, q15, [\state, #16 * 14] - stp q16, q17, [\state, #16 * 16] - stp q18, q19, [\state, #16 * 18] - stp q20, q21, [\state, #16 * 20] - stp q22, q23, [\state, #16 * 22] - stp q24, q25, [\state, #16 * 24] - stp q26, q27, [\state, #16 * 26] - stp q28, q29, [\state, #16 * 28] - stp q30, q31, [\state, #16 * 30]! - mrs x\tmpnr, fpsr - str w\tmpnr, [\state, #16 * 2] - mrs x\tmpnr, fpcr - str w\tmpnr, [\state, #16 * 2 + 4] -.endm - -.macro fpsimd_restore_fpcr state, tmp - /* - * Writes to fpcr may be self-synchronising, so avoid restoring - * the register if it hasn't changed. - */ - mrs \tmp, fpcr - cmp \tmp, \state - b.eq 9999f - msr fpcr, \state -9999: -.endm - -/* Clobbers \state */ -.macro fpsimd_restore state, tmpnr - ldp q0, q1, [\state, #16 * 0] - ldp q2, q3, [\state, #16 * 2] - ldp q4, q5, [\state, #16 * 4] - ldp q6, q7, [\state, #16 * 6] - ldp q8, q9, [\state, #16 * 8] - ldp q10, q11, [\state, #16 * 10] - ldp q12, q13, [\state, #16 * 12] - ldp q14, q15, [\state, #16 * 14] - ldp q16, q17, [\state, #16 * 16] - ldp q18, q19, [\state, #16 * 18] - ldp q20, q21, [\state, #16 * 20] - ldp q22, q23, [\state, #16 * 22] - ldp q24, q25, [\state, #16 * 24] - ldp q26, q27, [\state, #16 * 26] - ldp q28, q29, [\state, #16 * 28] - ldp q30, q31, [\state, #16 * 30]! - ldr w\tmpnr, [\state, #16 * 2] - msr fpsr, x\tmpnr - ldr w\tmpnr, [\state, #16 * 2 + 4] - fpsimd_restore_fpcr x\tmpnr, \state -.endm - -/* Sanity-check macros to help avoid encoding garbage instructions */ - -.macro _check_general_reg nr - .if (\nr) < 0 || (\nr) > 30 - .error "Bad register number \nr." - .endif -.endm - -.macro _sve_check_zreg znr - .if (\znr) < 0 || (\znr) > 31 - .error "Bad Scalable Vector Extension vector register number \znr." - .endif -.endm - -.macro _sve_check_preg pnr - .if (\pnr) < 0 || (\pnr) > 15 - .error "Bad Scalable Vector Extension predicate register number \pnr." - .endif -.endm - -.macro _check_num n, min, max - .if (\n) < (\min) || (\n) > (\max) - .error "Number \n out of range [\min,\max]" - .endif -.endm - -.macro _sme_check_wv v - .if (\v) < 12 || (\v) > 15 - .error "Bad vector select register \v." - .endif -.endm - -/* SVE instruction encodings for non-SVE-capable assemblers */ -/* (pre binutils 2.28, all kernel capable clang versions support SVE) */ - -/* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */ -.macro _sve_str_v nz, nxbase, offset=0 - _sve_check_zreg \nz - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0xe5804000 \ - | (\nz) \ - | ((\nxbase) << 5) \ - | (((\offset) & 7) << 10) \ - | (((\offset) & 0x1f8) << 13) -.endm - -/* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */ -.macro _sve_ldr_v nz, nxbase, offset=0 - _sve_check_zreg \nz - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0x85804000 \ - | (\nz) \ - | ((\nxbase) << 5) \ - | (((\offset) & 7) << 10) \ - | (((\offset) & 0x1f8) << 13) -.endm - -/* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */ -.macro _sve_str_p np, nxbase, offset=0 - _sve_check_preg \np - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0xe5800000 \ - | (\np) \ - | ((\nxbase) << 5) \ - | (((\offset) & 7) << 10) \ - | (((\offset) & 0x1f8) << 13) -.endm - -/* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */ -.macro _sve_ldr_p np, nxbase, offset=0 - _sve_check_preg \np - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0x85800000 \ - | (\np) \ - | ((\nxbase) << 5) \ - | (((\offset) & 7) << 10) \ - | (((\offset) & 0x1f8) << 13) -.endm - -/* RDVL X\nx, #\imm */ -.macro _sve_rdvl nx, imm - _check_general_reg \nx - _check_num (\imm), -0x20, 0x1f - .inst 0x04bf5000 \ - | (\nx) \ - | (((\imm) & 0x3f) << 5) -.endm - -/* RDFFR (unpredicated): RDFFR P\np.B */ -.macro _sve_rdffr np - _sve_check_preg \np - .inst 0x2519f000 \ - | (\np) -.endm - -/* WRFFR P\np.B */ -.macro _sve_wrffr np - _sve_check_preg \np - .inst 0x25289000 \ - | ((\np) << 5) -.endm - -/* PFALSE P\np.B */ -.macro _sve_pfalse np - _sve_check_preg \np - .inst 0x2518e400 \ - | (\np) -.endm - -/* SME instruction encodings for non-SME-capable assemblers */ -/* (pre binutils 2.38/LLVM 13) */ - -/* RDSVL X\nx, #\imm */ -.macro _sme_rdsvl nx, imm - _check_general_reg \nx - _check_num (\imm), -0x20, 0x1f - .inst 0x04bf5800 \ - | (\nx) \ - | (((\imm) & 0x3f) << 5) -.endm - -/* - * STR (vector from ZA array): - * STR ZA[\nw, #\offset], [X\nxbase, #\offset, MUL VL] - */ -.macro _sme_str_zav nw, nxbase, offset=0 - _sme_check_wv \nw - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0xe1200000 \ - | (((\nw) & 3) << 13) \ - | ((\nxbase) << 5) \ - | ((\offset) & 7) -.endm - -/* - * LDR (vector to ZA array): - * LDR ZA[\nw, #\offset], [X\nxbase, #\offset, MUL VL] - */ -.macro _sme_ldr_zav nw, nxbase, offset=0 - _sme_check_wv \nw - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0xe1000000 \ - | (((\nw) & 3) << 13) \ - | ((\nxbase) << 5) \ - | ((\offset) & 7) -.endm - -/* - * LDR (ZT0) - * - * LDR ZT0, nx - */ -.macro _ldr_zt nx - _check_general_reg \nx - .inst 0xe11f8000 \ - | (\nx << 5) -.endm - -/* - * STR (ZT0) - * - * STR ZT0, nx - */ -.macro _str_zt nx - _check_general_reg \nx - .inst 0xe13f8000 \ - | (\nx << 5) -.endm - -.macro __for from:req, to:req - .if (\from) == (\to) - _for__body %\from - .else - __for %\from, %((\from) + ((\to) - (\from)) / 2) - __for %((\from) + ((\to) - (\from)) / 2 + 1), %\to - .endif -.endm - -.macro _for var:req, from:req, to:req, insn:vararg - .macro _for__body \var:req - .noaltmacro - \insn - .altmacro - .endm - - .altmacro - __for \from, \to - .noaltmacro - - .purgem _for__body -.endm - -/* Update ZCR_EL1.LEN with the new VQ */ -.macro sve_load_vq xvqminus1, xtmp, xtmp2 - mrs_s \xtmp, SYS_ZCR_EL1 - bic \xtmp2, \xtmp, ZCR_ELx_LEN_MASK - orr \xtmp2, \xtmp2, \xvqminus1 - cmp \xtmp2, \xtmp - b.eq 921f - msr_s SYS_ZCR_EL1, \xtmp2 //self-synchronising -921: -.endm - -/* Update SMCR_EL1.LEN with the new VQ */ -.macro sme_load_vq xvqminus1, xtmp, xtmp2 - mrs_s \xtmp, SYS_SMCR_EL1 - bic \xtmp2, \xtmp, SMCR_ELx_LEN_MASK - orr \xtmp2, \xtmp2, \xvqminus1 - cmp \xtmp2, \xtmp - b.eq 921f - msr_s SYS_SMCR_EL1, \xtmp2 //self-synchronising -921: -.endm - -/* Preserve the first 128-bits of Znz and zero the rest. */ -.macro _sve_flush_z nz - _sve_check_zreg \nz - mov v\nz\().16b, v\nz\().16b -.endm - -.macro sve_flush_z - _for n, 0, 31, _sve_flush_z \n -.endm -.macro sve_flush_p - _for n, 0, 15, _sve_pfalse \n -.endm -.macro sve_flush_ffr - _sve_wrffr 0 -.endm - -.macro sve_save nxbase, xpfpsr, save_ffr, nxtmp - _for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34 - _for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16 - cbz \save_ffr, 921f - _sve_rdffr 0 - b 922f -921: - _sve_pfalse 0 // Zero out FFR -922: - _sve_str_p 0, \nxbase - _sve_ldr_p 0, \nxbase, -16 - mrs x\nxtmp, fpsr - str w\nxtmp, [\xpfpsr] - mrs x\nxtmp, fpcr - str w\nxtmp, [\xpfpsr, #4] -.endm - -.macro sve_load nxbase, xpfpsr, restore_ffr, nxtmp - _for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34 - cbz \restore_ffr, 921f - _sve_ldr_p 0, \nxbase - _sve_wrffr 0 -921: - _for n, 0, 15, _sve_ldr_p \n, \nxbase, \n - 16 - - ldr w\nxtmp, [\xpfpsr] - msr fpsr, x\nxtmp - ldr w\nxtmp, [\xpfpsr, #4] - msr fpcr, x\nxtmp -.endm - -.macro sme_save_za nxbase, xvl, nw - mov w\nw, #0 - -423: - _sme_str_zav \nw, \nxbase - add x\nxbase, x\nxbase, \xvl - add x\nw, x\nw, #1 - cmp \xvl, x\nw - bne 423b -.endm - -.macro sme_load_za nxbase, xvl, nw - mov w\nw, #0 - -423: - _sme_ldr_zav \nw, \nxbase - add x\nxbase, x\nxbase, \xvl - add x\nw, x\nw, #1 - cmp \xvl, x\nw - bne 423b -.endm diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 8cbd1e96fd50..21c8e400107c 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -270,7 +270,7 @@ static inline void __iomem *ioremap_prot(phys_addr_t phys, size_t size, pgprot_t user_prot) { pgprot_t prot; - ptdesc_t user_prot_val = pgprot_val(user_prot); + ptval_t user_prot_val = pgprot_val(user_prot); if (WARN_ON_ONCE(!(user_prot_val & PTE_USER))) return NULL; diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index a49042bfa801..9209c54350c7 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -732,20 +732,6 @@ struct kvm_cpu_context { u64 *vncr_array; }; -struct cpu_sve_state { - __u64 zcr_el1; - - /* - * Ordering is important since __sve_save_state/__sve_restore_state - * relies on it. - */ - __u32 fpsr; - __u32 fpcr; - - /* Must be SVE_VQ_BYTES (128 bit) aligned. */ - __u8 sve_regs[]; -}; - /* * This structure is instantiated on a per-CPU basis, and contains * data that is: @@ -771,12 +757,9 @@ struct kvm_host_data { /* * Hyp VA. - * sve_state is only used in pKVM and if system_supports_sve(). + * sve_regs is only used in pKVM and if system_supports_sve(). */ - struct cpu_sve_state *sve_state; - - /* Used by pKVM only. */ - u64 fpmr; + struct arm64_sve_state *sve_regs; /* Ownership of the FP regs */ enum { @@ -870,7 +853,7 @@ struct kvm_vcpu_arch { * floating point code saves the register state of a task it * records which view it saved in fp_type. */ - void *sve_state; + struct arm64_sve_state *sve_state; enum fp_type fp_type; unsigned int sve_max_vl; @@ -1114,10 +1097,6 @@ struct kvm_vcpu_arch { #define NESTED_SERROR_PENDING __vcpu_single_flag(sflags, BIT(8)) -/* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */ -#define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) + \ - sve_ffr_offset((vcpu)->arch.sve_max_vl)) - #define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl) #define vcpu_sve_zcr_elx(vcpu) \ diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 8d06b62e7188..ad19de1d0654 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -121,11 +121,6 @@ void __debug_save_host_buffers_nvhe(struct kvm_vcpu *vcpu); void __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu); #endif -void __fpsimd_save_state(struct user_fpsimd_state *fp_regs); -void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs); -void __sve_save_state(void *sve_pffr, u32 *fpsr, int save_ffr); -void __sve_restore_state(void *sve_pffr, u32 *fpsr, int restore_ffr); - u64 __guest_enter(struct kvm_vcpu *vcpu); bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt, u32 func_id); diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index 2954b311128c..74fedd9c5ff0 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -188,8 +188,7 @@ static inline size_t pkvm_host_sve_state_size(void) if (!system_supports_sve()) return 0; - return size_add(sizeof(struct cpu_sve_state), - SVE_SIG_REGS_SIZE(sve_vq_from_vl(kvm_host_sve_max_vl))); + return SVE_SIG_REGS_SIZE(sve_vq_from_vl(kvm_host_sve_max_vl)); } struct pkvm_mapping { diff --git a/arch/arm64/include/asm/linkage.h b/arch/arm64/include/asm/linkage.h index 40bd17add539..d1f7a16729d2 100644 --- a/arch/arm64/include/asm/linkage.h +++ b/arch/arm64/include/asm/linkage.h @@ -43,4 +43,8 @@ SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ bti c ; +#define _THIS_IP_ ({ unsigned long __ip; asm volatile("adr %0, ." : "=r" (__ip)); __ip; }) + +#define __bss_pgtbl __section(".bss..pgtbl") __aligned(PAGE_SIZE) + #endif diff --git a/arch/arm64/include/asm/pgtable-types.h b/arch/arm64/include/asm/pgtable-types.h index 265e8301d7ba..2f2f5527930f 100644 --- a/arch/arm64/include/asm/pgtable-types.h +++ b/arch/arm64/include/asm/pgtable-types.h @@ -17,13 +17,13 @@ * Generic page table descriptor format from which * all level specific descriptors can be derived. */ -typedef u64 ptdesc_t; +typedef u64 ptval_t; -typedef ptdesc_t pteval_t; -typedef ptdesc_t pmdval_t; -typedef ptdesc_t pudval_t; -typedef ptdesc_t p4dval_t; -typedef ptdesc_t pgdval_t; +typedef ptval_t pteval_t; +typedef ptval_t pmdval_t; +typedef ptval_t pudval_t; +typedef ptval_t p4dval_t; +typedef ptval_t pgdval_t; /* * These are used to make use of C type-checking.. @@ -54,7 +54,7 @@ typedef struct { pgdval_t pgd; } pgd_t; #define pgd_val(x) ((x).pgd) #define __pgd(x) ((pgd_t) { (x) } ) -typedef struct { ptdesc_t pgprot; } pgprot_t; +typedef struct { ptval_t pgprot; } pgprot_t; #define pgprot_val(x) ((x).pgprot) #define __pgprot(x) ((pgprot_t) { (x) } ) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 4dfa42b7d053..c9e4e00a9af2 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -1007,7 +1007,7 @@ static inline pud_t *p4d_pgtable(p4d_t p4d) static inline phys_addr_t pud_offset_phys(p4d_t *p4dp, unsigned long addr) { - BUG_ON(!pgtable_l4_enabled()); + VM_WARN_ON_ONCE(!pgtable_l4_enabled()); return p4d_page_paddr(READ_ONCE(*p4dp)) + pud_index(addr) * sizeof(pud_t); } @@ -1130,7 +1130,7 @@ static inline p4d_t *pgd_to_folded_p4d(pgd_t *pgdp, unsigned long addr) static inline phys_addr_t p4d_offset_phys(pgd_t *pgdp, unsigned long addr) { - BUG_ON(!pgtable_l5_enabled()); + VM_WARN_ON_ONCE(!pgtable_l5_enabled()); return pgd_page_paddr(READ_ONCE(*pgdp)) + p4d_index(addr) * sizeof(p4d_t); } diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index e30c4c8e3a7a..c2a627f39314 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -130,6 +130,9 @@ enum fp_type { FP_STATE_SVE, }; +struct arm64_sve_state; /* Opaque type */ +struct arm64_sme_state; /* Opaque type */ + struct cpu_context { unsigned long x19; unsigned long x20; @@ -164,8 +167,8 @@ struct thread_struct { enum fp_type fp_type; /* registers FPSIMD or SVE? */ unsigned int fpsimd_cpu; - void *sve_state; /* SVE registers, if any */ - void *sme_state; /* ZA and ZT state, if any */ + struct arm64_sve_state *sve_state; /* SVE registers, if any */ + struct arm64_sme_state *sme_state; /* ZA and ZT state, if any */ unsigned int vl[ARM64_VEC_MAX]; /* vector length */ unsigned int vl_onexec[ARM64_VEC_MAX]; /* vl after next exec */ unsigned long fault_address; /* fault info */ diff --git a/arch/arm64/include/asm/ptdump.h b/arch/arm64/include/asm/ptdump.h index baff24004459..5b374a6ab34a 100644 --- a/arch/arm64/include/asm/ptdump.h +++ b/arch/arm64/include/asm/ptdump.h @@ -26,8 +26,8 @@ struct ptdump_info { }; struct ptdump_prot_bits { - ptdesc_t mask; - ptdesc_t val; + ptval_t mask; + ptval_t val; const char *set; const char *clear; }; @@ -36,7 +36,7 @@ struct ptdump_pg_level { const struct ptdump_prot_bits *bits; char name[4]; int num; - ptdesc_t mask; + ptval_t mask; }; /* @@ -53,7 +53,7 @@ struct ptdump_pg_state { const struct mm_struct *mm; unsigned long start_address; int level; - ptdesc_t current_prot; + ptval_t current_prot; bool check_wx; unsigned long wx_pages; unsigned long uxn_pages; diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index c0bf5b398041..d52ac8c17190 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -725,9 +725,9 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b sme_dvmsync_add_pending(batch, mm); } -static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval) +static inline bool __pte_flags_need_flush(ptval_t oldval, ptval_t newval) { - ptdesc_t diff = oldval ^ newval; + ptval_t diff = oldval ^ newval; /* invalid to valid transition requires no flush */ if (!(oldval & PTE_VALID)) diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 06f83ca8de56..10272ddb4d6f 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -147,5 +147,13 @@ #define HWCAP3_MTE_STORE_ONLY (1UL << 1) #define HWCAP3_LSFE (1UL << 2) #define HWCAP3_LS64 (1UL << 3) +#define HWCAP3_SVE_B16MM (1UL << 4) +#define HWCAP3_SVE2P3 (1UL << 5) +#define HWCAP3_SME_LUT6 (1UL << 6) +#define HWCAP3_SME2P3 (1UL << 7) +#define HWCAP3_F16MM (1UL << 8) +#define HWCAP3_F16F32DOT (1UL << 9) +#define HWCAP3_F16F32MM (1UL << 10) +#define HWCAP3_SVE_LUT6 (1UL << 11) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 74b76bb70452..d2690c3ec528 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -27,7 +27,7 @@ KCOV_INSTRUMENT_idle.o := n # Object file lists. obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ - entry-common.o entry-fpsimd.o process.o ptrace.o \ + entry-common.o process.o ptrace.o \ setup.o signal.o sys.o stacktrace.o time.o traps.o \ io.o vdso.o hyp-stub.o psci.o cpu_ops.o \ return_address.o cpuinfo.o cpu_errata.o \ diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 5377e4c2eba2..1995e1198648 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -340,7 +340,37 @@ static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { ERRATA_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1), }, #endif - {}, +#ifdef CONFIG_ARM64_ERRATUM_4118414 + { + ERRATA_MIDR_RANGE_LIST(((const struct midr_range[]) { + MIDR_ALL_VERSIONS(MIDR_C1_PREMIUM), + MIDR_ALL_VERSIONS(MIDR_C1_ULTRA), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A77), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X3), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X4), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X925), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3AE), + MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), + MIDR_ALL_VERSIONS(MIDR_MICROSOFT_AZURE_COBALT_100), + {} + })), + }, +#endif + {} }; #endif @@ -608,6 +638,18 @@ static const struct midr_range erratum_ac04_cpu_23_list[] = { }; #endif +#ifdef CONFIG_ARM64_WORKAROUND_DISABLE_CNP +static const struct midr_range cnp_erratum_cpus[] = { +#ifdef CONFIG_NVIDIA_CARMEL_CNP_ERRATUM + MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL), +#endif +#ifdef CONFIG_HISILICON_ERRATUM_162100125 + MIDR_ALL_VERSIONS(MIDR_HISI_HIP09), +#endif + {}, +}; +#endif + const struct arm64_cpu_capabilities arm64_errata[] = { #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE { @@ -693,7 +735,7 @@ const struct arm64_cpu_capabilities arm64_errata[] = { #endif #ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI { - .desc = "Qualcomm erratum 1009, or ARM erratum 1286807, 2441009", + .desc = "Broken broadcast TLBI completion", .capability = ARM64_WORKAROUND_REPEAT_TLBI, .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, .matches = cpucap_multi_entry_cap_matches, @@ -801,12 +843,11 @@ const struct arm64_cpu_capabilities arm64_errata[] = { 1, 0), }, #endif -#ifdef CONFIG_NVIDIA_CARMEL_CNP_ERRATUM +#ifdef CONFIG_ARM64_WORKAROUND_DISABLE_CNP { - /* NVIDIA Carmel */ - .desc = "NVIDIA Carmel CNP erratum", - .capability = ARM64_WORKAROUND_NVIDIA_CARMEL_CNP, - ERRATA_MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL), + .desc = "NVIDIA Carmel CNP erratum, or Hisilicon erratum 162100125", + .capability = ARM64_WORKAROUND_DISABLE_CNP, + ERRATA_MIDR_RANGE_LIST(cnp_erratum_cpus), }, #endif #ifdef CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index e592520e13b7..799d7a09edfd 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -365,6 +365,8 @@ static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = { static const struct arm64_ftr_bits ftr_id_aa64smfr0[] = { ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_FA64_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), + FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_LUT6_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_LUTv2_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), @@ -419,6 +421,7 @@ static const struct arm64_ftr_bits ftr_id_aa64fpfr0[] = { ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8DP2_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8MM8_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8MM4_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F16MM2_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E4M3_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E5M2_SHIFT, 1, 0), ARM64_FTR_END, @@ -1164,6 +1167,14 @@ static __init void detect_system_supports_pseudo_nmi(void) static inline void detect_system_supports_pseudo_nmi(void) { } #endif +static bool detect_ftr_has_mpam(void) +{ + u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); + u64 pfr1 = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1); + + return id_aa64pfr0_mpam(pfr0) || id_aa64pfr1_mpamfrac(pfr1); +} + void __init init_cpu_features(struct cpuinfo_arm64 *info) { /* Before we start using the tables, make sure it is sorted */ @@ -1211,7 +1222,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) cpacr_restore(cpacr); } - if (id_aa64pfr0_mpam(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) { + if (detect_ftr_has_mpam()) { info->reg_mpamidr = read_cpuid(MPAMIDR_EL1); init_cpu_ftr_reg(SYS_MPAMIDR_EL1, info->reg_mpamidr); } @@ -1467,7 +1478,7 @@ void update_cpu_features(int cpu, cpacr_restore(cpacr); } - if (id_aa64pfr0_mpam(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) { + if (detect_ftr_has_mpam()) { info->reg_mpamidr = read_cpuid(MPAMIDR_EL1); taint |= check_update_ftr_reg(SYS_MPAMIDR_EL1, cpu, info->reg_mpamidr, boot->reg_mpamidr); @@ -1785,7 +1796,7 @@ has_useable_cnp(const struct arm64_cpu_capabilities *entry, int scope) if (is_kdump_kernel()) return false; - if (cpus_have_cap(ARM64_WORKAROUND_NVIDIA_CARMEL_CNP)) + if (cpus_have_cap(ARM64_WORKAROUND_DISABLE_CNP)) return false; return has_cpuid_feature(entry, scope); @@ -2486,7 +2497,7 @@ cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap) static bool test_has_mpam(const struct arm64_cpu_capabilities *entry, int scope) { - if (!has_cpuid_feature(entry, scope)) + if (!detect_ftr_has_mpam()) return false; /* Check firmware actually enabled MPAM on this cpu. */ @@ -3093,7 +3104,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .capability = ARM64_MPAM, .matches = test_has_mpam, .cpu_enable = cpu_enable_mpam, - ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, MPAM, 1) }, { .desc = "Memory Partitioning And Monitoring Virtualisation", @@ -3284,6 +3294,8 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64ISAR0_EL1, SM4, IMP, CAP_HWCAP, KERNEL_HWCAP_SM4), HWCAP_CAP(ID_AA64ISAR0_EL1, DP, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDDP), HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDFHM), + HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, F16F32DOT, CAP_HWCAP, KERNEL_HWCAP_F16F32DOT), + HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, F16F32MM, CAP_HWCAP, KERNEL_HWCAP_F16F32MM), HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM, CAP_HWCAP, KERNEL_HWCAP_FLAGM), HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM2, CAP_HWCAP, KERNEL_HWCAP_FLAGM2), HWCAP_CAP(ID_AA64ISAR0_EL1, RNDR, IMP, CAP_HWCAP, KERNEL_HWCAP_RNG), @@ -3313,7 +3325,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64ISAR3_EL1, LSFE, IMP, CAP_HWCAP, KERNEL_HWCAP_LSFE), HWCAP_CAP(ID_AA64MMFR2_EL1, AT, IMP, CAP_HWCAP, KERNEL_HWCAP_USCAT), #ifdef CONFIG_ARM64_SVE + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ISAR2_EL1, LUT, LUT6, CAP_HWCAP, KERNEL_HWCAP_SVE_LUT6), HWCAP_CAP(ID_AA64PFR0_EL1, SVE, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE), + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p3, CAP_HWCAP, KERNEL_HWCAP_SVE2P3), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p2, CAP_HWCAP, KERNEL_HWCAP_SVE2P2), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p1, CAP_HWCAP, KERNEL_HWCAP_SVE2P1), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2), @@ -3323,6 +3337,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BitPerm, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBITPERM), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE_B16B16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, BFSCALE, CAP_HWCAP, KERNEL_HWCAP_SVE_BFSCALE), + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, B16MM, CAP_HWCAP, KERNEL_HWCAP_SVE_B16MM), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBF16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, EBF16, CAP_HWCAP, KERNEL_HWCAP_SVE_EBF16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SHA3, IMP, CAP_HWCAP, KERNEL_HWCAP_SVESHA3), @@ -3362,7 +3377,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { #ifdef CONFIG_ARM64_SME HWCAP_CAP(ID_AA64PFR1_EL1, SME, IMP, CAP_HWCAP, KERNEL_HWCAP_SME), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, FA64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, LUT6, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_LUT6), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, LUTv2, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_LUTV2), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p3, CAP_HWCAP, KERNEL_HWCAP_SME2P3), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p2, CAP_HWCAP, KERNEL_HWCAP_SME2P2), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p1, CAP_HWCAP, KERNEL_HWCAP_SME2P1), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2, CAP_HWCAP, KERNEL_HWCAP_SME2), @@ -3393,6 +3410,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64FPFR0_EL1, F8DP2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8DP2), HWCAP_CAP(ID_AA64FPFR0_EL1, F8MM8, IMP, CAP_HWCAP, KERNEL_HWCAP_F8MM8), HWCAP_CAP(ID_AA64FPFR0_EL1, F8MM4, IMP, CAP_HWCAP, KERNEL_HWCAP_F8MM4), + HWCAP_CAP(ID_AA64FPFR0_EL1, F16MM2, IMP, CAP_HWCAP, KERNEL_HWCAP_F16MM), HWCAP_CAP(ID_AA64FPFR0_EL1, F8E4M3, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E4M3), HWCAP_CAP(ID_AA64FPFR0_EL1, F8E5M2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E5M2), #ifdef CONFIG_ARM64_POE diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 6149bc91251d..d50e2a9b066b 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -164,6 +164,14 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_MTE_FAR] = "mtefar", [KERNEL_HWCAP_MTE_STORE_ONLY] = "mtestoreonly", [KERNEL_HWCAP_LSFE] = "lsfe", + [KERNEL_HWCAP_SVE_B16MM] = "sveb16mm", + [KERNEL_HWCAP_SVE2P3] = "sve2p3", + [KERNEL_HWCAP_SME_LUT6] = "smelut6", + [KERNEL_HWCAP_SME2P3] = "sme2p3", + [KERNEL_HWCAP_F16MM] = "f16mm", + [KERNEL_HWCAP_F16F32DOT] = "f16f32dot", + [KERNEL_HWCAP_F16F32MM] = "f16f32mm", + [KERNEL_HWCAP_SVE_LUT6] = "svelut6", }; #ifdef CONFIG_COMPAT diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index a81cb4aa4738..30cd7f804398 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -31,7 +31,7 @@ static bool region_is_misaligned(const efi_memory_desc_t *md) * executable, everything else can be mapped with the XN bits * set. Also take the new (optional) RO/XP bits into account. */ -static __init ptdesc_t create_mapping_protection(efi_memory_desc_t *md) +static __init ptval_t create_mapping_protection(efi_memory_desc_t *md) { u64 attr = md->attribute; u32 type = md->type; @@ -85,7 +85,7 @@ static __init ptdesc_t create_mapping_protection(efi_memory_desc_t *md) int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) { - ptdesc_t prot_val = create_mapping_protection(md); + ptval_t prot_val = create_mapping_protection(md); bool page_mappings_only = (md->type == EFI_RUNTIME_SERVICES_CODE || md->type == EFI_RUNTIME_SERVICES_DATA); diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index c7a23f7c2212..ceb4eb11232a 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -254,12 +254,8 @@ static inline void fpsimd_syscall_enter(void) if (!system_supports_sve()) return; - if (test_thread_flag(TIF_SVE)) { - unsigned int sve_vq_minus_one; - - sve_vq_minus_one = sve_vq_from_vl(task_get_sve_vl(current)) - 1; - sve_flush_live(true, sve_vq_minus_one); - } + if (test_thread_flag(TIF_SVE)) + sve_flush_live(); /* * Any live non-FPSIMD SVE state has been zeroed. Allow diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S deleted file mode 100644 index 6325db1a2179..000000000000 --- a/arch/arm64/kernel/entry-fpsimd.S +++ /dev/null @@ -1,134 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * FP/SIMD state saving and restoring - * - * Copyright (C) 2012 ARM Ltd. - * Author: Catalin Marinas - */ - -#include - -#include -#include - -/* - * Save the FP registers. - * - * x0 - pointer to struct fpsimd_state - */ -SYM_FUNC_START(fpsimd_save_state) - fpsimd_save x0, 8 - ret -SYM_FUNC_END(fpsimd_save_state) - -/* - * Load the FP registers. - * - * x0 - pointer to struct fpsimd_state - */ -SYM_FUNC_START(fpsimd_load_state) - fpsimd_restore x0, 8 - ret -SYM_FUNC_END(fpsimd_load_state) - -#ifdef CONFIG_ARM64_SVE - -/* - * Save the SVE state - * - * x0 - pointer to buffer for state - * x1 - pointer to storage for FPSR - * x2 - Save FFR if non-zero - */ -SYM_FUNC_START(sve_save_state) - sve_save 0, x1, x2, 3 - ret -SYM_FUNC_END(sve_save_state) - -/* - * Load the SVE state - * - * x0 - pointer to buffer for state - * x1 - pointer to storage for FPSR - * x2 - Restore FFR if non-zero - */ -SYM_FUNC_START(sve_load_state) - sve_load 0, x1, x2, 4 - ret -SYM_FUNC_END(sve_load_state) - -SYM_FUNC_START(sve_get_vl) - _sve_rdvl 0, 1 - ret -SYM_FUNC_END(sve_get_vl) - -SYM_FUNC_START(sve_set_vq) - sve_load_vq x0, x1, x2 - ret -SYM_FUNC_END(sve_set_vq) - -/* - * Zero all SVE registers but the first 128-bits of each vector - * - * VQ must already be configured by caller, any further updates of VQ - * will need to ensure that the register state remains valid. - * - * x0 = include FFR? - * x1 = VQ - 1 - */ -SYM_FUNC_START(sve_flush_live) - cbz x1, 1f // A VQ-1 of 0 is 128 bits so no extra Z state - sve_flush_z -1: sve_flush_p - tbz x0, #0, 2f - sve_flush_ffr -2: ret -SYM_FUNC_END(sve_flush_live) - -#endif /* CONFIG_ARM64_SVE */ - -#ifdef CONFIG_ARM64_SME - -SYM_FUNC_START(sme_get_vl) - _sme_rdsvl 0, 1 - ret -SYM_FUNC_END(sme_get_vl) - -SYM_FUNC_START(sme_set_vq) - sme_load_vq x0, x1, x2 - ret -SYM_FUNC_END(sme_set_vq) - -/* - * Save the ZA and ZT state - * - * x0 - pointer to buffer for state - * x1 - number of ZT registers to save - */ -SYM_FUNC_START(sme_save_state) - _sme_rdsvl 2, 1 // x2 = VL/8 - sme_save_za 0, x2, 12 // Leaves x0 pointing to the end of ZA - - cbz x1, 1f - _str_zt 0 -1: - ret -SYM_FUNC_END(sme_save_state) - -/* - * Load the ZA and ZT state - * - * x0 - pointer to buffer for state - * x1 - number of ZT registers to save - */ -SYM_FUNC_START(sme_load_state) - _sme_rdsvl 2, 1 // x2 = VL/8 - sme_load_za 0, x2, 12 // Leaves x0 pointing to the end of ZA - - cbz x1, 1f - _ldr_zt 0 -1: - ret -SYM_FUNC_END(sme_load_state) - -#endif /* CONFIG_ARM64_SME */ diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 60a45d600b46..25dc5afe9ba0 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -377,8 +377,10 @@ static void task_fpsimd_load(void) if (!thread_sm_enabled(¤t->thread)) WARN_ON_ONCE(!test_and_set_thread_flag(TIF_SVE)); - if (test_thread_flag(TIF_SVE)) - sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1); + if (test_thread_flag(TIF_SVE)) { + unsigned long vq = sve_vq_from_vl(task_get_sve_vl(current)); + sysreg_clear_set_s(SYS_ZCR_EL1, ZCR_ELx_LEN, vq - 1); + } restore_sve_regs = true; restore_ffr = true; @@ -403,8 +405,10 @@ static void task_fpsimd_load(void) unsigned long sme_vl = task_get_sme_vl(current); /* Ensure VL is set up for restoring data */ - if (test_thread_flag(TIF_SME)) - sme_set_vq(sve_vq_from_vl(sme_vl) - 1); + if (test_thread_flag(TIF_SME)) { + unsigned long vq = sve_vq_from_vl(sme_vl); + sysreg_clear_set_s(SYS_SMCR_EL1, SMCR_ELx_LEN, vq - 1); + } write_sysreg_s(current->thread.svcr, SYS_SVCR); @@ -421,9 +425,8 @@ static void task_fpsimd_load(void) if (restore_sve_regs) { WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE); - sve_load_state(sve_pffr(¤t->thread), - ¤t->thread.uw.fpsimd_state.fpsr, - restore_ffr); + sve_load_state(current->thread.sve_state, restore_ffr); + fpsimd_load_common(¤t->thread.uw.fpsimd_state); } else { WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD); fpsimd_load_state(¤t->thread.uw.fpsimd_state); @@ -503,9 +506,8 @@ static void fpsimd_save_user_state(void) return; } - sve_save_state((char *)last->sve_state + - sve_ffr_offset(vl), - &last->st->fpsr, save_ffr); + sve_save_state(last->sve_state, save_ffr); + fpsimd_save_common(last->st); *last->fp_type = FP_STATE_SVE; } else { fpsimd_save_state(last->st); @@ -636,7 +638,8 @@ static __uint128_t arm64_cpu_to_le128(__uint128_t x) #define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x) -static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst, +static void __fpsimd_to_sve(struct arm64_sve_state *sst, + struct user_fpsimd_state const *fst, unsigned int vq) { unsigned int i; @@ -663,7 +666,7 @@ static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst, static inline void fpsimd_to_sve(struct task_struct *task) { unsigned int vq; - void *sst = task->thread.sve_state; + struct arm64_sve_state *sst = task->thread.sve_state; struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; if (!system_supports_sve() && !system_supports_sme()) @@ -687,7 +690,7 @@ static inline void fpsimd_to_sve(struct task_struct *task) static inline void sve_to_fpsimd(struct task_struct *task) { unsigned int vq, vl; - void const *sst = task->thread.sve_state; + const struct arm64_sve_state *sst = task->thread.sve_state; struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state; unsigned int i; __uint128_t const *p; @@ -786,7 +789,7 @@ void fpsimd_sync_from_effective_state(struct task_struct *task) void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task) { unsigned int vq; - void *sst = task->thread.sve_state; + struct arm64_sve_state *sst = task->thread.sve_state; struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; if (task->thread.fp_type != FP_STATE_SVE) @@ -804,7 +807,8 @@ static int change_live_vector_length(struct task_struct *task, { unsigned int sve_vl = task_get_sve_vl(task); unsigned int sme_vl = task_get_sme_vl(task); - void *sve_state = NULL, *sme_state = NULL; + struct arm64_sve_state *sve_state = NULL; + struct arm64_sme_state *sme_state = NULL; if (type == ARM64_VEC_SME) sme_vl = vl; @@ -1293,31 +1297,6 @@ void sme_suspend_exit(void) #endif /* CONFIG_ARM64_SME */ -static void sve_init_regs(void) -{ - /* - * Convert the FPSIMD state to SVE, zeroing all the state that - * is not shared with FPSIMD. If (as is likely) the current - * state is live in the registers then do this there and - * update our metadata for the current task including - * disabling the trap, otherwise update our in-memory copy. - * We are guaranteed to not be in streaming mode, we can only - * take a SVE trap when not in streaming mode and we can't be - * in streaming mode when taking a SME trap. - */ - if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { - unsigned long vq_minus_one = - sve_vq_from_vl(task_get_sve_vl(current)) - 1; - sve_set_vq(vq_minus_one); - sve_flush_live(true, vq_minus_one); - fpsimd_bind_task_to_cpu(); - } else { - fpsimd_to_sve(current); - current->thread.fp_type = FP_STATE_SVE; - fpsimd_flush_task_state(current); - } -} - /* * Trapped SVE access * @@ -1349,13 +1328,23 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs) WARN_ON(1); /* SVE access shouldn't have trapped */ /* - * Even if the task can have used streaming mode we can only - * generate SVE access traps in normal SVE mode and - * transitioning out of streaming mode may discard any - * streaming mode state. Always clear the high bits to avoid - * any potential errors tracking what is properly initialised. + * Convert the FPSIMD state to SVE. Stale SVE state can be present in + * registers or memory, so we must zero all state that is not shared + * with FPSIMD. + * + * SVE traps cannot be taken from streaming mode, so there cannot be + * any effective streaming mode SVE state. */ - sve_init_regs(); + if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { + unsigned long vq = sve_vq_from_vl(task_get_sve_vl(current)); + sysreg_clear_set_s(SYS_ZCR_EL1, ZCR_ELx_LEN, vq - 1); + sve_flush_live(); + fpsimd_bind_task_to_cpu(); + } else { + fpsimd_to_sve(current); + current->thread.fp_type = FP_STATE_SVE; + fpsimd_flush_task_state(current); + } put_cpu_fpsimd_context(); } @@ -1479,9 +1468,8 @@ void do_sme_acc(unsigned long esr, struct pt_regs *regs) WARN_ON(1); if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { - unsigned long vq_minus_one = - sve_vq_from_vl(task_get_sme_vl(current)) - 1; - sme_set_vq(vq_minus_one); + unsigned long vq = sve_vq_from_vl(task_get_sme_vl(current)); + sysreg_clear_set_s(SYS_SMCR_EL1, SMCR_ELx_LEN, vq - 1); fpsimd_bind_task_to_cpu(); } else { @@ -1656,8 +1644,8 @@ static void fpsimd_flush_thread_vl(enum vec_type type) void fpsimd_flush_thread(void) { - void *sve_state = NULL; - void *sme_state = NULL; + struct arm64_sve_state *sve_state = NULL; + struct arm64_sme_state *sme_state = NULL; if (!system_supports_fpsimd()) return; diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c index 15dedb385b9e..9fafd826002b 100644 --- a/arch/arm64/kernel/irq.c +++ b/arch/arm64/kernel/irq.c @@ -10,6 +10,7 @@ * Copyright (C) 2012 ARM Ltd. */ +#include #include #include #include @@ -32,34 +33,43 @@ DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts); DEFINE_PER_CPU(unsigned long *, irq_stack_ptr); - DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr); #ifdef CONFIG_SHADOW_CALL_STACK DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr); #endif -static void init_irq_scs(void) +static int __init init_irq_scs(void) { int cpu; + void *s; if (!scs_is_enabled()) - return; + return 0; - for_each_possible_cpu(cpu) - per_cpu(irq_shadow_call_stack_ptr, cpu) = - scs_alloc(early_cpu_to_node(cpu)); + for_each_possible_cpu(cpu) { + s = scs_alloc(early_cpu_to_node(cpu)); + if (!s) + return -ENOMEM; + per_cpu(irq_shadow_call_stack_ptr, cpu) = s; + } + + return 0; } -static void __init init_irq_stacks(void) +static int __init init_irq_stacks(void) { int cpu; unsigned long *p; for_each_possible_cpu(cpu) { p = arch_alloc_vmap_stack(IRQ_STACK_SIZE, early_cpu_to_node(cpu)); + if (!p) + return -ENOMEM; per_cpu(irq_stack_ptr, cpu) = p; } + + return 0; } #ifdef CONFIG_SOFTIRQ_ON_OWN_STACK @@ -109,8 +119,9 @@ int __init set_handle_fiq(void (*handle_fiq)(struct pt_regs *)) void __init init_IRQ(void) { - init_irq_stacks(); - init_irq_scs(); + if (init_irq_stacks() || init_irq_scs()) + panic("Failed to allocate IRQ stack resources\n"); + irqchip_init(); if (system_uses_irq_prio_masking()) { diff --git a/arch/arm64/kernel/patching.c b/arch/arm64/kernel/patching.c index 1041bc67a3ee..09f019c6547a 100644 --- a/arch/arm64/kernel/patching.c +++ b/arch/arm64/kernel/patching.c @@ -116,8 +116,7 @@ static void *__text_poke(text_poke_f func, void *addr, void *src, size_t len) while (patched < len) { ptr = addr + patched; - size = min_t(size_t, PAGE_SIZE - offset_in_page(ptr), - len - patched); + size = min(PAGE_SIZE - offset_in_page(ptr), len - patched); waddr = patch_map(ptr, FIX_TEXT_POKE0); func(waddr, src, patched, size); diff --git a/arch/arm64/kernel/pi/map_kernel.c b/arch/arm64/kernel/pi/map_kernel.c index a852264958c3..fb44cbdd2f29 100644 --- a/arch/arm64/kernel/pi/map_kernel.c +++ b/arch/arm64/kernel/pi/map_kernel.c @@ -165,7 +165,7 @@ static void noinline __section(".idmap.text") set_ttbr0_for_lpa2(phys_addr_t ttb static void __init remap_idmap_for_lpa2(void) { /* clear the bits that change meaning once LPA2 is turned on */ - ptdesc_t mask = PTE_SHARED; + ptval_t mask = PTE_SHARED; /* * We have to clear bits [9:8] in all block or page descriptors in the diff --git a/arch/arm64/kernel/pi/map_range.c b/arch/arm64/kernel/pi/map_range.c index de52cd85c691..761b14893f74 100644 --- a/arch/arm64/kernel/pi/map_range.c +++ b/arch/arm64/kernel/pi/map_range.c @@ -31,7 +31,7 @@ void __init map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, u64 va_offset) { u64 cmask = (level == 3) ? CONT_PTE_SIZE - 1 : U64_MAX; - ptdesc_t protval = pgprot_val(prot) & ~PTE_TYPE_MASK; + ptval_t protval = pgprot_val(prot) & ~PTE_TYPE_MASK; int lshift = (3 - level) * PTDESC_TABLE_SHIFT; u64 lmask = (PAGE_SIZE << lshift) - 1; @@ -88,7 +88,7 @@ void __init map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, } } -asmlinkage phys_addr_t __init create_init_idmap(pgd_t *pg_dir, ptdesc_t clrmask) +asmlinkage phys_addr_t __init create_init_idmap(pgd_t *pg_dir, ptval_t clrmask) { phys_addr_t ptep = (phys_addr_t)pg_dir + PAGE_SIZE; /* MMU is off */ pgprot_t text_prot = PAGE_KERNEL_ROX; diff --git a/arch/arm64/kernel/pi/pi.h b/arch/arm64/kernel/pi/pi.h index aec3172d4003..5dfd8484d200 100644 --- a/arch/arm64/kernel/pi/pi.h +++ b/arch/arm64/kernel/pi/pi.h @@ -35,4 +35,4 @@ void map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, asmlinkage void early_map_kernel(u64 boot_status, phys_addr_t fdt); -asmlinkage phys_addr_t create_init_idmap(pgd_t *pgd, ptdesc_t clrmask); +asmlinkage phys_addr_t create_init_idmap(pgd_t *pgd, ptval_t clrmask); diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index b3801f532b10..7bb6553fec08 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -61,7 +62,7 @@ static void update_mitigation_state(enum mitigation_state *oldp, ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "Mitigation: __user pointer sanitization\n"); + return sysfs_emit(buf, "Mitigation: __user pointer sanitization\n"); } /* @@ -126,7 +127,7 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, switch (spectre_v2_state) { case SPECTRE_UNAFFECTED: if (bhb_state == SPECTRE_UNAFFECTED) - return sprintf(buf, "Not affected\n"); + return sysfs_emit(buf, "Not affected\n"); /* * Platforms affected by Spectre-BHB can't report @@ -136,13 +137,13 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, fallthrough; case SPECTRE_MITIGATED: if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled()) - return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n"); + return sysfs_emit(buf, "Vulnerable: Unprivileged eBPF enabled\n"); - return sprintf(buf, "Mitigation: %s%s\n", v2_str, bhb_str); + return sysfs_emit(buf, "Mitigation: %s%s\n", v2_str, bhb_str); case SPECTRE_VULNERABLE: fallthrough; default: - return sprintf(buf, "Vulnerable\n"); + return sysfs_emit(buf, "Vulnerable\n"); } } @@ -438,13 +439,13 @@ ssize_t cpu_show_spec_store_bypass(struct device *dev, { switch (spectre_v4_state) { case SPECTRE_UNAFFECTED: - return sprintf(buf, "Not affected\n"); + return sysfs_emit(buf, "Not affected\n"); case SPECTRE_MITIGATED: - return sprintf(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n"); + return sysfs_emit(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n"); case SPECTRE_VULNERABLE: fallthrough; default: - return sprintf(buf, "Vulnerable\n"); + return sysfs_emit(buf, "Vulnerable\n"); } } diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 1d0e0e6a5b92..d46022f72075 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -745,15 +745,21 @@ void __init smp_init_cpus(void) else acpi_parse_and_init_cpus(); - if (cpu_count > nr_cpu_ids) - pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n", - cpu_count, nr_cpu_ids); - if (!bootcpu_valid) { pr_err("missing boot CPU MPIDR, not enabling secondaries\n"); return; } + /* + * For the nosmp/maxcpus=0 case, do not mark the secondary CPUs + * possible. + */ + if (!setup_max_cpus) + return; + + if (cpu_count > nr_cpu_ids) + pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n", + cpu_count, nr_cpu_ids); /* * We need to set the cpu_logical_map entries before enabling * the cpus so that cpu processor description entries (DT cpu nodes diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 8aaf404980a7..af1d72020976 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -349,9 +349,15 @@ SECTIONS _edata = .; /* start of zero-init region */ - BSS_SECTION(SBSS_ALIGN, 0, 0) + BSS_SECTION(SBSS_ALIGN, 0, PAGE_SIZE) __pi___bss_start = __bss_start; + /* page table BSS starts here - preceding data/BSS is omitted from the linear map */ + .pgtbl : ALIGN(PAGE_SIZE) { + *(.bss..pgtbl) + } + ASSERT(ADDR(.pgtbl) == __bss_stop, ".pgtbl must follow BSS") + . = ALIGN(PAGE_SIZE); __pi_init_pg_dir = .; . += INIT_DIR_SIZE; diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 9453321ef8c6..7def2fad21e8 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2502,10 +2502,10 @@ static void __init teardown_hyp_mode(void) continue; if (free_sve) { - struct cpu_sve_state *sve_state; + struct arm64_sve_state *sve_regs; - sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state; - free_pages((unsigned long) sve_state, pkvm_host_sve_state_order()); + sve_regs = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs; + free_pages((unsigned long) sve_regs, pkvm_host_sve_state_order()); } free_pages(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu], nvhe_percpu_order()); @@ -2630,7 +2630,7 @@ static int init_pkvm_host_sve_state(void) if (!page) return -ENOMEM; - per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = page_address(page); + per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs = page_address(page); } /* @@ -2667,11 +2667,11 @@ static void finalize_init_hyp_mode(void) if (system_supports_sve() && is_protected_kvm_enabled()) { for_each_possible_cpu(cpu) { - struct cpu_sve_state *sve_state; + struct arm64_sve_state *sve_regs; - sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state; - per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = - kern_hyp_va(sve_state); + sve_regs = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs; + per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs = + kern_hyp_va(sve_regs); } } } diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 332c453b87cf..b01d6622b872 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -500,7 +500,7 @@ static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) if (!kvm_arm_vcpu_sve_finalized(vcpu)) return -EPERM; - if (copy_to_user(uptr, vcpu->arch.sve_state + region.koffset, + if (copy_to_user(uptr, (void *)vcpu->arch.sve_state + region.koffset, region.klen) || clear_user(uptr + region.klen, region.upad)) return -EFAULT; @@ -526,7 +526,7 @@ static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) if (!kvm_arm_vcpu_sve_finalized(vcpu)) return -EPERM; - if (copy_from_user(vcpu->arch.sve_state + region.koffset, uptr, + if (copy_from_user((void *)vcpu->arch.sve_state + region.koffset, uptr, region.klen)) return -EFAULT; diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S index 11a10d8f5beb..308100ed25de 100644 --- a/arch/arm64/kvm/hyp/entry.S +++ b/arch/arm64/kvm/hyp/entry.S @@ -8,7 +8,6 @@ #include #include -#include #include #include #include diff --git a/arch/arm64/kvm/hyp/fpsimd.S b/arch/arm64/kvm/hyp/fpsimd.S deleted file mode 100644 index e950875e31ce..000000000000 --- a/arch/arm64/kvm/hyp/fpsimd.S +++ /dev/null @@ -1,33 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2015 - ARM Ltd - * Author: Marc Zyngier - */ - -#include - -#include - - .text - -SYM_FUNC_START(__fpsimd_save_state) - fpsimd_save x0, 1 - ret -SYM_FUNC_END(__fpsimd_save_state) - -SYM_FUNC_START(__fpsimd_restore_state) - fpsimd_restore x0, 1 - ret -SYM_FUNC_END(__fpsimd_restore_state) - -SYM_FUNC_START(__sve_restore_state) - mov x2, #1 - sve_load 0, x1, x2, 3 - ret -SYM_FUNC_END(__sve_restore_state) - -SYM_FUNC_START(__sve_save_state) - mov x2, #1 - sve_save 0, x1, x2, 3 - ret -SYM_FUNC_END(__sve_save_state) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index e9b36a3b27bb..d549d550b6e1 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -471,9 +471,8 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) * vCPU. Start off with the max VL so we can load the SVE state. */ sve_cond_update_zcr_vq(zcr_el2, SYS_ZCR_EL2); - __sve_restore_state(vcpu_sve_pffr(vcpu), - &vcpu->arch.ctxt.fp_regs.fpsr, - true); + sve_load_state(kern_hyp_va(vcpu->arch.sve_state), true); + fpsimd_load_common(&vcpu->arch.ctxt.fp_regs); /* * The effective VL for a VM could differ from the max VL when running a @@ -490,13 +489,13 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) static inline void __hyp_sve_save_host(void) { - struct cpu_sve_state *sve_state = *host_data_ptr(sve_state); + struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); + struct arm64_sve_state *sve_regs = *host_data_ptr(sve_regs); - sve_state->zcr_el1 = read_sysreg_el1(SYS_ZCR); + ctxt_sys_reg(hctxt, ZCR_EL1) = read_sysreg_el1(SYS_ZCR); write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); - __sve_save_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_sve_max_vl), - &sve_state->fpsr, - true); + sve_save_state(sve_regs, true); + fpsimd_save_common(&hctxt->fp_regs); } static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu) @@ -560,6 +559,8 @@ static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu) static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu) { + struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); + /* * Non-protected kvm relies on the host restoring its sve state. * Protected kvm restores the host's sve state as not to reveal that @@ -568,11 +569,11 @@ static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu) if (system_supports_sve()) { __hyp_sve_save_host(); } else { - __fpsimd_save_state(host_data_ptr(host_ctxt.fp_regs)); + fpsimd_save_state(&hctxt->fp_regs); } if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm))) - *host_data_ptr(fpmr) = read_sysreg_s(SYS_FPMR); + ctxt_sys_reg(hctxt, FPMR) = read_sysreg_s(SYS_FPMR); } @@ -628,7 +629,7 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) if (sve_guest) __hyp_sve_restore_guest(vcpu); else - __fpsimd_restore_state(&vcpu->arch.ctxt.fp_regs); + fpsimd_load_state(&vcpu->arch.ctxt.fp_regs); if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm))) write_sysreg_s(__vcpu_sys_reg(vcpu, FPMR), SYS_FPMR); diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index 62cdfbff7562..f57450ebcb49 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile @@ -26,7 +26,7 @@ hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o hyp-main.o hyp-smp.o psci-relay.o early_alloc.o page_alloc.o \ cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ - ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o ../vgic-v5-sr.o + ../hyp-entry.o ../exception.o ../pgtable.o ../vgic-v5-sr.o hyp-obj-y += ../../../kernel/smccc-call.o hyp-obj-$(CONFIG_LIST_HARDENED) += list_debug.o hyp-obj-$(CONFIG_NVHE_EL2_TRACING) += clock.o trace.o events.o diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 06db299c37a8..51dad35000f5 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -35,13 +35,15 @@ static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) * on the VL, so use a consistent (i.e., the maximum) guest VL. */ sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2); - __sve_save_state(vcpu_sve_pffr(vcpu), &vcpu->arch.ctxt.fp_regs.fpsr, true); + sve_save_state(kern_hyp_va(vcpu->arch.sve_state), true); + fpsimd_save_common(&vcpu->arch.ctxt.fp_regs); write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); } static void __hyp_sve_restore_host(void) { - struct cpu_sve_state *sve_state = *host_data_ptr(sve_state); + struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); + struct arm64_sve_state *sve_regs = *host_data_ptr(sve_regs); /* * On saving/restoring host sve state, always use the maximum VL for @@ -53,10 +55,9 @@ static void __hyp_sve_restore_host(void) * need to be revisited. */ write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); - __sve_restore_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_sve_max_vl), - &sve_state->fpsr, - true); - write_sysreg_el1(sve_state->zcr_el1, SYS_ZCR); + sve_load_state(sve_regs, true); + fpsimd_load_common(&hctxt->fp_regs); + write_sysreg_el1(ctxt_sys_reg(hctxt, ZCR_EL1), SYS_ZCR); } static void fpsimd_sve_flush(void) @@ -66,6 +67,7 @@ static void fpsimd_sve_flush(void) static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) { + struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); bool has_fpmr; if (!guest_owns_fp_regs()) @@ -80,7 +82,7 @@ static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) if (vcpu_has_sve(vcpu)) __hyp_sve_save_guest(vcpu); else - __fpsimd_save_state(&vcpu->arch.ctxt.fp_regs); + fpsimd_save_state(&vcpu->arch.ctxt.fp_regs); has_fpmr = kvm_has_fpmr(kern_hyp_va(vcpu->kvm)); if (has_fpmr) @@ -89,10 +91,10 @@ static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) if (system_supports_sve()) __hyp_sve_restore_host(); else - __fpsimd_restore_state(host_data_ptr(host_ctxt.fp_regs)); + fpsimd_load_state(&hctxt->fp_regs); if (has_fpmr) - write_sysreg_s(*host_data_ptr(fpmr), SYS_FPMR); + write_sysreg_s(ctxt_sys_reg(hctxt, FPMR), SYS_FPMR); *host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED; } diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index d461981616d9..75b00c323310 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -82,9 +82,9 @@ static int pkvm_create_host_sve_mappings(void) for (i = 0; i < hyp_nr_cpus; i++) { struct kvm_host_data *host_data = per_cpu_ptr(&kvm_host_data, i); - struct cpu_sve_state *sve_state = host_data->sve_state; + struct arm64_sve_state *sve_regs = host_data->sve_regs; - start = kern_hyp_va(sve_state); + start = kern_hyp_va(sve_regs); end = start + PAGE_ALIGN(pkvm_host_sve_state_size()); ret = pkvm_create_mappings(start, end, PAGE_HYP); if (ret) diff --git a/arch/arm64/kvm/hyp/vhe/Makefile b/arch/arm64/kvm/hyp/vhe/Makefile index 9695328bbd96..d6b3475145c0 100644 --- a/arch/arm64/kvm/hyp/vhe/Makefile +++ b/arch/arm64/kvm/hyp/vhe/Makefile @@ -10,4 +10,4 @@ CFLAGS_switch.o += -Wno-override-init obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ - ../fpsimd.o ../hyp-entry.o ../exception.o ../vgic-v5-sr.o + ../hyp-entry.o ../exception.o ../vgic-v5-sr.o diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 4da9281312eb..6e95514d5a76 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1479,6 +1479,11 @@ static void sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn, if (!kvm_has_mte(kvm)) return; + if (is_zero_pfn(pfn)) { + WARN_ON_ONCE(nr_pages != 1); + return; + } + if (folio_test_hugetlb(folio)) { /* Hugetlb has MTE flags set on head page only */ if (folio_try_hugetlb_mte_tagging(folio)) { diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c index c5c5425791da..f66a0016dd02 100644 --- a/arch/arm64/mm/fixmap.c +++ b/arch/arm64/mm/fixmap.c @@ -31,9 +31,9 @@ static_assert(NR_BM_PMD_TABLES == 1); #define BM_PTE_TABLE_IDX(addr) __BM_TABLE_IDX(addr, PMD_SHIFT) -static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __page_aligned_bss; -static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; -static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; +static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __bss_pgtbl; +static pmd_t bm_pmd[PTRS_PER_PMD] __bss_pgtbl __maybe_unused; +static pud_t bm_pud[PTRS_PER_PUD] __bss_pgtbl __maybe_unused; static inline pte_t *fixmap_pte(unsigned long addr) { diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c index abeb81bf6ebd..3fcad956fdf7 100644 --- a/arch/arm64/mm/kasan_init.c +++ b/arch/arm64/mm/kasan_init.c @@ -214,7 +214,7 @@ asmlinkage void __init kasan_early_init(void) * shadow pud_t[]/p4d_t[], which could end up getting corrupted * when the linear region is mapped. */ - static pte_t tbl[PTRS_PER_PTE] __page_aligned_bss; + static pte_t tbl[PTRS_PER_PTE] __bss_pgtbl; pgd_t *pgdp = pgd_offset_k(KASAN_SHADOW_START); set_pgd(pgdp, __pgd(__pa_symbol(tbl) | PGD_TYPE_TABLE)); diff --git a/arch/arm64/mm/mmap.c b/arch/arm64/mm/mmap.c index 92b2f5097a96..32e0771d6477 100644 --- a/arch/arm64/mm/mmap.c +++ b/arch/arm64/mm/mmap.c @@ -34,7 +34,7 @@ static pgprot_t protection_map[16] __ro_after_init = { [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_SHARED_EXEC }; -static ptdesc_t gcs_page_prot __ro_after_init = _PAGE_GCS_RO; +static ptval_t gcs_page_prot __ro_after_init = _PAGE_GCS_RO; /* * You really shouldn't be using read() or write() on /dev/mem. This might go @@ -87,7 +87,7 @@ arch_initcall(adjust_protection_map); pgprot_t vm_get_page_prot(vm_flags_t vm_flags) { - ptdesc_t prot; + ptval_t prot; /* Short circuit GCS to avoid bloating the table. */ if (system_supports_gcs() && (vm_flags & VM_SHADOW_STACK)) { diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 8242f93f05e4..9f354971b7e4 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -134,10 +134,6 @@ bool pgattr_change_is_safe(pteval_t old, pteval_t new) if (pte_pfn(__pte(old)) != pte_pfn(__pte(new))) return false; - /* live contiguous mappings may not be manipulated at all */ - if ((old | new) & PTE_CONT) - return false; - /* Transitioning from Non-Global to Global is unsafe */ if (old & ~new & PTE_NG) return false; @@ -187,6 +183,17 @@ static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end, } while (ptep++, addr += PAGE_SIZE, addr != end); } +static bool pte_range_has_valid_noncont(pte_t *ptep) +{ + for (int i = 0; i < CONT_PTES; i++) { + pte_t pte = __ptep_get(&ptep[i]); + + if (pte_valid(pte) && !pte_cont(pte)) + return true; + } + return false; +} + static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, @@ -224,7 +231,8 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, /* use a contiguous mapping if the range is suitably aligned */ if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) && - (flags & NO_CONT_MAPPINGS) == 0) + (flags & NO_CONT_MAPPINGS) == 0 && + !pte_range_has_valid_noncont(ptep)) __prot = __pgprot(pgprot_val(prot) | PTE_CONT); init_pte(ptep, addr, next, phys, __prot); @@ -256,8 +264,9 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, /* try section mapping first */ if (((addr | next | phys) & ~PMD_MASK) == 0 && - (flags & NO_BLOCK_MAPPINGS) == 0) { - pmd_set_huge(pmdp, phys, prot); + (flags & NO_BLOCK_MAPPINGS) == 0 && + !pmd_table(old_pmd)) { + WARN_ON(!pmd_set_huge(pmdp, phys, prot)); /* * After the PMD entry has been populated once, we @@ -273,8 +282,8 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, if (ret) return ret; - BUG_ON(pmd_val(old_pmd) != 0 && - pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); + VM_WARN_ON_ONCE(pmd_val(old_pmd) != 0 && + pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); } phys += next - addr; } while (pmdp++, addr = next, addr != end); @@ -282,6 +291,17 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, return 0; } +static bool pmd_range_has_valid_noncont(pmd_t *pmdp) +{ + for (int i = 0; i < CONT_PMDS; i++) { + pte_t pte = pmd_pte(READ_ONCE(pmdp[i])); + + if (pte_valid(pte) && !pte_cont(pte)) + return true; + } + return false; +} + static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, @@ -323,7 +343,8 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, /* use a contiguous mapping if the range is suitably aligned */ if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) && - (flags & NO_CONT_MAPPINGS) == 0) + (flags & NO_CONT_MAPPINGS) == 0 && + !pmd_range_has_valid_noncont(pmdp)) __prot = __pgprot(pgprot_val(prot) | PTE_CONT); ret = init_pmd(pmdp, addr, next, phys, __prot, pgtable_alloc, flags); @@ -379,8 +400,9 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, */ if (pud_sect_supported() && ((addr | next | phys) & ~PUD_MASK) == 0 && - (flags & NO_BLOCK_MAPPINGS) == 0) { - pud_set_huge(pudp, phys, prot); + (flags & NO_BLOCK_MAPPINGS) == 0 && + !pud_table(old_pud)) { + WARN_ON(!pud_set_huge(pudp, phys, prot)); /* * After the PUD entry has been populated once, we @@ -394,8 +416,8 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, if (ret) goto out; - BUG_ON(pud_val(old_pud) != 0 && - pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); + VM_WARN_ON_ONCE(pud_val(old_pud) != 0 && + pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); } phys += next - addr; } while (pudp++, addr = next, addr != end); @@ -445,8 +467,8 @@ static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end, if (ret) goto out; - BUG_ON(p4d_val(old_p4d) != 0 && - p4d_val(old_p4d) != READ_ONCE(p4d_val(*p4dp))); + VM_WARN_ON_ONCE(p4d_val(old_p4d) != 0 && + p4d_val(old_p4d) != READ_ONCE(p4d_val(*p4dp))); phys += next - addr; } while (p4dp++, addr = next, addr != end); @@ -1000,8 +1022,7 @@ void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, &phys, virt); return; } - early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, - NO_CONT_MAPPINGS); + early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 0); } void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, @@ -1028,18 +1049,17 @@ static void update_mapping_prot(phys_addr_t phys, unsigned long virt, return; } - early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, - NO_CONT_MAPPINGS); + early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 0); /* flush the TLBs after updating live kernel mappings */ flush_tlb_kernel_range(virt, virt + size); } -static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start, - phys_addr_t end, pgprot_t prot, int flags) +static void __init __map_memblock(phys_addr_t start, phys_addr_t end, + pgprot_t prot, int flags) { - early_create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, - prot, early_pgtable_alloc, flags); + early_create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start), + end - start, prot, early_pgtable_alloc, flags); } void __init mark_linear_text_alias_ro(void) @@ -1067,36 +1087,24 @@ static int __init parse_kfence_early_init(char *arg) } early_param("kfence.sample_interval", parse_kfence_early_init); -static phys_addr_t __init arm64_kfence_alloc_pool(void) +static void __init arm64_kfence_map_pool(void) { phys_addr_t kfence_pool; if (!kfence_early_init) - return 0; + return; kfence_pool = memblock_phys_alloc(KFENCE_POOL_SIZE, PAGE_SIZE); if (!kfence_pool) { pr_err("failed to allocate kfence pool\n"); kfence_early_init = false; - return 0; + return; } - /* Temporarily mark as NOMAP. */ - memblock_mark_nomap(kfence_pool, KFENCE_POOL_SIZE); - - return kfence_pool; -} - -static void __init arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) -{ - if (!kfence_pool) - return; - /* KFENCE pool needs page-level mapping. */ - __map_memblock(pgdp, kfence_pool, kfence_pool + KFENCE_POOL_SIZE, + __map_memblock(kfence_pool, kfence_pool + KFENCE_POOL_SIZE, pgprot_tagged(PAGE_KERNEL), - NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS); - memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE); + NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS | NO_EXEC_MAPPINGS); __kfence_pool = phys_to_virt(kfence_pool); } @@ -1128,18 +1136,18 @@ bool arch_kfence_init_pool(void) } #else /* CONFIG_KFENCE */ -static inline phys_addr_t arm64_kfence_alloc_pool(void) { return 0; } -static inline void arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) { } +static inline void arm64_kfence_map_pool(void) { } #endif /* CONFIG_KFENCE */ -static void __init map_mem(pgd_t *pgdp) +static void __init map_mem(void) { static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN); phys_addr_t kernel_start = __pa_symbol(_text); - phys_addr_t kernel_end = __pa_symbol(__init_begin); + phys_addr_t init_begin = __pa_symbol(__init_begin); + phys_addr_t init_end = __pa_symbol(__init_end); + phys_addr_t kernel_end = __pa_symbol(__bss_stop); phys_addr_t start, end; - phys_addr_t early_kfence_pool; int flags = NO_EXEC_MAPPINGS; u64 i; @@ -1156,7 +1164,7 @@ static void __init map_mem(pgd_t *pgdp) BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end) && pgd_index(_PAGE_OFFSET(VA_BITS_MIN)) != PTRS_PER_PGD - 1); - early_kfence_pool = arm64_kfence_alloc_pool(); + arm64_kfence_map_pool(); linear_map_requires_bbml2 = !force_pte_mapping() && can_set_direct_map(); @@ -1164,40 +1172,37 @@ static void __init map_mem(pgd_t *pgdp) flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; /* - * Take care not to create a writable alias for the - * read-only text and rodata sections of the kernel image. - * So temporarily mark them as NOMAP to skip mappings in - * the following for-loop + * Map the linear alias of the [_text, __init_begin) interval first + * so that its write permissions can be removed later without the need + * to split any block mappings created by the loop below. + * + * Write permissions are needed for alternatives patching, and will be + * removed later by mark_linear_text_alias_ro() above. This makes the + * contents of the region accessible to subsystems such as hibernate, + * but protects it from inadvertent modification or execution. */ - memblock_mark_nomap(kernel_start, kernel_end - kernel_start); + __map_memblock(kernel_start, init_begin, pgprot_tagged(PAGE_KERNEL), + flags); + + /* Map the kernel data/bss so it can be remapped later */ + __map_memblock(init_end, kernel_end, pgprot_tagged(PAGE_KERNEL), + flags); /* map all the memory banks */ for_each_mem_range(i, &start, &end) { - if (start >= end) - break; /* * The linear map must allow allocation tags reading/writing * if MTE is present. Otherwise, it has the same attributes as * PAGE_KERNEL. */ - __map_memblock(pgdp, start, end, pgprot_tagged(PAGE_KERNEL), + __map_memblock(start, end, pgprot_tagged(PAGE_KERNEL), flags); } - /* - * Map the linear alias of the [_text, __init_begin) interval - * as non-executable now, and remove the write permission in - * mark_linear_text_alias_ro() below (which will be called after - * alternative patching has completed). This makes the contents - * of the region accessible to subsystems such as hibernate, - * but protects it from inadvertent modification or execution. - * Note that contiguous mappings cannot be remapped in this way, - * so we should avoid them here. - */ - __map_memblock(pgdp, kernel_start, kernel_end, - PAGE_KERNEL, NO_CONT_MAPPINGS); - memblock_clear_nomap(kernel_start, kernel_end - kernel_start); - arm64_kfence_map_pool(early_kfence_pool, pgdp); + /* Map the kernel data/bss read-only in the linear map */ + __map_memblock(init_end, kernel_end, PAGE_KERNEL_RO, flags); + flush_tlb_kernel_range((unsigned long)lm_alias(__init_end), + (unsigned long)lm_alias(__bss_stop)); } void mark_rodata_ro(void) @@ -1419,7 +1424,7 @@ static void __init create_idmap(void) void __init paging_init(void) { - map_mem(swapper_pg_dir); + map_mem(); memblock_allow_resize(); diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index ce035e1b4eaf..bbe98ac9ad8c 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -21,7 +21,7 @@ struct page_change_data { pgprot_t clear_mask; }; -static ptdesc_t set_pageattr_masks(ptdesc_t val, struct mm_walk *walk) +static ptval_t set_pageattr_masks(ptval_t val, struct mm_walk *walk) { struct page_change_data *masks = walk->private; diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c index ab9899ca1e5f..1c20144700d7 100644 --- a/arch/arm64/mm/ptdump.c +++ b/arch/arm64/mm/ptdump.c @@ -194,7 +194,7 @@ void note_page(struct ptdump_state *pt_st, unsigned long addr, int level, struct ptdump_pg_state *st = container_of(pt_st, struct ptdump_pg_state, ptdump); struct ptdump_pg_level *pg_level = st->pg_level; static const char units[] = "KMGTPE"; - ptdesc_t prot = 0; + ptval_t prot = 0; /* check if the current level has been folded dynamically */ if (st->mm && ((level == 1 && mm_p4d_folded(st->mm)) || diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 811c2479e82d..9b85a84f6fd4 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -120,7 +120,7 @@ WORKAROUND_CAVIUM_TX2_219_PRFM WORKAROUND_CAVIUM_TX2_219_TVM WORKAROUND_CLEAN_CACHE WORKAROUND_DEVICE_LOAD_ACQUIRE -WORKAROUND_NVIDIA_CARMEL_CNP +WORKAROUND_DISABLE_CNP WORKAROUND_PMUV3_IMPDEF_TRAPS WORKAROUND_QCOM_FALKOR_E1003 WORKAROUND_QCOM_ORYON_CNTVOFF diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 6c3ff14e561e..bc1788b1662b 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -3790,6 +3790,51 @@ Field 1 ZA Field 0 SM EndSysreg +Sysreg FPCR 3 3 4 4 0 +Res0 63:27 +Field 26 AHP +Field 25 DN +Field 24 FZ +Enum 23:22 RMode + 0b00 RN + 0b01 RP + 0b10 RM + 0b11 RZ +EndEnum +Field 21:20 Stride +Field 19 FZ16 +Field 18:16 Len +Field 15 IDE +Res0 14 +Field 13 EBF +Field 12 IXE +Field 11 UFE +Field 10 OFE +Field 9 DZE +Field 8 IOE +Res0 7:3 +Field 2 NEP +Field 1 AH +Field 0 FIZ +EndSysreg + +Sysreg FPSR 3 3 4 4 1 +Res0 63:32 +Field 31 N +Field 30 Z +Field 29 C +Field 28 V +Field 27 QC +Res0 26:8 +Field 7 IDC +Res0 6:5 +Field 4 IXC +Field 3 UFC +Field 2 OFC +Field 1 DZC +Field 0 IOC +EndSysreg + Sysreg FPMR 3 3 4 4 2 Res0 63:38 Field 37:32 LSCALE2 @@ -4597,6 +4642,35 @@ Sysreg GCSPR_EL2 3 4 2 5 1 Fields GCSPR_ELx EndSysreg +Sysreg HDBSSBR_EL2 3 4 2 3 2 +Res0 63:56 +Field 55:12 BADDR +Res0 11:4 +Enum 3:0 SZ + 0b0000 4KB + 0b0001 8KB + 0b0010 16KB + 0b0011 32KB + 0b0100 64KB + 0b0101 128KB + 0b0110 256KB + 0b0111 512KB + 0b1000 1MB + 0b1001 2MB +EndEnum +EndSysreg + +Sysreg HDBSSPROD_EL2 3 4 2 3 3 +Res0 63:32 +Enum 31:26 FSC + 0b000000 OK + 0b010000 ExternalAbort + 0b101000 GPF +EndEnum +Res0 25:19 +Field 18:0 INDEX +EndSysreg + Sysreg DACR32_EL2 3 4 3 0 0 Res0 63:32 Field 31:30 D15 diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c index f84e0337cc02..44ff9f684bef 100644 --- a/arch/powerpc/lib/code-patching.c +++ b/arch/powerpc/lib/code-patching.c @@ -60,9 +60,6 @@ struct patch_context { static DEFINE_PER_CPU(struct patch_context, cpu_patching_context); -static int map_patch_area(void *addr, unsigned long text_poke_addr); -static void unmap_patch_area(unsigned long addr); - static bool mm_patch_enabled(void) { return IS_ENABLED(CONFIG_SMP) && radix_enabled(); @@ -117,11 +114,11 @@ static int text_area_cpu_up(unsigned int cpu) // Map/unmap the area to ensure all page tables are pre-allocated addr = (unsigned long)area->addr; - err = map_patch_area(empty_zero_page, addr); + err = map_kernel_page(addr, __pa_symbol(empty_zero_page), PAGE_KERNEL_RO); if (err) return err; - unmap_patch_area(addr); + unmap_kernel_page(addr); this_cpu_write(cpu_patching_context.area, area); this_cpu_write(cpu_patching_context.addr, addr); @@ -233,51 +230,6 @@ static unsigned long get_patch_pfn(void *addr) return __pa_symbol(addr) >> PAGE_SHIFT; } -/* - * This can be called for kernel text or a module. - */ -static int map_patch_area(void *addr, unsigned long text_poke_addr) -{ - unsigned long pfn = get_patch_pfn(addr); - - return map_kernel_page(text_poke_addr, (pfn << PAGE_SHIFT), PAGE_KERNEL); -} - -static void unmap_patch_area(unsigned long addr) -{ - pte_t *ptep; - pmd_t *pmdp; - pud_t *pudp; - p4d_t *p4dp; - pgd_t *pgdp; - - pgdp = pgd_offset_k(addr); - if (WARN_ON(pgd_none(*pgdp))) - return; - - p4dp = p4d_offset(pgdp, addr); - if (WARN_ON(p4d_none(*p4dp))) - return; - - pudp = pud_offset(p4dp, addr); - if (WARN_ON(pud_none(*pudp))) - return; - - pmdp = pmd_offset(pudp, addr); - if (WARN_ON(pmd_none(*pmdp))) - return; - - ptep = pte_offset_kernel(pmdp, addr); - if (WARN_ON(pte_none(*ptep))) - return; - - /* - * In hash, pte_clear flushes the tlb, in radix, we have to - */ - pte_clear(&init_mm, addr, ptep); - flush_tlb_kernel_range(addr, addr + PAGE_SIZE); -} - static int __do_patch_mem_mm(void *addr, unsigned long val, bool is_dword) { int err; diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index 4e40d5e96be9..110308bdef01 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -331,9 +331,6 @@ void __init mem_init(void) /* Set this up early, so we can take care of the zero page */ cpu_cache_init(); - /* clear the zero-page */ - __flush_wback_region(empty_zero_page, PAGE_SIZE); - vsyscall_init(); pr_info("virtual kernel memory layout:\n" diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig index ab90932fc2d0..245e7bb763b9 100644 --- a/drivers/perf/Kconfig +++ b/drivers/perf/Kconfig @@ -188,7 +188,7 @@ config FUJITSU_UNCORE_PMU monitoring Uncore events. config QCOM_L2_PMU - bool "Qualcomm Technologies L2-cache PMU" + bool "Qualcomm L2-cache PMU" depends on ARCH_QCOM && ARM64 && ACPI select QCOM_KRYO_L2_ACCESSORS help @@ -198,7 +198,7 @@ config QCOM_L2_PMU monitoring L2 cache events. config QCOM_L3_PMU - bool "Qualcomm Technologies L3-cache PMU" + bool "Qualcomm L3-cache PMU" depends on ARCH_QCOM && ARM64 && ACPI select QCOM_IRQ_COMBINER help diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index f5305c8fdca4..6e5cc4086a9e 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -197,13 +197,14 @@ enum cmn_model { CMN600 = 1, CMN650 = 2, - CMN700 = 4, - CI700 = 8, + CI700 = 4, + CMN700 = 8, CMNS3 = 16, /* ...and then we can use bitmap tricks for commonality */ CMN_ANY = -1, NOT_CMN600 = -2, - CMN_650ON = CMN650 | CMN700 | CMNS3, + CMN_700ON = ~(CMN700 - 1), + CMN_650ON = CMN_700ON | CMN650, }; /* Actual part numbers and revision IDs defined by the hardware */ @@ -919,14 +920,14 @@ static struct attribute *arm_cmn_event_attrs[] = { CMN_EVENT_DVM(NOT_CMN600, txsnp_stall, 0x0a), CMN_EVENT_DVM(NOT_CMN600, trkfull, 0x0b), CMN_EVENT_DVM_OCC(NOT_CMN600, trk_occupancy, 0x0c), - CMN_EVENT_DVM_OCC(CMN700, trk_occupancy_cxha, 0x0d), - CMN_EVENT_DVM_OCC(CMN700, trk_occupancy_pdn, 0x0e), - CMN_EVENT_DVM(CMN700, trk_alloc, 0x0f), - CMN_EVENT_DVM(CMN700, trk_cxha_alloc, 0x10), - CMN_EVENT_DVM(CMN700, trk_pdn_alloc, 0x11), - CMN_EVENT_DVM(CMN700, txsnp_stall_limit, 0x12), - CMN_EVENT_DVM(CMN700, rxsnp_stall_starv, 0x13), - CMN_EVENT_DVM(CMN700, txsnp_sync_stall_op, 0x14), + CMN_EVENT_DVM_OCC(CMN_700ON, trk_occupancy_cxha, 0x0d), + CMN_EVENT_DVM_OCC(CMN_700ON, trk_occupancy_pdn, 0x0e), + CMN_EVENT_DVM(CMN_700ON, trk_alloc, 0x0f), + CMN_EVENT_DVM(CMN_700ON, trk_cxha_alloc, 0x10), + CMN_EVENT_DVM(CMN_700ON, trk_pdn_alloc, 0x11), + CMN_EVENT_DVM(CMN_700ON, txsnp_stall_limit, 0x12), + CMN_EVENT_DVM(CMN_700ON, rxsnp_stall_starv, 0x13), + CMN_EVENT_DVM(CMN_700ON, txsnp_sync_stall_op, 0x14), CMN_EVENT_HNF(CMN_ANY, cache_miss, 0x01), CMN_EVENT_HNF(CMN_ANY, slc_sf_cache_access, 0x02), diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 988fc291241d..b69f99488111 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -224,6 +224,24 @@ static inline void _mpam_write_monsel_reg(struct mpam_msc *msc, u16 reg, u32 val #define mpam_write_monsel_reg(msc, reg, val) _mpam_write_monsel_reg(msc, MSMON_##reg, val) +static bool mpam_msc_check_aidr(struct mpam_msc *msc) +{ + u32 aidr = __mpam_read_reg(msc, MPAMF_AIDR); + u32 major = FIELD_GET(MPAMF_AIDR_ARCH_MAJOR_REV, aidr); + u32 minor = FIELD_GET(MPAMF_AIDR_ARCH_MINOR_REV, aidr); + + /* + * v0.0 and >v2.x aren't supported, but anything else should be backward + * compatible to v0.1 or v1.0. + */ + if (!major && !minor) + return false; + if (major > 1) + return false; + + return true; +} + static u64 mpam_msc_read_idr(struct mpam_msc *msc) { u64 idr_high = 0, idr_low; @@ -945,9 +963,8 @@ static int mpam_msc_hw_probe(struct mpam_msc *msc) lockdep_assert_held(&msc->probe_lock); - idr = __mpam_read_reg(msc, MPAMF_AIDR); - if ((idr & MPAMF_AIDR_ARCH_MAJOR_REV) != MPAM_ARCHITECTURE_V1) { - dev_err_once(dev, "MSC does not match MPAM architecture v1.x\n"); + if (!mpam_msc_check_aidr(msc)) { + dev_err_once(dev, "MSC does not match architecture v1.x\n"); return -EIO; } diff --git a/include/linux/linkage.h b/include/linux/linkage.h index b11660b706c5..53fe1f48fd28 100644 --- a/include/linux/linkage.h +++ b/include/linux/linkage.h @@ -39,6 +39,10 @@ #define __page_aligned_data __section(".data..page_aligned") __aligned(PAGE_SIZE) #define __page_aligned_bss __section(".bss..page_aligned") __aligned(PAGE_SIZE) +#ifndef __bss_pgtbl +#define __bss_pgtbl __page_aligned_bss +#endif + /* * For assembly routines. * diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 07483ed9b3ce..eca14547f9c1 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -2027,7 +2027,7 @@ static inline unsigned long zero_pfn(unsigned long addr) return zero_page_pfn; } -extern uint8_t empty_zero_page[PAGE_SIZE]; +extern const uint8_t empty_zero_page[PAGE_SIZE]; extern struct page *__zero_page; static inline struct page *_zero_page(unsigned long addr) diff --git a/mm/kasan/init.c b/mm/kasan/init.c index 9c880f607c6a..66a883887987 100644 --- a/mm/kasan/init.c +++ b/mm/kasan/init.c @@ -26,10 +26,10 @@ * - Latter it reused it as zero shadow to cover large ranges of memory * that allowed to access, but not handled by kasan (vmalloc/vmemmap ...). */ -unsigned char kasan_early_shadow_page[PAGE_SIZE] __page_aligned_bss; +unsigned char kasan_early_shadow_page[PAGE_SIZE] __bss_pgtbl; #if CONFIG_PGTABLE_LEVELS > 4 -p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D] __page_aligned_bss; +p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D] __bss_pgtbl; static inline bool kasan_p4d_table(pgd_t pgd) { return pgd_page(pgd) == virt_to_page(lm_alias(kasan_early_shadow_p4d)); @@ -41,7 +41,7 @@ static inline bool kasan_p4d_table(pgd_t pgd) } #endif #if CONFIG_PGTABLE_LEVELS > 3 -pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __page_aligned_bss; +pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __bss_pgtbl; static inline bool kasan_pud_table(p4d_t p4d) { return p4d_page(p4d) == virt_to_page(lm_alias(kasan_early_shadow_pud)); @@ -53,7 +53,7 @@ static inline bool kasan_pud_table(p4d_t p4d) } #endif #if CONFIG_PGTABLE_LEVELS > 2 -pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __page_aligned_bss; +pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __bss_pgtbl; static inline bool kasan_pmd_table(pud_t pud) { return pud_page(pud) == virt_to_page(lm_alias(kasan_early_shadow_pmd)); @@ -65,7 +65,7 @@ static inline bool kasan_pmd_table(pud_t pud) } #endif pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS] - __page_aligned_bss; + __bss_pgtbl; static inline bool kasan_pte_table(pmd_t pmd) { diff --git a/mm/mm_init.c b/mm/mm_init.c index f9f8e1af921c..46cf001238c5 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -57,7 +57,7 @@ unsigned long zero_page_pfn __ro_after_init; EXPORT_SYMBOL(zero_page_pfn); #ifndef __HAVE_COLOR_ZERO_PAGE -uint8_t empty_zero_page[PAGE_SIZE] __page_aligned_bss; +const uint8_t empty_zero_page[PAGE_SIZE] __aligned(PAGE_SIZE); EXPORT_SYMBOL(empty_zero_page); struct page *__zero_page __ro_after_init; diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c index e22703d6b97c..19fca95f7c22 100644 --- a/tools/testing/selftests/arm64/abi/hwcap.c +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -108,6 +108,24 @@ static void f8mm8_sigill(void) asm volatile(".inst 0x6e80ec00"); } +static void f16f32dot_sigill(void) +{ + /* FDOT V0.2S, V0.4H, V0.2H[0] */ + asm volatile(".inst 0xf409000"); +} + +static void f16f32mm_sigill(void) +{ + /* FMMLA V0.4S, V0.8H, V0.8H */ + asm volatile(".inst 0x4e40ec00"); +} + +static void f16mm_sigill(void) +{ + /* FMMLA V0.8H, V0.8H, V0.8H */ + asm volatile(".inst 0x4ec0ec00"); +} + static void faminmax_sigill(void) { /* FAMIN V0.4H, V0.4H, V0.4H */ @@ -191,6 +209,12 @@ static void lut_sigill(void) asm volatile(".inst 0x4e801000"); } +static void sve_lut6_sigill(void) +{ + /* LUTI6 Z0.H, { Z0.H, Z1.H }, Z0[0] */ + asm volatile(".inst 0x4560ac00"); +} + static void mops_sigill(void) { char dst[1], src[1]; @@ -282,6 +306,18 @@ static void sme2p2_sigill(void) asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); } +static void sme2p3_sigill(void) +{ + /* SMSTART SM */ + asm volatile("msr S0_3_C4_C3_3, xzr" : : : ); + + /* ADDQP Z0.B, Z0.B, Z0.B */ + asm volatile(".inst 0x4207800" : : : "z0"); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + static void sme_aes_sigill(void) { /* SMSTART SM */ @@ -378,6 +414,18 @@ static void smef8f32_sigill(void) asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); } +static void smelut6_sigill(void) +{ + /* SMSTART */ + asm volatile("msr S0_3_C4_C7_3, xzr" : : : ); + + /* LUTI6 { Z0.B-Z3.B }, ZT0, { Z0-Z2 } */ + asm volatile(".inst 0xc08a0000" : : : ); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + static void smelutv2_sigill(void) { /* SMSTART */ @@ -486,6 +534,12 @@ static void sve2p2_sigill(void) asm volatile(".inst 0x4cea000" : : : "z0"); } +static void sve2p3_sigill(void) +{ + /* ADDQP Z0.B, Z0.B, Z0.B */ + asm volatile(".inst 0x4207800" : : : "z0"); +} + static void sveaes_sigill(void) { /* AESD z0.b, z0.b, z0.b */ @@ -504,6 +558,12 @@ static void sveb16b16_sigill(void) asm volatile(".inst 0x65000000" : : : ); } +static void sveb16mm_sigill(void) +{ + /* BFMMLA Z0.H, Z0.H, Z0.H */ + asm volatile(".inst 0x64e0e000" : : : ); +} + static void svebfscale_sigill(void) { /* BFSCALE Z0.H, P0/M, Z0.H, Z0.H */ @@ -729,6 +789,27 @@ static const struct hwcap_data { .cpuinfo = "f8mm4", .sigill_fn = f8mm4_sigill, }, + { + .name = "F16MM", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_F16MM, + .cpuinfo = "f16mm", + .sigill_fn = f16mm_sigill, + }, + { + .name = "F16F32DOT", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_F16F32DOT, + .cpuinfo = "f16f32dot", + .sigill_fn = f16f32dot_sigill, + }, + { + .name = "F16F32MM", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_F16F32MM, + .cpuinfo = "f16f32mm", + .sigill_fn = f16f32mm_sigill, + }, { .name = "FAMINMAX", .at_hwcap = AT_HWCAP2, @@ -918,6 +999,13 @@ static const struct hwcap_data { .cpuinfo = "sme2p2", .sigill_fn = sme2p2_sigill, }, + { + .name = "SME 2.3", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_SME2P3, + .cpuinfo = "sme2p3", + .sigill_fn = sme2p3_sigill, + }, { .name = "SME AES", .at_hwcap = AT_HWCAP, @@ -967,6 +1055,13 @@ static const struct hwcap_data { .cpuinfo = "smef8f32", .sigill_fn = smef8f32_sigill, }, + { + .name = "SME LUT6", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_SME_LUT6, + .cpuinfo = "smelut6", + .sigill_fn = smelut6_sigill, + }, { .name = "SME LUTV2", .at_hwcap = AT_HWCAP2, @@ -1052,6 +1147,13 @@ static const struct hwcap_data { .cpuinfo = "sve2p2", .sigill_fn = sve2p2_sigill, }, + { + .name = "SVE 2.3", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_SVE2P3, + .cpuinfo = "sve2p3", + .sigill_fn = sve2p3_sigill, + }, { .name = "SVE AES", .at_hwcap = AT_HWCAP2, @@ -1066,6 +1168,13 @@ static const struct hwcap_data { .cpuinfo = "sveaes2", .sigill_fn = sveaes2_sigill, }, + { + .name = "SVE B16MM", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_SVE_B16MM, + .cpuinfo = "sveb16mm", + .sigill_fn = sveb16mm_sigill, + }, { .name = "SVE BFSCALE", .at_hwcap = AT_HWCAP, @@ -1087,6 +1196,13 @@ static const struct hwcap_data { .cpuinfo = "svef16mm", .sigill_fn = svef16mm_sigill, }, + { + .name = "SVE_LUT6", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_SVE_LUT6, + .cpuinfo = "svelut6", + .sigill_fn = sve_lut6_sigill, + }, { .name = "SVE2 B16B16", .at_hwcap = AT_HWCAP2, diff --git a/tools/testing/selftests/arm64/signal/test_signals.h b/tools/testing/selftests/arm64/signal/test_signals.h index ee75a2c25ce7..c7c343494cb8 100644 --- a/tools/testing/selftests/arm64/signal/test_signals.h +++ b/tools/testing/selftests/arm64/signal/test_signals.h @@ -36,6 +36,7 @@ enum { FSME_FA64_BIT, FSME2_BIT, FGCS_BIT, + FPOE_BIT, FMAX_END }; @@ -45,6 +46,7 @@ enum { #define FEAT_SME_FA64 (1UL << FSME_FA64_BIT) #define FEAT_SME2 (1UL << FSME2_BIT) #define FEAT_GCS (1UL << FGCS_BIT) +#define FEAT_POE (1UL << FPOE_BIT) /* * A descriptor used to describe and configure a test case. diff --git a/tools/testing/selftests/arm64/signal/test_signals_utils.c b/tools/testing/selftests/arm64/signal/test_signals_utils.c index 5d3621921cfe..4b12dbd7669d 100644 --- a/tools/testing/selftests/arm64/signal/test_signals_utils.c +++ b/tools/testing/selftests/arm64/signal/test_signals_utils.c @@ -31,6 +31,7 @@ static char const *const feats_names[FMAX_END] = { " FA64 ", " SME2 ", " GCS ", + " POE ", }; #define MAX_FEATS_SZ 128 @@ -341,6 +342,8 @@ int test_init(struct tdescr *td) td->feats_supported |= FEAT_SME2; if (getauxval(AT_HWCAP) & HWCAP_GCS) td->feats_supported |= FEAT_GCS; + if (getauxval(AT_HWCAP2) & HWCAP2_POE) + td->feats_supported |= FEAT_POE; if (feats_ok(td)) { if (td->feats_required & td->feats_supported) fprintf(stderr, diff --git a/tools/testing/selftests/arm64/signal/test_signals_utils.h b/tools/testing/selftests/arm64/signal/test_signals_utils.h index 36fc12b3cd60..2c7b8c64a35a 100644 --- a/tools/testing/selftests/arm64/signal/test_signals_utils.h +++ b/tools/testing/selftests/arm64/signal/test_signals_utils.h @@ -57,6 +57,22 @@ static inline __attribute__((always_inline)) uint64_t get_gcspr_el0(void) return val; } +#define SYS_POR_EL0 "S3_3_C10_C2_4" + +static inline uint64_t get_por_el0(void) +{ + uint64_t val; + + asm volatile("mrs %0, " SYS_POR_EL0 "\n" : "=r"(val)); + + return val; +} + +static inline void set_por_el0(uint64_t val) +{ + asm volatile("msr " SYS_POR_EL0 ", %0\n" :: "r"(val)); +} + static inline bool feats_ok(struct tdescr *td) { if (td->feats_incompatible & td->feats_supported) diff --git a/tools/testing/selftests/arm64/signal/testcases/poe_missing_poe_context.c b/tools/testing/selftests/arm64/signal/testcases/poe_missing_poe_context.c new file mode 100644 index 000000000000..3f22d8cf6106 --- /dev/null +++ b/tools/testing/selftests/arm64/signal/testcases/poe_missing_poe_context.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026 Arm Ltd + * + * Verify that the POR_EL0 register is left untouched on sigreturn if the + * POE frame record is missing. + */ + +#include + +#include "test_signals_utils.h" +#include "testcases.h" + +#define POR_EL0_INIT 0x07ul +#define POR_EL0_CUSTOM 0x77ul + +static bool failed_check; + +static bool modify_por_el0(struct tdescr *td) +{ + set_por_el0(POR_EL0_CUSTOM); + + return true; +} + +static int signal_remove_poe_context(struct tdescr *td, siginfo_t *si, + ucontext_t *uc) +{ + struct _aarch64_ctx *ctx = GET_UC_RESV_HEAD(uc); + size_t resv_size = GET_UCP_RESV_SIZE(uc); + struct _aarch64_ctx *poe_ctx_head; + + poe_ctx_head = get_header(ctx, POE_MAGIC, resv_size, NULL); + if (!poe_ctx_head) { + fprintf(stderr, "Missing poe_context record\n"); + failed_check = true; + return 0; + } + + /* + * Actually removing the record would require moving down the next + * records. An easier option is to turn it into an ESR record, which is + * ignored by sigreturn(). + */ + poe_ctx_head->magic = ESR_MAGIC; + + return 0; +} + +static void check_por_el0_preserved(struct tdescr *td) +{ + uint64_t por_el0 = get_por_el0(); + + if (por_el0 == POR_EL0_INIT) { + fprintf(stderr, "POR_EL0 preserved\n"); + } else { + fprintf(stderr, "POR_EL0 unexpectedly set to %lx\n", por_el0); + failed_check = true; + } + + td->pass = !failed_check; +} + +struct tdescr tde = { + .name = "POR_EL0 missing poe_context", + .descr = "Remove poe_context record and check POR_EL0 is preserved", + .feats_required = FEAT_POE, + .timeout = 3, + .sig_trig = SIGUSR1, + .init = modify_por_el0, + .run = signal_remove_poe_context, + .check_result = check_por_el0_preserved, +}; diff --git a/tools/testing/selftests/arm64/signal/testcases/poe_restore.c b/tools/testing/selftests/arm64/signal/testcases/poe_restore.c new file mode 100644 index 000000000000..9f9a61a4214d --- /dev/null +++ b/tools/testing/selftests/arm64/signal/testcases/poe_restore.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026 Arm Ltd + * + * Verify that the POR_EL0 register is saved and restored as expected on signal + * entry/return. + */ + +#include + +#include "test_signals_utils.h" +#include "testcases.h" + +#define POR_EL0_INIT 0x07ul +#define POR_EL0_CUSTOM 0x77ul + +static bool failed_check; + +static bool modify_por_el0(struct tdescr *td) +{ + set_por_el0(POR_EL0_CUSTOM); + + return true; +} + +static int signal_check_por_el0_reset(struct tdescr *td, siginfo_t *si, + ucontext_t *uc) +{ + uint64_t signal_por_el0 = get_por_el0(); + + if (signal_por_el0 != POR_EL0_INIT) { + fprintf(stderr, "POR_EL0 is %lx in signal handler (expected %lx)\n", + signal_por_el0, POR_EL0_INIT); + failed_check = true; + } + + return 0; +} + +static void check_por_el0_restored(struct tdescr *td) +{ + uint64_t por_el0 = get_por_el0(); + + if (por_el0 == POR_EL0_CUSTOM) { + fprintf(stderr, "POR_EL0 restored\n"); + } else { + fprintf(stderr, "POR_EL0 was %lx but is now %lx\n", + POR_EL0_CUSTOM, por_el0); + failed_check = true; + } + + td->pass = !failed_check; +} + +struct tdescr tde = { + .name = "POR_EL0 restore", + .descr = "Validate that POR_EL0 is saved/restored on signal entry/return", + .feats_required = FEAT_POE, + .timeout = 3, + .sig_trig = SIGUSR1, + .init = modify_por_el0, + .run = signal_check_por_el0_reset, + .check_result = check_por_el0_restored, +}; diff --git a/tools/testing/selftests/arm64/signal/testcases/poe_siginfo.c b/tools/testing/selftests/arm64/signal/testcases/poe_siginfo.c index 36bd9940ee05..e15fedf4da6e 100644 --- a/tools/testing/selftests/arm64/signal/testcases/poe_siginfo.c +++ b/tools/testing/selftests/arm64/signal/testcases/poe_siginfo.c @@ -21,21 +21,6 @@ static union { char buf[1024 * 128]; } context; -#define SYS_POR_EL0 "S3_3_C10_C2_4" - -static uint64_t get_por_el0(void) -{ - uint64_t val; - - asm volatile( - "mrs %0, " SYS_POR_EL0 "\n" - : "=r"(val) - : - : ); - - return val; -} - int poe_present(struct tdescr *td, siginfo_t *si, ucontext_t *uc) { struct _aarch64_ctx *head = GET_BUF_RESV_HEAD(context); diff --git a/tools/testing/selftests/mm/pkey-arm64.h b/tools/testing/selftests/mm/pkey-arm64.h index 8e9685e03c44..c5a78a2f211d 100644 --- a/tools/testing/selftests/mm/pkey-arm64.h +++ b/tools/testing/selftests/mm/pkey-arm64.h @@ -130,9 +130,10 @@ static inline u64 get_pkey_bits(u64 reg, int pkey) static inline void aarch64_write_signal_pkey(ucontext_t *uctxt, u64 pkey) { struct _aarch64_ctx *ctx = GET_UC_RESV_HEAD(uctxt); + size_t resv_size = GET_UCP_RESV_SIZE(uctxt); struct poe_context *poe_ctx = (struct poe_context *) get_header(ctx, POE_MAGIC, - sizeof(uctxt->uc_mcontext), NULL); + resv_size, NULL); if (poe_ctx) poe_ctx->por_el0 = pkey; }