github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 01:32:21 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

arm64 updates for 7.2

Browse Source 
CPU errata handling:
 - Extend CnP disabling workaround to HiSilicon HIP09 hardware.
 - Work around eternally broken broadcast TLB invalidation on more CPUs.
 - Documentation and code cleanups.
 
 CPU features:
 - Add new hwcaps for the 2025 dpISA extensions.
 
 Floating point / SVE / SME:
 - Significant cleanup to the low-level state management code in the core
   architecture code and KVM.
 - Use correct register widths during SVE/SME save/restore assembly.
 - Expose SVE/SME save/restore memory accesses to sanitisers.
 
 Memory management:
 - Preparatory work for unmapping the kernel data and bss sections from
   the linear map.
 
 Miscellaneous:
 - Inline DAIF manipulation helpers so they can be used safely from
   non-instrumentable code.
 - Fix handling of the 'nosmp' cmdline option to avoid marking secondary
   cores as "possible".
 
 MPAM:
 - Add support for v0.1 of the MPAM architecture.
 
 Perf:
 - Update HiSilicon PMU MAINTAINERS entry.
 - Fix event encodings for the DVM node in the CMN driver.
 
 Selftests:
 - Extend sigframe tests to cover POE context.
 - Add coverage for the newly added 2025 dpISA hwcaps.
 
 System registers:
 - Add new registers and ESR encodings for the HDBSS feature.
 
 Plus minor fixes and cleanups across the board.
 -----BEGIN PGP SIGNATURE-----
 
 iQFEBAABCgAuFiEEPxTL6PPUbjXGY88ct6xw3ITBYzQFAmoun+0QHHdpbGxAa2Vy
 bmVsLm9yZwAKCRC3rHDchMFjNFEvB/9scDYXHUCxQOqoRB5TPUi63pgl5vl83IfI
 TWE2r2wNsrjqXlAV2FDbzuTWxLpACS+vpk5ltsb57aUfENQ5wZEFCDvJlbpNZuHS
 5/zAQmCajPpgC5TFDo7rbgSv2BSR4TXuFfvGS349+a82c9C12EUulBFvR1DDVElh
 FCYT5PfHNu1Khas4YiUln+9WSqS8qSHfYsz4Qb/lDaZcAmh22jQBPZQFU7fMlZST
 Fr/nzSy6cxkARotqU7RlRQLgtDHBEqTYrNHCP0WSQ2aPJN8SeFUyjEcQas/Zak+S
 Bme8OHcqn4pZPll5TfHUTaJp91oIJAA9uX5RSwE+G3NvAGNbptuw
 =LXuh
 -----END PGP SIGNATURE-----

Merge tag 'arm64-upstream' of gitolite.kernel.org:pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Will Deacon:
 "It feels like the new world of AI tooling has slowed us down a little
  on the feature side when compared to the fixes side. The extra rounds
  of Sashiko review have also pushed a few things out until next time.

  Still, there's some good foundational stuff here for the fpsimd code
  and hardening work towards removing the predictable linear alias of
  the kernel image.

  CPU errata handling:
   - Extend CnP disabling workaround to HiSilicon HIP09 hardware.
   - Work around eternally broken broadcast TLB invalidation on more
     CPUs.
   - Documentation and code cleanups.

  CPU features:
   - Add new hwcaps for the 2025 dpISA extensions.

  Floating point / SVE / SME:
   - Significant cleanup to the low-level state management code in the
     core architecture code and KVM.
   - Use correct register widths during SVE/SME save/restore assembly.
   - Expose SVE/SME save/restore memory accesses to sanitisers.

  Memory management:
   - Preparatory work for unmapping the kernel data and bss sections
     from the linear map.

  Miscellaneous:
   - Inline DAIF manipulation helpers so they can be used safely from
     non-instrumentable code.
   - Fix handling of the 'nosmp' cmdline option to avoid marking
     secondary cores as "possible".

  MPAM:
   - Add support for v0.1 of the MPAM architecture.

  Perf:
   - Update HiSilicon PMU MAINTAINERS entry.
   - Fix event encodings for the DVM node in the CMN driver.

  Selftests:
   - Extend sigframe tests to cover POE context.
   - Add coverage for the newly added 2025 dpISA hwcaps.

  System registers:
   - Add new registers and ESR encodings for the HDBSS feature.

  Plus minor fixes and cleanups across the board"

* tag 'arm64-upstream' of gitolite.kernel.org:pub/scm/linux/kernel/git/arm64/linux: (73 commits)
  arm64: errata: Mitigate TLBI errata on Microsoft Azure Cobalt 100 CPU
  arm64: errata: Mitigate TLBI errata on NVIDIA Olympus CPU
  arm64: errata: Mitigate TLBI errata on various Arm CPUs
  arm64: cputype: Add C1-Premium definitions
  arm64: cputype: Add C1-Ultra definitions
  Revert "arm64: mm: Unmap kernel data/bss entirely from the linear map"
  Revert "arm64: mm: Defer remap of linear alias of data/bss"
  arm64: arch_timer: reuse arch_timer_read_cnt{p,v}ct_el0() helpers
  arm64/mm: Rename ptdesc_t
  arm64: mm: Defer remap of linear alias of data/bss
  KVM: arm64: Omit tag sync on stage-2 mappings of the zero page
  arm64: Avoid double evaluation of __ptep_get()
  kasan: Move generic KASAN page tables out of BSS too
  arm64: Rename page table BSS section to .bss..pgtbl
  arm64: patching: replace min_t with min in __text_poke
  perf/arm-cmn: Fix DVM node events
  arm64: fpsimd: Remove <asm/fpsimdmacros.h>
  arm64: fpsimd: Move SME save/restore inline
  arm64: fpsimd: Move sve_flush_live() inline
  arm64: fpsimd: Move SVE save/restore inline
  ...
This commit is contained in:
Linus Torvalds 2026-06-16 05:18:04 +05:30
commit 80476f22b8
76 changed files with 1275 additions and 966 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
Documentation/arch/arm64/elf_hwcaps.rst
View File

@ -451,6 +451,33 @@ HWCAP3_LS64
of CPU. User should only use ld64b/st64b on supported target (device)
memory location, otherwise fallback to the non-atomic alternatives.
HWCAP3_SVE_B16MM
Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and
ID_AA64ZFR0_EL1.B16B16 == 0b0011
HWCAP3_SVE2P3
Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and
ID_AA64ZFR0_EL1.SVEver == 0b0100
HWCAP3_SME_LUT6
Functionality implied by ID_AA64SMFR0_EL1.LUT6 == 0b1
HWCAP3_SME2P3
Functionality implied by ID_AA64SMFR0_EL1.SMEver == 0b0100
HWCAP3_F16MM
Functionality implied by ID_AA64FPFR0_EL1.F16MM2 == 0b1
HWCAP3_F16F32DOT
Functionality implied by ID_AA64ISAR0_EL1.FHM == 0b0010
HWCAP3_F16F32MM
Functionality implied by ID_AA64ISAR0_EL1.FHM == 0b0011
HWCAP3_SVE_LUT6
Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and
ID_AA64ISAR2_EL1.LUT == 0b0010.
4. Unused AT_HWCAP bits
-----------------------

93
Documentation/arch/arm64/silicon-errata.rst
View File

@ -116,7 +116,8 @@ stable kernels.
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A73 | #858921 | ARM64_ERRATUM_858921 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A76 | #1188873,1418040| ARM64_ERRATUM_1418040 |
| ARM | Cortex-A76 | #1188873, | ARM64_ERRATUM_1418040 |
| | | #1418040 | |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A76 | #1165522 | ARM64_ERRATUM_1165522 |
+----------------+-----------------+-----------------+-----------------------------+
@ -128,15 +129,28 @@ stable kernels.
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A76 | #3324349 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A76 | #4193800 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A76AE | #4193801 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A77 | #1491015 | N/A |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A77 | #1508412 | ARM64_ERRATUM_1508412 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A77 | #3324348 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A77 | #4193798 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A78 | #3324344 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A78C | #3324346,3324347| ARM64_ERRATUM_3194386 |
| ARM | Cortex-A78 | #4193791 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A78AE | #4193793 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A78C | #3324346, | ARM64_ERRATUM_3194386 |
| | | #3324347 | |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A78C | #4193794 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A710 | #2119858 | ARM64_ERRATUM_2119858 |
+----------------+-----------------+-----------------+-----------------------------+
@ -146,6 +160,8 @@ stable kernels.
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A710 | #3324338 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A710 | #4193788 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A715 | #2645198 | ARM64_ERRATUM_2645198 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A715 | #3456084 | ARM64_ERRATUM_3194386 |
@ -158,30 +174,44 @@ stable kernels.
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-X1 | #3324344 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-X1 | #4193791 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-X1C | #3324346 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-X1C | #4193792 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-X2 | #2119858 | ARM64_ERRATUM_2119858 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-X2 | #2224489 | ARM64_ERRATUM_2224489 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-X2 | #3324338 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-X2 | #4193788 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-X3 | #3324335 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-X3 | #4193786 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-X4 | #3194386 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-X4 | #4118414 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-X925 | #3324334 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-N1 | #1188873,1418040| ARM64_ERRATUM_1418040 |
| ARM | Cortex-X925 | #4193781 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-N1 | #1349291 | N/A |
| ARM | Neoverse-N1 | #1188873, | ARM64_ERRATUM_1418040 |
| | | #1418040 | |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-N1 | #1490853 | N/A |
| ARM | Neoverse-N1 | #1349291, | N/A |
| | | #1490853 | |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-N1 | #1542419 | ARM64_ERRATUM_1542419 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-N1 | #3324349 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-N1 | #4193800 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-N2 | #2139208 | ARM64_ERRATUM_2139208 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-N2 | #2067961 | ARM64_ERRATUM_2067961 |
@ -190,24 +220,41 @@ stable kernels.
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-N2 | #3324339 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-N2 | #4193789 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-N3 | #3456111 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-V1 | #1619801 | N/A |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-V1 | #3324341 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-V1 | #4193790 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-V2 | #3324336 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-V2 | #4193787 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-V3 | #3312417 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-V3 | #4193784 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-V3AE | #3312417 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Neoverse-V3AE | #4193784 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | C1-Premium | #4193780 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | C1-Pro | #4193714 | ARM64_ERRATUM_4193714 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | MMU-500 | #841119,826419 | ARM_SMMU_MMU_500_CPRE_ERRATA|
| | | #562869,1047329 | |
| ARM | C1-Ultra | #4193780 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | MMU-600 | #1076982,1209401| N/A |
| ARM | MMU-500 | #562869, | ARM_SMMU_MMU_500_CPRE_ERRATA|
| | | #841119, | |
| | | #826419, | |
| | | #1047329 | |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | MMU-600 | #1076982, | N/A |
| | | #1209401 | |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | MMU-700 | #2133013, | N/A |
| | | #2268618, | |
@ -230,11 +277,13 @@ stable kernels.
| Broadcom | Brahma-B53 | N/A | ARM64_ERRATUM_843419 |
+----------------+-----------------+-----------------+-----------------------------+
+----------------+-----------------+-----------------+-----------------------------+
| Cavium | ThunderX ITS | #22375,24313 | CAVIUM_ERRATUM_22375 |
| Cavium | ThunderX ITS | #22375, | CAVIUM_ERRATUM_22375 |
| | | #24313 | |
+----------------+-----------------+-----------------+-----------------------------+
| Cavium | ThunderX ITS | #23144 | CAVIUM_ERRATUM_23144 |
+----------------+-----------------+-----------------+-----------------------------+
| Cavium | ThunderX GICv3 | #23154,38545 | CAVIUM_ERRATUM_23154 |
| Cavium | ThunderX GICv3 | #23154, | CAVIUM_ERRATUM_23154 |
| | | #38545 | |
+----------------+-----------------+-----------------+-----------------------------+
| Cavium | ThunderX GICv3 | #38539 | N/A |
+----------------+-----------------+-----------------+-----------------------------+
@ -244,9 +293,8 @@ stable kernels.
+----------------+-----------------+-----------------+-----------------------------+
| Cavium | ThunderX SMMUv2 | #27704 | N/A |
+----------------+-----------------+-----------------+-----------------------------+
| Cavium | ThunderX2 SMMUv3| #74 | N/A |
+----------------+-----------------+-----------------+-----------------------------+
| Cavium | ThunderX2 SMMUv3| #126 | N/A |
| Cavium | ThunderX2 SMMUv3| #74, | N/A |
| | | #126 | |
+----------------+-----------------+-----------------+-----------------------------+
| Cavium | ThunderX2 Core | #219 | CAVIUM_TX2_ERRATUM_219 |
+----------------+-----------------+-----------------+-----------------------------+
@ -256,13 +304,13 @@ stable kernels.
+----------------+-----------------+-----------------+-----------------------------+
| NVIDIA | Carmel Core | N/A | NVIDIA_CARMEL_CNP_ERRATUM |
+----------------+-----------------+-----------------+-----------------------------+
| NVIDIA | Olympus core | T410-OLY-1029 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+
| NVIDIA | T241 GICv3/4.x | T241-FABRIC-4 | N/A |
+----------------+-----------------+-----------------+-----------------------------+
| NVIDIA | T241 MPAM | T241-MPAM-1 | N/A |
+----------------+-----------------+-----------------+-----------------------------+
| NVIDIA | T241 MPAM | T241-MPAM-4 | N/A |
+----------------+-----------------+-----------------+-----------------------------+
| NVIDIA | T241 MPAM | T241-MPAM-6 | N/A |
| NVIDIA | T241 MPAM | T241-MPAM-1, | N/A |
| | | T241-MPAM-4, | |
| | | T241-MPAM-6 | |
+----------------+-----------------+-----------------+-----------------------------+
+----------------+-----------------+-----------------+-----------------------------+
| Freescale/NXP | LS2080A/LS1043A | A-008585 | FSL_ERRATUM_A008585 |
@ -270,9 +318,8 @@ stable kernels.
+----------------+-----------------+-----------------+-----------------------------+
| Hisilicon | Hip0{5,6,7} | #161010101 | HISILICON_ERRATUM_161010101 |
+----------------+-----------------+-----------------+-----------------------------+
| Hisilicon | Hip0{6,7} | #161010701 | N/A |
+----------------+-----------------+-----------------+-----------------------------+
| Hisilicon | Hip0{6,7} | #161010803 | N/A |
| Hisilicon | Hip0{6,7} | #161010701, | N/A |
| | | #161010803 | |
+----------------+-----------------+-----------------+-----------------------------+
| Hisilicon | Hip07 | #161600802 | HISILICON_ERRATUM_161600802 |
+----------------+-----------------+-----------------+-----------------------------+
@ -284,6 +331,8 @@ stable kernels.
+----------------+-----------------+-----------------+-----------------------------+
| Hisilicon | Hip09 | #162100801 | HISILICON_ERRATUM_162100801 |
+----------------+-----------------+-----------------+-----------------------------+
| Hisilicon | Hip09 | #162100125 | HISILICON_ERRATUM_162100125 |
+----------------+-----------------+-----------------+-----------------------------+
+----------------+-----------------+-----------------+-----------------------------+
| Qualcomm Tech. | Kryo/Falkor v1 | E1003 | QCOM_FALKOR_ERRATUM_1003 |
+----------------+-----------------+-----------------+-----------------------------+
@ -323,3 +372,5 @@ stable kernels.
+----------------+-----------------+-----------------+-----------------------------+
| Microsoft | Azure Cobalt 100| #3324339 | ARM64_ERRATUM_3194386 |
+----------------+-----------------+-----------------+-----------------------------+
| Microsoft | Azure Cobalt 100| #4193789 | ARM64_ERRATUM_4118414 |
+----------------+-----------------+-----------------+-----------------------------+

2
MAINTAINERS
View File

@ -11635,7 +11635,7 @@ F: Documentation/devicetree/bindings/net/hisilicon*.txt
F: drivers/net/ethernet/hisilicon/
HISILICON PMU DRIVER
M: Jonathan Cameron <jonathan.cameron@huawei.com>
M: Yushan Wang <wangyushan12@huawei.com>
S: Supported
W: http://www.hisilicon.com
F: Documentation/admin-guide/perf/hisi-pcie-pmu.rst

63
arch/arm64/Kconfig
View File

@ -1153,6 +1153,44 @@ config ARM64_ERRATUM_4193714
If unsure, say Y.
config ARM64_ERRATUM_4118414
bool "Various: Completion of affected memory accesses might not be guaranteed by completion of a TLBI"
default y
select ARM64_WORKAROUND_REPEAT_TLBI
help
This option adds a workaround for the following errata:
* ARM C1-Premium erratum 4193780
* ARM C1-Ultra erratum 4193780
* ARM Cortex-A76 erratum 4193800
* ARM Cortex-A76AE erratum 4193801
* ARM Cortex-A77 erratum 4193798
* ARM Cortex-A78 erratum 4193791
* ARM Cortex-A78AE erratum 4193793
* ARM Cortex-A78C erratum 4193794
* ARM Cortex-A710 erratum 4193788
* ARM Cortex-X1 erratum 4193791
* ARM Cortex-X1C erratum 4193792
* ARM Cortex-X2 erratum 4193788
* ARM Cortex-X3 erratum 4193786
* ARM Cortex-X4 erratum 4118414
* ARM Cortex-X925 erratum 4193781
* ARM Neoverse-N1 erratum 4193800
* ARM Neoverse-N2 erratum 4193789
* ARM Neoverse-V1 erratum 4193790
* ARM Neoverse-V2 erratum 4193787
* ARM Neoverse-V3 erratum 4193784
* ARM Neoverse-V3AE erratum 4193784
* Microsoft Azure Cobalt 100 4193789
* NVIDIA Olympus erratum T410-OLY-1029
On affected cores, some memory accesses might not be completed by
broadcast TLB invalidation.
This issue is also known as CVE-2025-10263.
If unsure, say Y.
config CAVIUM_ERRATUM_22375
bool "Cavium erratum 22375, 24313"
default y
@ -1272,6 +1310,22 @@ config HISILICON_ERRATUM_162100801
If unsure, say Y.
config HISILICON_ERRATUM_162100125
bool "Hisilicon erratum 162100125"
default y
select ARM64_WORKAROUND_DISABLE_CNP
help
On HiSilicon HIP09, TLB entry matching behavior when CNP
(TTBRx.CNP=1) is enabled differs from the ARM architecture
specification.
TLB entries may be incorrectly shared between CPUs, potentially
causing TLB conflicts and stale mappings.
Disable CNP support for affected HiSilicon HIP09 cores.
If unsure, say Y.
config QCOM_FALKOR_ERRATUM_1003
bool "Falkor E1003: Incorrect translation due to ASID change"
default y
@ -1314,9 +1368,13 @@ config QCOM_FALKOR_ERRATUM_E1041
If unsure, say Y.
config ARM64_WORKAROUND_DISABLE_CNP
bool
config NVIDIA_CARMEL_CNP_ERRATUM
bool "NVIDIA Carmel CNP: CNP on Carmel semantically different than ARM cores"
default y
select ARM64_WORKAROUND_DISABLE_CNP
help
If CNP is enabled on Carmel cores, non-sharable TLBIs on a core will not
invalidate shared TLB entries installed by a different core, as it would
@ -2246,10 +2304,15 @@ config ARM64_SVE
booting the kernel. If unsure and you are not observing these
symptoms, you should assume that it is safe to say Y.
config AS_HAS_SME
# Supported by LLVM 13+ and binutils 2.38+
def_bool $(as-instr,.arch_extension sme)
config ARM64_SME
bool "ARM Scalable Matrix Extension support"
default y
depends on ARM64_SVE
depends on AS_HAS_SME
help
The Scalable Matrix Extension (SME) is an extension to the AArch64
execution state which utilises a substantial subset of the SVE

12
arch/arm64/include/asm/arch_timer.h
View File

@ -178,12 +178,8 @@ static __always_inline u64 __arch_counter_get_cntpct_stable(void)
static __always_inline u64 __arch_counter_get_cntpct(void)
{
u64 cnt;
u64 cnt = arch_timer_read_cntpct_el0();
asm volatile(ALTERNATIVE("isb\n mrs %0, cntpct_el0",
"nop\n" __mrs_s("%0", SYS_CNTPCTSS_EL0),
ARM64_HAS_ECV)
: "=r" (cnt));
arch_counter_enforce_ordering(cnt);
return cnt;
}
@ -199,12 +195,8 @@ static __always_inline u64 __arch_counter_get_cntvct_stable(void)
static __always_inline u64 __arch_counter_get_cntvct(void)
{
u64 cnt;
u64 cnt = arch_timer_read_cntvct_el0();
asm volatile(ALTERNATIVE("isb\n mrs %0, cntvct_el0",
"nop\n" __mrs_s("%0", SYS_CNTVCTSS_EL0),
ARM64_HAS_ECV)
: "=r" (cnt));
arch_counter_enforce_ordering(cnt);
return cnt;
}

4
arch/arm64/include/asm/cpucaps.h
View File

@ -58,8 +58,8 @@ cpucap_is_possible(const unsigned int cap)
return IS_ENABLED(CONFIG_ARM64_ERRATUM_2658417);
case ARM64_WORKAROUND_CAVIUM_23154:
return IS_ENABLED(CONFIG_CAVIUM_ERRATUM_23154);
case ARM64_WORKAROUND_NVIDIA_CARMEL_CNP:
return IS_ENABLED(CONFIG_NVIDIA_CARMEL_CNP_ERRATUM);
case ARM64_WORKAROUND_DISABLE_CNP:
return IS_ENABLED(CONFIG_ARM64_WORKAROUND_DISABLE_CNP);
case ARM64_WORKAROUND_REPEAT_TLBI:
return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI);
case ARM64_WORKAROUND_SPECULATIVE_SSBS:

7
arch/arm64/include/asm/cpufeature.h
View File

@ -620,6 +620,13 @@ static inline bool id_aa64pfr0_mpam(u64 pfr0)
return val > 0;
}
static inline bool id_aa64pfr1_mpamfrac(u64 pfr1)
{
u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_EL1_MPAM_frac_SHIFT);
return val > 0;
}
static inline bool id_aa64pfr1_mte(u64 pfr1)
{
u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_EL1_MTE_SHIFT);

4
arch/arm64/include/asm/cputype.h
View File

@ -97,8 +97,10 @@
#define ARM_CPU_PART_CORTEX_X925 0xD85
#define ARM_CPU_PART_CORTEX_A725 0xD87
#define ARM_CPU_PART_CORTEX_A720AE 0xD89
#define ARM_CPU_PART_C1_ULTRA 0xD8C
#define ARM_CPU_PART_NEOVERSE_N3 0xD8E
#define ARM_CPU_PART_C1_PRO 0xD8B
#define ARM_CPU_PART_C1_PREMIUM 0xD90
#define APM_CPU_PART_XGENE 0x000
#define APM_CPU_VAR_POTENZA 0x00
@ -189,8 +191,10 @@
#define MIDR_CORTEX_X925 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X925)
#define MIDR_CORTEX_A725 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A725)
#define MIDR_CORTEX_A720AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A720AE)
#define MIDR_C1_ULTRA MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_ULTRA)
#define MIDR_NEOVERSE_N3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N3)
#define MIDR_C1_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PRO)
#define MIDR_C1_PREMIUM MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PREMIUM)
#define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
#define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX)
#define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX)

10
arch/arm64/include/asm/daifflags.h
View File

@ -19,7 +19,7 @@
/* mask/save/unmask/restore all exceptions, including interrupts. */
static inline void local_daif_mask(void)
static __always_inline void local_daif_mask(void)
{
WARN_ON(system_has_prio_mask_debugging() &&
(read_sysreg_s(SYS_ICC_PMR_EL1) == (GIC_PRIO_IRQOFF |
@ -38,7 +38,7 @@ static inline void local_daif_mask(void)
trace_hardirqs_off();
}
static inline unsigned long local_daif_save_flags(void)
static __always_inline unsigned long local_daif_save_flags(void)
{
unsigned long flags;
@ -53,7 +53,7 @@ static inline unsigned long local_daif_save_flags(void)
return flags;
}
static inline unsigned long local_daif_save(void)
static __always_inline unsigned long local_daif_save(void)
{
unsigned long flags;
@ -64,7 +64,7 @@ static inline unsigned long local_daif_save(void)
return flags;
}
static inline void local_daif_restore(unsigned long flags)
static __always_inline void local_daif_restore(unsigned long flags)
{
bool irq_disabled = flags & PSR_I_BIT;
@ -124,7 +124,7 @@ static inline void local_daif_restore(unsigned long flags)
* Called by synchronous exception handlers to restore the DAIF bits that were
* modified by taking an exception.
*/
static inline void local_daif_inherit(struct pt_regs *regs)
static __always_inline void local_daif_inherit(struct pt_regs *regs)
{
unsigned long flags = regs->pstate & DAIF_MASK;

14
arch/arm64/include/asm/device.h
View File

@ -1,14 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2012 ARM Ltd.
*/
#ifndef __ASM_DEVICE_H
#define __ASM_DEVICE_H
struct dev_archdata {
};
struct pdev_archdata {
};
#endif

4
arch/arm64/include/asm/el2_setup.h
View File

@ -510,7 +510,9 @@
#endif
.macro finalise_el2_state
check_override id_aa64pfr0, ID_AA64PFR0_EL1_MPAM_SHIFT, .Linit_mpam_\@, .Lskip_mpam_\@, x1, x2
check_override id_aa64pfr0, ID_AA64PFR0_EL1_MPAM_SHIFT, .Linit_mpam_\@, .Lmpam_minor_\@, x1, x2
.Lmpam_minor_\@:
check_override id_aa64pfr1, ID_AA64PFR1_EL1_MPAM_frac_SHIFT, .Linit_mpam_\@, .Lskip_mpam_\@, x1, x2
.Linit_mpam_\@:
mov x0, #MPAM2_EL2_EnMPAMSM_MASK

2
arch/arm64/include/asm/esr.h
View File

@ -160,6 +160,8 @@
#define ESR_ELx_CM (UL(1) << ESR_ELx_CM_SHIFT)
/* ISS2 field definitions for Data Aborts */
#define ESR_ELx_HDBSSF_SHIFT (11)
#define ESR_ELx_HDBSSF (UL(1) << ESR_ELx_HDBSSF_SHIFT)
#define ESR_ELx_TnD_SHIFT (10)
#define ESR_ELx_TnD (UL(1) << ESR_ELx_TnD_SHIFT)
#define ESR_ELx_TagAccess_SHIFT (9)

386
arch/arm64/include/asm/fpsimd.h
View File

@ -22,6 +22,11 @@
#include <linux/stddef.h>
#include <linux/types.h>
#define __FPSIMD_PREAMBLE ".arch_extension fp\n" \
".arch_extension simd\n"
#define __SVE_PREAMBLE ".arch_extension sve\n"
#define __SME_PREAMBLE ".arch_extension sme\n"
/* Masks for extracting the FPSR and FPCR from the FPSCR */
#define VFP_FPSCR_STAT_MASK 0xf800009f
#define VFP_FPSCR_CTRL_MASK 0x07f79f00
@ -71,8 +76,82 @@ static inline void cpacr_restore(unsigned long cpacr)
struct task_struct;
extern void fpsimd_save_state(struct user_fpsimd_state *state);
extern void fpsimd_load_state(struct user_fpsimd_state *state);
static inline void fpsimd_save_common(struct user_fpsimd_state *state)
{
state->fpsr = read_sysreg_s(SYS_FPSR);
state->fpcr = read_sysreg_s(SYS_FPCR);
}
static inline void fpsimd_load_common(const struct user_fpsimd_state *state)
{
write_sysreg_s(state->fpsr, SYS_FPSR);
write_sysreg_s(state->fpcr, SYS_FPCR);
}
static inline void fpsimd_save_vregs(struct user_fpsimd_state *state)
{
instrument_write(state->vregs, sizeof(state->vregs));
asm volatile(
__FPSIMD_PREAMBLE
" stp q0, q1, [%[vregs], #16 * 0]\n"
" stp q2, q3, [%[vregs], #16 * 2]\n"
" stp q4, q5, [%[vregs], #16 * 4]\n"
" stp q6, q7, [%[vregs], #16 * 6]\n"
" stp q8, q9, [%[vregs], #16 * 8]\n"
" stp q10, q11, [%[vregs], #16 * 10]\n"
" stp q12, q13, [%[vregs], #16 * 12]\n"
" stp q14, q15, [%[vregs], #16 * 14]\n"
" stp q16, q17, [%[vregs], #16 * 16]\n"
" stp q18, q19, [%[vregs], #16 * 18]\n"
" stp q20, q21, [%[vregs], #16 * 20]\n"
" stp q22, q23, [%[vregs], #16 * 22]\n"
" stp q24, q25, [%[vregs], #16 * 24]\n"
" stp q26, q27, [%[vregs], #16 * 26]\n"
" stp q28, q29, [%[vregs], #16 * 28]\n"
" stp q30, q31, [%[vregs], #16 * 30]\n"
: "=Q" (state->vregs)
: [vregs] "r" (state->vregs)
);
}
static inline void fpsimd_load_vregs(const struct user_fpsimd_state *state)
{
instrument_read(state->vregs, sizeof(state->vregs));
asm volatile(
__FPSIMD_PREAMBLE
" ldp q0, q1, [%[vregs], #16 * 0]\n"
" ldp q2, q3, [%[vregs], #16 * 2]\n"
" ldp q4, q5, [%[vregs], #16 * 4]\n"
" ldp q6, q7, [%[vregs], #16 * 6]\n"
" ldp q8, q9, [%[vregs], #16 * 8]\n"
" ldp q10, q11, [%[vregs], #16 * 10]\n"
" ldp q12, q13, [%[vregs], #16 * 12]\n"
" ldp q14, q15, [%[vregs], #16 * 14]\n"
" ldp q16, q17, [%[vregs], #16 * 16]\n"
" ldp q18, q19, [%[vregs], #16 * 18]\n"
" ldp q20, q21, [%[vregs], #16 * 20]\n"
" ldp q22, q23, [%[vregs], #16 * 22]\n"
" ldp q24, q25, [%[vregs], #16 * 24]\n"
" ldp q26, q27, [%[vregs], #16 * 26]\n"
" ldp q28, q29, [%[vregs], #16 * 28]\n"
" ldp q30, q31, [%[vregs], #16 * 30]\n"
:
: "Q" (state->vregs),
[vregs] "r" (state->vregs)
);
}
static inline void fpsimd_save_state(struct user_fpsimd_state *state)
{
fpsimd_save_vregs(state);
fpsimd_save_common(state);
}
static inline void fpsimd_load_state(const struct user_fpsimd_state *state)
{
fpsimd_load_vregs(state);
fpsimd_load_common(state);
}
extern void fpsimd_thread_switch(struct task_struct *next);
extern void fpsimd_flush_thread(void);
@ -83,8 +162,8 @@ extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
struct cpu_fp_state {
struct user_fpsimd_state *st;
void *sve_state;
void *sme_state;
struct arm64_sve_state *sve_state;
struct arm64_sme_state *sme_state;
u64 *svcr;
u64 *fpmr;
unsigned int sve_vl;
@ -116,40 +195,166 @@ extern void task_smstop_sm(struct task_struct *task);
/* Maximum VL that SVE/SME VL-agnostic software can transparently support */
#define VL_ARCH_MAX 0x100
/* Offset of FFR in the SVE register dump */
static inline size_t sve_ffr_offset(int vl)
{
return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
}
static inline void *sve_pffr(struct thread_struct *thread)
{
unsigned int vl;
if (system_supports_sme() && thread_sm_enabled(thread))
vl = thread_get_sme_vl(thread);
else
vl = thread_get_sve_vl(thread);
return (char *)thread->sve_state + sve_ffr_offset(vl);
}
static inline void *thread_zt_state(struct thread_struct *thread)
{
/* The ZT register state is stored immediately after the ZA state */
unsigned int sme_vq = sve_vq_from_vl(thread_get_sme_vl(thread));
return thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq);
return (void *)thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq);
}
extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr);
extern void sve_load_state(void const *state, u32 const *pfpsr,
int restore_ffr);
extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
extern unsigned int sve_get_vl(void);
extern void sve_set_vq(unsigned long vq_minus_1);
extern void sme_set_vq(unsigned long vq_minus_1);
extern void sme_save_state(void *state, int zt);
extern void sme_load_state(void const *state, int zt);
static inline unsigned int sve_get_vl(void)
{
unsigned int vl;
asm volatile(
__SVE_PREAMBLE
" rdvl %x[vl], #1\n"
: [vl] "=r" (vl)
);
return vl;
}
#define FOR_EACH_Z_REG(idx_str, asm_str) \
" .irp " idx_str ",0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31\n" \
asm_str "\n" \
" .endr\n"
#define FOR_EACH_P_REG(idx_str, asm_str) \
" .irp " idx_str ",0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \
asm_str "\n" \
" .endr\n"
static inline void __sve_save_z(struct arm64_sve_state *state, unsigned long vl)
{
instrument_write(state, SVE_NUM_ZREGS * vl);
asm volatile(
__SVE_PREAMBLE
FOR_EACH_Z_REG("n", "str z\\n, [%[zregs], #\\n, MUL VL]")
:
: [zregs] "r" (state)
: "memory"
);
}
static inline void __sve_load_z(const struct arm64_sve_state *state, unsigned long vl)
{
instrument_read(state, SVE_NUM_ZREGS * vl);
asm volatile(
__SVE_PREAMBLE
FOR_EACH_Z_REG("n", "ldr z\\n, [%[zregs], #\\n, MUL VL]")
:
: [zregs] "r" (state)
: "memory"
);
}
static inline void __sve_save_p(struct arm64_sve_state *state, unsigned long vl, bool ffr)
{
void *pregs = (void *)state + SVE_NUM_ZREGS * vl;
unsigned long pl = vl / 8;
void *pffr = pregs + SVE_NUM_PREGS * pl;
instrument_write(pregs, SVE_NUM_PREGS * pl);
asm volatile(
__SVE_PREAMBLE
FOR_EACH_P_REG("n", "str p\\n, [%[pregs], #\\n, MUL VL]\n")
:
: [pregs] "r" (pregs)
: "memory"
);
instrument_write(pffr, pl);
if (ffr) {
asm volatile(
__SVE_PREAMBLE
" rdffr p0.b\n"
" str p0, [%[pffr]]\n"
" ldr p0, [%[pregs]]\n"
:
: [pregs] "r" (pregs),
[pffr] "r" (pffr)
: "memory"
);
} else {
asm volatile(
__SVE_PREAMBLE
" pfalse p0.b\n"
" str p0, [%[pffr]]\n"
" ldr p0, [%[pregs]]\n"
:
: [pregs] "r" (pregs),
[pffr] "r" (pffr)
: "memory"
);
}
}
static inline void __sve_load_p(const struct arm64_sve_state *state, unsigned long vl, bool ffr)
{
const void *pregs = (const void *)state + SVE_NUM_ZREGS * vl;
unsigned long pl = vl / 8;
const void *pffr = pregs + SVE_NUM_PREGS * pl;
if (ffr) {
instrument_read(pffr, pl);
asm volatile(
__SVE_PREAMBLE
" ldr p0, [%[pffr]]\n"
" wrffr p0.b\n"
:
: [pffr] "r" (pffr)
: "memory"
);
}
instrument_read(pregs, SVE_NUM_PREGS * pl);
asm volatile(
__SVE_PREAMBLE
FOR_EACH_P_REG("n", "ldr p\\n, [%[pregs], #\\n, MUL VL]\n")
:
: [pregs] "r" (pregs)
: "memory"
);
}
static inline void sve_save_state(struct arm64_sve_state *state, bool ffr)
{
unsigned long vl = sve_get_vl();
__sve_save_z(state, vl);
__sve_save_p(state, vl, ffr);
}
static inline void sve_load_state(const struct arm64_sve_state *state, bool ffr)
{
unsigned long vl = sve_get_vl();
__sve_load_z(state, vl);
__sve_load_p(state, vl, ffr);
}
/*
* Zero all SVE registers except for the first 128 bits of each vector.
*
* The caller must ensure that the VL has been configured and the CPU must be
* in non-streaming mode.
*/
static inline void sve_flush_live(void)
{
unsigned long vl = sve_get_vl();
if (vl > sizeof(__uint128_t)) {
asm volatile(
__FPSIMD_PREAMBLE
FOR_EACH_Z_REG("n", "mov v\\n\\().16b, v\\n\\().16b")
);
}
asm volatile(
__SVE_PREAMBLE
FOR_EACH_P_REG("n", "pfalse p\\n\\().b")
" wrffr p0.b\n"
);
}
struct arm64_cpu_capabilities;
extern void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__unused);
@ -402,8 +607,20 @@ static inline int sme_max_virtualisable_vl(void)
return vec_max_virtualisable_vl(ARM64_VEC_SME);
}
static inline unsigned int sme_get_vl(void)
{
unsigned int vl;
asm volatile(
__SME_PREAMBLE
" rdsvl %x[vl], #1\n"
: [vl] "=r" (vl)
);
return vl;
}
extern void sme_alloc(struct task_struct *task, bool flush);
extern unsigned int sme_get_vl(void);
extern int sme_set_current_vl(unsigned long arg);
extern int sme_get_current_vl(void);
extern void sme_suspend_exit(void);
@ -418,6 +635,106 @@ static inline size_t __sme_state_size(unsigned int sme_vl)
return size;
}
static inline void __sme_save_za(struct arm64_sme_state *state, unsigned long svl)
{
/*
* The <Wv> argument to LDR/STR (array vector) can only encode W12-W15.
* The "Ucj" constraint exists for this, but is only supported by GCC
* 14.1.0+ and LLVM 18.1.0+.
*/
register unsigned int v asm ("w12");
instrument_write(state, svl * svl);
for (v = 0; v < svl; v++) {
void *pav = (void *)state + v * svl;
asm volatile(
__SME_PREAMBLE
" str za[%w[v], #0], [%[pav]]\n"
:
: [v] "r" (v),
[pav] "r" (pav)
: "memory"
);
}
}
static inline void __sme_load_za(const struct arm64_sme_state *state, unsigned long svl)
{
/* See comment in __sme_save_za */
register unsigned int v asm ("w12");
instrument_read(state, svl * svl);
for (v = 0; v < svl; v++) {
void *pav = (void *)state + v * svl;
asm volatile(
__SME_PREAMBLE
" ldr za[%w[v], #0], [%[pav]]\n"
:
: [v] "r" (v),
[pav] "r" (pav)
: "memory"
);
}
}
static inline void __sme_save_zt(struct arm64_sme_state *state, unsigned long svl)
{
void *pzt = (void *)state + svl * svl;
instrument_write(pzt, 64);
asm volatile(
__DEFINE_ASM_GPR_NUMS
/*
* STR ZT0, [<Xn|SP>]
* Supported by binutils 2.41+.
* Supported by LLVM 16+
*/
" .inst 0xe13f8000 | ((.L__gpr_num_%[pzt]) << 5)\n"
:
: [pzt] "r" (pzt)
: "memory"
);
}
static inline void __sme_load_zt(const struct arm64_sme_state *state, unsigned long svl)
{
void *pzt = (void *)state + svl * svl;
instrument_read(pzt, 64);
asm volatile(
__DEFINE_ASM_GPR_NUMS
/*
* LDR ZT0, [<Xn|SP>]
* Supported by binutils 2.41+.
* Supported by LLVM 16+
*/
" .inst 0xe11f8000 | ((.L__gpr_num_%[pzt]) << 5)\n"
:
: [pzt] "r" (pzt)
: "memory"
);
}
static inline void sme_save_state(struct arm64_sme_state *state, bool zt)
{
unsigned long svl = sme_get_vl();
__sme_save_za(state, svl);
if (zt)
__sme_save_zt(state, svl);
}
static inline void sme_load_state(const struct arm64_sme_state *state, bool zt)
{
unsigned long svl = sme_get_vl();
__sme_load_za(state, svl);
if (zt)
__sme_load_zt(state, svl);
}
/*
* Return how many bytes of memory are required to store the full SME
* specific state for task, given task's currently configured vector
@ -474,6 +791,9 @@ static inline size_t sme_state_size(struct task_struct const *task)
return 0;
}
static inline void sme_save_state(struct arm64_sme_state *state, bool zt) { BUILD_BUG(); }
static inline void sme_load_state(const struct arm64_sme_state *state, bool zt) { BUILD_BUG(); }
static inline void sme_enter_from_user_mode(void) { }
static inline void sme_exit_to_user_mode(void) { }

357
arch/arm64/include/asm/fpsimdmacros.h
View File

@ -1,357 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* FP/SIMD state saving and restoring macros
*
* Copyright (C) 2012 ARM Ltd.
* Author: Catalin Marinas <catalin.marinas@arm.com>
*/
#include <asm/assembler.h>
.macro fpsimd_save state, tmpnr
stp q0, q1, [\state, #16 * 0]
stp q2, q3, [\state, #16 * 2]
stp q4, q5, [\state, #16 * 4]
stp q6, q7, [\state, #16 * 6]
stp q8, q9, [\state, #16 * 8]
stp q10, q11, [\state, #16 * 10]
stp q12, q13, [\state, #16 * 12]
stp q14, q15, [\state, #16 * 14]
stp q16, q17, [\state, #16 * 16]
stp q18, q19, [\state, #16 * 18]
stp q20, q21, [\state, #16 * 20]
stp q22, q23, [\state, #16 * 22]
stp q24, q25, [\state, #16 * 24]
stp q26, q27, [\state, #16 * 26]
stp q28, q29, [\state, #16 * 28]
stp q30, q31, [\state, #16 * 30]!
mrs x\tmpnr, fpsr
str w\tmpnr, [\state, #16 * 2]
mrs x\tmpnr, fpcr
str w\tmpnr, [\state, #16 * 2 + 4]
.endm
.macro fpsimd_restore_fpcr state, tmp
/*
* Writes to fpcr may be self-synchronising, so avoid restoring
* the register if it hasn't changed.
*/
mrs \tmp, fpcr
cmp \tmp, \state
b.eq 9999f
msr fpcr, \state
9999:
.endm
/* Clobbers \state */
.macro fpsimd_restore state, tmpnr
ldp q0, q1, [\state, #16 * 0]
ldp q2, q3, [\state, #16 * 2]
ldp q4, q5, [\state, #16 * 4]
ldp q6, q7, [\state, #16 * 6]
ldp q8, q9, [\state, #16 * 8]
ldp q10, q11, [\state, #16 * 10]
ldp q12, q13, [\state, #16 * 12]
ldp q14, q15, [\state, #16 * 14]
ldp q16, q17, [\state, #16 * 16]
ldp q18, q19, [\state, #16 * 18]
ldp q20, q21, [\state, #16 * 20]
ldp q22, q23, [\state, #16 * 22]
ldp q24, q25, [\state, #16 * 24]
ldp q26, q27, [\state, #16 * 26]
ldp q28, q29, [\state, #16 * 28]
ldp q30, q31, [\state, #16 * 30]!
ldr w\tmpnr, [\state, #16 * 2]
msr fpsr, x\tmpnr
ldr w\tmpnr, [\state, #16 * 2 + 4]
fpsimd_restore_fpcr x\tmpnr, \state
.endm
/* Sanity-check macros to help avoid encoding garbage instructions */
.macro _check_general_reg nr
.if (\nr) < 0 || (\nr) > 30
.error "Bad register number \nr."
.endif
.endm
.macro _sve_check_zreg znr
.if (\znr) < 0 || (\znr) > 31
.error "Bad Scalable Vector Extension vector register number \znr."
.endif
.endm
.macro _sve_check_preg pnr
.if (\pnr) < 0 || (\pnr) > 15
.error "Bad Scalable Vector Extension predicate register number \pnr."
.endif
.endm
.macro _check_num n, min, max
.if (\n) < (\min) || (\n) > (\max)
.error "Number \n out of range [\min,\max]"
.endif
.endm
.macro _sme_check_wv v
.if (\v) < 12 || (\v) > 15
.error "Bad vector select register \v."
.endif
.endm
/* SVE instruction encodings for non-SVE-capable assemblers */
/* (pre binutils 2.28, all kernel capable clang versions support SVE) */
/* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */
.macro _sve_str_v nz, nxbase, offset=0
_sve_check_zreg \nz
_check_general_reg \nxbase
_check_num (\offset), -0x100, 0xff
.inst 0xe5804000 \
| (\nz) \
| ((\nxbase) << 5) \
| (((\offset) & 7) << 10) \
| (((\offset) & 0x1f8) << 13)
.endm
/* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */
.macro _sve_ldr_v nz, nxbase, offset=0
_sve_check_zreg \nz
_check_general_reg \nxbase
_check_num (\offset), -0x100, 0xff
.inst 0x85804000 \
| (\nz) \
| ((\nxbase) << 5) \
| (((\offset) & 7) << 10) \
| (((\offset) & 0x1f8) << 13)
.endm
/* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */
.macro _sve_str_p np, nxbase, offset=0
_sve_check_preg \np
_check_general_reg \nxbase
_check_num (\offset), -0x100, 0xff
.inst 0xe5800000 \
| (\np) \
| ((\nxbase) << 5) \
| (((\offset) & 7) << 10) \
| (((\offset) & 0x1f8) << 13)
.endm
/* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */
.macro _sve_ldr_p np, nxbase, offset=0
_sve_check_preg \np
_check_general_reg \nxbase
_check_num (\offset), -0x100, 0xff
.inst 0x85800000 \
| (\np) \
| ((\nxbase) << 5) \
| (((\offset) & 7) << 10) \
| (((\offset) & 0x1f8) << 13)
.endm
/* RDVL X\nx, #\imm */
.macro _sve_rdvl nx, imm
_check_general_reg \nx
_check_num (\imm), -0x20, 0x1f
.inst 0x04bf5000 \
| (\nx) \
| (((\imm) & 0x3f) << 5)
.endm
/* RDFFR (unpredicated): RDFFR P\np.B */
.macro _sve_rdffr np
_sve_check_preg \np
.inst 0x2519f000 \
| (\np)
.endm
/* WRFFR P\np.B */
.macro _sve_wrffr np
_sve_check_preg \np
.inst 0x25289000 \
| ((\np) << 5)
.endm
/* PFALSE P\np.B */
.macro _sve_pfalse np
_sve_check_preg \np
.inst 0x2518e400 \
| (\np)
.endm
/* SME instruction encodings for non-SME-capable assemblers */
/* (pre binutils 2.38/LLVM 13) */
/* RDSVL X\nx, #\imm */
.macro _sme_rdsvl nx, imm
_check_general_reg \nx
_check_num (\imm), -0x20, 0x1f
.inst 0x04bf5800 \
| (\nx) \
| (((\imm) & 0x3f) << 5)
.endm
/*
* STR (vector from ZA array):
* STR ZA[\nw, #\offset], [X\nxbase, #\offset, MUL VL]
*/
.macro _sme_str_zav nw, nxbase, offset=0
_sme_check_wv \nw
_check_general_reg \nxbase
_check_num (\offset), -0x100, 0xff
.inst 0xe1200000 \
| (((\nw) & 3) << 13) \
| ((\nxbase) << 5) \
| ((\offset) & 7)
.endm
/*
* LDR (vector to ZA array):
* LDR ZA[\nw, #\offset], [X\nxbase, #\offset, MUL VL]
*/
.macro _sme_ldr_zav nw, nxbase, offset=0
_sme_check_wv \nw
_check_general_reg \nxbase
_check_num (\offset), -0x100, 0xff
.inst 0xe1000000 \
| (((\nw) & 3) << 13) \
| ((\nxbase) << 5) \
| ((\offset) & 7)
.endm
/*
* LDR (ZT0)
*
* LDR ZT0, nx
*/
.macro _ldr_zt nx
_check_general_reg \nx
.inst 0xe11f8000 \
| (\nx << 5)
.endm
/*
* STR (ZT0)
*
* STR ZT0, nx
*/
.macro _str_zt nx
_check_general_reg \nx
.inst 0xe13f8000 \
| (\nx << 5)
.endm
.macro __for from:req, to:req
.if (\from) == (\to)
_for__body %\from
.else
__for %\from, %((\from) + ((\to) - (\from)) / 2)
__for %((\from) + ((\to) - (\from)) / 2 + 1), %\to
.endif
.endm
.macro _for var:req, from:req, to:req, insn:vararg
.macro _for__body \var:req
.noaltmacro
\insn
.altmacro
.endm
.altmacro
__for \from, \to
.noaltmacro
.purgem _for__body
.endm
/* Update ZCR_EL1.LEN with the new VQ */
.macro sve_load_vq xvqminus1, xtmp, xtmp2
mrs_s \xtmp, SYS_ZCR_EL1
bic \xtmp2, \xtmp, ZCR_ELx_LEN_MASK
orr \xtmp2, \xtmp2, \xvqminus1
cmp \xtmp2, \xtmp
b.eq 921f
msr_s SYS_ZCR_EL1, \xtmp2 //self-synchronising
921:
.endm
/* Update SMCR_EL1.LEN with the new VQ */
.macro sme_load_vq xvqminus1, xtmp, xtmp2
mrs_s \xtmp, SYS_SMCR_EL1
bic \xtmp2, \xtmp, SMCR_ELx_LEN_MASK
orr \xtmp2, \xtmp2, \xvqminus1
cmp \xtmp2, \xtmp
b.eq 921f
msr_s SYS_SMCR_EL1, \xtmp2 //self-synchronising
921:
.endm
/* Preserve the first 128-bits of Znz and zero the rest. */
.macro _sve_flush_z nz
_sve_check_zreg \nz
mov v\nz\().16b, v\nz\().16b
.endm
.macro sve_flush_z
_for n, 0, 31, _sve_flush_z \n
.endm
.macro sve_flush_p
_for n, 0, 15, _sve_pfalse \n
.endm
.macro sve_flush_ffr
_sve_wrffr 0
.endm
.macro sve_save nxbase, xpfpsr, save_ffr, nxtmp
_for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34
_for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16
cbz \save_ffr, 921f
_sve_rdffr 0
b 922f
921:
_sve_pfalse 0 // Zero out FFR
922:
_sve_str_p 0, \nxbase
_sve_ldr_p 0, \nxbase, -16
mrs x\nxtmp, fpsr
str w\nxtmp, [\xpfpsr]
mrs x\nxtmp, fpcr
str w\nxtmp, [\xpfpsr, #4]
.endm
.macro sve_load nxbase, xpfpsr, restore_ffr, nxtmp
_for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34
cbz \restore_ffr, 921f
_sve_ldr_p 0, \nxbase
_sve_wrffr 0
921:
_for n, 0, 15, _sve_ldr_p \n, \nxbase, \n - 16
ldr w\nxtmp, [\xpfpsr]
msr fpsr, x\nxtmp
ldr w\nxtmp, [\xpfpsr, #4]
msr fpcr, x\nxtmp
.endm
.macro sme_save_za nxbase, xvl, nw
mov w\nw, #0
423:
_sme_str_zav \nw, \nxbase
add x\nxbase, x\nxbase, \xvl
add x\nw, x\nw, #1
cmp \xvl, x\nw
bne 423b
.endm
.macro sme_load_za nxbase, xvl, nw
mov w\nw, #0
423:
_sme_ldr_zav \nw, \nxbase
add x\nxbase, x\nxbase, \xvl
add x\nw, x\nw, #1
cmp \xvl, x\nw
bne 423b
.endm

2
arch/arm64/include/asm/io.h
View File

@ -270,7 +270,7 @@ static inline void __iomem *ioremap_prot(phys_addr_t phys, size_t size,
pgprot_t user_prot)
{
pgprot_t prot;
ptdesc_t user_prot_val = pgprot_val(user_prot);
ptval_t user_prot_val = pgprot_val(user_prot);
if (WARN_ON_ONCE(!(user_prot_val & PTE_USER)))
return NULL;

27
arch/arm64/include/asm/kvm_host.h
View File

@ -732,20 +732,6 @@ struct kvm_cpu_context {
u64 *vncr_array;
};
struct cpu_sve_state {
__u64 zcr_el1;
/*
* Ordering is important since __sve_save_state/__sve_restore_state
* relies on it.
*/
__u32 fpsr;
__u32 fpcr;
/* Must be SVE_VQ_BYTES (128 bit) aligned. */
__u8 sve_regs[];
};
/*
* This structure is instantiated on a per-CPU basis, and contains
* data that is:
@ -771,12 +757,9 @@ struct kvm_host_data {
/*
* Hyp VA.
* sve_state is only used in pKVM and if system_supports_sve().
* sve_regs is only used in pKVM and if system_supports_sve().
*/
struct cpu_sve_state *sve_state;
/* Used by pKVM only. */
u64 fpmr;
struct arm64_sve_state *sve_regs;
/* Ownership of the FP regs */
enum {
@ -870,7 +853,7 @@ struct kvm_vcpu_arch {
* floating point code saves the register state of a task it
* records which view it saved in fp_type.
*/
void *sve_state;
struct arm64_sve_state *sve_state;
enum fp_type fp_type;
unsigned int sve_max_vl;
@ -1114,10 +1097,6 @@ struct kvm_vcpu_arch {
#define NESTED_SERROR_PENDING __vcpu_single_flag(sflags, BIT(8))
/* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */
#define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) + \
sve_ffr_offset((vcpu)->arch.sve_max_vl))
#define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl)
#define vcpu_sve_zcr_elx(vcpu) \

5
arch/arm64/include/asm/kvm_hyp.h
View File

@ -121,11 +121,6 @@ void __debug_save_host_buffers_nvhe(struct kvm_vcpu *vcpu);
void __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu);
#endif
void __fpsimd_save_state(struct user_fpsimd_state *fp_regs);
void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs);
void __sve_save_state(void *sve_pffr, u32 *fpsr, int save_ffr);
void __sve_restore_state(void *sve_pffr, u32 *fpsr, int restore_ffr);
u64 __guest_enter(struct kvm_vcpu *vcpu);
bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt, u32 func_id);

3
arch/arm64/include/asm/kvm_pkvm.h
View File

@ -188,8 +188,7 @@ static inline size_t pkvm_host_sve_state_size(void)
if (!system_supports_sve())
return 0;
return size_add(sizeof(struct cpu_sve_state),
SVE_SIG_REGS_SIZE(sve_vq_from_vl(kvm_host_sve_max_vl)));
return SVE_SIG_REGS_SIZE(sve_vq_from_vl(kvm_host_sve_max_vl));
}
struct pkvm_mapping {

4
arch/arm64/include/asm/linkage.h
View File

@ -43,4 +43,8 @@
SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \
bti c ;
#define _THIS_IP_ ({ unsigned long __ip; asm volatile("adr %0, ." : "=r" (__ip)); __ip; })
#define __bss_pgtbl __section(".bss..pgtbl") __aligned(PAGE_SIZE)
#endif

14
arch/arm64/include/asm/pgtable-types.h
View File

@ -17,13 +17,13 @@
* Generic page table descriptor format from which
* all level specific descriptors can be derived.
*/
typedef u64 ptdesc_t;
typedef u64 ptval_t;
typedef ptdesc_t pteval_t;
typedef ptdesc_t pmdval_t;
typedef ptdesc_t pudval_t;
typedef ptdesc_t p4dval_t;
typedef ptdesc_t pgdval_t;
typedef ptval_t pteval_t;
typedef ptval_t pmdval_t;
typedef ptval_t pudval_t;
typedef ptval_t p4dval_t;
typedef ptval_t pgdval_t;
/*
* These are used to make use of C type-checking..
@ -54,7 +54,7 @@ typedef struct { pgdval_t pgd; } pgd_t;
#define pgd_val(x) ((x).pgd)
#define __pgd(x) ((pgd_t) { (x) } )
typedef struct { ptdesc_t pgprot; } pgprot_t;
typedef struct { ptval_t pgprot; } pgprot_t;
#define pgprot_val(x) ((x).pgprot)
#define __pgprot(x) ((pgprot_t) { (x) } )

4
arch/arm64/include/asm/pgtable.h
View File

@ -1007,7 +1007,7 @@ static inline pud_t *p4d_pgtable(p4d_t p4d)
static inline phys_addr_t pud_offset_phys(p4d_t *p4dp, unsigned long addr)
{
BUG_ON(!pgtable_l4_enabled());
VM_WARN_ON_ONCE(!pgtable_l4_enabled());
return p4d_page_paddr(READ_ONCE(*p4dp)) + pud_index(addr) * sizeof(pud_t);
}
@ -1130,7 +1130,7 @@ static inline p4d_t *pgd_to_folded_p4d(pgd_t *pgdp, unsigned long addr)
static inline phys_addr_t p4d_offset_phys(pgd_t *pgdp, unsigned long addr)
{
BUG_ON(!pgtable_l5_enabled());
VM_WARN_ON_ONCE(!pgtable_l5_enabled());
return pgd_page_paddr(READ_ONCE(*pgdp)) + p4d_index(addr) * sizeof(p4d_t);
}

7
arch/arm64/include/asm/processor.h
View File

@ -130,6 +130,9 @@ enum fp_type {
FP_STATE_SVE,
};
struct arm64_sve_state; /* Opaque type */
struct arm64_sme_state; /* Opaque type */
struct cpu_context {
unsigned long x19;
unsigned long x20;
@ -164,8 +167,8 @@ struct thread_struct {
enum fp_type fp_type; /* registers FPSIMD or SVE? */
unsigned int fpsimd_cpu;
void *sve_state; /* SVE registers, if any */
void *sme_state; /* ZA and ZT state, if any */
struct arm64_sve_state *sve_state; /* SVE registers, if any */
struct arm64_sme_state *sme_state; /* ZA and ZT state, if any */
unsigned int vl[ARM64_VEC_MAX]; /* vector length */
unsigned int vl_onexec[ARM64_VEC_MAX]; /* vl after next exec */
unsigned long fault_address; /* fault info */

8
arch/arm64/include/asm/ptdump.h
View File

@ -26,8 +26,8 @@ struct ptdump_info {
};
struct ptdump_prot_bits {
ptdesc_t mask;
ptdesc_t val;
ptval_t mask;
ptval_t val;
const char *set;
const char *clear;
};
@ -36,7 +36,7 @@ struct ptdump_pg_level {
const struct ptdump_prot_bits *bits;
char name[4];
int num;
ptdesc_t mask;
ptval_t mask;
};
/*
@ -53,7 +53,7 @@ struct ptdump_pg_state {
const struct mm_struct *mm;
unsigned long start_address;
int level;
ptdesc_t current_prot;
ptval_t current_prot;
bool check_wx;
unsigned long wx_pages;
unsigned long uxn_pages;

4
arch/arm64/include/asm/tlbflush.h
View File

@ -725,9 +725,9 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b
sme_dvmsync_add_pending(batch, mm);
}
static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval)
static inline bool __pte_flags_need_flush(ptval_t oldval, ptval_t newval)
{
ptdesc_t diff = oldval ^ newval;
ptval_t diff = oldval ^ newval;
/* invalid to valid transition requires no flush */
if (!(oldval & PTE_VALID))

8
arch/arm64/include/uapi/asm/hwcap.h
View File

@ -147,5 +147,13 @@
#define HWCAP3_MTE_STORE_ONLY (1UL << 1)
#define HWCAP3_LSFE (1UL << 2)
#define HWCAP3_LS64 (1UL << 3)
#define HWCAP3_SVE_B16MM (1UL << 4)
#define HWCAP3_SVE2P3 (1UL << 5)
#define HWCAP3_SME_LUT6 (1UL << 6)
#define HWCAP3_SME2P3 (1UL << 7)
#define HWCAP3_F16MM (1UL << 8)
#define HWCAP3_F16F32DOT (1UL << 9)
#define HWCAP3_F16F32MM (1UL << 10)
#define HWCAP3_SVE_LUT6 (1UL << 11)
#endif /* _UAPI__ASM_HWCAP_H */

2
arch/arm64/kernel/Makefile
View File

@ -27,7 +27,7 @@ KCOV_INSTRUMENT_idle.o := n
# Object file lists.
obj-y := debug-monitors.o entry.o irq.o fpsimd.o \
entry-common.o entry-fpsimd.o process.o ptrace.o \
entry-common.o process.o ptrace.o \
setup.o signal.o sys.o stacktrace.o time.o traps.o \
io.o vdso.o hyp-stub.o psci.o cpu_ops.o \
return_address.o cpuinfo.o cpu_errata.o \

55
arch/arm64/kernel/cpu_errata.c
View File

@ -340,7 +340,37 @@ static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = {
ERRATA_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1),
},
#endif
{},
#ifdef CONFIG_ARM64_ERRATUM_4118414
{
ERRATA_MIDR_RANGE_LIST(((const struct midr_range[]) {
MIDR_ALL_VERSIONS(MIDR_C1_PREMIUM),
MIDR_ALL_VERSIONS(MIDR_C1_ULTRA),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A76),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A76AE),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A77),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A78),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A710),
MIDR_ALL_VERSIONS(MIDR_CORTEX_X1),
MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C),
MIDR_ALL_VERSIONS(MIDR_CORTEX_X2),
MIDR_ALL_VERSIONS(MIDR_CORTEX_X3),
MIDR_ALL_VERSIONS(MIDR_CORTEX_X4),
MIDR_ALL_VERSIONS(MIDR_CORTEX_X925),
MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1),
MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1),
MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2),
MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3),
MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3AE),
MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS),
MIDR_ALL_VERSIONS(MIDR_MICROSOFT_AZURE_COBALT_100),
{}
})),
},
#endif
{}
};
#endif
@ -608,6 +638,18 @@ static const struct midr_range erratum_ac04_cpu_23_list[] = {
};
#endif
#ifdef CONFIG_ARM64_WORKAROUND_DISABLE_CNP
static const struct midr_range cnp_erratum_cpus[] = {
#ifdef CONFIG_NVIDIA_CARMEL_CNP_ERRATUM
MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL),
#endif
#ifdef CONFIG_HISILICON_ERRATUM_162100125
MIDR_ALL_VERSIONS(MIDR_HISI_HIP09),
#endif
{},
};
#endif
const struct arm64_cpu_capabilities arm64_errata[] = {
#ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
{
@ -693,7 +735,7 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
#endif
#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI
{
.desc = "Qualcomm erratum 1009, or ARM erratum 1286807, 2441009",
.desc = "Broken broadcast TLBI completion",
.capability = ARM64_WORKAROUND_REPEAT_TLBI,
.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
.matches = cpucap_multi_entry_cap_matches,
@ -801,12 +843,11 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
1, 0),
},
#endif
#ifdef CONFIG_NVIDIA_CARMEL_CNP_ERRATUM
#ifdef CONFIG_ARM64_WORKAROUND_DISABLE_CNP
{
/* NVIDIA Carmel */
.desc = "NVIDIA Carmel CNP erratum",
.capability = ARM64_WORKAROUND_NVIDIA_CARMEL_CNP,
ERRATA_MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL),
.desc = "NVIDIA Carmel CNP erratum, or Hisilicon erratum 162100125",
.capability = ARM64_WORKAROUND_DISABLE_CNP,
ERRATA_MIDR_RANGE_LIST(cnp_erratum_cpus),
},
#endif
#ifdef CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE

28
arch/arm64/kernel/cpufeature.c
View File

@ -365,6 +365,8 @@ static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = {
static const struct arm64_ftr_bits ftr_id_aa64smfr0[] = {
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_FA64_SHIFT, 1, 0),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_LUT6_SHIFT, 1, 0),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_LUTv2_SHIFT, 1, 0),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
@ -419,6 +421,7 @@ static const struct arm64_ftr_bits ftr_id_aa64fpfr0[] = {
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8DP2_SHIFT, 1, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8MM8_SHIFT, 1, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8MM4_SHIFT, 1, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F16MM2_SHIFT, 1, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E4M3_SHIFT, 1, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E5M2_SHIFT, 1, 0),
ARM64_FTR_END,
@ -1164,6 +1167,14 @@ static __init void detect_system_supports_pseudo_nmi(void)
static inline void detect_system_supports_pseudo_nmi(void) { }
#endif
static bool detect_ftr_has_mpam(void)
{
u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
u64 pfr1 = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1);
return id_aa64pfr0_mpam(pfr0) || id_aa64pfr1_mpamfrac(pfr1);
}
void __init init_cpu_features(struct cpuinfo_arm64 *info)
{
/* Before we start using the tables, make sure it is sorted */
@ -1211,7 +1222,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info)
cpacr_restore(cpacr);
}
if (id_aa64pfr0_mpam(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) {
if (detect_ftr_has_mpam()) {
info->reg_mpamidr = read_cpuid(MPAMIDR_EL1);
init_cpu_ftr_reg(SYS_MPAMIDR_EL1, info->reg_mpamidr);
}
@ -1467,7 +1478,7 @@ void update_cpu_features(int cpu,
cpacr_restore(cpacr);
}
if (id_aa64pfr0_mpam(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) {
if (detect_ftr_has_mpam()) {
info->reg_mpamidr = read_cpuid(MPAMIDR_EL1);
taint |= check_update_ftr_reg(SYS_MPAMIDR_EL1, cpu,
info->reg_mpamidr, boot->reg_mpamidr);
@ -1785,7 +1796,7 @@ has_useable_cnp(const struct arm64_cpu_capabilities *entry, int scope)
if (is_kdump_kernel())
return false;
if (cpus_have_cap(ARM64_WORKAROUND_NVIDIA_CARMEL_CNP))
if (cpus_have_cap(ARM64_WORKAROUND_DISABLE_CNP))
return false;
return has_cpuid_feature(entry, scope);
@ -2486,7 +2497,7 @@ cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap)
static bool
test_has_mpam(const struct arm64_cpu_capabilities *entry, int scope)
{
if (!has_cpuid_feature(entry, scope))
if (!detect_ftr_has_mpam())
return false;
/* Check firmware actually enabled MPAM on this cpu. */
@ -3093,7 +3104,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.capability = ARM64_MPAM,
.matches = test_has_mpam,
.cpu_enable = cpu_enable_mpam,
ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, MPAM, 1)
},
{
.desc = "Memory Partitioning And Monitoring Virtualisation",
@ -3284,6 +3294,8 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
HWCAP_CAP(ID_AA64ISAR0_EL1, SM4, IMP, CAP_HWCAP, KERNEL_HWCAP_SM4),
HWCAP_CAP(ID_AA64ISAR0_EL1, DP, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDDP),
HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDFHM),
HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, F16F32DOT, CAP_HWCAP, KERNEL_HWCAP_F16F32DOT),
HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, F16F32MM, CAP_HWCAP, KERNEL_HWCAP_F16F32MM),
HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM, CAP_HWCAP, KERNEL_HWCAP_FLAGM),
HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM2, CAP_HWCAP, KERNEL_HWCAP_FLAGM2),
HWCAP_CAP(ID_AA64ISAR0_EL1, RNDR, IMP, CAP_HWCAP, KERNEL_HWCAP_RNG),
@ -3313,7 +3325,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
HWCAP_CAP(ID_AA64ISAR3_EL1, LSFE, IMP, CAP_HWCAP, KERNEL_HWCAP_LSFE),
HWCAP_CAP(ID_AA64MMFR2_EL1, AT, IMP, CAP_HWCAP, KERNEL_HWCAP_USCAT),
#ifdef CONFIG_ARM64_SVE
HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ISAR2_EL1, LUT, LUT6, CAP_HWCAP, KERNEL_HWCAP_SVE_LUT6),
HWCAP_CAP(ID_AA64PFR0_EL1, SVE, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE),
HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p3, CAP_HWCAP, KERNEL_HWCAP_SVE2P3),
HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p2, CAP_HWCAP, KERNEL_HWCAP_SVE2P2),
HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p1, CAP_HWCAP, KERNEL_HWCAP_SVE2P1),
HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2),
@ -3323,6 +3337,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BitPerm, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBITPERM),
HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE_B16B16),
HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, BFSCALE, CAP_HWCAP, KERNEL_HWCAP_SVE_BFSCALE),
HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, B16MM, CAP_HWCAP, KERNEL_HWCAP_SVE_B16MM),
HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBF16),
HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, EBF16, CAP_HWCAP, KERNEL_HWCAP_SVE_EBF16),
HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SHA3, IMP, CAP_HWCAP, KERNEL_HWCAP_SVESHA3),
@ -3362,7 +3377,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
#ifdef CONFIG_ARM64_SME
HWCAP_CAP(ID_AA64PFR1_EL1, SME, IMP, CAP_HWCAP, KERNEL_HWCAP_SME),
HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, FA64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64),
HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, LUT6, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_LUT6),
HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, LUTv2, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_LUTV2),
HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p3, CAP_HWCAP, KERNEL_HWCAP_SME2P3),
HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p2, CAP_HWCAP, KERNEL_HWCAP_SME2P2),
HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p1, CAP_HWCAP, KERNEL_HWCAP_SME2P1),
HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2, CAP_HWCAP, KERNEL_HWCAP_SME2),
@ -3393,6 +3410,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
HWCAP_CAP(ID_AA64FPFR0_EL1, F8DP2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8DP2),
HWCAP_CAP(ID_AA64FPFR0_EL1, F8MM8, IMP, CAP_HWCAP, KERNEL_HWCAP_F8MM8),
HWCAP_CAP(ID_AA64FPFR0_EL1, F8MM4, IMP, CAP_HWCAP, KERNEL_HWCAP_F8MM4),
HWCAP_CAP(ID_AA64FPFR0_EL1, F16MM2, IMP, CAP_HWCAP, KERNEL_HWCAP_F16MM),
HWCAP_CAP(ID_AA64FPFR0_EL1, F8E4M3, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E4M3),
HWCAP_CAP(ID_AA64FPFR0_EL1, F8E5M2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E5M2),
#ifdef CONFIG_ARM64_POE

8
arch/arm64/kernel/cpuinfo.c
View File

@ -164,6 +164,14 @@ static const char *const hwcap_str[] = {
[KERNEL_HWCAP_MTE_FAR] = "mtefar",
[KERNEL_HWCAP_MTE_STORE_ONLY] = "mtestoreonly",
[KERNEL_HWCAP_LSFE] = "lsfe",
[KERNEL_HWCAP_SVE_B16MM] = "sveb16mm",
[KERNEL_HWCAP_SVE2P3] = "sve2p3",
[KERNEL_HWCAP_SME_LUT6] = "smelut6",
[KERNEL_HWCAP_SME2P3] = "sme2p3",
[KERNEL_HWCAP_F16MM] = "f16mm",
[KERNEL_HWCAP_F16F32DOT] = "f16f32dot",
[KERNEL_HWCAP_F16F32MM] = "f16f32mm",
[KERNEL_HWCAP_SVE_LUT6] = "svelut6",
};
#ifdef CONFIG_COMPAT

4
arch/arm64/kernel/efi.c
View File

@ -31,7 +31,7 @@ static bool region_is_misaligned(const efi_memory_desc_t *md)
* executable, everything else can be mapped with the XN bits
* set. Also take the new (optional) RO/XP bits into account.
*/
static __init ptdesc_t create_mapping_protection(efi_memory_desc_t *md)
static __init ptval_t create_mapping_protection(efi_memory_desc_t *md)
{
u64 attr = md->attribute;
u32 type = md->type;
@ -85,7 +85,7 @@ static __init ptdesc_t create_mapping_protection(efi_memory_desc_t *md)
int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md)
{
ptdesc_t prot_val = create_mapping_protection(md);
ptval_t prot_val = create_mapping_protection(md);
bool page_mappings_only = (md->type == EFI_RUNTIME_SERVICES_CODE ||
md->type == EFI_RUNTIME_SERVICES_DATA);

8
arch/arm64/kernel/entry-common.c
View File

@ -254,12 +254,8 @@ static inline void fpsimd_syscall_enter(void)
if (!system_supports_sve())
return;
if (test_thread_flag(TIF_SVE)) {
unsigned int sve_vq_minus_one;
sve_vq_minus_one = sve_vq_from_vl(task_get_sve_vl(current)) - 1;
sve_flush_live(true, sve_vq_minus_one);
}
if (test_thread_flag(TIF_SVE))
sve_flush_live();
/*
* Any live non-FPSIMD SVE state has been zeroed. Allow

134
arch/arm64/kernel/entry-fpsimd.S
View File

@ -1,134 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* FP/SIMD state saving and restoring
*
* Copyright (C) 2012 ARM Ltd.
* Author: Catalin Marinas <catalin.marinas@arm.com>
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/fpsimdmacros.h>
/*
* Save the FP registers.
*
* x0 - pointer to struct fpsimd_state
*/
SYM_FUNC_START(fpsimd_save_state)
fpsimd_save x0, 8
ret
SYM_FUNC_END(fpsimd_save_state)
/*
* Load the FP registers.
*
* x0 - pointer to struct fpsimd_state
*/
SYM_FUNC_START(fpsimd_load_state)
fpsimd_restore x0, 8
ret
SYM_FUNC_END(fpsimd_load_state)
#ifdef CONFIG_ARM64_SVE
/*
* Save the SVE state
*
* x0 - pointer to buffer for state
* x1 - pointer to storage for FPSR
* x2 - Save FFR if non-zero
*/
SYM_FUNC_START(sve_save_state)
sve_save 0, x1, x2, 3
ret
SYM_FUNC_END(sve_save_state)
/*
* Load the SVE state
*
* x0 - pointer to buffer for state
* x1 - pointer to storage for FPSR
* x2 - Restore FFR if non-zero
*/
SYM_FUNC_START(sve_load_state)
sve_load 0, x1, x2, 4
ret
SYM_FUNC_END(sve_load_state)
SYM_FUNC_START(sve_get_vl)
_sve_rdvl 0, 1
ret
SYM_FUNC_END(sve_get_vl)
SYM_FUNC_START(sve_set_vq)
sve_load_vq x0, x1, x2
ret
SYM_FUNC_END(sve_set_vq)
/*
* Zero all SVE registers but the first 128-bits of each vector
*
* VQ must already be configured by caller, any further updates of VQ
* will need to ensure that the register state remains valid.
*
* x0 = include FFR?
* x1 = VQ - 1
*/
SYM_FUNC_START(sve_flush_live)
cbz x1, 1f // A VQ-1 of 0 is 128 bits so no extra Z state
sve_flush_z
1: sve_flush_p
tbz x0, #0, 2f
sve_flush_ffr
2: ret
SYM_FUNC_END(sve_flush_live)
#endif /* CONFIG_ARM64_SVE */
#ifdef CONFIG_ARM64_SME
SYM_FUNC_START(sme_get_vl)
_sme_rdsvl 0, 1
ret
SYM_FUNC_END(sme_get_vl)
SYM_FUNC_START(sme_set_vq)
sme_load_vq x0, x1, x2
ret
SYM_FUNC_END(sme_set_vq)
/*
* Save the ZA and ZT state
*
* x0 - pointer to buffer for state
* x1 - number of ZT registers to save
*/
SYM_FUNC_START(sme_save_state)
_sme_rdsvl 2, 1 // x2 = VL/8
sme_save_za 0, x2, 12 // Leaves x0 pointing to the end of ZA
cbz x1, 1f
_str_zt 0
1:
ret
SYM_FUNC_END(sme_save_state)
/*
* Load the ZA and ZT state
*
* x0 - pointer to buffer for state
* x1 - number of ZT registers to save
*/
SYM_FUNC_START(sme_load_state)
_sme_rdsvl 2, 1 // x2 = VL/8
sme_load_za 0, x2, 12 // Leaves x0 pointing to the end of ZA
cbz x1, 1f
_ldr_zt 0
1:
ret
SYM_FUNC_END(sme_load_state)
#endif /* CONFIG_ARM64_SME */

90
arch/arm64/kernel/fpsimd.c
View File

@ -377,8 +377,10 @@ static void task_fpsimd_load(void)
if (!thread_sm_enabled(&current->thread))
WARN_ON_ONCE(!test_and_set_thread_flag(TIF_SVE));
if (test_thread_flag(TIF_SVE))
sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1);
if (test_thread_flag(TIF_SVE)) {
unsigned long vq = sve_vq_from_vl(task_get_sve_vl(current));
sysreg_clear_set_s(SYS_ZCR_EL1, ZCR_ELx_LEN, vq - 1);
}
restore_sve_regs = true;
restore_ffr = true;
@ -403,8 +405,10 @@ static void task_fpsimd_load(void)
unsigned long sme_vl = task_get_sme_vl(current);
/* Ensure VL is set up for restoring data */
if (test_thread_flag(TIF_SME))
sme_set_vq(sve_vq_from_vl(sme_vl) - 1);
if (test_thread_flag(TIF_SME)) {
unsigned long vq = sve_vq_from_vl(sme_vl);
sysreg_clear_set_s(SYS_SMCR_EL1, SMCR_ELx_LEN, vq - 1);
}
write_sysreg_s(current->thread.svcr, SYS_SVCR);
@ -421,9 +425,8 @@ static void task_fpsimd_load(void)
if (restore_sve_regs) {
WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE);
sve_load_state(sve_pffr(&current->thread),
&current->thread.uw.fpsimd_state.fpsr,
restore_ffr);
sve_load_state(current->thread.sve_state, restore_ffr);
fpsimd_load_common(&current->thread.uw.fpsimd_state);
} else {
WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD);
fpsimd_load_state(&current->thread.uw.fpsimd_state);
@ -503,9 +506,8 @@ static void fpsimd_save_user_state(void)
return;
}
sve_save_state((char *)last->sve_state +
sve_ffr_offset(vl),
&last->st->fpsr, save_ffr);
sve_save_state(last->sve_state, save_ffr);
fpsimd_save_common(last->st);
*last->fp_type = FP_STATE_SVE;
} else {
fpsimd_save_state(last->st);
@ -636,7 +638,8 @@ static __uint128_t arm64_cpu_to_le128(__uint128_t x)
#define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x)
static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst,
static void __fpsimd_to_sve(struct arm64_sve_state *sst,
struct user_fpsimd_state const *fst,
unsigned int vq)
{
unsigned int i;
@ -663,7 +666,7 @@ static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst,
static inline void fpsimd_to_sve(struct task_struct *task)
{
unsigned int vq;
void *sst = task->thread.sve_state;
struct arm64_sve_state *sst = task->thread.sve_state;
struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
if (!system_supports_sve() && !system_supports_sme())
@ -687,7 +690,7 @@ static inline void fpsimd_to_sve(struct task_struct *task)
static inline void sve_to_fpsimd(struct task_struct *task)
{
unsigned int vq, vl;
void const *sst = task->thread.sve_state;
const struct arm64_sve_state *sst = task->thread.sve_state;
struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;
unsigned int i;
__uint128_t const *p;
@ -786,7 +789,7 @@ void fpsimd_sync_from_effective_state(struct task_struct *task)
void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task)
{
unsigned int vq;
void *sst = task->thread.sve_state;
struct arm64_sve_state *sst = task->thread.sve_state;
struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
if (task->thread.fp_type != FP_STATE_SVE)
@ -804,7 +807,8 @@ static int change_live_vector_length(struct task_struct *task,
{
unsigned int sve_vl = task_get_sve_vl(task);
unsigned int sme_vl = task_get_sme_vl(task);
void *sve_state = NULL, *sme_state = NULL;
struct arm64_sve_state *sve_state = NULL;
struct arm64_sme_state *sme_state = NULL;
if (type == ARM64_VEC_SME)
sme_vl = vl;
@ -1293,31 +1297,6 @@ void sme_suspend_exit(void)
#endif /* CONFIG_ARM64_SME */
static void sve_init_regs(void)
{
/*
* Convert the FPSIMD state to SVE, zeroing all the state that
* is not shared with FPSIMD. If (as is likely) the current
* state is live in the registers then do this there and
* update our metadata for the current task including
* disabling the trap, otherwise update our in-memory copy.
* We are guaranteed to not be in streaming mode, we can only
* take a SVE trap when not in streaming mode and we can't be
* in streaming mode when taking a SME trap.
*/
if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
unsigned long vq_minus_one =
sve_vq_from_vl(task_get_sve_vl(current)) - 1;
sve_set_vq(vq_minus_one);
sve_flush_live(true, vq_minus_one);
fpsimd_bind_task_to_cpu();
} else {
fpsimd_to_sve(current);
current->thread.fp_type = FP_STATE_SVE;
fpsimd_flush_task_state(current);
}
}
/*
* Trapped SVE access
*
@ -1349,13 +1328,23 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs)
WARN_ON(1); /* SVE access shouldn't have trapped */
/*
* Even if the task can have used streaming mode we can only
* generate SVE access traps in normal SVE mode and
* transitioning out of streaming mode may discard any
* streaming mode state. Always clear the high bits to avoid
* any potential errors tracking what is properly initialised.
* Convert the FPSIMD state to SVE. Stale SVE state can be present in
* registers or memory, so we must zero all state that is not shared
* with FPSIMD.
*
* SVE traps cannot be taken from streaming mode, so there cannot be
* any effective streaming mode SVE state.
*/
sve_init_regs();
if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
unsigned long vq = sve_vq_from_vl(task_get_sve_vl(current));
sysreg_clear_set_s(SYS_ZCR_EL1, ZCR_ELx_LEN, vq - 1);
sve_flush_live();
fpsimd_bind_task_to_cpu();
} else {
fpsimd_to_sve(current);
current->thread.fp_type = FP_STATE_SVE;
fpsimd_flush_task_state(current);
}
put_cpu_fpsimd_context();
}
@ -1479,9 +1468,8 @@ void do_sme_acc(unsigned long esr, struct pt_regs *regs)
WARN_ON(1);
if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
unsigned long vq_minus_one =
sve_vq_from_vl(task_get_sme_vl(current)) - 1;
sme_set_vq(vq_minus_one);
unsigned long vq = sve_vq_from_vl(task_get_sme_vl(current));
sysreg_clear_set_s(SYS_SMCR_EL1, SMCR_ELx_LEN, vq - 1);
fpsimd_bind_task_to_cpu();
} else {
@ -1656,8 +1644,8 @@ static void fpsimd_flush_thread_vl(enum vec_type type)
void fpsimd_flush_thread(void)
{
void *sve_state = NULL;
void *sme_state = NULL;
struct arm64_sve_state *sve_state = NULL;
struct arm64_sme_state *sme_state = NULL;
if (!system_supports_fpsimd())
return;

29
arch/arm64/kernel/irq.c
View File

@ -10,6 +10,7 @@
* Copyright (C) 2012 ARM Ltd.
*/
#include <linux/errno.h>
#include <linux/hardirq.h>
#include <linux/init.h>
#include <linux/irq.h>
@ -32,34 +33,43 @@ DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts);
DEFINE_PER_CPU(unsigned long *, irq_stack_ptr);
DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
#ifdef CONFIG_SHADOW_CALL_STACK
DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
#endif
static void init_irq_scs(void)
static int __init init_irq_scs(void)
{
int cpu;
void *s;
if (!scs_is_enabled())
return;
return 0;
for_each_possible_cpu(cpu)
per_cpu(irq_shadow_call_stack_ptr, cpu) =
scs_alloc(early_cpu_to_node(cpu));
for_each_possible_cpu(cpu) {
s = scs_alloc(early_cpu_to_node(cpu));
if (!s)
return -ENOMEM;
per_cpu(irq_shadow_call_stack_ptr, cpu) = s;
}
return 0;
}
static void __init init_irq_stacks(void)
static int __init init_irq_stacks(void)
{
int cpu;
unsigned long *p;
for_each_possible_cpu(cpu) {
p = arch_alloc_vmap_stack(IRQ_STACK_SIZE, early_cpu_to_node(cpu));
if (!p)
return -ENOMEM;
per_cpu(irq_stack_ptr, cpu) = p;
}
return 0;
}
#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK
@ -109,8 +119,9 @@ int __init set_handle_fiq(void (*handle_fiq)(struct pt_regs *))
void __init init_IRQ(void)
{
init_irq_stacks();
init_irq_scs();
if (init_irq_stacks() || init_irq_scs())
panic("Failed to allocate IRQ stack resources\n");
irqchip_init();
if (system_uses_irq_prio_masking()) {

3
arch/arm64/kernel/patching.c
View File

@ -116,8 +116,7 @@ static void *__text_poke(text_poke_f func, void *addr, void *src, size_t len)
while (patched < len) {
ptr = addr + patched;
size = min_t(size_t, PAGE_SIZE - offset_in_page(ptr),
len - patched);
size = min(PAGE_SIZE - offset_in_page(ptr), len - patched);
waddr = patch_map(ptr, FIX_TEXT_POKE0);
func(waddr, src, patched, size);

2
arch/arm64/kernel/pi/map_kernel.c
View File

@ -165,7 +165,7 @@ static void noinline __section(".idmap.text") set_ttbr0_for_lpa2(phys_addr_t ttb
static void __init remap_idmap_for_lpa2(void)
{
/* clear the bits that change meaning once LPA2 is turned on */
ptdesc_t mask = PTE_SHARED;
ptval_t mask = PTE_SHARED;
/*
* We have to clear bits [9:8] in all block or page descriptors in the

4
arch/arm64/kernel/pi/map_range.c
View File

@ -31,7 +31,7 @@ void __init map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa,
u64 va_offset)
{
u64 cmask = (level == 3) ? CONT_PTE_SIZE - 1 : U64_MAX;
ptdesc_t protval = pgprot_val(prot) & ~PTE_TYPE_MASK;
ptval_t protval = pgprot_val(prot) & ~PTE_TYPE_MASK;
int lshift = (3 - level) * PTDESC_TABLE_SHIFT;
u64 lmask = (PAGE_SIZE << lshift) - 1;
@ -88,7 +88,7 @@ void __init map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa,
}
}
asmlinkage phys_addr_t __init create_init_idmap(pgd_t *pg_dir, ptdesc_t clrmask)
asmlinkage phys_addr_t __init create_init_idmap(pgd_t *pg_dir, ptval_t clrmask)
{
phys_addr_t ptep = (phys_addr_t)pg_dir + PAGE_SIZE; /* MMU is off */
pgprot_t text_prot = PAGE_KERNEL_ROX;

2
arch/arm64/kernel/pi/pi.h
View File

@ -35,4 +35,4 @@ void map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa,
asmlinkage void early_map_kernel(u64 boot_status, phys_addr_t fdt);
asmlinkage phys_addr_t create_init_idmap(pgd_t *pgd, ptdesc_t clrmask);
asmlinkage phys_addr_t create_init_idmap(pgd_t *pgd, ptval_t clrmask);

17
arch/arm64/kernel/proton-pack.c
View File

@ -24,6 +24,7 @@
#include <linux/nospec.h>
#include <linux/prctl.h>
#include <linux/sched/task_stack.h>
#include <linux/sysfs.h>
#include <asm/debug-monitors.h>
#include <asm/insn.h>
@ -61,7 +62,7 @@ static void update_mitigation_state(enum mitigation_state *oldp,
ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr,
char *buf)
{
return sprintf(buf, "Mitigation: __user pointer sanitization\n");
return sysfs_emit(buf, "Mitigation: __user pointer sanitization\n");
}
/*
@ -126,7 +127,7 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
switch (spectre_v2_state) {
case SPECTRE_UNAFFECTED:
if (bhb_state == SPECTRE_UNAFFECTED)
return sprintf(buf, "Not affected\n");
return sysfs_emit(buf, "Not affected\n");
/*
* Platforms affected by Spectre-BHB can't report
@ -136,13 +137,13 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
fallthrough;
case SPECTRE_MITIGATED:
if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled())
return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n");
return sysfs_emit(buf, "Vulnerable: Unprivileged eBPF enabled\n");
return sprintf(buf, "Mitigation: %s%s\n", v2_str, bhb_str);
return sysfs_emit(buf, "Mitigation: %s%s\n", v2_str, bhb_str);
case SPECTRE_VULNERABLE:
fallthrough;
default:
return sprintf(buf, "Vulnerable\n");
return sysfs_emit(buf, "Vulnerable\n");
}
}
@ -438,13 +439,13 @@ ssize_t cpu_show_spec_store_bypass(struct device *dev,
{
switch (spectre_v4_state) {
case SPECTRE_UNAFFECTED:
return sprintf(buf, "Not affected\n");
return sysfs_emit(buf, "Not affected\n");
case SPECTRE_MITIGATED:
return sprintf(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n");
return sysfs_emit(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n");
case SPECTRE_VULNERABLE:
fallthrough;
default:
return sprintf(buf, "Vulnerable\n");
return sysfs_emit(buf, "Vulnerable\n");
}
}

14
arch/arm64/kernel/smp.c
View File

@ -745,15 +745,21 @@ void __init smp_init_cpus(void)
else
acpi_parse_and_init_cpus();
if (cpu_count > nr_cpu_ids)
pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n",
cpu_count, nr_cpu_ids);
if (!bootcpu_valid) {
pr_err("missing boot CPU MPIDR, not enabling secondaries\n");
return;
}
/*
* For the nosmp/maxcpus=0 case, do not mark the secondary CPUs
* possible.
*/
if (!setup_max_cpus)
return;
if (cpu_count > nr_cpu_ids)
pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n",
cpu_count, nr_cpu_ids);
/*
* We need to set the cpu_logical_map entries before enabling
* the cpus so that cpu processor description entries (DT cpu nodes

8
arch/arm64/kernel/vmlinux.lds.S
View File

@ -349,9 +349,15 @@ SECTIONS
_edata = .;
/* start of zero-init region */
BSS_SECTION(SBSS_ALIGN, 0, 0)
BSS_SECTION(SBSS_ALIGN, 0, PAGE_SIZE)
__pi___bss_start = __bss_start;
/* page table BSS starts here - preceding data/BSS is omitted from the linear map */
.pgtbl : ALIGN(PAGE_SIZE) {
*(.bss..pgtbl)
}
ASSERT(ADDR(.pgtbl) == __bss_stop, ".pgtbl must follow BSS")
. = ALIGN(PAGE_SIZE);
__pi_init_pg_dir = .;
. += INIT_DIR_SIZE;

16
arch/arm64/kvm/arm.c
View File

@ -2502,10 +2502,10 @@ static void __init teardown_hyp_mode(void)
continue;
if (free_sve) {
struct cpu_sve_state *sve_state;
struct arm64_sve_state *sve_regs;
sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state;
free_pages((unsigned long) sve_state, pkvm_host_sve_state_order());
sve_regs = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs;
free_pages((unsigned long) sve_regs, pkvm_host_sve_state_order());
}
free_pages(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu], nvhe_percpu_order());
@ -2630,7 +2630,7 @@ static int init_pkvm_host_sve_state(void)
if (!page)
return -ENOMEM;
per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = page_address(page);
per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs = page_address(page);
}
/*
@ -2667,11 +2667,11 @@ static void finalize_init_hyp_mode(void)
if (system_supports_sve() && is_protected_kvm_enabled()) {
for_each_possible_cpu(cpu) {
struct cpu_sve_state *sve_state;
struct arm64_sve_state *sve_regs;
sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state;
per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state =
kern_hyp_va(sve_state);
sve_regs = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs;
per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs =
kern_hyp_va(sve_regs);
}
}
}

4
arch/arm64/kvm/guest.c
View File

@ -500,7 +500,7 @@ static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
if (!kvm_arm_vcpu_sve_finalized(vcpu))
return -EPERM;
if (copy_to_user(uptr, vcpu->arch.sve_state + region.koffset,
if (copy_to_user(uptr, (void *)vcpu->arch.sve_state + region.koffset,
region.klen) ||
clear_user(uptr + region.klen, region.upad))
return -EFAULT;
@ -526,7 +526,7 @@ static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
if (!kvm_arm_vcpu_sve_finalized(vcpu))
return -EPERM;
if (copy_from_user(vcpu->arch.sve_state + region.koffset, uptr,
if (copy_from_user((void *)vcpu->arch.sve_state + region.koffset, uptr,
region.klen))
return -EFAULT;

1
arch/arm64/kvm/hyp/entry.S
View File

@ -8,7 +8,6 @@
#include <asm/alternative.h>
#include <asm/assembler.h>
#include <asm/fpsimdmacros.h>
#include <asm/kvm.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_asm.h>

33
arch/arm64/kvm/hyp/fpsimd.S
View File

@ -1,33 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2015 - ARM Ltd
* Author: Marc Zyngier <marc.zyngier@arm.com>
*/
#include <linux/linkage.h>
#include <asm/fpsimdmacros.h>
.text
SYM_FUNC_START(__fpsimd_save_state)
fpsimd_save x0, 1
ret
SYM_FUNC_END(__fpsimd_save_state)
SYM_FUNC_START(__fpsimd_restore_state)
fpsimd_restore x0, 1
ret
SYM_FUNC_END(__fpsimd_restore_state)
SYM_FUNC_START(__sve_restore_state)
mov x2, #1
sve_load 0, x1, x2, 3
ret
SYM_FUNC_END(__sve_restore_state)
SYM_FUNC_START(__sve_save_state)
mov x2, #1
sve_save 0, x1, x2, 3
ret
SYM_FUNC_END(__sve_save_state)

23
arch/arm64/kvm/hyp/include/hyp/switch.h
View File

@ -471,9 +471,8 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
* vCPU. Start off with the max VL so we can load the SVE state.
*/
sve_cond_update_zcr_vq(zcr_el2, SYS_ZCR_EL2);
__sve_restore_state(vcpu_sve_pffr(vcpu),
&vcpu->arch.ctxt.fp_regs.fpsr,
true);
sve_load_state(kern_hyp_va(vcpu->arch.sve_state), true);
fpsimd_load_common(&vcpu->arch.ctxt.fp_regs);
/*
* The effective VL for a VM could differ from the max VL when running a
@ -490,13 +489,13 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
static inline void __hyp_sve_save_host(void)
{
struct cpu_sve_state *sve_state = *host_data_ptr(sve_state);
struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt);
struct arm64_sve_state *sve_regs = *host_data_ptr(sve_regs);
sve_state->zcr_el1 = read_sysreg_el1(SYS_ZCR);
ctxt_sys_reg(hctxt, ZCR_EL1) = read_sysreg_el1(SYS_ZCR);
write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2);
__sve_save_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_sve_max_vl),
&sve_state->fpsr,
true);
sve_save_state(sve_regs, true);
fpsimd_save_common(&hctxt->fp_regs);
}
static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu)
@ -560,6 +559,8 @@ static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu)
static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu)
{
struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt);
/*
* Non-protected kvm relies on the host restoring its sve state.
* Protected kvm restores the host's sve state as not to reveal that
@ -568,11 +569,11 @@ static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu)
if (system_supports_sve()) {
__hyp_sve_save_host();
} else {
__fpsimd_save_state(host_data_ptr(host_ctxt.fp_regs));
fpsimd_save_state(&hctxt->fp_regs);
}
if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm)))
*host_data_ptr(fpmr) = read_sysreg_s(SYS_FPMR);
ctxt_sys_reg(hctxt, FPMR) = read_sysreg_s(SYS_FPMR);
}
@ -628,7 +629,7 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
if (sve_guest)
__hyp_sve_restore_guest(vcpu);
else
__fpsimd_restore_state(&vcpu->arch.ctxt.fp_regs);
fpsimd_load_state(&vcpu->arch.ctxt.fp_regs);
if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm)))
write_sysreg_s(__vcpu_sys_reg(vcpu, FPMR), SYS_FPMR);

2
arch/arm64/kvm/hyp/nvhe/Makefile
View File

@ -26,7 +26,7 @@ hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o
hyp-main.o hyp-smp.o psci-relay.o early_alloc.o page_alloc.o \
cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o
hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o ../vgic-v5-sr.o
../hyp-entry.o ../exception.o ../pgtable.o ../vgic-v5-sr.o
hyp-obj-y += ../../../kernel/smccc-call.o
hyp-obj-$(CONFIG_LIST_HARDENED) += list_debug.o
hyp-obj-$(CONFIG_NVHE_EL2_TRACING) += clock.o trace.o events.o

20
arch/arm64/kvm/hyp/nvhe/hyp-main.c
View File

@ -35,13 +35,15 @@ static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu)
* on the VL, so use a consistent (i.e., the maximum) guest VL.
*/
sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
__sve_save_state(vcpu_sve_pffr(vcpu), &vcpu->arch.ctxt.fp_regs.fpsr, true);
sve_save_state(kern_hyp_va(vcpu->arch.sve_state), true);
fpsimd_save_common(&vcpu->arch.ctxt.fp_regs);
write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2);
}
static void __hyp_sve_restore_host(void)
{
struct cpu_sve_state *sve_state = *host_data_ptr(sve_state);
struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt);
struct arm64_sve_state *sve_regs = *host_data_ptr(sve_regs);
/*
* On saving/restoring host sve state, always use the maximum VL for
@ -53,10 +55,9 @@ static void __hyp_sve_restore_host(void)
* need to be revisited.
*/
write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2);
__sve_restore_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_sve_max_vl),
&sve_state->fpsr,
true);
write_sysreg_el1(sve_state->zcr_el1, SYS_ZCR);
sve_load_state(sve_regs, true);
fpsimd_load_common(&hctxt->fp_regs);
write_sysreg_el1(ctxt_sys_reg(hctxt, ZCR_EL1), SYS_ZCR);
}
static void fpsimd_sve_flush(void)
@ -66,6 +67,7 @@ static void fpsimd_sve_flush(void)
static void fpsimd_sve_sync(struct kvm_vcpu *vcpu)
{
struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt);
bool has_fpmr;
if (!guest_owns_fp_regs())
@ -80,7 +82,7 @@ static void fpsimd_sve_sync(struct kvm_vcpu *vcpu)
if (vcpu_has_sve(vcpu))
__hyp_sve_save_guest(vcpu);
else
__fpsimd_save_state(&vcpu->arch.ctxt.fp_regs);
fpsimd_save_state(&vcpu->arch.ctxt.fp_regs);
has_fpmr = kvm_has_fpmr(kern_hyp_va(vcpu->kvm));
if (has_fpmr)
@ -89,10 +91,10 @@ static void fpsimd_sve_sync(struct kvm_vcpu *vcpu)
if (system_supports_sve())
__hyp_sve_restore_host();
else
__fpsimd_restore_state(host_data_ptr(host_ctxt.fp_regs));
fpsimd_load_state(&hctxt->fp_regs);
if (has_fpmr)
write_sysreg_s(*host_data_ptr(fpmr), SYS_FPMR);
write_sysreg_s(ctxt_sys_reg(hctxt, FPMR), SYS_FPMR);
*host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED;
}

4
arch/arm64/kvm/hyp/nvhe/setup.c
View File

@ -82,9 +82,9 @@ static int pkvm_create_host_sve_mappings(void)
for (i = 0; i < hyp_nr_cpus; i++) {
struct kvm_host_data *host_data = per_cpu_ptr(&kvm_host_data, i);
struct cpu_sve_state *sve_state = host_data->sve_state;
struct arm64_sve_state *sve_regs = host_data->sve_regs;
start = kern_hyp_va(sve_state);
start = kern_hyp_va(sve_regs);
end = start + PAGE_ALIGN(pkvm_host_sve_state_size());
ret = pkvm_create_mappings(start, end, PAGE_HYP);
if (ret)

2
arch/arm64/kvm/hyp/vhe/Makefile
View File

@ -10,4 +10,4 @@ CFLAGS_switch.o += -Wno-override-init
obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o
obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
../fpsimd.o ../hyp-entry.o ../exception.o ../vgic-v5-sr.o
../hyp-entry.o ../exception.o ../vgic-v5-sr.o

5
arch/arm64/kvm/mmu.c
View File

@ -1479,6 +1479,11 @@ static void sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn,
if (!kvm_has_mte(kvm))
return;
if (is_zero_pfn(pfn)) {
WARN_ON_ONCE(nr_pages != 1);
return;
}
if (folio_test_hugetlb(folio)) {
/* Hugetlb has MTE flags set on head page only */
if (folio_try_hugetlb_mte_tagging(folio)) {

6
arch/arm64/mm/fixmap.c
View File

@ -31,9 +31,9 @@ static_assert(NR_BM_PMD_TABLES == 1);
#define BM_PTE_TABLE_IDX(addr) __BM_TABLE_IDX(addr, PMD_SHIFT)
static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __page_aligned_bss;
static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused;
static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused;
static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __bss_pgtbl;
static pmd_t bm_pmd[PTRS_PER_PMD] __bss_pgtbl __maybe_unused;
static pud_t bm_pud[PTRS_PER_PUD] __bss_pgtbl __maybe_unused;
static inline pte_t *fixmap_pte(unsigned long addr)
{

2
arch/arm64/mm/kasan_init.c
View File

@ -214,7 +214,7 @@ asmlinkage void __init kasan_early_init(void)
* shadow pud_t[]/p4d_t[], which could end up getting corrupted
* when the linear region is mapped.
*/
static pte_t tbl[PTRS_PER_PTE] __page_aligned_bss;
static pte_t tbl[PTRS_PER_PTE] __bss_pgtbl;
pgd_t *pgdp = pgd_offset_k(KASAN_SHADOW_START);
set_pgd(pgdp, __pgd(__pa_symbol(tbl) | PGD_TYPE_TABLE));

4
arch/arm64/mm/mmap.c
View File

@ -34,7 +34,7 @@ static pgprot_t protection_map[16] __ro_after_init = {
[VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_SHARED_EXEC
};
static ptdesc_t gcs_page_prot __ro_after_init = _PAGE_GCS_RO;
static ptval_t gcs_page_prot __ro_after_init = _PAGE_GCS_RO;
/*
* You really shouldn't be using read() or write() on /dev/mem. This might go
@ -87,7 +87,7 @@ arch_initcall(adjust_protection_map);
pgprot_t vm_get_page_prot(vm_flags_t vm_flags)
{
ptdesc_t prot;
ptval_t prot;
/* Short circuit GCS to avoid bloating the table. */
if (system_supports_gcs() && (vm_flags & VM_SHADOW_STACK)) {

145
arch/arm64/mm/mmu.c
View File

@ -134,10 +134,6 @@ bool pgattr_change_is_safe(pteval_t old, pteval_t new)
if (pte_pfn(__pte(old)) != pte_pfn(__pte(new)))
return false;
/* live contiguous mappings may not be manipulated at all */
if ((old | new) & PTE_CONT)
return false;
/* Transitioning from Non-Global to Global is unsafe */
if (old & ~new & PTE_NG)
return false;
@ -187,6 +183,17 @@ static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end,
} while (ptep++, addr += PAGE_SIZE, addr != end);
}
static bool pte_range_has_valid_noncont(pte_t *ptep)
{
for (int i = 0; i < CONT_PTES; i++) {
pte_t pte = __ptep_get(&ptep[i]);
if (pte_valid(pte) && !pte_cont(pte))
return true;
}
return false;
}
static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
unsigned long end, phys_addr_t phys,
pgprot_t prot,
@ -224,7 +231,8 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
/* use a contiguous mapping if the range is suitably aligned */
if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) &&
(flags & NO_CONT_MAPPINGS) == 0)
(flags & NO_CONT_MAPPINGS) == 0 &&
!pte_range_has_valid_noncont(ptep))
__prot = __pgprot(pgprot_val(prot) | PTE_CONT);
init_pte(ptep, addr, next, phys, __prot);
@ -256,8 +264,9 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,
/* try section mapping first */
if (((addr | next | phys) & ~PMD_MASK) == 0 &&
(flags & NO_BLOCK_MAPPINGS) == 0) {
pmd_set_huge(pmdp, phys, prot);
(flags & NO_BLOCK_MAPPINGS) == 0 &&
!pmd_table(old_pmd)) {
WARN_ON(!pmd_set_huge(pmdp, phys, prot));
/*
* After the PMD entry has been populated once, we
@ -273,8 +282,8 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,
if (ret)
return ret;
BUG_ON(pmd_val(old_pmd) != 0 &&
pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp)));
VM_WARN_ON_ONCE(pmd_val(old_pmd) != 0 &&
pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp)));
}
phys += next - addr;
} while (pmdp++, addr = next, addr != end);
@ -282,6 +291,17 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,
return 0;
}
static bool pmd_range_has_valid_noncont(pmd_t *pmdp)
{
for (int i = 0; i < CONT_PMDS; i++) {
pte_t pte = pmd_pte(READ_ONCE(pmdp[i]));
if (pte_valid(pte) && !pte_cont(pte))
return true;
}
return false;
}
static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
unsigned long end, phys_addr_t phys,
pgprot_t prot,
@ -323,7 +343,8 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
/* use a contiguous mapping if the range is suitably aligned */
if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) &&
(flags & NO_CONT_MAPPINGS) == 0)
(flags & NO_CONT_MAPPINGS) == 0 &&
!pmd_range_has_valid_noncont(pmdp))
__prot = __pgprot(pgprot_val(prot) | PTE_CONT);
ret = init_pmd(pmdp, addr, next, phys, __prot, pgtable_alloc, flags);
@ -379,8 +400,9 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
*/
if (pud_sect_supported() &&
((addr | next | phys) & ~PUD_MASK) == 0 &&
(flags & NO_BLOCK_MAPPINGS) == 0) {
pud_set_huge(pudp, phys, prot);
(flags & NO_BLOCK_MAPPINGS) == 0 &&
!pud_table(old_pud)) {
WARN_ON(!pud_set_huge(pudp, phys, prot));
/*
* After the PUD entry has been populated once, we
@ -394,8 +416,8 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
if (ret)
goto out;
BUG_ON(pud_val(old_pud) != 0 &&
pud_val(old_pud) != READ_ONCE(pud_val(*pudp)));
VM_WARN_ON_ONCE(pud_val(old_pud) != 0 &&
pud_val(old_pud) != READ_ONCE(pud_val(*pudp)));
}
phys += next - addr;
} while (pudp++, addr = next, addr != end);
@ -445,8 +467,8 @@ static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,
if (ret)
goto out;
BUG_ON(p4d_val(old_p4d) != 0 &&
p4d_val(old_p4d) != READ_ONCE(p4d_val(*p4dp)));
VM_WARN_ON_ONCE(p4d_val(old_p4d) != 0 &&
p4d_val(old_p4d) != READ_ONCE(p4d_val(*p4dp)));
phys += next - addr;
} while (p4dp++, addr = next, addr != end);
@ -1000,8 +1022,7 @@ void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
&phys, virt);
return;
}
early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
NO_CONT_MAPPINGS);
early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 0);
}
void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
@ -1028,18 +1049,17 @@ static void update_mapping_prot(phys_addr_t phys, unsigned long virt,
return;
}
early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
NO_CONT_MAPPINGS);
early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 0);
/* flush the TLBs after updating live kernel mappings */
flush_tlb_kernel_range(virt, virt + size);
}
static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start,
phys_addr_t end, pgprot_t prot, int flags)
static void __init __map_memblock(phys_addr_t start, phys_addr_t end,
pgprot_t prot, int flags)
{
early_create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start,
prot, early_pgtable_alloc, flags);
early_create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start),
end - start, prot, early_pgtable_alloc, flags);
}
void __init mark_linear_text_alias_ro(void)
@ -1067,36 +1087,24 @@ static int __init parse_kfence_early_init(char *arg)
}
early_param("kfence.sample_interval", parse_kfence_early_init);
static phys_addr_t __init arm64_kfence_alloc_pool(void)
static void __init arm64_kfence_map_pool(void)
{
phys_addr_t kfence_pool;
if (!kfence_early_init)
return 0;
return;
kfence_pool = memblock_phys_alloc(KFENCE_POOL_SIZE, PAGE_SIZE);
if (!kfence_pool) {
pr_err("failed to allocate kfence pool\n");
kfence_early_init = false;
return 0;
return;
}
/* Temporarily mark as NOMAP. */
memblock_mark_nomap(kfence_pool, KFENCE_POOL_SIZE);
return kfence_pool;
}
static void __init arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp)
{
if (!kfence_pool)
return;
/* KFENCE pool needs page-level mapping. */
__map_memblock(pgdp, kfence_pool, kfence_pool + KFENCE_POOL_SIZE,
__map_memblock(kfence_pool, kfence_pool + KFENCE_POOL_SIZE,
pgprot_tagged(PAGE_KERNEL),
NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS);
memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE);
NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS | NO_EXEC_MAPPINGS);
__kfence_pool = phys_to_virt(kfence_pool);
}
@ -1128,18 +1136,18 @@ bool arch_kfence_init_pool(void)
}
#else /* CONFIG_KFENCE */
static inline phys_addr_t arm64_kfence_alloc_pool(void) { return 0; }
static inline void arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) { }
static inline void arm64_kfence_map_pool(void) { }
#endif /* CONFIG_KFENCE */
static void __init map_mem(pgd_t *pgdp)
static void __init map_mem(void)
{
static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN);
phys_addr_t kernel_start = __pa_symbol(_text);
phys_addr_t kernel_end = __pa_symbol(__init_begin);
phys_addr_t init_begin = __pa_symbol(__init_begin);
phys_addr_t init_end = __pa_symbol(__init_end);
phys_addr_t kernel_end = __pa_symbol(__bss_stop);
phys_addr_t start, end;
phys_addr_t early_kfence_pool;
int flags = NO_EXEC_MAPPINGS;
u64 i;
@ -1156,7 +1164,7 @@ static void __init map_mem(pgd_t *pgdp)
BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end) &&
pgd_index(_PAGE_OFFSET(VA_BITS_MIN)) != PTRS_PER_PGD - 1);
early_kfence_pool = arm64_kfence_alloc_pool();
arm64_kfence_map_pool();
linear_map_requires_bbml2 = !force_pte_mapping() && can_set_direct_map();
@ -1164,40 +1172,37 @@ static void __init map_mem(pgd_t *pgdp)
flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
/*
* Take care not to create a writable alias for the
* read-only text and rodata sections of the kernel image.
* So temporarily mark them as NOMAP to skip mappings in
* the following for-loop
* Map the linear alias of the [_text, __init_begin) interval first
* so that its write permissions can be removed later without the need
* to split any block mappings created by the loop below.
*
* Write permissions are needed for alternatives patching, and will be
* removed later by mark_linear_text_alias_ro() above. This makes the
* contents of the region accessible to subsystems such as hibernate,
* but protects it from inadvertent modification or execution.
*/
memblock_mark_nomap(kernel_start, kernel_end - kernel_start);
__map_memblock(kernel_start, init_begin, pgprot_tagged(PAGE_KERNEL),
flags);
/* Map the kernel data/bss so it can be remapped later */
__map_memblock(init_end, kernel_end, pgprot_tagged(PAGE_KERNEL),
flags);
/* map all the memory banks */
for_each_mem_range(i, &start, &end) {
if (start >= end)
break;
/*
* The linear map must allow allocation tags reading/writing
* if MTE is present. Otherwise, it has the same attributes as
* PAGE_KERNEL.
*/
__map_memblock(pgdp, start, end, pgprot_tagged(PAGE_KERNEL),
__map_memblock(start, end, pgprot_tagged(PAGE_KERNEL),
flags);
}
/*
* Map the linear alias of the [_text, __init_begin) interval
* as non-executable now, and remove the write permission in
* mark_linear_text_alias_ro() below (which will be called after
* alternative patching has completed). This makes the contents
* of the region accessible to subsystems such as hibernate,
* but protects it from inadvertent modification or execution.
* Note that contiguous mappings cannot be remapped in this way,
* so we should avoid them here.
*/
__map_memblock(pgdp, kernel_start, kernel_end,
PAGE_KERNEL, NO_CONT_MAPPINGS);
memblock_clear_nomap(kernel_start, kernel_end - kernel_start);
arm64_kfence_map_pool(early_kfence_pool, pgdp);
/* Map the kernel data/bss read-only in the linear map */
__map_memblock(init_end, kernel_end, PAGE_KERNEL_RO, flags);
flush_tlb_kernel_range((unsigned long)lm_alias(__init_end),
(unsigned long)lm_alias(__bss_stop));
}
void mark_rodata_ro(void)
@ -1419,7 +1424,7 @@ static void __init create_idmap(void)
void __init paging_init(void)
{
map_mem(swapper_pg_dir);
map_mem();
memblock_allow_resize();

2
arch/arm64/mm/pageattr.c
View File

@ -21,7 +21,7 @@ struct page_change_data {
pgprot_t clear_mask;
};
static ptdesc_t set_pageattr_masks(ptdesc_t val, struct mm_walk *walk)
static ptval_t set_pageattr_masks(ptval_t val, struct mm_walk *walk)
{
struct page_change_data *masks = walk->private;

2
arch/arm64/mm/ptdump.c
View File

@ -194,7 +194,7 @@ void note_page(struct ptdump_state *pt_st, unsigned long addr, int level,
struct ptdump_pg_state *st = container_of(pt_st, struct ptdump_pg_state, ptdump);
struct ptdump_pg_level *pg_level = st->pg_level;
static const char units[] = "KMGTPE";
ptdesc_t prot = 0;
ptval_t prot = 0;
/* check if the current level has been folded dynamically */
if (st->mm && ((level == 1 && mm_p4d_folded(st->mm)) ||

2
arch/arm64/tools/cpucaps
View File

@ -120,7 +120,7 @@ WORKAROUND_CAVIUM_TX2_219_PRFM
WORKAROUND_CAVIUM_TX2_219_TVM
WORKAROUND_CLEAN_CACHE
WORKAROUND_DEVICE_LOAD_ACQUIRE
WORKAROUND_NVIDIA_CARMEL_CNP
WORKAROUND_DISABLE_CNP
WORKAROUND_PMUV3_IMPDEF_TRAPS
WORKAROUND_QCOM_FALKOR_E1003
WORKAROUND_QCOM_ORYON_CNTVOFF

74
arch/arm64/tools/sysreg
View File

@ -3790,6 +3790,51 @@ Field 1 ZA
Field 0 SM
EndSysreg
Sysreg FPCR 3 3 4 4 0
Res0 63:27
Field 26 AHP
Field 25 DN
Field 24 FZ
Enum 23:22 RMode
0b00 RN
0b01 RP
0b10 RM
0b11 RZ
EndEnum
Field 21:20 Stride
Field 19 FZ16
Field 18:16 Len
Field 15 IDE
Res0 14
Field 13 EBF
Field 12 IXE
Field 11 UFE
Field 10 OFE
Field 9 DZE
Field 8 IOE
Res0 7:3
Field 2 NEP
Field 1 AH
Field 0 FIZ
EndSysreg
Sysreg FPSR 3 3 4 4 1
Res0 63:32
Field 31 N
Field 30 Z
Field 29 C
Field 28 V
Field 27 QC
Res0 26:8
Field 7 IDC
Res0 6:5
Field 4 IXC
Field 3 UFC
Field 2 OFC
Field 1 DZC
Field 0 IOC
EndSysreg
Sysreg FPMR 3 3 4 4 2
Res0 63:38
Field 37:32 LSCALE2
@ -4597,6 +4642,35 @@ Sysreg GCSPR_EL2 3 4 2 5 1
Fields GCSPR_ELx
EndSysreg
Sysreg HDBSSBR_EL2 3 4 2 3 2
Res0 63:56
Field 55:12 BADDR
Res0 11:4
Enum 3:0 SZ
0b0000 4KB
0b0001 8KB
0b0010 16KB
0b0011 32KB
0b0100 64KB
0b0101 128KB
0b0110 256KB
0b0111 512KB
0b1000 1MB
0b1001 2MB
EndEnum
EndSysreg
Sysreg HDBSSPROD_EL2 3 4 2 3 3
Res0 63:32
Enum 31:26 FSC
0b000000 OK
0b010000 ExternalAbort
0b101000 GPF
EndEnum
Res0 25:19
Field 18:0 INDEX
EndSysreg
Sysreg DACR32_EL2 3 4 3 0 0
Res0 63:32
Field 31:30 D15

52
arch/powerpc/lib/code-patching.c
View File

@ -60,9 +60,6 @@ struct patch_context {
static DEFINE_PER_CPU(struct patch_context, cpu_patching_context);
static int map_patch_area(void *addr, unsigned long text_poke_addr);
static void unmap_patch_area(unsigned long addr);
static bool mm_patch_enabled(void)
{
return IS_ENABLED(CONFIG_SMP) && radix_enabled();
@ -117,11 +114,11 @@ static int text_area_cpu_up(unsigned int cpu)
// Map/unmap the area to ensure all page tables are pre-allocated
addr = (unsigned long)area->addr;
err = map_patch_area(empty_zero_page, addr);
err = map_kernel_page(addr, __pa_symbol(empty_zero_page), PAGE_KERNEL_RO);
if (err)
return err;
unmap_patch_area(addr);
unmap_kernel_page(addr);
this_cpu_write(cpu_patching_context.area, area);
this_cpu_write(cpu_patching_context.addr, addr);
@ -233,51 +230,6 @@ static unsigned long get_patch_pfn(void *addr)
return __pa_symbol(addr) >> PAGE_SHIFT;
}
/*
* This can be called for kernel text or a module.
*/
static int map_patch_area(void *addr, unsigned long text_poke_addr)
{
unsigned long pfn = get_patch_pfn(addr);
return map_kernel_page(text_poke_addr, (pfn << PAGE_SHIFT), PAGE_KERNEL);
}
static void unmap_patch_area(unsigned long addr)
{
pte_t *ptep;
pmd_t *pmdp;
pud_t *pudp;
p4d_t *p4dp;
pgd_t *pgdp;
pgdp = pgd_offset_k(addr);
if (WARN_ON(pgd_none(*pgdp)))
return;
p4dp = p4d_offset(pgdp, addr);
if (WARN_ON(p4d_none(*p4dp)))
return;
pudp = pud_offset(p4dp, addr);
if (WARN_ON(pud_none(*pudp)))
return;
pmdp = pmd_offset(pudp, addr);
if (WARN_ON(pmd_none(*pmdp)))
return;
ptep = pte_offset_kernel(pmdp, addr);
if (WARN_ON(pte_none(*ptep)))
return;
/*
* In hash, pte_clear flushes the tlb, in radix, we have to
*/
pte_clear(&init_mm, addr, ptep);
flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
}
static int __do_patch_mem_mm(void *addr, unsigned long val, bool is_dword)
{
int err;

3
arch/sh/mm/init.c
View File

@ -331,9 +331,6 @@ void __init mem_init(void)
/* Set this up early, so we can take care of the zero page */
cpu_cache_init();
/* clear the zero-page */
__flush_wback_region(empty_zero_page, PAGE_SIZE);
vsyscall_init();
pr_info("virtual kernel memory layout:\n"

4
drivers/perf/Kconfig
View File

@ -188,7 +188,7 @@ config FUJITSU_UNCORE_PMU
monitoring Uncore events.
config QCOM_L2_PMU
bool "Qualcomm Technologies L2-cache PMU"
bool "Qualcomm L2-cache PMU"
depends on ARCH_QCOM && ARM64 && ACPI
select QCOM_KRYO_L2_ACCESSORS
help
@ -198,7 +198,7 @@ config QCOM_L2_PMU
monitoring L2 cache events.
config QCOM_L3_PMU
bool "Qualcomm Technologies L3-cache PMU"
bool "Qualcomm L3-cache PMU"
depends on ARCH_QCOM && ARM64 && ACPI
select QCOM_IRQ_COMBINER
help

23
drivers/perf/arm-cmn.c
View File

@ -197,13 +197,14 @@
enum cmn_model {
CMN600 = 1,
CMN650 = 2,
CMN700 = 4,
CI700 = 8,
CI700 = 4,
CMN700 = 8,
CMNS3 = 16,
/* ...and then we can use bitmap tricks for commonality */
CMN_ANY = -1,
NOT_CMN600 = -2,
CMN_650ON = CMN650 | CMN700 | CMNS3,
CMN_700ON = ~(CMN700 - 1),
CMN_650ON = CMN_700ON | CMN650,
};
/* Actual part numbers and revision IDs defined by the hardware */
@ -919,14 +920,14 @@ static struct attribute *arm_cmn_event_attrs[] = {
CMN_EVENT_DVM(NOT_CMN600, txsnp_stall, 0x0a),
CMN_EVENT_DVM(NOT_CMN600, trkfull, 0x0b),
CMN_EVENT_DVM_OCC(NOT_CMN600, trk_occupancy, 0x0c),
CMN_EVENT_DVM_OCC(CMN700, trk_occupancy_cxha, 0x0d),
CMN_EVENT_DVM_OCC(CMN700, trk_occupancy_pdn, 0x0e),
CMN_EVENT_DVM(CMN700, trk_alloc, 0x0f),
CMN_EVENT_DVM(CMN700, trk_cxha_alloc, 0x10),
CMN_EVENT_DVM(CMN700, trk_pdn_alloc, 0x11),
CMN_EVENT_DVM(CMN700, txsnp_stall_limit, 0x12),
CMN_EVENT_DVM(CMN700, rxsnp_stall_starv, 0x13),
CMN_EVENT_DVM(CMN700, txsnp_sync_stall_op, 0x14),
CMN_EVENT_DVM_OCC(CMN_700ON, trk_occupancy_cxha, 0x0d),
CMN_EVENT_DVM_OCC(CMN_700ON, trk_occupancy_pdn, 0x0e),
CMN_EVENT_DVM(CMN_700ON, trk_alloc, 0x0f),
CMN_EVENT_DVM(CMN_700ON, trk_cxha_alloc, 0x10),
CMN_EVENT_DVM(CMN_700ON, trk_pdn_alloc, 0x11),
CMN_EVENT_DVM(CMN_700ON, txsnp_stall_limit, 0x12),
CMN_EVENT_DVM(CMN_700ON, rxsnp_stall_starv, 0x13),
CMN_EVENT_DVM(CMN_700ON, txsnp_sync_stall_op, 0x14),
CMN_EVENT_HNF(CMN_ANY, cache_miss, 0x01),
CMN_EVENT_HNF(CMN_ANY, slc_sf_cache_access, 0x02),

23
drivers/resctrl/mpam_devices.c
View File

@ -224,6 +224,24 @@ static inline void _mpam_write_monsel_reg(struct mpam_msc *msc, u16 reg, u32 val
#define mpam_write_monsel_reg(msc, reg, val) _mpam_write_monsel_reg(msc, MSMON_##reg, val)
static bool mpam_msc_check_aidr(struct mpam_msc *msc)
{
u32 aidr = __mpam_read_reg(msc, MPAMF_AIDR);
u32 major = FIELD_GET(MPAMF_AIDR_ARCH_MAJOR_REV, aidr);
u32 minor = FIELD_GET(MPAMF_AIDR_ARCH_MINOR_REV, aidr);
/*
* v0.0 and >v2.x aren't supported, but anything else should be backward
* compatible to v0.1 or v1.0.
*/
if (!major && !minor)
return false;
if (major > 1)
return false;
return true;
}
static u64 mpam_msc_read_idr(struct mpam_msc *msc)
{
u64 idr_high = 0, idr_low;
@ -945,9 +963,8 @@ static int mpam_msc_hw_probe(struct mpam_msc *msc)
lockdep_assert_held(&msc->probe_lock);
idr = __mpam_read_reg(msc, MPAMF_AIDR);
if ((idr & MPAMF_AIDR_ARCH_MAJOR_REV) != MPAM_ARCHITECTURE_V1) {
dev_err_once(dev, "MSC does not match MPAM architecture v1.x\n");
if (!mpam_msc_check_aidr(msc)) {
dev_err_once(dev, "MSC does not match architecture v1.x\n");
return -EIO;
}

4
include/linux/linkage.h
View File

@ -39,6 +39,10 @@
#define __page_aligned_data __section(".data..page_aligned") __aligned(PAGE_SIZE)
#define __page_aligned_bss __section(".bss..page_aligned") __aligned(PAGE_SIZE)
#ifndef __bss_pgtbl
#define __bss_pgtbl __page_aligned_bss
#endif
/*
* For assembly routines.
*

2
include/linux/pgtable.h
View File

@ -2027,7 +2027,7 @@ static inline unsigned long zero_pfn(unsigned long addr)
return zero_page_pfn;
}
extern uint8_t empty_zero_page[PAGE_SIZE];
extern const uint8_t empty_zero_page[PAGE_SIZE];
extern struct page *__zero_page;
static inline struct page *_zero_page(unsigned long addr)

10
mm/kasan/init.c
View File

@ -26,10 +26,10 @@
* - Latter it reused it as zero shadow to cover large ranges of memory
* that allowed to access, but not handled by kasan (vmalloc/vmemmap ...).
*/
unsigned char kasan_early_shadow_page[PAGE_SIZE] __page_aligned_bss;
unsigned char kasan_early_shadow_page[PAGE_SIZE] __bss_pgtbl;
#if CONFIG_PGTABLE_LEVELS > 4
p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D] __page_aligned_bss;
p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D] __bss_pgtbl;
static inline bool kasan_p4d_table(pgd_t pgd)
{
return pgd_page(pgd) == virt_to_page(lm_alias(kasan_early_shadow_p4d));
@ -41,7 +41,7 @@ static inline bool kasan_p4d_table(pgd_t pgd)
}
#endif
#if CONFIG_PGTABLE_LEVELS > 3
pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __page_aligned_bss;
pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __bss_pgtbl;
static inline bool kasan_pud_table(p4d_t p4d)
{
return p4d_page(p4d) == virt_to_page(lm_alias(kasan_early_shadow_pud));
@ -53,7 +53,7 @@ static inline bool kasan_pud_table(p4d_t p4d)
}
#endif
#if CONFIG_PGTABLE_LEVELS > 2
pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __page_aligned_bss;
pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __bss_pgtbl;
static inline bool kasan_pmd_table(pud_t pud)
{
return pud_page(pud) == virt_to_page(lm_alias(kasan_early_shadow_pmd));
@ -65,7 +65,7 @@ static inline bool kasan_pmd_table(pud_t pud)
}
#endif
pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS]
__page_aligned_bss;
__bss_pgtbl;
static inline bool kasan_pte_table(pmd_t pmd)
{

2
mm/mm_init.c
View File

@ -57,7 +57,7 @@ unsigned long zero_page_pfn __ro_after_init;
EXPORT_SYMBOL(zero_page_pfn);
#ifndef __HAVE_COLOR_ZERO_PAGE
uint8_t empty_zero_page[PAGE_SIZE] __page_aligned_bss;
const uint8_t empty_zero_page[PAGE_SIZE] __aligned(PAGE_SIZE);
EXPORT_SYMBOL(empty_zero_page);
struct page *__zero_page __ro_after_init;

116
tools/testing/selftests/arm64/abi/hwcap.c
View File

@ -108,6 +108,24 @@ static void f8mm8_sigill(void)
asm volatile(".inst 0x6e80ec00");
}
static void f16f32dot_sigill(void)
{
/* FDOT V0.2S, V0.4H, V0.2H[0] */
asm volatile(".inst 0xf409000");
}
static void f16f32mm_sigill(void)
{
/* FMMLA V0.4S, V0.8H, V0.8H */
asm volatile(".inst 0x4e40ec00");
}
static void f16mm_sigill(void)
{
/* FMMLA V0.8H, V0.8H, V0.8H */
asm volatile(".inst 0x4ec0ec00");
}
static void faminmax_sigill(void)
{
/* FAMIN V0.4H, V0.4H, V0.4H */
@ -191,6 +209,12 @@ static void lut_sigill(void)
asm volatile(".inst 0x4e801000");
}
static void sve_lut6_sigill(void)
{
/* LUTI6 Z0.H, { Z0.H, Z1.H }, Z0[0] */
asm volatile(".inst 0x4560ac00");
}
static void mops_sigill(void)
{
char dst[1], src[1];
@ -282,6 +306,18 @@ static void sme2p2_sigill(void)
asm volatile("msr S0_3_C4_C6_3, xzr" : : : );
}
static void sme2p3_sigill(void)
{
/* SMSTART SM */
asm volatile("msr S0_3_C4_C3_3, xzr" : : : );
/* ADDQP Z0.B, Z0.B, Z0.B */
asm volatile(".inst 0x4207800" : : : "z0");
/* SMSTOP */
asm volatile("msr S0_3_C4_C6_3, xzr" : : : );
}
static void sme_aes_sigill(void)
{
/* SMSTART SM */
@ -378,6 +414,18 @@ static void smef8f32_sigill(void)
asm volatile("msr S0_3_C4_C6_3, xzr" : : : );
}
static void smelut6_sigill(void)
{
/* SMSTART */
asm volatile("msr S0_3_C4_C7_3, xzr" : : : );
/* LUTI6 { Z0.B-Z3.B }, ZT0, { Z0-Z2 } */
asm volatile(".inst 0xc08a0000" : : : );
/* SMSTOP */
asm volatile("msr S0_3_C4_C6_3, xzr" : : : );
}
static void smelutv2_sigill(void)
{
/* SMSTART */
@ -486,6 +534,12 @@ static void sve2p2_sigill(void)
asm volatile(".inst 0x4cea000" : : : "z0");
}
static void sve2p3_sigill(void)
{
/* ADDQP Z0.B, Z0.B, Z0.B */
asm volatile(".inst 0x4207800" : : : "z0");
}
static void sveaes_sigill(void)
{
/* AESD z0.b, z0.b, z0.b */
@ -504,6 +558,12 @@ static void sveb16b16_sigill(void)
asm volatile(".inst 0x65000000" : : : );
}
static void sveb16mm_sigill(void)
{
/* BFMMLA Z0.H, Z0.H, Z0.H */
asm volatile(".inst 0x64e0e000" : : : );
}
static void svebfscale_sigill(void)
{
/* BFSCALE Z0.H, P0/M, Z0.H, Z0.H */
@ -729,6 +789,27 @@ static const struct hwcap_data {
.cpuinfo = "f8mm4",
.sigill_fn = f8mm4_sigill,
},
{
.name = "F16MM",
.at_hwcap = AT_HWCAP3,
.hwcap_bit = HWCAP3_F16MM,
.cpuinfo = "f16mm",
.sigill_fn = f16mm_sigill,
},
{
.name = "F16F32DOT",
.at_hwcap = AT_HWCAP3,
.hwcap_bit = HWCAP3_F16F32DOT,
.cpuinfo = "f16f32dot",
.sigill_fn = f16f32dot_sigill,
},
{
.name = "F16F32MM",
.at_hwcap = AT_HWCAP3,
.hwcap_bit = HWCAP3_F16F32MM,
.cpuinfo = "f16f32mm",
.sigill_fn = f16f32mm_sigill,
},
{
.name = "FAMINMAX",
.at_hwcap = AT_HWCAP2,
@ -918,6 +999,13 @@ static const struct hwcap_data {
.cpuinfo = "sme2p2",
.sigill_fn = sme2p2_sigill,
},
{
.name = "SME 2.3",
.at_hwcap = AT_HWCAP3,
.hwcap_bit = HWCAP3_SME2P3,
.cpuinfo = "sme2p3",
.sigill_fn = sme2p3_sigill,
},
{
.name = "SME AES",
.at_hwcap = AT_HWCAP,
@ -967,6 +1055,13 @@ static const struct hwcap_data {
.cpuinfo = "smef8f32",
.sigill_fn = smef8f32_sigill,
},
{
.name = "SME LUT6",
.at_hwcap = AT_HWCAP3,
.hwcap_bit = HWCAP3_SME_LUT6,
.cpuinfo = "smelut6",
.sigill_fn = smelut6_sigill,
},
{
.name = "SME LUTV2",
.at_hwcap = AT_HWCAP2,
@ -1052,6 +1147,13 @@ static const struct hwcap_data {
.cpuinfo = "sve2p2",
.sigill_fn = sve2p2_sigill,
},
{
.name = "SVE 2.3",
.at_hwcap = AT_HWCAP3,
.hwcap_bit = HWCAP3_SVE2P3,
.cpuinfo = "sve2p3",
.sigill_fn = sve2p3_sigill,
},
{
.name = "SVE AES",
.at_hwcap = AT_HWCAP2,
@ -1066,6 +1168,13 @@ static const struct hwcap_data {
.cpuinfo = "sveaes2",
.sigill_fn = sveaes2_sigill,
},
{
.name = "SVE B16MM",
.at_hwcap = AT_HWCAP3,
.hwcap_bit = HWCAP3_SVE_B16MM,
.cpuinfo = "sveb16mm",
.sigill_fn = sveb16mm_sigill,
},
{
.name = "SVE BFSCALE",
.at_hwcap = AT_HWCAP,
@ -1087,6 +1196,13 @@ static const struct hwcap_data {
.cpuinfo = "svef16mm",
.sigill_fn = svef16mm_sigill,
},
{
.name = "SVE_LUT6",
.at_hwcap = AT_HWCAP3,
.hwcap_bit = HWCAP3_SVE_LUT6,
.cpuinfo = "svelut6",
.sigill_fn = sve_lut6_sigill,
},
{
.name = "SVE2 B16B16",
.at_hwcap = AT_HWCAP2,

2
tools/testing/selftests/arm64/signal/test_signals.h
View File

@ -36,6 +36,7 @@ enum {
FSME_FA64_BIT,
FSME2_BIT,
FGCS_BIT,
FPOE_BIT,
FMAX_END
};
@ -45,6 +46,7 @@ enum {
#define FEAT_SME_FA64 (1UL << FSME_FA64_BIT)
#define FEAT_SME2 (1UL << FSME2_BIT)
#define FEAT_GCS (1UL << FGCS_BIT)
#define FEAT_POE (1UL << FPOE_BIT)
/*
* A descriptor used to describe and configure a test case.

3
tools/testing/selftests/arm64/signal/test_signals_utils.c
View File

@ -31,6 +31,7 @@ static char const *const feats_names[FMAX_END] = {
" FA64 ",
" SME2 ",
" GCS ",
" POE ",
};
#define MAX_FEATS_SZ 128
@ -341,6 +342,8 @@ int test_init(struct tdescr *td)
td->feats_supported |= FEAT_SME2;
if (getauxval(AT_HWCAP) & HWCAP_GCS)
td->feats_supported |= FEAT_GCS;
if (getauxval(AT_HWCAP2) & HWCAP2_POE)
td->feats_supported |= FEAT_POE;
if (feats_ok(td)) {
if (td->feats_required & td->feats_supported)
fprintf(stderr,

16
tools/testing/selftests/arm64/signal/test_signals_utils.h
View File

@ -57,6 +57,22 @@ static inline __attribute__((always_inline)) uint64_t get_gcspr_el0(void)
return val;
}
#define SYS_POR_EL0 "S3_3_C10_C2_4"
static inline uint64_t get_por_el0(void)
{
uint64_t val;
asm volatile("mrs %0, " SYS_POR_EL0 "\n" : "=r"(val));
return val;
}
static inline void set_por_el0(uint64_t val)
{
asm volatile("msr " SYS_POR_EL0 ", %0\n" :: "r"(val));
}
static inline bool feats_ok(struct tdescr *td)
{
if (td->feats_incompatible & td->feats_supported)

73
tools/testing/selftests/arm64/signal/testcases/poe_missing_poe_context.c Normal file
View File

@ -0,0 +1,73 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2026 Arm Ltd
*
* Verify that the POR_EL0 register is left untouched on sigreturn if the
* POE frame record is missing.
*/
#include <asm/sigcontext.h>
#include "test_signals_utils.h"
#include "testcases.h"
#define POR_EL0_INIT 0x07ul
#define POR_EL0_CUSTOM 0x77ul
static bool failed_check;
static bool modify_por_el0(struct tdescr *td)
{
set_por_el0(POR_EL0_CUSTOM);
return true;
}
static int signal_remove_poe_context(struct tdescr *td, siginfo_t *si,
ucontext_t *uc)
{
struct _aarch64_ctx *ctx = GET_UC_RESV_HEAD(uc);
size_t resv_size = GET_UCP_RESV_SIZE(uc);
struct _aarch64_ctx *poe_ctx_head;
poe_ctx_head = get_header(ctx, POE_MAGIC, resv_size, NULL);
if (!poe_ctx_head) {
fprintf(stderr, "Missing poe_context record\n");
failed_check = true;
return 0;
}
/*
* Actually removing the record would require moving down the next
* records. An easier option is to turn it into an ESR record, which is
* ignored by sigreturn().
*/
poe_ctx_head->magic = ESR_MAGIC;
return 0;
}
static void check_por_el0_preserved(struct tdescr *td)
{
uint64_t por_el0 = get_por_el0();
if (por_el0 == POR_EL0_INIT) {
fprintf(stderr, "POR_EL0 preserved\n");
} else {
fprintf(stderr, "POR_EL0 unexpectedly set to %lx\n", por_el0);
failed_check = true;
}
td->pass = !failed_check;
}
struct tdescr tde = {
.name = "POR_EL0 missing poe_context",
.descr = "Remove poe_context record and check POR_EL0 is preserved",
.feats_required = FEAT_POE,
.timeout = 3,
.sig_trig = SIGUSR1,
.init = modify_por_el0,
.run = signal_remove_poe_context,
.check_result = check_por_el0_preserved,
};

64
tools/testing/selftests/arm64/signal/testcases/poe_restore.c Normal file
View File

@ -0,0 +1,64 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2026 Arm Ltd
*
* Verify that the POR_EL0 register is saved and restored as expected on signal
* entry/return.
*/
#include <asm/sigcontext.h>
#include "test_signals_utils.h"
#include "testcases.h"
#define POR_EL0_INIT 0x07ul
#define POR_EL0_CUSTOM 0x77ul
static bool failed_check;
static bool modify_por_el0(struct tdescr *td)
{
set_por_el0(POR_EL0_CUSTOM);
return true;
}
static int signal_check_por_el0_reset(struct tdescr *td, siginfo_t *si,
ucontext_t *uc)
{
uint64_t signal_por_el0 = get_por_el0();
if (signal_por_el0 != POR_EL0_INIT) {
fprintf(stderr, "POR_EL0 is %lx in signal handler (expected %lx)\n",
signal_por_el0, POR_EL0_INIT);
failed_check = true;
}
return 0;
}
static void check_por_el0_restored(struct tdescr *td)
{
uint64_t por_el0 = get_por_el0();
if (por_el0 == POR_EL0_CUSTOM) {
fprintf(stderr, "POR_EL0 restored\n");
} else {
fprintf(stderr, "POR_EL0 was %lx but is now %lx\n",
POR_EL0_CUSTOM, por_el0);
failed_check = true;
}
td->pass = !failed_check;
}
struct tdescr tde = {
.name = "POR_EL0 restore",
.descr = "Validate that POR_EL0 is saved/restored on signal entry/return",
.feats_required = FEAT_POE,
.timeout = 3,
.sig_trig = SIGUSR1,
.init = modify_por_el0,
.run = signal_check_por_el0_reset,
.check_result = check_por_el0_restored,
};

15
tools/testing/selftests/arm64/signal/testcases/poe_siginfo.c
View File

@ -21,21 +21,6 @@ static union {
char buf[1024 * 128];
} context;
#define SYS_POR_EL0 "S3_3_C10_C2_4"
static uint64_t get_por_el0(void)
{
uint64_t val;
asm volatile(
"mrs %0, " SYS_POR_EL0 "\n"
: "=r"(val)
:
: );
return val;
}
int poe_present(struct tdescr *td, siginfo_t *si, ucontext_t *uc)
{
struct _aarch64_ctx *head = GET_BUF_RESV_HEAD(context);

3
tools/testing/selftests/mm/pkey-arm64.h
View File

@ -130,9 +130,10 @@ static inline u64 get_pkey_bits(u64 reg, int pkey)
static inline void aarch64_write_signal_pkey(ucontext_t *uctxt, u64 pkey)
{
struct _aarch64_ctx *ctx = GET_UC_RESV_HEAD(uctxt);
size_t resv_size = GET_UCP_RESV_SIZE(uctxt);
struct poe_context *poe_ctx =
(struct poe_context *) get_header(ctx, POE_MAGIC,
sizeof(uctxt->uc_mcontext), NULL);
resv_size, NULL);
if (poe_ctx)
poe_ctx->por_el0 = pkey;
}